commit 709385cbaa5a590228becfab6e0bd2ea83858ca9 Author: upstream-mirror Date: Tue Sep 15 20:16:01 2026 +0500 snapshot of https://github.com/nats-io/nats-server.git (main, 2026-09-15; blobs > 131072B omitted) diff --git a/.coveralls.yml b/.coveralls.yml new file mode 100644 index 0000000..cf27a37 --- /dev/null +++ b/.coveralls.yml @@ -0,0 +1 @@ +service_name: travis-pro diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..9bd393a --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,5 @@ +* @nats-io/server +*.go @nats-io/server-dev +doc/ @nats-io/server-dev +test/ @nats-io/server-dev +locksordering.txt @nats-io/server-dev diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..b77080d --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,5 @@ +# These are supported funding model platforms + +# NATS.io +custom: "https://crowdfunding.linuxfoundation.org/initiatives/nats-io" + diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..ad10539 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Discussion + url: https://github.com/nats-io/nats-server/discussions + about: Ideal for ideas, feedback, or longer form questions. + - name: Chat + url: https://slack.nats.io + about: Ideal for short, one-off questions, general conversation, and meeting other NATS users! diff --git a/.github/ISSUE_TEMPLATE/defect.yml b/.github/ISSUE_TEMPLATE/defect.yml new file mode 100644 index 0000000..fdbfc30 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/defect.yml @@ -0,0 +1,54 @@ +--- +name: Defect +description: Report a defect, such as a bug or regression. +labels: + - defect +body: + - type: textarea + id: observed + attributes: + label: Observed behavior + description: Describe the unexpected behavior or performance regression you are observing. + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected behavior + description: Describe the expected behavior or performance characteristics. + validations: + required: true + - type: textarea + id: versions + attributes: + label: Server and client version + description: |- + **One of the first things we are likely to ask you is if the defect reproduces on the latest version, so please check that first!** + For the server, use `nats-server --version`, check the startup log output, or the image tag pulled from Docker. + For the CLI client, use `nats --version`. + For language-specific clients, check the version downloaded by the language dependency manager. + validations: + required: true + - type: textarea + id: environment + attributes: + label: Host environment + description: |- + Specify any relevant details about the host environment the server and/or client was running in, + such as operating system, CPU architecture, container runtime, etc. + validations: + required: false + - type: textarea + id: steps + attributes: + label: Steps to reproduce + description: Provide as many concrete steps to reproduce the defect. Please also include stream info and consumer info if this issue is JetStream-related. + validations: + required: false + - type: checkboxes + id: no-ai-meat-proxy + attributes: + label: Submission acknowledgment + options: + - label: I am a human being writing in my own words and not an AI agent. I will not use an AI agent to communicate on my behalf in this issue, either directly or via copy-paste. + required: true diff --git a/.github/ISSUE_TEMPLATE/proposal.yml b/.github/ISSUE_TEMPLATE/proposal.yml new file mode 100644 index 0000000..c28e7af --- /dev/null +++ b/.github/ISSUE_TEMPLATE/proposal.yml @@ -0,0 +1,35 @@ +--- +name: Proposal +description: Propose an enhancement or new feature. +labels: + - proposal +body: + - type: textarea + id: change + attributes: + label: Proposed change + description: This could be a behavior change, enhanced API, or a new feature. + validations: + required: true + - type: textarea + id: usecase + attributes: + label: Use case + description: What is the use case or general motivation for this proposal? + validations: + required: true + - type: textarea + id: contribute + attributes: + label: Contribution + description: |- + Are you intending or interested in contributing code for this proposal if accepted? + validations: + required: false + - type: checkboxes + id: no-ai-meat-proxy + attributes: + label: Submission acknowledgment + options: + - label: I am a human being writing in my own words and not an AI agent. I will not use an AI agent to communicate on my behalf in this issue, either directly or via copy-paste. + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..ae5cd79 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1 @@ + diff --git a/.github/actions/nightly-release/action.yaml b/.github/actions/nightly-release/action.yaml new file mode 100644 index 0000000..aee79ef --- /dev/null +++ b/.github/actions/nightly-release/action.yaml @@ -0,0 +1,47 @@ +name: Nightly Docker Releaser +description: Builds nightly docker images + +inputs: + hub_username: + description: Docker hub username + required: true + + hub_password: + description: Docker hub password + required: true + + name: + description: The name of the build + default: nightly + required: false + +runs: + using: composite + steps: + - name: Log in to Docker Hub + shell: bash + run: docker login -u "${{ inputs.hub_username }}" -p "${{ inputs.hub_password }}" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Generate build metadata + shell: sh + run: | + echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV + echo "GIT_COMMIT=$(cd ${{ inputs.workdir }}/nats-server && git rev-parse --short HEAD)" >> $GITHUB_ENV + echo "BUILD_NAME=$(echo ${{ inputs.name }} | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9.-]/-/g')" >> $GITHUB_ENV + + - name: Build and push Docker images + uses: docker/build-push-action@v6 + with: + context: "${{ github.workspace }}/src/github.com/nats-io" + file: "${{ github.workspace }}/src/github.com/nats-io/nats-server/docker/Dockerfile.nightly" + build-args: | + VERSION=nightly-${{ env.BUILD_DATE }} + GIT_COMMIT=${{ env.GIT_COMMIT }} + platforms: linux/amd64,linux/arm64 + push: true + tags: | + synadia/nats-server:${{ env.BUILD_NAME }} + synadia/nats-server:${{ env.BUILD_NAME }}-${{ env.BUILD_DATE }} diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f0b9262 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" \ No newline at end of file diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml new file mode 100644 index 0000000..b6d823a --- /dev/null +++ b/.github/workflows/claude.yml @@ -0,0 +1,64 @@ +name: Claude Code + +# GITHUB_TOKEN needs contents:read and actions:read — required by +# claude-code-action for restoring trusted config files from the base branch. +# All other GitHub API access uses the App token. +permissions: + contents: read + actions: read + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + pull_request_target: + types: [opened, reopened] + +jobs: + claude: + name: Claude Review + uses: synadia-io/ai-workflows/.github/workflows/claude.yml@v2 + if: contains( + fromJson('["OWNER","MEMBER","COLLABORATOR"]'), + github.event.comment.author_association || github.event.pull_request.author_association + ) + with: + gh_app_id: ${{ vars.CLAUDE_GH_APP_ID }} + checkout_mode: "base" + # review_focus is used only by the auto-review job (pull_request_target), not the + # interactive @claude job — so the check below only needs github.event.pull_request, + # which pull_request_target populates. On issue_comment events that field is absent, + # so this evaluates to startsWith(null, 'backports/') -> false; harmless, since the + # interactive job ignores review_focus. Can't be fixed in-expression: issue_comment + # payloads carry no head ref at all. + review_focus: | + ${{ startsWith(github.event.pull_request.head.ref, 'backports/') && ' + This is a BACKPORT / RELEASE pull request. Every cherry-picked commit + originates from a PR that was already independently reviewed and approved + on the source branch. Do NOT re-review the individual code changes and do + NOT comment on the implementation of cherry-picked hunks — assume they are + correct. + + Instead, focus exclusively on: + - Cherry-pick completeness. Inspect the PRs/commits included in this + backport. For each source PR, check its milestone, labels, and target + branch to decide whether it belongs in this release line. Flag any PR + that targets this release but is MISSING here, and any included PR that + does not belong. + - Non-clean picks. Call out commits that appear to have been modified + during the cherry-pick (conflict resolution, adapted context) — those + are the only changes that warrant a closer look. + - Release overview. Produce a concise summary of what this release ships, + grouped by area (JetStream, Raft/NRG, leafnodes, security, etc.), so + reviewers can sanity-check the scope at a glance. + ' || ' + Additionally focus on: + - Performance implications (hot paths, allocations, lock contention) + - Concurrency safety (goroutine leaks, race conditions, deadlocks) + - Raft consensus and JetStream clustering correctness + - Security boundaries (authentication, authorization, TLS handling) + ' }} + secrets: + claude_oauth_token: ${{ secrets.CLAUDE_OAUTH_TOKEN }} + gh_app_private_key: ${{ secrets.CLAUDE_GH_APP_PRIVATE_KEY }} diff --git a/.github/workflows/cov.yaml b/.github/workflows/cov.yaml new file mode 100644 index 0000000..e680d36 --- /dev/null +++ b/.github/workflows/cov.yaml @@ -0,0 +1,53 @@ +name: NATS Server Code Coverage +on: + workflow_dispatch: {} + + schedule: + - cron: "40 4 * * *" + +permissions: + contents: read + +jobs: + nightly_coverage: + runs-on: ubuntu-latest + + env: + GOPATH: /home/runner/work/nats-server + GO111MODULE: "on" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + path: src/github.com/nats-io/nats-server + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version: stable + cache-dependency-path: src/github.com/nats-io/nats-server/go.sum + + - name: Run code coverage + shell: bash --noprofile --norc -x -eo pipefail {0} + run: | + set -e + cd src/github.com/nats-io/nats-server + ./scripts/cov.sh upload + set +e + + - name: Convert coverage.out to coverage.lcov + # Use commit hash here to avoid a re-tagging attack, as this is a third-party action + # Commit e4612787670fc5b5f49026b8c29c5569921de1db = tag v1.2.0 + uses: jandelgado/gcov2lcov-action@e4612787670fc5b5f49026b8c29c5569921de1db + with: + infile: acc.out + working-directory: src/github.com/nats-io/nats-server + + - name: Coveralls + # Use commit hash here to avoid a re-tagging attack, as this is a third-party action + # Commit 5cbfd81b66ca5d10c19b062c04de0199c215fb6e = tag v2.3.7 + uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e + with: + github-token: ${{ secrets.github_token }} + file: src/github.com/nats-io/nats-server/coverage.lcov diff --git a/.github/workflows/long-tests.yaml b/.github/workflows/long-tests.yaml new file mode 100644 index 0000000..c4fbaa1 --- /dev/null +++ b/.github/workflows/long-tests.yaml @@ -0,0 +1,33 @@ +name: NATS Server Long Tests + +on: + # Allow manual trigger (any branch) + workflow_dispatch: + # Run daily at 12:30 on default branch + schedule: + - cron: "30 12 * * *" + +permissions: + contents: read + +concurrency: + # At most one of these workflow per ref running + group: ${{ github.workflow }}-${{ github.ref }} + # New one cancels in-progress one + cancel-in-progress: true + +jobs: + js-long: + name: Long JetStream tests + runs-on: ${{ vars.GHA_WORKER_MEDIUM || 'ubuntu-latest' }} + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Run tests + run: go test -race -v -run='^TestLong.*' ./server -tags=include_js_long_tests -count=1 -vet=off -timeout=60m -shuffle on -p 1 -failfast diff --git a/.github/workflows/mqtt-test.yaml b/.github/workflows/mqtt-test.yaml new file mode 100644 index 0000000..b49358d --- /dev/null +++ b/.github/workflows/mqtt-test.yaml @@ -0,0 +1,53 @@ +name: MQTT External Tests +on: [pull_request] + +permissions: + contents: read + +jobs: + test: + env: + GOPATH: /home/runner/work/nats-server + GO111MODULE: "on" + runs-on: ${{ vars.GHA_WORKER_SMALL || 'ubuntu-latest' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + path: src/github.com/nats-io/nats-server + + - name: Setup Go + uses: actions/setup-go@v6 + with: + go-version: stable + cache-dependency-path: src/github.com/nats-io/nats-server/go.sum + + - name: Set up testing tools and environment + shell: bash --noprofile --norc -eo pipefail {0} + id: setup + run: | + wget https://github.com/hivemq/mqtt-cli/releases/download/v4.20.0/mqtt-cli-4.20.0.deb + sudo apt install ./mqtt-cli-4.20.0.deb + # Pinned to the mqtt-test commit that adds --pipeline (PR #5); + # bump to @v0.2.0 once it is tagged. + go install github.com/ConnectEverything/mqtt-test@6138352776422dfd16eeb3333d078358d8b85fea + + - name: Run tests (3 times to detect flappers) + shell: bash --noprofile --norc -eo pipefail {0} + run: | + cd src/github.com/nats-io/nats-server + go test -v --count=3 --run='TestXMQTT' ./server + + - name: Run tests with --race + shell: bash --noprofile --norc -eo pipefail {0} + run: | + cd src/github.com/nats-io/nats-server + go test -v --race --failfast --run='TestXMQTT' ./server + + - name: Run benchmarks + shell: bash --noprofile --norc -eo pipefail {0} + run: | + cd src/github.com/nats-io/nats-server + go test --run='-' --count=3 --bench 'BenchmarkXMQTT' --benchtime=100x ./server + + # TODO: compare benchmarks diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml new file mode 100644 index 0000000..93cf2d6 --- /dev/null +++ b/.github/workflows/nightly.yaml @@ -0,0 +1,51 @@ +name: Docker Nightly +on: + workflow_dispatch: + inputs: + target: + description: "Override source branch (optional)" + type: string + required: false + + schedule: + - cron: "40 4 * * *" + +permissions: + contents: read + +jobs: + run: + runs-on: ${{ vars.GHA_WORKER_RELEASE || 'ubuntu-latest' }} + permissions: + contents: write + steps: + - name: Checkout NATS + uses: actions/checkout@v6 + with: + path: src/github.com/nats-io/nats-server + ref: ${{ inputs.target || 'main' }} + fetch-depth: 0 + fetch-tags: true + + - name: Checkout NSC + uses: actions/checkout@v6 + with: + repository: nats-io/nsc + path: src/github.com/nats-io/nsc + fetch-depth: 1 + fetch-tags: true + + - name: Checkout NATS CLI + uses: actions/checkout@v6 + with: + repository: nats-io/natscli + path: src/github.com/nats-io/natscli + fetch-depth: 1 + fetch-tags: true + + - name: Build Docker nightly + uses: ./src/github.com/nats-io/nats-server/.github/actions/nightly-release + with: + name: ${{ inputs.target || 'nightly' }} + hub_username: "${{ secrets.DOCKER_USERNAME }}" + hub_password: "${{ secrets.DOCKER_PASSWORD }}" diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml new file mode 100644 index 0000000..a4d9f49 --- /dev/null +++ b/.github/workflows/pull-requests.yml @@ -0,0 +1,59 @@ +name: NATS Server PRs + +on: + pull_request_target: + types: [opened, synchronize, reopened, ready_for_review] + workflow_dispatch: + +permissions: + contents: read + pull-requests: write + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + signoffs: + name: Sign-offs + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + steps: + - name: Checkout base repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Fetch PR head + run: | + set -euo pipefail + git fetch --no-tags origin "$BASE_SHA" "pull/${PR_NUMBER}/head:pr-head" + + - name: Check all PR commits are signed off + run: | + set -euo pipefail + + range="${BASE_SHA}..pr-head" + missing=0 + missing_list="" + + while read -r c; do + msg="$(git log -1 --pretty=%B "$c")" + if ! printf '%s\n' "$msg" | grep -q '^Signed-off-by:'; then + echo "::error ::Commit $c has not been signed off" + missing=1 + missing_list+=$'\n- '"$c" + fi + done < <(git rev-list --no-merges "$range") + + if [ "$missing" -ne 0 ]; then + body="$(printf 'Please review the `CONTRIBUTING.md` guide as some commits are missing `Signed-off-by:` in their commit messages.\n\nTo correct, please amend the following commits and force-push:\n\n%s' "$missing_list")" + gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --body "$body" + exit 1 + fi diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..ade9add --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,54 @@ +name: NATS Server Releases +on: + push: + tags: + - v* + +permissions: + contents: read + +jobs: + run: + name: GitHub Release + runs-on: ${{ vars.GHA_WORKER_RELEASE || 'ubuntu-latest' }} + permissions: + contents: write + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version: "stable" + + - name: Check version matches tag + env: + TRAVIS_TAG: ${{ github.ref_name }} + run: | + go test -race -v -run=TestVersionMatchesTag ./server -ldflags="-X=github.com/nats-io/nats-server/v2/server.serverVersion=$TRAVIS_TAG" -count=1 -vet=off + + - name: Install cosign + # Use commit hash here to avoid a re-tagging attack, as this is a third-party action + # Commit 6f9f17788090df1f26f669e9d70d6ae9567deba6 = tag v4.1.2 + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 + + - name: Install syft + # Use commit hash here to avoid a re-tagging attack, as this is a third-party action + # Commit e22c389904149dbc22b58101806040fa8d37a610 = tag v0.24.0 + uses: anchore/sbom-action/download-syft@e22c389904149dbc22b58101806040fa8d37a610 + with: + syft-version: "v1.42.3" + + - name: Create release + uses: goreleaser/goreleaser-action@v7 + with: + distribution: goreleaser + version: "~> v2" + args: release --clean + env: + GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/stale-issues.yaml b/.github/workflows/stale-issues.yaml new file mode 100644 index 0000000..4880a50 --- /dev/null +++ b/.github/workflows/stale-issues.yaml @@ -0,0 +1,21 @@ +name: Stale Issues +on: + schedule: + - cron: "30 1 * * *" + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + runs-on: ubuntu-latest + + steps: + - uses: actions/stale@v10 + with: + stale-issue-label: stale + stale-pr-label: stale + days-before-stale: 56 # Mark stale after 8 weeks (56 days) of inactivity + days-before-close: -1 # Disable auto-closing + exempt-all-milestones: true # Any issue/PR within a milestone will be omitted diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 0000000..9d689f2 --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,372 @@ +name: NATS Server Tests + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +env: + RACE: ${{ (github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/heads/release/') && github.event_name != 'pull_request') && '-race' || '' }} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/heads/release/') }} + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version-file: "go.mod" + + - name: Run golangci-lint + # Use commit hash here to avoid a re-tagging attack, as this is a third-party action + # Commit 82606bf257cbaff209d206a39f5134f0cfbfd2ee = tag v9.2.1 + uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee + with: + version: v2.13.0 + skip-cache: true + skip-save-cache: true + args: --timeout=5m --config=.golangci.yml + + build-latest: + name: Build (Latest Go) + needs: [lint] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Build NATS Server 64-bit + run: GOARCH=amd64 GOOS=linux go build + + - name: Build NATS Server 32-bit + run: GOARCH=386 GOOS=linux go build + + build-supported: + name: Build (Minimum Go) + needs: [lint] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version-file: "go.mod" + + - name: Build NATS Server 64-bit + run: GOARCH=amd64 GOOS=linux go build + + - name: Build NATS Server 32-bit + run: GOARCH=386 GOOS=linux go build + + # Using GitHub-supplied workers for Windows for now. + # Note that the below testing steps depend on the Linux build + # only, as the Windows builds take a fair bit longer to set up. + build-windows: + name: Build (Minimum Go, ${{ matrix.os }}) + needs: [lint] + strategy: + fail-fast: false + matrix: + os: [windows-2022, windows-2025] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version-file: "go.mod" + + - name: Build NATS Server + run: go build + + store: + name: Test Stores + needs: [build-latest, build-supported, lint] + runs-on: ${{ vars.GHA_WORKER_MEDIUM || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Run unit tests + run: ./scripts/runTestsOnTravis.sh store_tests + + js-no-cluster: + name: Test JetStream + needs: [build-latest, build-supported, lint] + runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Run unit tests + run: ./scripts/runTestsOnTravis.sh js_tests + + raft: + name: Test Raft + needs: [build-latest, build-supported, lint] + runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Run unit tests + run: ./scripts/runTestsOnTravis.sh raft_tests + + js-consumers: + name: Test JetStream Consumers + needs: [build-latest, build-supported, lint] + runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Run unit tests + run: ./scripts/runTestsOnTravis.sh js_consumer_tests + + js-cluster-1: + name: Test JetStream Cluster 1 + needs: [build-latest, build-supported, lint] + runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Run unit tests + run: ./scripts/runTestsOnTravis.sh js_cluster_tests_1 + + js-cluster-2: + name: Test JetStream Cluster 2 + needs: [build-latest, build-supported, lint] + runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Run unit tests + run: ./scripts/runTestsOnTravis.sh js_cluster_tests_2 + + js-cluster-3: + name: Test JetStream Cluster 3 + needs: [build-latest, build-supported, lint] + runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Run unit tests + run: ./scripts/runTestsOnTravis.sh js_cluster_tests_3 + + js-cluster-4: + name: Test JetStream Cluster 4 + needs: [build-latest, build-supported, lint] + runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Run unit tests + run: ./scripts/runTestsOnTravis.sh js_cluster_tests_4 + + js-supercluster: + name: Test JetStream Supercluster + needs: [build-latest, build-supported, lint] + runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Run unit tests + run: ./scripts/runTestsOnTravis.sh js_super_cluster_tests + + no-race-1: + name: Test No-Race 1 + needs: [build-latest, build-supported, lint] + runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Run unit tests + run: ./scripts/runTestsOnTravis.sh no_race_1_tests + + no-race-2: + name: Test No-Race 2 + needs: [build-latest, build-supported, lint] + runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Run unit tests + run: ./scripts/runTestsOnTravis.sh no_race_2_tests + + mqtt: + name: Test MQTT + needs: [build-latest, build-supported, lint] + runs-on: ${{ vars.GHA_WORKER_MEDIUM || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Run unit tests + run: ./scripts/runTestsOnTravis.sh mqtt_tests + + msgtrace: + name: Test Message Tracing + needs: [build-latest, build-supported, lint] + runs-on: ${{ vars.GHA_WORKER_MEDIUM || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Run unit tests + run: ./scripts/runTestsOnTravis.sh msgtrace_tests + + jwt: + name: Test JWT + needs: [build-latest, build-supported, lint] + runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Run unit tests + run: ./scripts/runTestsOnTravis.sh jwt_tests + + server-pkg-non-js: + name: Test Remaining Server Tests + needs: [build-latest, build-supported, lint] + runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Run unit tests + run: ./scripts/runTestsOnTravis.sh srv_pkg_non_js_tests + + non-server-pkg: + name: Test Other Packages + needs: [build-latest, build-supported, lint] + runs-on: ${{ vars.GHA_WORKER_MEDIUM || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: stable + + - name: Run unit tests + run: ./scripts/runTestsOnTravis.sh non_srv_pkg_tests diff --git a/.github/workflows/vuln.yaml b/.github/workflows/vuln.yaml new file mode 100644 index 0000000..bfd0ffa --- /dev/null +++ b/.github/workflows/vuln.yaml @@ -0,0 +1,33 @@ +name: Vulnerability Scan + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: "0 6 * * *" + +permissions: + contents: read + security-events: write + +jobs: + govulncheck: + runs-on: ${{ vars.GHA_WORKER_SMALL || 'ubuntu-latest' }} + + steps: + - name: Run govulncheck + # Use commit hash here to avoid a re-tagging attack, as this is a third-party action + # Commit b625fbe08f3bccbe446d94fbf87fcc875a4f50ee = tag v1.0.4 + uses: golang/govulncheck-action@b625fbe08f3bccbe446d94fbf87fcc875a4f50ee + with: + go-package: ./... + output-format: sarif + output-file: govulncheck.sarif + + - name: Upload SARIF results + if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork + uses: github/codeql-action/upload-sarif@v4 + with: + sarif_file: govulncheck.sarif diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dd7aaf4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,55 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test +dist + +# Configuration Files +*.conf +*.cfg + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe + +# Eclipse +.project + +# IntelliJ +.idea/ + +# Emacs +*~ +\#*\# +.\#* + +# Visual Studio Code +.vscode + +# Mac +.DS_Store + +# bin +nats-server +gnatsd +check + +# coverage +coverage.out + +# Cross compiled binaries +pkg diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..2af0412 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,90 @@ +version: "2" +run: + concurrency: 4 + modules-download-mode: readonly + issues-exit-code: 1 + tests: true +output: + formats: + text: + path: stdout + print-linter-name: true + print-issued-lines: true +linters: + default: none + enable: + - forbidigo + - govet + - ineffassign + - misspell + - staticcheck + - unused + settings: + errcheck: + check-type-assertions: false + check-blank: false + forbidigo: + forbid: + - pattern: ^fmt\.Print(f|ln)?$ + govet: + settings: + printf: + funcs: + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf + misspell: + locale: US + staticcheck: + checks: + - all + - -QF* + - -ST1003 + - -ST1016 + - -SA1019 + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + rules: + - linters: + - forbidigo + path: main.go + - linters: + - forbidigo + source: "nats-server: v%s" + - linters: + - forbidigo + path: server/opts.go + - linters: + - forbidigo + path: _test.go + paths: + - .github + - doc + - docker + - logos + - scripts + - util + - third_party$ + - builtin$ + - examples$ +formatters: + enable: + - goimports + exclusions: + generated: lax + paths: + - .github + - doc + - docker + - logos + - scripts + - util + - third_party$ + - builtin$ + - examples$ diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..7b7d5eb --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,118 @@ +project_name: nats-server +version: 2 + +release: + github: + owner: '{{ envOrDefault "GITHUB_REPOSITORY_OWNER" "nats-io" }}' + name: '{{ envOrDefault "GITHUB_REPOSITORY_NAME" "nats-server" }}' + name_template: "Release {{.Tag}}" + draft: true + +changelog: + disable: true + +builds: + - main: . + binary: nats-server + flags: + - -trimpath + ldflags: + - -w -X 'github.com/nats-io/nats-server/v2/server.gitCommit={{.ShortCommit}}' -X 'github.com/nats-io/nats-server/v2/server.serverVersion={{.Tag}}' + env: + # This is the toolchain version we use for releases. To override, set the env var, e.g.: + # GORELEASER_TOOLCHAIN="go1.22.8" TARGET='linux_amd64' goreleaser build --snapshot --clean --single-target + - GOTOOLCHAIN={{ envOrDefault "GORELEASER_TOOLCHAIN" "go1.27.1" }} + - GO111MODULE=on + - CGO_ENABLED=0 + goos: + - darwin + - linux + - windows + - freebsd + goarch: + - amd64 + - arm + - arm64 + - 386 + - loong64 + - mips64le + - s390x + - ppc64le + # RISC-V currently only supported on Linux + - riscv64 + goarm: + - 6 + - 7 + ignore: + - goos: windows + goarch: arm + - goos: darwin + goarch: 386 + - goos: freebsd + goarch: arm + - goos: freebsd + goarch: arm64 + - goos: freebsd + goarch: 386 + - goos: darwin + goarch: riscv64 + - goos: windows + goarch: riscv64 + - goos: freebsd + goarch: riscv64 + mod_timestamp: "{{ .CommitTimestamp }}" + +nfpms: + - file_name_template: "{{.ProjectName}}-{{.Tag}}-{{.Arch}}{{if .Arm}}{{.Arm}}{{end}}" + homepage: https://nats.io + description: High-Performance server for NATS, the cloud native messaging system. + maintainer: Ivan Kozlovic + license: Apache 2.0 + vendor: Synadia Inc. + formats: + - deb + - rpm + mtime: "{{ .CommitDate }}" + contents: + - src: /usr/bin/nats-server + dst: /usr/local/bin/nats-server + type: "symlink" + rpm: + buildhost: synadia.com + +archives: + - name_template: "{{.ProjectName}}-{{.Tag}}-{{.Os}}-{{.Arch}}{{if .Arm}}{{.Arm}}{{end}}" + id: targz-archives + wrap_in_directory: true + formats: ["tar.gz"] + format_overrides: + - goos: windows + formats: ["zip"] + builds_info: + owner: root + group: root + mode: 0755 + files: + - src: README.md + info: + owner: root + group: root + mode: 0644 + mtime: "{{ .CommitDate }}" + - src: LICENSE + info: + owner: root + group: root + mode: 0644 + mtime: "{{ .CommitDate }}" + +checksum: + name_template: "SHA256SUMS" + algorithm: sha256 + +sboms: + - artifacts: binary + documents: + [ + "{{.ProjectName}}-{{.Tag}}-{{.Os}}-{{.Arch}}{{if .Arm}}{{.Arm}}{{end}}.sbom.spdx.json", + ] diff --git a/AMBASSADORS.md b/AMBASSADORS.md new file mode 100644 index 0000000..82f38c3 --- /dev/null +++ b/AMBASSADORS.md @@ -0,0 +1,5 @@ +# Ambassadors + +The NATS ambassador program recognizes community members that go above and beyond in their contributions to the community and the ecosystem. Learn more [here](https://nats.io/community#nats-ambassador-program). + +- [Maurice van Veen](https://nats.io/community#maurice-van-veen) diff --git a/CODE-OF-CONDUCT.md b/CODE-OF-CONDUCT.md new file mode 100644 index 0000000..b850d49 --- /dev/null +++ b/CODE-OF-CONDUCT.md @@ -0,0 +1,3 @@ +## Community Code of Conduct + +NATS follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..99c1c67 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,52 @@ +# Contributing + +Thanks for your interest in contributing! This document contains `nats-io/nats-server` specific contributing details. If you are a first-time contributor, please refer to the general [NATS Contributor Guide](https://nats.io/contributing/) to get a comprehensive overview of contributing to the NATS project. + +## Getting started + +There are three general ways you can contribute to this repo: + +- Proposing an enhancement or new feature +- Reporting a bug or regression +- Contributing changes to the source code + +For the first two, refer to the [GitHub Issues](https://github.com/nats-io/nats-server/issues/new/choose) which guides you through the available options along with the needed information to collect. + +## Contributing changes + +_Prior to opening a pull request, it is recommended to open an issue first to ensure the maintainers can review intended changes. This saves time for both you and us. Exceptions to this rule include fixing non-functional source such as code comments, documentation or other supporting files._ + +Proposing source code changes is done through GitHub's standard pull request workflow. + +If your branch is a work-in-progress then please start by creating your pull requests as draft, by clicking the down-arrow next to the `Create pull request` button and instead selecting `Create draft pull request`. + +This will defer the automatic process of requesting a review from the NATS team and significantly reduces noise until you are ready. Once you are happy, you can click the `Ready for review` button. + +### Guidelines + +A good pull request includes: + +- A succinct yet descriptive title describing, in a few words, what is fixed/improved/optimised by this change. +- A high-level description of the changes, including an overview of _why_ the changes are relevant or what problem you are trying to solve. Include links to any issues that are related by adding comments like `Resolves #NNN` to your description. See [Linking a Pull Request to an Issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue) for more information. +- An up-to-date parent commit. Please make sure you are pulling in the latest `main` branch and rebasing your work on top of it, i.e. `git rebase main`. +- Unit tests where appropriate. Bug fixes will benefit from the addition of regression tests and performance improvements will benefit from the addition of new benchmarks where possible. New features will **NOT** be accepted without suitable test coverage! +- No more commits than necessary. Sometimes having multiple commits is useful for telling a story or isolating changes from one another, but please squash down any unnecessary commits that may just be for clean-up, comments or small changes. +- No additional external dependencies that aren't absolutely essential. Please do everything you can to avoid pulling in additional libraries/dependencies into `go.mod` as we will be very critical of these. + +### AI Policy + +We are happy to accept AI-assisted contributions, but we are not happy to talk to your AI agent during the process. When submitting an issue or a pull request, communicate with us in your own words and [do not meat-proxy](https://nomeatproxy.com). If we wanted to talk to an AI, we can do it ourselves! If you are submitting code, we expect you to understand how the code works and that it has been appropriately tested, even if an AI has written or reviewed it. + +We also use AI review tools for pull requests, but it is always a human that makes the final decision on whether or not to accept it, so please engage faithfully during the code review process. + +### Sign-off + +In order to accept a contribution, you will first need to certify that the contribution is your original work and that you license the work to the project under the [Apache-2.0 license](https://github.com/nats-io/nats-server/blob/main/LICENSE). + +This is done by using `Signed-off-by: Your Name ` statement which should appear in all of your commit messages. Please note that we can only accept sign-offs under a legal name. Nicknames and aliases are not permitted. + +To perform a sign-off when committing with `git`, use `git commit -s` (or `--signoff`) to add the `Signed-off-by:` trailer to your commit message. + +## Get help + +If you have questions about the contribution process, please start a [GitHub discussion](https://github.com/nats-io/nats-server/discussions), join the [NATS Slack](https://slack.nats.io/), or send your question to the [NATS Google Group](https://groups.google.com/forum/#!forum/natsio). diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md new file mode 100644 index 0000000..fb3717a --- /dev/null +++ b/DEPENDENCIES.md @@ -0,0 +1,18 @@ +# External Dependencies + +This file lists the dependencies used in this repository. + +| Dependency | License | +|-|-| +| Go | BSD 3-Clause "New" or "Revised" License | +| github.com/nats-io/nats-server/v2 | Apache License 2.0 | +| github.com/google/go-tpm | Apache License 2.0 | +| github.com/klauspost/compress | BSD 3-Clause "New" or "Revised" License | +| github.com/minio/highwayhash | Apache License 2.0 | +| github.com/nats-io/jwt/v2 | Apache License 2.0 | +| github.com/nats-io/nats.go | Apache License 2.0 | +| github.com/nats-io/nkeys | Apache License 2.0 | +| github.com/nats-io/nuid | Apache License 2.0 | +| golang.org/x/crypto | BSD 3-Clause "New" or "Revised" License | +| golang.org/x/sys | BSD 3-Clause "New" or "Revised" License | +| golang.org/x/time | BSD 3-Clause "New" or "Revised" License | diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 0000000..5b2bc43 --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,3 @@ +# NATS Server Governance + +NATS Server is part of the NATS project and is subject to the [NATS Governance](https://github.com/nats-io/nats-general/blob/main/GOVERNANCE.md). diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MAINTAINERS.md b/MAINTAINERS.md new file mode 100644 index 0000000..c89f1c2 --- /dev/null +++ b/MAINTAINERS.md @@ -0,0 +1,10 @@ +# Maintainers + +Maintainership is on a per project basis. Reference [NATS Governance Model](https://github.com/nats-io/nats-general/blob/main/GOVERNANCE.md). + +### Maintainers + - Derek Collison [@derekcollison](https://github.com/derekcollison) + - Ivan Kozlovic [@kozlovic](https://github.com/kozlovic) + - Waldemar Quevedo [@wallyqs](https://github.com/wallyqs) + - Oleg Shaldybin [@olegshaldybin](https://github.com/olegshaldybin) + - R.I. Pienaar [@ripienaar](https://github.com/ripienaar) diff --git a/README.md b/README.md new file mode 100644 index 0000000..4487552 --- /dev/null +++ b/README.md @@ -0,0 +1,76 @@ +

+ NATS Logo +

+ +[NATS](https://nats.io) is a simple, secure and performant communications system for digital systems, services and devices. NATS is part of the Cloud Native Computing Foundation ([CNCF](https://cncf.io)). NATS has over [40 client language implementations](https://nats.io/download/), and its server can run on-premise, in the cloud, at the edge, and even on a Raspberry Pi. NATS can secure and simplify design and operation of modern distributed systems. + +[![License][License-Image]][License-Url] [![Build][Build-Status-Image]][Build-Status-Url] [![Release][Release-Image]][Release-Url] [![Slack][Slack-Image]][Slack-Url] [![Coverage][Coverage-Image]][Coverage-Url] [![Docker Downloads][Docker-Image]][Docker-Url] [![GitHub Downloads][GitHub-Image]][Somsubhra-URL] [![CII Best Practices][CIIBestPractices-Image]][CIIBestPractices-Url] [![Artifact Hub][ArtifactHub-Image]][ArtifactHub-Url] + +## Documentation + +- [Official Website](https://nats.io) +- [Official Documentation](https://docs.nats.io) +- [FAQ](https://docs.nats.io/reference/faq) +- Watch [a video overview](https://rethink.synadia.com/episodes/1/) of NATS. +- Watch [this video from SCALE 13x](https://www.youtube.com/watch?v=sm63oAVPqAM) to learn more about its origin story and design philosophy. + +## Contact + +- [Twitter](https://twitter.com/nats_io): Follow us on Twitter! +- [Google Groups](https://groups.google.com/forum/#!forum/natsio): Where you can ask questions +- [Slack](https://natsio.slack.com): Click [here](https://slack.nats.io) to join. You can ask questions to our maintainers and to the rich and active community. + +## Contributing + +If you are interested in contributing to NATS, read about our... + +- [Contributing guide](./CONTRIBUTING.md) +- [Report issues or propose Pull Requests](https://github.com/nats-io) + +[License-Url]: https://www.apache.org/licenses/LICENSE-2.0 +[License-Image]: https://img.shields.io/badge/License-Apache2-blue.svg +[Docker-Image]: https://img.shields.io/docker/pulls/_/nats.svg +[Docker-Url]: https://hub.docker.com/_/nats +[Slack-Image]: https://img.shields.io/badge/chat-on%20slack-green +[Slack-Url]: https://slack.nats.io +[Fossa-Url]: https://app.fossa.io/projects/git%2Bgithub.com%2Fnats-io%2Fnats-server?ref=badge_shield +[Fossa-Image]: https://app.fossa.io/api/projects/git%2Bgithub.com%2Fnats-io%2Fnats-server.svg?type=shield +[Build-Status-Url]: https://github.com/nats-io/nats-server/actions/workflows/tests.yaml +[Build-Status-Image]: https://github.com/nats-io/nats-server/actions/workflows/tests.yaml/badge.svg?branch=main +[Release-Url]: https://github.com/nats-io/nats-server/releases/latest +[Release-Image]: https://img.shields.io/github/v/release/nats-io/nats-server +[Coverage-Url]: https://coveralls.io/r/nats-io/nats-server?branch=main +[Coverage-image]: https://coveralls.io/repos/github/nats-io/nats-server/badge.svg?branch=main +[ReportCard-Url]: https://goreportcard.com/report/nats-io/nats-server +[ReportCard-Image]: https://goreportcard.com/badge/github.com/nats-io/nats-server +[CIIBestPractices-Url]: https://bestpractices.coreinfrastructure.org/projects/1895 +[CIIBestPractices-Image]: https://bestpractices.coreinfrastructure.org/projects/1895/badge +[ArtifactHub-Url]: https://artifacthub.io/packages/helm/nats/nats +[ArtifactHub-Image]: https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/nats +[GitHub-Release]: https://github.com/nats-io/nats-server/releases/ +[GitHub-Image]: https://img.shields.io/github/downloads/nats-io/nats-server/total.svg?logo=github +[Somsubhra-url]: https://somsubhra.github.io/github-release-stats/?username=nats-io&repository=nats-server + +## Roadmap + +The NATS product roadmap can be found [here](https://nats.io/about/#roadmap). + +## Adopters + +Who uses NATS? See our [list of users](https://nats.io/#who-uses-nats) on [https://nats.io](https://nats.io). + +## Security + +### Security Audit + +A third party security audit was performed by Trail of Bits following engagement by the Open Source Technology Improvement Fund (OSTIF). You can see the [full report from April 2025 here](https://github.com/trailofbits/publications/blob/master/reviews/2025-04-ostif-nats-securityreview.pdf). + +### Reporting Security Vulnerabilities + +If you've found a vulnerability or a potential vulnerability in the NATS server, please let us know at +[nats-security](mailto:security@nats.io). + +## License + +Unless otherwise noted, the NATS source files are distributed +under the Apache Version 2.0 license found in the LICENSE file. diff --git a/RELEASES.md b/RELEASES.md new file mode 100644 index 0000000..3d8e9e5 --- /dev/null +++ b/RELEASES.md @@ -0,0 +1,9 @@ +# Release Workflow + +- As of NATS 2.11, the server follows a 6-month release cycle. 2.11 was released in March 2025. +- Version tagging follows semantic versioning with security fixes clearly marked. +- The `main` branch is for active development of features and bug fixes. Release branches are created for preparing patch releases. Releases are tagged with semantic versions +- The current and previous minor series are supported in terms of bug fixes and security patches. +- Binaries are released via [GitHub Releases](https://github.com/nats-io/nats-server/releases) and Docker images are available as an [official image](https://hub.docker.com/_/nats). +- Preview releases for the next minor version, e.g. 2.14.0, become available once feature development is complete. +- Tagged releases as well as nightly Docker images of `main` (for testing) are available in the [synadia/nats-server](https://hub.docker.com/r/synadia/nats-server/tags) Docker repository. diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..aa6aef4 --- /dev/null +++ b/TODO.md @@ -0,0 +1,54 @@ + +# General + +- [ ] Auth for queue groups? +- [ ] Blacklist or ERR escalation to close connection for auth/permissions +- [ ] Protocol updates, MAP, MPUB, etc +- [ ] Multiple listen endpoints +- [ ] Websocket / HTTP2 strategy +- [ ] T series reservations +- [ ] _SYS. server events? +- [ ] No downtime restart +- [ ] Signal based reload of configuration +- [ ] brew, apt-get, rpm, chocately (windows) +- [ ] IOVec pools and writev for high fanout? +- [ ] Modify cluster support for single message across routes between pub/sub and d-queue +- [ ] Memory limits/warnings? +- [ ] Limit number of subscriptions a client can have, total memory usage etc. +- [ ] Multi-tenant accounts with isolation of subject space +- [ ] Pedantic state +- [X] _SYS.> reserved for server events? +- [X] Listen configure key vs addr and port +- [X] Add ENV and variable support to dconf? ucl? +- [X] Buffer pools/sync pools? +- [X] Multiple Authorization / Access +- [X] Write dynamic socket buffer sizes +- [X] Read dynamic socket buffer sizes +- [X] Info updates contain other implicit route servers +- [X] Sublist better at high concurrency, cache uses writelock always currently +- [X] Switch to 1.4/1.5 and use maps vs hashmaps in sublist +- [X] NewSource on Rand to lower lock contention on QueueSubs, or redesign! +- [X] Default sort by cid on connz +- [X] Track last activity time per connection? +- [X] Add total connections to varz so we won't miss spikes, etc. +- [X] Add starttime and uptime to connz list. +- [X] Gossip Protocol for discovery for clustering +- [X] Add in HTTP requests to varz? +- [X] Add favico and help link for monitoring? +- [X] Better user/pass support using bcrypt etc. +- [X] SSL/TLS support +- [X] Add support for / to point to varz, connz, etc.. +- [X] Support sort options for /connz via nats-top +- [X] Dropped message statistics (slow consumers) +- [X] Add current time to each monitoring endpoint +- [X] varz uptime do days and only integer secs +- [X] Place version in varz (same info sent to clients) +- [X] Place server ID/UUID in varz +- [X] nats-top equivalent, utils +- [X] Connz report routes (/routez) +- [X] Docker +- [X] Remove reliance on `ps` +- [X] Syslog support +- [X] Client support for language and version +- [X] Fix benchmarks on linux +- [X] Daemon mode? Won't fix diff --git a/conf/fuzz.go b/conf/fuzz.go new file mode 100644 index 0000000..e42a82e --- /dev/null +++ b/conf/fuzz.go @@ -0,0 +1,24 @@ +// Copyright 2020-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build gofuzz + +package conf + +func Fuzz(data []byte) int { + _, err := Parse(string(data)) + if err != nil { + return 0 + } + return 1 +} diff --git a/conf/lex.go b/conf/lex.go new file mode 100644 index 0000000..80a9a11 --- /dev/null +++ b/conf/lex.go @@ -0,0 +1,1319 @@ +// Copyright 2013-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Customized heavily from +// https://github.com/BurntSushi/toml/blob/master/lex.go, which is based on +// Rob Pike's talk: http://cuddle.googlecode.com/hg/talk/lex.html + +// The format supported is less restrictive than today's formats. +// Supports mixed Arrays [], nested Maps {}, multiple comment types (# and //) +// Also supports key value assignments using '=' or ':' or whiteSpace() +// e.g. foo = 2, foo : 2, foo 2 +// maps can be assigned with no key separator as well +// semicolons as value terminators in key/value assignments are optional +// +// see lex_test.go for more examples. + +package conf + +import ( + "encoding/hex" + "fmt" + "strings" + "unicode" + "unicode/utf8" +) + +type itemType int + +const ( + itemError itemType = iota + itemNIL // used in the parser to indicate no type + itemEOF + itemKey + itemText + itemString + itemBool + itemInteger + itemFloat + itemDatetime + itemArrayStart + itemArrayEnd + itemMapStart + itemMapEnd + itemCommentStart + itemVariable + itemInclude +) + +const ( + eof = 0 + mapStart = '{' + mapEnd = '}' + keySepEqual = '=' + keySepColon = ':' + arrayStart = '[' + arrayEnd = ']' + arrayValTerm = ',' + mapValTerm = ',' + commentHashStart = '#' + commentSlashStart = '/' + dqStringStart = '"' + dqStringEnd = '"' + sqStringStart = '\'' + sqStringEnd = '\'' + optValTerm = ';' + topOptStart = '{' + topOptValTerm = ',' + topOptTerm = '}' + blockStart = '(' + blockEnd = ')' + mapEndString = string(mapEnd) +) + +type stateFn func(lx *lexer) stateFn + +type lexer struct { + input string + start int + pos int + width int + line int + state stateFn + items chan item + + // A stack of state functions used to maintain context. + // The idea is to reuse parts of the state machine in various places. + // For example, values can appear at the top level or within arbitrarily + // nested arrays. The last state on the stack is used after a value has + // been lexed. Similarly for comments. + stack []stateFn + + // Used for processing escapable substrings in double-quoted and raw strings + stringParts []string + stringStateFn stateFn + + // lstart is the start position of the current line. + lstart int + + // ilstart is the start position of the line from the current item. + ilstart int +} + +type item struct { + typ itemType + val string + line int + pos int +} + +func (lx *lexer) nextItem() item { + for { + select { + case item := <-lx.items: + return item + default: + lx.state = lx.state(lx) + } + } +} + +func lex(input string) *lexer { + lx := &lexer{ + input: input, + state: lexTop, + line: 1, + items: make(chan item, 10), + stack: make([]stateFn, 0, 10), + stringParts: []string{}, + } + return lx +} + +func (lx *lexer) push(state stateFn) { + lx.stack = append(lx.stack, state) +} + +func (lx *lexer) pop() stateFn { + if len(lx.stack) == 0 { + return lx.errorf("BUG in lexer: no states to pop.") + } + li := len(lx.stack) - 1 + last := lx.stack[li] + lx.stack = lx.stack[0:li] + return last +} + +func (lx *lexer) emit(typ itemType) { + val := strings.Join(lx.stringParts, "") + lx.input[lx.start:lx.pos] + // Position of item in line where it started. + pos := lx.pos - lx.ilstart - len(val) + lx.items <- item{typ, val, lx.line, pos} + lx.start = lx.pos + lx.ilstart = lx.lstart +} + +func (lx *lexer) emitString() { + var finalString string + if len(lx.stringParts) > 0 { + finalString = strings.Join(lx.stringParts, "") + lx.input[lx.start:lx.pos] + lx.stringParts = []string{} + } else { + finalString = lx.input[lx.start:lx.pos] + } + // Position of string in line where it started. + pos := lx.pos - lx.ilstart - len(finalString) + lx.items <- item{itemString, finalString, lx.line, pos} + lx.start = lx.pos + lx.ilstart = lx.lstart +} + +func (lx *lexer) addCurrentStringPart(offset int) { + lx.stringParts = append(lx.stringParts, lx.input[lx.start:lx.pos-offset]) + lx.start = lx.pos +} + +func (lx *lexer) addStringPart(s string) stateFn { + lx.stringParts = append(lx.stringParts, s) + lx.start = lx.pos + return lx.stringStateFn +} + +func (lx *lexer) hasEscapedParts() bool { + return len(lx.stringParts) > 0 +} + +func (lx *lexer) next() (r rune) { + if lx.pos >= len(lx.input) { + lx.width = 0 + return eof + } + + if lx.input[lx.pos] == '\n' { + lx.line++ + + // Mark start position of current line. + lx.lstart = lx.pos + } + r, lx.width = utf8.DecodeRuneInString(lx.input[lx.pos:]) + lx.pos += lx.width + + return r +} + +// ignore skips over the pending input before this point. +func (lx *lexer) ignore() { + lx.start = lx.pos + lx.ilstart = lx.lstart +} + +// backup steps back one rune. Can be called only once per call of next. +func (lx *lexer) backup() { + lx.pos -= lx.width + if lx.pos < len(lx.input) && lx.input[lx.pos] == '\n' { + lx.line-- + } +} + +// peek returns but does not consume the next rune in the input. +func (lx *lexer) peek() rune { + r := lx.next() + lx.backup() + return r +} + +// errorf stops all lexing by emitting an error and returning `nil`. +// Note that any value that is a character is escaped if it's a special +// character (new lines, tabs, etc.). +func (lx *lexer) errorf(format string, values ...any) stateFn { + for i, value := range values { + if v, ok := value.(rune); ok { + values[i] = escapeSpecial(v) + } + } + + // Position of error in current line. + pos := lx.pos - lx.lstart + lx.items <- item{ + itemError, + fmt.Sprintf(format, values...), + lx.line, + pos, + } + return nil +} + +// lexTop consumes elements at the top level of data structure. +func lexTop(lx *lexer) stateFn { + r := lx.next() + if unicode.IsSpace(r) { + return lexSkip(lx, lexTop) + } + + switch r { + case topOptStart: + lx.push(lexTop) + return lexSkip(lx, lexBlockStart) + case commentHashStart: + lx.push(lexTop) + return lexCommentStart + case commentSlashStart: + rn := lx.next() + if rn == commentSlashStart { + lx.push(lexTop) + return lexCommentStart + } + lx.backup() + fallthrough + case eof: + if lx.pos > lx.start { + return lx.errorf("Unexpected EOF.") + } + lx.emit(itemEOF) + return nil + } + + // At this point, the only valid item can be a key, so we back up + // and let the key lexer do the rest. + lx.backup() + lx.push(lexTopValueEnd) + return lexKeyStart +} + +// lexTopValueEnd is entered whenever a top-level value has been consumed. +// It must see only whitespace, and will turn back to lexTop upon a new line. +// If it sees EOF, it will quit the lexer successfully. +func lexTopValueEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case r == commentHashStart: + // a comment will read to a new line for us. + lx.push(lexTop) + return lexCommentStart + case r == commentSlashStart: + rn := lx.next() + if rn == commentSlashStart { + lx.push(lexTop) + return lexCommentStart + } + lx.backup() + fallthrough + case isWhitespace(r): + return lexTopValueEnd + case isNL(r) || r == eof || r == optValTerm || r == topOptValTerm || r == topOptTerm: + lx.ignore() + return lexTop + } + return lx.errorf("Expected a top-level value to end with a new line, "+ + "comment or EOF, but got '%v' instead.", r) +} + +func lexBlockStart(lx *lexer) stateFn { + r := lx.next() + if unicode.IsSpace(r) { + return lexSkip(lx, lexBlockStart) + } + + switch r { + case topOptStart: + lx.push(lexBlockEnd) + return lexSkip(lx, lexBlockStart) + case topOptTerm: + lx.ignore() + return lx.pop() + case commentHashStart: + lx.push(lexBlockStart) + return lexCommentStart + case commentSlashStart: + rn := lx.next() + if rn == commentSlashStart { + lx.push(lexBlockStart) + return lexCommentStart + } + lx.backup() + fallthrough + case eof: + if lx.pos > lx.start { + return lx.errorf("Unexpected EOF.") + } + lx.emit(itemEOF) + return nil + } + + // At this point, the only valid item can be a key, so we back up + // and let the key lexer do the rest. + lx.backup() + lx.push(lexBlockValueEnd) + return lexKeyStart +} + +// lexBlockValueEnd is entered whenever a block-level value has been consumed. +// It must see only whitespace, and will turn back to lexBlockStart upon a new line. +// If it sees EOF, it will quit the lexer successfully. +func lexBlockValueEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case r == commentHashStart: + // a comment will read to a new line for us. + lx.push(lexBlockValueEnd) + return lexCommentStart + case r == commentSlashStart: + rn := lx.next() + if rn == commentSlashStart { + lx.push(lexBlockValueEnd) + return lexCommentStart + } + lx.backup() + fallthrough + case isWhitespace(r): + return lexBlockValueEnd + case isNL(r) || r == optValTerm || r == topOptValTerm: + lx.ignore() + return lexBlockStart + case r == topOptTerm: + lx.backup() + return lexBlockEnd + } + return lx.errorf("Expected a block-level value to end with a new line, "+ + "comment or EOF, but got '%v' instead.", r) +} + +// lexBlockEnd is entered whenever a block-level value has been consumed. +// It must see only whitespace, and will turn back to lexTop upon a "}". +func lexBlockEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case r == commentHashStart: + // a comment will read to a new line for us. + lx.push(lexBlockStart) + return lexCommentStart + case r == commentSlashStart: + rn := lx.next() + if rn == commentSlashStart { + lx.push(lexBlockStart) + return lexCommentStart + } + lx.backup() + fallthrough + case isNL(r) || isWhitespace(r): + return lexBlockEnd + case r == optValTerm || r == topOptValTerm: + lx.ignore() + return lexBlockStart + case r == topOptTerm: + lx.ignore() + return lx.pop() + } + return lx.errorf("Expected a block-level to end with a '}', but got '%v' instead.", r) +} + +// lexKeyStart consumes a key name up until the first non-whitespace character. +// lexKeyStart will ignore whitespace. It will also eat enclosing quotes. +func lexKeyStart(lx *lexer) stateFn { + r := lx.peek() + switch { + case isKeySeparator(r): + return lx.errorf("Unexpected key separator '%v'", r) + case unicode.IsSpace(r): + lx.next() + return lexSkip(lx, lexKeyStart) + case r == dqStringStart: + lx.next() + return lexSkip(lx, lexDubQuotedKey) + case r == sqStringStart: + lx.next() + return lexSkip(lx, lexQuotedKey) + } + lx.ignore() + lx.next() + return lexKey +} + +// lexDubQuotedKey consumes the text of a key between quotes. +func lexDubQuotedKey(lx *lexer) stateFn { + r := lx.peek() + if r == dqStringEnd { + lx.emit(itemKey) + lx.next() + return lexSkip(lx, lexKeyEnd) + } else if r == eof { + if lx.pos > lx.start { + return lx.errorf("Unexpected EOF.") + } + lx.emit(itemEOF) + return nil + } + lx.next() + return lexDubQuotedKey +} + +// lexQuotedKey consumes the text of a key between quotes. +func lexQuotedKey(lx *lexer) stateFn { + r := lx.peek() + if r == sqStringEnd { + lx.emit(itemKey) + lx.next() + return lexSkip(lx, lexKeyEnd) + } else if r == eof { + if lx.pos > lx.start { + return lx.errorf("Unexpected EOF.") + } + lx.emit(itemEOF) + return nil + } + lx.next() + return lexQuotedKey +} + +// keyCheckKeyword will check for reserved keywords as the key value when the key is +// separated with a space. +func (lx *lexer) keyCheckKeyword(fallThrough, push stateFn) stateFn { + key := strings.ToLower(lx.input[lx.start:lx.pos]) + switch key { + case "include": + lx.ignore() + if push != nil { + lx.push(push) + } + return lexIncludeStart + } + lx.emit(itemKey) + return fallThrough +} + +// lexIncludeStart will consume the whitespace til the start of the value. +func lexIncludeStart(lx *lexer) stateFn { + r := lx.next() + if isWhitespace(r) { + return lexSkip(lx, lexIncludeStart) + } + lx.backup() + return lexInclude +} + +// lexIncludeQuotedString consumes the inner contents of a string. It assumes that the +// beginning '"' has already been consumed and ignored. It will not interpret any +// internal contents. +func lexIncludeQuotedString(lx *lexer) stateFn { + r := lx.next() + switch { + case r == sqStringEnd: + lx.backup() + lx.emit(itemInclude) + lx.next() + lx.ignore() + return lx.pop() + case r == eof: + return lx.errorf("Unexpected EOF in quoted include") + } + return lexIncludeQuotedString +} + +// lexIncludeDubQuotedString consumes the inner contents of a string. It assumes that the +// beginning '"' has already been consumed and ignored. It will not interpret any +// internal contents. +func lexIncludeDubQuotedString(lx *lexer) stateFn { + r := lx.next() + switch { + case r == dqStringEnd: + lx.backup() + lx.emit(itemInclude) + lx.next() + lx.ignore() + return lx.pop() + case r == eof: + return lx.errorf("Unexpected EOF in double quoted include") + } + return lexIncludeDubQuotedString +} + +// lexIncludeString consumes the inner contents of a raw string. +func lexIncludeString(lx *lexer) stateFn { + r := lx.next() + switch { + case isNL(r) || r == eof || r == optValTerm || r == mapEnd || isWhitespace(r): + lx.backup() + lx.emit(itemInclude) + return lx.pop() + case r == sqStringEnd: + lx.backup() + lx.emit(itemInclude) + lx.next() + lx.ignore() + return lx.pop() + } + return lexIncludeString +} + +// lexInclude will consume the include value. +func lexInclude(lx *lexer) stateFn { + r := lx.next() + switch { + case r == sqStringStart: + lx.ignore() // ignore the " or ' + return lexIncludeQuotedString + case r == dqStringStart: + lx.ignore() // ignore the " or ' + return lexIncludeDubQuotedString + case r == arrayStart: + return lx.errorf("Expected include value but found start of an array") + case r == mapStart: + return lx.errorf("Expected include value but found start of a map") + case r == blockStart: + return lx.errorf("Expected include value but found start of a block") + case unicode.IsDigit(r), r == '-': + return lx.errorf("Expected include value but found start of a number") + case r == '\\': + return lx.errorf("Expected include value but found escape sequence") + case isNL(r): + return lx.errorf("Expected include value but found new line") + } + lx.backup() + return lexIncludeString +} + +// lexKey consumes the text of a key. Assumes that the first character (which +// is not whitespace) has already been consumed. +func lexKey(lx *lexer) stateFn { + r := lx.peek() + if unicode.IsSpace(r) { + // Spaces signal we could be looking at a keyword, e.g. include. + // Keywords will eat the keyword and set the appropriate return stateFn. + return lx.keyCheckKeyword(lexKeyEnd, nil) + } else if isKeySeparator(r) || r == eof { + lx.emit(itemKey) + return lexKeyEnd + } + lx.next() + return lexKey +} + +// lexKeyEnd consumes the end of a key (up to the key separator). +// Assumes that the first whitespace character after a key (or the '=' or ':' +// separator) has NOT been consumed. +func lexKeyEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case unicode.IsSpace(r): + return lexSkip(lx, lexKeyEnd) + case isKeySeparator(r): + return lexSkip(lx, lexValue) + case r == eof: + lx.emit(itemEOF) + return nil + } + // We start the value here + lx.backup() + return lexValue +} + +// lexValue starts the consumption of a value anywhere a value is expected. +// lexValue will ignore whitespace. +// After a value is lexed, the last state on the next is popped and returned. +func lexValue(lx *lexer) stateFn { + // We allow whitespace to precede a value, but NOT new lines. + // In array syntax, the array states are responsible for ignoring new lines. + r := lx.next() + if isWhitespace(r) { + return lexSkip(lx, lexValue) + } + + switch { + case r == arrayStart: + lx.ignore() + lx.emit(itemArrayStart) + return lexArrayValue + case r == mapStart: + lx.ignore() + lx.emit(itemMapStart) + return lexMapKeyStart + case r == sqStringStart: + lx.ignore() // ignore the " or ' + return lexQuotedString + case r == dqStringStart: + lx.ignore() // ignore the " or ' + lx.stringStateFn = lexDubQuotedString + return lexDubQuotedString + case r == '-': + return lexNegNumberStart + case r == blockStart: + lx.ignore() + return lexBlock + case unicode.IsDigit(r): + lx.backup() // avoid an extra state and use the same as above + return lexNumberOrDateOrStringOrIPStart + case r == '.': // special error case, be kind to users + return lx.errorf("Floats must start with a digit") + case isNL(r): + return lx.errorf("Expected value but found new line") + } + lx.backup() + lx.stringStateFn = lexString + return lexString +} + +// lexArrayValue consumes one value in an array. It assumes that '[' or ',' +// have already been consumed. All whitespace and new lines are ignored. +func lexArrayValue(lx *lexer) stateFn { + r := lx.next() + switch { + case unicode.IsSpace(r): + return lexSkip(lx, lexArrayValue) + case r == commentHashStart: + lx.push(lexArrayValue) + return lexCommentStart + case r == commentSlashStart: + rn := lx.next() + if rn == commentSlashStart { + lx.push(lexArrayValue) + return lexCommentStart + } + lx.backup() + fallthrough + case r == arrayValTerm: + return lx.errorf("Unexpected array value terminator '%v'.", arrayValTerm) + case r == arrayEnd: + return lexArrayEnd + } + + lx.backup() + lx.push(lexArrayValueEnd) + return lexValue +} + +// lexArrayValueEnd consumes the cruft between values of an array. Namely, +// it ignores whitespace and expects either a ',' or a ']'. +func lexArrayValueEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r): + return lexSkip(lx, lexArrayValueEnd) + case r == commentHashStart: + lx.push(lexArrayValueEnd) + return lexCommentStart + case r == commentSlashStart: + rn := lx.next() + if rn == commentSlashStart { + lx.push(lexArrayValueEnd) + return lexCommentStart + } + lx.backup() + fallthrough + case r == arrayValTerm || isNL(r): + return lexSkip(lx, lexArrayValue) // Move onto next + case r == arrayEnd: + return lexArrayEnd + } + return lx.errorf("Expected an array value terminator %q or an array "+ + "terminator %q, but got '%v' instead.", arrayValTerm, arrayEnd, r) +} + +// lexArrayEnd finishes the lexing of an array. It assumes that a ']' has +// just been consumed. +func lexArrayEnd(lx *lexer) stateFn { + lx.ignore() + lx.emit(itemArrayEnd) + return lx.pop() +} + +// lexMapKeyStart consumes a key name up until the first non-whitespace +// character. +// lexMapKeyStart will ignore whitespace. +func lexMapKeyStart(lx *lexer) stateFn { + r := lx.peek() + switch { + case isKeySeparator(r): + return lx.errorf("Unexpected key separator '%v'.", r) + case r == arrayEnd: + return lx.errorf("Unexpected array end '%v' processing map.", r) + case unicode.IsSpace(r): + lx.next() + return lexSkip(lx, lexMapKeyStart) + case r == mapEnd: + lx.next() + return lexSkip(lx, lexMapEnd) + case r == commentHashStart: + lx.next() + lx.push(lexMapKeyStart) + return lexCommentStart + case r == commentSlashStart: + lx.next() + rn := lx.next() + if rn == commentSlashStart { + lx.push(lexMapKeyStart) + return lexCommentStart + } + lx.backup() + case r == sqStringStart: + lx.next() + return lexSkip(lx, lexMapQuotedKey) + case r == dqStringStart: + lx.next() + return lexSkip(lx, lexMapDubQuotedKey) + case r == eof: + return lx.errorf("Unexpected EOF processing map.") + } + lx.ignore() + lx.next() + return lexMapKey +} + +// lexMapQuotedKey consumes the text of a key between quotes. +func lexMapQuotedKey(lx *lexer) stateFn { + if r := lx.peek(); r == eof { + return lx.errorf("Unexpected EOF processing quoted map key.") + } else if r == sqStringEnd { + lx.emit(itemKey) + lx.next() + return lexSkip(lx, lexMapKeyEnd) + } + lx.next() + return lexMapQuotedKey +} + +// lexMapDubQuotedKey consumes the text of a key between quotes. +func lexMapDubQuotedKey(lx *lexer) stateFn { + if r := lx.peek(); r == eof { + return lx.errorf("Unexpected EOF processing double quoted map key.") + } else if r == dqStringEnd { + lx.emit(itemKey) + lx.next() + return lexSkip(lx, lexMapKeyEnd) + } + lx.next() + return lexMapDubQuotedKey +} + +// lexMapKey consumes the text of a key. Assumes that the first character (which +// is not whitespace) has already been consumed. +func lexMapKey(lx *lexer) stateFn { + if r := lx.peek(); r == eof { + return lx.errorf("Unexpected EOF processing map key.") + } else if unicode.IsSpace(r) { + // Spaces signal we could be looking at a keyword, e.g. include. + // Keywords will eat the keyword and set the appropriate return stateFn. + return lx.keyCheckKeyword(lexMapKeyEnd, lexMapValueEnd) + } else if isKeySeparator(r) { + lx.emit(itemKey) + return lexMapKeyEnd + } + lx.next() + return lexMapKey +} + +// lexMapKeyEnd consumes the end of a key (up to the key separator). +// Assumes that the first whitespace character after a key (or the '=' +// separator) has NOT been consumed. +func lexMapKeyEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case unicode.IsSpace(r): + return lexSkip(lx, lexMapKeyEnd) + case isKeySeparator(r): + return lexSkip(lx, lexMapValue) + } + // We start the value here + lx.backup() + return lexMapValue +} + +// lexMapValue consumes one value in a map. It assumes that '{' or ',' +// have already been consumed. All whitespace and new lines are ignored. +// Map values can be separated by ',' or simple NLs. +func lexMapValue(lx *lexer) stateFn { + r := lx.next() + switch { + case unicode.IsSpace(r): + return lexSkip(lx, lexMapValue) + case r == mapValTerm: + return lx.errorf("Unexpected map value terminator %q.", mapValTerm) + case r == mapEnd: + return lexSkip(lx, lexMapEnd) + } + lx.backup() + lx.push(lexMapValueEnd) + return lexValue +} + +// lexMapValueEnd consumes the cruft between values of a map. Namely, +// it ignores whitespace and expects either a ',' or a '}'. +func lexMapValueEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r): + return lexSkip(lx, lexMapValueEnd) + case r == commentHashStart: + lx.push(lexMapValueEnd) + return lexCommentStart + case r == commentSlashStart: + rn := lx.next() + if rn == commentSlashStart { + lx.push(lexMapValueEnd) + return lexCommentStart + } + lx.backup() + fallthrough + case r == optValTerm || r == mapValTerm || isNL(r): + return lexSkip(lx, lexMapKeyStart) // Move onto next + case r == mapEnd: + return lexSkip(lx, lexMapEnd) + } + return lx.errorf("Expected a map value terminator %q or a map "+ + "terminator %q, but got '%v' instead.", mapValTerm, mapEnd, r) +} + +// lexMapEnd finishes the lexing of a map. It assumes that a '}' has +// just been consumed. +func lexMapEnd(lx *lexer) stateFn { + lx.ignore() + lx.emit(itemMapEnd) + return lx.pop() +} + +// Checks if the unquoted string was actually a boolean +func (lx *lexer) isBool() bool { + str := strings.ToLower(lx.input[lx.start:lx.pos]) + return str == "true" || str == "false" || + str == "on" || str == "off" || + str == "yes" || str == "no" +} + +// Check if the unquoted string is a variable reference, starting with $. +func (lx *lexer) isVariable() bool { + if lx.start >= len(lx.input) { + return false + } + if lx.input[lx.start] == '$' { + lx.start += 1 + return true + } + return false +} + +// lexQuotedString consumes the inner contents of a string. It assumes that the +// beginning '"' has already been consumed and ignored. It will not interpret any +// internal contents. +func lexQuotedString(lx *lexer) stateFn { + r := lx.next() + switch { + case r == sqStringEnd: + lx.backup() + lx.emit(itemString) + lx.next() + lx.ignore() + return lx.pop() + case r == eof: + if lx.pos > lx.start { + return lx.errorf("Unexpected EOF.") + } + lx.emit(itemEOF) + return nil + } + return lexQuotedString +} + +// lexDubQuotedString consumes the inner contents of a string. It assumes that the +// beginning '"' has already been consumed and ignored. It will not interpret any +// internal contents. +func lexDubQuotedString(lx *lexer) stateFn { + r := lx.next() + switch { + case r == '\\': + lx.addCurrentStringPart(1) + return lexStringEscape + case r == dqStringEnd: + lx.backup() + lx.emitString() + lx.next() + lx.ignore() + return lx.pop() + case r == eof: + if lx.pos > lx.start { + return lx.errorf("Unexpected EOF.") + } + lx.emit(itemEOF) + return nil + } + return lexDubQuotedString +} + +// lexString consumes the inner contents of a raw string. +func lexString(lx *lexer) stateFn { + r := lx.next() + switch { + case r == '\\': + lx.addCurrentStringPart(1) + return lexStringEscape + // Termination of non-quoted strings + case isNL(r) || r == eof || r == optValTerm || + r == arrayValTerm || r == arrayEnd || r == mapEnd || + isWhitespace(r): + + lx.backup() + if lx.hasEscapedParts() { + lx.emitString() + } else if lx.isBool() { + lx.emit(itemBool) + } else if lx.isVariable() { + lx.emit(itemVariable) + } else { + lx.emitString() + } + return lx.pop() + case r == sqStringEnd: + lx.backup() + lx.emitString() + lx.next() + lx.ignore() + return lx.pop() + } + return lexString +} + +// lexBlock consumes the inner contents as a string. It assumes that the +// beginning '(' has already been consumed and ignored. It will continue +// processing until it finds a ')' on a new line by itself. +func lexBlock(lx *lexer) stateFn { + r := lx.next() + switch { + case r == blockEnd: + lx.backup() + lx.backup() + + // Looking for a ')' character on a line by itself, if the previous + // character isn't a new line, then break so we keep processing the block. + if lx.next() != '\n' { + lx.next() + break + } + lx.next() + + // Make sure the next character is a new line or an eof. We want a ')' on a + // bare line by itself. + switch lx.next() { + case '\n', eof: + lx.backup() + lx.backup() + lx.emit(itemString) + lx.next() + lx.ignore() + return lx.pop() + } + lx.backup() + case r == eof: + return lx.errorf("Unexpected EOF processing block.") + } + return lexBlock +} + +// lexStringEscape consumes an escaped character. It assumes that the preceding +// '\\' has already been consumed. +func lexStringEscape(lx *lexer) stateFn { + r := lx.next() + switch r { + case 'x': + return lexStringBinary + case 't': + return lx.addStringPart("\t") + case 'n': + return lx.addStringPart("\n") + case 'r': + return lx.addStringPart("\r") + case '"': + return lx.addStringPart("\"") + case '\\': + return lx.addStringPart("\\") + } + return lx.errorf("Invalid escape character '%v'. Only the following "+ + "escape characters are allowed: \\xXX, \\t, \\n, \\r, \\\", \\\\.", r) +} + +// lexStringBinary consumes two hexadecimal digits following '\x'. It assumes +// that the '\x' has already been consumed. +func lexStringBinary(lx *lexer) stateFn { + r := lx.next() + if isNL(r) { + return lx.errorf("Expected two hexadecimal digits after '\\x', but hit end of line") + } + r = lx.next() + if isNL(r) { + return lx.errorf("Expected two hexadecimal digits after '\\x', but hit end of line") + } + offset := lx.pos - 2 + byteString, err := hex.DecodeString(lx.input[offset:lx.pos]) + if err != nil { + return lx.errorf("Expected two hexadecimal digits after '\\x', but got '%s'", lx.input[offset:lx.pos]) + } + lx.addStringPart(string(byteString)) + return lx.stringStateFn +} + +// lexNumberOrDateOrStringOrIPStart consumes either a (positive) +// integer, a float, a datetime, or IP, or String that started with a +// number. It assumes that NO negative sign has been consumed, that +// is triggered above. +func lexNumberOrDateOrStringOrIPStart(lx *lexer) stateFn { + r := lx.next() + if !unicode.IsDigit(r) { + if r == '.' { + return lx.errorf("Floats must start with a digit, not '.'.") + } + return lx.errorf("Expected a digit but got '%v'.", r) + } + return lexNumberOrDateOrStringOrIP +} + +// lexNumberOrDateOrStringOrIP consumes either a (positive) integer, +// float, datetime, IP or string without quotes that starts with a +// number. +func lexNumberOrDateOrStringOrIP(lx *lexer) stateFn { + r := lx.next() + switch { + case r == '-': + if lx.pos-lx.start != 5 { + return lx.errorf("All ISO8601 dates must be in full Zulu form.") + } + return lexDateAfterYear + case unicode.IsDigit(r): + return lexNumberOrDateOrStringOrIP + case r == '.': + // Assume float at first, but could be IP + return lexFloatStart + case isNumberSuffix(r): + return lexConvenientNumber + case !(isNL(r) || r == eof || r == mapEnd || r == optValTerm || r == mapValTerm || isWhitespace(r) || unicode.IsDigit(r)): + // Treat it as a string value once we get a rune that + // is not a number. + lx.stringStateFn = lexString + return lexString + } + lx.backup() + lx.emit(itemInteger) + return lx.pop() +} + +// lexConvenientNumber is when we have a suffix, e.g. 1k or 1Mb +func lexConvenientNumber(lx *lexer) stateFn { + r := lx.next() + switch { + case r == 'b' || r == 'B' || r == 'i' || r == 'I': + return lexConvenientNumber + } + lx.backup() + if isNL(r) || r == eof || r == mapEnd || r == optValTerm || r == mapValTerm || isWhitespace(r) || unicode.IsDigit(r) { + lx.emit(itemInteger) + return lx.pop() + } + // This is not a number, so treat it as a string. + lx.stringStateFn = lexString + return lexString +} + +// lexDateAfterYear consumes a full Zulu Datetime in ISO8601 format. +// It assumes that "YYYY-" has already been consumed. +func lexDateAfterYear(lx *lexer) stateFn { + formats := []rune{ + // digits are '0'. + // everything else is direct equality. + '0', '0', '-', '0', '0', + 'T', + '0', '0', ':', '0', '0', ':', '0', '0', + 'Z', + } + for _, f := range formats { + r := lx.next() + if f == '0' { + if !unicode.IsDigit(r) { + return lx.errorf("Expected digit in ISO8601 datetime, "+ + "but found '%v' instead.", r) + } + } else if f != r { + return lx.errorf("Expected '%v' in ISO8601 datetime, "+ + "but found '%v' instead.", f, r) + } + } + lx.emit(itemDatetime) + return lx.pop() +} + +// lexNegNumberStart consumes either an integer or a float. It assumes that a +// negative sign has already been read, but that *no* digits have been consumed. +// lexNegNumberStart will move to the appropriate integer or float states. +func lexNegNumberStart(lx *lexer) stateFn { + // we MUST see a digit. Even floats have to start with a digit. + r := lx.next() + if !unicode.IsDigit(r) { + if r == '.' { + return lx.errorf("Floats must start with a digit, not '.'.") + } + return lx.errorf("Expected a digit but got '%v'.", r) + } + return lexNegNumber +} + +// lexNegNumber consumes a negative integer or a float after seeing the first digit. +func lexNegNumber(lx *lexer) stateFn { + r := lx.next() + switch { + case unicode.IsDigit(r): + return lexNegNumber + case r == '.': + return lexFloatStart + case isNumberSuffix(r): + return lexConvenientNumber + } + lx.backup() + lx.emit(itemInteger) + return lx.pop() +} + +// lexFloatStart starts the consumption of digits of a float after a '.'. +// Namely, at least one digit is required. +func lexFloatStart(lx *lexer) stateFn { + r := lx.next() + if !unicode.IsDigit(r) { + return lx.errorf("Floats must have a digit after the '.', but got "+ + "'%v' instead.", r) + } + return lexFloat +} + +// lexFloat consumes the digits of a float after a '.'. +// Assumes that one digit has been consumed after a '.' already. +func lexFloat(lx *lexer) stateFn { + r := lx.next() + if unicode.IsDigit(r) { + return lexFloat + } + + // Not a digit, if its another '.', need to see if we falsely assumed a float. + if r == '.' { + return lexIPAddr + } + + lx.backup() + lx.emit(itemFloat) + return lx.pop() +} + +// lexIPAddr consumes IP addrs, like 127.0.0.1:4222 +func lexIPAddr(lx *lexer) stateFn { + r := lx.next() + if unicode.IsDigit(r) || r == '.' || r == ':' || r == '-' { + return lexIPAddr + } + lx.backup() + lx.emit(itemString) + return lx.pop() +} + +// lexCommentStart begins the lexing of a comment. It will emit +// itemCommentStart and consume no characters, passing control to lexComment. +func lexCommentStart(lx *lexer) stateFn { + lx.ignore() + lx.emit(itemCommentStart) + return lexComment +} + +// lexComment lexes an entire comment. It assumes that '#' has been consumed. +// It will consume *up to* the first new line character, and pass control +// back to the last state on the stack. +func lexComment(lx *lexer) stateFn { + r := lx.peek() + if isNL(r) || r == eof { + lx.emit(itemText) + return lx.pop() + } + lx.next() + return lexComment +} + +// lexSkip ignores all slurped input and moves on to the next state. +func lexSkip(lx *lexer, nextState stateFn) stateFn { + return func(lx *lexer) stateFn { + lx.ignore() + return nextState + } +} + +// Tests to see if we have a number suffix +func isNumberSuffix(r rune) bool { + return r == 'k' || r == 'K' || r == 'm' || r == 'M' || r == 'g' || r == 'G' || r == 't' || r == 'T' || r == 'p' || r == 'P' || r == 'e' || r == 'E' +} + +// Tests for both key separators +func isKeySeparator(r rune) bool { + return r == keySepEqual || r == keySepColon +} + +// isWhitespace returns true if `r` is a whitespace character according +// to the spec. +func isWhitespace(r rune) bool { + return r == '\t' || r == ' ' +} + +func isNL(r rune) bool { + return r == '\n' || r == '\r' +} + +func (itype itemType) String() string { + switch itype { + case itemError: + return "Error" + case itemNIL: + return "NIL" + case itemEOF: + return "EOF" + case itemText: + return "Text" + case itemString: + return "String" + case itemBool: + return "Bool" + case itemInteger: + return "Integer" + case itemFloat: + return "Float" + case itemDatetime: + return "DateTime" + case itemKey: + return "Key" + case itemArrayStart: + return "ArrayStart" + case itemArrayEnd: + return "ArrayEnd" + case itemMapStart: + return "MapStart" + case itemMapEnd: + return "MapEnd" + case itemCommentStart: + return "CommentStart" + case itemVariable: + return "Variable" + case itemInclude: + return "Include" + } + panic(fmt.Sprintf("BUG: Unknown type '%s'.", itype.String())) +} + +func (item item) String() string { + return fmt.Sprintf("(%s, '%s', %d, %d)", item.typ.String(), item.val, item.line, item.pos) +} + +func escapeSpecial(c rune) string { + switch c { + case '\n': + return "\\n" + } + return string(c) +} diff --git a/conf/lex_test.go b/conf/lex_test.go new file mode 100644 index 0000000..5f54add --- /dev/null +++ b/conf/lex_test.go @@ -0,0 +1,1645 @@ +package conf + +import "testing" + +// Test to make sure we get what we expect. +func expect(t *testing.T, lx *lexer, items []item) { + t.Helper() + for i := 0; i < len(items); i++ { + item := lx.nextItem() + _ = item.String() + if item.typ == itemEOF { + break + } + if item != items[i] { + t.Fatalf("Testing: '%s'\nExpected %q, received %q\n", + lx.input, items[i], item) + } + if item.typ == itemError { + break + } + } +} + +func TestPlainValue(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo") + expect(t, lx, expectedItems) +} + +func TestSimpleKeyStringValues(t *testing.T) { + // Double quotes + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "bar", 1, 7}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo = \"bar\"") + expect(t, lx, expectedItems) + + // Single quotes + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemString, "bar", 1, 7}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo = 'bar'") + expect(t, lx, expectedItems) + + // No spaces + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemString, "bar", 1, 5}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo='bar'") + expect(t, lx, expectedItems) + + // NL + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemString, "bar", 1, 5}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo='bar'\r\n") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemString, "bar", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo=\t'bar'\t") + expect(t, lx, expectedItems) +} + +func TestComplexStringValues(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "bar\\r\\n \\t", 1, 7}, + {itemEOF, "", 2, 0}, + } + + lx := lex("foo = 'bar\\r\\n \\t'") + expect(t, lx, expectedItems) +} + +func TestStringStartingWithNumber(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "3xyz", 1, 6}, + {itemEOF, "", 2, 0}, + } + + lx := lex(`foo = 3xyz`) + expect(t, lx, expectedItems) + + lx = lex(`foo = 3xyz,`) + expect(t, lx, expectedItems) + + lx = lex(`foo = 3xyz;`) + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 2, 9}, + {itemString, "3xyz", 2, 15}, + {itemEOF, "", 2, 0}, + } + content := ` + foo = 3xyz + ` + lx = lex(content) + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "map", 2, 9}, + {itemMapStart, "", 2, 14}, + {itemKey, "foo", 3, 11}, + {itemString, "3xyz", 3, 17}, + {itemMapEnd, "", 3, 22}, + {itemEOF, "", 2, 0}, + } + content = ` + map { + foo = 3xyz} + ` + lx = lex(content) + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "map", 2, 9}, + {itemMapStart, "", 2, 14}, + {itemKey, "foo", 3, 11}, + {itemString, "3xyz", 3, 17}, + {itemMapEnd, "", 4, 10}, + {itemEOF, "", 2, 0}, + } + content = ` + map { + foo = 3xyz; + } + ` + lx = lex(content) + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "map", 2, 9}, + {itemMapStart, "", 2, 14}, + {itemKey, "foo", 3, 11}, + {itemString, "3xyz", 3, 17}, + {itemKey, "bar", 4, 11}, + {itemString, "4wqs", 4, 17}, + {itemMapEnd, "", 5, 10}, + {itemEOF, "", 2, 0}, + } + content = ` + map { + foo = 3xyz, + bar = 4wqs + } + ` + lx = lex(content) + expect(t, lx, expectedItems) +} + +func TestBinaryString(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "e", 1, 9}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo = \\x65") + expect(t, lx, expectedItems) +} + +func TestBinaryStringLatin1(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "\xe9", 1, 9}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo = \\xe9") + expect(t, lx, expectedItems) +} + +func TestSimpleKeyIntegerValues(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "123", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo = 123") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "123", 1, 4}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo=123") + expect(t, lx, expectedItems) + lx = lex("foo=123\r\n") + expect(t, lx, expectedItems) +} + +func TestSimpleKeyNegativeIntegerValues(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "-123", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo = -123") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "-123", 1, 4}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo=-123") + expect(t, lx, expectedItems) + lx = lex("foo=-123\r\n") + expect(t, lx, expectedItems) +} + +func TestConvenientIntegerValues(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "1k", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo = 1k") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "1K", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo = 1K") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "1m", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo = 1m") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "1M", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo = 1M") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "1g", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo = 1g") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "1G", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo = 1G") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "1MB", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo = 1MB") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "1Gb", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo = 1Gb") + expect(t, lx, expectedItems) + + // Negative versions + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "-1m", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo = -1m") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "-1GB", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo = -1GB ") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemString, "1Ghz", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo = 1Ghz") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemString, "2Pie", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo = 2Pie") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemString, "3Mbs", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo = 3Mbs,") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "4Gb", 1, 6}, + {itemKey, "bar", 1, 11}, + {itemString, "5Gø", 1, 17}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo = 4Gb, bar = 5Gø") + expect(t, lx, expectedItems) +} + +func TestSimpleKeyFloatValues(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemFloat, "22.2", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo = 22.2") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemFloat, "22.2", 1, 4}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo=22.2") + expect(t, lx, expectedItems) + lx = lex("foo=22.2\r\n") + expect(t, lx, expectedItems) +} + +func TestBadBinaryStringEndingAfterZeroHexChars(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemError, "Expected two hexadecimal digits after '\\x', but hit end of line", 2, 1}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo = xyz\\x\n") + expect(t, lx, expectedItems) +} + +func TestBadBinaryStringEndingAfterOneHexChar(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemError, "Expected two hexadecimal digits after '\\x', but hit end of line", 2, 1}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo = xyz\\xF\n") + expect(t, lx, expectedItems) +} + +func TestBadBinaryStringWithZeroHexChars(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemError, "Expected two hexadecimal digits after '\\x', but got ']\"'", 1, 12}, + {itemEOF, "", 1, 0}, + } + lx := lex(`foo = "[\x]"`) + expect(t, lx, expectedItems) +} + +func TestBadBinaryStringWithOneHexChar(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemError, "Expected two hexadecimal digits after '\\x', but got 'e]'", 1, 12}, + {itemEOF, "", 1, 0}, + } + lx := lex(`foo = "[\xe]"`) + expect(t, lx, expectedItems) +} + +func TestBadFloatValues(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemError, "Floats must start with a digit", 1, 7}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo = .2") + expect(t, lx, expectedItems) +} + +func TestBadKey(t *testing.T) { + expectedItems := []item{ + {itemError, "Unexpected key separator ':'", 1, 1}, + {itemEOF, "", 1, 0}, + } + lx := lex(" :foo = 22") + expect(t, lx, expectedItems) +} + +func TestSimpleKeyBoolValues(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemBool, "true", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo = true") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemBool, "true", 1, 4}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo=true") + expect(t, lx, expectedItems) + lx = lex("foo=true\r\n") + expect(t, lx, expectedItems) +} + +func TestComments(t *testing.T) { + expectedItems := []item{ + {itemCommentStart, "", 1, 1}, + {itemText, " This is a comment", 1, 1}, + {itemEOF, "", 1, 0}, + } + lx := lex("# This is a comment") + expect(t, lx, expectedItems) + lx = lex("# This is a comment\r\n") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemCommentStart, "", 1, 2}, + {itemText, " This is a comment", 1, 2}, + {itemEOF, "", 1, 0}, + } + lx = lex("// This is a comment\r\n") + expect(t, lx, expectedItems) +} + +func TestTopValuesWithComments(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "123", 1, 6}, + {itemCommentStart, "", 1, 12}, + {itemText, " This is a comment", 1, 12}, + {itemEOF, "", 1, 0}, + } + + lx := lex("foo = 123 // This is a comment") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "123", 1, 4}, + {itemCommentStart, "", 1, 12}, + {itemText, " This is a comment", 1, 12}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo=123 # This is a comment") + expect(t, lx, expectedItems) +} + +func TestRawString(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "bar", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo = bar") + expect(t, lx, expectedItems) + lx = lex(`foo = bar' `) + expect(t, lx, expectedItems) +} + +func TestDateValues(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemDatetime, "2016-05-04T18:53:41Z", 1, 6}, + {itemEOF, "", 1, 0}, + } + + lx := lex("foo = 2016-05-04T18:53:41Z") + expect(t, lx, expectedItems) +} + +func TestVariableValues(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemVariable, "bar", 1, 7}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo = $bar") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemVariable, "bar", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo =$bar") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemVariable, "bar", 1, 5}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo $bar") + expect(t, lx, expectedItems) +} + +func TestArrays(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemArrayStart, "", 1, 7}, + {itemInteger, "1", 1, 7}, + {itemInteger, "2", 1, 10}, + {itemInteger, "3", 1, 13}, + {itemString, "bar", 1, 17}, + {itemArrayEnd, "", 1, 22}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo = [1, 2, 3, 'bar']") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemArrayStart, "", 1, 7}, + {itemInteger, "1", 1, 7}, + {itemInteger, "2", 1, 9}, + {itemInteger, "3", 1, 11}, + {itemString, "bar", 1, 14}, + {itemArrayEnd, "", 1, 19}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo = [1,2,3,'bar']") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemArrayStart, "", 1, 7}, + {itemInteger, "1", 1, 7}, + {itemInteger, "2", 1, 10}, + {itemInteger, "3", 1, 12}, + {itemString, "bar", 1, 15}, + {itemArrayEnd, "", 1, 20}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo = [1, 2,3,'bar']") + expect(t, lx, expectedItems) +} + +var mlArray = ` +# top level comment +foo = [ + 1, # One + 2, // Two + 3 # Three + 'bar' , + "bar" +] +` + +func TestMultilineArrays(t *testing.T) { + expectedItems := []item{ + {itemCommentStart, "", 2, 2}, + {itemText, " top level comment", 2, 2}, + {itemKey, "foo", 3, 1}, + {itemArrayStart, "", 3, 8}, + {itemInteger, "1", 4, 2}, + {itemCommentStart, "", 4, 6}, + {itemText, " One", 4, 6}, + {itemInteger, "2", 5, 2}, + {itemCommentStart, "", 5, 7}, + {itemText, " Two", 5, 7}, + {itemInteger, "3", 6, 2}, + {itemCommentStart, "", 6, 5}, + {itemText, " Three", 6, 5}, + {itemString, "bar", 7, 3}, + {itemString, "bar", 8, 3}, + {itemArrayEnd, "", 9, 2}, + {itemEOF, "", 9, 0}, + } + lx := lex(mlArray) + expect(t, lx, expectedItems) +} + +var mlArrayNoSep = ` +# top level comment +foo = [ + 1 // foo + 2 + 3 + 'bar' + "bar" +] +` + +func TestMultilineArraysNoSep(t *testing.T) { + expectedItems := []item{ + {itemCommentStart, "", 2, 2}, + {itemText, " top level comment", 2, 2}, + {itemKey, "foo", 3, 1}, + {itemArrayStart, "", 3, 8}, + {itemInteger, "1", 4, 2}, + {itemCommentStart, "", 4, 6}, + {itemText, " foo", 4, 6}, + {itemInteger, "2", 5, 2}, + {itemInteger, "3", 6, 2}, + {itemString, "bar", 7, 3}, + {itemString, "bar", 8, 3}, + {itemArrayEnd, "", 9, 2}, + {itemEOF, "", 9, 0}, + } + lx := lex(mlArrayNoSep) + expect(t, lx, expectedItems) +} + +func TestSimpleMap(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemMapStart, "", 1, 7}, + {itemKey, "ip", 1, 7}, + {itemString, "127.0.0.1", 1, 11}, + {itemKey, "port", 1, 23}, + {itemInteger, "4242", 1, 30}, + {itemMapEnd, "", 1, 35}, + {itemEOF, "", 1, 0}, + } + + lx := lex("foo = {ip='127.0.0.1', port = 4242}") + expect(t, lx, expectedItems) +} + +var mlMap = ` +foo = { + ip = '127.0.0.1' # the IP + port= 4242 // the port +} +` + +func TestMultilineMap(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 2, 1}, + {itemMapStart, "", 2, 8}, + {itemKey, "ip", 3, 3}, + {itemString, "127.0.0.1", 3, 9}, + {itemCommentStart, "", 3, 21}, + {itemText, " the IP", 3, 21}, + {itemKey, "port", 4, 3}, + {itemInteger, "4242", 4, 9}, + {itemCommentStart, "", 4, 16}, + {itemText, " the port", 4, 16}, + {itemMapEnd, "", 5, 2}, + {itemEOF, "", 5, 0}, + } + + lx := lex(mlMap) + expect(t, lx, expectedItems) +} + +var nestedMap = ` +foo = { + host = { + ip = '127.0.0.1' + port= 4242 + } +} +` + +func TestNestedMaps(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 2, 1}, + {itemMapStart, "", 2, 8}, + {itemKey, "host", 3, 3}, + {itemMapStart, "", 3, 11}, + {itemKey, "ip", 4, 5}, + {itemString, "127.0.0.1", 4, 11}, + {itemKey, "port", 5, 5}, + {itemInteger, "4242", 5, 11}, + {itemMapEnd, "", 6, 4}, + {itemMapEnd, "", 7, 2}, + {itemEOF, "", 7, 0}, + } + + lx := lex(nestedMap) + expect(t, lx, expectedItems) +} + +func TestQuotedKeys(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "123", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo : 123") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 1}, + {itemInteger, "123", 1, 8}, + {itemEOF, "", 1, 0}, + } + lx = lex("'foo' : 123") + expect(t, lx, expectedItems) + lx = lex("\"foo\" : 123") + expect(t, lx, expectedItems) +} + +func TestQuotedKeysWithSpace(t *testing.T) { + expectedItems := []item{ + {itemKey, " foo", 1, 1}, + {itemInteger, "123", 1, 9}, + {itemEOF, "", 1, 0}, + } + lx := lex("' foo' : 123") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, " foo", 1, 1}, + {itemInteger, "123", 1, 9}, + {itemEOF, "", 1, 0}, + } + lx = lex("\" foo\" : 123") + expect(t, lx, expectedItems) +} + +func TestColonKeySep(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "123", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo : 123") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "123", 1, 4}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo:123") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "123", 1, 5}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo: 123") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "123", 1, 6}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo: 123\r\n") + expect(t, lx, expectedItems) +} + +func TestWhitespaceKeySep(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "123", 1, 4}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo 123") + expect(t, lx, expectedItems) + lx = lex("foo 123") + expect(t, lx, expectedItems) + lx = lex("foo\t123") + expect(t, lx, expectedItems) + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemInteger, "123", 1, 5}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo\t\t123\r\n") + expect(t, lx, expectedItems) +} + +var escString = ` +foo = \t +bar = \r +baz = \n +q = \" +bs = \\ +` + +func TestEscapedString(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 2, 1}, + {itemString, "\t", 2, 9}, + {itemKey, "bar", 3, 1}, + {itemString, "\r", 3, 9}, + {itemKey, "baz", 4, 1}, + {itemString, "\n", 4, 9}, + {itemKey, "q", 5, 1}, + {itemString, "\"", 5, 9}, + {itemKey, "bs", 6, 1}, + {itemString, "\\", 6, 9}, + {itemEOF, "", 6, 0}, + } + lx := lex(escString) + expect(t, lx, expectedItems) +} + +func TestCompoundStringES(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "\\end", 1, 8}, + {itemEOF, "", 2, 0}, + } + lx := lex(`foo = "\\end"`) + expect(t, lx, expectedItems) +} + +func TestCompoundStringSE(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "start\\", 1, 8}, + {itemEOF, "", 2, 0}, + } + lx := lex(`foo = "start\\"`) + expect(t, lx, expectedItems) +} + +func TestCompoundStringEE(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "Eq", 1, 12}, + {itemEOF, "", 2, 0}, + } + lx := lex(`foo = \x45\x71`) + expect(t, lx, expectedItems) +} + +func TestCompoundStringSEE(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "startEq", 1, 12}, + {itemEOF, "", 2, 0}, + } + lx := lex(`foo = start\x45\x71`) + expect(t, lx, expectedItems) +} + +func TestCompoundStringSES(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "start|end", 1, 9}, + {itemEOF, "", 2, 0}, + } + lx := lex(`foo = start\x7Cend`) + expect(t, lx, expectedItems) +} + +func TestCompoundStringEES(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "<>end", 1, 12}, + {itemEOF, "", 2, 0}, + } + lx := lex(`foo = \x3c\x3eend`) + expect(t, lx, expectedItems) +} + +func TestCompoundStringESE(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "", 1, 12}, + {itemEOF, "", 2, 0}, + } + lx := lex(`foo = \x3cmiddle\x3E`) + expect(t, lx, expectedItems) +} + +func TestBadStringEscape(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemError, "Invalid escape character 'y'. Only the following escape characters are allowed: \\xXX, \\t, \\n, \\r, \\\", \\\\.", 1, 8}, + {itemEOF, "", 2, 0}, + } + lx := lex(`foo = \y`) + expect(t, lx, expectedItems) +} + +func TestNonBool(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "\\true", 1, 7}, + {itemEOF, "", 2, 0}, + } + lx := lex(`foo = \\true`) + expect(t, lx, expectedItems) +} + +func TestNonVariable(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "\\$var", 1, 7}, + {itemEOF, "", 2, 0}, + } + lx := lex(`foo = \\$var`) + expect(t, lx, expectedItems) +} + +func TestEmptyStringDQ(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "", 1, 7}, + {itemEOF, "", 2, 0}, + } + lx := lex(`foo = ""`) + expect(t, lx, expectedItems) +} + +func TestEmptyStringSQ(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "", 1, 7}, + {itemEOF, "", 2, 0}, + } + lx := lex(`foo = ''`) + expect(t, lx, expectedItems) +} + +var nestedWhitespaceMap = ` +foo { + host { + ip = '127.0.0.1' + port= 4242 + } +} +` + +func TestNestedWhitespaceMaps(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 2, 1}, + {itemMapStart, "", 2, 7}, + {itemKey, "host", 3, 3}, + {itemMapStart, "", 3, 10}, + {itemKey, "ip", 4, 5}, + {itemString, "127.0.0.1", 4, 11}, + {itemKey, "port", 5, 5}, + {itemInteger, "4242", 5, 11}, + {itemMapEnd, "", 6, 4}, + {itemMapEnd, "", 7, 2}, + {itemEOF, "", 7, 0}, + } + + lx := lex(nestedWhitespaceMap) + expect(t, lx, expectedItems) +} + +var semicolons = ` +foo = 123; +bar = 'baz'; +baz = 'boo' +map { + id = 1; +} +` + +func TestOptionalSemicolons(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 2, 1}, + {itemInteger, "123", 2, 7}, + {itemKey, "bar", 3, 1}, + {itemString, "baz", 3, 8}, + {itemKey, "baz", 4, 1}, + {itemString, "boo", 4, 8}, + {itemKey, "map", 5, 1}, + {itemMapStart, "", 5, 6}, + {itemKey, "id", 6, 2}, + {itemInteger, "1", 6, 7}, + {itemMapEnd, "", 7, 2}, + {itemEOF, "", 8, 0}, + } + + lx := lex(semicolons) + expect(t, lx, expectedItems) +} + +func TestSemicolonChaining(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemString, "1", 1, 5}, + {itemKey, "bar", 1, 9}, + {itemFloat, "2.2", 1, 13}, + {itemKey, "baz", 1, 18}, + {itemBool, "true", 1, 22}, + {itemEOF, "", 1, 0}, + } + + lx := lex("foo='1'; bar=2.2; baz=true;") + expect(t, lx, expectedItems) +} + +var noquotes = ` +foo = 123 +bar = baz +baz=boo +map { + id:one + id2 : onetwo +} +t true +f false +tstr "true" +tkey = two +fkey = five # This should be a string +` + +func TestNonQuotedStrings(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 2, 1}, + {itemInteger, "123", 2, 7}, + {itemKey, "bar", 3, 1}, + {itemString, "baz", 3, 7}, + {itemKey, "baz", 4, 1}, + {itemString, "boo", 4, 5}, + {itemKey, "map", 5, 1}, + {itemMapStart, "", 5, 6}, + {itemKey, "id", 6, 2}, + {itemString, "one", 6, 5}, + {itemKey, "id2", 7, 2}, + {itemString, "onetwo", 7, 8}, + {itemMapEnd, "", 8, 2}, + {itemKey, "t", 9, 1}, + {itemBool, "true", 9, 3}, + {itemKey, "f", 10, 1}, + {itemBool, "false", 10, 3}, + {itemKey, "tstr", 11, 1}, + {itemString, "true", 11, 7}, + {itemKey, "tkey", 12, 1}, + {itemString, "two", 12, 8}, + {itemKey, "fkey", 13, 1}, + {itemString, "five", 13, 8}, + {itemCommentStart, "", 13, 14}, + {itemText, " This should be a string", 13, 14}, + {itemEOF, "", 14, 0}, + } + lx := lex(noquotes) + expect(t, lx, expectedItems) +} + +var danglingquote = ` +listen: "localhost:4242 + +http: localhost:8222 +` + +func TestDanglingQuotedString(t *testing.T) { + expectedItems := []item{ + {itemKey, "listen", 2, 1}, + {itemError, "Unexpected EOF.", 5, 1}, + } + lx := lex(danglingquote) + expect(t, lx, expectedItems) +} + +var keydanglingquote = ` +foo = " +listen: " + +http: localhost:8222 + +" +` + +func TestKeyDanglingQuotedString(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 2, 1}, + {itemString, "\nlisten: ", 3, 8}, + {itemKey, "http", 5, 1}, + {itemString, "localhost:8222", 5, 7}, + {itemError, "Unexpected EOF.", 8, 1}, + } + lx := lex(keydanglingquote) + expect(t, lx, expectedItems) +} + +var danglingsquote = ` +listen: 'localhost:4242 + +http: localhost:8222 +` + +func TestDanglingSingleQuotedString(t *testing.T) { + expectedItems := []item{ + {itemKey, "listen", 2, 1}, + {itemError, "Unexpected EOF.", 5, 1}, + } + lx := lex(danglingsquote) + expect(t, lx, expectedItems) +} + +var keydanglingsquote = ` +foo = ' +listen: ' + +http: localhost:8222 + +' +` + +func TestKeyDanglingSingleQuotedString(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 2, 1}, + {itemString, "\nlisten: ", 3, 8}, + {itemKey, "http", 5, 1}, + {itemString, "localhost:8222", 5, 7}, + {itemError, "Unexpected EOF.", 8, 1}, + } + lx := lex(keydanglingsquote) + expect(t, lx, expectedItems) +} + +var mapdanglingbracket = ` +listen = 4222 + +cluster = { + + foo = bar + +` + +func TestMapDanglingBracket(t *testing.T) { + expectedItems := []item{ + {itemKey, "listen", 2, 1}, + {itemInteger, "4222", 2, 10}, + {itemKey, "cluster", 4, 1}, + {itemMapStart, "", 4, 12}, + {itemKey, "foo", 6, 3}, + {itemString, "bar", 6, 9}, + {itemError, "Unexpected EOF processing map.", 8, 1}, + } + lx := lex(mapdanglingbracket) + expect(t, lx, expectedItems) +} + +var blockdanglingparens = ` +listen = 4222 + +quote = ( + + foo = bar + +` + +func TestBlockDanglingParens(t *testing.T) { + expectedItems := []item{ + {itemKey, "listen", 2, 1}, + {itemInteger, "4222", 2, 10}, + {itemKey, "quote", 4, 1}, + {itemError, "Unexpected EOF processing block.", 8, 1}, + } + lx := lex(blockdanglingparens) + expect(t, lx, expectedItems) +} + +func TestMapQuotedKeys(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemMapStart, "", 1, 7}, + {itemKey, "bar", 1, 8}, + {itemInteger, "4242", 1, 15}, + {itemMapEnd, "", 1, 20}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo = {'bar' = 4242}") + expect(t, lx, expectedItems) + lx = lex("foo = {\"bar\" = 4242}") + expect(t, lx, expectedItems) +} + +func TestSpecialCharsMapQuotedKeys(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemMapStart, "", 1, 7}, + {itemKey, "bar-1.2.3", 1, 8}, + {itemMapStart, "", 1, 22}, + {itemKey, "port", 1, 23}, + {itemInteger, "4242", 1, 28}, + {itemMapEnd, "", 1, 34}, + {itemMapEnd, "", 1, 35}, + {itemEOF, "", 1, 0}, + } + lx := lex("foo = {'bar-1.2.3' = { port:4242 }}") + expect(t, lx, expectedItems) + lx = lex("foo = {\"bar-1.2.3\" = { port:4242 }}") + expect(t, lx, expectedItems) +} + +var mlnestedmap = ` +systems { + allinone { + description: "This is a description." + } +} +` + +func TestDoubleNestedMapsNewLines(t *testing.T) { + expectedItems := []item{ + {itemKey, "systems", 2, 1}, + {itemMapStart, "", 2, 10}, + {itemKey, "allinone", 3, 3}, + {itemMapStart, "", 3, 13}, + {itemKey, "description", 4, 5}, + {itemString, "This is a description.", 4, 19}, + {itemMapEnd, "", 5, 4}, + {itemMapEnd, "", 6, 2}, + {itemEOF, "", 7, 0}, + } + lx := lex(mlnestedmap) + expect(t, lx, expectedItems) +} + +var blockexample = ` +numbers ( +1234567890 +) +` + +func TestBlockString(t *testing.T) { + expectedItems := []item{ + {itemKey, "numbers", 2, 1}, + {itemString, "\n1234567890\n", 4, 10}, + } + lx := lex(blockexample) + expect(t, lx, expectedItems) +} + +func TestBlockStringEOF(t *testing.T) { + expectedItems := []item{ + {itemKey, "numbers", 2, 1}, + {itemString, "\n1234567890\n", 4, 10}, + } + blockbytes := []byte(blockexample[0 : len(blockexample)-1]) + blockbytes = append(blockbytes, 0) + lx := lex(string(blockbytes)) + expect(t, lx, expectedItems) +} + +var mlblockexample = ` +numbers ( + 12(34)56 + ( + 7890 + ) +) +` + +func TestBlockStringMultiLine(t *testing.T) { + expectedItems := []item{ + {itemKey, "numbers", 2, 1}, + {itemString, "\n 12(34)56\n (\n 7890\n )\n", 7, 10}, + } + lx := lex(mlblockexample) + expect(t, lx, expectedItems) +} + +func TestUnquotedIPAddr(t *testing.T) { + expectedItems := []item{ + {itemKey, "listen", 1, 0}, + {itemString, "127.0.0.1:4222", 1, 8}, + {itemEOF, "", 1, 0}, + } + lx := lex("listen: 127.0.0.1:4222") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "listen", 1, 0}, + {itemString, "127.0.0.1", 1, 8}, + {itemEOF, "", 1, 0}, + } + lx = lex("listen: 127.0.0.1") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "listen", 1, 0}, + {itemString, "apcera.me:80", 1, 8}, + {itemEOF, "", 1, 0}, + } + lx = lex("listen: apcera.me:80") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "listen", 1, 0}, + {itemString, "nats.io:-1", 1, 8}, + {itemEOF, "", 1, 0}, + } + lx = lex("listen: nats.io:-1") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "listen", 1, 0}, + {itemInteger, "-1", 1, 8}, + {itemEOF, "", 1, 0}, + } + lx = lex("listen: -1") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "listen", 1, 0}, + {itemString, ":-1", 1, 8}, + {itemEOF, "", 1, 0}, + } + lx = lex("listen: :-1") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "listen", 1, 0}, + {itemString, ":80", 1, 9}, + {itemEOF, "", 1, 0}, + } + lx = lex("listen = :80") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "listen", 1, 0}, + {itemArrayStart, "", 1, 10}, + {itemString, "localhost:4222", 1, 10}, + {itemString, "localhost:4333", 1, 26}, + {itemArrayEnd, "", 1, 41}, + {itemEOF, "", 1, 0}, + } + lx = lex("listen = [localhost:4222, localhost:4333]") + expect(t, lx, expectedItems) +} + +var arrayOfMaps = ` +authorization { + users = [ + {user: alice, password: foo} + {user: bob, password: bar} + ] + timeout: 0.5 +} +` + +func TestArrayOfMaps(t *testing.T) { + expectedItems := []item{ + {itemKey, "authorization", 2, 1}, + {itemMapStart, "", 2, 16}, + {itemKey, "users", 3, 5}, + {itemArrayStart, "", 3, 14}, + {itemMapStart, "", 4, 8}, + {itemKey, "user", 4, 8}, + {itemString, "alice", 4, 14}, + {itemKey, "password", 4, 21}, + {itemString, "foo", 4, 31}, + {itemMapEnd, "", 4, 35}, + {itemMapStart, "", 5, 8}, + {itemKey, "user", 5, 8}, + {itemString, "bob", 5, 14}, + {itemKey, "password", 5, 21}, + {itemString, "bar", 5, 31}, + {itemMapEnd, "", 5, 35}, + {itemArrayEnd, "", 6, 6}, + {itemKey, "timeout", 7, 5}, + {itemFloat, "0.5", 7, 14}, + {itemMapEnd, "", 8, 2}, + {itemEOF, "", 9, 0}, + } + lx := lex(arrayOfMaps) + expect(t, lx, expectedItems) +} + +func TestInclude(t *testing.T) { + expectedItems := []item{ + {itemInclude, "users.conf", 1, 9}, + {itemEOF, "", 1, 0}, + } + lx := lex("include \"users.conf\"") + expect(t, lx, expectedItems) + + lx = lex("include 'users.conf'") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemInclude, "users.conf", 1, 8}, + {itemEOF, "", 1, 0}, + } + lx = lex("include users.conf") + expect(t, lx, expectedItems) +} + +func TestMapInclude(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1, 0}, + {itemMapStart, "", 1, 5}, + {itemInclude, "users.conf", 1, 14}, + {itemMapEnd, "", 1, 26}, + {itemEOF, "", 1, 0}, + } + + lx := lex("foo { include users.conf }") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemMapStart, "", 1, 5}, + {itemInclude, "users.conf", 1, 13}, + {itemMapEnd, "", 1, 24}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo {include users.conf}") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemMapStart, "", 1, 5}, + {itemInclude, "users.conf", 1, 15}, + {itemMapEnd, "", 1, 28}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo { include 'users.conf' }") + expect(t, lx, expectedItems) + + expectedItems = []item{ + {itemKey, "foo", 1, 0}, + {itemMapStart, "", 1, 5}, + {itemInclude, "users.conf", 1, 15}, + {itemMapEnd, "", 1, 27}, + {itemEOF, "", 1, 0}, + } + lx = lex("foo { include \"users.conf\"}") + expect(t, lx, expectedItems) +} + +func TestJSONCompat(t *testing.T) { + for _, test := range []struct { + name string + input string + expected []item + }{ + { + name: "should omit initial and final brackets at top level with a single item", + input: ` + { + "http_port": 8223 + } + `, + expected: []item{ + {itemKey, "http_port", 3, 28}, + {itemInteger, "8223", 3, 40}, + {itemKey, "}", 4, 25}, + {itemEOF, "", 0, 0}, + }, + }, + { + name: "should omit trailing commas at top level with two items", + input: ` + { + "http_port": 8223, + "port": 4223 + } + `, + expected: []item{ + {itemKey, "http_port", 3, 28}, + {itemInteger, "8223", 3, 40}, + {itemKey, "port", 4, 28}, + {itemInteger, "4223", 4, 35}, + {itemKey, "}", 5, 25}, + {itemEOF, "", 0, 0}, + }, + }, + { + name: "should omit trailing commas at top level with multiple items", + input: ` + { + "http_port": 8223, + "port": 4223, + "max_payload": "5MB", + "debug": true, + "max_control_line": 1024 + } + `, + expected: []item{ + {itemKey, "http_port", 3, 28}, + {itemInteger, "8223", 3, 40}, + {itemKey, "port", 4, 28}, + {itemInteger, "4223", 4, 35}, + {itemKey, "max_payload", 5, 28}, + {itemString, "5MB", 5, 43}, + {itemKey, "debug", 6, 28}, + {itemBool, "true", 6, 36}, + {itemKey, "max_control_line", 7, 28}, + {itemInteger, "1024", 7, 47}, + {itemKey, "}", 8, 25}, + {itemEOF, "", 0, 0}, + }, + }, + { + name: "should support JSON not prettified", + input: `{"http_port": 8224,"port": 4224} + `, + expected: []item{ + {itemKey, "http_port", 1, 2}, + {itemInteger, "8224", 1, 14}, + {itemKey, "port", 1, 20}, + {itemInteger, "4224", 1, 27}, + {itemEOF, "", 0, 0}, + }, + }, + { + name: "should support JSON not prettified with final bracket after newline", + input: `{"http_port": 8225,"port": 4225 + } + `, + expected: []item{ + {itemKey, "http_port", 1, 2}, + {itemInteger, "8225", 1, 14}, + {itemKey, "port", 1, 20}, + {itemInteger, "4225", 1, 27}, + {itemKey, "}", 2, 25}, + {itemEOF, "", 0, 0}, + }, + }, + { + name: "should support uglified JSON with inner blocks", + input: `{"http_port": 8227,"port": 4227,"write_deadline": "1h","cluster": {"port": 6222,"routes": ["nats://127.0.0.1:4222","nats://127.0.0.1:4223","nats://127.0.0.1:4224"]}} + `, + expected: []item{ + {itemKey, "http_port", 1, 2}, + {itemInteger, "8227", 1, 14}, + {itemKey, "port", 1, 20}, + {itemInteger, "4227", 1, 27}, + {itemKey, "write_deadline", 1, 33}, + {itemString, "1h", 1, 51}, + {itemKey, "cluster", 1, 56}, + {itemMapStart, "", 1, 67}, + {itemKey, "port", 1, 68}, + {itemInteger, "6222", 1, 75}, + {itemKey, "routes", 1, 81}, + {itemArrayStart, "", 1, 91}, + {itemString, "nats://127.0.0.1:4222", 1, 92}, + {itemString, "nats://127.0.0.1:4223", 1, 116}, + {itemString, "nats://127.0.0.1:4224", 1, 140}, + {itemArrayEnd, "", 1, 163}, + {itemMapEnd, "", 1, 164}, + {itemKey, "}", 14, 25}, + {itemEOF, "", 0, 0}, + }, + }, + { + name: "should support prettified JSON with inner blocks", + input: ` + { + "http_port": 8227, + "port": 4227, + "write_deadline": "1h", + "cluster": { + "port": 6222, + "routes": [ + "nats://127.0.0.1:4222", + "nats://127.0.0.1:4223", + "nats://127.0.0.1:4224" + ] + } + } + `, + expected: []item{ + {itemKey, "http_port", 3, 28}, + {itemInteger, "8227", 3, 40}, + {itemKey, "port", 4, 28}, + {itemInteger, "4227", 4, 35}, + {itemKey, "write_deadline", 5, 28}, + {itemString, "1h", 5, 46}, + {itemKey, "cluster", 6, 28}, + {itemMapStart, "", 6, 39}, + {itemKey, "port", 7, 30}, + {itemInteger, "6222", 7, 37}, + {itemKey, "routes", 8, 30}, + {itemArrayStart, "", 8, 40}, + {itemString, "nats://127.0.0.1:4222", 9, 32}, + {itemString, "nats://127.0.0.1:4223", 10, 32}, + {itemString, "nats://127.0.0.1:4224", 11, 32}, + {itemArrayEnd, "", 12, 30}, + {itemMapEnd, "", 13, 28}, + {itemKey, "}", 14, 25}, + {itemEOF, "", 0, 0}, + }, + }, + { + name: "should support JSON with blocks", + input: `{ + "jetstream": { + "store_dir": "/tmp/nats" + "max_mem": 1000000, + }, + "port": 4222, + "server_name": "nats1" + } + `, + expected: []item{ + {itemKey, "jetstream", 2, 28}, + {itemMapStart, "", 2, 41}, + {itemKey, "store_dir", 3, 30}, + {itemString, "/tmp/nats", 3, 43}, + {itemKey, "max_mem", 4, 30}, + {itemInteger, "1000000", 4, 40}, + {itemMapEnd, "", 5, 28}, + {itemKey, "port", 6, 28}, + {itemInteger, "4222", 6, 35}, + {itemKey, "server_name", 7, 28}, + {itemString, "nats1", 7, 43}, + {itemKey, "}", 8, 25}, + {itemEOF, "", 0, 0}, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + lx := lex(test.input) + expect(t, lx, test.expected) + }) + } +} diff --git a/conf/parse.go b/conf/parse.go new file mode 100644 index 0000000..8dbb71f --- /dev/null +++ b/conf/parse.go @@ -0,0 +1,514 @@ +// Copyright 2013-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package conf supports a configuration file format used by gnatsd. It is +// a flexible format that combines the best of traditional +// configuration formats and newer styles such as JSON and YAML. +package conf + +// The format supported is less restrictive than today's formats. +// Supports mixed Arrays [], nested Maps {}, multiple comment types (# and //) +// Also supports key value assignments using '=' or ':' or whiteSpace() +// e.g. foo = 2, foo : 2, foo 2 +// maps can be assigned with no key separator as well +// semicolons as value terminators in key/value assignments are optional +// +// see parse_test.go for more examples. + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + "unicode" +) + +const _EMPTY_ = "" + +type parser struct { + mapping map[string]any + lx *lexer + + // The current scoped context, can be array or map + ctx any + + // stack of contexts, either map or array/slice stack + ctxs []any + + // Keys stack + keys []string + + // Keys stack as items + ikeys []item + + // The config file path, empty by default. + fp string + + // pedantic reports error when configuration is not correct. + pedantic bool + + // Tracks environment variable references, to avoid cycles + envVarReferences map[string]bool +} + +// Parse will return a map of keys to any, although concrete types +// underly them. The values supported are string, bool, int64, float64, DateTime. +// Arrays and nested Maps are also supported. +func Parse(data string) (map[string]any, error) { + p, err := parse(data, "", false) + if err != nil { + return nil, err + } + return p.mapping, nil +} + +// ParseWithChecks is equivalent to Parse but runs in pedantic mode. +func ParseWithChecks(data string) (map[string]any, error) { + p, err := parse(data, "", true) + if err != nil { + return nil, err + } + return p.mapping, nil +} + +// ParseFile is a helper to open file, etc. and parse the contents. +func ParseFile(fp string) (map[string]any, error) { + data, err := os.ReadFile(fp) + if err != nil { + return nil, fmt.Errorf("error opening config file: %v", err) + } + + p, err := parse(string(data), fp, false) + if err != nil { + return nil, err + } + return p.mapping, nil +} + +// ParseFileWithChecks is equivalent to ParseFile but runs in pedantic mode. +func ParseFileWithChecks(fp string) (map[string]any, error) { + data, err := os.ReadFile(fp) + if err != nil { + return nil, err + } + + p, err := parse(string(data), fp, true) + if err != nil { + return nil, err + } + + return p.mapping, nil +} + +// configDigest returns a digest for the parsed config. +func configDigest(m map[string]any) (string, error) { + digest := sha256.New() + e := json.NewEncoder(digest) + if err := e.Encode(m); err != nil { + return _EMPTY_, err + } + return fmt.Sprintf("sha256:%x", digest.Sum(nil)), nil +} + +// ParseFileWithChecksDigest returns the processed config and a digest +// that represents the configuration. +func ParseFileWithChecksDigest(fp string) (map[string]any, string, error) { + m, err := ParseFileWithChecks(fp) + if err != nil { + return nil, _EMPTY_, err + } + digest, err := configDigest(m) + if err != nil { + return nil, _EMPTY_, err + } + return m, digest, nil +} + +type token struct { + item item + value any + usedVariable bool + sourceFile string +} + +func (t *token) MarshalJSON() ([]byte, error) { + return json.Marshal(t.value) +} + +func (t *token) Value() any { + return t.value +} + +func (t *token) Line() int { + return t.item.line +} + +func (t *token) IsUsedVariable() bool { + return t.usedVariable +} + +func (t *token) SourceFile() string { + return t.sourceFile +} + +func (t *token) Position() int { + return t.item.pos +} + +func newParser(data, fp string, pedantic bool) *parser { + return &parser{ + mapping: make(map[string]any), + lx: lex(data), + ctxs: make([]any, 0, 4), + keys: make([]string, 0, 4), + ikeys: make([]item, 0, 4), + fp: filepath.Dir(fp), + pedantic: pedantic, + envVarReferences: make(map[string]bool), + } +} + +func parse(data, fp string, pedantic bool) (*parser, error) { + p := newParser(data, fp, pedantic) + if err := p.parse(fp); err != nil { + return nil, err + } + return p, nil +} + +func parseEnv(data string, parent *parser) (*parser, error) { + p := newParser(data, "", false) + p.envVarReferences = parent.envVarReferences + if err := p.parse(""); err != nil { + return nil, err + } + return p, nil +} + +func (p *parser) parse(fp string) error { + p.pushContext(p.mapping) + + var prevItem item + for { + it := p.next() + if it.typ == itemEOF { + // Here we allow the final character to be a bracket '}' + // in order to support JSON like configurations. + if prevItem.typ == itemKey && prevItem.val != mapEndString { + return fmt.Errorf("config is invalid (%s:%d:%d)", fp, it.line, it.pos) + } + break + } + prevItem = it + if err := p.processItem(it, fp); err != nil { + return err + } + } + return nil +} + +func (p *parser) next() item { + return p.lx.nextItem() +} + +func (p *parser) pushContext(ctx any) { + p.ctxs = append(p.ctxs, ctx) + p.ctx = ctx +} + +func (p *parser) popContext() any { + if len(p.ctxs) == 0 { + panic("BUG in parser, context stack empty") + } + li := len(p.ctxs) - 1 + last := p.ctxs[li] + p.ctxs = p.ctxs[0:li] + p.ctx = p.ctxs[len(p.ctxs)-1] + return last +} + +func (p *parser) pushKey(key string) { + p.keys = append(p.keys, key) +} + +func (p *parser) popKey() string { + if len(p.keys) == 0 { + panic("BUG in parser, keys stack empty") + } + li := len(p.keys) - 1 + last := p.keys[li] + p.keys = p.keys[0:li] + return last +} + +func (p *parser) pushItemKey(key item) { + p.ikeys = append(p.ikeys, key) +} + +func (p *parser) popItemKey() item { + if len(p.ikeys) == 0 { + panic("BUG in parser, item keys stack empty") + } + li := len(p.ikeys) - 1 + last := p.ikeys[li] + p.ikeys = p.ikeys[0:li] + return last +} + +func (p *parser) processItem(it item, fp string) error { + setValue := func(it item, v any) { + if p.pedantic { + p.setValue(&token{it, v, false, fp}) + } else { + p.setValue(v) + } + } + + switch it.typ { + case itemError: + return fmt.Errorf("Parse error on line %d: '%s'", it.line, it.val) + case itemKey: + // Keep track of the keys as items and strings, + // we do this in order to be able to still support + // includes without many breaking changes. + p.pushKey(it.val) + + if p.pedantic { + p.pushItemKey(it) + } + case itemMapStart: + newCtx := make(map[string]any) + p.pushContext(newCtx) + case itemMapEnd: + setValue(it, p.popContext()) + case itemString: + // FIXME(dlc) sanitize string? + setValue(it, it.val) + case itemInteger: + lastDigit := 0 + for _, r := range it.val { + if !unicode.IsDigit(r) && r != '-' { + break + } + lastDigit++ + } + numStr := it.val[:lastDigit] + num, err := strconv.ParseInt(numStr, 10, 64) + if err != nil { + if e, ok := err.(*strconv.NumError); ok && + e.Err == strconv.ErrRange { + return fmt.Errorf("integer '%s' is out of the range", it.val) + } + return fmt.Errorf("expected integer, but got '%s'", it.val) + } + // Process a suffix + suffix := strings.ToLower(strings.TrimSpace(it.val[lastDigit:])) + + switch suffix { + case "": + setValue(it, num) + case "k": + setValue(it, num*1000) + case "kb", "ki", "kib": + setValue(it, num*1024) + case "m": + setValue(it, num*1000*1000) + case "mb", "mi", "mib": + setValue(it, num*1024*1024) + case "g": + setValue(it, num*1000*1000*1000) + case "gb", "gi", "gib": + setValue(it, num*1024*1024*1024) + case "t": + setValue(it, num*1000*1000*1000*1000) + case "tb", "ti", "tib": + setValue(it, num*1024*1024*1024*1024) + case "p": + setValue(it, num*1000*1000*1000*1000*1000) + case "pb", "pi", "pib": + setValue(it, num*1024*1024*1024*1024*1024) + case "e": + setValue(it, num*1000*1000*1000*1000*1000*1000) + case "eb", "ei", "eib": + setValue(it, num*1024*1024*1024*1024*1024*1024) + } + case itemFloat: + num, err := strconv.ParseFloat(it.val, 64) + if err != nil { + if e, ok := err.(*strconv.NumError); ok && + e.Err == strconv.ErrRange { + return fmt.Errorf("float '%s' is out of the range", it.val) + } + return fmt.Errorf("expected float, but got '%s'", it.val) + } + setValue(it, num) + case itemBool: + switch strings.ToLower(it.val) { + case "true", "yes", "on": + setValue(it, true) + case "false", "no", "off": + setValue(it, false) + default: + return fmt.Errorf("expected boolean value, but got '%s'", it.val) + } + + case itemDatetime: + dt, err := time.Parse("2006-01-02T15:04:05Z", it.val) + if err != nil { + return fmt.Errorf( + "expected Zulu formatted DateTime, but got '%s'", it.val) + } + setValue(it, dt) + case itemArrayStart: + var array = make([]any, 0) + p.pushContext(array) + case itemArrayEnd: + array := p.ctx + p.popContext() + setValue(it, array) + case itemVariable: + value, found, err := p.lookupVariable(it.val) + if err != nil { + return fmt.Errorf("variable reference for '%s' on line %d could not be parsed: %s", + it.val, it.line, err) + } + if !found { + return fmt.Errorf("variable reference for '%s' on line %d can not be found", + it.val, it.line) + } + + if p.pedantic { + switch tk := value.(type) { + case *token: + // Mark the looked up variable as used, and make + // the variable reference become handled as a token. + tk.usedVariable = true + p.setValue(&token{it, tk.Value(), false, fp}) + default: + // Special case to add position context to bcrypt references. + p.setValue(&token{it, value, false, fp}) + } + } else { + p.setValue(value) + } + case itemInclude: + var ( + m map[string]any + err error + ) + if p.pedantic { + m, err = ParseFileWithChecks(filepath.Join(p.fp, it.val)) + } else { + m, err = ParseFile(filepath.Join(p.fp, it.val)) + } + if err != nil { + return fmt.Errorf("error parsing include file '%s', %v", it.val, err) + } + for k, v := range m { + p.pushKey(k) + + if p.pedantic { + switch tk := v.(type) { + case *token: + p.pushItemKey(tk.item) + } + } + p.setValue(v) + } + } + + return nil +} + +// Used to map an environment value into a temporary map to pass to secondary Parse call. +const pkey = "pk" + +// We special case raw strings here that are bcrypt'd. This allows us not to force quoting the strings +const bcryptPrefix = "2a$" + +// lookupVariable will lookup a variable reference. It will use block scoping on keys +// it has seen before, with the top level scoping being the environment variables. We +// ignore array contexts and only process the map contexts.. +// +// Returns true for ok if it finds something, similar to map. +func (p *parser) lookupVariable(varReference string) (any, bool, error) { + // Do special check to see if it is a raw bcrypt string. + if strings.HasPrefix(varReference, bcryptPrefix) { + return "$" + varReference, true, nil + } + + // Loop through contexts currently on the stack. + for i := len(p.ctxs) - 1; i >= 0; i-- { + ctx := p.ctxs[i] + // Process if it is a map context + if m, ok := ctx.(map[string]any); ok { + if v, ok := m[varReference]; ok { + return v, ok, nil + } + } + } + + // If we are here, we have exhausted our context maps and still not found anything. + // Detect reference cycles + if p.envVarReferences[varReference] { + return nil, false, fmt.Errorf("variable reference cycle for '%s'", varReference) + } + p.envVarReferences[varReference] = true + defer delete(p.envVarReferences, varReference) + + // Parse from the environment + if vStr, ok := os.LookupEnv(varReference); ok { + // Everything we get here will be a string value, so we need to process as a parser would. + if subp, err := parseEnv(fmt.Sprintf("%s=%s", pkey, vStr), p); err == nil { + v, ok := subp.mapping[pkey] + return v, ok, nil + } else { + return nil, false, err + } + } + return nil, false, nil +} + +func (p *parser) setValue(val any) { + // Test to see if we are on an array or a map + + // Array processing + if ctx, ok := p.ctx.([]any); ok { + p.ctx = append(ctx, val) + p.ctxs[len(p.ctxs)-1] = p.ctx + } + + // Map processing + if ctx, ok := p.ctx.(map[string]any); ok { + key := p.popKey() + + if p.pedantic { + // Change the position to the beginning of the key + // since more useful when reporting errors. + switch v := val.(type) { + case *token: + it := p.popItemKey() + v.item.pos = it.pos + v.item.line = it.line + ctx[key] = v + } + } else { + // FIXME(dlc), make sure to error if redefining same key? + ctx[key] = val + } + } +} diff --git a/conf/parse_test.go b/conf/parse_test.go new file mode 100644 index 0000000..1194ab0 --- /dev/null +++ b/conf/parse_test.go @@ -0,0 +1,1034 @@ +package conf + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +// Test to make sure we get what we expect. + +func test(t *testing.T, data string, ex map[string]any) { + t.Helper() + m, err := Parse(data) + if err != nil { + t.Fatalf("Received err: %v\n", err) + } + if m == nil { + t.Fatal("Received nil map") + } + + if !reflect.DeepEqual(m, ex) { + t.Fatalf("Not Equal:\nReceived: '%+v'\nExpected: '%+v'\n", m, ex) + } + + // Also test ParseWithChecks + n, err := ParseWithChecks(data) + if err != nil { + t.Fatalf("Received err: %v\n", err) + } + if n == nil { + t.Fatal("Received nil map") + } + // Compare with the original results. + a, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + b, err := json.Marshal(n) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(b, a) { + t.Fatalf("Not Equal:\nReceived: '%+v'\nExpected: '%+v'\n", string(a), string(b)) + } +} + +func TestSimpleTopLevel(t *testing.T) { + ex := map[string]any{ + "foo": "1", + "bar": float64(2.2), + "baz": true, + "boo": int64(22), + } + test(t, "foo='1'; bar=2.2; baz=true; boo=22", ex) +} + +func TestBools(t *testing.T) { + ex := map[string]any{ + "foo": true, + } + test(t, "foo=true", ex) + test(t, "foo=TRUE", ex) + test(t, "foo=true", ex) + test(t, "foo=yes", ex) + test(t, "foo=on", ex) +} + +var varSample = ` + index = 22 + foo = $index +` + +func TestSimpleVariable(t *testing.T) { + ex := map[string]any{ + "index": int64(22), + "foo": int64(22), + } + test(t, varSample, ex) +} + +var varNestedSample = ` + index = 22 + nest { + index = 11 + foo = $index + } + bar = $index +` + +func TestNestedVariable(t *testing.T) { + ex := map[string]any{ + "index": int64(22), + "nest": map[string]any{ + "index": int64(11), + "foo": int64(11), + }, + "bar": int64(22), + } + test(t, varNestedSample, ex) +} + +func TestMissingVariable(t *testing.T) { + _, err := Parse("foo=$index") + if err == nil { + t.Fatalf("Expected an error for a missing variable, got none") + } + if !strings.HasPrefix(err.Error(), "variable reference") { + t.Fatalf("Wanted a variable reference err, got %q\n", err) + } +} + +func TestEnvVariable(t *testing.T) { + ex := map[string]any{ + "foo": int64(22), + } + evar := "__UNIQ22__" + os.Setenv(evar, "22") + defer os.Unsetenv(evar) + test(t, fmt.Sprintf("foo = $%s", evar), ex) +} + +func TestEnvVariableString(t *testing.T) { + ex := map[string]any{ + "foo": "xyz", + } + evar := "__UNIQ22__" + os.Setenv(evar, "xyz") + defer os.Unsetenv(evar) + test(t, fmt.Sprintf("foo = $%s", evar), ex) +} + +func TestEnvVariableStringStartingWithNumber(t *testing.T) { + evar := "__UNIQ22__" + os.Setenv(evar, "3xyz") + defer os.Unsetenv(evar) + + _, err := Parse("foo = $%s") + if err == nil { + t.Fatalf("Expected err not being able to process string: %v\n", err) + } +} + +func TestEnvVariableStringStartingWithNumberAndSizeUnit(t *testing.T) { + ex := map[string]any{ + "foo": "3Gyz", + } + evar := "__UNIQ22__" + os.Setenv(evar, "3Gyz") + defer os.Unsetenv(evar) + test(t, fmt.Sprintf("foo = $%s", evar), ex) +} + +func TestEnvVariableStringStartingWithNumberUsingQuotes(t *testing.T) { + ex := map[string]any{ + "foo": "3xyz", + } + evar := "__UNIQ22__" + os.Setenv(evar, "'3xyz'") + defer os.Unsetenv(evar) + test(t, fmt.Sprintf("foo = $%s", evar), ex) +} + +func TestBcryptVariable(t *testing.T) { + ex := map[string]any{ + "password": "$2a$11$ooo", + } + test(t, "password: $2a$11$ooo", ex) +} + +var easynum = ` +k = 8k +kb = 4kb +ki = 3ki +kib = 4ki +m = 1m +mb = 2MB +mi = 2Mi +mib = 64MiB +g = 2g +gb = 22GB +gi = 22Gi +gib = 22GiB +tb = 22TB +ti = 22Ti +tib = 22TiB +pb = 22PB +pi = 22Pi +pib = 22PiB +` + +func TestConvenientNumbers(t *testing.T) { + ex := map[string]any{ + "k": int64(8 * 1000), + "kb": int64(4 * 1024), + "ki": int64(3 * 1024), + "kib": int64(4 * 1024), + "m": int64(1000 * 1000), + "mb": int64(2 * 1024 * 1024), + "mi": int64(2 * 1024 * 1024), + "mib": int64(64 * 1024 * 1024), + "g": int64(2 * 1000 * 1000 * 1000), + "gb": int64(22 * 1024 * 1024 * 1024), + "gi": int64(22 * 1024 * 1024 * 1024), + "gib": int64(22 * 1024 * 1024 * 1024), + "tb": int64(22 * 1024 * 1024 * 1024 * 1024), + "ti": int64(22 * 1024 * 1024 * 1024 * 1024), + "tib": int64(22 * 1024 * 1024 * 1024 * 1024), + "pb": int64(22 * 1024 * 1024 * 1024 * 1024 * 1024), + "pi": int64(22 * 1024 * 1024 * 1024 * 1024 * 1024), + "pib": int64(22 * 1024 * 1024 * 1024 * 1024 * 1024), + } + test(t, easynum, ex) +} + +func TestParseFileWithChecksDigestPreservesConfigKeyUsedAsVariable(t *testing.T) { + confFile := filepath.Join(t.TempDir(), "nats.conf") + if err := os.WriteFile(confFile, []byte(` + port = 4222 + monitor_port = $port + `), 0666); err != nil { + t.Fatal(err) + } + + m, _, err := ParseFileWithChecksDigest(confFile) + if err != nil { + t.Fatal(err) + } + if _, ok := m["port"]; !ok { + t.Fatal("expected config key used as a variable reference to be preserved") + } +} + +var sample1 = ` +foo { + host { + ip = '127.0.0.1' + port = 4242 + } + servers = [ "a.com", "b.com", "c.com"] +} +` + +func TestSample1(t *testing.T) { + ex := map[string]any{ + "foo": map[string]any{ + "host": map[string]any{ + "ip": "127.0.0.1", + "port": int64(4242), + }, + "servers": []any{"a.com", "b.com", "c.com"}, + }, + } + test(t, sample1, ex) +} + +var cluster = ` +cluster { + port: 4244 + + authorization { + user: route_user + password: top_secret + timeout: 1 + } + + # Routes are actively solicited and connected to from this server. + # Other servers can connect to us if they supply the correct credentials + # in their routes definitions from above. + + // Test both styles of comments + + routes = [ + nats-route://foo:bar@apcera.me:4245 + nats-route://foo:bar@apcera.me:4246 + ] +} +` + +func TestSample2(t *testing.T) { + ex := map[string]any{ + "cluster": map[string]any{ + "port": int64(4244), + "authorization": map[string]any{ + "user": "route_user", + "password": "top_secret", + "timeout": int64(1), + }, + "routes": []any{ + "nats-route://foo:bar@apcera.me:4245", + "nats-route://foo:bar@apcera.me:4246", + }, + }, + } + + test(t, cluster, ex) +} + +var sample3 = ` +foo { + expr = '(true == "false")' + text = 'This is a multi-line +text block.' +} +` + +func TestSample3(t *testing.T) { + ex := map[string]any{ + "foo": map[string]any{ + "expr": "(true == \"false\")", + "text": "This is a multi-line\ntext block.", + }, + } + test(t, sample3, ex) +} + +var sample4 = ` + array [ + { abc: 123 } + { xyz: "word" } + ] +` + +func TestSample4(t *testing.T) { + ex := map[string]any{ + "array": []any{ + map[string]any{"abc": int64(123)}, + map[string]any{"xyz": "word"}, + }, + } + test(t, sample4, ex) +} + +var sample5 = ` + now = 2016-05-04T18:53:41Z + gmt = false + +` + +func TestSample5(t *testing.T) { + dt, _ := time.Parse("2006-01-02T15:04:05Z", "2016-05-04T18:53:41Z") + ex := map[string]any{ + "now": dt, + "gmt": false, + } + test(t, sample5, ex) +} + +func TestIncludes(t *testing.T) { + ex := map[string]any{ + "listen": "127.0.0.1:4222", + "authorization": map[string]any{ + "ALICE_PASS": "$2a$10$UHR6GhotWhpLsKtVP0/i6.Nh9.fuY73cWjLoJjb2sKT8KISBcUW5q", + "BOB_PASS": "$2a$11$dZM98SpGeI7dCFFGSpt.JObQcix8YHml4TBUZoge9R1uxnMIln5ly", + "users": []any{ + map[string]any{ + "user": "alice", + "password": "$2a$10$UHR6GhotWhpLsKtVP0/i6.Nh9.fuY73cWjLoJjb2sKT8KISBcUW5q"}, + map[string]any{ + "user": "bob", + "password": "$2a$11$dZM98SpGeI7dCFFGSpt.JObQcix8YHml4TBUZoge9R1uxnMIln5ly"}, + }, + "timeout": float64(0.5), + }, + } + + m, err := ParseFile("simple.conf") + if err != nil { + t.Fatalf("Received err: %v\n", err) + } + if m == nil { + t.Fatal("Received nil map") + } + + if !reflect.DeepEqual(m, ex) { + t.Fatalf("Not Equal:\nReceived: '%+v'\nExpected: '%+v'\n", m, ex) + } +} + +var varIncludedVariablesSample = ` +authorization { + + include "./includes/passwords.conf" + + CAROL_PASS: foo + + users = [ + {user: alice, password: $ALICE_PASS} + {user: bob, password: $BOB_PASS} + {user: carol, password: $CAROL_PASS} + ] +} +` + +func TestIncludeVariablesWithChecks(t *testing.T) { + p, err := parse(varIncludedVariablesSample, "", true) + if err != nil { + t.Fatalf("Received err: %v\n", err) + } + key := "authorization" + m, ok := p.mapping[key] + if !ok { + t.Errorf("Expected %q to be in the config", key) + } + expectKeyVal := func(t *testing.T, m any, expectedKey string, expectedVal string, expectedLine, expectedPos int) { + t.Helper() + tk := m.(*token) + v := tk.Value() + vv := v.(map[string]any) + value, ok := vv[expectedKey] + if !ok { + t.Errorf("Expected key %q", expectedKey) + } + tk, ok = value.(*token) + if !ok { + t.Fatalf("Expected token %v", value) + } + if tk.Line() != expectedLine { + t.Errorf("Expected token to be at line %d, got: %d", expectedLine, tk.Line()) + } + if tk.Position() != expectedPos { + t.Errorf("Expected token to be at position %d, got: %d", expectedPos, tk.Position()) + } + v = tk.Value() + if v != expectedVal { + t.Errorf("Expected %q, got: %s", expectedVal, v) + } + } + expectKeyVal(t, m, "ALICE_PASS", "$2a$10$UHR6GhotWhpLsKtVP0/i6.Nh9.fuY73cWjLoJjb2sKT8KISBcUW5q", 2, 1) + expectKeyVal(t, m, "BOB_PASS", "$2a$11$dZM98SpGeI7dCFFGSpt.JObQcix8YHml4TBUZoge9R1uxnMIln5ly", 3, 1) + expectKeyVal(t, m, "CAROL_PASS", "foo", 6, 3) +} + +func TestParserNoInfiniteLoop(t *testing.T) { + for _, test := range []string{`A@@Føøøø?˛ø:{øøøø˙˙`, `include "9/�`} { + if _, err := Parse(test); err == nil { + t.Fatal("expected an error") + } else if !strings.Contains(err.Error(), "Unexpected EOF") { + t.Fatal("expected unexpected eof error") + } + } +} + +func TestParseWithNoValuesAreInvalid(t *testing.T) { + for _, test := range []struct { + name string + conf string + err string + }{ + { + "invalid key without values", + `aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, + "config is invalid (:1:41)", + }, + { + "invalid untrimmed key without values", + ` aaaaaaaaaaaaaaaaaaaaaaaaaaa`, + "config is invalid (:1:41)", + }, + { + "invalid untrimmed key without values", + ` aaaaaaaaaaaaaaaaaaaaaaaaaaa `, + "config is invalid (:1:41)", + }, + { + "invalid keys after comments", + ` + # with comments and no spaces to create key values + # is also an invalid config. + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + `, + "config is invalid (:5:25)", + }, + { + "comma separated without values are invalid", + ` + a,a,a,a,a,a,a,a,a,a,a + `, + "config is invalid (:3:25)", + }, + } { + t.Run(test.name, func(t *testing.T) { + if _, err := parse(test.conf, "", true); err == nil { + t.Error("expected an error") + } else if !strings.Contains(err.Error(), test.err) { + t.Errorf("expected invalid conf error, got: %v", err) + } + }) + } +} + +func TestParseWithNoValuesEmptyConfigsAreValid(t *testing.T) { + for _, test := range []struct { + name string + conf string + }{ + { + "empty conf", + "", + }, + { + "empty conf with line breaks", + ` + + + `, + }, + { + "just comments with no values", + ` + # just comments with no values + # is still valid. + `, + }, + } { + t.Run(test.name, func(t *testing.T) { + if _, err := parse(test.conf, "", true); err != nil { + t.Errorf("unexpected error: %v", err) + } + }) + } +} + +func TestParseWithTrailingBracketsAreValid(t *testing.T) { + for _, test := range []struct { + name string + conf string + }{ + { + "empty conf", + "{}", + }, + { + "just comments with no values", + ` + { + # comments in the body + } + `, + }, + { + // trailing brackets accidentally can become keys, + // this is valid since needed to support JSON like configs.. + "trailing brackets after config", + ` + accounts { users = [{}]} + } + `, + }, + { + "wrapped in brackets", + `{ + accounts { users = [{}]} + } + `, + }, + } { + t.Run(test.name, func(t *testing.T) { + if _, err := parse(test.conf, "", true); err != nil { + t.Errorf("unexpected error: %v", err) + } + }) + } +} + +func TestParseWithNoValuesIncludes(t *testing.T) { + for _, test := range []struct { + input string + includes map[string]string + err string + linepos string + }{ + { + `# includes + accounts { + foo { include 'foo.conf'} + bar { users = [{user = "bar"}] } + quux { include 'quux.conf'} + } + `, + map[string]string{ + "foo.conf": ``, + "quux.conf": `?????????????`, + }, + "error parsing include file 'quux.conf', config is invalid", + "quux.conf:1:1", + }, + { + `# includes + accounts { + foo { include 'foo.conf'} + bar { include 'bar.conf'} + quux { include 'quux.conf'} + } + `, + map[string]string{ + "foo.conf": ``, // Empty configs are ok + "bar.conf": `AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`, + "quux.conf": ` + # just some comments, + # and no key values also ok. + `, + }, + "error parsing include file 'bar.conf', config is invalid", + "bar.conf:1:34", + }, + } { + t.Run("", func(t *testing.T) { + sdir := t.TempDir() + f, err := os.CreateTemp(sdir, "nats.conf-") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(f.Name(), []byte(test.input), 066); err != nil { + t.Error(err) + } + if test.includes != nil { + for includeFile, contents := range test.includes { + inf, err := os.Create(filepath.Join(sdir, includeFile)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(inf.Name(), []byte(contents), 066); err != nil { + t.Error(err) + } + } + } + if _, err := parse(test.input, f.Name(), true); err == nil { + t.Error("expected an error") + } else if !strings.Contains(err.Error(), test.err) || !strings.Contains(err.Error(), test.linepos) { + t.Errorf("expected invalid conf error, got: %v", err) + } + }) + } +} + +func TestJSONParseCompat(t *testing.T) { + for _, test := range []struct { + name string + input string + includes map[string]string + expected map[string]any + }{ + { + "JSON with nested blocks", + ` + { + "http_port": 8227, + "port": 4227, + "write_deadline": "1h", + "cluster": { + "port": 6222, + "routes": [ + "nats://127.0.0.1:4222", + "nats://127.0.0.1:4223", + "nats://127.0.0.1:4224" + ] + } + } + `, + nil, + map[string]any{ + "http_port": int64(8227), + "port": int64(4227), + "write_deadline": "1h", + "cluster": map[string]any{ + "port": int64(6222), + "routes": []any{ + "nats://127.0.0.1:4222", + "nats://127.0.0.1:4223", + "nats://127.0.0.1:4224", + }, + }, + }, + }, + { + "JSON with nested blocks", + `{ + "jetstream": { + "store_dir": "/tmp/nats" + "max_mem": 1000000, + }, + "port": 4222, + "server_name": "nats1" + } + `, + nil, + map[string]any{ + "jetstream": map[string]any{ + "store_dir": "/tmp/nats", + "max_mem": int64(1_000_000), + }, + "port": int64(4222), + "server_name": "nats1", + }, + }, + { + "JSON empty object in one line", + `{}`, + nil, + map[string]any{}, + }, + { + "JSON empty object with line breaks", + ` + { + } + `, + nil, + map[string]any{}, + }, + { + "JSON includes", + ` + accounts { + foo { include 'foo.json' } + bar { include 'bar.json' } + quux { include 'quux.json' } + } + `, + map[string]string{ + "foo.json": `{ "users": [ {"user": "foo"} ] }`, + "bar.json": `{ + "users": [ {"user": "bar"} ] + }`, + "quux.json": `{}`, + }, + map[string]any{ + "accounts": map[string]any{ + "foo": map[string]any{ + "users": []any{ + map[string]any{ + "user": "foo", + }, + }, + }, + "bar": map[string]any{ + "users": []any{ + map[string]any{ + "user": "bar", + }, + }, + }, + "quux": map[string]any{}, + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + sdir := t.TempDir() + f, err := os.CreateTemp(sdir, "nats.conf-") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(f.Name(), []byte(test.input), 066); err != nil { + t.Error(err) + } + if test.includes != nil { + for includeFile, contents := range test.includes { + inf, err := os.Create(filepath.Join(sdir, includeFile)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(inf.Name(), []byte(contents), 066); err != nil { + t.Error(err) + } + } + } + m, err := ParseFile(f.Name()) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if !reflect.DeepEqual(m, test.expected) { + t.Fatalf("Not Equal:\nReceived: '%+v'\nExpected: '%+v'\n", m, test.expected) + } + }) + } +} + +func TestBlocks(t *testing.T) { + for _, test := range []struct { + name string + input string + expected map[string]any + err string + linepos string + }{ + { + "inline block", + `{ listen: 0.0.0.0:4222 }`, + map[string]any{ + "listen": "0.0.0.0:4222", + }, + "", "", + }, + { + "newline block", + `{ + listen: 0.0.0.0:4222 + }`, + map[string]any{ + "listen": "0.0.0.0:4222", + }, + "", "", + }, + { + "newline block with trailing comment", + ` + { + listen: 0.0.0.0:4222 + } + # wibble + `, + map[string]any{ + "listen": "0.0.0.0:4222", + }, + "", "", + }, + { + "nested newline blocks with trailing comment", + ` + { + { + listen: 0.0.0.0:4222 // random comment + } + # wibble1 + } + # wibble2 + `, + map[string]any{ + "listen": "0.0.0.0:4222", + }, + "", "", + }, + { + "top line values in block scope", + ` + { + "debug": False + "prof_port": 8221 + "server_name": "aws-useast2-natscj1-1" + } + `, + map[string]any{ + "debug": false, + "prof_port": int64(8221), + "server_name": "aws-useast2-natscj1-1", + }, + "", "", + }, + { + "comment in block scope after value parse", + ` + { + "debug": False + "server_name": "gcp-asianortheast3-natscj1-1" + + # Profile port specification. + "prof_port": 8221 + } + `, + map[string]any{ + "debug": false, + "prof_port": int64(8221), + "server_name": "gcp-asianortheast3-natscj1-1", + }, + "", "", + }, + } { + t.Run(test.name, func(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "nats.conf-") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(f.Name(), []byte(test.input), 066); err != nil { + t.Error(err) + } + if m, err := ParseFile(f.Name()); err == nil { + if !reflect.DeepEqual(m, test.expected) { + t.Fatalf("Not Equal:\nReceived: '%+v'\nExpected: '%+v'\n", m, test.expected) + } + } else if !strings.Contains(err.Error(), test.err) || !strings.Contains(err.Error(), test.linepos) { + t.Errorf("expected invalid conf error, got: %v", err) + } else if err != nil { + t.Error(err) + } + }) + } +} + +func TestParseDigest(t *testing.T) { + for _, test := range []struct { + input string + includes map[string]string + digest string + }{ + { + `foo = bar`, + nil, + "sha256:226e49e13d16e5e8aa0d62e58cd63361bf097d3e2b2444aa3044334628a2e8de", + }, + { + `# Comments and whitespace have no effect + foo = bar + `, + nil, + "sha256:226e49e13d16e5e8aa0d62e58cd63361bf097d3e2b2444aa3044334628a2e8de", + }, + { + `# Syntax changes have no effect + 'foo': 'bar' + `, + nil, + "sha256:226e49e13d16e5e8aa0d62e58cd63361bf097d3e2b2444aa3044334628a2e8de", + }, + { + `# Syntax changes have no effect + { 'foo': 'bar' } + `, + nil, + "sha256:226e49e13d16e5e8aa0d62e58cd63361bf097d3e2b2444aa3044334628a2e8de", + }, + { + `# substitutions + BAR_USERS = { users = [ {user = "bar"} ]} + hello = 'world' + accounts { + QUUX_USERS = [ { user: quux }] + bar = $BAR_USERS + quux = { users = $QUUX_USERS } + } + very { nested { env { VAR = 'NESTED', quux = $VAR }}} + `, + nil, + "sha256:bddb282343249c26d3edcef9bfaa9d2711505fc67210380e871405ba394a9186", + }, + { + `# substitutions, same as previous one without env vars. + hello = 'world' + accounts { + bar = { users = [ { user = "bar" } ]} + quux = { users = [ { user: quux } ]} + } + very { nested { env { quux = 'NESTED' }}} + `, + nil, + "sha256:34f8faf3f269fe7509edc4742f20c8c4a7ad51fe21f8b361764314b533ac3ab5", + }, + { + `# substitutions + BAR_USERS = { users = [ {user = "foo"} ]} + bar = $BAR_USERS + accounts { + users = $BAR_USERS + } + `, + nil, + "sha256:b0e2ba0b8ec2cb75681b5c5d61789cda0c9942c2f9fe16cdb7f6703364485360", + }, + { + `# substitutions + bar = { users = [ {user = "foo"} ]} + accounts { + users = { users = [ {user = "foo"} ]} + } + `, + nil, + "sha256:f5d943b4ed22b80c6199203f8a7eaa8eb68ef7b2d46ef6b1b26f05e21f8beb13", + }, + { + `# includes + accounts { + foo { include 'foo.conf'} + bar { users = [{user = "bar"}] } + quux { include 'quux.conf'} + } + `, + map[string]string{ + "foo.conf": ` users = [{user = "foo"}]`, + "quux.conf": ` users = [{user = "quux"}]`, + }, + "sha256:e72d70c91b64b0f880f86decb95ec2600cbdcf8bdcd2355fce5ebc54a84a77e9", + }, + { + `# includes + accounts { + foo { include 'foo.conf'} + bar { include 'bar.conf'} + quux { include 'quux.conf'} + } + `, + map[string]string{ + "foo.conf": ` users = [{user = "foo"}]`, + "bar.conf": ` users = [{user = "bar"}]`, + "quux.conf": ` users = [{user = "quux"}]`, + }, + "sha256:e72d70c91b64b0f880f86decb95ec2600cbdcf8bdcd2355fce5ebc54a84a77e9", + }, + } { + t.Run("", func(t *testing.T) { + sdir := t.TempDir() + f, err := os.CreateTemp(sdir, "nats.conf-") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(f.Name(), []byte(test.input), 066); err != nil { + t.Error(err) + } + if test.includes != nil { + for includeFile, contents := range test.includes { + inf, err := os.Create(filepath.Join(sdir, includeFile)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(inf.Name(), []byte(contents), 066); err != nil { + t.Error(err) + } + } + } + _, digest, err := ParseFileWithChecksDigest(f.Name()) + if err != nil { + t.Fatal(err) + } + if digest != test.digest { + t.Errorf("\ngot: %s\nexpected: %s", digest, test.digest) + } + }) + } +} diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 0000000..e431f8a --- /dev/null +++ b/doc/README.md @@ -0,0 +1,3 @@ +# Architecture Decision Records + +The NATS ADR documents have moved to their [own repository](https://github.com/nats-io/nats-architecture-and-design/) diff --git a/docker/Dockerfile.nightly b/docker/Dockerfile.nightly new file mode 100644 index 0000000..3878c5a --- /dev/null +++ b/docker/Dockerfile.nightly @@ -0,0 +1,49 @@ +FROM --platform=$BUILDPLATFORM golang:alpine AS builder + +ARG VERSION="nightly" +ARG GIT_COMMIT + +ARG TARGETOS +ARG TARGETARCH + +ENV GOOS=$TARGETOS \ + GOARCH=$TARGETARCH \ + GO111MODULE=on \ + CGO_ENABLED=0 + +RUN apk add --no-cache ca-certificates +RUN update-ca-certificates + +WORKDIR /src +RUN mkdir -p /src/out/$GOOS/$GOARCH + +COPY ./nats-server/ /src/nats-server/ +COPY ./nsc/ /src/nsc/ +COPY ./natscli/ /src/natscli/ + +RUN cd /src/nats-server && go build -trimpath \ + -ldflags "-w -X server.serverVersion=${VERSION},server.gitCommit=${GIT_COMMIT}" \ + -o /src/out/$TARGETOS/$TARGETARCH/nats-server . + +RUN cd /src/natscli && go build -trimpath \ + -ldflags "-w -X main.version=${VERSION}" \ + -o /src/out/$TARGETOS/$TARGETARCH/nats ./nats + +RUN cd /src/nsc && go build -trimpath \ + -o /src/out/$TARGETOS/$TARGETARCH/nsc . + +FROM --platform=$TARGETPLATFORM alpine:latest + +ARG TARGETOS +ARG TARGETARCH + +RUN apk add --no-cache tzdata + +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=builder /src/out/$TARGETOS/$TARGETARCH/nats-server /bin/nats-server +COPY --from=builder /src/out/$TARGETOS/$TARGETARCH/nats /bin/nats +COPY --from=builder /src/out/$TARGETOS/$TARGETARCH/nsc /bin/nsc + +EXPOSE 4222 8222 6222 5222 + +ENTRYPOINT ["/bin/nats-server"] diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6ba7f51 --- /dev/null +++ b/go.mod @@ -0,0 +1,19 @@ +module github.com/nats-io/nats-server/v2 + +go 1.26.0 + +toolchain go1.26.8 + +require ( + github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op + github.com/google/go-tpm v0.9.8 + github.com/klauspost/compress v1.20.0 + github.com/minio/highwayhash v1.0.4 + github.com/nats-io/jwt/v2 v2.8.2 + github.com/nats-io/nats.go v1.51.0 + github.com/nats-io/nkeys v0.4.16 + github.com/nats-io/nuid v1.0.1 + golang.org/x/crypto v0.57.0 + golang.org/x/sys v0.48.0 + golang.org/x/time v0.16.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..d49ad70 --- /dev/null +++ b/go.sum @@ -0,0 +1,23 @@ +github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= +github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc= +github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc= +github.com/nats-io/nats.go v1.51.0 h1:ByW84XTz6W03GSSsygsZcA+xgKK8vPGaa/FCAAEHnAI= +github.com/nats-io/nats.go v1.51.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= +golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= diff --git a/internal/antithesis/noop.go b/internal/antithesis/noop.go new file mode 100644 index 0000000..3255e88 --- /dev/null +++ b/internal/antithesis/noop.go @@ -0,0 +1,27 @@ +// Copyright 2022-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file is used iff the `enable_antithesis_sdk` build tag is not present +//go:build !enable_antithesis_sdk + +package antithesis + +import ( + "testing" +) + +// AssertUnreachable this implementation is a NOOP +func AssertUnreachable(_ testing.TB, _ string, _ map[string]any) {} + +// Assert this implementation is a NOOP +func Assert(_ testing.TB, _ bool, _ string, _ map[string]any) {} diff --git a/internal/antithesis/test_assert.go b/internal/antithesis/test_assert.go new file mode 100644 index 0000000..23bac3a --- /dev/null +++ b/internal/antithesis/test_assert.go @@ -0,0 +1,111 @@ +// Copyright 2022-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file is used iff the `enable_antithesis_sdk` build tag is present +//go:build enable_antithesis_sdk + +package antithesis + +import ( + "encoding/json" + "fmt" + "runtime/debug" + "testing" + + "github.com/antithesishq/antithesis-sdk-go/assert" +) + +// This file provides assertions utility functions suitable for use in tests. +// It is a thin wrapper around the Antithesis SDK. +// +// Notice that unlike other assertions libraries, a violation does not halt execution or fail the test. +// The effects of violating an assertion, are: +// 1. Print the violation message, prefixed by the test name in which it happened +// 2. Print the stack for the calling goroutine so each failed test has a clear trace of the failure +// 3. Invoke the underlying Antithesis assertion +// +// N.B. Enabling this module outside for tests running outside of Antithesis will enable 1 and 2 above, but not 3. +// therefore it can be useful to output additional test failure details when running tests locally or in CI. + +// AssertUnreachable is used to flag code branches that should not get invoked ever. +// Example: +// +// pubAck, err := js.Publish(...) +// +// if err != nil { +// antithesis.AssertUnreachable(t, "Publish failed", map[string]any{"error": err.Error()}) +// t.Fatalf("Publish failed with error: %s", err) +// } +func AssertUnreachable(t testing.TB, message string, details map[string]any) { + // Always print a message + fmt.Printf("{*} [%s] Assert Unreachable violation: %s\n", t.Name(), message) + if details != nil && len(details) > 0 { + fmt.Printf("{*} Details:\n") + jsonDetails, err := json.MarshalIndent(details, "", " ") + if err != nil { + panic(err) + } + fmt.Println(string(jsonDetails)) + } + + // Always print the stack trace + fmt.Printf("{*} Stack trace:\n") + debug.PrintStack() + + // N.B. as of today, message (as-is) is the unique identifier of the event + // Therefore this will de-duplicate the same assertion failing in 2 different tests + // But not the same assertion failing at 2 different lines of the same test + messageWithTestName := fmt.Sprintf("[%s] %s", t.Name(), message) + + // Fire assertion violation event (if Antithesis is enabled) + assert.Unreachable(messageWithTestName, details) +} + +// Assert is used to check that some given condition is always true, +// Example: +// +// antithesis.Assert(t, sequence > lastSequence, "Non-monotonic stream sequence number", map[string]any{ +// "stream": streamName, +// "connection_id": nc.Id(), +// "sequence": sequence, +// "lastSequence": lastSequence, +// }) +func Assert(t testing.TB, condition bool, message string, details map[string]any) { + // Condition is true, nothing to do + if condition { + return + } + + // Always print a message + fmt.Printf("{*} [%s] Assert violation: %s\n", t.Name(), message) + if details != nil && len(details) > 0 { + fmt.Printf("{*} Details:\n") + jsonDetails, err := json.MarshalIndent(details, "", " ") + if err != nil { + panic(err) + } + fmt.Println(string(jsonDetails)) + } + + // Always print the stack trace + fmt.Printf("{*} Stack trace:\n") + debug.PrintStack() + + // N.B. as of today, message (as-is) is the unique identifier of the event + // Therefore this will de-duplicate the same assertion failing in 2 different tests + // But not the same assertion failing at 2 different lines of the same test + messageWithTestName := fmt.Sprintf("[%s] %s", t.Name(), message) + + // Fire assertion violation event (if Antithesis is enabled) + assert.AlwaysOrUnreachable(false, messageWithTestName, details) +} diff --git a/internal/ldap/dn.go b/internal/ldap/dn.go new file mode 100644 index 0000000..e155ad4 --- /dev/null +++ b/internal/ldap/dn.go @@ -0,0 +1,305 @@ +// Copyright (c) 2011-2015 Michael Mitton (mmitton@gmail.com) +// Portions copyright (c) 2015-2016 go-ldap Authors +package ldap + +import ( + "bytes" + "crypto/x509/pkix" + "encoding/asn1" + enchex "encoding/hex" + "errors" + "fmt" + "strings" +) + +var attributeTypeNames = map[string]string{ + "2.5.4.3": "CN", + "2.5.4.5": "SERIALNUMBER", + "2.5.4.6": "C", + "2.5.4.7": "L", + "2.5.4.8": "ST", + "2.5.4.9": "STREET", + "2.5.4.10": "O", + "2.5.4.11": "OU", + "2.5.4.17": "POSTALCODE", + // FIXME: Add others. + "0.9.2342.19200300.100.1.25": "DC", +} + +// AttributeTypeAndValue represents an attributeTypeAndValue from https://tools.ietf.org/html/rfc4514 +type AttributeTypeAndValue struct { + // Type is the attribute type + Type string + // Value is the attribute value + Value string +} + +// RelativeDN represents a relativeDistinguishedName from https://tools.ietf.org/html/rfc4514 +type RelativeDN struct { + Attributes []*AttributeTypeAndValue +} + +// DN represents a distinguishedName from https://tools.ietf.org/html/rfc4514 +type DN struct { + RDNs []*RelativeDN +} + +// FromCertSubject takes a pkix.Name from a cert and returns a DN +// that uses the same set. Does not support multi value RDNs. +func FromCertSubject(subject pkix.Name) (*DN, error) { + dn := &DN{ + RDNs: make([]*RelativeDN, 0), + } + for i := len(subject.Names) - 1; i >= 0; i-- { + name := subject.Names[i] + oidString := name.Type.String() + typeName, ok := attributeTypeNames[oidString] + if !ok { + return nil, fmt.Errorf("invalid type name: %+v", name) + } + v, ok := name.Value.(string) + if !ok { + return nil, fmt.Errorf("invalid type value: %+v", v) + } + rdn := &RelativeDN{ + Attributes: []*AttributeTypeAndValue{ + { + Type: typeName, + Value: v, + }, + }, + } + dn.RDNs = append(dn.RDNs, rdn) + } + return dn, nil +} + +// FromRawCertSubject takes a raw subject from a certificate +// and uses asn1.Unmarshal to get the individual RDNs in the +// original order, including multi-value RDNs. +func FromRawCertSubject(rawSubject []byte) (*DN, error) { + dn := &DN{ + RDNs: make([]*RelativeDN, 0), + } + var rdns pkix.RDNSequence + _, err := asn1.Unmarshal(rawSubject, &rdns) + if err != nil { + return nil, err + } + + for i := len(rdns) - 1; i >= 0; i-- { + rdn := rdns[i] + if len(rdn) == 0 { + continue + } + + r := &RelativeDN{} + attrs := make([]*AttributeTypeAndValue, 0) + for j := len(rdn) - 1; j >= 0; j-- { + atv := rdn[j] + + typeName := "" + name := atv.Type.String() + typeName, ok := attributeTypeNames[name] + if !ok { + return nil, fmt.Errorf("invalid type name: %+v", name) + } + value, ok := atv.Value.(string) + if !ok { + return nil, fmt.Errorf("invalid type value: %+v", atv.Value) + } + attr := &AttributeTypeAndValue{ + Type: typeName, + Value: value, + } + attrs = append(attrs, attr) + } + r.Attributes = attrs + dn.RDNs = append(dn.RDNs, r) + } + + return dn, nil +} + +// ParseDN returns a distinguishedName or an error. +// The function respects https://tools.ietf.org/html/rfc4514 +func ParseDN(str string) (*DN, error) { + dn := new(DN) + dn.RDNs = make([]*RelativeDN, 0) + rdn := new(RelativeDN) + rdn.Attributes = make([]*AttributeTypeAndValue, 0) + buffer := bytes.Buffer{} + attribute := new(AttributeTypeAndValue) + escaping := false + + unescapedTrailingSpaces := 0 + stringFromBuffer := func() string { + s := buffer.String() + s = s[0 : len(s)-unescapedTrailingSpaces] + buffer.Reset() + unescapedTrailingSpaces = 0 + return s + } + + for i := 0; i < len(str); i++ { + char := str[i] + switch { + case escaping: + unescapedTrailingSpaces = 0 + escaping = false + switch char { + case ' ', '"', '#', '+', ',', ';', '<', '=', '>', '\\': + buffer.WriteByte(char) + continue + } + // Not a special character, assume hex encoded octet + if len(str) == i+1 { + return nil, errors.New("got corrupted escaped character") + } + + dst := []byte{0} + n, err := enchex.Decode([]byte(dst), []byte(str[i:i+2])) + if err != nil { + return nil, fmt.Errorf("failed to decode escaped character: %s", err) + } else if n != 1 { + return nil, fmt.Errorf("expected 1 byte when un-escaping, got %d", n) + } + buffer.WriteByte(dst[0]) + i++ + case char == '\\': + unescapedTrailingSpaces = 0 + escaping = true + case char == '=': + attribute.Type = stringFromBuffer() + // Special case: If the first character in the value is # the following data + // is BER encoded. Throw an error since not supported right now. + if len(str) > i+1 && str[i+1] == '#' { + return nil, errors.New("unsupported BER encoding") + } + case char == ',' || char == '+': + // We're done with this RDN or value, push it + if len(attribute.Type) == 0 { + return nil, errors.New("incomplete type, value pair") + } + attribute.Value = stringFromBuffer() + rdn.Attributes = append(rdn.Attributes, attribute) + attribute = new(AttributeTypeAndValue) + if char == ',' { + dn.RDNs = append(dn.RDNs, rdn) + rdn = new(RelativeDN) + rdn.Attributes = make([]*AttributeTypeAndValue, 0) + } + case char == ' ' && buffer.Len() == 0: + // ignore unescaped leading spaces + continue + default: + if char == ' ' { + // Track unescaped spaces in case they are trailing and we need to remove them + unescapedTrailingSpaces++ + } else { + // Reset if we see a non-space char + unescapedTrailingSpaces = 0 + } + buffer.WriteByte(char) + } + } + if buffer.Len() > 0 { + if len(attribute.Type) == 0 { + return nil, errors.New("DN ended with incomplete type, value pair") + } + attribute.Value = stringFromBuffer() + rdn.Attributes = append(rdn.Attributes, attribute) + dn.RDNs = append(dn.RDNs, rdn) + } + return dn, nil +} + +// Equal returns true if the DNs are equal as defined by rfc4517 4.2.15 (distinguishedNameMatch). +// Returns true if they have the same number of relative distinguished names +// and corresponding relative distinguished names (by position) are the same. +func (d *DN) Equal(other *DN) bool { + if len(d.RDNs) != len(other.RDNs) { + return false + } + for i := range d.RDNs { + if !d.RDNs[i].Equal(other.RDNs[i]) { + return false + } + } + return true +} + +// RDNsMatch returns true if the individual RDNs of the DNs +// are the same regardless of ordering. +func (d *DN) RDNsMatch(other *DN) bool { + if len(d.RDNs) != len(other.RDNs) { + return false + } + matched := make([]bool, len(other.RDNs)) + for _, irdn := range d.RDNs { + found := false + for j, ordn := range other.RDNs { + if !matched[j] && irdn.Equal(ordn) { + matched[j] = true + found = true + break + } + } + if !found { + return false + } + } + return true +} + +// AncestorOf returns true if the other DN consists of at least one RDN followed by all the RDNs of the current DN. +// "ou=widgets,o=acme.com" is an ancestor of "ou=sprockets,ou=widgets,o=acme.com" +// "ou=widgets,o=acme.com" is not an ancestor of "ou=sprockets,ou=widgets,o=foo.com" +// "ou=widgets,o=acme.com" is not an ancestor of "ou=widgets,o=acme.com" +func (d *DN) AncestorOf(other *DN) bool { + if len(d.RDNs) >= len(other.RDNs) { + return false + } + // Take the last `len(d.RDNs)` RDNs from the other DN to compare against + otherRDNs := other.RDNs[len(other.RDNs)-len(d.RDNs):] + for i := range d.RDNs { + if !d.RDNs[i].Equal(otherRDNs[i]) { + return false + } + } + return true +} + +// Equal returns true if the RelativeDNs are equal as defined by rfc4517 4.2.15 (distinguishedNameMatch). +// Relative distinguished names are the same if and only if they have the same number of AttributeTypeAndValues +// and each attribute of the first RDN is the same as the attribute of the second RDN with the same attribute type. +// The order of attributes is not significant. +// Case of attribute types is not significant. +func (r *RelativeDN) Equal(other *RelativeDN) bool { + if len(r.Attributes) != len(other.Attributes) { + return false + } + return r.hasAllAttributes(other.Attributes) && other.hasAllAttributes(r.Attributes) +} + +func (r *RelativeDN) hasAllAttributes(attrs []*AttributeTypeAndValue) bool { + for _, attr := range attrs { + found := false + for _, myattr := range r.Attributes { + if myattr.Equal(attr) { + found = true + break + } + } + if !found { + return false + } + } + return true +} + +// Equal returns true if the AttributeTypeAndValue is equivalent to the specified AttributeTypeAndValue +// Case of the attribute type is not significant +func (a *AttributeTypeAndValue) Equal(other *AttributeTypeAndValue) bool { + return strings.EqualFold(a.Type, other.Type) && a.Value == other.Value +} diff --git a/internal/ldap/dn_test.go b/internal/ldap/dn_test.go new file mode 100644 index 0000000..6d7baa3 --- /dev/null +++ b/internal/ldap/dn_test.go @@ -0,0 +1,225 @@ +// Copyright (c) 2011-2015 Michael Mitton (mmitton@gmail.com) +// Portions copyright (c) 2015-2016 go-ldap Authors +// Static-Check Fixes Copyright 2024 The NATS Authors + +package ldap + +import ( + "reflect" + "testing" +) + +func TestSuccessfulDNParsing(t *testing.T) { + testcases := map[string]DN{ + "": {[]*RelativeDN{}}, + "cn=Jim\\2C \\22Hasse Hö\\22 Hansson!,dc=dummy,dc=com": {[]*RelativeDN{ + {[]*AttributeTypeAndValue{{"cn", "Jim, \"Hasse Hö\" Hansson!"}}}, + {[]*AttributeTypeAndValue{{"dc", "dummy"}}}, + {[]*AttributeTypeAndValue{{"dc", "com"}}}}}, + "UID=jsmith,DC=example,DC=net": {[]*RelativeDN{ + {[]*AttributeTypeAndValue{{"UID", "jsmith"}}}, + {[]*AttributeTypeAndValue{{"DC", "example"}}}, + {[]*AttributeTypeAndValue{{"DC", "net"}}}}}, + "OU=Sales+CN=J. Smith,DC=example,DC=net": {[]*RelativeDN{ + {[]*AttributeTypeAndValue{ + {"OU", "Sales"}, + {"CN", "J. Smith"}}}, + {[]*AttributeTypeAndValue{{"DC", "example"}}}, + {[]*AttributeTypeAndValue{{"DC", "net"}}}}}, + // + // "1.3.6.1.4.1.1466.0=#04024869": {[]*RelativeDN{ + // {[]*AttributeTypeAndValue{{"1.3.6.1.4.1.1466.0", "Hi"}}}}}, + // "1.3.6.1.4.1.1466.0=#04024869,DC=net": {[]*RelativeDN{ + // {[]*AttributeTypeAndValue{{"1.3.6.1.4.1.1466.0", "Hi"}}}, + // {[]*AttributeTypeAndValue{{"DC", "net"}}}}}, + "CN=Lu\\C4\\8Di\\C4\\87": {[]*RelativeDN{ + {[]*AttributeTypeAndValue{{"CN", "Lučić"}}}}}, + " CN = Lu\\C4\\8Di\\C4\\87 ": {[]*RelativeDN{ + {[]*AttributeTypeAndValue{{"CN", "Lučić"}}}}}, + ` A = 1 , B = 2 `: {[]*RelativeDN{ + {[]*AttributeTypeAndValue{{"A", "1"}}}, + {[]*AttributeTypeAndValue{{"B", "2"}}}}}, + ` A = 1 + B = 2 `: {[]*RelativeDN{ + {[]*AttributeTypeAndValue{ + {"A", "1"}, + {"B", "2"}}}}}, + ` \ \ A\ \ = \ \ 1\ \ , \ \ B\ \ = \ \ 2\ \ `: {[]*RelativeDN{ + {[]*AttributeTypeAndValue{{" A ", " 1 "}}}, + {[]*AttributeTypeAndValue{{" B ", " 2 "}}}}}, + ` \ \ A\ \ = \ \ 1\ \ + \ \ B\ \ = \ \ 2\ \ `: {[]*RelativeDN{ + {[]*AttributeTypeAndValue{ + {" A ", " 1 "}, + {" B ", " 2 "}}}}}, + } + + for test, answer := range testcases { + dn, err := ParseDN(test) + if err != nil { + t.Error(err.Error()) + continue + } + if !reflect.DeepEqual(dn, &answer) { + t.Errorf("Parsed DN %s is not equal to the expected structure", test) + t.Logf("Expected:") + for _, rdn := range answer.RDNs { + for _, attribs := range rdn.Attributes { + t.Logf("#%v\n", attribs) + } + } + t.Logf("Actual:") + for _, rdn := range dn.RDNs { + for _, attribs := range rdn.Attributes { + t.Logf("#%v\n", attribs) + } + } + } + } +} + +func TestErrorDNParsing(t *testing.T) { + testcases := map[string]string{ + "*": "DN ended with incomplete type, value pair", + "cn=Jim\\0Test": "failed to decode escaped character: encoding/hex: invalid byte: U+0054 'T'", + "cn=Jim\\0": "got corrupted escaped character", + "DC=example,=net": "DN ended with incomplete type, value pair", + // "1=#0402486": "failed to decode BER encoding: encoding/hex: odd length hex string", + "test,DC=example,DC=com": "incomplete type, value pair", + "=test,DC=example,DC=com": "incomplete type, value pair", + } + + for test, answer := range testcases { + _, err := ParseDN(test) + if err == nil { + t.Errorf("Expected %s to fail parsing but succeeded\n", test) + } else if err.Error() != answer { + t.Errorf("Unexpected error on %s:\n%s\nvs.\n%s\n", test, answer, err.Error()) + } + } +} + +func TestDNEqual(t *testing.T) { + testcases := []struct { + A string + B string + Equal bool + }{ + // Exact match + {"", "", true}, + {"o=A", "o=A", true}, + {"o=A", "o=B", false}, + + {"o=A,o=B", "o=A,o=B", true}, + {"o=A,o=B", "o=A,o=C", false}, + + {"o=A+o=B", "o=A+o=B", true}, + {"o=A+o=B", "o=A+o=C", false}, + + // Case mismatch in type is ignored + {"o=A", "O=A", true}, + {"o=A,o=B", "o=A,O=B", true}, + {"o=A+o=B", "o=A+O=B", true}, + + // Case mismatch in value is significant + {"o=a", "O=A", false}, + {"o=a,o=B", "o=A,O=B", false}, + {"o=a+o=B", "o=A+O=B", false}, + + // Multi-valued RDN order mismatch is ignored + {"o=A+o=B", "O=B+o=A", true}, + // Number of RDN attributes is significant + {"o=A+o=B", "O=B+o=A+O=B", false}, + + // Missing values are significant + {"o=A+o=B", "O=B+o=A+O=C", false}, // missing values matter + {"o=A+o=B+o=C", "O=B+o=A", false}, // missing values matter + + // Whitespace tests + // Matching + { + "cn=John Doe, ou=People, dc=sun.com", + "cn=John Doe, ou=People, dc=sun.com", + true, + }, + // Difference in leading/trailing chars is ignored + { + "cn=John Doe, ou=People, dc=sun.com", + "cn=John Doe,ou=People,dc=sun.com", + true, + }, + // Difference in values is significant + { + "cn=John Doe, ou=People, dc=sun.com", + "cn=John Doe, ou=People, dc=sun.com", + false, + }, + } + + for i, tc := range testcases { + a, err := ParseDN(tc.A) + if err != nil { + t.Errorf("%d: %v", i, err) + continue + } + b, err := ParseDN(tc.B) + if err != nil { + t.Errorf("%d: %v", i, err) + continue + } + if expected, actual := tc.Equal, a.Equal(b); expected != actual { + t.Errorf("%d: when comparing '%s' and '%s' expected %v, got %v", i, tc.A, tc.B, expected, actual) + continue + } + if expected, actual := tc.Equal, b.Equal(a); expected != actual { + t.Errorf("%d: when comparing '%s' and '%s' expected %v, got %v", i, tc.A, tc.B, expected, actual) + continue + } + } +} + +func TestDNAncestor(t *testing.T) { + testcases := []struct { + A string + B string + Ancestor bool + }{ + // Exact match returns false + {"", "", false}, + {"o=A", "o=A", false}, + {"o=A,o=B", "o=A,o=B", false}, + {"o=A+o=B", "o=A+o=B", false}, + + // Mismatch + {"ou=C,ou=B,o=A", "ou=E,ou=D,ou=B,o=A", false}, + + // Descendant + {"ou=C,ou=B,o=A", "ou=E,ou=C,ou=B,o=A", true}, + } + + for i, tc := range testcases { + a, err := ParseDN(tc.A) + if err != nil { + t.Errorf("%d: %v", i, err) + continue + } + b, err := ParseDN(tc.B) + if err != nil { + t.Errorf("%d: %v", i, err) + continue + } + if expected, actual := tc.Ancestor, a.AncestorOf(b); expected != actual { + t.Errorf("%d: when comparing '%s' and '%s' expected %v, got %v", i, tc.A, tc.B, expected, actual) + continue + } + } +} + +func TestDNReverseMatchDuplicates(t *testing.T) { + // A has two CN=Alice RDNs and one O=NATS. + // B has one CN=Alice, one O=NATS, and one OU=Users. + // They have the same length (3). + dnA, _ := ParseDN("CN=Alice,CN=Alice,O=NATS") + dnB, _ := ParseDN("CN=Alice,O=NATS,OU=Users") + if dnA.RDNsMatch(dnB) { + t.Fatal("Expected RDNsMatch to return false for different RDN sets, but it returned true") + } +} diff --git a/internal/ocsp/ocsp.go b/internal/ocsp/ocsp.go new file mode 100644 index 0000000..f317c1f --- /dev/null +++ b/internal/ocsp/ocsp.go @@ -0,0 +1,276 @@ +// Copyright 2019-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testhelper + +import ( + "crypto" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" + "sync" + "testing" + "time" + + "golang.org/x/crypto/ocsp" +) + +const ( + defaultResponseTTL = 4 * time.Second + defaultAddress = "127.0.0.1:8888" +) + +func NewOCSPResponderCustomAddress(t *testing.T, issuerCertPEM, issuerKeyPEM string, addr string) *http.Server { + t.Helper() + return NewOCSPResponderBase(t, issuerCertPEM, issuerCertPEM, issuerKeyPEM, false, addr, defaultResponseTTL, "") +} + +func NewOCSPResponder(t *testing.T, issuerCertPEM, issuerKeyPEM string) *http.Server { + t.Helper() + return NewOCSPResponderBase(t, issuerCertPEM, issuerCertPEM, issuerKeyPEM, false, defaultAddress, defaultResponseTTL, "") +} + +func NewOCSPResponderDesignatedCustomAddress(t *testing.T, issuerCertPEM, respCertPEM, respKeyPEM string, addr string) *http.Server { + t.Helper() + return NewOCSPResponderBase(t, issuerCertPEM, respCertPEM, respKeyPEM, true, addr, defaultResponseTTL, "") +} + +func NewOCSPResponderPreferringHTTPMethod(t *testing.T, issuerCertPEM, issuerKeyPEM, method string) *http.Server { + t.Helper() + return NewOCSPResponderBase(t, issuerCertPEM, issuerCertPEM, issuerKeyPEM, false, defaultAddress, defaultResponseTTL, method) +} + +func NewOCSPResponderCustomTimeout(t *testing.T, issuerCertPEM, issuerKeyPEM string, responseTTL time.Duration) *http.Server { + t.Helper() + return NewOCSPResponderBase(t, issuerCertPEM, issuerCertPEM, issuerKeyPEM, false, defaultAddress, responseTTL, "") +} + +func NewOCSPResponderBase(t *testing.T, issuerCertPEM, respCertPEM, respKeyPEM string, embed bool, addr string, responseTTL time.Duration, method string) *http.Server { + t.Helper() + var mu sync.Mutex + status := make(map[string]int) + + issuerCert := parseCertPEM(t, issuerCertPEM) + respCert := parseCertPEM(t, respCertPEM) + respKey := parseKeyPEM(t, respKeyPEM) + + mux := http.NewServeMux() + // The "/statuses/" endpoint is for directly setting a key-value pair in + // the CA's status database. + mux.HandleFunc("/statuses/", func(rw http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + + key := r.URL.Path[len("/statuses/"):] + switch r.Method { + case "GET": + mu.Lock() + n, ok := status[key] + if !ok { + n = ocsp.Unknown + } + mu.Unlock() + + fmt.Fprintf(rw, "%s %d", key, n) + case "POST": + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + + n, err := strconv.Atoi(string(data)) + if err != nil { + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + + mu.Lock() + status[key] = n + mu.Unlock() + + fmt.Fprintf(rw, "%s %d", key, n) + default: + http.Error(rw, "Method Not Allowed", http.StatusMethodNotAllowed) + return + } + }) + // The "/" endpoint is for normal OCSP requests. This actually parses an + // OCSP status request and signs a response with a CA. Lightly based off: + // https://www.ietf.org/rfc/rfc2560.txt + mux.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) { + var reqData []byte + var err error + + switch { + case r.Method == "GET": + if method != "" && r.Method != method { + http.Error(rw, "", http.StatusBadRequest) + return + } + reqData, err = base64.StdEncoding.DecodeString(r.URL.Path[1:]) + case r.Method == "POST": + if method != "" && r.Method != method { + http.Error(rw, "", http.StatusBadRequest) + return + } + reqData, err = io.ReadAll(r.Body) + r.Body.Close() + default: + http.Error(rw, "Method Not Allowed", http.StatusMethodNotAllowed) + return + } + if err != nil { + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + + ocspReq, err := ocsp.ParseRequest(reqData) + if err != nil { + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + + mu.Lock() + n, ok := status[ocspReq.SerialNumber.String()] + if !ok { + n = ocsp.Unknown + } + mu.Unlock() + + tmpl := ocsp.Response{ + Status: n, + SerialNumber: ocspReq.SerialNumber, + ThisUpdate: time.Now(), + } + if responseTTL != 0 { + tmpl.NextUpdate = tmpl.ThisUpdate.Add(responseTTL) + } + if embed { + tmpl.Certificate = respCert + } + respData, err := ocsp.CreateResponse(issuerCert, respCert, tmpl, respKey) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + + rw.Header().Set("Content-Type", "application/ocsp-response") + rw.Header().Set("Content-Length", fmt.Sprint(len(respData))) + + fmt.Fprint(rw, string(respData)) + }) + + srv := &http.Server{ + Addr: addr, + Handler: mux, + ReadTimeout: time.Second * 5, + } + go srv.ListenAndServe() + time.Sleep(1 * time.Second) + return srv +} + +func parseCertPEM(t *testing.T, certPEM string) *x509.Certificate { + t.Helper() + block := parsePEM(t, certPEM) + + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("failed to parse cert '%s': %s", certPEM, err) + } + return cert +} + +func parseKeyPEM(t *testing.T, keyPEM string) crypto.Signer { + t.Helper() + block := parsePEM(t, keyPEM) + + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + key, err = x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + t.Fatalf("failed to parse ikey %s: %s", keyPEM, err) + } + } + keyc := key.(crypto.Signer) + return keyc +} + +func parsePEM(t *testing.T, pemPath string) *pem.Block { + t.Helper() + data, err := os.ReadFile(pemPath) + if err != nil { + t.Fatal(err) + } + + block, _ := pem.Decode(data) + if block == nil { + t.Fatalf("failed to decode PEM %s", pemPath) + } + return block +} + +func GetOCSPStatus(s tls.ConnectionState) (*ocsp.Response, error) { + if len(s.VerifiedChains) == 0 { + return nil, fmt.Errorf("missing TLS verified chains") + } + chain := s.VerifiedChains[0] + + if got, want := len(chain), 2; got < want { + return nil, fmt.Errorf("incomplete cert chain, got %d, want at least %d", got, want) + } + leaf, issuer := chain[0], chain[1] + + resp, err := ocsp.ParseResponseForCert(s.OCSPResponse, leaf, issuer) + if err != nil { + return nil, fmt.Errorf("failed to parse OCSP response: %w", err) + } + if err := resp.CheckSignatureFrom(issuer); err != nil { + return resp, err + } + return resp, nil +} + +func SetOCSPStatus(t *testing.T, ocspURL, certPEM string, status int) { + t.Helper() + + cert := parseCertPEM(t, certPEM) + + hc := &http.Client{Timeout: 10 * time.Second} + resp, err := hc.Post( + fmt.Sprintf("%s/statuses/%s", ocspURL, cert.SerialNumber), + "", + strings.NewReader(fmt.Sprint(status)), + ) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("failed to read OCSP HTTP response body: %s", err) + } + + if got, want := resp.Status, "200 OK"; got != want { + t.Error(strings.TrimSpace(string(data))) + t.Fatalf("unexpected OCSP HTTP set status, got %q, want %q", got, want) + } +} diff --git a/internal/testhelper/logging.go b/internal/testhelper/logging.go new file mode 100644 index 0000000..5fb7084 --- /dev/null +++ b/internal/testhelper/logging.go @@ -0,0 +1,127 @@ +// Copyright 2019-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testhelper + +// These routines need to be accessible in both the server and test +// directories, and tests importing a package don't get exported symbols from +// _test.go files in the imported package, so we put them here where they can +// be used freely. + +import ( + "fmt" + "strings" + "sync" + "testing" +) + +type DummyLogger struct { + sync.Mutex + Msg string + AllMsgs []string +} + +func (l *DummyLogger) CheckContent(t *testing.T, expectedStr string) { + t.Helper() + l.Lock() + defer l.Unlock() + if l.Msg != expectedStr { + t.Fatalf("Expected log to be: %v, got %v", expectedStr, l.Msg) + } +} + +func (l *DummyLogger) aggregate() { + if l.AllMsgs != nil { + l.AllMsgs = append(l.AllMsgs, l.Msg) + } +} + +func (l *DummyLogger) Noticef(format string, v ...any) { + l.Lock() + defer l.Unlock() + l.Msg = fmt.Sprintf(format, v...) + l.aggregate() +} +func (l *DummyLogger) Errorf(format string, v ...any) { + l.Lock() + defer l.Unlock() + l.Msg = fmt.Sprintf(format, v...) + l.aggregate() +} +func (l *DummyLogger) Warnf(format string, v ...any) { + l.Lock() + defer l.Unlock() + l.Msg = fmt.Sprintf(format, v...) + l.aggregate() +} +func (l *DummyLogger) Fatalf(format string, v ...any) { + l.Lock() + defer l.Unlock() + l.Msg = fmt.Sprintf(format, v...) + l.aggregate() +} +func (l *DummyLogger) Debugf(format string, v ...any) { + l.Lock() + defer l.Unlock() + l.Msg = fmt.Sprintf(format, v...) + l.aggregate() +} +func (l *DummyLogger) Tracef(format string, v ...any) { + l.Lock() + defer l.Unlock() + l.Msg = fmt.Sprintf(format, v...) + l.aggregate() +} + +// NewDummyLogger creates a dummy logger and allows to ask for logs to be +// retained instead of just keeping the most recent. Use retain to provide an +// initial size estimate on messages (not to provide a max capacity). +func NewDummyLogger(retain uint) *DummyLogger { + l := &DummyLogger{} + if retain > 0 { + l.AllMsgs = make([]string, 0, retain) + } + return l +} + +func (l *DummyLogger) Drain() { + l.Lock() + defer l.Unlock() + if l.AllMsgs == nil { + return + } + l.AllMsgs = make([]string, 0, len(l.AllMsgs)) +} + +func (l *DummyLogger) CheckForProhibited(t *testing.T, reason, needle string) { + t.Helper() + l.Lock() + defer l.Unlock() + + if l.AllMsgs == nil { + t.Fatal("DummyLogger.CheckForProhibited called without AllMsgs being collected") + } + + // Collect _all_ matches, rather than have to re-test repeatedly. + // This will particularly help with less deterministic tests with multiple matches. + shouldFail := false + for i := range l.AllMsgs { + if strings.Contains(l.AllMsgs[i], needle) { + t.Errorf("log contains %s: %v", reason, l.AllMsgs[i]) + shouldFail = true + } + } + if shouldFail { + t.FailNow() + } +} diff --git a/locksordering.txt b/locksordering.txt new file mode 100644 index 0000000..43ca843 --- /dev/null +++ b/locksordering.txt @@ -0,0 +1,67 @@ +Here is the list of some established lock ordering. + +In this list, A -> B means that you can have A.Lock() then B.Lock(), not the opposite. + +jetStream -> jsAccount -> Server -> client -> Account + +jetStream -> jsAccount -> stream -> consumer + +The "prMu" lock protects the peer reconcile signaling state on jetStream, so +signaling is cheap and never contends on the JS lock. It must not be held while +acquiring the JetStream lock. + + jetStream -> prMu + +A lock to protect jetstream account's usage has been introduced: jsAccount.usageMu. +This lock is independent and can be invoked under any other lock: jsAccount -> jsa.usageMu, stream -> jsa.usageMu, etc... + +A lock to protect the account's leafnodes list was also introduced to +allow that lock to be held and the acquire a client lock which is not +possible with the normal account lock. + +accountLeafList -> client + +AccountResolver interface has various implementations, but assume: AccountResolver -> Server + +A reloadMu lock was added to prevent newly connecting clients racing with the configuration reload. +This must be taken out as soon as a reload is about to happen before any other locks: + + reloadMu -> Server + reloadMu -> optsMu + +The "jscmMu" lock in the Account is used to serialise calls to checkJetStreamMigrate and +clearObserverState so that they cannot interleave which would leave Raft nodes in +inconsistent observer states. + + jscmMu -> Account -> jsAccount + jscmMu -> stream.clsMu + jscmMu -> RaftNode + +The "clsMu" lock protects the consumer list on a stream, used for signalling consumer activity. + + stream -> clsMu + +The "clMu" and "ddMu" locks protect clustered and dedupe state respectively. +The stream lock (`mset.mu`) is optional, but if holding "clMu" or "ddMu", +locking the stream lock afterward would violate locking order. + + stream -> clMu + stream -> clMu -> ddMu + +The "mset.batches.mu" lock protects the batching state without needing to hold the stream lock. +If "clMu" is used to commit a batch, it should only be acquired while already holding the batch lock. + + stream -> mset.batches.mu -> clMu + +The "isolateMu" lock isolates direct get and msg get reads from writes, so those reads do not +observe partially applied rollups or atomic batches. Other read paths (consumer delivery, +mirrors/sources, info requests, etc.) are sequentially consistent and may observe a prefix of +an inflight batch. It must be acquired before the stream lock, never while holding it. + + isolateMu -> stream + +The "eventIdsMu" lock protects the server's event ID generator. It may be acquired +while holding the Server lock, but the Server lock must not be acquired while +holding eventIdsMu. + + Server -> eventIdsMu diff --git a/logger/log.go b/logger/log.go new file mode 100644 index 0000000..83889fb --- /dev/null +++ b/logger/log.go @@ -0,0 +1,405 @@ +// Copyright 2012-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package logger provides logging facilities for the NATS server +package logger + +import ( + "fmt" + "log" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" +) + +// Default file permissions for log files. +const defaultLogPerms = os.FileMode(0640) + +// Logger is the server logger +type Logger struct { + sync.Mutex + logger *log.Logger + debug bool + trace bool + infoLabel string + warnLabel string + errorLabel string + fatalLabel string + debugLabel string + traceLabel string + fl *fileLogger +} + +type LogOption interface { + isLoggerOption() +} + +// LogUTC controls whether timestamps in the log output should be UTC or local time. +type LogUTC bool + +func (l LogUTC) isLoggerOption() {} + +func logFlags(time bool, opts ...LogOption) int { + flags := 0 + if time { + flags = log.LstdFlags | log.Lmicroseconds + } + + for _, opt := range opts { + switch v := opt.(type) { + case LogUTC: + if time && bool(v) { + flags |= log.LUTC + } + } + } + + return flags +} + +// NewStdLogger creates a logger with output directed to Stderr +func NewStdLogger(time, debug, trace, colors, pid bool, opts ...LogOption) *Logger { + flags := logFlags(time, opts...) + + pre := "" + if pid { + pre = pidPrefix() + } + + l := &Logger{ + logger: log.New(os.Stderr, pre, flags), + debug: debug, + trace: trace, + } + + if colors { + setColoredLabelFormats(l) + } else { + setPlainLabelFormats(l) + } + + return l +} + +// NewFileLogger creates a logger with output directed to a file +func NewFileLogger(filename string, time, debug, trace, pid bool, opts ...LogOption) *Logger { + flags := logFlags(time, opts...) + + pre := "" + if pid { + pre = pidPrefix() + } + + fl, err := newFileLogger(filename, pre, time) + if err != nil { + log.Fatalf("error opening file: %v", err) + return nil + } + + l := &Logger{ + logger: log.New(fl, pre, flags), + debug: debug, + trace: trace, + fl: fl, + } + fl.Lock() + fl.l = l + fl.Unlock() + + setPlainLabelFormats(l) + return l +} + +type writerAndCloser interface { + Write(b []byte) (int, error) + Close() error + Name() string +} + +type fileLogger struct { + out int64 + canRotate int32 + sync.Mutex + l *Logger + f writerAndCloser + limit int64 + olimit int64 + pid string + time bool + closed bool + maxNumFiles int +} + +func newFileLogger(filename, pidPrefix string, time bool) (*fileLogger, error) { + fileflags := os.O_WRONLY | os.O_APPEND | os.O_CREATE + f, err := os.OpenFile(filename, fileflags, defaultLogPerms) + if err != nil { + return nil, err + } + stats, err := f.Stat() + if err != nil { + f.Close() + return nil, err + } + fl := &fileLogger{ + canRotate: 0, + f: f, + out: stats.Size(), + pid: pidPrefix, + time: time, + } + return fl, nil +} + +func (l *fileLogger) setLimit(limit int64) { + l.Lock() + l.olimit, l.limit = limit, limit + atomic.StoreInt32(&l.canRotate, 1) + rotateNow := l.out > l.limit + l.Unlock() + if rotateNow { + l.l.Noticef("Rotating logfile...") + } +} + +func (l *fileLogger) setMaxNumFiles(max int) { + l.Lock() + l.maxNumFiles = max + l.Unlock() +} + +func (l *fileLogger) logDirect(label, format string, v ...any) int { + var entrya = [256]byte{} + var entry = entrya[:0] + if l.pid != "" { + entry = append(entry, l.pid...) + } + if l.time { + now := time.Now() + year, month, day := now.Date() + hour, min, sec := now.Clock() + microsec := now.Nanosecond() / 1000 + entry = append(entry, fmt.Sprintf("%04d/%02d/%02d %02d:%02d:%02d.%06d ", + year, month, day, hour, min, sec, microsec)...) + } + entry = append(entry, label...) + entry = append(entry, fmt.Sprintf(format, v...)...) + entry = append(entry, '\r', '\n') + l.f.Write(entry) + return len(entry) +} + +func (l *fileLogger) logPurge(fname string) { + var backups []string + lDir := filepath.Dir(fname) + lBase := filepath.Base(fname) + entries, err := os.ReadDir(lDir) + if err != nil { + l.logDirect(l.l.errorLabel, "Unable to read directory %q for log purge (%v), will attempt next rotation", lDir, err) + return + } + for _, entry := range entries { + if entry.IsDir() || entry.Name() == lBase || !strings.HasPrefix(entry.Name(), lBase) { + continue + } + if stamp, found := strings.CutPrefix(entry.Name(), fmt.Sprintf("%s%s", lBase, ".")); found { + _, err := time.Parse("2006:01:02:15:04:05.999999999", strings.Replace(stamp, ".", ":", 5)) + if err == nil { + backups = append(backups, entry.Name()) + } + } + } + currBackups := len(backups) + maxBackups := l.maxNumFiles - 1 + if currBackups > maxBackups { + // backups sorted oldest to latest based on timestamped lexical filename (ReadDir) + for i := 0; i < currBackups-maxBackups; i++ { + if err := os.Remove(filepath.Join(lDir, string(os.PathSeparator), backups[i])); err != nil { + l.logDirect(l.l.errorLabel, "Unable to remove backup log file %q (%v), will attempt next rotation", backups[i], err) + // Bail fast, we'll try again next rotation + return + } + l.logDirect(l.l.infoLabel, "Purged log file %q", backups[i]) + } + } +} + +func (l *fileLogger) Write(b []byte) (int, error) { + if atomic.LoadInt32(&l.canRotate) == 0 { + n, err := l.f.Write(b) + if err == nil { + atomic.AddInt64(&l.out, int64(n)) + } + return n, err + } + l.Lock() + n, err := l.f.Write(b) + if err == nil { + l.out += int64(n) + if l.out > l.limit { + if err := l.f.Close(); err != nil { + l.limit *= 2 + l.logDirect(l.l.errorLabel, "Unable to close logfile for rotation (%v), will attempt next rotation at size %v", err, l.limit) + l.Unlock() + return n, err + } + fname := l.f.Name() + now := time.Now() + bak := fmt.Sprintf("%s.%04d.%02d.%02d.%02d.%02d.%02d.%09d", fname, + now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), + now.Second(), now.Nanosecond()) + os.Rename(fname, bak) + fileflags := os.O_WRONLY | os.O_APPEND | os.O_CREATE + f, err := os.OpenFile(fname, fileflags, defaultLogPerms) + if err != nil { + l.Unlock() + panic(fmt.Sprintf("Unable to re-open the logfile %q after rotation: %v", fname, err)) + } + l.f = f + n := l.logDirect(l.l.infoLabel, "Rotated log, backup saved as %q", bak) + l.out = int64(n) + l.limit = l.olimit + if l.maxNumFiles > 0 { + l.logPurge(fname) + } + } + } + l.Unlock() + return n, err +} + +func (l *fileLogger) close() error { + l.Lock() + if l.closed { + l.Unlock() + return nil + } + l.closed = true + l.Unlock() + return l.f.Close() +} + +// SetSizeLimit sets the size of a logfile after which a backup +// is created with the file name + "year.month.day.hour.min.sec.nanosec" +// and the current log is truncated. +func (l *Logger) SetSizeLimit(limit int64) error { + l.Lock() + if l.fl == nil { + l.Unlock() + return fmt.Errorf("can set log size limit only for file logger") + } + fl := l.fl + l.Unlock() + fl.setLimit(limit) + return nil +} + +// SetMaxNumFiles sets the number of archived log files that will be retained +func (l *Logger) SetMaxNumFiles(max int) error { + l.Lock() + if l.fl == nil { + l.Unlock() + return fmt.Errorf("can set log max number of files only for file logger") + } + fl := l.fl + l.Unlock() + fl.setMaxNumFiles(max) + return nil +} + +// NewTestLogger creates a logger with output directed to Stderr with a prefix. +// Useful for tracing in tests when multiple servers are in the same pid +func NewTestLogger(prefix string, time bool) *Logger { + flags := 0 + if time { + flags = log.LstdFlags | log.Lmicroseconds + } + l := &Logger{ + logger: log.New(os.Stderr, prefix, flags), + debug: true, + trace: true, + } + setColoredLabelFormats(l) + return l +} + +// Close implements the io.Closer interface to clean up +// resources in the server's logger implementation. +// Caller must ensure threadsafety. +func (l *Logger) Close() error { + if l.fl != nil { + return l.fl.close() + } + return nil +} + +// Generate the pid prefix string +func pidPrefix() string { + return fmt.Sprintf("[%d] ", os.Getpid()) +} + +func setPlainLabelFormats(l *Logger) { + l.infoLabel = "[INF] " + l.debugLabel = "[DBG] " + l.warnLabel = "[WRN] " + l.errorLabel = "[ERR] " + l.fatalLabel = "[FTL] " + l.traceLabel = "[TRC] " +} + +func setColoredLabelFormats(l *Logger) { + colorFormat := "[\x1b[%sm%s\x1b[0m] " + l.infoLabel = fmt.Sprintf(colorFormat, "32", "INF") + l.debugLabel = fmt.Sprintf(colorFormat, "36", "DBG") + l.warnLabel = fmt.Sprintf(colorFormat, "0;93", "WRN") + l.errorLabel = fmt.Sprintf(colorFormat, "31", "ERR") + l.fatalLabel = fmt.Sprintf(colorFormat, "31", "FTL") + l.traceLabel = fmt.Sprintf(colorFormat, "33", "TRC") +} + +// Noticef logs a notice statement +func (l *Logger) Noticef(format string, v ...any) { + l.logger.Printf(l.infoLabel+format, v...) +} + +// Warnf logs a notice statement +func (l *Logger) Warnf(format string, v ...any) { + l.logger.Printf(l.warnLabel+format, v...) +} + +// Errorf logs an error statement +func (l *Logger) Errorf(format string, v ...any) { + l.logger.Printf(l.errorLabel+format, v...) +} + +// Fatalf logs a fatal error +func (l *Logger) Fatalf(format string, v ...any) { + l.logger.Fatalf(l.fatalLabel+format, v...) +} + +// Debugf logs a debug statement +func (l *Logger) Debugf(format string, v ...any) { + if l.debug { + l.logger.Printf(l.debugLabel+format, v...) + } +} + +// Tracef logs a trace statement +func (l *Logger) Tracef(format string, v ...any) { + if l.trace { + l.logger.Printf(l.traceLabel+format, v...) + } +} diff --git a/logger/log_test.go b/logger/log_test.go new file mode 100644 index 0000000..89a0dd6 --- /dev/null +++ b/logger/log_test.go @@ -0,0 +1,327 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logger + +import ( + "bytes" + "fmt" + "io" + "log" + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +func TestStdLogger(t *testing.T) { + logger := NewStdLogger(false, false, false, false, false) + + flags := logger.logger.Flags() + if flags != 0 { + t.Fatalf("Expected %v, received %v\n", 0, flags) + } + + if logger.debug { + t.Fatalf("Expected %t, received %t\n", false, logger.debug) + } + + if logger.trace { + t.Fatalf("Expected %t, received %t\n", false, logger.trace) + } +} + +func TestStdLoggerWithDebugTraceAndTime(t *testing.T) { + logger := NewStdLogger(true, true, true, false, false) + + flags := logger.logger.Flags() + if flags != log.LstdFlags|log.Lmicroseconds { + t.Fatalf("Expected %d, received %d\n", log.LstdFlags, flags) + } + + if !logger.debug { + t.Fatalf("Expected %t, received %t\n", true, logger.debug) + } + + if !logger.trace { + t.Fatalf("Expected %t, received %t\n", true, logger.trace) + } +} + +func TestStdLoggerNotice(t *testing.T) { + expectOutput(t, func() { + logger := NewStdLogger(false, false, false, false, false) + logger.Noticef("foo") + }, "[INF] foo\n") +} + +func TestStdLoggerNoticeWithColor(t *testing.T) { + expectOutput(t, func() { + logger := NewStdLogger(false, false, false, true, false) + logger.Noticef("foo") + }, "[\x1b[32mINF\x1b[0m] foo\n") +} + +func TestStdLoggerDebug(t *testing.T) { + expectOutput(t, func() { + logger := NewStdLogger(false, true, false, false, false) + logger.Debugf("foo %s", "bar") + }, "[DBG] foo bar\n") +} + +func TestStdLoggerDebugWithOutDebug(t *testing.T) { + expectOutput(t, func() { + logger := NewStdLogger(false, false, false, false, false) + logger.Debugf("foo") + }, "") +} + +func TestStdLoggerTrace(t *testing.T) { + expectOutput(t, func() { + logger := NewStdLogger(false, false, true, false, false) + logger.Tracef("foo") + }, "[TRC] foo\n") +} + +func TestStdLoggerTraceWithOutDebug(t *testing.T) { + expectOutput(t, func() { + logger := NewStdLogger(false, false, false, false, false) + logger.Tracef("foo") + }, "") +} + +func TestFileLogger(t *testing.T) { + tmpDir := t.TempDir() + file := createFileAtDir(t, tmpDir, "nats-server:log_") + file.Close() + + logger := NewFileLogger(file.Name(), false, false, false, false) + defer logger.Close() + logger.Noticef("foo") + + buf, err := os.ReadFile(file.Name()) + if err != nil { + t.Fatalf("Could not read logfile: %v", err) + } + if len(buf) <= 0 { + t.Fatal("Expected a non-zero length logfile") + } + + if string(buf) != "[INF] foo\n" { + t.Fatalf("Expected '%s', received '%s'\n", "[INFO] foo", string(buf)) + } + + file = createFileAtDir(t, tmpDir, "nats-server:log_") + file.Close() + + logger = NewFileLogger(file.Name(), true, false, true, true) + defer logger.Close() + logger.Errorf("foo") + + buf, err = os.ReadFile(file.Name()) + if err != nil { + t.Fatalf("Could not read logfile: %v", err) + } + if len(buf) <= 0 { + t.Fatal("Expected a non-zero length logfile") + } + str := string(buf) + errMsg := fmt.Sprintf("Expected '%s', received '%s'\n", "[pid] [ERR] foo", str) + pidEnd := strings.Index(str, " ") + infoStart := strings.LastIndex(str, "[ERR]") + if pidEnd == -1 || infoStart == -1 { + t.Fatalf("%v", errMsg) + } + pid := str[0:pidEnd] + if pid[0] != '[' || pid[len(pid)-1] != ']' { + t.Fatalf("%v", errMsg) + } + + date := str[pidEnd:infoStart] + dateRegExp := "[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}" + reg, err := regexp.Compile(dateRegExp) + if err != nil { + t.Fatalf("Compile date regexp error: %v", err) + } + if matched := reg.Match([]byte(date)); !matched { + t.Fatalf("Date string '%s' does not match '%s'", date, dateRegExp) + } + + if !strings.HasSuffix(str, "[ERR] foo\n") { + t.Fatalf("%v", errMsg) + } +} + +func TestFileLoggerSizeLimit(t *testing.T) { + // Create std logger + logger := NewStdLogger(true, false, false, false, true) + if err := logger.SetSizeLimit(1000); err == nil || + !strings.Contains(err.Error(), "only for file logger") { + t.Fatalf("Expected error about being able to use only for file logger, got %v", err) + } + logger.Close() + + tmpDir := t.TempDir() + + file := createFileAtDir(t, tmpDir, "log_") + file.Close() + + logger = NewFileLogger(file.Name(), true, false, false, true) + defer logger.Close() + logger.SetSizeLimit(1000) + + for i := 0; i < 50; i++ { + logger.Noticef("This is a line in the log file") + } + + files, err := os.ReadDir(tmpDir) + if err != nil { + t.Fatalf("Error reading logs dir: %v", err) + } + if len(files) == 1 { + t.Fatalf("Expected file to have been rotated") + } + lastBackup := files[len(files)-1] + if err := logger.Close(); err != nil { + t.Fatalf("Error closing log: %v", err) + } + content, err := os.ReadFile(file.Name()) + if err != nil { + t.Fatalf("Error loading latest log: %v", err) + } + if !bytes.Contains(content, []byte("Rotated log")) || + !bytes.Contains(content, []byte(lastBackup.Name())) { + t.Fatalf("Should be statement about rotated log and backup name, got %s", content) + } + + tmpDir = t.TempDir() + + // Recreate logger and don't set a limit + file = createFileAtDir(t, tmpDir, "log_") + file.Close() + logger = NewFileLogger(file.Name(), true, false, false, true) + defer logger.Close() + for i := 0; i < 50; i++ { + logger.Noticef("This is line %d in the log file", i+1) + } + files, err = os.ReadDir(tmpDir) + if err != nil { + t.Fatalf("Error reading logs dir: %v", err) + } + if len(files) != 1 { + t.Fatalf("Expected file to not be rotated") + } + + // Now set a limit that is below current size + logger.SetSizeLimit(1000) + // Should have triggered rotation + files, err = os.ReadDir(tmpDir) + if err != nil { + t.Fatalf("Error reading logs dir: %v", err) + } + if len(files) <= 1 { + t.Fatalf("Expected file to have been rotated") + } + if err := logger.Close(); err != nil { + t.Fatalf("Error closing log: %v", err) + } + lastBackup = files[len(files)-1] + content, err = os.ReadFile(file.Name()) + if err != nil { + t.Fatalf("Error loading latest log: %v", err) + } + if !bytes.Contains(content, []byte("Rotated log")) || + !bytes.Contains(content, []byte(lastBackup.Name())) { + t.Fatalf("Should be statement about rotated log and backup name, got %s", content) + } + + logger = NewFileLogger(file.Name(), true, false, false, true) + defer logger.Close() + logger.SetSizeLimit(1000) + + // Check error on rotate. + logger.Lock() + logger.fl.Lock() + failClose := &fileLogFailClose{logger.fl.f, true} + logger.fl.f = failClose + logger.fl.Unlock() + logger.Unlock() + // Write a big line that will force rotation. + // Since we fail to close the log file, we should have bumped the limit to 2000 + logger.Noticef("This is a big line: %v", make([]byte, 1000)) + + // Remove the failure + failClose.fail = false + // Write a big line that makes rotation happen + logger.Noticef("This is a big line: %v", make([]byte, 2000)) + // Close + logger.Close() + + files, err = os.ReadDir(tmpDir) + if err != nil { + t.Fatalf("Error reading logs dir: %v", err) + } + lastBackup = files[len(files)-1] + content, err = os.ReadFile(filepath.Join(tmpDir, lastBackup.Name())) + if err != nil { + t.Fatalf("Error reading backup file: %v", err) + } + if !bytes.Contains(content, []byte("on purpose")) || !bytes.Contains(content, []byte("size 2000")) { + t.Fatalf("Expected error that file could not rotated and max size bumped to 2000, got %s", content) + } +} + +type fileLogFailClose struct { + writerAndCloser + fail bool +} + +func (l *fileLogFailClose) Close() error { + if l.fail { + return fmt.Errorf("on purpose") + } + return l.writerAndCloser.Close() +} + +func expectOutput(t *testing.T, f func(), expected string) { + old := os.Stderr // keep backup of the real stderr + r, w, _ := os.Pipe() + os.Stderr = w + + f() + + outC := make(chan string) + // copy the output in a separate goroutine so printing can't block indefinitely + go func() { + var buf bytes.Buffer + io.Copy(&buf, r) + outC <- buf.String() + }() + + os.Stderr.Close() + os.Stderr = old // restoring the real stdout + out := <-outC + if out != expected { + t.Fatalf("Expected '%s', received '%s'\n", expected, out) + } +} + +func createFileAtDir(t *testing.T, dir, prefix string) *os.File { + t.Helper() + f, err := os.CreateTemp(dir, prefix) + if err != nil { + t.Fatal(err) + } + return f +} diff --git a/logger/syslog.go b/logger/syslog.go new file mode 100644 index 0000000..0eb134e --- /dev/null +++ b/logger/syslog.go @@ -0,0 +1,132 @@ +// Copyright 2012-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !windows + +package logger + +import ( + "fmt" + "log" + "log/syslog" + "net/url" + "os" + "strings" +) + +// SysLogger provides a system logger facility +type SysLogger struct { + writer *syslog.Writer + debug bool + trace bool +} + +// SetSyslogName sets the name to use for the syslog. +// Currently used only on Windows. +func SetSyslogName(name string) {} + +// GetSysLoggerTag generates the tag name for use in syslog statements. If +// the executable is linked, the name of the link will be used as the tag, +// otherwise, the name of the executable is used. "nats-server" is the default +// for the NATS server. +func GetSysLoggerTag() string { + procName := os.Args[0] + if strings.ContainsRune(procName, os.PathSeparator) { + parts := strings.FieldsFunc(procName, func(c rune) bool { + return c == os.PathSeparator + }) + procName = parts[len(parts)-1] + } + return procName +} + +// NewSysLogger creates a new system logger +func NewSysLogger(debug, trace bool) *SysLogger { + w, err := syslog.New(syslog.LOG_DAEMON|syslog.LOG_NOTICE, GetSysLoggerTag()) + if err != nil { + log.Fatalf("error connecting to syslog: %q", err.Error()) + } + + return &SysLogger{ + writer: w, + debug: debug, + trace: trace, + } +} + +// NewRemoteSysLogger creates a new remote system logger +func NewRemoteSysLogger(fqn string, debug, trace bool) *SysLogger { + network, addr := getNetworkAndAddr(fqn) + w, err := syslog.Dial(network, addr, syslog.LOG_DEBUG, GetSysLoggerTag()) + if err != nil { + log.Fatalf("error connecting to syslog: %q", err.Error()) + } + + return &SysLogger{ + writer: w, + debug: debug, + trace: trace, + } +} + +func getNetworkAndAddr(fqn string) (network, addr string) { + u, err := url.Parse(fqn) + if err != nil { + log.Fatal(err) + } + + network = u.Scheme + if network == "udp" || network == "tcp" { + addr = u.Host + } else if network == "unix" { + addr = u.Path + } else { + log.Fatalf("error invalid network type: %q", u.Scheme) + } + + return +} + +// Noticef logs a notice statement +func (l *SysLogger) Noticef(format string, v ...any) { + l.writer.Notice(fmt.Sprintf(format, v...)) +} + +// Warnf logs a warning statement +func (l *SysLogger) Warnf(format string, v ...any) { + l.writer.Warning(fmt.Sprintf(format, v...)) +} + +// Fatalf logs a fatal error +func (l *SysLogger) Fatalf(format string, v ...any) { + l.writer.Crit(fmt.Sprintf(format, v...)) +} + +// Errorf logs an error statement +func (l *SysLogger) Errorf(format string, v ...any) { + l.writer.Err(fmt.Sprintf(format, v...)) +} + +// Debugf logs a debug statement +func (l *SysLogger) Debugf(format string, v ...any) { + if l.debug { + l.writer.Debug(fmt.Sprintf(format, v...)) + } +} + +// Tracef logs a trace statement +func (l *SysLogger) Tracef(format string, v ...any) { + if l.trace { + l.writer.Notice(fmt.Sprintf(format, v...)) + } +} diff --git a/logger/syslog_test.go b/logger/syslog_test.go new file mode 100644 index 0000000..8a35187 --- /dev/null +++ b/logger/syslog_test.go @@ -0,0 +1,237 @@ +// Copyright 2012-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !windows + +package logger + +import ( + "fmt" + "log" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +var serverFQN string + +func TestSysLogger(t *testing.T) { + logger := NewSysLogger(false, false) + + if logger.debug { + t.Fatalf("Expected %t, received %t\n", false, logger.debug) + } + + if logger.trace { + t.Fatalf("Expected %t, received %t\n", false, logger.trace) + } +} + +func TestSysLoggerWithDebugAndTrace(t *testing.T) { + logger := NewSysLogger(true, true) + + if !logger.debug { + t.Fatalf("Expected %t, received %t\n", true, logger.debug) + } + + if !logger.trace { + t.Fatalf("Expected %t, received %t\n", true, logger.trace) + } +} + +func testTag(t *testing.T, exePath, expected string) { + os.Args[0] = exePath + if result := GetSysLoggerTag(); result != expected { + t.Fatalf("Expected %s, received %s", expected, result) + } +} + +func restoreArg(orig string) { + os.Args[0] = orig +} + +func TestSysLoggerTagGen(t *testing.T) { + origArg := os.Args[0] + defer restoreArg(origArg) + + testTag(t, "nats-server", "nats-server") + testTag(t, filepath.Join(".", "nats-server"), "nats-server") + testTag(t, filepath.Join("home", "bin", "nats-server"), "nats-server") + testTag(t, filepath.Join("..", "..", "nats-server"), "nats-server") + testTag(t, "nats-server.service1", "nats-server.service1") + testTag(t, "nats-server_service1", "nats-server_service1") + testTag(t, "nats-server-service1", "nats-server-service1") + testTag(t, "nats-server service1", "nats-server service1") +} + +func TestSysLoggerTag(t *testing.T) { + origArg := os.Args[0] + defer restoreArg(origArg) + + os.Args[0] = "ServerLoggerTag" + + done := make(chan string) + startServer(done) + logger := NewRemoteSysLogger(serverFQN, true, true) + logger.Noticef("foo") + + line := <-done + data := strings.Split(line, "[") + if len(data) != 2 { + t.Fatalf("Unexpected syslog line %s\n", line) + } + + if !strings.Contains(data[0], os.Args[0]) { + t.Fatalf("Expected '%s', received '%s'\n", os.Args[0], data[0]) + } +} + +func TestRemoteSysLogger(t *testing.T) { + done := make(chan string) + startServer(done) + logger := NewRemoteSysLogger(serverFQN, true, true) + + if !logger.debug { + t.Fatalf("Expected %t, received %t\n", true, logger.debug) + } + + if !logger.trace { + t.Fatalf("Expected %t, received %t\n", true, logger.trace) + } + logger.Noticef("foo") + <-done +} + +func TestRemoteSysLoggerNotice(t *testing.T) { + done := make(chan string) + startServer(done) + logger := NewRemoteSysLogger(serverFQN, true, true) + + logger.Noticef("foo %s", "bar") + expectSyslogOutput(t, <-done, "foo bar\n") +} + +func TestRemoteSysLoggerDebug(t *testing.T) { + done := make(chan string) + startServer(done) + logger := NewRemoteSysLogger(serverFQN, true, true) + + logger.Debugf("foo %s", "qux") + expectSyslogOutput(t, <-done, "foo qux\n") +} + +func TestRemoteSysLoggerDebugDisabled(t *testing.T) { + done := make(chan string) + startServer(done) + logger := NewRemoteSysLogger(serverFQN, false, false) + + logger.Debugf("foo %s", "qux") + rcvd := <-done + if rcvd != "" { + t.Fatalf("Unexpected syslog response %s\n", rcvd) + } +} + +func TestRemoteSysLoggerTrace(t *testing.T) { + done := make(chan string) + startServer(done) + logger := NewRemoteSysLogger(serverFQN, true, true) + + logger.Tracef("foo %s", "qux") + expectSyslogOutput(t, <-done, "foo qux\n") +} + +func TestRemoteSysLoggerTraceDisabled(t *testing.T) { + done := make(chan string) + startServer(done) + logger := NewRemoteSysLogger(serverFQN, true, false) + + logger.Tracef("foo %s", "qux") + rcvd := <-done + if rcvd != "" { + t.Fatalf("Unexpected syslog response %s\n", rcvd) + } +} + +func TestGetNetworkAndAddrUDP(t *testing.T) { + n, a := getNetworkAndAddr("udp://foo.com:1000") + + if n != "udp" { + t.Fatalf("Unexpected network %s\n", n) + } + + if a != "foo.com:1000" { + t.Fatalf("Unexpected addr %s\n", a) + } +} + +func TestGetNetworkAndAddrTCP(t *testing.T) { + n, a := getNetworkAndAddr("tcp://foo.com:1000") + + if n != "tcp" { + t.Fatalf("Unexpected network %s\n", n) + } + + if a != "foo.com:1000" { + t.Fatalf("Unexpected addr %s\n", a) + } +} + +func TestGetNetworkAndAddrUnix(t *testing.T) { + n, a := getNetworkAndAddr("unix:///foo.sock") + + if n != "unix" { + t.Fatalf("Unexpected network %s\n", n) + } + + if a != "/foo.sock" { + t.Fatalf("Unexpected addr %s\n", a) + } +} +func expectSyslogOutput(t *testing.T, line string, expected string) { + data := strings.Split(line, "]: ") + if len(data) != 2 { + t.Fatalf("Unexpected syslog line %s\n", line) + } + + if data[1] != expected { + t.Fatalf("Expected '%s', received '%s'\n", expected, data[1]) + } +} + +func runSyslog(c net.PacketConn, done chan<- string) { + var buf [4096]byte + var rcvd string + for { + n, _, err := c.ReadFrom(buf[:]) + if err != nil || n == 0 { + break + } + rcvd += string(buf[:n]) + } + done <- rcvd +} + +func startServer(done chan<- string) { + c, e := net.ListenPacket("udp", "127.0.0.1:0") + if e != nil { + log.Fatalf("net.ListenPacket failed udp :0 %v", e) + } + + serverFQN = fmt.Sprintf("udp://%s", c.LocalAddr().String()) + c.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) + go runSyslog(c, done) +} diff --git a/logger/syslog_windows.go b/logger/syslog_windows.go new file mode 100644 index 0000000..176776b --- /dev/null +++ b/logger/syslog_windows.go @@ -0,0 +1,112 @@ +// Copyright 2012-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package logger logs to the windows event log +package logger + +import ( + "fmt" + "os" + "strings" + + "golang.org/x/sys/windows/svc/eventlog" +) + +var natsEventSource = "NATS-Server" + +// SetSyslogName sets the name to use for the system log event source +func SetSyslogName(name string) { + natsEventSource = name +} + +// SysLogger logs to the windows event logger +type SysLogger struct { + writer *eventlog.Log + debug bool + trace bool +} + +// NewSysLogger creates a log using the windows event logger +func NewSysLogger(debug, trace bool) *SysLogger { + if err := eventlog.InstallAsEventCreate(natsEventSource, eventlog.Info|eventlog.Error|eventlog.Warning); err != nil { + if !strings.Contains(err.Error(), "registry key already exists") { + panic(fmt.Sprintf("could not access event log: %v", err)) + } + } + + w, err := eventlog.Open(natsEventSource) + if err != nil { + panic(fmt.Sprintf("could not open event log: %v", err)) + } + + return &SysLogger{ + writer: w, + debug: debug, + trace: trace, + } +} + +// NewRemoteSysLogger creates a remote event logger +func NewRemoteSysLogger(fqn string, debug, trace bool) *SysLogger { + w, err := eventlog.OpenRemote(fqn, natsEventSource) + if err != nil { + panic(fmt.Sprintf("could not open event log: %v", err)) + } + + return &SysLogger{ + writer: w, + debug: debug, + trace: trace, + } +} + +func formatMsg(tag, format string, v ...any) string { + orig := fmt.Sprintf(format, v...) + return fmt.Sprintf("pid[%d][%s]: %s", os.Getpid(), tag, orig) +} + +// Noticef logs a notice statement +func (l *SysLogger) Noticef(format string, v ...any) { + l.writer.Info(1, formatMsg("NOTICE", format, v...)) +} + +// Warnf logs a warning statement +func (l *SysLogger) Warnf(format string, v ...any) { + l.writer.Info(1, formatMsg("WARN", format, v...)) +} + +// Fatalf logs a fatal error +func (l *SysLogger) Fatalf(format string, v ...any) { + msg := formatMsg("FATAL", format, v...) + l.writer.Error(5, msg) + panic(msg) +} + +// Errorf logs an error statement +func (l *SysLogger) Errorf(format string, v ...any) { + l.writer.Error(2, formatMsg("ERROR", format, v...)) +} + +// Debugf logs a debug statement +func (l *SysLogger) Debugf(format string, v ...any) { + if l.debug { + l.writer.Info(3, formatMsg("DEBUG", format, v...)) + } +} + +// Tracef logs a trace statement +func (l *SysLogger) Tracef(format string, v ...any) { + if l.trace { + l.writer.Info(4, formatMsg("TRACE", format, v...)) + } +} diff --git a/logger/syslog_windows_test.go b/logger/syslog_windows_test.go new file mode 100644 index 0000000..6a3190b --- /dev/null +++ b/logger/syslog_windows_test.go @@ -0,0 +1,140 @@ +// Copyright 2012-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package logger + +import ( + "os/exec" + "strings" + "testing" + + "golang.org/x/sys/windows/svc/eventlog" +) + +// Skips testing if we do not have privledges to run this test. +// This lets us skip the tests for general (non admin/system) users. +func checkPrivledges(t *testing.T) { + src := "NATS-eventlog-testsource" + defer eventlog.Remove(src) + if err := eventlog.InstallAsEventCreate(src, eventlog.Info|eventlog.Error|eventlog.Warning); err != nil { + if strings.Contains(err.Error(), "Access is denied") { + // Skip this test because elevated privileges are required. + t.SkipNow() + } + // let the tests report other types of errors + } +} + +// lastLogEntryContains reads the last entry (/c:1 /rd:true) written +// to the event log by the NATS-Server source, returning true if the +// passed text was found, false otherwise. +func lastLogEntryContains(t *testing.T, text string) bool { + var output []byte + var err error + + cmd := exec.Command("wevtutil.exe", "qe", "Application", "/q:*[System[Provider[@Name='NATS-Server']]]", + "/rd:true", "/c:1") + if output, err = cmd.Output(); err != nil { + t.Fatalf("Unable to execute command: %v", err) + } + return strings.Contains(string(output), text) +} + +// TestSysLogger tests event logging on windows +func TestSysLogger(t *testing.T) { + checkPrivledges(t) + logger := NewSysLogger(false, false) + if logger.debug { + t.Fatalf("Expected %t, received %t\n", false, logger.debug) + } + + if logger.trace { + t.Fatalf("Expected %t, received %t\n", false, logger.trace) + } + logger.Noticef("%s", "Noticef") + if !lastLogEntryContains(t, "[NOTICE]: Noticef") { + t.Fatalf("missing log entry") + } + + logger.Errorf("%s", "Errorf") + if !lastLogEntryContains(t, "[ERROR]: Errorf") { + t.Fatalf("missing log entry") + } + + logger.Tracef("%s", "Tracef") + if lastLogEntryContains(t, "Tracef") { + t.Fatalf("should not contain log entry") + } + + logger.Debugf("%s", "Debugf") + if lastLogEntryContains(t, "Debugf") { + t.Fatalf("should not contain log entry") + } +} + +// TestSysLoggerWithDebugAndTrace tests event logging +func TestSysLoggerWithDebugAndTrace(t *testing.T) { + checkPrivledges(t) + logger := NewSysLogger(true, true) + if !logger.debug { + t.Fatalf("Expected %t, received %t\n", true, logger.debug) + } + + if !logger.trace { + t.Fatalf("Expected %t, received %t\n", true, logger.trace) + } + + logger.Tracef("%s", "Tracef") + if !lastLogEntryContains(t, "[TRACE]: Tracef") { + t.Fatalf("missing log entry") + } + + logger.Debugf("%s", "Debugf") + if !lastLogEntryContains(t, "[DEBUG]: Debugf") { + t.Fatalf("missing log entry") + } +} + +// TestRemoteSysLoggerWithDebugAndTrace tests remote event logging +func TestRemoteSysLoggerWithDebugAndTrace(t *testing.T) { + checkPrivledges(t) + logger := NewRemoteSysLogger("", true, true) + if !logger.debug { + t.Fatalf("Expected %t, received %t\n", true, logger.debug) + } + + if !logger.trace { + t.Fatalf("Expected %t, received %t\n", true, logger.trace) + } + logger.Tracef("NATS %s", "[TRACE]: Remote Noticef") + if !lastLogEntryContains(t, "Remote Noticef") { + t.Fatalf("missing log entry") + } +} + +func TestSysLoggerFatalf(t *testing.T) { + defer func() { + if r := recover(); r != nil { + if !lastLogEntryContains(t, "[FATAL]: Fatalf") { + t.Fatalf("missing log entry") + } + } + }() + + checkPrivledges(t) + logger := NewSysLogger(true, true) + logger.Fatalf("%s", "Fatalf") + t.Fatalf("did not panic when expected to") +} diff --git a/logos/nats-horizontal-color.png b/logos/nats-horizontal-color.png new file mode 100644 index 0000000..6e24b57 Binary files /dev/null and b/logos/nats-horizontal-color.png differ diff --git a/logos/nats-server.png b/logos/nats-server.png new file mode 100644 index 0000000..822cbd1 Binary files /dev/null and b/logos/nats-server.png differ diff --git a/main.go b/main.go new file mode 100644 index 0000000..2c6281a --- /dev/null +++ b/main.go @@ -0,0 +1,135 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +//go:generate go run server/errors_gen.go + +import ( + "flag" + "fmt" + "os" + + "github.com/nats-io/nats-server/v2/server" +) + +var usageStr = ` +Usage: nats-server [options] + +Server Options: + -a, --addr, --net Bind to host address (default: 0.0.0.0) + -p, --port Use port for clients (default: 4222) + -n, --name + --server_name Server name (default: auto) + -P, --pid File to store PID + -m, --http_port Use port for http monitoring + -ms,--https_port Use port for https monitoring + -c, --config Configuration file + -t Test configuration and exit + -sl,--signal [=] Send signal to nats-server process (ldm, stop, quit, term, reopen, reload) + can be either a PID (e.g. 1) or the path to a PID file (e.g. /var/run/nats-server.pid) + --client_advertise Client URL to advertise to other servers + --ports_file_dir Creates a ports file in the specified directory (_.ports). + +Logging Options: + -l, --log File to redirect log output + -T, --logtime Timestamp log entries (default: true) + --logtime_utc Timestamps in UTC instead of local timezone (default: false) + -s, --syslog Log to syslog or windows event log + -r, --remote_syslog Syslog server addr (udp://localhost:514) + -D, --debug Enable debugging output + -V, --trace Trace the raw protocol + -VV Verbose trace (traces system account as well) + -DV Debug and trace + -DVV Debug and verbose trace (traces system account as well) + --log_size_limit Logfile size limit (default: auto) + --max_traced_msg_len Maximum printable length for traced messages (default: unlimited) + +JetStream Options: + -js, --jetstream Enable JetStream functionality + -sd, --store_dir Set the storage directory + +Authorization Options: + --user User required for connections + --pass Password required for connections + --auth Authorization token required for connections + +TLS Options: + --tls Enable TLS, do not verify clients (default: false) + --tlscert Server certificate file + --tlskey Private key for server certificate + --tlsverify Enable TLS, verify client certificates + --tlscacert Client certificate CA for verification + +Cluster Options: + --routes Routes to solicit and connect + --cluster Cluster URL for solicited routes + --cluster_name Cluster Name, if not set one will be dynamically generated + --no_advertise Do not advertise known cluster information to clients + --cluster_advertise Cluster URL to advertise to other servers + --connect_retries For implicit routes, number of connect retries + --cluster_listen Cluster url from which members can solicit routes + +Profiling Options: + --profile Profiling HTTP port + +Common Options: + -h, --help Show this message + -v, --version Show version + --help_tls TLS help +` + +// usage will print out the flag options for the server. +func usage() { + fmt.Printf("%s\n", usageStr) + os.Exit(0) +} + +func main() { + exe := "nats-server" + + // Create a FlagSet and sets the usage + fs := flag.NewFlagSet(exe, flag.ExitOnError) + fs.Usage = usage + + // Configure the options from the flags/config file + opts, err := server.ConfigureOptions(fs, os.Args[1:], + server.PrintServerAndExit, + fs.Usage, + server.PrintTLSHelpAndDie) + if err != nil { + server.PrintAndDie(fmt.Sprintf("%s: %s", exe, err)) + } else if opts.CheckConfig { + fmt.Fprintf(os.Stderr, "%s: configuration file %s is valid (%s)\n", exe, opts.ConfigFile, opts.ConfigDigest()) + os.Exit(0) + } + + // Redact secret arguments before expvar reads them. + server.RedactArgs(os.Args) + + // Create the server with appropriate options. + s, err := server.NewServer(opts) + if err != nil { + server.PrintAndDie(fmt.Sprintf("%s: %s", exe, err)) + } + + // Configure the logger based on the flags. + s.ConfigureLogger() + + // Start things up. Block here until done. + if err := server.Run(s); err != nil { + server.PrintAndDie(err.Error()) + } + + s.WaitForShutdown() +} diff --git a/scripts/cov.sh b/scripts/cov.sh new file mode 100755 index 0000000..5e34d44 --- /dev/null +++ b/scripts/cov.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Run from directory above via ./scripts/cov.sh + +check_file () { + # If the content of the file is simply "mode: atomic", then it means that the + # code coverage did not complete due to a panic in one of the tests. + if [[ $(cat ./cov/$2) == "mode: atomic" ]]; then + echo "#############################################" + echo "## Code coverage for $1 package failed ##" + echo "#############################################" + exit 1 + fi +} + +# Do not globally set the -e flag because we don't a flapper to prevent the push to coverall. + +export GO111MODULE="on" + +go install github.com/wadey/gocovmerge@latest +# Fail fast by checking if we can run gocovmerge +gocovmerge +if [[ $? != 0 ]]; then + echo "Unable to run gocovmerge" + exit 1 +fi + +rm -rf ./cov +mkdir cov +# +# Since it is difficult to get a full run without a flapper, do not use `-failfast`. +# It is better to have one flapper or two and still get the report than have +# to re-run the whole code coverage. One or two failed tests should not affect +# so much the code coverage. +# +# However, we need to take into account that if there is a panic in one test, all +# other tests in that package will not run, which then would cause the code coverage +# to drastically be lowered. In that case, we don't want the code coverage to be +# uploaded. +# +go test -v -covermode=atomic -coverprofile=./cov/conf.out ./conf -timeout=1h -tags=skip_no_race_tests +check_file "conf" "conf.out" +go test -v -covermode=atomic -coverprofile=./cov/internal.out ./internal/ldap -timeout=1h -tags=skip_no_race_tests +check_file "internal" "internal.out" +go test -v -covermode=atomic -coverprofile=./cov/log.out ./logger -timeout=1h -tags=skip_no_race_tests +check_file "logger" "log.out" +go test -v -covermode=atomic -coverprofile=./cov/server_avl.out ./server/avl -timeout=1h -tags=skip_no_race_tests +check_file "server_avl" "server_avl.out" +go test -v -covermode=atomic -coverprofile=./cov/server.out ./server -timeout=1h -tags=skip_no_race_tests +check_file "server" "server.out" +go test -v -covermode=atomic -coverprofile=./cov/test.out -coverpkg=./server ./test -timeout=1h -tags=skip_no_race_tests +check_file "test" "test.out" + +# At this point, if that fails, we want the caller to know about the failure. +set -e +gocovmerge ./cov/*.out > acc.out +rm -rf ./cov + +# If no argument passed, launch a browser to see the results. +if [[ $1 == "" ]]; then + go tool cover -html=acc.out +fi +set +e diff --git a/scripts/runTestsOnTravis.sh b/scripts/runTestsOnTravis.sh new file mode 100755 index 0000000..596f85e --- /dev/null +++ b/scripts/runTestsOnTravis.sh @@ -0,0 +1,138 @@ +#!/bin/sh + +set -ex + +if [ "$1" = "compile" ]; then + # First check that NATS builds. + go build -v; + + # Now run the linters. + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.4; + golangci-lint run; + if [ "$TRAVIS_TAG" != "" ]; then + go test -v -run=TestVersionMatchesTag ./server -ldflags="-X=github.com/nats-io/nats-server/v2/server.serverVersion=$TRAVIS_TAG" -count=1 -vet=off + fi + +elif [ "$1" = "build_only" ]; then + go build -v; + +elif [ "$1" = "no_race_1_tests" ]; then + + # Run tests without the `-race` flag. By convention, those tests start + # with `TestNoRace`. This will include all test files that have the + # "&& !skip_no_race_1_tests" in the go build directive. + + go test -v -p=1 -run=TestNoRace ./... -tags=skip_no_race_2_tests -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "no_race_2_tests" ]; then + + # Run tests without the `-race` flag. By convention, those tests start + # with `TestNoRace`. This will include all test files that have the + # "&& !skip_no_race_2_tests" in the go build directive. + + go test -v -p=1 -run=TestNoRace ./... -tags=skip_no_race_1_tests -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "store_tests" ]; then + + # Run store tests. By convention, all file store tests start with `TestFileStore`, + # and memory store tests start with `TestMemStore`. + + go test $RACE -v -p=1 -run=Test\(Store\|FileStore\|MemStore\) ./server -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "js_tests" ]; then + + # Run JetStream non-clustered tests. By convention, all JS tests start + # with `TestJetStream`. We exclude the clustered, super-clustered and + # consumer tests by using the appropriate tags. + + go test $RACE -v -p=1 -run=TestJetStream ./server -tags=skip_js_cluster_tests,skip_js_cluster_tests_2,skip_js_cluster_tests_3,skip_js_cluster_tests_4,skip_js_super_cluster_tests,skip_js_consumer_tests -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "js_consumer_tests" ]; then + + # Run JetStream consumer tests. All tests start with TestJetStreamConsumer prefix. + + go test $RACE -v -p=1 -run=TestJetStreamConsumer ./server -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "raft_tests" ]; then + + # Run the RAFT tests. All tests start with TestNRG prefix. + + go test $RACE -v -p=1 ./server/... -run="^TestNRG" -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "js_cluster_tests_1" ]; then + + # Run JetStream clustered tests. By convention, all JS cluster tests + # start with `TestJetStreamCluster`. Will run the first batch of tests, + # excluding others with use of proper tags. + + go test $RACE -v -p=1 -run=TestJetStreamCluster ./server -tags=skip_js_cluster_tests_2,skip_js_cluster_tests_3,skip_js_cluster_tests_4 -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "js_cluster_tests_2" ]; then + + # Run JetStream clustered tests. By convention, all JS cluster tests + # start with `TestJetStreamCluster`. Will run the second batch of tests, + # excluding others with use of proper tags. + + go test $RACE -v -p=1 -run=TestJetStreamCluster ./server -tags=skip_js_cluster_tests,skip_js_cluster_tests_3,skip_js_cluster_tests_4 -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "js_cluster_tests_3" ]; then + + # Run JetStream clustered tests. By convention, all JS cluster tests + # start with `TestJetStreamCluster`. Will run the third batch of tests, + # excluding others with use of proper tags. + # + + go test $RACE -v -p=1 -run=TestJetStreamCluster ./server -tags=skip_js_cluster_tests,skip_js_cluster_tests_2,skip_js_cluster_tests_4 -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "js_cluster_tests_4" ]; then + + # Run JetStream clustered tests. By convention, all JS cluster tests + # start with `TestJetStreamCluster`. Will run the third batch of tests, + # excluding others with use of proper tags. + # + + go test $RACE -v -p=1 -run=TestJetStreamCluster ./server -tags=skip_js_cluster_tests,skip_js_cluster_tests_2,skip_js_cluster_tests_3 -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "js_super_cluster_tests" ]; then + + # Run JetStream super clustered tests. By convention, all JS super cluster + # tests start with `TestJetStreamSuperCluster`. + + go test $RACE -v -p=1 -run=TestJetStreamSuperCluster ./server -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "mqtt_tests" ]; then + + # Run MQTT tests. By convention, all MQTT tests start with `TestMQTT`. + + go test $RACE -v -p=1 -run=TestMQTT ./server -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "msgtrace_tests" ]; then + + # Run Message Tracing tests. By convention, all message tracing tests start with `TestMsgTrace`. + + go test $RACE -v -p=1 -run=TestMsgTrace ./server -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "srv_pkg_non_js_tests" ]; then + + # Run all non JetStream tests in the server package. We exclude the + # store tests by using the `skip_store_tests` build tag, the JS tests + # by using `skip_js_tests`, MQTT tests by using `skip_mqtt_tests` and + # message tracing tests by using `skip_msgtrace_tests`. + # Ignore JWT and NRG tests here as they have their own matrix run. + + # Also including the ldflag with the version since this includes the `TestVersionMatchesTag`. + go test $RACE -v -p=1 ./server/... -run="^Test(N[^R]|NR[^G]|J[^W]|JW[^T]|[^JN])" -ldflags="-X=github.com/nats-io/nats-server/v2/server.serverVersion=$TRAVIS_TAG" -tags=skip_store_tests,skip_js_tests,skip_mqtt_tests,skip_msgtrace_tests,skip_no_race_tests -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "jwt_tests" ]; then + + # Run the JWT tests. All tests start with TestJWT. + + go test $RACE -v -p=1 ./server/... -run="^TestJWT" -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "non_srv_pkg_tests" ]; then + + # Run all tests of all non server package. + + go test $RACE -v -p=1 $(go list ./... | grep -v "/server") -count=1 -vet=off -timeout=30m -failfast + +fi diff --git a/scripts/updateCopyrights.sh b/scripts/updateCopyrights.sh new file mode 100755 index 0000000..8c47c32 --- /dev/null +++ b/scripts/updateCopyrights.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Ensure script is run at the root of a git repository +git rev-parse --is-inside-work-tree &>/dev/null || { echo "Not inside a git repository"; exit 1; } + +# Find all .go files tracked by git +git ls-files "*.go" | while read -r file; do + # Skip files that don't have a copyright belonging to "The NATS Authors" + current_copyright=$(grep -oE "^// Copyright [0-9]{4}(-[0-9]{4})? The NATS Authors" "$file" || echo "") + [[ -z "$current_copyright" ]] && continue + + # Get the last commit year for the file, ignore commit messages containing the word "copyright" + last_year=$(git log --follow --format="%ad" --date=format:%Y --grep="(C|c)opyright" --invert-grep -n 1 -- "$file") + existing_years=$(echo "$current_copyright" | grep -oE "[0-9]{4}(-[0-9]{4})?") + + # Determine the new copyright range + if [[ "$existing_years" =~ ^([0-9]{4})-([0-9]{4})$ ]]; then + first_year=${BASH_REMATCH[1]} + new_copyright="// Copyright $first_year-$last_year The NATS Authors" + elif [[ "$existing_years" =~ ^([0-9]{4})$ ]]; then + first_year=${BASH_REMATCH[1]} + if [[ "$first_year" == "$last_year" ]]; then + new_copyright="// Copyright $first_year The NATS Authors" + else + new_copyright="// Copyright $first_year-$last_year The NATS Authors" + fi + else + continue # If the format is somehow incorrect, skip the file + fi + + # Update the first line + if sed --version &>/dev/null; then + # Linux sed + sed -i "1s|^// Copyright.*|$new_copyright|" "$file" + else + # BSD/macOS sed, needs -i '' + sed -i '' "1s|^// Copyright.*|$new_copyright|" "$file" + fi +done diff --git a/server/README-MQTT.md b/server/README-MQTT.md new file mode 100644 index 0000000..6b420a8 --- /dev/null +++ b/server/README-MQTT.md @@ -0,0 +1,595 @@ +**MQTT Implementation Overview** + +Revision 1.1 + +Authors: Ivan Kozlovic, Lev Brouk + +NATS Server currently supports most of MQTT 3.1.1. This document describes how +it is implemented. + +It is strongly recommended to review the [MQTT v3.1.1 +specifications](https://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html) +and get a detailed understanding before proceeding with this document. + +# Contents + +1. [Concepts](#1-concepts) + - [Server, client](#server-client) + - [Connection, client ID, session](#connection-client-id-session) + - [Packets, messages, and subscriptions](#packets-messages-and-subscriptions) + - [Quality of Service (QoS), publish identifier (PI)](#quality-of-service-qos-publish-identifier-pi) + - [Retained message](#retained-message) + - [Will message](#will-message) +2. [Use of JetStream](#2-use-of-jetstream) + - [JetStream API](#jetstream-api) + - [Streams](#streams) + - [Consumers and Internal NATS Subscriptions](#consumers-and-internal-nats-subscriptions) +3. [Lifecycles](#3-lifecycles) + - [Connection, Session](#connection-session) + - [Subscription](#subscription) + - [Message](#message) + - [Retained messages](#retained-messages) +4. [Implementation Notes](#4-implementation-notes) + - [Hooking into NATS I/O](#hooking-into-nats-io) + - [Session Management](#session-management) + - [Processing QoS acks: PUBACK, PUBREC, PUBCOMP](#processing-qos-acks-puback-pubrec-pubcomp) + - [Subject Wildcards](#subject-wildcards) +5. [Known issues](#5-known-issues) + +# 1. Concepts + +## Server, client + +In the MQTT specification there are concepts of **Client** and **Server**, used +somewhat interchangeably with those of **Sender** and **Receiver**. A **Server** +acts as a **Receiver** when it gets `PUBLISH` messages from a **Sender** +**Client**, and acts as a **Sender** when it delivers them to subscribed +**Clients**. + +In the NATS server implementation there are also concepts (types) `server` and +`client`. `client` is an internal representation of a (connected) client and +runs its own read and write loops. Both of these have an `mqtt` field that if +set makes them behave as MQTT-compliant. + +The code and comments may sometimes be confusing as they refer to `server` and +`client` sometimes ambiguously between MQTT and NATS. + +## Connection, client ID, session + +When an MQTT client connects to a server, it must send a `CONNECT` packet to +create an **MQTT Connection**. The packet must include a **Client Identifier**. +The server will then create or load a previously saved **Session** for the (hash +of) the client ID. + +## Packets, messages, and subscriptions + +The low level unit of transmission in MQTT is a **Packet**. Examples of packets +are: `CONNECT`, `SUBSCRIBE`, `SUBACK`, `PUBLISH`, `PUBCOMP`, etc. + +An **MQTT Message** starts with a `PUBLISH` packet that a client sends to the +server. It is then matched against the current **MQTT Subscriptions** and is +delivered to them as appropriate. During the message delivery the server acts as +an MQTT client, and the receiver acts as an MQTT server. + +Internally we use **NATS Messages** and **NATS Subscriptions** to facilitate +message delivery. This may be somewhat confusing as the code refers to `msg` and +`sub`. What may be even more confusing is that some MQTT packets (specifically, +`PUBREL`) are represented as NATS messages, and that the original MQTT packet +"metadata" may be encoded as NATS message headers. + +## Quality of Service (QoS), publish identifier (PI) + +MQTT specifies 3 levels of quality of service (**QoS**): + +- `0` for at most once. A single delivery attempt. +- `1` for at least once. Will try to redeliver until acknowledged by the + receiver. +- `2` for exactly once. See the [SPEC REF] for the acknowledgement flow. + +QoS 1 and 2 messages need to be identified with publish identifiers (**PI**s). A +PI is a 16-bit integer that must uniquely identify a message for the duration of +the required exchange of acknowledgment packets. + +Note that the QoS applies separately to the transmission of a message from a +sender client to the server, and from the server to the receiving client. There +is no protocol-level acknowledgements between the receiver and the original +sender. The sender passes the ownership of messages to the server, and the +server then delivers them at maximum possible QoS to the receivers +(subscribers). The PIs for in-flight outgoing messages are issued and stored per +session. + +## Retained message + +A **Retained Message** is not part of any MQTT session and is not removed when the +session that produced it goes away. Instead, the server needs to persist a +_single_ retained message per topic. When a subscription is started, the server +needs to send the “matching” retained messages, that is, messages that would +have been delivered to the new subscription should that subscription had been +running prior to the publication of this message. + +Retained messages are removed when the server receives a retained message with +an empty body. Still, this retained message that serves as a “delete” of a +retained message will be processed as a normal published message. + +Retained messages can have QoS. + +## Will message + +The `CONNECT` packet can contain information about a **Will Message** that needs to +be sent to any client subscribing on the Will topic/subject in the event that +the client is disconnected implicitly, that is, not as a result as the client +sending the `DISCONNECT` packet. + +Will messages can have the retain flag and QoS. + +# 2. Use of JetStream + +The MQTT implementation relies heavily on JetStream. We use it to: + +- Persist (and restore) the [Session](#connection-client-id-session) state. +- Store and retrieve [Retained messages](#retained-message). +- Persist incoming [QoS 1 and + 2](#quality-of-service-qos-publish-identifier-pi) messages, and + re-deliver if needed. +- Store and de-duplicate incoming [QoS + 2](#quality-of-service-qos-publish-identifier-pi) messages. +- Persist and re-deliver outgoing [QoS + 2](#quality-of-service-qos-publish-identifier-pi) `PUBREL` packets. + +Here is the overview of how we set up and use JetStream **streams**, +**consumers**, and **internal NATS subscriptions**. + +## JetStream API + +All interactions with JetStream are performed via `mqttJSA` that sends NATS +requests to JetStream. Most are processed synchronously and await a response, +some (e.g. `jsa.sendAck()`) are sent asynchronously. JetStream API is usually +referred to as `jsa` in the code. No special locking is required to use `jsa`, +however the asynchronous use of JetStream may create race conditions with +delivery callbacks. + +## Streams + +We create the following streams unless they already exist. Failing to ensure the +streams would prevent the client from connecting. + +Each stream is created with a replica value that is determined by the size of +the cluster but limited to 3. It can also be overwritten by the stream_replicas +option in the MQTT configuration block. + +The streams are created the first time an Account Session Manager is initialized +and are used by all sessions in it. Note that to avoid race conditions, some +subscriptions are created first. The streams are never deleted. See +`mqttCreateAccountSessionManager()` for details. + +1. `$MQTT_sess` stores persisted **Session** records. It filters on + `"$MQTT.sess.>` subject and has a “limits” policy with `MaxMsgsPer` setting + of 1. +2. `$MQTT_msgs` is used for **QoS 1 and 2 message delivery**. + It filters on `$MQTT.msgs.>` subject and has an “interest” policy. +3. `$MQTT_rmsgs` stores **Retained Messages**. They are all + stored (and filtered) on a single subject `$MQTT.rmsg`. This stream has a + limits policy. +4. `$MQTT_qos2in` stores and deduplicates **Incoming QoS 2 Messages**. It + filters on `$MQTT.qos2.in.>` and has a "limits" policy with `MaxMsgsPer` of + 1. +5. `$MQTT_out` stores **Outgoing QoS 2** `PUBREL` packets. It filters on + `$MQTT.out.>` and has a "interest" retention policy. + +## Consumers and Internal NATS Subscriptions + +### Account Scope + +- A durable consumer for [Retained Messages](#retained-message) - + `$MQTT_rmsgs_` +- A subscription to handle all [jsa](#jetstream-api) replies for the account. +- A subscription to replies to "session persist" requests, so that we can detect + the use of a session with the same client ID anywhere in the cluster. +- 2 subscriptions to support [retained messages](#retained-message): + `$MQTT.sub.` for the messages themselves, and one to receive replies to + "delete retained message" JS API (on the JS reply subject var). + +### Session Scope + +When a new QoS 2 MQTT subscription is detected in a session, we ensure that +there is a durable consumer for [QoS +2](#quality-of-service-qos-publish-identifier-pi) `PUBREL`s out for delivery - +`$MQTT_PUBREL_` + +### Subscription Scope + +For all MQTT subscriptions, regardless of their QoS, we create internal NATS subscriptions to + +- `subject` (directly encoded from `topic`). This subscription is used to + deliver QoS 0 messages, and messages originating from NATS. +- if needed, `subject fwc` complements `subject` for topics like `topic.#` to + include `topic` itself, see [top-level wildcards](#subject-wildcards) + +For QoS 1 or 2 MQTT subscriptions we ensure: + +- A durable consumer for messages out for delivery - `_` +- An internal subscription to `$MQTT.sub.` to deliver the messages to the + receiving client. + +### (Old) Notes + +As indicated before, for a QoS1 or QoS2 subscription, the server will create a +JetStream consumer with the appropriate subject filter. If the subscription +already existed, then only the NATS subscription is created for the JetStream +consumer’s delivery subject. + +Note that JS consumers can be created with an “Replicas” override, which from +recent discussion is problematic with “Interest” policy streams, which +“$MQTT_msgs” is. + +We do handle situations where a subscription on the same subject filter is sent +with a different QoS as per MQTT specifications. If the existing was on QoS 1 or +2, and the “new” is for QoS 0, then we delete the existing JS consumer. + +Subscriptions that are QoS 0 have a NATS subscription with the callback function +being `mqttDeliverMsgCbQos0()`; while QoS 1 and 2 have a NATS subscription with +callback `mqttDeliverMsgCbQos12()`. Both those functions have comments that +describe the reason for their existence and what they are doing. For instance +the `mqttDeliverMsgCbQos0()` callback will reject any producing client that is +of type JETSTREAM, so that it handles only non JetStream (QoS 1 and 2) messages. + +Both these functions end-up calling mqttDeliver() which will first enqueue the +possible retained messages buffer before delivering any new message. The message +itself being delivered is serialized in MQTT format and enqueued to the client’s +outbound buffer and call to addToPCD is made so that it is flushed out of the +readloop. + +# 3. Lifecycles + +## Connection, Session + +An MQTT connection is created when a listening MQTT server receives a `CONNECT` +packet. See `mqttProcessConnect()`. A connection is associated with a session. +Steps: + +1. Ensure that we have an `AccountSessionManager` so we can have an + `mqttSession`. Lazily initialize JetStream streams, and internal consumers + and subscriptions. See `getOrCreateMQTTAccountSessionManager()`. +2. Find and disconnect any previous session/client for the same ID. See + `mqttProcessConnect()`. +3. Ensure we have an `mqttSession` - create a new or load a previously persisted + one. If the clean flag is set in `CONNECT`, clean the session. see + `mqttSession.clear()` +4. Initialize session's subscriptions, if any. +5. Always send back a `CONNACK` packet. If there were errors in previous steps, + include the error. + +An MQTT connection can be closed for a number of reasons, including receiving a +`DISCONNECT` from the client, explicit internal errors processing MQTT packets, +or the server receiving another `CONNECT` packet with the same client ID. See +`mqttHandleClosedClient()` and `mqttHandleWill()`. Steps: + +1. Send out the Will Message if applicable (if not caused by a `DISCONNECT` packet) +2. Delete the JetStream consumers for to QoS 1 and 2 packet delivery through + JS API calls (if "clean" session flag is set) +3. Delete the session record from the “$MQTT_sess” stream, based on recorded + stream sequence. (if "clean" session flag is set) +4. Close the client connection. + +On an explicit disconnect, that is, the client sends the DISCONNECT packet, the +server will NOT send the Will, as per specifications. + +For sessions that had the “clean” flag, the JS consumers corresponding to QoS 1 +subscriptions are deleted through JS API calls, the session record is then +deleted (based on recorded stream sequence) from the “$MQTT_sess” stream. + +Finally, the client connection is closed + +Sessions are persisted on disconnect, and on subscriptions changes. + +## Subscription + +Receiving an MQTT `SUBSCRIBE` packet creates new subscriptions, or updates +existing subscriptions in a session. Each `SUBSCRIBE` packet may contain several +specific subscriptions (`topic` + QoS in each). We always respond with a +`SUBACK`, which may indicate which subscriptions errored out. + +For each subscription in the packet, we: + +1. Ignore it if `topic` starts with `$MQTT.sub.`. +2. Set up QoS 0 message delivery - an internal NATS subscription on `topic`. +3. Replay any retained messages for `topic`, once as QoS 0. +4. If we already have a subscription on `topic`, update its QoS +5. If this is a QoS 2 subscription in the session, ensure we have the [PUBREL + consumer](#session-scope) for the session. +6. If this is a QoS 1 or 2 subscription, ensure we have the [Message + consumer](#subscription-scope) for this subscription (or delete one if it + exists and this is now a QoS 0 sub). +7. Add an extra subscription for the [top-level wildcard](#subject-wildcards) case. +8. Update the session, persist it if changed. + +When a session is restored (no clean flag), we go through the same steps to +re-subscribe to its stored subscription, except step #8 which would have been +redundant. + +When we get an `UNSUBSCRIBE` packet, it can contain multiple subscriptions to +unsubscribe. The parsing will generate a slice of mqttFilter objects that +contain the “filter” (the topic with possibly wildcard of the subscription) and +the QoS value. The server goes through the list and deletes the JS consumer (if +QoS 1 or 2) and unsubscribes the NATS subscription for the delivery subject (if +it was a QoS 1 or 2) or on the actual topic/subject. In case of the “#” +wildcard, the server will handle the “level up” subscriptions that NATS had to +create. + +Again, we update the session and persist it as needed in the `$MQTT_sess` +stream. + +## Message + +1. Detect an incoming PUBLISH packet, parse and check the message QoS. Fill out + the session's `mqttPublish` struct that contains information about the + published message. (see `mqttParse()`, `mqttParsePub()`) +2. Process the message according to its QoS (see `mqttProcessPub()`) + + - QoS 0: + - Initiate message delivery + - QoS 1: + - Initiate message delivery + - Send back a `PUBACK` + - QoS 2: + - Store the message in `$MQTT_qos2in` stream, using a PI-specific subject. + Since `MaxMsgsPer` is set to 1, we will ignore duplicates on the PI. + - Send back a `PUBREC` + - "Wait" for a `PUBREL`, then initiate message delivery + - Remove the previously stored QoS2 message + - Send back a `PUBCOMP` + +3. Initiate message delivery (see `mqttInitiateMsgDelivery()`) + + - Convert the MQTT `topic` into a NATS `subject` using + `mqttTopicToNATSPubSubject()` function. If there is a known subject + mapping, then we select the new subject using `selectMappedSubject()` + function and then convert back this subject into an MQTT topic using + `natsSubjectToMQTTTopic()` function. + - Re-serialize the `PUBLISH` packet received as a NATS message. Use NATS + headers for the metadata, and the deliverable MQTT `PUBLISH` packet as the + contents. + - Publish the messages as `subject` (and `subject fwc` if applicable, see + [subject wildcards](#subject-wildcards)). Use the "standard" NATS + `c.processInboundClientMsg()` to do that. `processInboundClientMsg()` will + distribute the message to any NATS subscriptions (including routes, + gateways, leafnodes) and the relevant MQTT subscriptions. + - Check for retained messages, process as needed. See + `c.processInboundClientMsg()` calling `c.mqttHandlePubRetain()` For MQTT + clients. + - If the message QoS is 1 or 2, store it in `$MQTT_msgs` stream as + `$MQTT.msgs.` for "at least once" delivery with retries. + +4. Let NATS and JetStream deliver to the internal subscriptions, and to the + receiving clients. See `mqttDeliverMsgCb...()` + + - The NATS message posted to `subject` (and `subject fwc`) will be delivered + to each relevant internal subscription by calling `mqttDeliverMsgCbQoS0()`. + The function has access to both the publishing and the receiving clients. + + - Ignore all irrelevant invocations. Specifically, do nothing if the + message needs to be delivered with a higher QoS - that will be handled by + the other, `...QoS12` callback. Note that if the original message was + publuished with a QoS 1 or 2, but the subscription has its maximum QoS + set to 0, the message will be delivered by this callback. + - Ignore "reserved" subscriptions, as per MQTT spec. + - Decode delivery `topic` from the NATS `subject`. + - Write (enqueue) outgoing `PUBLISH` packet. + - **DONE for QoS 0** + + - The NATS message posted to JetStream as `$MQTT.msgs.subject` will be + consumed by subscription-specific consumers. Note that MQTT subscriptions + with max QoS 0 do not have JetStream consumers. They are handled by the + QoS0 callback. + + The consumers will deliver it to the `$MQTT.sub.` + subject for their respective NATS subscriptions by calling + `mqttDeliverMsgCbQoS12()`. This callback too has access to both the + publishing and the receiving clients. + + - Ignore "reserved" subscriptions, as per MQTT spec. + - See if this is a re-delivery from JetStream by checking `sess.cpending` + for the JS reply subject. If so, use the existing PI and treat this as a + duplicate redelivery. + - Otherwise, assign the message a new PI (see `trackPublish()` and + `bumpPI()`) and store it in `sess.cpending` and `sess.pendingPublish`, + along with the JS reply subject that can be used to remove this pending + message from the consumer once it's delivered to the receipient. + - Decode delivery `topic` from the NATS `subject`. + - Write (enqueue) outgoing `PUBLISH` packet. + +5. QoS 1: "Wait" for a `PUBACK`. See `mqttProcessPubAck()`. + + - When received, remove the PI from the tracking maps, send an ACK to + consumer to remove the message. + - **DONE for QoS 1** + +6. QoS 2: "Wait" for a `PUBREC`. When received, we need to do all the same + things as in the QoS 1 `PUBACK` case, but we need to send out a `PUBREL`, and + continue using the same PI until the delivery flow is complete and we get + back a `PUBCOMP`. For that, we add the PI to `sess.pendingPubRel`, and to + `sess.cpending` with the PubRel consumer durable name. + + We also compose and store a headers-only NATS message signifying a `PUBREL` + out for delivery, and store it in the `$MQTT_qos2out` stream, as + `$MQTT.qos2.out.`. + +7. QoS 2: Deliver `PUBREL`. The PubRel session-specific consumer will publish to + internal subscription on `$MQTT.qos2.delivery`, calling + `mqttDeliverPubRelCb()`. We store the ACK reply subject in `cpending` to + remove the JS message on `PUBCOMP`, compose and send out a `PUBREL` packet. + +8. QoS 2: "Wait" for a `PUBCOMP`. See `mqttProcessPubComp()`. + - When received, remove the PI from the tracking maps, send an ACK to + consumer to remove the `PUBREL` message. + - **DONE for QoS 2** + +## Retained messages + +When we process an inbound `PUBLISH` and submit it to +`processInboundClientMsg()` function, for MQTT clients it will invoke +`mqttHandlePubRetain()` which checks if the published message is “retained” or +not. + +If it is, then we construct a record representing the retained message and store +it in the `$MQTT_rmsg` stream, under the single `$MQTT.rmsg` subject. The stored +record (in JSON) contains information about the subject, topic, MQTT flags, user +that produced this message and the message content itself. It is stored and the +stream sequence is remembered in the memory structure that contains retained +messages. + +Note that when creating an account session manager, the retained messages stream +is read from scratch to load all the messages through the use of a JS consumer. +The associated subscription will process the recovered retained messages or any +new that comes from the network. + +A retained message is added to a map and a subscription is created and inserted +into a sublist that will be used to perform a ReverseMatch() when a subscription +is started and we want to find all retained messages that the subscription would +have received if it had been running prior to the message being published. + +If a retained message on topic “foo” already exists, then the server has to +delete the old message at the stream sequence we saved when storing it. + +This could have been done with having retained messages stored under +`$MQTT.rmsg.` as opposed to all under a single subject, and make use of +the `MaxMsgsPer` field set to 1. The `MaxMsgsPer` option was introduced well into +the availability of MQTT and changes to the sessions was made in [PR +#2501](https://github.com/nats-io/nats-server/pull/2501), with a conversion of +existing streams such as `$MQTT*sess*` into a single stream with unique +subjects, but the changes were not made to the retained messages stream. + +There are also subscriptions for the handling of retained messages which are +messages that are asked by the publisher to be retained by the MQTT server to be +delivered to matching subscriptions when they start. There is a single message +per topic. Retained messages are deleted when the user sends a retained message +(there is a flag in the PUBLISH protocol) on a given topic with an empty body. +The difficulty with retained messages is to handle them in a cluster since all +servers need to be aware of their presence so that they can deliver them to +subscriptions that those servers may become the leader for. + +- `$MQTT_rmsgs` which has a “limits” policy and holds retained messages, all + under `$MQTT.rmsg` single subject. Not sure why I did not use MaxMsgsPer for + this stream and not filter `$MQTT.rmsg.>`. + +The first step when processing a new subscription is to gather the retained +messages that would be a match for this subscription. To do so, the server will +serialize into a buffer all messages for the account session manager’s sublist’s +ReverseMatch result. We use the returned subscriptions’ subject to find from a +map appropriate retained message (see `serializeRetainedMsgsForSub()` for +details). + +# 4. Implementation Notes + +## Hooking into NATS I/O + +### Starting the accept loop + +The MQTT accept loop is started when the server detects that an MQTT port has +been defined in the configuration file. It works similarly to all other accept +loops. Note that for MQTT over websocket, the websocket port has to be defined +and MQTT clients will connect to that port instead of the MQTT port and need to +provide `/mqtt` as part of the URL to redirect the creation of the client to an +MQTT client (with websocket support) instead of a regular NATS with websocket. +See the branching done in `startWebsocketServer()`. See `startMQTT()`. + +### Starting the read/write loops + +When a TCP connection is accepted, the internal go routine will invoke +`createMQTTClient()`. This function will set a `c.mqtt` object that will make it +become an MQTT client (through the `isMqtt()` helper function). The `readLoop()` +and `writeLoop()` are started similarly to other clients. However, the read loop +will branch out to `mqttParse()` instead when detecting that this is an MQTT +client. + +## Session Management + +### Account Session Manager + +`mqttAccountSessionManager` is an object that holds the state of all sessions in +an account. It also manages the lifecycle of JetStream streams and internal +subscriptions for processing JS API replies, session updates, etc. See +`mqttCreateAccountSessionManager()`. It is lazily initialized upon the first +MQTT `CONNECT` packet received. Account session manager is referred to as `asm` +in the code. + +Note that creating the account session manager (and attempting to create the +streams) is done only once per account on a given server, since once created the +account session manager for a given account would be found in the sessions map +of the mqttSessionManager object. + +### Find and disconnect previous session/client + +Once all that is done, we now go to the creation of the session object itself. +For that, we first need to make sure that it does not already exist, meaning +that it is registered on the server - or anywhere in the cluster. Note that MQTT +dictates that if a session with the same ID connects, the OLD session needs to +be closed, not the new one being created. NATS Server complies with this +requirement. + +Once a session is detected to already exists, the old one (as described above) +is closed and the new one accepted, however, the session ID is maintained in a +flappers map so that we detect situations where sessions with the same ID are +started multiple times causing the previous one to be closed. When that +detection occurs, the newly created session is put in “jail” for a second to +avoid a very rapid succession of connect/disconnect. This has already been seen +by users since there was some issue there where we would schedule the connection +closed instead of waiting in place which was causing a panic. + +We also protect from multiple clients on a given server trying to connect with +the same ID at the “same time” while the processing of a CONNECT of a session is +not yet finished. This is done with the use of a sessLocked map, keyed by the +session ID. + +### Create or restore the session + +If everything is good up to that point, the server will either create or restore +a session from the stream. This is done in the `createOrRestoreSession()` +function. The client/session ID is hashed and added to the session’s stream +subject along with the JS domain to prevent clients connecting from different +domains to “pollute” the session stream of a given domain. + +Since each session constitutes a subject and the stream has a maximum of 1 +message per subject, we attempt to load the last message on the formed subject. +If we don’t find it, then the session object is created “empty”, while if we +find a record, we create the session object based on the record persisted on the +stream. + +If the session was restored from the JS stream, we keep track of the stream +sequence where the record was located. When we save the session (even if it +already exists) we will use this sequence number to set the +`JSExpectedLastSubjSeq` header so that we handle possibly different servers in a +(super)cluster to detect the race of clients trying to use the same session ID, +since only one of the write should succeed. On success, the session’s new +sequence is remembered by the server that did the write. + +When created or restored, the CONNACK can now be sent back to the client, and if +there were any recovered subscriptions, they are now processed. + +## Processing QoS acks: PUBACK, PUBREC, PUBCOMP + +When the server delivers a message with QoS 1 or 2 (also a `PUBREL` for QoS 2) to a subscribed client, the client will send back an acknowledgement. See `mqttProcessPubAck()`, `mqttProcessPubRec()`, and `mqttProcessPubComp()` + +While the specific logic for each packet differs, these handlers all update the +session's PI mappings (`cpending`, `pendingPublish`, `pendingPubRel`), and if +needed send an ACK to JetStream to remove the message from its consumer and stop +the re-delivery attempts. + +## Subject Wildcards + +Note that MQTT subscriptions have wildcards too, the `“+”` wildcard is equivalent +to NATS’s `“*”` wildcard, however, MQTT’s wildcard `“#”` is similar to `“>”`, except +that it also includes the level above. That is, a subscription on `“foo/#”` would +receive messages on `“foo/bar/baz”`, but also on `“foo”`. + +So, for MQTT subscriptions enging with a `'#'` we are forced to create 2 +internal NATS subscriptions, one on `“foo”` and one on `“foo.>”`. + +# 5. Known issues +- "active" redelivery for QoS from JetStream (compliant, just a note) +- JetStream QoS redelivery happens out of (original) order +- finish delivery of in-flight messages after UNSUB +- finish delivery of in-flight messages after a reconnect +- consider replacing `$MQTT_msgs` with `$MQTT_out`. +- consider using unique `$MQTT.rmsg.>` and `MaxMsgsPer` for retained messages. +- add a cli command to list/clean old sessions diff --git a/server/README.md b/server/README.md new file mode 100644 index 0000000..38e8621 --- /dev/null +++ b/server/README.md @@ -0,0 +1,17 @@ +# Tests + +Tests that run on Travis have been split into jobs that run in their own VM in parallel. This reduces the overall running time but also is allowing recycling of a job when we get a flapper as opposed to have to recycle the whole test suite. + +## JetStream Tests + +For JetStream tests, we need to observe a naming convention so that no tests are omitted when running on Travis. + +The script `runTestsOnTravis.sh` will run a given job based on the definition found in "`.travis.yml`". + +As for the naming convention: + +- All JetStream test name should start with `TestJetStream` +- Cluster tests should go into `jetstream_cluster_test.go` and start with `TestJetStreamCluster` +- Super-cluster tests should go into `jetstream_super_cluster_test.go` and start with `TestJetStreamSuperCluster` + +Not following this convention means that some tests may not be executed on Travis. diff --git a/server/archive/archive.go b/server/archive/archive.go new file mode 100644 index 0000000..a8485e3 --- /dev/null +++ b/server/archive/archive.go @@ -0,0 +1,356 @@ +// Copyright 2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package archive + +import ( + "bufio" + "encoding/binary" + "errors" + "io" +) + +const MagicBytes = "NATSARC1" + +// maxNameLen bounds the entry name length accepted when reading an archive, +// avoiding an unbounded allocation on corrupt or malicious input. The name is +// the only variable-length field read directly from the stream. +const maxNameLen = 1 << 20 + +var ( + ErrClosed = errors.New("archive: closed") + ErrInvalidArchive = errors.New("archive: invalid archive stream") + ErrIncompleteEntry = errors.New("archive: entry not fully written") + ErrNoActiveEntry = errors.New("archive: no active entry") + ErrWriteTooLong = errors.New("archive: write exceeds declared entry size") + ErrNilHeader = errors.New("archive: nil header") + ErrNegativeEntrySize = errors.New("archive: negative entry size") +) + +// Header describes one archive entry. +// +// On the wire each entry is the encoded header fields followed by the payload. +// HeaderSize and PayloadSize describe how that payload is split (e.g. message +// headers vs. body); the payload length is their sum and is not stored +// separately. Sequence is always encoded, with 0 meaning "unset". +type Header struct { + Name string + HeaderSize int64 + PayloadSize int64 + Timestamp int64 + Sequence uint64 +} + +func (h *Header) clone() *Header { + if h == nil { + return nil + } + c := *h + return &c +} + +// Writer writes a stream of archive entries. +type Writer struct { + w io.Writer + wroteMagic bool + header *Header + remaining int64 + closed bool +} + +// NewWriter returns a new stream writer. +func NewWriter(w io.Writer) *Writer { + return &Writer{w: w} +} + +// WriteHeader starts a new archive entry. +func (a *Writer) WriteHeader(hdr *Header) error { + if a.closed { + return ErrClosed + } + if hdr == nil { + return ErrNilHeader + } + if hdr.HeaderSize < 0 || hdr.PayloadSize < 0 { + return ErrNegativeEntrySize + } + if a.header != nil && a.remaining > 0 { + return ErrIncompleteEntry + } + totalPayloadSize := hdr.HeaderSize + hdr.PayloadSize + if totalPayloadSize < 0 { + return ErrNegativeEntrySize + } + if !a.wroteMagic { + if _, err := io.WriteString(a.w, MagicBytes); err != nil { + return err + } + a.wroteMagic = true + } + if err := a.writeUvarint(uint64(len(hdr.Name))); err != nil { + return err + } + if err := a.writeVarint(hdr.Timestamp); err != nil { + return err + } + if err := a.writeUvarint(hdr.Sequence); err != nil { + return err + } + if err := a.writeUvarint(uint64(hdr.HeaderSize)); err != nil { + return err + } + if err := a.writeUvarint(uint64(hdr.PayloadSize)); err != nil { + return err + } + if _, err := io.WriteString(a.w, hdr.Name); err != nil { + return err + } + a.header = hdr.clone() + a.remaining = totalPayloadSize + if a.remaining == 0 { + a.header = nil + } + return nil +} + +// Write writes payload bytes for the current entry. +func (a *Writer) Write(p []byte) (int, error) { + if a.closed { + return 0, ErrClosed + } + if len(p) == 0 { + return 0, nil + } + if a.header == nil { + return 0, ErrNoActiveEntry + } + wanted := len(p) + if int64(wanted) > a.remaining { + wanted = int(a.remaining) + } + n, err := a.w.Write(p[:wanted]) + a.remaining -= int64(n) + if err != nil { + return n, err + } + if n < wanted { + return n, io.ErrShortWrite + } + if len(p) > wanted { + if a.remaining == 0 { + a.header = nil + } + return n, ErrWriteTooLong + } + if a.remaining == 0 { + a.header = nil + } + return n, nil +} + +// Close finalizes the stream writer. +func (a *Writer) Close() error { + if a.closed { + return nil + } + a.closed = true + if a.header != nil && a.remaining > 0 { + return ErrIncompleteEntry + } + return nil +} + +// Flush flushes the underlying writer if it supports flushing. +func (a *Writer) Flush() error { + if a.closed { + return ErrClosed + } + if f, ok := a.w.(interface{ Flush() error }); ok { + return f.Flush() + } + if f, ok := a.w.(interface{ Flush() }); ok { + f.Flush() + } + return nil +} + +// Reader reads a stream of archive entries. +type Reader struct { + r *bufio.Reader + header *Header + remaining int64 + seenMagic bool + err error +} + +// NewReader returns a new stream reader. +func NewReader(r io.Reader) *Reader { + return &Reader{r: bufio.NewReader(r)} +} + +func (u *Reader) discardCurrent() error { + if u.remaining == 0 { + u.header = nil + return nil + } + _, err := io.CopyN(io.Discard, u.r, u.remaining) + u.remaining = 0 + u.header = nil + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return ErrInvalidArchive + } + return err +} + +// Next advances to the next entry and returns its header. +func (u *Reader) Next() (*Header, error) { + if u.err != nil { + return nil, u.err + } + if err := u.discardCurrent(); err != nil { + u.err = err + return nil, err + } + if !u.seenMagic { + magic := make([]byte, len(MagicBytes)) + if _, err := io.ReadFull(u.r, magic); err != nil { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + u.err = io.EOF + return nil, io.EOF + } + u.err = err + return nil, err + } + if string(magic) != MagicBytes { + u.err = ErrInvalidArchive + return nil, ErrInvalidArchive + } + u.seenMagic = true + } + // A clean io.EOF on the first field is the normal end of the stream. + nameLen, err := u.readUvarint() + if err != nil { + u.err = err + return nil, err + } + timestamp, err := u.readVarint() + if err != nil { + u.err = noEOF(err) + return nil, u.err + } + sequence, err := u.readUvarint() + if err != nil { + u.err = noEOF(err) + return nil, u.err + } + headerSize, err := u.readUvarint() + if err != nil { + u.err = noEOF(err) + return nil, u.err + } + payloadSize, err := u.readUvarint() + if err != nil { + u.err = noEOF(err) + return nil, u.err + } + // HeaderSize/PayloadSize are stored as int64: a value above MaxInt64 + // converts to a negative int64, and an overflowing sum lands negative too, + // so the sign checks below cover both cases. + hdrSize, plSize := int64(headerSize), int64(payloadSize) + total := hdrSize + plSize + if nameLen > maxNameLen || hdrSize < 0 || plSize < 0 || total < 0 { + u.err = ErrInvalidArchive + return nil, ErrInvalidArchive + } + name := make([]byte, nameLen) + if _, err := io.ReadFull(u.r, name); err != nil { + u.err = ErrInvalidArchive + return nil, ErrInvalidArchive + } + u.header = &Header{ + Name: string(name), + Timestamp: timestamp, + Sequence: sequence, + HeaderSize: hdrSize, + PayloadSize: plSize, + } + u.remaining = total + return u.header.clone(), nil +} + +// Read reads bytes from the current entry payload. +func (u *Reader) Read(p []byte) (int, error) { + if u.err != nil { + return 0, u.err + } + if len(p) == 0 { + return 0, nil + } + if u.header == nil || u.remaining == 0 { + return 0, io.EOF + } + + if int64(len(p)) > u.remaining { + p = p[:int(u.remaining)] + } + n, err := io.ReadFull(u.r, p) + u.remaining -= int64(n) + if err != nil { + u.err = ErrInvalidArchive + return n, ErrInvalidArchive + } + if u.remaining == 0 { + u.header = nil + } + return n, nil +} + +// noEOF converts a clean io.EOF into ErrInvalidArchive. It is used for header +// fields after the first: reaching EOF there means the entry was truncated +// rather than that the stream ended cleanly on an entry boundary. +func noEOF(err error) error { + if err != nil && errors.Is(err, io.EOF) { + return ErrInvalidArchive + } + return err +} + +func (a *Writer) writeVarint(v int64) error { + var tmp [binary.MaxVarintLen64]byte + n := binary.PutVarint(tmp[:], v) + _, err := a.w.Write(tmp[:n]) + return err +} + +func (a *Writer) writeUvarint(v uint64) error { + var tmp [binary.MaxVarintLen64]byte + n := binary.PutUvarint(tmp[:], v) + _, err := a.w.Write(tmp[:n]) + return err +} + +func (u *Reader) readVarint() (int64, error) { + v, err := binary.ReadVarint(u.r) + if err != nil && !errors.Is(err, io.EOF) { + return v, ErrInvalidArchive + } + return v, err +} + +func (u *Reader) readUvarint() (uint64, error) { + v, err := binary.ReadUvarint(u.r) + if err != nil && !errors.Is(err, io.EOF) { + return v, ErrInvalidArchive + } + return v, err +} diff --git a/server/archive/archive_test.go b/server/archive/archive_test.go new file mode 100644 index 0000000..ebbed28 --- /dev/null +++ b/server/archive/archive_test.go @@ -0,0 +1,244 @@ +// Copyright 2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package archive + +import ( + "bytes" + "errors" + "io" + "testing" +) + +func TestRoundTrip(t *testing.T) { + var buf bytes.Buffer + + w := NewWriter(&buf) + + want := []struct { + hdr Header + data string + }{ + { + hdr: Header{ + Name: "alpha.txt", + Timestamp: 100, + HeaderSize: 4, + PayloadSize: 7, + }, + data: "hello alpha", + }, + { + hdr: Header{ + Name: "nested/beta.txt", + Timestamp: 100, + Sequence: 42, + HeaderSize: 3, + PayloadSize: 9, + }, + data: "beta payload", + }, + } + + for i := range want { + if err := w.WriteHeader(&want[i].hdr); err != nil { + t.Fatalf("write header %d: %v", i, err) + } + if _, err := io.WriteString(w, want[i].data); err != nil { + t.Fatalf("write payload %d: %v", i, err) + } + } + if err := w.Close(); err != nil { + t.Fatalf("close writer: %v", err) + } + + r := NewReader(bytes.NewReader(buf.Bytes())) + + for i := range want { + hdr, err := r.Next() + if err != nil { + t.Fatalf("next %d: %v", i, err) + } + if hdr.Name != want[i].hdr.Name { + t.Fatalf("header %d name mismatch: got %q want %q", i, hdr.Name, want[i].hdr.Name) + } + if hdr.Timestamp != want[i].hdr.Timestamp { + t.Fatalf("header %d timestamp mismatch: got %d want %d", i, hdr.Timestamp, want[i].hdr.Timestamp) + } + if hdr.Sequence != want[i].hdr.Sequence { + t.Fatalf("header %d sequence mismatch: got %d want %d", i, hdr.Sequence, want[i].hdr.Sequence) + } + if hdr.HeaderSize != want[i].hdr.HeaderSize { + t.Fatalf("header %d header size mismatch: got %d want %d", i, hdr.HeaderSize, want[i].hdr.HeaderSize) + } + if hdr.PayloadSize != want[i].hdr.PayloadSize { + t.Fatalf("header %d payload size mismatch: got %d want %d", i, hdr.PayloadSize, want[i].hdr.PayloadSize) + } + got, err := io.ReadAll(r) + if err != nil { + t.Fatalf("read payload %d: %v", i, err) + } + if string(got) != want[i].data { + t.Fatalf("payload %d mismatch: got %q want %q", i, got, want[i].data) + } + } + + if _, err := r.Next(); !errors.Is(err, io.EOF) { + t.Fatalf("expected EOF, got %v", err) + } +} + +func TestNextSkipsUnreadPayload(t *testing.T) { + var buf bytes.Buffer + + w := NewWriter(&buf) + first := Header{Name: "first", HeaderSize: 0, PayloadSize: int64(len("abcdef"))} + if err := w.WriteHeader(&first); err != nil { + t.Fatalf("write first header: %v", err) + } + if _, err := io.WriteString(w, "abcdef"); err != nil { + t.Fatalf("write first payload: %v", err) + } + + second := Header{Name: "second", HeaderSize: 0, PayloadSize: int64(len("second-body"))} + if err := w.WriteHeader(&second); err != nil { + t.Fatalf("write second header: %v", err) + } + if _, err := io.WriteString(w, "second-body"); err != nil { + t.Fatalf("write second payload: %v", err) + } + + r := NewReader(bytes.NewReader(buf.Bytes())) + + hdr, err := r.Next() + if err != nil { + t.Fatalf("next first: %v", err) + } + if hdr.Name != "first" { + t.Fatalf("unexpected first header name %q", hdr.Name) + } + + buf1 := make([]byte, 3) + if _, err := io.ReadFull(r, buf1); err != nil { + t.Fatalf("read partial first payload: %v", err) + } + if string(buf1) != "abc" { + t.Fatalf("unexpected partial payload %q", buf1) + } + + hdr, err = r.Next() + if err != nil { + t.Fatalf("next second: %v", err) + } + if hdr.Name != "second" { + t.Fatalf("unexpected second header name %q", hdr.Name) + } + + got, err := io.ReadAll(r) + if err != nil { + t.Fatalf("read second payload: %v", err) + } + if string(got) != "second-body" { + t.Fatalf("unexpected second payload %q", got) + } +} + +// validArchive returns a well-formed archive holding a single entry whose +// payload is the 6-byte string "abcdef" split as HeaderSize=2, PayloadSize=4. +func validArchive(t *testing.T) []byte { + t.Helper() + var buf bytes.Buffer + w := NewWriter(&buf) + hdr := Header{Name: "first", Timestamp: 7, Sequence: 3, HeaderSize: 2, PayloadSize: 4} + if err := w.WriteHeader(&hdr); err != nil { + t.Fatalf("write header: %v", err) + } + if _, err := io.WriteString(w, "abcdef"); err != nil { + t.Fatalf("write payload: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("close: %v", err) + } + return buf.Bytes() +} + +func TestEmptyArchiveReturnsEOF(t *testing.T) { + r := NewReader(bytes.NewReader(nil)) + if _, err := r.Next(); !errors.Is(err, io.EOF) { + t.Fatalf("expected io.EOF on empty archive, got %v", err) + } +} + +func TestInvalidMagicIsRejected(t *testing.T) { + r := NewReader(bytes.NewReader([]byte("not-a-valid-archive-stream"))) + if _, err := r.Next(); !errors.Is(err, ErrInvalidArchive) { + t.Fatalf("expected ErrInvalidArchive, got %v", err) + } +} + +// A clean EOF on the first header field means end-of-stream, but an EOF +// partway through the header means the entry was truncated. +func TestTruncatedHeaderIsInvalid(t *testing.T) { + full := validArchive(t) + // Keep the magic plus the single name-length byte, dropping the rest. + r := NewReader(bytes.NewReader(full[:len(MagicBytes)+1])) + if _, err := r.Next(); !errors.Is(err, ErrInvalidArchive) { + t.Fatalf("expected ErrInvalidArchive, got %v", err) + } +} + +func TestTruncatedPayloadIsInvalid(t *testing.T) { + full := validArchive(t) + // Drop the last 3 payload bytes; the header still decodes cleanly. + r := NewReader(bytes.NewReader(full[:len(full)-3])) + hdr, err := r.Next() + if err != nil { + t.Fatalf("next: %v", err) + } + if hdr.HeaderSize+hdr.PayloadSize != 6 { + t.Fatalf("unexpected payload size %d", hdr.HeaderSize+hdr.PayloadSize) + } + if _, err := io.ReadAll(r); !errors.Is(err, ErrInvalidArchive) { + t.Fatalf("expected ErrInvalidArchive reading payload, got %v", err) + } +} + +// The header round-trips when its fields are read directly from the stream, +// including a large Sequence (uvarint) and a negative Timestamp (varint). +func TestHeaderFieldEncoding(t *testing.T) { + var buf bytes.Buffer + w := NewWriter(&buf) + want := Header{Name: "nested/x.dat", Timestamp: -123, Sequence: 1 << 60, HeaderSize: 0, PayloadSize: 0} + if err := w.WriteHeader(&want); err != nil { + t.Fatalf("write header: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + r := NewReader(bytes.NewReader(buf.Bytes())) + got, err := r.Next() + if err != nil { + t.Fatalf("next: %v", err) + } + if *got != want { + t.Fatalf("header round-trip mismatch: got %+v want %+v", *got, want) + } + // A zero-length payload yields immediate EOF, then a clean end-of-stream. + if n, err := r.Read(make([]byte, 4)); n != 0 || !errors.Is(err, io.EOF) { + t.Fatalf("expected (0, EOF) reading empty payload, got (%d, %v)", n, err) + } + if _, err := r.Next(); !errors.Is(err, io.EOF) { + t.Fatalf("expected io.EOF after last entry, got %v", err) + } +} diff --git a/server/ats/ats.go b/server/ats/ats.go new file mode 100644 index 0000000..50045bf --- /dev/null +++ b/server/ats/ats.go @@ -0,0 +1,86 @@ +// Copyright 2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ats controls the go routines for the access time service. +// This allows more efficient unixnano operations for cache access times. +// We will have one per binary (usually per server). +package ats + +import ( + "sync/atomic" + "time" +) + +// Update every 100ms for gathering access time in unix nano. +const TickInterval = 100 * time.Millisecond + +var ( + // Our unix nano time. + utime atomic.Int64 + // How may registered users do we have, controls lifetime of Go routine. + refs atomic.Int64 + // To signal the shutdown of the Go routine. + done chan struct{} +) + +func init() { + // Initialize our done chan. + done = make(chan struct{}, 1) +} + +// Register usage. This will happen on filestore creation. +func Register() { + if v := refs.Add(1); v == 1 { + // This is the first to register (could also go up and down), + // so spin up Go routine and grab initial time. + utime.Store(time.Now().UnixNano()) + + go func() { + ticker := time.NewTicker(TickInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + utime.Store(time.Now().UnixNano()) + case <-done: + return + } + } + }() + } +} + +// Unregister usage. We will shutdown the go routine if no more registered users. +func Unregister() { + if v := refs.Add(-1); v == 0 { + done <- struct{}{} + } else if v < 0 { + refs.Store(0) + panic("unbalanced unregister for access time state") + } +} + +// Will load the access time from an atomic. +// If no one has registered this will return 0 or stale data. +// It is the responsibility of the user to properly register and unregister. +func AccessTime() int64 { + // Return last updated time. + v := utime.Load() + if v == 0 { + // Always register a time, the worst case is a stale time. + // On startup, we can register in parallel and could previously panic. + v = time.Now().UnixNano() + utime.Store(v) + } + return v +} diff --git a/server/ats/ats_test.go b/server/ats/ats_test.go new file mode 100644 index 0000000..63451b6 --- /dev/null +++ b/server/ats/ats_test.go @@ -0,0 +1,98 @@ +// Copyright 2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ats + +import ( + "runtime" + "testing" + "time" +) + +func TestNotRunningValue(t *testing.T) { + // Set back to zero in case this test gets run multiple times via --count. + utime.Store(0) + at := AccessTime() + if at == 0 { + t.Fatal("Expected non-zero access time") + } + + atn := AccessTime() + if atn != at { + t.Fatal("Did not expect updates to access time") + } +} + +func TestRegisterAndUnregister(t *testing.T) { + ngrp := runtime.NumGoroutine() + + Register() + if r := refs.Load(); r != 1 { + t.Fatalf("Expected refs to be 1, got %d", r) + } + // Make sure we have a non-zero time. + at := AccessTime() + if at == 0 { + t.Fatal("Expected non-zero access time") + } + // Make sure we are updating access time. + time.Sleep(2 * TickInterval) + atn := AccessTime() + if atn == at { + t.Fatal("Expected access time to be updated but it was not") + } + + // Now unregister. + Unregister() + if r := refs.Load(); r != 0 { + t.Fatalf("Expected refs to be 0, got %d", r) + } + time.Sleep(TickInterval) + + at = AccessTime() + time.Sleep(2 * TickInterval) + atn = AccessTime() + if atn != at { + t.Fatal("Did not expect updates to access time") + } + + // Check that we have no additional go routines running. + ngra := runtime.NumGoroutine() + if ngra != ngrp { + t.Fatalf("Expected same number of go routines after removing all registered: %d vs %d", ngrp, ngra) + } + + // Check that we spin back up on going from zero to one registered after spinning down. + Register() + defer Unregister() + + at = AccessTime() + time.Sleep(2 * TickInterval) + atn = AccessTime() + if atn == at { + t.Fatal("Expected access time to be updated but it was not") + } + ngra = runtime.NumGoroutine() + if ngra != ngrp+1 { + t.Fatalf("Expected to see additional go routine: %d vs %d", ngrp, ngra) + } +} + +func TestUnbalancedUnregister(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("Expected function to panic, but it did not") + } + }() + Unregister() +} diff --git a/server/auth.go b/server/auth.go new file mode 100644 index 0000000..9e70aff --- /dev/null +++ b/server/auth.go @@ -0,0 +1,1837 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "crypto/sha256" + "crypto/subtle" + "crypto/tls" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/base64" + "encoding/hex" + "fmt" + "net" + "net/url" + "regexp" + "slices" + "strings" + "sync/atomic" + "time" + + "github.com/nats-io/jwt/v2" + "github.com/nats-io/nats-server/v2/internal/ldap" + "github.com/nats-io/nkeys" + "golang.org/x/crypto/bcrypt" +) + +// Authentication is an interface for implementing authentication +type Authentication interface { + // Check if a client is authorized to connect + Check(c ClientAuthentication) bool +} + +// ClientAuthentication is an interface for client authentication +type ClientAuthentication interface { + // GetOpts gets options associated with a client + GetOpts() *ClientOpts + // GetTLSConnectionState if TLS is enabled, TLS ConnectionState, nil otherwise + GetTLSConnectionState() *tls.ConnectionState + // RegisterUser optionally map a user after auth. + RegisterUser(*User) + // RemoteAddress expose the connection information of the client + RemoteAddress() net.Addr + // GetNonce is the nonce presented to the user in the INFO line + GetNonce() []byte + // Kind indicates what type of connection this is matching defined constants like CLIENT, ROUTER, GATEWAY, LEAF etc + Kind() int + //Gets the ID associated with a client + GetID() uint64 +} + +// NkeyUser is for multiple nkey based users +type NkeyUser struct { + Nkey string `json:"user"` + Issued int64 `json:"issued,omitempty"` // this is a copy of the issued at (iat) field in the jwt + Permissions *Permissions `json:"permissions,omitempty"` + Account *Account `json:"account,omitempty"` + SigningKey string `json:"signing_key,omitempty"` + AllowedConnectionTypes map[string]struct{} `json:"connection_types,omitempty"` + ProxyRequired bool `json:"proxy_required,omitempty"` + defaultPerms bool +} + +// User is for multiple accounts/users. +type User struct { + Username string `json:"user"` + Password string `json:"password"` + Permissions *Permissions `json:"permissions,omitempty"` + Account *Account `json:"account,omitempty"` + ConnectionDeadline time.Time `json:"connection_deadline,omitempty"` + AllowedConnectionTypes map[string]struct{} `json:"connection_types,omitempty"` + ProxyRequired bool `json:"proxy_required,omitempty"` +} + +// clone performs a deep copy of the User struct, returning a new clone with +// all values copied. +func (u *User) clone() *User { + if u == nil { + return nil + } + clone := &User{} + *clone = *u + // Account is not cloned because it is always by reference to an existing struct. + clone.Permissions = u.Permissions.clone() + + if u.AllowedConnectionTypes != nil { + clone.AllowedConnectionTypes = make(map[string]struct{}) + for k, v := range u.AllowedConnectionTypes { + clone.AllowedConnectionTypes[k] = v + } + } + + return clone +} + +// clone performs a deep copy of the NkeyUser struct, returning a new clone with +// all values copied. +func (n *NkeyUser) clone() *NkeyUser { + if n == nil { + return nil + } + clone := &NkeyUser{} + *clone = *n + // Account is not cloned because it is always by reference to an existing struct. + clone.Permissions = n.Permissions.clone() + + if n.AllowedConnectionTypes != nil { + clone.AllowedConnectionTypes = make(map[string]struct{}) + for k, v := range n.AllowedConnectionTypes { + clone.AllowedConnectionTypes[k] = v + } + } + + return clone +} + +// SubjectPermission is an individual allow and deny struct for publish +// and subscribe authorizations. +type SubjectPermission struct { + Allow []string `json:"allow,omitempty"` + Deny []string `json:"deny,omitempty"` +} + +// ResponsePermission can be used to allow responses to any reply subject +// that is received on a valid subscription. +type ResponsePermission struct { + MaxMsgs int `json:"max"` + Expires time.Duration `json:"ttl"` +} + +// Permissions are the allowed subjects on a per +// publish or subscribe basis. +type Permissions struct { + Publish *SubjectPermission `json:"publish"` + Subscribe *SubjectPermission `json:"subscribe"` + Response *ResponsePermission `json:"responses,omitempty"` +} + +// RoutePermissions are similar to user permissions +// but describe what a server can import/export from and to +// another server. +type RoutePermissions struct { + Import *SubjectPermission `json:"import"` + Export *SubjectPermission `json:"export"` +} + +// clone will clone an individual subject permission. +func (p *SubjectPermission) clone() *SubjectPermission { + if p == nil { + return nil + } + clone := &SubjectPermission{} + if p.Allow != nil { + clone.Allow = make([]string, len(p.Allow)) + copy(clone.Allow, p.Allow) + } + if p.Deny != nil { + clone.Deny = make([]string, len(p.Deny)) + copy(clone.Deny, p.Deny) + } + return clone +} + +// clone performs a deep copy of the Permissions struct, returning a new clone +// with all values copied. +func (p *Permissions) clone() *Permissions { + if p == nil { + return nil + } + clone := &Permissions{} + if p.Publish != nil { + clone.Publish = p.Publish.clone() + } + if p.Subscribe != nil { + clone.Subscribe = p.Subscribe.clone() + } + if p.Response != nil { + clone.Response = &ResponsePermission{ + MaxMsgs: p.Response.MaxMsgs, + Expires: p.Response.Expires, + } + } + return clone +} + +// wsCanMapCerts returns true if the websocket listener collects client +// certificates, which mqtt and leafnode connections over it need to be mapped. +func (o *Options) wsCanMapCerts() bool { + return o.Websocket.Port != 0 && o.Websocket.TLSConfig != nil && + o.Websocket.TLSConfig.ClientAuth >= tls.RequestClientCert +} + +// canTLSMap returns true if verify_and_map on any listener that uses the users +// table could select this user, which makes it a certificate identity. Cluster +// and gateway map onto their own configured user, so they do not count. +func (o *Options) canTLSMap(u *User) bool { + // The mapping lookup skips users not allowed on the connection type. + allowed := func(cts ...string) bool { + if len(u.AllowedConnectionTypes) == 0 { + return true + } + for _, ct := range cts { + if _, ok := u.AllowedConnectionTypes[ct]; ok { + return true + } + } + return false + } + // Only a listener that is started can map, and in-process connections have + // no TLS state, so they never do. + if o.TLSMap && !o.DontListen && allowed(jwt.ConnectionTypeStandard) { + return true + } + if o.Websocket.TLSMap && o.Websocket.Port != 0 && allowed(jwt.ConnectionTypeWebsocket) { + return true + } + // Mqtt over websocket maps with the mqtt listener, not the websocket one, + // but takes its certificate from the websocket transport. + if o.MQTT.TLSMap && o.MQTT.Port != 0 && + (allowed(jwt.ConnectionTypeMqtt) || + o.wsCanMapCerts() && allowed(jwt.ConnectionTypeMqttWS)) { + return true + } + // Leafnodes only use the users table when the leafnode block sets no + // credentials of its own, see isLeafNodeAuthorized. + return o.LeafNode.TLSMap && o.LeafNode.Port != 0 && + o.LeafNode.Username == _EMPTY_ && o.LeafNode.Nkey == _EMPTY_ && + len(o.LeafNode.Users) == 0 && + (allowed(jwt.ConnectionTypeLeafnode) || + o.wsCanMapCerts() && allowed(jwt.ConnectionTypeLeafnodeWS)) +} + +// checkAuthforWarnings will look for insecure settings and log concerns. +// Lock is assumed held. +func (s *Server) checkAuthforWarnings() { + warn := false + opts := s.getOpts() + if opts.Password != _EMPTY_ && !isBcrypt(opts.Password) { + warn = true + } + for _, u := range s.users { + // Skip warn if using TLS certs based auth + // unless a password has been left in the config. + if u.Password == _EMPTY_ && opts.canTLSMap(u) { + continue + } + // Check if this is our internal sys client created on the fly. + if s.sysAccOnlyNoAuthUser != _EMPTY_ && u.Username == s.sysAccOnlyNoAuthUser { + continue + } + if !isBcrypt(u.Password) { + warn = true + break + } + } + if warn { + // Warning about using plaintext passwords. + s.Warnf("Plaintext passwords detected, use nkeys or bcrypt") + } +} + +// If Users or Nkeys options have definitions without an account defined, +// assign them to the default global account. +// Lock should be held. +func (s *Server) assignGlobalAccountToOrphanUsers(nkeys map[string]*NkeyUser, users map[string]*User) { + for _, u := range users { + if u.Account == nil { + u.Account = s.gacc + } + } + for _, u := range nkeys { + if u.Account == nil { + u.Account = s.gacc + } + } +} + +// If the given permissions has a ResponsePermission +// set, ensure that defaults are set (if values are 0) +// and that a Publish permission is set, and Allow +// is disabled if not explicitly set. +func validateResponsePermissions(p *Permissions) { + if p == nil || p.Response == nil { + return + } + if p.Publish == nil { + p.Publish = &SubjectPermission{} + } + if p.Publish.Allow == nil { + // We turn off the blanket allow statement. + p.Publish.Allow = []string{} + } + // If there is a response permission, ensure + // that if value is 0, we set the default value. + if p.Response.MaxMsgs == 0 { + p.Response.MaxMsgs = DEFAULT_ALLOW_RESPONSE_MAX_MSGS + } + if p.Response.Expires == 0 { + p.Response.Expires = DEFAULT_ALLOW_RESPONSE_EXPIRATION + } +} + +// configureAuthorization will do any setup needed for authorization. +// Lock is assumed held. +func (s *Server) configureAuthorization() { + opts := s.getOpts() + if opts == nil { + return + } + + // Check for multiple users first + // This just checks and sets up the user map if we have multiple users. + if opts.CustomClientAuthentication != nil { + s.info.AuthRequired = true + } else if s.trustedKeys != nil { + s.info.AuthRequired = true + } else if opts.Nkeys != nil || opts.Users != nil { + s.nkeys, s.users = s.buildNkeysAndUsersFromOptions(opts.Nkeys, opts.Users) + s.info.AuthRequired = true + } else if opts.Username != _EMPTY_ || opts.Authorization != _EMPTY_ { + s.info.AuthRequired = true + } else { + s.users = nil + s.nkeys = nil + s.info.AuthRequired = false + } + + // Do similar for websocket config + s.wsConfigAuth(&opts.Websocket) + // And for mqtt config + s.mqttConfigAuth(&opts.MQTT) + + // Check for server configured auth callouts. + if opts.AuthCallout != nil { + s.mu.Unlock() + // Give operator log entries if not valid account and auth_users. + _, err := s.lookupAccount(opts.AuthCallout.Account) + s.mu.Lock() + if err != nil { + s.Errorf("Authorization callout account %q not valid", opts.AuthCallout.Account) + } + for _, u := range opts.AuthCallout.AuthUsers { + // Check for user in users and nkeys since this is server config. + var found bool + if len(s.users) > 0 { + _, found = s.users[u] + } + if !found && len(s.nkeys) > 0 { + _, found = s.nkeys[u] + } + if !found { + s.Errorf("Authorization callout user %q not valid: %v", u, err) + } + } + } +} + +// Takes the given slices of NkeyUser and User options and build +// corresponding maps used by the server. The users are cloned +// so that server does not reference options. +// The global account is assigned to users that don't have an +// existing account. +// Server lock is held on entry. +func (s *Server) buildNkeysAndUsersFromOptions(nko []*NkeyUser, uo []*User) (map[string]*NkeyUser, map[string]*User) { + var nkeys map[string]*NkeyUser + var users map[string]*User + + if nko != nil { + nkeys = make(map[string]*NkeyUser, len(nko)) + for _, u := range nko { + copy := u.clone() + if u.Account != nil { + if v, ok := s.accounts.Load(u.Account.Name); ok { + copy.Account = v.(*Account) + } + } + if copy.Permissions != nil { + validateResponsePermissions(copy.Permissions) + } + nkeys[u.Nkey] = copy + } + } + if uo != nil { + users = make(map[string]*User, len(uo)) + for _, u := range uo { + copy := u.clone() + if u.Account != nil { + if v, ok := s.accounts.Load(u.Account.Name); ok { + copy.Account = v.(*Account) + } + } + if copy.Permissions != nil { + validateResponsePermissions(copy.Permissions) + } + users[u.Username] = copy + } + } + s.assignGlobalAccountToOrphanUsers(nkeys, users) + return nkeys, users +} + +// checkAuthentication will check based on client type and +// return boolean indicating if client is authorized. +func (s *Server) checkAuthentication(c *client) bool { + switch c.kind { + case CLIENT: + return s.isClientAuthorized(c) + case ROUTER: + return s.isRouterAuthorized(c) + case GATEWAY: + return s.isGatewayAuthorized(c) + case LEAF: + return s.isLeafNodeAuthorized(c) + default: + return false + } +} + +// isClientAuthorized will check the client against the proper authorization method and data. +// This could be nkey, token, or username/password based. +func (s *Server) isClientAuthorized(c *client) bool { + opts := s.getOpts() + + // Check custom auth first, then jwts, then nkeys, then + // multiple users with TLS map if enabled, then token, + // then single user/pass. + if opts.CustomClientAuthentication != nil && !opts.CustomClientAuthentication.Check(c) { + return false + } + + if opts.CustomClientAuthentication == nil && !s.processClientOrLeafAuthentication(c, opts) { + return false + } + + if c.kind == CLIENT || c.kind == LEAF { + // Generate an event if we have a system account. + s.accountConnectEvent(c) + } + + return true +} + +// returns false if the client needs to be disconnected +func (c *client) matchesPinnedCert(tlsPinnedCerts PinnedCertSet) bool { + if tlsPinnedCerts == nil { + return true + } + tlsState := c.GetTLSConnectionState() + if tlsState == nil || len(tlsState.PeerCertificates) == 0 || tlsState.PeerCertificates[0] == nil { + c.Debugf("Failed pinned cert test as client did not provide a certificate") + return false + } + sha := sha256.Sum256(tlsState.PeerCertificates[0].RawSubjectPublicKeyInfo) + keyId := hex.EncodeToString(sha[:]) + if _, ok := tlsPinnedCerts[keyId]; !ok { + c.Debugf("Failed pinned cert test for key id: %s", keyId) + return false + } + return true +} + +var ( + mustacheRE = regexp.MustCompile(`{{2}([^}]+)}{2}`) + maxPermTemplateSubjectExpansions = 4096 + errPermTemplateExpansionLimit error = fmt.Errorf("template expansion exceeds limit") +) + +func processUserPermissionsTemplate(lim jwt.UserPermissionLimits, ujwt *jwt.UserClaims, acc *Account) (jwt.UserPermissionLimits, error) { + nArrayCartesianProduct := func(a ...[]string) [][]string { + c := 1 + for _, a := range a { + c *= len(a) + } + if c == 0 { + return nil + } + p := make([][]string, c) + b := make([]string, c*len(a)) + n := make([]int, len(a)) + s := 0 + for i := range p { + e := s + len(a) + pi := b[s:e] + p[i] = pi + s = e + for j, n := range n { + pi[j] = a[j][n] + } + for j := len(n) - 1; j >= 0; j-- { + n[j]++ + if n[j] < len(a[j]) { + break + } + n[j] = 0 + } + } + return p + } + isTag := func(op string) []string { + if len(op) >= 4 && strings.EqualFold("tag(", op[:4]) && strings.HasSuffix(op, ")") { + v := strings.TrimPrefix(op, "tag(") + v = strings.TrimSuffix(v, ")") + return []string{"tag", v} + } else if len(op) >= 12 && strings.EqualFold("account-tag(", op[:12]) && strings.HasSuffix(op, ")") { + v := strings.TrimPrefix(op, "account-tag(") + v = strings.TrimSuffix(v, ")") + return []string{"account-tag", v} + } + return nil + } + applyTemplate := func(list jwt.StringList, failOnBadSubject bool) (jwt.StringList, error) { + found := false + FOR_FIND: + for i := 0; i < len(list); i++ { + // check if templates are present + if mustacheRE.MatchString(list[i]) { + found = true + break FOR_FIND + } + } + if !found { + return list, nil + } + // process the templates + emittedList := make([]string, 0, len(list)) + for i := 0; i < len(list); i++ { + // find all the templates {{}} in this acl + tokens := mustacheRE.FindAllString(list[i], -1) + srcs := make([]string, len(tokens)) + values := make([][]string, len(tokens)) + hasTags := false + for tokenNum, tk := range tokens { + srcs[tokenNum] = tk + op := strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(tk, "{{"), "}}")) + if strings.EqualFold("name()", op) { + values[tokenNum] = []string{ujwt.Name} + } else if strings.EqualFold("subject()", op) { + values[tokenNum] = []string{ujwt.Subject} + } else if strings.EqualFold("account-name()", op) { + acc.mu.RLock() + values[tokenNum] = []string{acc.nameTag} + acc.mu.RUnlock() + } else if strings.EqualFold("account-subject()", op) { + // this always has an issuer account since this is a scoped signer + values[tokenNum] = []string{ujwt.IssuerAccount} + } else if isTag(op) != nil { + hasTags = true + match := isTag(op) + var tags jwt.TagList + if match[0] == "account-tag" { + acc.mu.RLock() + tags = acc.tags + acc.mu.RUnlock() + } else { + tags = ujwt.Tags + } + tagPrefix := fmt.Sprintf("%s:", strings.ToLower(match[1])) + var valueList []string + for _, tag := range tags { + if strings.HasPrefix(tag, tagPrefix) { + tagValue := strings.TrimPrefix(tag, tagPrefix) + valueList = append(valueList, tagValue) + } + } + if len(valueList) != 0 { + values[tokenNum] = valueList + } else if failOnBadSubject { + return nil, fmt.Errorf("generated invalid subject %q: %q is not defined", list[i], match[1]) + } else { + // generate an invalid subject? + values[tokenNum] = []string{" "} + } + } else { + return nil, fmt.Errorf("template operation in %q: %q is not defined", list[i], op) + } + } + if !hasTags { + subj := list[i] + for idx, m := range srcs { + subj = strings.Replace(subj, m, values[idx][0], -1) + } + if IsValidSubject(subj) { + emittedList = append(emittedList, subj) + } else if failOnBadSubject { + return nil, fmt.Errorf("generated invalid subject") + } + } else { + expCount := 1 + for _, v := range values { + if len(v) == 0 { + expCount = 0 + break + } + if expCount > maxPermTemplateSubjectExpansions/len(v) { + return nil, fmt.Errorf("%w: %d", errPermTemplateExpansionLimit, maxPermTemplateSubjectExpansions) + } + expCount *= len(v) + } + if len(emittedList) > maxPermTemplateSubjectExpansions-expCount { + return nil, fmt.Errorf("%w: %d", errPermTemplateExpansionLimit, maxPermTemplateSubjectExpansions) + } + a := nArrayCartesianProduct(values...) + for _, aa := range a { + subj := list[i] + for j := 0; j < len(srcs); j++ { + subj = strings.Replace(subj, srcs[j], aa[j], -1) + } + if IsValidSubject(subj) { + emittedList = append(emittedList, subj) + } else if failOnBadSubject { + return nil, fmt.Errorf("generated invalid subject") + } + } + } + } + return emittedList, nil + } + + subAllowWasNotEmpty := len(lim.Permissions.Sub.Allow) > 0 + pubAllowWasNotEmpty := len(lim.Permissions.Pub.Allow) > 0 + + var err error + if lim.Permissions.Sub.Allow, err = applyTemplate(lim.Permissions.Sub.Allow, false); err != nil { + return jwt.UserPermissionLimits{}, err + } else if lim.Permissions.Sub.Deny, err = applyTemplate(lim.Permissions.Sub.Deny, true); err != nil { + return jwt.UserPermissionLimits{}, err + } else if lim.Permissions.Pub.Allow, err = applyTemplate(lim.Permissions.Pub.Allow, false); err != nil { + return jwt.UserPermissionLimits{}, err + } else if lim.Permissions.Pub.Deny, err = applyTemplate(lim.Permissions.Pub.Deny, true); err != nil { + return jwt.UserPermissionLimits{}, err + } + + // if pub/sub allow were not empty, but are empty post template processing, add in a "deny >" to compensate + if subAllowWasNotEmpty && len(lim.Permissions.Sub.Allow) == 0 { + lim.Permissions.Sub.Deny.Add(">") + } + if pubAllowWasNotEmpty && len(lim.Permissions.Pub.Allow) == 0 { + lim.Permissions.Pub.Deny.Add(">") + } + return lim, nil +} + +func (s *Server) processClientOrLeafAuthentication(c *client, opts *Options) (authorized bool) { + var ( + nkey *NkeyUser + ujwt string + juc *jwt.UserClaims + acc *Account + user *User + ok bool + err error + ao bool // auth override + ) + + // Little helper that will log the error as a debug statement, set the auth error in + // the connection and return false to indicate authentication failure. + setProxyAuthError := func(err error) bool { + c.Debugf(err.Error()) + c.setAuthError(err) + return false + } + + // Indicate if this connection came from a trusted proxy. Note that if + // trustedProxy could be false even if the connection is proxied, but it + // means that there was no trusted proxy configured. + trustedProxy, ok := s.proxyCheck(c, opts) + if trustedProxy && !ok { + return setProxyAuthError(ErrAuthProxyNotTrusted) + } + + var proxyRequired bool + // Check if we have auth callouts enabled at the server level or in the bound account. + defer func() { + authErr := c.getAuthError() + if authErr == nil { + authErr = ErrAuthentication + } + reason := getAuthErrClosedState(authErr).String() + // No-op + if juc == nil && opts.AuthCallout == nil { + if !authorized { + s.sendAccountAuthErrorEvent(c, c.acc, reason) + } + return + } + // We have a juc, check if externally managed, i.e. should be delegated + // to the auth callout service. + if juc != nil && !acc.hasExternalAuth() { + if !authorized { + s.sendAccountAuthErrorEvent(c, c.acc, reason) + } + return + } + // Check config-mode. The global account is a condition since users that + // are not found in the config are implicitly bound to the global account. + // This means those users should be implicitly delegated to auth callout + // if configured. Exclude LEAF connections from this check. + if c.kind != LEAF && juc == nil && opts.AuthCallout != nil && c.acc.Name != globalAccountName { + // If no allowed accounts are defined, then all accounts are in scope. + // Otherwise see if the account is in the list. + delegated := len(opts.AuthCallout.AllowedAccounts) == 0 + if !delegated { + for _, n := range opts.AuthCallout.AllowedAccounts { + if n == c.acc.Name { + delegated = true + break + } + } + } + + // Not delegated, so return with previous authorized result. + if !delegated { + if !authorized { + s.sendAccountAuthErrorEvent(c, c.acc, reason) + } + return + } + } + + // We have auth callout set here. + var skip bool + // Check if we are on the list of auth_users. + userID := c.getRawAuthUser() + if juc != nil { + skip = acc.isExternalAuthUser(userID) + } else { + for _, u := range opts.AuthCallout.AuthUsers { + if userID == u { + skip = true + break + } + } + } + + // If we are here we have an auth callout defined and we have failed auth so far + // so we will callout to our auth backend for processing. + if !skip { + authorized, reason = s.processClientOrLeafCallout(c, opts, proxyRequired, trustedProxy, ujwt) + } + // Check if we are authorized and in the auth callout account, and if so add in deny publish permissions for the auth subject. + if authorized { + var authAccountName string + if juc == nil && opts.AuthCallout != nil { + authAccountName = opts.AuthCallout.Account + } else if juc != nil { + authAccountName = acc.Name + } + c.mu.Lock() + if c.acc != nil && c.acc.Name == authAccountName { + c.mergeDenyPermissions(pub, []string{AuthCalloutSubject}) + } + c.mu.Unlock() + } else { + // If we are here we failed external authorization. + // Send an account scoped event. Server config mode acc will be nil, + // so lookup the auth callout assigned account, that is where this will be sent. + if acc == nil { + acc, _ = s.lookupAccount(opts.AuthCallout.Account) + } + s.sendAccountAuthErrorEvent(c, acc, reason) + } + }() + + s.mu.Lock() + authRequired := s.info.AuthRequired + if !authRequired { + // If no auth required for regular clients, then check if + // we have an override for MQTT or Websocket clients. + switch c.clientType() { + case MQTT: + authRequired = s.mqtt.authOverride + case WS: + authRequired = s.websocket.authOverride + } + } + if !authRequired { + // TODO(dlc) - If they send us credentials should we fail? + s.mu.Unlock() + if c.kind == LEAF { + // Auth is not required, register the leaf node with the selected account. + // Otherwise, auth needs to match client auth. + return s.registerLeafWithAccount(c, opts.LeafNode.Account) + } + return true + } + var ( + username string + password string + token string + noAuthUser string + pinnedAcounts map[string]struct{} + ) + tlsMap := opts.TLSMap + if c.kind == CLIENT { + switch c.clientType() { + case MQTT: + mo := &opts.MQTT + // Always override TLSMap. + tlsMap = mo.TLSMap + // The rest depends on if there was any auth override in + // the mqtt's config. + if s.mqtt.authOverride { + noAuthUser = mo.NoAuthUser + username = mo.Username + password = mo.Password + token = mo.Token + ao = true + } + case WS: + wo := &opts.Websocket + // Always override TLSMap. + tlsMap = wo.TLSMap + // The rest depends on if there was any auth override in + // the websocket's config. + if s.websocket.authOverride { + noAuthUser = wo.NoAuthUser + username = wo.Username + password = wo.Password + token = wo.Token + ao = true + } + } + } else { + tlsMap = opts.LeafNode.TLSMap + } + + if !ao { + noAuthUser = opts.NoAuthUser + // If a leaf connects using websocket, and websocket{} block has a no_auth_user + // use that one instead. + if c.kind == LEAF && c.isWebsocket() && opts.Websocket.NoAuthUser != _EMPTY_ { + noAuthUser = opts.Websocket.NoAuthUser + } + username = opts.Username + password = opts.Password + token = opts.Authorization + } + + // MQTT can carry JWTs in the password field. Reconstruct it here for auth + // processing and auth callout, but do not populate c.opts.JWT yet or it would + // be exposed through monitoring and advisory paths even when the password is + // not actually a JWT. + if ujwt == _EMPTY_ && c.isMqtt() && c.opts.JWT == _EMPTY_ { + // Don't set juc here, leave that to the next s.trustedKeys != nil block, + // so that we don't try to trust a JWT when we aren't in operator mode. We + // will allow it to be passed through auth callout though. + if _, err := jwt.DecodeUserClaims(c.opts.Password); err == nil { + ujwt = c.opts.Password + } + } + + // Check if we have trustedKeys defined in the server. If so we require a user jwt. + if s.trustedKeys != nil { + if ujwt == _EMPTY_ { + // Need to be sure that it's a NATS JWT, otherwise we will not correctly + // attempt the default sentinel below. + if _, err = jwt.DecodeUserClaims(c.opts.JWT); err == nil { + ujwt = c.opts.JWT + } + } + if ujwt == _EMPTY_ { + // Didn't fall through with a valid NATS JWT, so try the default sentinel + // if configured. + if opts.DefaultSentinel != _EMPTY_ { + c.opts.JWT = opts.DefaultSentinel + ujwt = c.opts.JWT + } + } + if ujwt == _EMPTY_ { + s.mu.Unlock() + c.Debugf("Authentication requires a user JWT") + return false + } + if juc, err = jwt.DecodeUserClaims(ujwt); err != nil { + s.mu.Unlock() + c.Debugf("User JWT not valid: %v", err) + return false + } + if proxyRequired = juc.ProxyRequired; proxyRequired && !trustedProxy { + s.mu.Unlock() + return setProxyAuthError(ErrAuthProxyRequired) + } + vr := jwt.CreateValidationResults() + juc.Validate(vr) + if vr.IsBlocking(true) { + s.mu.Unlock() + c.Debugf("User JWT no longer valid: %+v", vr) + return false + } + pinnedAcounts = opts.resolverPinnedAccounts + } + + // Check if we have nkeys or users for client. + hasNkeys := len(s.nkeys) > 0 + hasUsers := len(s.users) > 0 + if hasNkeys { + if (c.kind == CLIENT || c.kind == LEAF) && noAuthUser != _EMPTY_ && + c.opts.Username == _EMPTY_ && c.opts.Password == _EMPTY_ && c.opts.Token == _EMPTY_ && c.opts.Nkey == _EMPTY_ { + if _, exists := s.nkeys[noAuthUser]; exists { + c.mu.Lock() + c.opts.Nkey = noAuthUser + c.mu.Unlock() + } + } + if c.opts.Nkey != _EMPTY_ { + nkey, ok = s.nkeys[c.opts.Nkey] + if !ok || !c.connectionTypeAllowed(nkey.AllowedConnectionTypes) { + s.mu.Unlock() + return false + } + } + } + if hasUsers && nkey == nil { + // Check if we are tls verify and are mapping users from the client_certificate. + if tlsMap { + authorized := checkClientTLSCertSubject(c, func(u string, certDN *ldap.DN, _ bool) (string, bool) { + // First do literal lookup using the resulting string representation + // of RDNSequence as implemented by the pkix package from Go. + if u != _EMPTY_ { + usr, ok := s.users[u] + if !ok || !c.connectionTypeAllowed(usr.AllowedConnectionTypes) { + return _EMPTY_, false + } + user = usr + return usr.Username, true + } + + if certDN == nil { + return _EMPTY_, false + } + + // Look through the accounts for a DN that is equal to the one + // presented by the certificate. + dns := make(map[*User]*ldap.DN) + for _, usr := range s.users { + if !c.connectionTypeAllowed(usr.AllowedConnectionTypes) { + continue + } + // TODO: Use this utility to make a full validation pass + // on start in case tlsmap feature is being used. + inputDN, err := ldap.ParseDN(usr.Username) + if err != nil { + continue + } + if inputDN.Equal(certDN) { + user = usr + return usr.Username, true + } + + // In case it did not match exactly, then collect the DNs + // and try to match later in case the DN was reordered. + dns[usr] = inputDN + } + + // Check in case the DN was reordered. + for usr, inputDN := range dns { + if inputDN.RDNsMatch(certDN) { + user = usr + return usr.Username, true + } + } + return _EMPTY_, false + }) + if !authorized { + s.mu.Unlock() + return false + } + if c.opts.Username != _EMPTY_ { + s.Warnf("User %q found in connect proto, but user required from cert", c.opts.Username) + } + // Already checked that the client didn't send a user in connect + // but we set it here to be able to identify it in the logs. + c.opts.Username = user.Username + } else { + if (c.kind == CLIENT || c.kind == LEAF) && noAuthUser != _EMPTY_ && + c.opts.Username == _EMPTY_ && c.opts.Password == _EMPTY_ && c.opts.Token == _EMPTY_ { + if u, exists := s.users[noAuthUser]; exists { + c.mu.Lock() + c.opts.Username = u.Username + c.opts.Password = u.Password + c.mu.Unlock() + } + } + if c.opts.Username != _EMPTY_ { + user, ok = s.users[c.opts.Username] + if !ok || !c.connectionTypeAllowed(user.AllowedConnectionTypes) { + s.mu.Unlock() + return false + } + } + } + } + s.mu.Unlock() + + // If we have a jwt and a userClaim, make sure we have the Account, etc associated. + // We need to look up the account. This will use an account resolver if one is present. + if juc != nil { + issuer := juc.Issuer + if juc.IssuerAccount != _EMPTY_ { + issuer = juc.IssuerAccount + } + if pinnedAcounts != nil { + if _, ok := pinnedAcounts[issuer]; !ok { + c.Debugf("Account %s not listed as operator pinned account", issuer) + atomic.AddUint64(&s.pinnedAccFail, 1) + return false + } + } + if acc, err = s.LookupAccount(issuer); acc == nil { + c.Debugf("Account JWT lookup error: %v", err) + return false + } + acc.mu.RLock() + aissuer := acc.Issuer + acc.mu.RUnlock() + if !s.isTrustedIssuer(aissuer) { + c.Debugf("Account JWT not signed by trusted operator") + return false + } + if scope, ok := acc.hasIssuer(juc.Issuer); !ok { + c.Debugf("User JWT issuer is not known") + return false + } else if scope != nil { + if err := scope.ValidateScopedSigner(juc); err != nil { + c.Debugf("User JWT is not valid: %v", err) + return false + } else if uSc, ok := scope.(*jwt.UserScope); !ok { + c.Debugf("User JWT is not valid") + return false + } else if juc.UserPermissionLimits, err = processUserPermissionsTemplate(uSc.Template, juc, acc); err != nil { + c.Debugf("User JWT generated invalid permissions") + return false + } + } + if acc.IsExpired() { + c.Debugf("Account JWT has expired") + return false + } + if juc.BearerToken && acc.failBearer() { + c.Debugf("Account does not allow bearer tokens") + return false + } + // We check the allowed connection types, but only after processing + // of scoped signer (so that it updates `juc` with what is defined + // in the account. + allowedConnTypes, err := convertAllowedConnectionTypes(juc.AllowedConnectionTypes) + if err != nil { + // We got an error, which means some connection types were unknown. As long as + // a valid one is returned, we proceed with auth. If not, we have to reject. + // In other words, suppose that JWT allows "WEBSOCKET" in the array. No error + // is returned and allowedConnTypes will contain "WEBSOCKET" only. + // Client will be rejected if not a websocket client, or proceed with rest of + // auth if it is. + // Now suppose JWT allows "WEBSOCKET, MQTT" and say MQTT is not known by this + // server. In this case, allowedConnTypes would contain "WEBSOCKET" and we + // would get `err` indicating that "MQTT" is an unknown connection type. + // If a websocket client connects, it should still be allowed, since after all + // the admin wanted to allow websocket and mqtt connection types. + // However, say that the JWT only allows "MQTT" (and again suppose this server + // does not know about MQTT connection type), then since the allowedConnTypes + // map would be empty (no valid types found), and since empty means allow-all, + // then we should reject because the intent was to allow connections for this + // user only as an MQTT client. + c.Debugf("%v", err) + if len(allowedConnTypes) == 0 { + return false + } + } + if !c.connectionTypeAllowed(allowedConnTypes) { + c.Debugf("Connection type not allowed") + return false + } + // Skip validation of nonce when presented with a bearer token. + // While support for bearer tokens was added for WebSockets, there is no + // security benefit in restricting their use to that client protocol: the + // client can just go use the other protocol. + if !juc.BearerToken { + // Verify the signature against the nonce. + if c.opts.Sig == _EMPTY_ { + c.Debugf("Signature missing") + return false + } + sig, err := base64.RawURLEncoding.DecodeString(c.opts.Sig) + if err != nil { + // Allow fallback to normal base64. + sig, err = base64.StdEncoding.DecodeString(c.opts.Sig) + if err != nil { + c.Debugf("Signature not valid base64") + return false + } + } + pub, err := nkeys.FromPublicKey(juc.Subject) + if err != nil { + c.Debugf("User nkey not valid: %v", err) + return false + } + if err := pub.Verify(c.nonce, sig); err != nil { + c.Debugf("Signature not verified") + return false + } + } + if acc.checkUserRevoked(juc.Subject, juc.IssuedAt) { + c.Debugf("User authentication revoked") + return false + } + if !validateSrc(juc, c.host) { + c.Errorf("Bad src Ip %s", c.host) + return false + } + allowNow, validFor := validateTimes(juc) + if !allowNow { + c.Errorf("Outside connect times") + return false + } + + nkey = buildInternalNkeyUser(juc, allowedConnTypes, acc) + if err := c.RegisterNkeyUser(nkey); err != nil { + return false + } + + // Warn about JetStream restrictions + if c.perms != nil { + deniedPub := []string{} + deniedSub := []string{} + for _, sub := range denyAllJs { + if c.perms.pub.deny != nil { + if c.perms.pub.deny.HasInterest(sub) { + deniedPub = append(deniedPub, sub) + } + } + if c.perms.sub.deny != nil { + if c.perms.sub.deny.HasInterest(sub) { + deniedSub = append(deniedSub, sub) + } + } + } + if len(deniedPub) > 0 || len(deniedSub) > 0 { + c.Debugf("Connected %s has JetStream denied on pub: %v sub: %v", c.kindString(), deniedPub, deniedSub) + } + } + + // Hold onto the user's public key. + c.mu.Lock() + c.pubKey = juc.Subject + // If this is a MQTT client, we purposefully didn't populate the JWT as it could contain + // a password or token. Now we know it's a valid JWT, we can populate it. + if c.isMqtt() { + c.opts.JWT = ujwt + } + c.tags = juc.Tags + c.nameTag = juc.Name + c.mu.Unlock() + + // Check if we need to set an auth timer if the user jwt expires. + c.setExpiration(juc.Claims(), validFor) + + acc.mu.RLock() + c.Debugf("Authenticated JWT: %s %q (claim-name: %q, claim-tags: %q) "+ + "signed with %q by Account %q (claim-name: %q, claim-tags: %q) signed with %q has mappings %t accused %p", + c.kindString(), juc.Subject, juc.Name, juc.Tags, juc.Issuer, issuer, acc.nameTag, acc.tags, acc.Issuer, acc.hasMappings(), acc) + acc.mu.RUnlock() + return true + } + + if nkey != nil { + if proxyRequired = nkey.ProxyRequired; proxyRequired && !trustedProxy { + return setProxyAuthError(ErrAuthProxyRequired) + } + // If we did not match noAuthUser check signature which is required. + if nkey.Nkey != noAuthUser { + if c.opts.Sig == _EMPTY_ { + c.Debugf("Signature missing") + return false + } + sig, err := base64.RawURLEncoding.DecodeString(c.opts.Sig) + if err != nil { + // Allow fallback to normal base64. + sig, err = base64.StdEncoding.DecodeString(c.opts.Sig) + if err != nil { + c.Debugf("Signature not valid base64") + return false + } + } + pub, err := nkeys.FromPublicKey(c.opts.Nkey) + if err != nil { + c.Debugf("User nkey not valid: %v", err) + return false + } + if err := pub.Verify(c.nonce, sig); err != nil { + c.Debugf("Signature not verified") + return false + } + } + if err := c.RegisterNkeyUser(nkey); err != nil { + return false + } + return true + } + if user != nil { + if proxyRequired = user.ProxyRequired; proxyRequired && !trustedProxy { + return setProxyAuthError(ErrAuthProxyRequired) + } + // A user with no password is a certificate only identity when mapping is + // enabled on another listener, so do not let comparePasswords match two + // empty passwords. The no_auth_user needs no credentials either way. + if !tlsMap && user.Password == _EMPTY_ && user.Username != noAuthUser && opts.canTLSMap(user) { + c.Debugf("User %q requires a client certificate", user.Username) + return false + } + ok = comparePasswords(user.Password, c.opts.Password) + // If we are authorized, register the user which will properly setup any permissions + // for pub/sub authorizations. + if ok { + c.RegisterUser(user) + } + return ok + } + + // Check for the use of simple auth. + if c.kind == CLIENT || c.kind == LEAF { + if proxyRequired = opts.ProxyRequired; proxyRequired && !trustedProxy { + return setProxyAuthError(ErrAuthProxyRequired) + } + if token != _EMPTY_ { + return comparePasswords(token, c.opts.Token) + } else if username != _EMPTY_ { + if username != c.opts.Username { + return false + } + return comparePasswords(password, c.opts.Password) + } + } + return false +} + +// If there are configured trusted proxies and this connection comes +// from a proxy whose signature can be verified by one of the known +// trusted key, this function will return `true, true`. If the signature +// cannot be verified by any, it will return `true, false`. +// If the connection is not proxied, or there are no configured trusted +// proxies, then this function returns `false, false`. +// +// Server lock MUST NOT be held on entry since this function will grab +// the read lock to extract the list of proxy trusted keys. The signature +// verification process will be done outside of the lock. +func (s *Server) proxyCheck(c *client, opts *Options) (bool, bool) { + // If there is no signature or no configured trusted proxy, return false. + psig := c.opts.ProxySig + if psig == _EMPTY_ || opts.Proxies == nil || len(opts.Proxies.Trusted) == 0 { + return false, false + } + // Decode the signature. + sig, err := base64.RawURLEncoding.DecodeString(psig) + if err != nil { + c.Debugf("Proxy signature not valid base64") + return true, false + } + // Go through the trusted keys and verify the signature. + s.mu.RLock() + keys := slices.Clone(s.proxiesKeyPairs) + s.mu.RUnlock() + for _, kp := range keys { + // We stop at the first that is valid. + if err := kp.Verify(c.nonce, sig); err == nil { + pub, _ := kp.PublicKey() + // Need to hold both locks so the proxiedConns is only + // registered while the client isn't closed. + s.mu.Lock() + c.mu.Lock() + if c.isClosed() { + c.mu.Unlock() + s.mu.Unlock() + return true, true + } + // Track which proxy public key is used by this connection. + c.proxyKey = pub + cid := c.cid + c.mu.Unlock() + // Track this proxied connection so that it can be closed + // if the trusted key is removed on configuration reload + if s.proxiedConns == nil { + s.proxiedConns = make(map[string]map[uint64]*client) + } + clients := s.proxiedConns[pub] + if clients == nil { + clients = make(map[uint64]*client) + } + clients[cid] = c + s.proxiedConns[pub] = clients + s.mu.Unlock() + return true, true + } + } + // We could not verify the signature, so indicate failure. + return true, false +} + +func getTLSAuthDCs(rdns *pkix.RDNSequence) string { + dcOID := asn1.ObjectIdentifier{0, 9, 2342, 19200300, 100, 1, 25} + dcs := []string{} + for _, rdn := range *rdns { + if len(rdn) == 0 { + continue + } + for _, atv := range rdn { + value, ok := atv.Value.(string) + if !ok { + continue + } + if atv.Type.Equal(dcOID) { + dcs = append(dcs, "DC="+value) + } + } + } + return strings.Join(dcs, ",") +} + +type tlsMapAuthFn func(string, *ldap.DN, bool) (string, bool) + +func checkClientTLSCertSubject(c *client, fn tlsMapAuthFn) bool { + tlsState := c.GetTLSConnectionState() + if tlsState == nil { + c.Debugf("User required in cert, no TLS connection state") + return false + } + if len(tlsState.PeerCertificates) == 0 { + c.Debugf("User required in cert, no peer certificates found") + return false + } + cert := tlsState.PeerCertificates[0] + if len(tlsState.PeerCertificates) > 1 { + c.Debugf("Multiple peer certificates found, selecting first") + } + + hasSANs := len(cert.DNSNames) > 0 + hasEmailAddresses := len(cert.EmailAddresses) > 0 + hasSubject := len(cert.Subject.String()) > 0 + hasURIs := len(cert.URIs) > 0 + if !hasSANs && !hasEmailAddresses && !hasSubject && !hasURIs { + c.Debugf("User required in cert, none found") + return false + } + + switch { + case hasEmailAddresses: + for _, u := range cert.EmailAddresses { + if match, ok := fn(u, nil, false); ok { + c.Debugf("Using email found in cert for auth [%q]", match) + return true + } + } + fallthrough + case hasSANs: + for _, u := range cert.DNSNames { + if match, ok := fn(u, nil, true); ok { + c.Debugf("Using SAN found in cert for auth [%q]", match) + return true + } + } + fallthrough + case hasURIs: + for _, u := range cert.URIs { + if match, ok := fn(u.String(), nil, false); ok { + c.Debugf("Using URI found in cert for auth [%q]", match) + return true + } + } + } + + // Use the string representation of the full RDN Sequence including + // the domain components in case there are any. + rdn := cert.Subject.ToRDNSequence().String() + + // Match using the raw subject to avoid ignoring attributes. + // https://github.com/golang/go/issues/12342 + dn, err := ldap.FromRawCertSubject(cert.RawSubject) + if err == nil { + if match, ok := fn(_EMPTY_, dn, false); ok { + c.Debugf("Using DistinguishedNameMatch for auth [%q]", match) + return true + } + c.Debugf("DistinguishedNameMatch could not be used for auth [%q]", rdn) + } + + var rdns pkix.RDNSequence + if _, err := asn1.Unmarshal(cert.RawSubject, &rdns); err == nil { + // If found domain components then include roughly following + // the order from https://tools.ietf.org/html/rfc2253 + // + // NOTE: The original sequence from string representation by ToRDNSequence does not follow + // the correct ordering, so this addition ofdomainComponents would likely be deprecated in + // another release in favor of using the correct ordered as parsed by the go-ldap library. + // + dcs := getTLSAuthDCs(&rdns) + if len(dcs) > 0 { + u := strings.Join([]string{rdn, dcs}, ",") + if match, ok := fn(u, nil, false); ok { + c.Debugf("Using RDNSequence for auth [%q]", match) + return true + } + c.Debugf("RDNSequence could not be used for auth [%q]", u) + } + } + + // If no match, then use the string representation of the RDNSequence + // from the subject without the domainComponents. + if match, ok := fn(rdn, nil, false); ok { + c.Debugf("Using certificate subject for auth [%q]", match) + return true + } + + c.Debugf("User in cert [%q], not found", rdn) + return false +} + +func dnsAltNameLabels(dnsAltName string) []string { + return strings.Split(strings.ToLower(dnsAltName), ".") +} + +// Check DNS name according to https://tools.ietf.org/html/rfc6125#section-6.4.1 +func dnsAltNameMatches(dnsAltNameLabels []string, urls []*url.URL) bool { +URLS: + for _, url := range urls { + if url == nil { + continue URLS + } + hostLabels := strings.Split(strings.ToLower(url.Hostname()), ".") + // Following https://tools.ietf.org/html/rfc6125#section-6.4.3, should not => will not, may => will not + // The wildcard * never matches multiple label and only matches the left most label. + if len(hostLabels) != len(dnsAltNameLabels) { + continue URLS + } + i := 0 + // only match wildcard on left most label + if dnsAltNameLabels[0] == "*" { + i++ + } + for ; i < len(dnsAltNameLabels); i++ { + if dnsAltNameLabels[i] != hostLabels[i] { + continue URLS + } + } + return true + } + return false +} + +// checkRouterAuth checks optional router authorization which can be nil or username/password. +func (s *Server) isRouterAuthorized(c *client) bool { + // Snapshot server options. + opts := s.getOpts() + + // Check custom auth first, then TLS map if enabled + // then single user/pass. + if opts.CustomRouterAuthentication != nil { + return opts.CustomRouterAuthentication.Check(c) + } + + if opts.Cluster.TLSMap || opts.Cluster.TLSCheckKnownURLs { + return checkClientTLSCertSubject(c, func(user string, _ *ldap.DN, isDNSAltName bool) (string, bool) { + if user == _EMPTY_ { + return _EMPTY_, false + } + if opts.Cluster.TLSCheckKnownURLs && isDNSAltName { + if dnsAltNameMatches(dnsAltNameLabels(user), opts.Routes) { + return _EMPTY_, true + } + } + if opts.Cluster.TLSMap && opts.Cluster.Username == user { + return _EMPTY_, true + } + return _EMPTY_, false + }) + } + + if opts.Cluster.Username == _EMPTY_ { + return true + } + + if opts.Cluster.Username != c.opts.Username { + return false + } + if !comparePasswords(opts.Cluster.Password, c.opts.Password) { + return false + } + return true +} + +// isGatewayAuthorized checks optional gateway authorization which can be nil or username/password. +func (s *Server) isGatewayAuthorized(c *client) bool { + // Snapshot server options. + opts := s.getOpts() + + // Check whether TLS map is enabled, otherwise use single user/pass. + if opts.Gateway.TLSMap || opts.Gateway.TLSCheckKnownURLs { + return checkClientTLSCertSubject(c, func(user string, _ *ldap.DN, isDNSAltName bool) (string, bool) { + if user == _EMPTY_ { + return _EMPTY_, false + } + if opts.Gateway.TLSCheckKnownURLs && isDNSAltName { + labels := dnsAltNameLabels(user) + for _, gw := range opts.Gateway.Gateways { + if gw != nil && dnsAltNameMatches(labels, gw.URLs) { + return _EMPTY_, true + } + } + } + if opts.Gateway.TLSMap && opts.Gateway.Username == user { + return _EMPTY_, true + } + return _EMPTY_, false + }) + } + + if opts.Gateway.Username == _EMPTY_ { + return true + } + + if opts.Gateway.Username != c.opts.Username { + return false + } + return comparePasswords(opts.Gateway.Password, c.opts.Password) +} + +func (s *Server) registerLeafWithAccount(c *client, account string) bool { + var err error + acc := s.globalAccount() + if account != _EMPTY_ { + acc, err = s.lookupAccount(account) + if err != nil { + s.Errorf("authentication of user %q failed, unable to lookup account %q: %v", + c.opts.Username, account, err) + return false + } + } + if err = c.registerWithAccount(acc); err != nil { + return false + } + return true +} + +// isLeafNodeAuthorized will check for auth for an inbound leaf node connection. +func (s *Server) isLeafNodeAuthorized(c *client) bool { + opts := s.getOpts() + + setProxyAuthError := func(err error) bool { + c.Debugf(err.Error()) + c.setAuthError(err) + return false + } + + isAuthorized := func(username, password, account string, proxyRequired bool) bool { + trustedProxy, ok := s.proxyCheck(c, opts) + if trustedProxy && !ok { + return setProxyAuthError(ErrAuthProxyNotTrusted) + } + // A given user may not be required, but if the boolean is set at the + // authorization top-level, then override. + if !proxyRequired && opts.LeafNode.ProxyRequired { + proxyRequired = true + } + if proxyRequired && !trustedProxy { + return setProxyAuthError(ErrAuthProxyRequired) + } + if username != c.opts.Username { + return false + } + if !comparePasswords(password, c.opts.Password) { + return false + } + return s.registerLeafWithAccount(c, account) + } + + // If leafnodes config has an authorization{} stanza, this takes precedence. + // The user in CONNECT must match. We will bind to the account associated + // with that user (from the leafnode's authorization{} config). + if opts.LeafNode.Username != _EMPTY_ { + return isAuthorized(opts.LeafNode.Username, opts.LeafNode.Password, opts.LeafNode.Account, + opts.LeafNode.ProxyRequired) + } else if opts.LeafNode.Nkey != _EMPTY_ { + trustedProxy, ok := s.proxyCheck(c, opts) + if trustedProxy && !ok { + return false + } + if opts.LeafNode.ProxyRequired && !trustedProxy { + return setProxyAuthError(ErrAuthProxyRequired) + } + if c.opts.Nkey != opts.LeafNode.Nkey { + return false + } + if c.opts.Sig == _EMPTY_ { + c.Debugf("Signature missing") + return false + } + sig, err := base64.RawURLEncoding.DecodeString(c.opts.Sig) + if err != nil { + // Allow fallback to normal base64. + sig, err = base64.StdEncoding.DecodeString(c.opts.Sig) + if err != nil { + c.Debugf("Signature not valid base64") + return false + } + } + pub, err := nkeys.FromPublicKey(c.opts.Nkey) + if err != nil { + c.Debugf("User nkey not valid: %v", err) + return false + } + if err := pub.Verify(c.nonce, sig); err != nil { + c.Debugf("Signature not verified") + return false + } + return s.registerLeafWithAccount(c, opts.LeafNode.Account) + } else if len(opts.LeafNode.Users) > 0 { + if opts.LeafNode.TLSMap { + var user *User + found := checkClientTLSCertSubject(c, func(u string, _ *ldap.DN, _ bool) (string, bool) { + // This is expected to be a very small array. + for _, usr := range opts.LeafNode.Users { + if u == usr.Username { + user = usr + return u, true + } + } + return _EMPTY_, false + }) + if !found { + return false + } + if c.opts.Username != _EMPTY_ { + s.Warnf("User %q found in connect proto, but user required from cert", c.opts.Username) + } + c.opts.Username = user.Username + // EMPTY will result in $G + accName := _EMPTY_ + if user.Account != nil { + accName = user.Account.GetName() + } + // This will authorize since are using an existing user, + // but it will also register with proper account. + return isAuthorized(user.Username, user.Password, accName, user.ProxyRequired) + } + + // This is expected to be a very small array. + for _, u := range opts.LeafNode.Users { + if u.Username == c.opts.Username { + var accName string + if u.Account != nil { + accName = u.Account.Name + } + return isAuthorized(u.Username, u.Password, accName, u.ProxyRequired) + } + } + return false + } + + // We are here if we accept leafnode connections without any credentials. + + // Still, if the CONNECT has some user info, we will bind to the + // user's account or to the specified default account (if provided) + // or to the global account. + return s.isClientAuthorized(c) +} + +// Support for bcrypt stored passwords and tokens. +var validBcryptPrefix = regexp.MustCompile(`^\$2[abxy]\$\d{2}\$.*`) + +// isBcrypt checks whether the given password or token is bcrypted. +func isBcrypt(password string) bool { + if strings.HasPrefix(password, "$") { + return validBcryptPrefix.MatchString(password) + } + + return false +} + +func comparePasswords(serverPassword, clientPassword string) bool { + // Check to see if the server password is a bcrypt hash + if isBcrypt(serverPassword) { + if err := bcrypt.CompareHashAndPassword([]byte(serverPassword), []byte(clientPassword)); err != nil { + return false + } + } else { + // stringToBytes should be constant-time near enough compared to + // turning a string into []byte normally. + spass := stringToBytes(serverPassword) + cpass := stringToBytes(clientPassword) + if subtle.ConstantTimeCompare(spass, cpass) == 0 { + return false + } + } + return true +} + +func validateAuth(o *Options) error { + if err := validatePinnedCerts(o.TLSPinnedCerts); err != nil { + return err + } + for _, u := range o.Users { + if err := validateAllowedConnectionTypes(u.AllowedConnectionTypes); err != nil { + return err + } + if err := validatePermissionSubjects(u.Permissions); err != nil { + return fmt.Errorf("invalid permissions for user %q: %w", u.Username, err) + } + } + for _, u := range o.Nkeys { + if err := validateAllowedConnectionTypes(u.AllowedConnectionTypes); err != nil { + return err + } + if err := validatePermissionSubjects(u.Permissions); err != nil { + return fmt.Errorf("invalid permissions for nkey %q: %w", u.Nkey, err) + } + } + return validateNoAuthUser(o, o.NoAuthUser) +} + +func validatePermissionSubjects(p *Permissions) error { + if p == nil { + return nil + } + if p.Publish != nil { + if err := checkPermSubjectArray(p.Publish.Allow, false); err != nil { + return fmt.Errorf("publish allow: %w", err) + } + if err := checkPermSubjectArray(p.Publish.Deny, false); err != nil { + return fmt.Errorf("publish deny: %w", err) + } + } + if p.Subscribe != nil { + if err := checkPermSubjectArray(p.Subscribe.Allow, true); err != nil { + return fmt.Errorf("subscribe allow: %w", err) + } + if err := checkPermSubjectArray(p.Subscribe.Deny, true); err != nil { + return fmt.Errorf("subscribe deny: %w", err) + } + } + return nil +} + +func validateAllowedConnectionTypes(m map[string]struct{}) error { + for ct := range m { + ctuc := strings.ToUpper(ct) + switch ctuc { + case jwt.ConnectionTypeStandard, jwt.ConnectionTypeWebsocket, + jwt.ConnectionTypeLeafnode, jwt.ConnectionTypeLeafnodeWS, + jwt.ConnectionTypeMqtt, jwt.ConnectionTypeMqttWS, + jwt.ConnectionTypeInProcess: + default: + return fmt.Errorf("unknown connection type %q", ct) + } + if ctuc != ct { + delete(m, ct) + m[ctuc] = struct{}{} + } + } + return nil +} + +func validateNoAuthUser(o *Options, noAuthUser string) error { + if noAuthUser == _EMPTY_ { + return nil + } + if len(o.TrustedOperators) > 0 { + return fmt.Errorf("no_auth_user not compatible with Trusted Operator") + } + + if o.Nkeys == nil && o.Users == nil { + return fmt.Errorf(`no_auth_user: "%s" present, but users/nkeys are not defined`, noAuthUser) + } + for _, u := range o.Users { + if u.Username == noAuthUser { + return nil + } + } + for _, u := range o.Nkeys { + if u.Nkey == noAuthUser { + return nil + } + } + return fmt.Errorf( + `no_auth_user: "%s" not present as user or nkey in authorization block or account configuration`, + noAuthUser) +} + +func validateProxies(o *Options) error { + if o.Proxies == nil { + return nil + } + for _, p := range o.Proxies.Trusted { + if !nkeys.IsValidPublicKey(p.Key) { + return fmt.Errorf("proxy trusted key %q is invalid", p.Key) + } + } + return nil +} + +// Create a list of nkeys.KeyPair corresponding to the public keys +// of the Proxies.TrustedKeys list. +// Server lock must be held on entry. +func (s *Server) processProxiesTrustedKeys() { + // We could be here on reload. + if s.proxiesKeyPairs != nil { + s.proxiesKeyPairs = s.proxiesKeyPairs[:0] + } + if opts := s.getOpts(); opts.Proxies == nil { + return + } + for _, p := range s.getOpts().Proxies.Trusted { + // Can't fail since we have already checked that it was a valid key. + kp, _ := nkeys.FromPublicKey(p.Key) + s.proxiesKeyPairs = append(s.proxiesKeyPairs, kp) + } +} + +// Returns the connection's `ClosedState` for the given authenication error. +func getAuthErrClosedState(authErr error) ClosedState { + switch authErr { + case ErrAuthProxyNotTrusted: + return ProxyNotTrusted + case ErrAuthProxyRequired: + return ProxyRequired + default: + return AuthenticationViolation + } +} diff --git a/server/auth_callout.go b/server/auth_callout.go new file mode 100644 index 0000000..9d03e8c --- /dev/null +++ b/server/auth_callout.go @@ -0,0 +1,503 @@ +// Copyright 2022-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "crypto/tls" + "encoding/pem" + "errors" + "fmt" + "time" + "unicode" + + "github.com/nats-io/jwt/v2" + "github.com/nats-io/nkeys" +) + +const ( + AuthCalloutSubject = "$SYS.REQ.USER.AUTH" + AuthRequestSubject = "nats-authorization-request" + AuthRequestXKeyHeader = "Nats-Server-Xkey" +) + +func titleCase(m string) string { + r := []rune(m) + if len(r) == 0 { + return _EMPTY_ + } + return string(append([]rune{unicode.ToUpper(r[0])}, r[1:]...)) +} + +// Process a callout on this client's behalf. +func (s *Server) processClientOrLeafCallout(c *client, opts *Options, proxyRequired, trustedProxy bool, ujwt string) (authorized bool, errStr string) { + isOperatorMode := len(opts.TrustedKeys) > 0 + + // this is the account the user connected in, or the one running the callout + var acc *Account + if !isOperatorMode && opts.AuthCallout != nil && opts.AuthCallout.Account != _EMPTY_ { + aname := opts.AuthCallout.Account + var err error + acc, err = s.LookupAccount(aname) + if err != nil { + errStr = fmt.Sprintf("No valid account %q for auth callout request: %v", aname, err) + s.Warnf(errStr) + return false, errStr + } + } else { + acc = c.acc + } + if acc == nil { + // FIX for https://github.com/nats-io/nats-server/issues/7841 + // hand rolled creds on leafnode became crasher here + errStr = fmt.Sprintf("%s not mapped to a callout account", c.kindString()) + s.Warnf(errStr) + return false, errStr + } + + // Check if we have been requested to encrypt. + var xkp nkeys.KeyPair + var xkey string + var pubAccXKey string + if !isOperatorMode && opts.AuthCallout != nil && opts.AuthCallout.XKey != _EMPTY_ { + pubAccXKey = opts.AuthCallout.XKey + } else if isOperatorMode { + pubAccXKey = acc.externalAuthXKey() + } + // If set grab server's xkey keypair and public key. + if pubAccXKey != _EMPTY_ { + // These are only set on creation, so lock not needed. + xkp, xkey = s.xkp, s.info.XKey + } + + // Create a keypair for the user. We will expect this public user to be in the signed response. + // This prevents replay attacks. + ukp, _ := nkeys.CreateUser() + pub, _ := ukp.PublicKey() + + reply := s.newRespInbox() + respCh := make(chan string, 1) + + decodeResponse := func(rc *client, rmsg []byte, acc *Account) (*jwt.UserClaims, error) { + account := acc.Name + _, msg := rc.msgParts(rmsg) + + // This signals not authorized. + // Since this is an account subscription will always have "\r\n". + if len(msg) <= LEN_CR_LF { + return nil, fmt.Errorf("auth callout violation: %q on account %q", "no reason supplied", account) + } + // Strip trailing CRLF. + msg = msg[:len(msg)-LEN_CR_LF] + encrypted := false + // If we sent an encrypted request the response could be encrypted as well. + // we are expecting the input to be `eyJ` if it is a JWT + if xkp != nil && len(msg) > 0 && !bytes.HasPrefix(msg, []byte(jwtPrefix)) { + var err error + msg, err = xkp.Open(msg, pubAccXKey) + if err != nil { + return nil, fmt.Errorf("error decrypting auth callout response on account %q: %v", account, err) + } + encrypted = true + } + + cr, err := jwt.DecodeAuthorizationResponseClaims(string(msg)) + if err != nil { + return nil, err + } + vr := jwt.CreateValidationResults() + cr.Validate(vr) + if len(vr.Issues) > 0 { + return nil, fmt.Errorf("authorization response had validation errors: %v", vr.Issues[0]) + } + + // the subject is the user id + if cr.Subject != pub { + return nil, errors.New("auth callout violation: auth callout response is not for expected user") + } + + // check the audience to be the server ID + if cr.Audience != s.info.ID { + return nil, errors.New("auth callout violation: auth callout response is not for server") + } + + // check if had an error message from the auth account + if cr.Error != _EMPTY_ { + return nil, fmt.Errorf("auth callout service returned an error: %v", cr.Error) + } + + // if response is encrypted none of this is needed + if isOperatorMode && !encrypted { + pkStr := cr.Issuer + if cr.IssuerAccount != _EMPTY_ { + pkStr = cr.IssuerAccount + } + if pkStr != account { + if _, ok := acc.hasIssuer(pkStr); !ok { + return nil, errors.New("auth callout signing key is unknown") + } + } + } + + return jwt.DecodeUserClaims(cr.Jwt) + } + + // getIssuerAccount returns the issuer (as per JWT) - it also asserts that + // only in operator mode we expect to receive `issuer_account`. + getIssuerAccount := func(arc *jwt.UserClaims, account string) (string, error) { + // Make sure correct issuer. + var issuer string + if opts.AuthCallout != nil { + issuer = opts.AuthCallout.Issuer + } else { + // Operator mode is who we send the request on unless switching accounts. + issuer = acc.Name + } + + // the jwt issuer can be a signing key + jwtIssuer := arc.Issuer + if arc.IssuerAccount != _EMPTY_ { + if !isOperatorMode { + // this should be invalid - effectively it would allow the auth callout + // to issue on another account which may be allowed given the configuration + // where the auth callout account can handle multiple different ones.. + return _EMPTY_, fmt.Errorf("error non operator mode account %q: attempted to use issuer_account", account) + } + jwtIssuer = arc.IssuerAccount + } + + if jwtIssuer != issuer { + if !isOperatorMode { + return _EMPTY_, fmt.Errorf("wrong issuer for auth callout response on account %q, expected %q got %q", account, issuer, jwtIssuer) + } else if !acc.isAllowedAcount(jwtIssuer) { + return _EMPTY_, fmt.Errorf("account %q not permitted as valid account option for auth callout for account %q", + arc.Issuer, account) + } + } + return jwtIssuer, nil + } + + getExpirationAndAllowedConnections := func(arc *jwt.UserClaims, account string) (time.Duration, map[string]struct{}, error) { + allowNow, expiration := validateTimes(arc) + if !allowNow { + c.Errorf("Outside connect times") + return 0, nil, fmt.Errorf("authorized user on account %q outside of valid connect times", account) + } + + allowedConnTypes, err := convertAllowedConnectionTypes(arc.User.AllowedConnectionTypes) + if err != nil { + c.Debugf("%v", err) + if len(allowedConnTypes) == 0 { + return 0, nil, fmt.Errorf("authorized user on account %q using invalid connection type", account) + } + } + return expiration, allowedConnTypes, nil + } + + assignAccountAndPermissions := func(arc *jwt.UserClaims, account string) (*Account, error) { + // Apply to this client. + var err error + issuerAccount, err := getIssuerAccount(arc, account) + if err != nil { + return nil, err + } + + // if we are not in operator mode, they can specify placement as a tag + var placement string + if !isOperatorMode { + // only allow placement if we are not in operator mode + placement = arc.Audience + } else { + placement = issuerAccount + } + + targetAcc, err := s.LookupAccount(placement) + if err != nil { + return nil, fmt.Errorf("no valid account %q for auth callout response on account %q: %v", placement, account, err) + } + if isOperatorMode { + // this will validate the signing key that emitted the user, and if it is a signing + // key it assigns the permissions from the target account + if scope, ok := targetAcc.hasIssuer(arc.Issuer); !ok { + return nil, fmt.Errorf("user JWT issuer %q is not known", arc.Issuer) + } else if scope != nil { + // this possibly has to be different because it could just be a plain issued by a non-scoped signing key + if err := scope.ValidateScopedSigner(arc); err != nil { + return nil, fmt.Errorf("user JWT is not valid: %v", err) + } else if uSc, ok := scope.(*jwt.UserScope); !ok { + return nil, fmt.Errorf("user JWT is not a valid scoped user") + } else if arc.User.UserPermissionLimits, err = processUserPermissionsTemplate(uSc.Template, arc, targetAcc); err != nil { + return nil, fmt.Errorf("user JWT generated invalid permissions: %v", err) + } + } + } + return targetAcc, nil + } + + processReply := func(_ *subscription, rc *client, racc *Account, subject, reply string, rmsg []byte) { + arc, err := decodeResponse(rc, rmsg, racc) + if err != nil { + c.authViolation() + respCh <- titleCase(err.Error()) + return + } + // If the caller had established that the user should go through a proxy, + // or if the `arc` JWT requires it, and we don't have a trusted proxy, + // reject the connection. + if (proxyRequired || arc.ProxyRequired) && !trustedProxy { + err = ErrAuthProxyRequired + c.setAuthError(err) + c.authViolation() + respCh <- titleCase(err.Error()) + return + } + vr := jwt.CreateValidationResults() + arc.Validate(vr) + if len(vr.Issues) > 0 { + c.authViolation() + respCh <- fmt.Sprintf("Error validating user JWT: %v", vr.Issues[0]) + return + } + + // Make sure that the user is what we requested. + if arc.Subject != pub { + c.authViolation() + respCh <- fmt.Sprintf("Expected authorized user of %q but got %q on account %q", pub, arc.Subject, racc.Name) + return + } + + expiration, allowedConnTypes, err := getExpirationAndAllowedConnections(arc, racc.Name) + if err != nil { + c.authViolation() + respCh <- titleCase(err.Error()) + return + } + + targetAcc, err := assignAccountAndPermissions(arc, racc.Name) + if err != nil { + c.authViolation() + respCh <- titleCase(err.Error()) + return + } + + // the JWT is cleared, because if in operator mode it may hold the JWT + // for the bearer token that connected to the callout if in operator mode + // the permissions are already set on the client, this prevents a decode + // on c.RegisterNKeyUser which would have wrong values + c.mu.Lock() + c.opts.JWT = _EMPTY_ + c.mu.Unlock() + + // Build internal user and bind to the targeted account. + nkuser := buildInternalNkeyUser(arc, allowedConnTypes, targetAcc) + if err := c.RegisterNkeyUser(nkuser); err != nil { + c.authViolation() + respCh <- fmt.Sprintf("Could not register auth callout user: %v", err) + return + } + + // See if the response wants to override the username. + if arc.Name != _EMPTY_ { + c.mu.Lock() + c.opts.Username = arc.Name + // Clear any others. + c.opts.Nkey = _EMPTY_ + c.pubKey = _EMPTY_ + c.opts.Token = _EMPTY_ + c.mu.Unlock() + } + + // Check if we need to set an auth timer if the user jwt expires. + c.setExpiration(arc.Claims(), expiration) + + respCh <- _EMPTY_ + } + + // create a subscription to receive a response from the authcallout + sub, err := acc.subscribeInternal(reply, processReply) + if err != nil { + errStr = fmt.Sprintf("Error setting up reply subscription for auth request: %v", err) + s.Warnf(errStr) + return false, errStr + } + defer acc.unsubscribeInternal(sub) + + // Build our request claims - jwt subject should be nkey + jwtSub := acc.Name + if opts.AuthCallout != nil { + jwtSub = opts.AuthCallout.Issuer + } + + // The public key of the server, if set is available on Varz.Key + // This means that when a service connects, it can now peer + // authenticate if it wants to - but that also means that it needs to be + // listening to cluster changes + claim := jwt.NewAuthorizationRequestClaims(jwtSub) + claim.Audience = AuthRequestSubject + // Set expected public user nkey. + claim.UserNkey = pub + + s.mu.RLock() + claim.Server = jwt.ServerID{ + Name: s.info.Name, + Host: s.info.Host, + ID: s.info.ID, + Version: s.info.Version, + Cluster: s.info.Cluster, + } + s.mu.RUnlock() + + // Tags + claim.Server.Tags = s.getOpts().Tags + + // Check if we have been requested to encrypt. + // FIXME: possibly this public key also needs to be on the + // Varz, because then it can be peer verified? + if xkp != nil { + claim.Server.XKey = xkey + } + + authTimeout := secondsToDuration(s.getOpts().AuthTimeout) + claim.Expires = time.Now().Add(time.Duration(authTimeout)).UTC().Unix() + + // Grab client info for the request. + c.mu.Lock() + c.fillClientInfo(&claim.ClientInformation) + c.fillConnectOpts(&claim.ConnectOptions, ujwt) + // If we have a sig in the client opts, fill in nonce. + if claim.ConnectOptions.SignedNonce != _EMPTY_ { + claim.ClientInformation.Nonce = string(c.nonce) + } + + // TLS + if c.flags.isSet(handshakeComplete) && c.nc != nil { + var ct jwt.ClientTLS + conn := c.nc.(*tls.Conn) + cs := conn.ConnectionState() + ct.Version = tlsVersion(cs.Version) + ct.Cipher = tls.CipherSuiteName(cs.CipherSuite) + // Check verified chains. + for _, vs := range cs.VerifiedChains { + var certs []string + for _, c := range vs { + blk := &pem.Block{ + Type: "CERTIFICATE", + Bytes: c.Raw, + } + certs = append(certs, string(pem.EncodeToMemory(blk))) + } + ct.VerifiedChains = append(ct.VerifiedChains, certs) + } + // If we do not have verified chains put in peer certs. + if len(ct.VerifiedChains) == 0 { + for _, c := range cs.PeerCertificates { + blk := &pem.Block{ + Type: "CERTIFICATE", + Bytes: c.Raw, + } + ct.Certs = append(ct.Certs, string(pem.EncodeToMemory(blk))) + } + } + claim.TLS = &ct + } + c.mu.Unlock() + + b, err := claim.Encode(s.kp) + if err != nil { + errStr = fmt.Sprintf("Error encoding auth request claim on account %q: %v", acc.Name, err) + s.Warnf(errStr) + return false, errStr + } + req := []byte(b) + var hdr []byte + + // Check if we have been asked to encrypt. + if xkp != nil { + req, err = xkp.Seal([]byte(req), pubAccXKey) + if err != nil { + errStr = fmt.Sprintf("Error encrypting auth request claim on account %q: %v", acc.Name, err) + s.Warnf(errStr) + return false, errStr + } + hdr = genHeader(hdr, AuthRequestXKeyHeader, xkey) + } + + // Send out our request. + if err := s.sendInternalAccountMsgWithReply(acc, AuthCalloutSubject, reply, hdr, req, false); err != nil { + errStr = fmt.Sprintf("Error sending authorization request: %v", err) + s.Debugf(errStr) + return false, errStr + } + select { + case errStr = <-respCh: + if authorized = errStr == _EMPTY_; !authorized { + s.Warnf(errStr) + } + case <-time.After(authTimeout): + s.Debugf(fmt.Sprintf("Authorization callout response not received in time on account %q", acc.Name)) + } + + return authorized, errStr +} + +// Fill in client information for the request. +// Lock should be held. +func (c *client) fillClientInfo(ci *jwt.ClientInformation) { + if c == nil || (c.kind != CLIENT && c.kind != LEAF && c.kind != JETSTREAM && c.kind != ACCOUNT) { + return + } + + // Do it this way to fail to compile if fields are added to jwt.ClientInformation. + *ci = jwt.ClientInformation{ + Host: c.host, + ID: c.cid, + User: c.getRawAuthUser(), + Name: c.opts.Name, + Tags: c.tags, + NameTag: c.nameTag, + Kind: c.kindString(), + Type: c.clientTypeString(), + MQTT: c.getMQTTClientID(), + } +} + +// Fill in client options. +// Lock should be held. +func (c *client) fillConnectOpts(opts *jwt.ConnectOptions, ujwt string) { + if c == nil || (c.kind != CLIENT && c.kind != LEAF && c.kind != JETSTREAM && c.kind != ACCOUNT) { + return + } + + o := c.opts + if ujwt == _EMPTY_ { + // The caller may supply a reconstructed JWT that should be sent to auth + // callout without storing it in c.opts.JWT. If not, fall back to the client + // option as before. + ujwt = o.JWT + } + + // Do it this way to fail to compile if fields are added to jwt.ClientInformation. + *opts = jwt.ConnectOptions{ + JWT: ujwt, + Nkey: o.Nkey, + SignedNonce: o.Sig, + Token: o.Token, + Username: o.Username, + Password: o.Password, + Name: o.Name, + Lang: o.Lang, + Version: o.Version, + Protocol: o.Protocol, + } +} diff --git a/server/auth_callout_test.go b/server/auth_callout_test.go new file mode 100644 index 0000000..69c9b6a --- /dev/null +++ b/server/auth_callout_test.go @@ -0,0 +1,3258 @@ +// Copyright 2022-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bufio" + "bytes" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "net" + "os" + "reflect" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/jwt/v2" + "github.com/nats-io/nats.go" + "github.com/nats-io/nkeys" +) + +// Helper function to decode an auth request. +func decodeAuthRequest(t *testing.T, ejwt []byte) (string, *jwt.ServerID, *jwt.ClientInformation, *jwt.ConnectOptions, *jwt.ClientTLS) { + t.Helper() + ac, err := jwt.DecodeAuthorizationRequestClaims(string(ejwt)) + require_NoError(t, err) + return ac.UserNkey, &ac.Server, &ac.ClientInformation, &ac.ConnectOptions, ac.TLS +} + +func authCalloutMQTTConnectPacket(clientID, user, pass string) []byte { + flags := byte(mqttConnFlagCleanSession) + if user != _EMPTY_ { + flags |= mqttConnFlagUsernameFlag + } + if pass != _EMPTY_ { + flags |= mqttConnFlagPasswordFlag + } + + pkLen := 2 + len(mqttProtoName) + + 1 + + 1 + + 2 + + 2 + len(clientID) + + if user != _EMPTY_ { + pkLen += 2 + len(user) + } + if pass != _EMPTY_ { + pkLen += 2 + len(pass) + } + + w := newMQTTWriter(0) + w.WriteByte(mqttPacketConnect) + w.WriteVarInt(pkLen) + w.WriteString(string(mqttProtoName)) + w.WriteByte(0x4) + w.WriteByte(flags) + w.WriteUint16(0) + w.WriteString(clientID) + if user != _EMPTY_ { + w.WriteString(user) + } + if pass != _EMPTY_ { + w.WriteBytes([]byte(pass)) + } + return w.Bytes() +} + +func authCalloutMQTTConnect(t testing.TB, host string, port int, clientID, user, pass string) (net.Conn, byte) { + t.Helper() + + c, err := net.Dial("tcp", net.JoinHostPort(host, fmt.Sprintf("%d", port))) + require_NoError(t, err) + + c.SetDeadline(time.Now().Add(2 * time.Second)) + defer c.SetDeadline(time.Time{}) + + _, err = c.Write(authCalloutMQTTConnectPacket(clientID, user, pass)) + require_NoError(t, err) + + var ack [4]byte + _, err = io.ReadFull(c, ack[:]) + require_NoError(t, err) + if ack[0] != mqttPacketConnectAck || ack[1] != 2 { + t.Fatalf("Expected MQTT CONNACK, got %v", ack[:]) + } + return c, ack[3] +} + +func TestTitleCaseEmptyString(t *testing.T) { + defer require_NoPanic(t) + require_Equal(t, titleCase(_EMPTY_), _EMPTY_) +} + +const ( + authCalloutPub = "UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON" + authCalloutSeed = "SUAP277QP7U4JMFFPVZHLJYEQJ2UHOTYVEIZJYAWRJXQLP4FRSEHYZJJOU" + authCalloutIssuer = "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + authCalloutIssuerSeed = "SAANDLKMXL6CUS3CP52WIXBEDN6YJ545GDKC65U5JZPPV6WH6ESWUA6YAI" + authCalloutIssuerSK = "SAAE46BB675HKZKSVJEUZAKKWIV6BJJO6XYE46Z3ZHO7TCI647M3V42IJE" +) + +func serviceResponse(t *testing.T, userID string, serverID string, uJwt string, errMsg string, expires time.Duration) []byte { + cr := jwt.NewAuthorizationResponseClaims(userID) + cr.Audience = serverID + cr.Error = errMsg + cr.Jwt = uJwt + if expires != 0 { + cr.Expires = time.Now().Add(expires).Unix() + } + aa, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) + require_NoError(t, err) + token, err := cr.Encode(aa) + require_NoError(t, err) + return []byte(token) +} + +func newScopedRole(t *testing.T, role string, pub []string, sub []string, allowResponses bool) (*jwt.UserScope, nkeys.KeyPair) { + akp, pk := createKey(t) + r := jwt.NewUserScope() + r.Key = pk + r.Template.Sub.Allow.Add(sub...) + r.Template.Pub.Allow.Add(pub...) + if allowResponses { + r.Template.Resp = &jwt.ResponsePermission{ + MaxMsgs: 1, + Expires: time.Second * 3, + } + } + r.Role = role + return r, akp +} + +// Will create a signed user jwt as an authorized user. +func createAuthUser(t *testing.T, user, name, account, issuerAccount string, akp nkeys.KeyPair, expires time.Duration, limits *jwt.UserPermissionLimits) string { + t.Helper() + + if akp == nil { + var err error + akp, err = nkeys.FromSeed([]byte(authCalloutIssuerSeed)) + require_NoError(t, err) + } + + uc := jwt.NewUserClaims(user) + if issuerAccount != "" { + if _, err := nkeys.FromPublicKey(issuerAccount); err != nil { + t.Fatalf("issuer account is not a public key: %v", err) + } + uc.IssuerAccount = issuerAccount + } + // The callout uses the audience as the target account + // only if in non-operator mode, otherwise the user JWT has + // correct attribution - issuer or issuer_account + if _, err := nkeys.FromPublicKey(account); err != nil { + // if it is not a public key, set the audience + uc.Audience = account + } + + if name != _EMPTY_ { + uc.Name = name + } + if expires != 0 { + uc.Expires = time.Now().Add(expires).Unix() + } + if limits != nil { + uc.UserPermissionLimits = *limits + } + + vr := jwt.CreateValidationResults() + uc.Validate(vr) + require_Len(t, len(vr.Errors()), 0) + + tok, err := uc.Encode(akp) + require_NoError(t, err) + + return tok +} + +type authTest struct { + t *testing.T + srv *Server + conf string + authClient *nats.Conn + clients []*nats.Conn +} + +func NewAuthTest(t *testing.T, config string, authHandler nats.MsgHandler, clientOptions ...nats.Option) *authTest { + a := &authTest{t: t} + a.conf = createConfFile(t, []byte(config)) + a.srv, _ = RunServerWithConfig(a.conf) + + var err error + a.authClient = a.ConnectCallout(clientOptions...) + _, err = a.authClient.Subscribe(AuthCalloutSubject, authHandler) + require_NoError(t, err) + return a +} + +func (at *authTest) NewClient(clientOptions ...nats.Option) (*nats.Conn, error) { + conn, err := nats.Connect(at.srv.ClientURL(), clientOptions...) + if err != nil { + return nil, err + } + at.clients = append(at.clients, conn) + return conn, nil +} + +func (at *authTest) ConnectCallout(clientOptions ...nats.Option) *nats.Conn { + conn, err := at.NewClient(clientOptions...) + if err != nil { + err = fmt.Errorf("callout client failed: %w", err) + } + require_NoError(at.t, err) + return conn +} + +func (at *authTest) Connect(clientOptions ...nats.Option) *nats.Conn { + conn, err := at.NewClient(clientOptions...) + require_NoError(at.t, err) + return conn +} + +func (at *authTest) WSNewClient(clientOptions ...nats.Option) (*nats.Conn, error) { + pi := at.srv.PortsInfo(10 * time.Millisecond) + require_False(at.t, pi == nil) + + // test cert is SAN to DNS localhost, not local IPs returned by server in test environments + wssUrl := strings.Replace(pi.WebSocket[0], "127.0.0.1", "localhost", 1) + + // Seeing 127.0.1.1 in some test environments... + wssUrl = strings.Replace(wssUrl, "127.0.1.1", "localhost", 1) + + conn, err := nats.Connect(wssUrl, clientOptions...) + if err != nil { + return nil, err + } + at.clients = append(at.clients, conn) + return conn, nil +} + +func (at *authTest) WSConnect(clientOptions ...nats.Option) *nats.Conn { + conn, err := at.WSNewClient(clientOptions...) + require_NoError(at.t, err) + return conn +} + +func (at *authTest) RequireConnectError(clientOptions ...nats.Option) { + _, err := at.NewClient(clientOptions...) + require_Error(at.t, err) +} + +func (at *authTest) Cleanup() { + if at.authClient != nil { + at.authClient.Close() + } + if at.srv != nil { + at.srv.Shutdown() + removeFile(at.t, at.conf) + } +} + +func TestAuthCalloutBasics(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: A + authorization { + timeout: 1s + users: [ { user: "auth", password: "pwd" } ] + auth_callout { + # Needs to be a public account nkey, will work for both server config and operator mode. + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + # users that will power the auth callout service. + auth_users: [ auth ] + } + } + ` + callouts := uint32(0) + handler := func(m *nats.Msg) { + atomic.AddUint32(&callouts, 1) + user, si, ci, opts, _ := decodeAuthRequest(t, m.Data) + require_True(t, si.Name == "A") + require_True(t, ci.Host == "127.0.0.1") + // Allow dlc user. + if (opts.Username == "dlc" && opts.Password == "zzz") || opts.Token == "SECRET_TOKEN" { + var j jwt.UserPermissionLimits + j.Pub.Allow.Add("$SYS.>") + j.Payload = 1024 + if opts.Token == "SECRET_TOKEN" { + // Token MUST NOT be exposed in user info. + require_Equal(t, ci.User, "[REDACTED]") + } + ujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, "", nil, 10*time.Minute, &j) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else if opts.Username == "proxy" { + var j jwt.UserPermissionLimits + j.ProxyRequired = true + ujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, "", nil, 10*time.Minute, &j) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else { + // Nil response signals no authentication. + m.Respond(nil) + } + } + at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer at.Cleanup() + + // This one should fail since bad password. + at.RequireConnectError(nats.UserInfo("dlc", "xxx")) + + // This one should fail because it will require to be proxied and it is not. + at.RequireConnectError(nats.UserInfo("proxy", "xxx")) + + // This one will use callout since not defined in server config. + nc := at.Connect(nats.UserInfo("dlc", "zzz")) + defer nc.Close() + + resp, err := nc.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response := ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + require_NoError(t, err) + + userInfo := response.Data.(*UserInfo) + + dlc := &UserInfo{ + UserID: "dlc", + Account: globalAccountName, + AccountName: globalAccountName, + Permissions: &Permissions{ + Publish: &SubjectPermission{ + Allow: []string{"$SYS.>"}, + Deny: []string{AuthCalloutSubject}, // Will be auto-added since in auth account. + }, + Subscribe: &SubjectPermission{}, + }, + } + expires := userInfo.Expires + userInfo.Expires = 0 + if !reflect.DeepEqual(dlc, userInfo) { + t.Fatalf("User info for %q did not match", "dlc") + } + if expires > 10*time.Minute || expires < (10*time.Minute-5*time.Second) { + t.Fatalf("Expected expires of ~%v, got %v", 10*time.Minute, expires) + } + + // Callout with a token should also work, regardless of it being redacted in the user info. + nc.Close() + nc = at.Connect(nats.Token("SECRET_TOKEN")) + defer nc.Close() + + resp, err = nc.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response = ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + require_NoError(t, err) + + userInfo = response.Data.(*UserInfo) + dlc = &UserInfo{ + // Token MUST NOT be exposed in user info. + UserID: "[REDACTED]", + Account: globalAccountName, + AccountName: globalAccountName, + Permissions: &Permissions{ + Publish: &SubjectPermission{ + Allow: []string{"$SYS.>"}, + Deny: []string{AuthCalloutSubject}, // Will be auto-added since in auth account. + }, + Subscribe: &SubjectPermission{}, + }, + } + expires = userInfo.Expires + userInfo.Expires = 0 + if !reflect.DeepEqual(dlc, userInfo) { + t.Fatalf("User info for %q did not match", "dlc") + } + if expires > 10*time.Minute || expires < (10*time.Minute-5*time.Second) { + t.Fatalf("Expected expires of ~%v, got %v", 10*time.Minute, expires) + } +} + +func TestAuthCalloutUpdateAccountClaimsClearsExternalAuthorization(t *testing.T) { + opts := defaultServerOptions + s := New(&opts) + defer s.Shutdown() + + _, authAccPub := createKey(t) + _, authUserPub := createKey(t) + _, allowedAccPub := createKey(t) + _, deniedAccPub := createKey(t) + + authAcc, err := s.RegisterAccount(authAccPub) + require_NoError(t, err) + + claim := jwt.NewAccountClaims(authAccPub) + claim.Name = "AUTH" + claim.EnableExternalAuthorization(authUserPub) + claim.Authorization.AllowedAccounts.Add(allowedAccPub) + claim.Authorization.XKey = "test-xkey" + s.UpdateAccountClaims(authAcc, claim) + + require_True(t, authAcc.hasExternalAuth()) + require_True(t, authAcc.isExternalAuthUser(authUserPub)) + require_True(t, authAcc.isAllowedAcount(allowedAccPub)) + require_False(t, authAcc.isAllowedAcount(deniedAccPub)) + require_Equal(t, authAcc.externalAuthXKey(), "test-xkey") + + claim = jwt.NewAccountClaims(authAccPub) + claim.Name = "AUTH" + s.UpdateAccountClaims(authAcc, claim) + + require_False(t, authAcc.hasExternalAuth()) + require_False(t, authAcc.isExternalAuthUser(authUserPub)) + require_False(t, authAcc.isAllowedAcount(allowedAccPub)) + require_Equal(t, authAcc.externalAuthXKey(), _EMPTY_) +} + +func TestAuthCalloutMQTTJwtPassedInConnectOptions(t *testing.T) { + storeDir := t.TempDir() + conf := fmt.Sprintf(` + listen: "127.0.0.1:-1" + server_name: A + jetstream: { max_mem_store: 256MB, max_file_store: 1GB, store_dir: %q } + mqtt { host: "127.0.0.1", port: -1 } + authorization { + timeout: 1s + users: [ { user: "auth", password: "pwd" } ] + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + auth_users: [ auth ] + } + } + `, storeDir) + + ukp, err := nkeys.CreateUser() + require_NoError(t, err) + upub, err := ukp.PublicKey() + require_NoError(t, err) + expectedJWT := createAuthUser(t, upub, _EMPTY_, globalAccountName, "", nil, 10*time.Minute, nil) + seenJWT := make(chan string, 1) + handler := func(m *nats.Msg) { + user, si, _, opts, _ := decodeAuthRequest(t, m.Data) + select { + case seenJWT <- opts.JWT: + default: + } + if opts.JWT != expectedJWT { + m.Respond(nil) + return + } + ujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, "", nil, 10*time.Minute, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } + + at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer at.Cleanup() + + mc, rc := authCalloutMQTTConnect(t, at.srv.getOpts().MQTT.Host, at.srv.getOpts().MQTT.Port, "mqtt-auth-callout", "ignored", expectedJWT) + defer mc.Close() + + select { + case got := <-seenJWT: + require_Equal(t, got, expectedJWT) + case <-time.After(2 * time.Second): + t.Fatal("Did not receive auth callout request") + } + require_Equal(t, rc, mqttConnAckRCConnectionAccepted) +} + +func TestAuthCalloutNonOperatorMQTTOpaquePassword(t *testing.T) { + storeDir := t.TempDir() + conf := fmt.Sprintf(` + listen: "127.0.0.1:-1" + server_name: A + jetstream: { max_mem_store: 256MB, max_file_store: 1GB, store_dir: %q } + mqtt { host: "127.0.0.1", port: -1 } + authorization { + timeout: 1s + users: [ { user: "auth", password: "pwd" } ] + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + auth_users: [ auth ] + } + } + `, storeDir) + + const opaquePassword = "external-jwt-or-token" + type seenConnectOptions struct { + jwt string + password string + } + seen := make(chan seenConnectOptions, 1) + handler := func(m *nats.Msg) { + user, si, _, opts, _ := decodeAuthRequest(t, m.Data) + select { + case seen <- seenConnectOptions{jwt: opts.JWT, password: opts.Password}: + default: + } + if opts.JWT != _EMPTY_ || opts.Password != opaquePassword { + m.Respond(nil) + return + } + ujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, "", nil, 10*time.Minute, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } + + at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer at.Cleanup() + + mc, rc := authCalloutMQTTConnect(t, at.srv.getOpts().MQTT.Host, at.srv.getOpts().MQTT.Port, "mqtt-auth-callout", "ignored", opaquePassword) + defer mc.Close() + + got := require_ChanRead(t, seen, 2*time.Second) + require_Equal(t, got.jwt, _EMPTY_) + require_Equal(t, got.password, opaquePassword) + require_Equal(t, rc, mqttConnAckRCConnectionAccepted) +} + +func TestAuthCalloutMultiAccounts(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: ZZ + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + FOO {} + BAR {} + BAZ {} + } + authorization { + timeout: 1s + auth_callout { + # Needs to be a public account nkey, will work for both server config and operator mode. + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth ] + } + } + ` + handler := func(m *nats.Msg) { + user, si, ci, opts, _ := decodeAuthRequest(t, m.Data) + require_True(t, si.Name == "ZZ") + require_True(t, ci.Host == "127.0.0.1") + // Allow dlc user and map to the BAZ account. + if opts.Username == "dlc" && opts.Password == "zzz" { + ujwt := createAuthUser(t, user, _EMPTY_, "BAZ", "", nil, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else { + // Nil response signals no authentication. + m.Respond(nil) + } + } + + at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer at.Cleanup() + + // This one will use callout since not defined in server config. + nc := at.Connect(nats.UserInfo("dlc", "zzz")) + defer nc.Close() + + resp, err := nc.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response := ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + require_NoError(t, err) + userInfo := response.Data.(*UserInfo) + + require_True(t, userInfo.UserID == "dlc") + require_True(t, userInfo.Account == "BAZ") +} + +func TestAuthCalloutAllowedAccounts(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: ZZ + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + FOO { users [ {user: "foo", password: "pwd"} ] } + BAR {} + SYS { users [ {user: "sys", password: "pwd"} ] } + } + system_account: SYS + no_auth_user: foo + authorization { + timeout: 1s + auth_callout { + # Needs to be a public account nkey, will work for both server config and operator mode. + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth ] + allowed_accounts: [ BAR ] + } + } + ` + handler := func(m *nats.Msg) { + t.Helper() + user, si, ci, opts, _ := decodeAuthRequest(t, m.Data) + require_True(t, si.Name == "ZZ") + require_True(t, ci.Host == "127.0.0.1") + // Allow dlc user and map to the BAZ account. + if opts.Username == "dlc" && opts.Password == "zzz" { + ujwt := createAuthUser(t, user, _EMPTY_, "BAR", "", nil, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else { + // Nil response signals no authentication. + m.Respond(nil) + } + } + + check := func(at *authTest, user, password, account string) { + t.Helper() + + var nc *nats.Conn + // Assume no auth user. + if password == "" { + nc = at.Connect() + } else { + nc = at.Connect(nats.UserInfo(user, password)) + } + defer nc.Close() + + resp, err := nc.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response := ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + require_NoError(t, err) + userInfo := response.Data.(*UserInfo) + + require_True(t, userInfo.UserID == user) + require_True(t, userInfo.Account == account) + } + + tests := []struct { + user string + password string + account string + }{ + {"dlc", "zzz", "BAR"}, + {"foo", "", "FOO"}, + {"sys", "pwd", "SYS"}, + } + + at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer at.Cleanup() + + for _, test := range tests { + t.Run(test.user, func(t *testing.T) { + at.t = t + check(at, test.user, test.password, test.account) + }) + } +} + +// A credential-less client that sends a non-CONNECT protocol op first (e.g. +// PING) must be run through the same auth callout as a CONNECT client. Prior to +// the fix, the pre-CONNECT no_auth_user fast path in the parser registered the +// user directly and never invoked the callout, letting a client the callout +// would have denied become fully authorized. +func TestAuthCalloutNoAuthUserPreConnectInvokesCallout(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: ZZ + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + FOO { users [ {user: "foo", password: "pwd"} ] } + SYS { users [ {user: "sys", password: "pwd"} ] } + } + system_account: SYS + no_auth_user: foo + authorization { + timeout: 1s + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth ] + # allowed_accounts absent => every account (including FOO, the + # no_auth_user's account) is in scope, so the callout is the gate. + } + } + ` + var calloutInvoked atomic.Int32 + handler := func(m *nats.Msg) { + calloutInvoked.Add(1) + // Deny everyone: a nil response signals no authentication. + m.Respond(nil) + } + + at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer at.Cleanup() + + // Raw TCP connection so we control the first protocol op. Send PING before + // any CONNECT, which is what triggers the pre-CONNECT fast path. + c, err := net.Dial("tcp", at.srv.Addr().String()) + require_NoError(t, err) + defer c.Close() + + br := bufio.NewReader(c) + // Read and discard the initial INFO line. + _, err = br.ReadString('\n') + require_NoError(t, err) + + _, err = c.Write([]byte("PING\r\n")) + require_NoError(t, err) + + require_NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) + line, err := br.ReadString('\n') + require_NoError(t, err) + + // The callout denied the client, so we must get an authorization violation, + // never a PONG. + require_False(t, strings.HasPrefix(line, "PONG")) + require_Contains(t, line, "Authorization Violation") + + // The callout must actually have been consulted on this path. + require_True(t, calloutInvoked.Load() > 0) +} + +// A PING-first no_auth_user admitted by an auth callout with an expiring JWT +// must still be disconnected at expiry. The pre-CONNECT fast path installs the +// JWT expiration timer and the auth-timeout timer in the same c.atmr slot, so +// it must clear the auth timer before authenticating (mirroring processConnect) +// rather than after, or it would drop the expiration timer and leave the client +// connected past expiry. +func TestAuthCalloutNoAuthUserPreConnectHonorsExpiration(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: ZZ + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + FOO { users [ {user: "foo", password: "pwd"} ] } + SYS { users [ {user: "sys", password: "pwd"} ] } + } + system_account: SYS + no_auth_user: foo + authorization { + timeout: 1s + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth ] + } + } + ` + handler := func(m *nats.Msg) { + user, si, _, _, _ := decodeAuthRequest(t, m.Data) + // Grant, mapping to FOO with a short JWT expiry so the server must + // install and honor an expiration timer. + ujwt := createAuthUser(t, user, "foo", "FOO", "", nil, time.Second, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } + + at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer at.Cleanup() + + c, err := net.Dial("tcp", at.srv.Addr().String()) + require_NoError(t, err) + defer c.Close() + + br := bufio.NewReader(c) + // Read and discard the initial INFO line. + _, err = br.ReadString('\n') + require_NoError(t, err) + + _, err = c.Write([]byte("PING\r\n")) + require_NoError(t, err) + + // The callout granted the connection, so the first reply is a PONG. + require_NoError(t, c.SetReadDeadline(time.Now().Add(2*time.Second))) + line, err := br.ReadString('\n') + require_NoError(t, err) + require_True(t, strings.HasPrefix(line, "PONG")) + + // The JWT expires ~1s out. The expiration timer must survive and fire, + // disconnecting us with an auth-expired error. + require_NoError(t, c.SetReadDeadline(time.Now().Add(5*time.Second))) + var expired bool + for { + line, err = br.ReadString('\n') + if err != nil { + // Server closed the connection after signaling expiry. + break + } + if strings.Contains(line, "User Authentication Expired") { + expired = true + } + } + require_True(t, expired) +} + +func TestAuthCalloutClientTLSCerts(t *testing.T) { + conf := ` + listen: "localhost:-1" + server_name: T + + tls { + cert_file = "../test/configs/certs/tlsauth/server.pem" + key_file = "../test/configs/certs/tlsauth/server-key.pem" + ca_file = "../test/configs/certs/tlsauth/ca.pem" + verify = true + } + + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + FOO {} + } + authorization { + timeout: 1s + auth_callout { + # Needs to be a public account nkey, will work for both server config and operator mode. + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth ] + } + } + ` + handler := func(m *nats.Msg) { + user, si, ci, _, ctls := decodeAuthRequest(t, m.Data) + require_True(t, si.Name == "T") + require_True(t, ci.Host == "127.0.0.1") + require_True(t, ctls != nil) + // Zero since we are verified and will be under verified chains. + require_True(t, len(ctls.Certs) == 0) + require_True(t, len(ctls.VerifiedChains) == 1) + // Since we have a CA. + require_True(t, len(ctls.VerifiedChains[0]) == 2) + blk, _ := pem.Decode([]byte(ctls.VerifiedChains[0][0])) + cert, err := x509.ParseCertificate(blk.Bytes) + require_NoError(t, err) + if strings.HasPrefix(cert.Subject.String(), "CN=example.com") { + // Override blank name here, server will substitute. + ujwt := createAuthUser(t, user, "dlc", "FOO", "", nil, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } + } + + ac := NewAuthTest(t, conf, handler, + nats.UserInfo("auth", "pwd"), + nats.ClientCert("../test/configs/certs/tlsauth/client2.pem", "../test/configs/certs/tlsauth/client2-key.pem"), + nats.RootCAs("../test/configs/certs/tlsauth/ca.pem")) + defer ac.Cleanup() + + // Will use client cert to determine user. + nc := ac.Connect( + nats.ClientCert("../test/configs/certs/tlsauth/client2.pem", "../test/configs/certs/tlsauth/client2-key.pem"), + nats.RootCAs("../test/configs/certs/tlsauth/ca.pem"), + ) + defer nc.Close() + + resp, err := nc.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response := ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + require_NoError(t, err) + userInfo := response.Data.(*UserInfo) + + require_True(t, userInfo.UserID == "dlc") + require_True(t, userInfo.Account == "FOO") +} + +func TestAuthCalloutVerifiedUserCalloutsWithSig(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: A + authorization { + timeout: 1s + users: [ + { user: "auth", password: "pwd" } + { nkey: "UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON" } + ] + auth_callout { + # Needs to be a public account nkey, will work for both server config and operator mode. + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + # users that will power the auth callout service. + auth_users: [ auth ] + } + } + ` + callouts := uint32(0) + handler := func(m *nats.Msg) { + atomic.AddUint32(&callouts, 1) + user, si, ci, opts, _ := decodeAuthRequest(t, m.Data) + require_True(t, si.Name == "A") + require_True(t, ci.Host == "127.0.0.1") + require_True(t, opts.SignedNonce != _EMPTY_) + require_True(t, ci.Nonce != _EMPTY_) + ujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, "", nil, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } + ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer ac.Cleanup() + + seedFile := createTempFile(t, _EMPTY_) + defer removeFile(t, seedFile.Name()) + seedFile.WriteString(authCalloutSeed) + nkeyOpt, err := nats.NkeyOptionFromSeed(seedFile.Name()) + require_NoError(t, err) + + nc := ac.Connect(nkeyOpt) + defer nc.Close() + + // Make sure that the callout was called. + if atomic.LoadUint32(&callouts) != 1 { + t.Fatalf("Expected callout to be called") + } + + resp, err := nc.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response := ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + require_NoError(t, err) + + userInfo := response.Data.(*UserInfo) + + dlc := &UserInfo{ + UserID: "UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON", + Account: globalAccountName, + AccountName: globalAccountName, + Permissions: &Permissions{ + Publish: &SubjectPermission{ + Deny: []string{AuthCalloutSubject}, // Will be auto-added since in auth account. + }, + Subscribe: &SubjectPermission{}, + }, + } + if !reflect.DeepEqual(dlc, userInfo) { + t.Fatalf("User info for %q did not match", "dlc") + } +} + +// For creating the authorized users in operator mode. +func createAuthServiceUser(t *testing.T, accKp nkeys.KeyPair) (pub, creds string) { + t.Helper() + ukp, _ := nkeys.CreateUser() + seed, _ := ukp.Seed() + upub, _ := ukp.PublicKey() + uclaim := newJWTTestUserClaims() + uclaim.Name = "auth-service" + uclaim.Subject = upub + vr := jwt.ValidationResults{} + uclaim.Validate(&vr) + require_Len(t, len(vr.Errors()), 0) + ujwt, err := uclaim.Encode(accKp) + require_NoError(t, err) + return upub, genCredsFile(t, ujwt, seed) +} + +func createBasicAccountUser(t *testing.T, accKp nkeys.KeyPair) (creds string) { + return createBasicAccount(t, "auth-client", accKp, true) +} + +func createBasicAccountLeaf(t *testing.T, accKp nkeys.KeyPair) (creds string) { + return createBasicAccount(t, "auth-leaf", accKp, false) +} + +func createBasicAccountBearer(t *testing.T, accKp nkeys.KeyPair) string { + t.Helper() + ukp, _ := nkeys.CreateUser() + upub, _ := ukp.PublicKey() + uclaim := newJWTTestUserClaims() + uclaim.Name = "default_sentinel" + uclaim.Subject = upub + uclaim.BearerToken = true + + uclaim.Permissions.Pub.Deny.Add(">") + uclaim.Permissions.Sub.Deny.Add(">") + + vr := jwt.ValidationResults{} + uclaim.Validate(&vr) + require_Len(t, len(vr.Errors()), 0) + ujwt, err := uclaim.Encode(accKp) + require_NoError(t, err) + return ujwt +} + +func createBasicAccount(t *testing.T, name string, accKp nkeys.KeyPair, addDeny bool) (creds string) { + t.Helper() + ukp, _ := nkeys.CreateUser() + seed, _ := ukp.Seed() + upub, _ := ukp.PublicKey() + uclaim := newJWTTestUserClaims() + uclaim.Subject = upub + uclaim.Name = name + if addDeny { + // For these deny all permission + uclaim.Permissions.Pub.Deny.Add(">") + uclaim.Permissions.Sub.Deny.Add(">") + } + vr := jwt.ValidationResults{} + uclaim.Validate(&vr) + require_Len(t, len(vr.Errors()), 0) + ujwt, err := uclaim.Encode(accKp) + require_NoError(t, err) + return genCredsFile(t, ujwt, seed) +} + +func createScopedUser(t *testing.T, accKp nkeys.KeyPair, sk nkeys.KeyPair) (creds string) { + t.Helper() + ukp, _ := nkeys.CreateUser() + seed, _ := ukp.Seed() + upub, _ := ukp.PublicKey() + uclaim := newJWTTestUserClaims() + apk, _ := accKp.PublicKey() + uclaim.IssuerAccount = apk + uclaim.Subject = upub + uclaim.Name = "scoped-user" + uclaim.SetScoped(true) + + // Uncomment this to set the sub limits + // uclaim.Limits.Subs = 0 + vr := jwt.ValidationResults{} + uclaim.Validate(&vr) + require_Len(t, len(vr.Errors()), 0) + ujwt, err := uclaim.Encode(sk) + require_NoError(t, err) + return genCredsFile(t, ujwt, seed) +} + +func TestAuthCalloutOperatorNoServerConfigCalloutAllowed(t *testing.T) { + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:-1 + operator: %s + resolver: MEM + authorization { + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + auth_users: [ auth ] + } + } + `, ojwt))) + defer removeFile(t, conf) + + opts := LoadConfig(conf) + _, err := NewServer(opts) + require_Error(t, err, errors.New("operators do not allow authorization callouts to be configured directly")) +} + +func TestAuthCalloutOperatorModeBasics(t *testing.T) { + _, spub := createKey(t) + sysClaim := jwt.NewAccountClaims(spub) + sysClaim.Name = "$SYS" + sysJwt, err := sysClaim.Encode(oKp) + require_NoError(t, err) + + // TEST account. + tkp, tpub := createKey(t) + tSigningKp, tSigningPub := createKey(t) + accClaim := jwt.NewAccountClaims(tpub) + accClaim.Name = "TEST" + accClaim.SigningKeys.Add(tSigningPub) + scope, scopedKp := newScopedRole(t, "foo", []string{"foo.>", "$SYS.REQ.USER.INFO"}, []string{"foo.>", "_INBOX.>"}, false) + accClaim.SigningKeys.AddScopedSigner(scope) + accJwt, err := accClaim.Encode(oKp) + require_NoError(t, err) + + // AUTH service account. + akp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) + require_NoError(t, err) + + apub, err := akp.PublicKey() + require_NoError(t, err) + + // The authorized user for the service. + upub, creds := createAuthServiceUser(t, akp) + defer removeFile(t, creds) + + authClaim := jwt.NewAccountClaims(apub) + authClaim.Name = "AUTH" + authClaim.EnableExternalAuthorization(upub) + authClaim.Authorization.AllowedAccounts.Add(tpub) + authJwt, err := authClaim.Encode(oKp) + require_NoError(t, err) + + defaultSentinel := createBasicAccountBearer(t, akp) + + conf := fmt.Sprintf(` + listen: 127.0.0.1:-1 + operator: %s + system_account: %s + resolver: MEM + resolver_preload: { + %s: %s + %s: %s + %s: %s + } + default_sentinel: %s + `, ojwt, spub, apub, authJwt, tpub, accJwt, spub, sysJwt, defaultSentinel) + + const secretToken = "--XX--" + const dummyToken = "--ZZ--" + const skKeyToken = "--SK--" + const scopedToken = "--Scoped--" + const badScopedToken = "--BADScoped--" + const defaultToken = "--Default--" + + dkp, notAllowAccountPub := createKey(t) + handler := func(m *nats.Msg) { + user, si, _, opts, _ := decodeAuthRequest(t, m.Data) + if opts.Token == secretToken { + ujwt := createAuthUser(t, user, "dlc", tpub, "", tkp, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else if opts.Token == dummyToken { + ujwt := createAuthUser(t, user, "dummy", notAllowAccountPub, "", dkp, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else if opts.Token == skKeyToken { + ujwt := createAuthUser(t, user, "sk", tpub, tpub, tSigningKp, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else if opts.Token == scopedToken { + // must have no limits set + ujwt := createAuthUser(t, user, "scoped", tpub, tpub, scopedKp, 0, &jwt.UserPermissionLimits{}) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else if opts.Token == badScopedToken { + // limits are nil - here which result in a default user - this will fail scoped + ujwt := createAuthUser(t, user, "bad-scoped", tpub, tpub, scopedKp, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else if opts.Token == defaultToken { + ujwt := createAuthUser(t, user, "default", tpub, "", tkp, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else { + m.Respond(nil) + } + } + + ac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) + defer ac.Cleanup() + resp, err := ac.authClient.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response := ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + require_NoError(t, err) + + userInfo := response.Data.(*UserInfo) + expected := &UserInfo{ + UserID: upub, + Account: apub, + AccountName: "AUTH", + UserName: "auth-service", + Permissions: &Permissions{ + Publish: &SubjectPermission{ + Deny: []string{AuthCalloutSubject}, // Will be auto-added since in auth account. + }, + Subscribe: &SubjectPermission{}, + }, + } + if !reflect.DeepEqual(expected, userInfo) { + t.Fatalf("User info did not match expected, expected auto-deny permissions on callout subject") + } + + // Bearer token etc.. + // This is used by all users, and the customization will be in other connect args. + // This needs to also be bound to the authorization account. + creds = createBasicAccountUser(t, akp) + defer removeFile(t, creds) + + // We require a token. + ac.RequireConnectError(nats.UserCredentials(creds)) + + // Send correct token. This should switch us to the test account. + nc := ac.Connect(nats.UserCredentials(creds), nats.Token(secretToken)) + require_NoError(t, err) + defer nc.Close() + + resp, err = nc.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response = ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + require_NoError(t, err) + + userInfo = response.Data.(*UserInfo) + + // Make sure we switch accounts. + if userInfo.Account != tpub { + t.Fatalf("Expected to be switched to %q, but got %q", tpub, userInfo.Account) + } + + // Now make sure that if the authorization service switches to an account that is not allowed, we reject. + ac.RequireConnectError(nats.UserCredentials(creds), nats.Token(dummyToken)) + + // Send the signing key token. This should switch us to the test account, but the user + // is signed with the account signing key + nc = ac.Connect(nats.UserCredentials(creds), nats.Token(skKeyToken)) + defer nc.Close() + + resp, err = nc.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response = ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + require_NoError(t, err) + + userInfo = response.Data.(*UserInfo) + if userInfo.Account != tpub { + t.Fatalf("Expected to be switched to %q, but got %q", tpub, userInfo.Account) + } + + // bad scoped user + ac.RequireConnectError(nats.UserCredentials(creds), nats.Token(badScopedToken)) + + // Send the signing key token. This should switch us to the test account, but the user + // is signed with the account signing key + nc = ac.Connect(nats.UserCredentials(creds), nats.Token(scopedToken)) + require_NoError(t, err) + defer nc.Close() + + resp, err = nc.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response = ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + require_NoError(t, err) + + userInfo = response.Data.(*UserInfo) + if userInfo.Account != tpub { + t.Fatalf("Expected to be switched to %q, but got %q", tpub, userInfo.Account) + } + require_True(t, len(userInfo.Permissions.Publish.Allow) == 2) + slices.Sort(userInfo.Permissions.Publish.Allow) + require_Equal(t, "foo.>", userInfo.Permissions.Publish.Allow[1]) + slices.Sort(userInfo.Permissions.Subscribe.Allow) + require_True(t, len(userInfo.Permissions.Subscribe.Allow) == 2) + require_Equal(t, "foo.>", userInfo.Permissions.Subscribe.Allow[1]) + + // this connects without a credential, so will be assigned the default sentinel + nc = ac.Connect(nats.Token(defaultToken)) + require_NoError(t, err) + defer nc.Close() + + resp, err = nc.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response = ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + ui := response.Data.(*UserInfo) + require_Equal(t, "default", ui.UserID) + require_NoError(t, err) + +} + +func testAuthCalloutScopedUser(t *testing.T, allowAnyAccount bool) { + _, spub := createKey(t) + sysClaim := jwt.NewAccountClaims(spub) + sysClaim.Name = "$SYS" + sysJwt, err := sysClaim.Encode(oKp) + require_NoError(t, err) + + // TEST account. + _, tpub := createKey(t) + _, tSigningPub := createKey(t) + accClaim := jwt.NewAccountClaims(tpub) + accClaim.Name = "TEST" + accClaim.SigningKeys.Add(tSigningPub) + scope, scopedKp := newScopedRole(t, "foo", []string{"foo.>", "$SYS.REQ.USER.INFO"}, []string{"foo.>", "_INBOX.>"}, true) + scope.Template.Limits.Subs = 10 + scope.Template.Limits.Payload = 512 + accClaim.SigningKeys.AddScopedSigner(scope) + accJwt, err := accClaim.Encode(oKp) + require_NoError(t, err) + + // AUTH service account. + akp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) + require_NoError(t, err) + + apub, err := akp.PublicKey() + require_NoError(t, err) + + // The authorized user for the service. + upub, creds := createAuthServiceUser(t, akp) + defer removeFile(t, creds) + + authClaim := jwt.NewAccountClaims(apub) + authClaim.Name = "AUTH" + authClaim.EnableExternalAuthorization(upub) + if allowAnyAccount { + authClaim.Authorization.AllowedAccounts.Add("*") + } else { + authClaim.Authorization.AllowedAccounts.Add(tpub) + } + // the scope for the bearer token which has no permissions + sentinelScope, authKP := newScopedRole(t, "sentinel", nil, nil, false) + sentinelScope.Template.Sub.Deny.Add(">") + sentinelScope.Template.Pub.Deny.Add(">") + sentinelScope.Template.Limits.Subs = 0 + sentinelScope.Template.Payload = 0 + authClaim.SigningKeys.AddScopedSigner(sentinelScope) + + authJwt, err := authClaim.Encode(oKp) + require_NoError(t, err) + + conf := fmt.Sprintf(` + listen: 127.0.0.1:-1 + operator: %s + system_account: %s + resolver: MEM + resolver_preload: { + %s: %s + %s: %s + %s: %s + } + `, ojwt, spub, apub, authJwt, tpub, accJwt, spub, sysJwt) + + const scopedToken = "--Scoped--" + handler := func(m *nats.Msg) { + user, si, _, opts, _ := decodeAuthRequest(t, m.Data) + if opts.Token == scopedToken { + // must have no limits set + ujwt := createAuthUser(t, user, "scoped", tpub, tpub, scopedKp, 0, &jwt.UserPermissionLimits{}) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else { + m.Respond(nil) + } + } + + ac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) + defer ac.Cleanup() + resp, err := ac.authClient.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response := ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + require_NoError(t, err) + + userInfo := response.Data.(*UserInfo) + expected := &UserInfo{ + UserID: upub, + Account: apub, + AccountName: "AUTH", + UserName: "auth-service", + Permissions: &Permissions{ + Publish: &SubjectPermission{ + Deny: []string{AuthCalloutSubject}, // Will be auto-added since in auth account. + }, + Subscribe: &SubjectPermission{}, + }, + } + if !reflect.DeepEqual(expected, userInfo) { + t.Fatalf("User info did not match expected, expected auto-deny permissions on callout subject") + } + + // Bearer token - this has no permissions see sentinelScope + // This is used by all users, and the customization will be in other connect args. + // This needs to also be bound to the authorization account. + creds = createScopedUser(t, akp, authKP) + defer removeFile(t, creds) + + // Send the signing key token. This should switch us to the test account, but the user + // is signed with the account signing key + nc := ac.Connect(nats.UserCredentials(creds), nats.Token(scopedToken)) + + resp, err = nc.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response = ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + require_NoError(t, err) + + userInfo = response.Data.(*UserInfo) + if userInfo.Account != tpub { + t.Fatalf("Expected to be switched to %q, but got %q", tpub, userInfo.Account) + } + require_True(t, len(userInfo.Permissions.Publish.Allow) == 2) + slices.Sort(userInfo.Permissions.Publish.Allow) + require_Equal(t, "foo.>", userInfo.Permissions.Publish.Allow[1]) + slices.Sort(userInfo.Permissions.Subscribe.Allow) + require_True(t, len(userInfo.Permissions.Subscribe.Allow) == 2) + require_Equal(t, "foo.>", userInfo.Permissions.Subscribe.Allow[1]) + + _, err = nc.Subscribe("foo.>", func(msg *nats.Msg) { + t.Log("got request on foo.>") + require_NoError(t, msg.Respond(nil)) + }) + require_NoError(t, err) + + m, err := nc.Request("foo.bar", nil, time.Second) + require_NoError(t, err) + require_NotNil(t, m) + t.Log("go response from foo.bar") + + nc.Close() +} + +func TestAuthCalloutScopedUserAssignedAccount(t *testing.T) { + testAuthCalloutScopedUser(t, false) +} + +func TestAuthCalloutScopedUserAllAccount(t *testing.T) { + testAuthCalloutScopedUser(t, true) +} + +const ( + curveSeed = "SXAAXMRAEP6JWWHNB6IKFL554IE6LZVT6EY5MBRICPILTLOPHAG73I3YX4" + curvePublic = "XAB3NANV3M6N7AHSQP2U5FRWKKUT7EG2ZXXABV4XVXYQRJGM4S2CZGHT" +) + +func TestAuthCalloutServerConfigEncryption(t *testing.T) { + tmpl := ` + listen: "127.0.0.1:-1" + server_name: A + authorization { + timeout: 1s + users: [ { user: "auth", password: "pwd" } ] + auth_callout { + # Needs to be a public account nkey, will work for both server config and operator mode. + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + # users that will power the auth callout service. + auth_users: [ auth ] + # This is a public xkey (x25519). The auth service has the private key. + xkey: "%s" + } + } + ` + conf := fmt.Sprintf(tmpl, curvePublic) + + rkp, err := nkeys.FromCurveSeed([]byte(curveSeed)) + require_NoError(t, err) + + handler := func(m *nats.Msg) { + // This will be encrypted. + _, err := jwt.DecodeAuthorizationRequestClaims(string(m.Data)) + require_Error(t, err) + + xkey := m.Header.Get(AuthRequestXKeyHeader) + require_True(t, xkey != _EMPTY_) + decrypted, err := rkp.Open(m.Data, xkey) + require_NoError(t, err) + user, si, ci, opts, _ := decodeAuthRequest(t, decrypted) + // The header xkey must match the signed xkey in server info. + require_True(t, si.XKey == xkey) + require_True(t, si.Name == "A") + require_True(t, ci.Host == "127.0.0.1") + // Allow dlc user. + if opts.Username == "dlc" && opts.Password == "zzz" { + ujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, "", nil, 10*time.Minute, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else if opts.Username == "dlc" && opts.Password == "xxx" { + ujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, "", nil, 10*time.Minute, nil) + // Encrypt this response. + data, err := rkp.Seal(serviceResponse(t, user, si.ID, ujwt, "", 0), si.XKey) // Server's public xkey. + require_NoError(t, err) + m.Respond(data) + } else { + // Nil response signals no authentication. + m.Respond(nil) + } + } + + ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer ac.Cleanup() + + nc := ac.Connect(nats.UserInfo("dlc", "zzz")) + defer nc.Close() + + // Authorization services can optionally encrypt the responses using the server's public xkey. + nc = ac.Connect(nats.UserInfo("dlc", "xxx")) + defer nc.Close() +} + +func TestAuthCalloutOperatorModeEncryption(t *testing.T) { + _, spub := createKey(t) + sysClaim := jwt.NewAccountClaims(spub) + sysClaim.Name = "$SYS" + sysJwt, err := sysClaim.Encode(oKp) + require_NoError(t, err) + + // TEST account. + tkp, tpub := createKey(t) + accClaim := jwt.NewAccountClaims(tpub) + accClaim.Name = "TEST" + accJwt, err := accClaim.Encode(oKp) + require_NoError(t, err) + + // AUTH service account. + akp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) + require_NoError(t, err) + + apub, err := akp.PublicKey() + require_NoError(t, err) + + // The authorized user for the service. + upub, creds := createAuthServiceUser(t, akp) + defer removeFile(t, creds) + + authClaim := jwt.NewAccountClaims(apub) + authClaim.Name = "AUTH" + authClaim.EnableExternalAuthorization(upub) + authClaim.Authorization.AllowedAccounts.Add(tpub) + authClaim.Authorization.XKey = curvePublic + + authJwt, err := authClaim.Encode(oKp) + require_NoError(t, err) + + conf := fmt.Sprintf(` + listen: 127.0.0.1:-1 + operator: %s + system_account: %s + resolver: MEM + resolver_preload: { + %s: %s + %s: %s + %s: %s + } + `, ojwt, spub, apub, authJwt, tpub, accJwt, spub, sysJwt) + + rkp, err := nkeys.FromCurveSeed([]byte(curveSeed)) + require_NoError(t, err) + + const tokenA = "--XX--" + const tokenB = "--ZZ--" + + handler := func(m *nats.Msg) { + // Make sure this is an encrypted request. + if bytes.HasPrefix(m.Data, []byte(jwtPrefix)) { + t.Fatalf("Request not encrypted") + } + xkey := m.Header.Get(AuthRequestXKeyHeader) + require_True(t, xkey != _EMPTY_) + decrypted, err := rkp.Open(m.Data, xkey) + require_NoError(t, err) + user, si, ci, opts, _ := decodeAuthRequest(t, decrypted) + // The header xkey must match the signed xkey in server info. + require_True(t, si.XKey == xkey) + require_True(t, ci.Host == "127.0.0.1") + if opts.Token == tokenA { + ujwt := createAuthUser(t, user, "dlc", tpub, "", tkp, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else if opts.Token == tokenB { + ujwt := createAuthUser(t, user, "rip", tpub, "", tkp, 0, nil) + // Encrypt this response. + data, err := rkp.Seal(serviceResponse(t, user, si.ID, ujwt, "", 0), si.XKey) // Server's public xkey. + require_NoError(t, err) + m.Respond(data) + } else { + m.Respond(nil) + } + } + + ac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) + defer ac.Cleanup() + + // Bearer token etc.. + // This is used by all users, and the customization will be in other connect args. + // This needs to also be bound to the authorization account. + creds = createBasicAccountUser(t, akp) + defer removeFile(t, creds) + + // This will receive an encrypted request to the auth service but send plaintext response. + nc := ac.Connect(nats.UserCredentials(creds), nats.Token(tokenA)) + defer nc.Close() + + // This will receive an encrypted request to the auth service and send an encrypted response. + nc = ac.Connect(nats.UserCredentials(creds), nats.Token(tokenB)) + defer nc.Close() +} + +func TestAuthCalloutServerTags(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: A + server_tags: ["foo", "bar"] + authorization { + users: [ { user: "auth", password: "pwd" } ] + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + auth_users: [ auth ] + } + } + ` + + tch := make(chan jwt.TagList, 1) + handler := func(m *nats.Msg) { + user, si, _, _, _ := decodeAuthRequest(t, m.Data) + tch <- si.Tags + ujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, "", nil, 10*time.Minute, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } + + ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer ac.Cleanup() + + nc := ac.Connect() + defer nc.Close() + + tags := <-tch + require_True(t, len(tags) == 2) + require_True(t, tags.Contains("foo")) + require_True(t, tags.Contains("bar")) +} + +func TestAuthCalloutServerClusterAndVersion(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: A + authorization { + users: [ { user: "auth", password: "pwd" } ] + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + auth_users: [ auth ] + } + } + cluster { name: HUB } + ` + ch := make(chan string, 2) + handler := func(m *nats.Msg) { + user, si, _, _, _ := decodeAuthRequest(t, m.Data) + ch <- si.Cluster + ch <- si.Version + ujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, "", nil, 10*time.Minute, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } + + ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer ac.Cleanup() + + nc := ac.Connect() + defer nc.Close() + + cluster := <-ch + require_True(t, cluster == "HUB") + + version := <-ch + require_True(t, len(version) > 0) + ok, err := versionAtLeastCheckError(version, 2, 10, 0) + require_NoError(t, err) + require_True(t, ok) +} + +func TestAuthCalloutErrorResponse(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: A + authorization { + users: [ { user: "auth", password: "pwd" } ] + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + auth_users: [ auth ] + } + } + ` + handler := func(m *nats.Msg) { + user, si, _, _, _ := decodeAuthRequest(t, m.Data) + m.Respond(serviceResponse(t, user, si.ID, "", "BAD AUTH", 0)) + } + + ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer ac.Cleanup() + + ac.RequireConnectError(nats.UserInfo("dlc", "zzz")) +} + +func TestAuthCalloutAuthUserFailDoesNotInvokeCallout(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: A + authorization { + users: [ { user: "auth", password: "pwd" } ] + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + auth_users: [ auth ] + } + } + ` + callouts := uint32(0) + handler := func(m *nats.Msg) { + user, si, _, _, _ := decodeAuthRequest(t, m.Data) + atomic.AddUint32(&callouts, 1) + m.Respond(serviceResponse(t, user, si.ID, "", "WRONG PASSWORD", 0)) + } + + ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer ac.Cleanup() + + ac.RequireConnectError(nats.UserInfo("auth", "zzz")) + + if atomic.LoadUint32(&callouts) != 0 { + t.Fatalf("Expected callout to not be called") + } +} + +func TestAuthCalloutAuthErrEvents(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: A + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + FOO {} + BAR {} + } + authorization { + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth ] + } + } + ` + + handler := func(m *nats.Msg) { + user, si, _, opts, _ := decodeAuthRequest(t, m.Data) + // Allow dlc user and map to the BAZ account. + if opts.Username == "dlc" && opts.Password == "zzz" { + ujwt := createAuthUser(t, user, _EMPTY_, "FOO", "", nil, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else if opts.Username == "dlc" { + m.Respond(serviceResponse(t, user, si.ID, "", "WRONG PASSWORD", 0)) + } else if opts.Username == "rip" { + m.Respond(serviceResponse(t, user, si.ID, "", "BAD CREDS", 0)) + } else if opts.Username == "proxy" { + var j jwt.UserPermissionLimits + j.ProxyRequired = true + ujwt := createAuthUser(t, user, _EMPTY_, "FOO", "", nil, 0, &j) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } + } + + ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer ac.Cleanup() + + // This is where the event fires, in this account. + sub, err := ac.authClient.SubscribeSync(authErrorAccountEventSubj) + require_NoError(t, err) + + // This one will use callout since not defined in server config. + nc := ac.Connect(nats.UserInfo("dlc", "zzz")) + defer nc.Close() + checkSubsPending(t, sub, 0) + + checkAuthErrEvent := func(user, pass, reason string) { + ac.RequireConnectError(nats.UserInfo(user, pass)) + + m, err := sub.NextMsg(time.Second) + require_NoError(t, err) + + var dm DisconnectEventMsg + err = json.Unmarshal(m.Data, &dm) + require_NoError(t, err) + + // Convert both reasons to lower case to do the comparison. + dmr := strings.ToLower(dm.Reason) + r := strings.ToLower(reason) + + if !strings.Contains(dmr, r) { + t.Fatalf("Expected %q reason, but got %q", r, dmr) + } + } + + checkAuthErrEvent("dlc", "xxx", "WRONG PASSWORD") + checkAuthErrEvent("rip", "abc", "BAD CREDS") + // The auth callout uses as the reason the error string, not a closed state. + checkAuthErrEvent("proxy", "proxy", ErrAuthProxyRequired.Error()) +} + +func TestAuthCalloutConnectEvents(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: A + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + FOO {} + BAR {} + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } + authorization { + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth, admin ] + } + } + ` + + handler := func(m *nats.Msg) { + user, si, _, opts, _ := decodeAuthRequest(t, m.Data) + // Allow dlc user and map to the BAZ account. + if opts.Username == "dlc" && opts.Password == "zzz" { + ujwt := createAuthUser(t, user, _EMPTY_, "FOO", "", nil, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else if opts.Username == "rip" && opts.Password == "xxx" { + ujwt := createAuthUser(t, user, _EMPTY_, "BAR", "", nil, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else { + m.Respond(serviceResponse(t, user, si.ID, "", "BAD CREDS", 0)) + } + } + + ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer ac.Cleanup() + + // Setup system user. + snc := ac.Connect(nats.UserInfo("admin", "s3cr3t!")) + defer snc.Close() + + // Allow this connect event to pass us by.. + time.Sleep(250 * time.Millisecond) + + // Watch for connect events. + csub, err := snc.SubscribeSync(fmt.Sprintf(connectEventSubj, "*")) + require_NoError(t, err) + + // Watch for disconnect events. + dsub, err := snc.SubscribeSync(fmt.Sprintf(disconnectEventSubj, "*")) + require_NoError(t, err) + + // Connections updates. Old + acOldSub, err := snc.SubscribeSync(fmt.Sprintf(accConnsEventSubjOld, "*")) + require_NoError(t, err) + + // Connections updates. New + acNewSub, err := snc.SubscribeSync(fmt.Sprintf(accConnsEventSubjNew, "*")) + require_NoError(t, err) + + snc.Flush() + + checkConnectEvents := func(user, pass, acc string) { + nc := ac.Connect(nats.UserInfo(user, pass)) + require_NoError(t, err) + + m, err := csub.NextMsg(time.Second) + require_NoError(t, err) + + var cm ConnectEventMsg + err = json.Unmarshal(m.Data, &cm) + require_NoError(t, err) + require_True(t, cm.Client.User == user) + require_True(t, cm.Client.Account == acc) + + // Check that we have updates, 1 each, for the connections updates. + m, err = acOldSub.NextMsg(time.Second) + require_NoError(t, err) + + var anc AccountNumConns + err = json.Unmarshal(m.Data, &anc) + require_NoError(t, err) + require_True(t, anc.AccountStat.Account == acc) + require_True(t, anc.AccountStat.Conns == 1) + + m, err = acNewSub.NextMsg(time.Second) + require_NoError(t, err) + + err = json.Unmarshal(m.Data, &anc) + require_NoError(t, err) + require_True(t, anc.AccountStat.Account == acc) + require_True(t, anc.AccountStat.Conns == 1) + + // Force the disconnect. + nc.Close() + + m, err = dsub.NextMsg(time.Second) + require_NoError(t, err) + + var dm DisconnectEventMsg + err = json.Unmarshal(m.Data, &dm) + require_NoError(t, err) + + m, err = acOldSub.NextMsg(time.Second) + require_NoError(t, err) + err = json.Unmarshal(m.Data, &anc) + require_NoError(t, err) + require_True(t, anc.AccountStat.Account == acc) + require_True(t, anc.AccountStat.Conns == 0) + + m, err = acNewSub.NextMsg(time.Second) + require_NoError(t, err) + err = json.Unmarshal(m.Data, &anc) + require_NoError(t, err) + require_True(t, anc.AccountStat.Account == acc) + require_True(t, anc.AccountStat.Conns == 0) + + // Make sure no double events sent. + time.Sleep(200 * time.Millisecond) + checkSubsPending(t, csub, 0) + checkSubsPending(t, dsub, 0) + checkSubsPending(t, acOldSub, 0) + checkSubsPending(t, acNewSub, 0) + } + + checkConnectEvents("dlc", "zzz", "FOO") + checkConnectEvents("rip", "xxx", "BAR") +} + +func TestAuthCalloutBadServer(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: A + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + FOO {} + } + authorization { + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth ] + } + } + ` + + handler := func(m *nats.Msg) { + user, _, _, _, _ := decodeAuthRequest(t, m.Data) + skp, err := nkeys.CreateServer() + require_NoError(t, err) + spk, err := skp.PublicKey() + require_NoError(t, err) + ujwt := createAuthUser(t, user, _EMPTY_, "FOO", "", nil, 0, nil) + m.Respond(serviceResponse(t, user, spk, ujwt, "", 0)) + } + + ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer ac.Cleanup() + + // This is where the event fires, in this account. + sub, err := ac.authClient.SubscribeSync(authErrorAccountEventSubj) + require_NoError(t, err) + + checkAuthErrEvent := func(user, pass, reason string) { + ac.RequireConnectError(nats.UserInfo(user, pass)) + + m, err := sub.NextMsg(time.Second) + require_NoError(t, err) + + var dm DisconnectEventMsg + err = json.Unmarshal(m.Data, &dm) + require_NoError(t, err) + + if !strings.Contains(dm.Reason, reason) { + t.Fatalf("Expected %q reason, but got %q", reason, dm.Reason) + } + } + checkAuthErrEvent("hello", "world", "response is not for server") +} + +func TestAuthCalloutBadUser(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: A + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + FOO {} + } + authorization { + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth ] + } + } + ` + + handler := func(m *nats.Msg) { + _, si, _, _, _ := decodeAuthRequest(t, m.Data) + kp, err := nkeys.CreateUser() + require_NoError(t, err) + upk, err := kp.PublicKey() + require_NoError(t, err) + ujwt := createAuthUser(t, upk, _EMPTY_, "FOO", "", nil, 0, nil) + m.Respond(serviceResponse(t, upk, si.ID, ujwt, "", 0)) + } + + ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer ac.Cleanup() + + // This is where the event fires, in this account. + sub, err := ac.authClient.SubscribeSync(authErrorAccountEventSubj) + require_NoError(t, err) + + checkAuthErrEvent := func(user, pass, reason string) { + ac.RequireConnectError(nats.UserInfo(user, pass)) + + m, err := sub.NextMsg(time.Second) + require_NoError(t, err) + + var dm DisconnectEventMsg + err = json.Unmarshal(m.Data, &dm) + require_NoError(t, err) + + if !strings.Contains(dm.Reason, reason) { + t.Fatalf("Expected %q reason, but got %q", reason, dm.Reason) + } + } + checkAuthErrEvent("hello", "world", "auth callout response is not for expected user") +} + +func TestAuthCalloutExpiredUser(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: A + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + FOO {} + } + authorization { + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth ] + } + } + ` + + handler := func(m *nats.Msg) { + user, si, _, _, _ := decodeAuthRequest(t, m.Data) + ujwt := createAuthUser(t, user, _EMPTY_, "FOO", "", nil, time.Second*-5, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } + + ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer ac.Cleanup() + + // This is where the event fires, in this account. + sub, err := ac.authClient.SubscribeSync(authErrorAccountEventSubj) + require_NoError(t, err) + + checkAuthErrEvent := func(user, pass, reason string) { + ac.RequireConnectError(nats.UserInfo(user, pass)) + + m, err := sub.NextMsg(time.Second) + require_NoError(t, err) + + var dm DisconnectEventMsg + err = json.Unmarshal(m.Data, &dm) + require_NoError(t, err) + + if !strings.Contains(dm.Reason, reason) { + t.Fatalf("Expected %q reason, but got %q", reason, dm.Reason) + } + } + checkAuthErrEvent("hello", "world", "claim is expired") +} + +func TestAuthCalloutExpiredResponse(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: A + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + FOO {} + } + authorization { + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth ] + } + } + ` + + handler := func(m *nats.Msg) { + user, si, _, _, _ := decodeAuthRequest(t, m.Data) + ujwt := createAuthUser(t, user, _EMPTY_, "FOO", "", nil, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", time.Second*-5)) + } + + ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer ac.Cleanup() + + // This is where the event fires, in this account. + sub, err := ac.authClient.SubscribeSync(authErrorAccountEventSubj) + require_NoError(t, err) + + checkAuthErrEvent := func(user, pass, reason string) { + ac.RequireConnectError(nats.UserInfo(user, pass)) + + m, err := sub.NextMsg(time.Second) + require_NoError(t, err) + + var dm DisconnectEventMsg + err = json.Unmarshal(m.Data, &dm) + require_NoError(t, err) + + if !strings.Contains(dm.Reason, reason) { + t.Fatalf("Expected %q reason, but got %q", reason, dm.Reason) + } + } + checkAuthErrEvent("hello", "world", "claim is expired") +} + +func TestAuthCalloutOperator_AnyAccount(t *testing.T) { + _, spub := createKey(t) + sysClaim := jwt.NewAccountClaims(spub) + sysClaim.Name = "$SYS" + sysJwt, err := sysClaim.Encode(oKp) + require_NoError(t, err) + + // A account. + akp, apk := createKey(t) + aClaim := jwt.NewAccountClaims(apk) + aClaim.Name = "A" + aJwt, err := aClaim.Encode(oKp) + require_NoError(t, err) + + // B account. + bkp, bpk := createKey(t) + bClaim := jwt.NewAccountClaims(bpk) + bClaim.Name = "B" + bJwt, err := bClaim.Encode(oKp) + require_NoError(t, err) + + // AUTH callout service account. + ckp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) + require_NoError(t, err) + + cpk, err := ckp.PublicKey() + require_NoError(t, err) + + // The authorized user for the service. + upub, creds := createAuthServiceUser(t, ckp) + defer removeFile(t, creds) + + authClaim := jwt.NewAccountClaims(cpk) + authClaim.Name = "AUTH" + authClaim.EnableExternalAuthorization(upub) + authClaim.Authorization.AllowedAccounts.Add("*") + authJwt, err := authClaim.Encode(oKp) + require_NoError(t, err) + + conf := fmt.Sprintf(` + listen: 127.0.0.1:-1 + operator: %s + system_account: %s + resolver: MEM + resolver_preload: { + %s: %s + %s: %s + %s: %s + %s: %s + } + `, ojwt, spub, cpk, authJwt, apk, aJwt, bpk, bJwt, spub, sysJwt) + + handler := func(m *nats.Msg) { + user, si, _, opts, _ := decodeAuthRequest(t, m.Data) + if opts.Token == "PutMeInA" { + ujwt := createAuthUser(t, user, "user_a", apk, "", akp, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else if opts.Token == "PutMeInB" { + ujwt := createAuthUser(t, user, "user_b", bpk, "", bkp, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else { + m.Respond(nil) + } + + } + + ac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) + defer ac.Cleanup() + resp, err := ac.authClient.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response := ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + require_NoError(t, err) + + // Bearer token etc.. + // This is used by all users, and the customization will be in other connect args. + // This needs to also be bound to the authorization account. + creds = createBasicAccountUser(t, ckp) + defer removeFile(t, creds) + + // We require a token. + ac.RequireConnectError(nats.UserCredentials(creds)) + + // Send correct token. This should switch us to the A account. + nc := ac.Connect(nats.UserCredentials(creds), nats.Token("PutMeInA")) + require_NoError(t, err) + defer nc.Close() + + resp, err = nc.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response = ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + require_NoError(t, err) + userInfo := response.Data.(*UserInfo) + require_Equal(t, userInfo.Account, apk) + + nc = ac.Connect(nats.UserCredentials(creds), nats.Token("PutMeInB")) + require_NoError(t, err) + defer nc.Close() + + resp, err = nc.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response = ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + require_NoError(t, err) + userInfo = response.Data.(*UserInfo) + require_Equal(t, userInfo.Account, bpk) +} + +func TestAuthCalloutWSClientTLSCerts(t *testing.T) { + conf := ` + server_name: T + listen: "localhost:-1" + + tls { + cert_file = "../test/configs/certs/tlsauth/server.pem" + key_file = "../test/configs/certs/tlsauth/server-key.pem" + ca_file = "../test/configs/certs/tlsauth/ca.pem" + verify = true + } + + websocket: { + listen: "localhost:-1" + tls { + cert_file = "../test/configs/certs/tlsauth/server.pem" + key_file = "../test/configs/certs/tlsauth/server-key.pem" + ca_file = "../test/configs/certs/tlsauth/ca.pem" + verify = true + } + } + + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + FOO {} + } + authorization { + timeout: 1s + auth_callout { + # Needs to be a public account nkey, will work for both server config and operator mode. + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth ] + } + } + ` + handler := func(m *nats.Msg) { + user, si, ci, _, ctls := decodeAuthRequest(t, m.Data) + require_Equal(t, si.Name, "T") + require_Equal(t, ci.Host, "127.0.0.1") + require_NotEqual(t, ctls, nil) + // Zero since we are verified and will be under verified chains. + require_Equal(t, len(ctls.Certs), 0) + require_Equal(t, len(ctls.VerifiedChains), 1) + // Since we have a CA. + require_Equal(t, len(ctls.VerifiedChains[0]), 2) + blk, _ := pem.Decode([]byte(ctls.VerifiedChains[0][0])) + cert, err := x509.ParseCertificate(blk.Bytes) + require_NoError(t, err) + if strings.HasPrefix(cert.Subject.String(), "CN=example.com") { + // Override blank name here, server will substitute. + ujwt := createAuthUser(t, user, "dlc", "FOO", "", nil, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } + } + + ac := NewAuthTest(t, conf, handler, + nats.UserInfo("auth", "pwd"), + nats.ClientCert("../test/configs/certs/tlsauth/client2.pem", "../test/configs/certs/tlsauth/client2-key.pem"), + nats.RootCAs("../test/configs/certs/tlsauth/ca.pem")) + defer ac.Cleanup() + + // Will use client cert to determine user. + nc := ac.WSConnect( + nats.ClientCert("../test/configs/certs/tlsauth/client2.pem", "../test/configs/certs/tlsauth/client2-key.pem"), + nats.RootCAs("../test/configs/certs/tlsauth/ca.pem"), + ) + defer nc.Close() + + resp, err := nc.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response := ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + require_NoError(t, err) + userInfo := response.Data.(*UserInfo) + + require_Equal(t, userInfo.UserID, "dlc") + require_Equal(t, userInfo.Account, "FOO") +} + +func testConfClientClose(t *testing.T, respondNil bool) { + conf := ` + listen: "127.0.0.1:-1" + server_name: ZZ + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + } + authorization { + timeout: 1s + auth_callout { + # Needs to be a public account nkey, will work for both server config and operator mode. + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth ] + } + } + ` + handler := func(m *nats.Msg) { + user, si, _, _, _ := decodeAuthRequest(t, m.Data) + if respondNil { + m.Respond(nil) + } else { + m.Respond(serviceResponse(t, user, si.ID, "", "not today", 0)) + } + } + + at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer at.Cleanup() + + // This one will use callout since not defined in server config. + _, err := at.NewClient(nats.UserInfo("a", "x")) + require_Error(t, err) + require_True(t, strings.Contains(strings.ToLower(err.Error()), nats.AUTHORIZATION_ERR)) +} + +func TestAuthCallout_ClientAuthErrorConf(t *testing.T) { + testConfClientClose(t, true) + testConfClientClose(t, false) +} + +func testAuthCall_ClientAuthErrorOperatorMode(t *testing.T, respondNil bool) { + _, spub := createKey(t) + sysClaim := jwt.NewAccountClaims(spub) + sysClaim.Name = "$SYS" + sysJwt, err := sysClaim.Encode(oKp) + require_NoError(t, err) + + // AUTH service account. + akp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) + require_NoError(t, err) + + apub, err := akp.PublicKey() + require_NoError(t, err) + + // The authorized user for the service. + upub, creds := createAuthServiceUser(t, akp) + defer removeFile(t, creds) + + authClaim := jwt.NewAccountClaims(apub) + authClaim.Name = "AUTH" + authClaim.EnableExternalAuthorization(upub) + authClaim.Authorization.AllowedAccounts.Add("*") + + // the scope for the bearer token which has no permissions + sentinelScope, authKP := newScopedRole(t, "sentinel", nil, nil, false) + sentinelScope.Template.Sub.Deny.Add(">") + sentinelScope.Template.Pub.Deny.Add(">") + sentinelScope.Template.Limits.Subs = 0 + sentinelScope.Template.Payload = 0 + authClaim.SigningKeys.AddScopedSigner(sentinelScope) + + authJwt, err := authClaim.Encode(oKp) + require_NoError(t, err) + + conf := fmt.Sprintf(` + listen: 127.0.0.1:-1 + operator: %s + system_account: %s + resolver: MEM + resolver_preload: { + %s: %s + %s: %s + } + `, ojwt, spub, apub, authJwt, spub, sysJwt) + + handler := func(m *nats.Msg) { + user, si, _, _, _ := decodeAuthRequest(t, m.Data) + if respondNil { + m.Respond(nil) + } else { + m.Respond(serviceResponse(t, user, si.ID, "", "not today", 0)) + } + } + + ac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) + defer ac.Cleanup() + + // Bearer token - this has no permissions see sentinelScope + // This is used by all users, and the customization will be in other connect args. + // This needs to also be bound to the authorization account. + creds = createScopedUser(t, akp, authKP) + defer removeFile(t, creds) + + // Send the signing key token. This should switch us to the test account, but the user + // is signed with the account signing key + _, err = ac.NewClient(nats.UserCredentials(creds)) + require_Error(t, err) + require_True(t, strings.Contains(strings.ToLower(err.Error()), nats.AUTHORIZATION_ERR)) +} + +func TestAuthCallout_ClientAuthErrorOperatorMode(t *testing.T) { + testAuthCall_ClientAuthErrorOperatorMode(t, true) + testAuthCall_ClientAuthErrorOperatorMode(t, false) +} + +func TestOperatorModeUserRevocation(t *testing.T) { + skp, spub := createKey(t) + sysClaim := jwt.NewAccountClaims(spub) + sysClaim.Name = "$SYS" + sysJwt, err := sysClaim.Encode(oKp) + require_NoError(t, err) + + // TEST account. + tkp, tpub := createKey(t) + accClaim := jwt.NewAccountClaims(tpub) + accClaim.Name = "TEST" + accJwt, err := accClaim.Encode(oKp) + require_NoError(t, err) + + // AUTH service account. + akp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) + require_NoError(t, err) + + apub, err := akp.PublicKey() + require_NoError(t, err) + + // The authorized user for the service. + upub, creds := createAuthServiceUser(t, akp) + defer removeFile(t, creds) + + authClaim := jwt.NewAccountClaims(apub) + authClaim.Name = "AUTH" + authClaim.EnableExternalAuthorization(upub) + authClaim.Authorization.AllowedAccounts.Add(tpub) + authJwt, err := authClaim.Encode(oKp) + require_NoError(t, err) + + jwtDir, err := os.MkdirTemp("", "") + require_NoError(t, err) + defer func() { + _ = os.RemoveAll(jwtDir) + }() + + conf := fmt.Sprintf(` + listen: 127.0.0.1:-1 + operator: %s + system_account: %s + resolver: { + type: "full" + dir: %s + allow_delete: false + interval: "2m" + timeout: "1.9s" + } + resolver_preload: { + %s: %s + %s: %s + %s: %s + } + `, ojwt, spub, jwtDir, apub, authJwt, tpub, accJwt, spub, sysJwt) + + const token = "--secret--" + + users := make(map[string]string) + handler := func(m *nats.Msg) { + user, si, _, opts, _ := decodeAuthRequest(t, m.Data) + if opts.Token == token { + // must have no limits set + ujwt := createAuthUser(t, user, "user", tpub, tpub, tkp, 0, &jwt.UserPermissionLimits{}) + users[opts.Name] = ujwt + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else { + m.Respond(nil) + } + } + + ac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) + defer ac.Cleanup() + + // create a system user + _, sysCreds := createAuthServiceUser(t, skp) + defer removeFile(t, sysCreds) + // connect the system user + sysNC, err := ac.NewClient(nats.UserCredentials(sysCreds)) + require_NoError(t, err) + defer sysNC.Close() + + // Bearer token etc.. + // This is used by all users, and the customization will be in other connect args. + // This needs to also be bound to the authorization account. + creds = createBasicAccountUser(t, akp) + defer removeFile(t, creds) + + var fwg sync.WaitGroup + + // connect three clients + nc := ac.Connect(nats.UserCredentials(creds), nats.Name("first"), nats.Token(token), nats.NoReconnect(), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + if err != nil && strings.Contains(err.Error(), "authentication revoked") { + fwg.Done() + } + })) + fwg.Add(1) + + var swg sync.WaitGroup + // connect another user + ncA := ac.Connect(nats.UserCredentials(creds), nats.Token(token), nats.Name("second"), nats.NoReconnect(), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + if err != nil && strings.Contains(err.Error(), "authentication revoked") { + swg.Done() + } + })) + swg.Add(1) + + ncB := ac.Connect(nats.UserCredentials(creds), nats.Token(token), nats.NoReconnect(), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + if err != nil && strings.Contains(err.Error(), "authentication revoked") { + swg.Done() + } + })) + swg.Add(1) + + require_NoError(t, err) + + // revoke the user first - look at the JWT we issued + uc, err := jwt.DecodeUserClaims(users["first"]) + require_NoError(t, err) + // revoke the user in account + accClaim.Revocations = make(map[string]int64) + accClaim.Revocations.Revoke(uc.Subject, time.Now().Add(time.Minute)) + accJwt, err = accClaim.Encode(oKp) + require_NoError(t, err) + // send the update request + updateAccount(t, sysNC, accJwt) + + // wait for the user to be disconnected with the error we expect + fwg.Wait() + require_Equal(t, nc.IsConnected(), false) + + // update the account to remove any revocations + accClaim.Revocations = make(map[string]int64) + accJwt, err = accClaim.Encode(oKp) + require_NoError(t, err) + updateAccount(t, sysNC, accJwt) + // we should still be connected on the other 2 clients + require_Equal(t, ncA.IsConnected(), true) + require_Equal(t, ncB.IsConnected(), true) + + // update the jwt and revoke all users + accClaim.Revocations.Revoke(jwt.All, time.Now().Add(time.Minute)) + accJwt, err = accClaim.Encode(oKp) + require_NoError(t, err) + updateAccount(t, sysNC, accJwt) + + swg.Wait() + require_Equal(t, ncA.IsConnected(), false) + require_Equal(t, ncB.IsConnected(), false) +} + +func updateAccount(t *testing.T, sys *nats.Conn, jwtToken string) { + ac, err := jwt.DecodeAccountClaims(jwtToken) + require_NoError(t, err) + r, err := sys.Request(fmt.Sprintf(`$SYS.REQ.ACCOUNT.%s.CLAIMS.UPDATE`, ac.Subject), []byte(jwtToken), time.Second*2) + require_NoError(t, err) + + type data struct { + Account string `json:"account"` + Code int `json:"code"` + } + type serverResponse struct { + Data data `json:"data"` + } + + var response serverResponse + err = json.Unmarshal(r.Data, &response) + require_NoError(t, err) + require_NotNil(t, response.Data) + require_Equal(t, response.Data.Code, int(200)) +} + +func TestAuthCalloutLeafNodeAndOperatorMode(t *testing.T) { + _, spub := createKey(t) + sysClaim := jwt.NewAccountClaims(spub) + sysClaim.Name = "$SYS" + sysJwt, err := sysClaim.Encode(oKp) + require_NoError(t, err) + + // A account. + akp, apk := createKey(t) + aClaim := jwt.NewAccountClaims(apk) + aClaim.Name = "A" + aJwt, err := aClaim.Encode(oKp) + require_NoError(t, err) + + // AUTH callout service account. + ckp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) + require_NoError(t, err) + + cpk, err := ckp.PublicKey() + require_NoError(t, err) + + // The authorized user for the service. + upub, creds := createAuthServiceUser(t, ckp) + defer removeFile(t, creds) + + authClaim := jwt.NewAccountClaims(cpk) + authClaim.Name = "AUTH" + authClaim.EnableExternalAuthorization(upub) + authClaim.Authorization.AllowedAccounts.Add("*") + authJwt, err := authClaim.Encode(oKp) + require_NoError(t, err) + + conf := fmt.Sprintf(` + listen: 127.0.0.1:-1 + operator: %s + system_account: %s + resolver: MEM + resolver_preload: { + %s: %s + %s: %s + %s: %s + } + leafnodes { + listen: "127.0.0.1:-1" + } + `, ojwt, spub, cpk, authJwt, apk, aJwt, spub, sysJwt) + + handler := func(m *nats.Msg) { + user, si, _, opts, _ := decodeAuthRequest(t, m.Data) + if (opts.Username == "leaf" && opts.Password == "pwd") || (opts.Token == "token") { + ujwt := createAuthUser(t, user, "user_a", apk, "", akp, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else { + m.Respond(nil) + } + } + + at := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) + defer at.Cleanup() + + ucreds := createBasicAccountUser(t, ckp) + defer removeFile(t, ucreds) + + // This should switch us to the A account. + nc := at.Connect(nats.UserCredentials(ucreds), nats.Token("token")) + defer nc.Close() + + natsSub(t, nc, "foo", func(m *nats.Msg) { + m.Respond([]byte("here")) + }) + natsFlush(t, nc) + + // Create creds for the leaf account. + lcreds := createBasicAccountLeaf(t, ckp) + defer removeFile(t, lcreds) + + hopts := at.srv.getOpts() + + for _, test := range []struct { + name string + up string + ok bool + }{ + {"bad token", "tokenx", false}, + {"bad username and password", "leaf:pwdx", false}, + {"token", "token", true}, + {"username and password", "leaf:pwd", true}, + } { + t.Run(test.name, func(t *testing.T) { + lconf := createConfFile(t, []byte(fmt.Sprintf(` + listen: "127.0.0.1:-1" + server_name: "LEAF" + leafnodes { + remotes [ + { + url: "nats://%s@127.0.0.1:%d" + credentials: "%s" + } + ] + } + `, test.up, hopts.LeafNode.Port, lcreds))) + leaf, _ := RunServerWithConfig(lconf) + defer leaf.Shutdown() + + if !test.ok { + // Expect failure to connect. Wait a bit before checking. + time.Sleep(50 * time.Millisecond) + checkLeafNodeConnectedCount(t, leaf, 0) + return + } + + checkLeafNodeConnected(t, leaf) + + checkSubInterest(t, leaf, globalAccountName, "foo", time.Second) + + ncl := natsConnect(t, leaf.ClientURL()) + defer ncl.Close() + + resp, err := ncl.Request("foo", []byte("hello"), time.Second) + require_NoError(t, err) + require_Equal(t, string(resp.Data), "here") + }) + } +} + +func TestAuthCalloutLeafNodeAndConfigMode(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + A {} + } + authorization { + timeout: 1s + auth_callout { + # Needs to be a public account nkey, will work for both server config and operator mode. + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth ] + } + } + leafnodes { + listen: "127.0.0.1:-1" + } + ` + handler := func(m *nats.Msg) { + user, si, _, opts, _ := decodeAuthRequest(t, m.Data) + if (opts.Username == "leaf" && opts.Password == "pwd") || (opts.Token == "token") { + ujwt := createAuthUser(t, user, _EMPTY_, "A", "", nil, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else { + m.Respond(nil) + } + } + + at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer at.Cleanup() + + // This should switch us to the A account. + nc := at.Connect(nats.Token("token")) + defer nc.Close() + + natsSub(t, nc, "foo", func(m *nats.Msg) { + m.Respond([]byte("here")) + }) + natsFlush(t, nc) + + hopts := at.srv.getOpts() + + for _, test := range []struct { + name string + up string + ok bool + }{ + {"bad token", "tokenx", false}, + {"bad username and password", "leaf:pwdx", false}, + {"token", "token", true}, + {"username and password", "leaf:pwd", true}, + } { + t.Run(test.name, func(t *testing.T) { + lconf := createConfFile(t, []byte(fmt.Sprintf(` + listen: "127.0.0.1:-1" + server_name: "LEAF" + leafnodes { + remotes [{url: "nats://%s@127.0.0.1:%d"}] + } + `, test.up, hopts.LeafNode.Port))) + leaf, _ := RunServerWithConfig(lconf) + defer leaf.Shutdown() + + if !test.ok { + // Expect failure to connect. Wait a bit before checking. + time.Sleep(50 * time.Millisecond) + checkLeafNodeConnectedCount(t, leaf, 0) + return + } + + checkLeafNodeConnected(t, leaf) + + checkSubInterest(t, leaf, globalAccountName, "foo", time.Second) + + ncl := natsConnect(t, leaf.ClientURL()) + defer ncl.Close() + + resp, err := ncl.Request("foo", []byte("hello"), time.Second) + require_NoError(t, err) + require_Equal(t, string(resp.Data), "here") + }) + } + +} + +func TestAuthCalloutProxyRequiredInUserNotInAuthJWT(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: A + accounts { + AUTH: { + users: [ { user: auth, password: auth } ] + } + APP: { + users: [ + { user: user, password: pwd } + { user: proxy, password: pwd, proxy_required: true } + ] + } + SYS: {} + } + system_account: SYS + + authorization { + timeout: 1s + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + auth_users: [ auth ] + account: AUTH + } + } + ` + var invoked atomic.Int32 + handler := func(m *nats.Msg) { + invoked.Add(1) + user, si, ci, opts, _ := decodeAuthRequest(t, m.Data) + require_True(t, si.Name == "A") + require_True(t, ci.Host == "127.0.0.1") + if opts.Username == "proxy" { + // If we don't set a ProxyRequired property explicitly here, but the + // user has it, so it should still be rejected. + ujwt := createAuthUser(t, user, _EMPTY_, "APP", _EMPTY_, nil, 10*time.Minute, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, _EMPTY_, 0)) + } else { + m.Respond(nil) + } + } + at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "auth")) + defer at.Cleanup() + + sub, err := at.authClient.SubscribeSync(authErrorAccountEventSubj) + require_NoError(t, err) + + // It should fail, even if the auth callout does not require proxy in its JWT. + // In other words, we reject if not proxied and the user config or the auth JWT + // requires proxy connection. + at.RequireConnectError(nats.UserInfo("proxy", "pwd")) + + m, err := sub.NextMsg(time.Second) + require_NoError(t, err) + + var dm DisconnectEventMsg + err = json.Unmarshal(m.Data, &dm) + require_NoError(t, err) + + // Convert both reasons to lower case to do the comparison. + dmr := strings.ToLower(dm.Reason) + r := strings.ToLower(ErrAuthProxyRequired.Error()) + if !strings.Contains(dmr, r) { + t.Fatalf("Expected %q reason, but got %q", r, dmr) + } + + // Auth callout should have been invoked once. + require_Equal(t, 1, int(invoked.Load())) +} + +func TestAuthCalloutOperatorModeMismatchedCalloutCreds(t *testing.T) { + _, spub := createKey(t) + sysClaim := jwt.NewAccountClaims(spub) + sysClaim.Name = "$SYS" + sysJwt, err := sysClaim.Encode(oKp) + require_NoError(t, err) + + // AUTH callout service account. + ckp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) + require_NoError(t, err) + cpk, err := ckp.PublicKey() + require_NoError(t, err) + + // The authorized user for the service. + upub, _ := createAuthServiceUser(t, ckp) + + authClaim := jwt.NewAccountClaims(cpk) + authClaim.Name = "AUTH" + authClaim.EnableExternalAuthorization(upub) + authClaim.Authorization.AllowedAccounts.Add("*") + authJwt, err := authClaim.Encode(oKp) + require_NoError(t, err) + + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:-1 + operator: %s + system_account: %s + resolver: MEM + resolver_preload: { + %s: %s + %s: %s + } + `, ojwt, spub, cpk, authJwt, spub, sysJwt))) + + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + // Create mismatched creds for the auth service user: + // JWT with the auth user's public key, but a different seed. + ukpBad, _ := nkeys.CreateUser() + seedBad, _ := ukpBad.Seed() + + uclaim := newJWTTestUserClaims() + uclaim.Name = "auth-service" + uclaim.Subject = upub // auth user's public key + ujwt, err := uclaim.Encode(ckp) + require_NoError(t, err) + badCreds := genCredsFile(t, ujwt, seedBad) + defer removeFile(t, badCreds) + + // Client auth will time out because the callout service didn't connect. + _, err = nats.Connect(s.ClientURL(), nats.UserCredentials(badCreds), nats.MaxReconnects(0)) + require_Error(t, err) + + // Server should still be running. + time.Sleep(500 * time.Millisecond) + require_True(t, s.Running()) +} + +func TestAuthCalloutLeafNodeOperatorModeMismatchedCreds(t *testing.T) { + _, spub := createKey(t) + sysClaim := jwt.NewAccountClaims(spub) + sysClaim.Name = "$SYS" + sysJwt, err := sysClaim.Encode(oKp) + require_NoError(t, err) + + // A account. + akp, apk := createKey(t) + aClaim := jwt.NewAccountClaims(apk) + aClaim.Name = "A" + aJwt, err := aClaim.Encode(oKp) + require_NoError(t, err) + + // AUTH callout service account. + ckp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) + require_NoError(t, err) + cpk, err := ckp.PublicKey() + require_NoError(t, err) + + // The authorized user for the service. + upub, creds := createAuthServiceUser(t, ckp) + defer removeFile(t, creds) + + authClaim := jwt.NewAccountClaims(cpk) + authClaim.Name = "AUTH" + authClaim.EnableExternalAuthorization(upub) + authClaim.Authorization.AllowedAccounts.Add("*") + authJwt, err := authClaim.Encode(oKp) + require_NoError(t, err) + + conf := fmt.Sprintf(` + listen: 127.0.0.1:-1 + operator: %s + system_account: %s + resolver: MEM + resolver_preload: { + %s: %s + %s: %s + %s: %s + } + leafnodes { + listen: "127.0.0.1:-1" + } + `, ojwt, spub, cpk, authJwt, apk, aJwt, spub, sysJwt) + + handler := func(m *nats.Msg) { + user, si, _, opts, _ := decodeAuthRequest(t, m.Data) + if opts.Token == "token" { + ujwt := createAuthUser(t, user, "user_a", apk, "", akp, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else { + m.Respond(nil) + } + } + + at := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) + defer at.Cleanup() + + // Good leaf node connection. + lcreds := createBasicAccountLeaf(t, ckp) + defer removeFile(t, lcreds) + + hopts := at.srv.getOpts() + lconf := createConfFile(t, []byte(fmt.Sprintf(` + listen: "127.0.0.1:-1" + server_name: "LEAF" + leafnodes { + remotes [ + { + url: "nats://token@127.0.0.1:%d" + credentials: "%s" + } + ] + } + `, hopts.LeafNode.Port, lcreds))) + leaf, _ := RunServerWithConfig(lconf) + defer leaf.Shutdown() + checkLeafNodeConnected(t, leaf) + + // Now create a leaf with mismatched JWT/seed - JWT from one user, + // seed from another. This should not crash the server. + ukp1, _ := nkeys.CreateUser() + seed1, _ := ukp1.Seed() + + ukp2, _ := nkeys.CreateUser() + upub2, _ := ukp2.PublicKey() + + // Create JWT for user2 but pair with seed from user1. + uclaim := newJWTTestUserClaims() + uclaim.Subject = upub2 + uclaim.Name = "mismatched-leaf" + ujwt, err := uclaim.Encode(ckp) + require_NoError(t, err) + badCreds := genCredsFile(t, ujwt, seed1) + defer removeFile(t, badCreds) + + lconf2 := createConfFile(t, []byte(fmt.Sprintf(` + listen: "127.0.0.1:-1" + server_name: "BAD_LEAF" + leafnodes { + remotes [ + { + url: "nats://token@127.0.0.1:%d" + credentials: "%s" + } + ] + } + `, hopts.LeafNode.Port, badCreds))) + leaf2, _ := RunServerWithConfig(lconf2) + defer leaf2.Shutdown() + + // Bad leaf should fail to connect but NOT crash the server. + time.Sleep(50 * time.Millisecond) + checkLeafNodeConnectedCount(t, leaf2, 0) + + // Verify the hub server is still running and healthy. + checkLeafNodeConnectedCount(t, at.srv, 1) +} + +func TestAuthCalloutRegisterWithAccountAfterClose(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: ZZ + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + FOO {} + } + authorization { + timeout: 1 + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth ] + } + } + ` + // Track whether the auth callout handler was called and collect the + // client pointer for manual registration later. + type calloutInfo struct { + user string + serverID string + processed chan struct{} + } + var ci calloutInfo + ci.processed = make(chan struct{}) + + handler := func(m *nats.Msg) { + user, si, _, opts, _ := decodeAuthRequest(t, m.Data) + if opts.Username == "zombie" && opts.Password == "pwd" { + ci.user = user + ci.serverID = si.ID + close(ci.processed) + // Do NOT respond. Let the auth callout timeout. + // This simulates the case where the response arrives late. + } else { + m.Respond(nil) + } + } + + at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer at.Cleanup() + s := at.srv + + fooAcc, err := s.lookupAccount("FOO") + require_NoError(t, err) + require_Equal(t, fooAcc.NumLocalConnections(), 0) + baseServerClients := s.NumClients() + + // Connect a client that will go through auth callout. The handler will + // NOT respond, so the callout will timeout and the client will be closed. + var connectErr error + done := make(chan struct{}) + go func() { + defer close(done) + _, connectErr = nats.Connect(s.ClientURL(), + nats.UserInfo("zombie", "pwd"), + nats.Timeout(5*time.Second), + nats.MaxReconnects(0), + ) + }() + + // Wait for the auth handler to be invoked (request received). + <-ci.processed + + // Wait for auth timeout (1s) + closeConnection to complete. + <-done + require_Error(t, connectErr) + require_Equal(t, fooAcc.NumLocalConnections(), 0) + require_Equal(t, s.NumClients(), baseServerClients) + + // Manually create a client, like auth callout would, so we have a reference to it and can close it. + globalAcc, err := s.lookupAccount(globalAccountName) + require_NoError(t, err) + c := &client{srv: s, kind: CLIENT, acc: globalAcc} + c.initClient() + globalAcc.addClient(c) + c.setNoReconnect() + c.closeConnection(ClientClosed) + + // Simulate what processReply does after the auth callout response arrives: + // It calls c.RegisterNkeyUser which calls registerWithAccount(targetAcc). + // registerWithAccount should not add the client as we've closed it above. + require_Error(t, c.registerWithAccount(fooAcc), ErrConnectionClosed) + require_Equal(t, fooAcc.NumLocalConnections(), 0) +} + +func TestAuthCalloutZombieInflatesAccountConnections(t *testing.T) { + conf := ` + listen: "127.0.0.1:-1" + server_name: ZZ + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + FOO { limits { max_conn: 5 } } + } + authorization { + timeout: 1 + auth_callout { + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth ] + } + } + ` + + handler := func(m *nats.Msg) { + user, si, _, opts, _ := decodeAuthRequest(t, m.Data) + if opts.Username == "legit" && opts.Password == "pwd" { + ujwt := createAuthUser(t, user, _EMPTY_, "FOO", "", nil, 0, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } else { + m.Respond(nil) + } + } + + at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer at.Cleanup() + s := at.srv + + globalAcc := s.globalAccount() + fooAcc, err := s.lookupAccount("FOO") + require_NoError(t, err) + + fooAcc.mu.RLock() + mconns := fooAcc.mconns + fooAcc.mu.RUnlock() + require_Equal(t, mconns, 5) + + // Step 1: Inject zombie connections into FOO account. + // This simulates what happens when processReply calls registerWithAccount + // on a client that has already been through closeConnection. + for range 3 { + c := &client{srv: s, kind: CLIENT, acc: globalAcc} + c.initClient() + globalAcc.addClient(c) + c.closeConnection(ClientClosed) + require_Error(t, c.registerWithAccount(fooAcc), ErrConnectionClosed) + } + require_Equal(t, fooAcc.NumLocalConnections(), 0) + + // Step 2: Connect legitimate clients. FOO has max_connections: 5. + // With 3 zombies, we should only be able to connect 2 real clients + // before hitting the limit, even though there are 0 real connections. + for range 5 { + nc, err := nats.Connect(s.ClientURL(), + nats.UserInfo("legit", "pwd"), + nats.MaxReconnects(0), + ) + require_NoError(t, err) + //goland:noinspection GoDeferInLoop + defer nc.Close() + } + require_Equal(t, fooAcc.NumLocalConnections(), 5) +} + +func TestAuthCalloutOperatorModeMQTTOpaquePasswordUsesDefaultSentinel(t *testing.T) { + _, spub := createKey(t) + sysClaim := jwt.NewAccountClaims(spub) + sysClaim.Name = "$SYS" + sysJwt, err := sysClaim.Encode(oKp) + require_NoError(t, err) + + tkp, tpub := createKey(t) + accClaim := jwt.NewAccountClaims(tpub) + accClaim.Name = "TEST" + accClaim.Limits.JetStreamLimits.Consumer = -1 + accClaim.Limits.JetStreamLimits.Streams = -1 + accClaim.Limits.JetStreamLimits.MemoryStorage = 1024 * 1024 + accClaim.Limits.JetStreamLimits.DiskStorage = 1024 * 1024 + accJwt, err := accClaim.Encode(oKp) + require_NoError(t, err) + + akp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) + require_NoError(t, err) + apub, err := akp.PublicKey() + require_NoError(t, err) + + upub, creds := createAuthServiceUser(t, akp) + defer removeFile(t, creds) + + authClaim := jwt.NewAccountClaims(apub) + authClaim.Name = "AUTH" + authClaim.EnableExternalAuthorization(upub) + authClaim.Authorization.AllowedAccounts.Add(tpub) + authJwt, err := authClaim.Encode(oKp) + require_NoError(t, err) + + defaultSentinel := createBasicAccountBearer(t, akp) + storeDir := t.TempDir() + conf := fmt.Sprintf(` + listen: "127.0.0.1:-1" + server_name: A + jetstream: { max_mem_store: 256MB, max_file_store: 1GB, store_dir: %q } + mqtt { host: "127.0.0.1", port: -1 } + operator: %s + system_account: %s + resolver: MEM + resolver_preload: { + %s: %s + %s: %s + %s: %s + } + default_sentinel: %s + `, storeDir, ojwt, spub, apub, authJwt, tpub, accJwt, spub, sysJwt, defaultSentinel) + + const opaquePassword = "external-jwt-or-token" + type seenConnectOptions struct { + jwt string + password string + } + seen := make(chan seenConnectOptions, 1) + handler := func(m *nats.Msg) { + user, si, _, opts, _ := decodeAuthRequest(t, m.Data) + select { + case seen <- seenConnectOptions{jwt: opts.JWT, password: opts.Password}: + default: + } + if opts.JWT != defaultSentinel || opts.Password != opaquePassword { + m.Respond(nil) + return + } + ujwt := createAuthUser(t, user, "mqtt", tpub, "", tkp, 10*time.Minute, nil) + m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) + } + + at := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) + defer at.Cleanup() + + mc, rc := authCalloutMQTTConnect(t, at.srv.getOpts().MQTT.Host, at.srv.getOpts().MQTT.Port, "mqtt-auth-callout", "ignored", opaquePassword) + defer mc.Close() + + got := require_ChanRead(t, seen, 2*time.Second) + require_Equal(t, got.jwt, defaultSentinel) + require_Equal(t, got.password, opaquePassword) + require_Equal(t, rc, mqttConnAckRCConnectionAccepted) +} diff --git a/server/auth_test.go b/server/auth_test.go new file mode 100644 index 0000000..c941689 --- /dev/null +++ b/server/auth_test.go @@ -0,0 +1,1246 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bufio" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net" + "net/url" + "os" + "reflect" + "strings" + "testing" + "time" + + "github.com/nats-io/jwt/v2" + "github.com/nats-io/nats.go" + "github.com/nats-io/nkeys" +) + +func TestUserCloneNilPermissions(t *testing.T) { + user := &User{ + Username: "foo", + Password: "bar", + } + + clone := user.clone() + + if !reflect.DeepEqual(user, clone) { + t.Fatalf("Cloned Users are incorrect.\nexpected: %+v\ngot: %+v", + user, clone) + } + + clone.Password = "baz" + if reflect.DeepEqual(user, clone) { + t.Fatal("Expected Users to be different") + } +} + +func TestUserClone(t *testing.T) { + user := &User{ + Username: "foo", + Password: "bar", + Permissions: &Permissions{ + Publish: &SubjectPermission{ + Allow: []string{"foo"}, + }, + Subscribe: &SubjectPermission{ + Allow: []string{"bar"}, + }, + }, + } + + clone := user.clone() + + if !reflect.DeepEqual(user, clone) { + t.Fatalf("Cloned Users are incorrect.\nexpected: %+v\ngot: %+v", + user, clone) + } + + clone.Permissions.Subscribe.Allow = []string{"baz"} + if reflect.DeepEqual(user, clone) { + t.Fatal("Expected Users to be different") + } +} + +func TestUserClonePermissionsNoLists(t *testing.T) { + user := &User{ + Username: "foo", + Password: "bar", + Permissions: &Permissions{}, + } + + clone := user.clone() + + if clone.Permissions.Publish != nil { + t.Fatalf("Expected Publish to be nil, got: %v", clone.Permissions.Publish) + } + if clone.Permissions.Subscribe != nil { + t.Fatalf("Expected Subscribe to be nil, got: %v", clone.Permissions.Subscribe) + } +} + +func TestUserCloneNoPermissions(t *testing.T) { + user := &User{ + Username: "foo", + Password: "bar", + } + + clone := user.clone() + + if clone.Permissions != nil { + t.Fatalf("Expected Permissions to be nil, got: %v", clone.Permissions) + } +} + +func TestUserCloneNil(t *testing.T) { + user := (*User)(nil) + clone := user.clone() + if clone != nil { + t.Fatalf("Expected nil, got: %+v", clone) + } +} + +func TestUserUnknownAllowedConnectionType(t *testing.T) { + o := DefaultOptions() + o.Users = []*User{{ + Username: "user", + Password: "pwd", + AllowedConnectionTypes: testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, "someNewType"}), + }} + _, err := NewServer(o) + if err == nil || !strings.Contains(err.Error(), "connection type") { + t.Fatalf("Expected error about unknown connection type, got %v", err) + } + + o.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{"websocket"}) + s, err := NewServer(o) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + s.mu.Lock() + user := s.opts.Users[0] + s.mu.Unlock() + for act := range user.AllowedConnectionTypes { + if act != jwt.ConnectionTypeWebsocket { + t.Fatalf("Expected map to have been updated with proper case, got %v", act) + } + } + // Same with NKey user now. + o.Users = nil + o.Nkeys = []*NkeyUser{{ + Nkey: "somekey", + AllowedConnectionTypes: testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, "someNewType"}), + }} + _, err = NewServer(o) + if err == nil || !strings.Contains(err.Error(), "connection type") { + t.Fatalf("Expected error about unknown connection type, got %v", err) + } + o.Nkeys[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{"websocket"}) + s, err = NewServer(o) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + s.mu.Lock() + nkey := s.opts.Nkeys[0] + s.mu.Unlock() + for act := range nkey.AllowedConnectionTypes { + if act != jwt.ConnectionTypeWebsocket { + t.Fatalf("Expected map to have been updated with proper case, got %v", act) + } + } +} + +func TestDNSAltNameMatching(t *testing.T) { + for idx, test := range []struct { + altName string + urls []string + match bool + }{ + {"foo", []string{"FOO"}, true}, + {"foo", []string{".."}, false}, + {"foo", []string{"."}, false}, + {"Foo", []string{"foO"}, true}, + {"FOO", []string{"foo"}, true}, + {"foo1", []string{"bar"}, false}, + {"multi", []string{"m", "mu", "mul", "multi"}, true}, + {"multi", []string{"multi", "m", "mu", "mul"}, true}, + {"foo.bar", []string{"foo", "foo.bar.bar", "foo.baz"}, false}, + {"foo.Bar", []string{"foo", "bar.foo", "Foo.Bar"}, true}, + {"foo.*", []string{"foo", "bar.foo", "Foo.Bar"}, false}, // only match left most + {"f*.bar", []string{"foo", "bar.foo", "Foo.Bar"}, false}, + {"*.bar", []string{"foo.bar"}, true}, + {"*", []string{"baz.bar", "bar", "z.y"}, true}, + {"*", []string{"bar"}, true}, + {"*", []string{"."}, false}, + {"*", []string{""}, true}, + {"*", []string{"*"}, true}, + {"bar.*", []string{"bar.*"}, true}, + {"*.Y-X-red-mgmt.default.svc", []string{"A.Y-X-red-mgmt.default.svc"}, true}, + {"*.Y-X-green-mgmt.default.svc", []string{"A.Y-X-green-mgmt.default.svc"}, true}, + {"*.Y-X-blue-mgmt.default.svc", []string{"A.Y-X-blue-mgmt.default.svc"}, true}, + {"Y-X-red-mgmt", []string{"Y-X-red-mgmt"}, true}, + {"Y-X-red-mgmt", []string{"X-X-red-mgmt"}, false}, + {"Y-X-red-mgmt", []string{"Y-X-green-mgmt"}, false}, + {"Y-X-red-mgmt", []string{"Y"}, false}, + {"Y-X-red-mgmt", []string{"Y-X"}, false}, + {"Y-X-red-mgmt", []string{"Y-X-red"}, false}, + {"Y-X-red-mgmt", []string{"X-red-mgmt"}, false}, + {"Y-X-green-mgmt", []string{"Y-X-green-mgmt"}, true}, + {"Y-X-blue-mgmt", []string{"Y-X-blue-mgmt"}, true}, + {"connect.Y.local", []string{"connect.Y.local"}, true}, + {"connect.Y.local", []string{".Y.local"}, false}, + {"connect.Y.local", []string{"..local"}, false}, + {"gcp.Y.local", []string{"gcp.Y.local"}, true}, + {"uswest1.gcp.Y.local", []string{"uswest1.gcp.Y.local"}, true}, + } { + urlSet := make([]*url.URL, len(test.urls)) + for i, u := range test.urls { + var err error + urlSet[i], err = url.Parse("nats://" + u) + if err != nil { + t.Fatal(err) + } + } + if dnsAltNameMatches(dnsAltNameLabels(test.altName), urlSet) != test.match { + t.Fatal("Test", idx, "Match miss match, expected:", test.match) + } + } +} + +func TestProcessUserPermissionsTemplateMalformedOpDoesNotPanic(t *testing.T) { + defer require_NoPanic(t) + + lim := jwt.UserPermissionLimits{} + lim.Permissions.Sub.Deny = jwt.StringList{"foo.{{x}}"} + + _, err := processUserPermissionsTemplate(lim, &jwt.UserClaims{}, &Account{}) + require_Error(t, err) +} + +func TestProcessUserPermissionsTemplateUnknownAllowOpDoesNotPanic(t *testing.T) { + defer require_NoPanic(t) + + lim := jwt.UserPermissionLimits{} + lim.Permissions.Pub.Allow = jwt.StringList{"foo.{{unknown()}}"} + + _, err := processUserPermissionsTemplate(lim, &jwt.UserClaims{}, &Account{}) + require_Error(t, err) +} + +func TestProcessUserPermissionsTemplateRejectsExcessiveTagExpansions(t *testing.T) { + defer require_NoPanic(t) + + lim := jwt.UserPermissionLimits{} + lim.Permissions.Pub.Allow = jwt.StringList{"foo.{{tag(a)}}.{{tag(b)}}"} + + uc := &jwt.UserClaims{} + for i := range 128 { + uc.Tags = append(uc.Tags, fmt.Sprintf("a:v%d", i)) + uc.Tags = append(uc.Tags, fmt.Sprintf("b:v%d", i)) + } + + _, err := processUserPermissionsTemplate(lim, uc, &Account{}) + require_Error(t, err) +} + +func TestNoAuthUser(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: "127.0.0.1:-1" + accounts { + FOO { users [{user: "foo", password: "pwd1"}] } + BAR { users [{user: "bar", password: "pwd2"}] } + } + no_auth_user: "foo" + `)) + defer os.Remove(conf) + s, o := RunServerWithConfig(conf) + defer s.Shutdown() + + for _, test := range []struct { + name string + usrInfo string + ok bool + account string + }{ + {"valid user/pwd", "bar:pwd2@", true, "BAR"}, + {"invalid pwd", "bar:wrong@", false, _EMPTY_}, + {"some token", "sometoken@", false, _EMPTY_}, + {"user used without pwd", "bar@", false, _EMPTY_}, // will be treated as a token + {"user with empty password", "bar:@", false, _EMPTY_}, + {"no user", _EMPTY_, true, "FOO"}, + } { + t.Run(test.name, func(t *testing.T) { + url := fmt.Sprintf("nats://%s127.0.0.1:%d", test.usrInfo, o.Port) + nc, err := nats.Connect(url) + if err != nil { + if test.ok { + t.Fatalf("Unexpected error: %v", err) + } + return + } else if !test.ok { + nc.Close() + t.Fatalf("Should have failed, did not") + } + var accName string + s.mu.Lock() + for _, c := range s.clients { + c.mu.Lock() + if c.acc != nil { + accName = c.acc.Name + } + c.mu.Unlock() + break + } + s.mu.Unlock() + nc.Close() + checkClientsCount(t, s, 0) + if accName != test.account { + t.Fatalf("The account should have been %q, got %q", test.account, accName) + } + }) + } +} + +func TestNoAuthUserNkey(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: "127.0.0.1:-1" + accounts { + FOO { users [{user: "foo", password: "pwd1"}] } + BAR { users [{nkey: "UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON"}] } + } + no_auth_user: "UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON" + `)) + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + // Make sure we connect ok and to the correct account. + nc := natsConnect(t, s.ClientURL()) + defer nc.Close() + resp, err := nc.Request(userDirectInfoSubj, nil, time.Second) + require_NoError(t, err) + response := ServerAPIResponse{Data: &UserInfo{}} + err = json.Unmarshal(resp.Data, &response) + require_NoError(t, err) + userInfo := response.Data.(*UserInfo) + require_Equal(t, userInfo.UserID, "UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON") + require_Equal(t, userInfo.Account, "BAR") +} + +func TestUserConnectionDeadline(t *testing.T) { + clientAuth := &DummyAuth{ + t: t, + register: true, + deadline: time.Now().Add(50 * time.Millisecond), + } + + opts := DefaultOptions() + opts.CustomClientAuthentication = clientAuth + + s := RunServer(opts) + defer s.Shutdown() + + var dcerr error + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + + nc, err := nats.Connect( + s.ClientURL(), + nats.UserInfo("valid", _EMPTY_), + nats.NoReconnect(), + nats.ErrorHandler(func(nc *nats.Conn, _ *nats.Subscription, err error) { + dcerr = err + cancel() + })) + if err != nil { + t.Fatalf("Expected client to connect, got: %s", err) + } + + <-ctx.Done() + + checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { + if nc.IsConnected() { + return fmt.Errorf("Expected to be disconnected") + } + return nil + }) + + if dcerr == nil || dcerr.Error() != "nats: authentication expired" { + t.Fatalf("Expected a auth expired error: got: %v", dcerr) + } +} + +func TestNoAuthUserNoConnectProto(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: "127.0.0.1:-1" + accounts { + A { users [{user: "foo", password: "pwd"}] } + } + authorization { timeout: 1 } + no_auth_user: "foo" + `)) + defer os.Remove(conf) + s, o := RunServerWithConfig(conf) + defer s.Shutdown() + + checkClients := func(n int) { + t.Helper() + time.Sleep(100 * time.Millisecond) + if nc := s.NumClients(); nc != n { + t.Fatalf("Expected %d clients, got %d", n, nc) + } + } + + conn, err := net.Dial("tcp", net.JoinHostPort(o.Host, fmt.Sprintf("%d", o.Port))) + require_NoError(t, err) + defer conn.Close() + checkClientsCount(t, s, 1) + + // With no auth user we should not require a CONNECT. + // Make sure we are good on not sending CONN first. + _, err = conn.Write([]byte("PUB foo 2\r\nok\r\n")) + require_NoError(t, err) + checkClients(1) + conn.Close() + + // Now make sure we still do get timed out though. + conn, err = net.Dial("tcp", net.JoinHostPort(o.Host, fmt.Sprintf("%d", o.Port))) + require_NoError(t, err) + defer conn.Close() + checkClientsCount(t, s, 1) + + time.Sleep(1200 * time.Millisecond) + checkClientsCount(t, s, 0) +} + +type captureProxyRequiredLogger struct { + DummyLogger + ch chan string +} + +func (l *captureProxyRequiredLogger) Debugf(format string, args ...any) { + msg := fmt.Sprintf(format, args...) + if strings.Contains(msg, ErrAuthProxyRequired.Error()) { + select { + case l.ch <- msg: + default: + } + } +} + +func TestAuthProxyRequired(t *testing.T) { + conf := createConfFile(t, []byte(` + port: -1 + authorization { + user: user + password: pwd + proxy_required: true + } + `)) + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + l := &captureProxyRequiredLogger{ch: make(chan string, 1)} + s.SetLogger(l, true, false) + + _, err := nats.Connect(s.ClientURL(), nats.UserInfo("user", "pwd")) + require_True(t, errors.Is(err, nats.ErrAuthorization)) + + checkLog := func() { + t.Helper() + select { + case <-l.ch: + return + case <-time.After(time.Second): + t.Fatal("Did not get log statement") + } + } + checkLog() + + drainLog := func() { + for { + select { + case <-l.ch: + default: + return + } + } + } + s.Shutdown() + drainLog() + + nkUsr1, err := nkeys.CreateUser() + require_NoError(t, err) + nkPub1, err := nkUsr1.PublicKey() + require_NoError(t, err) + + nkUsr2, err := nkeys.CreateUser() + require_NoError(t, err) + nkPub2, err := nkUsr2.PublicKey() + require_NoError(t, err) + + conf = createConfFile(t, fmt.Appendf(nil, ` + port: -1 + authorization { + users: [ + {user: user1, password: pwd1} + {user: user2, password: pwd2, proxy_required: true} + {user: user3, password: pwd3, proxy_required: false} + {nkey: "%s", proxy_required: true} + {nkey: "%s", proxy_required: false} + ] + } + `, nkPub1, nkPub2)) + s, _ = RunServerWithConfig(conf) + defer s.Shutdown() + + s.SetLogger(l, true, false) + + checkClients := func() { + t.Helper() + // Should connect ok. + nc := natsConnect(t, s.ClientURL(), nats.UserInfo("user1", "pwd1")) + nc.Close() + + // Should not, since it requires going through proxy. + _, err = nats.Connect(s.ClientURL(), nats.UserInfo("user2", "pwd2")) + require_True(t, errors.Is(err, nats.ErrAuthorization)) + checkLog() + + // Should connect ok. + nc = natsConnect(t, s.ClientURL(), nats.UserInfo("user3", "pwd3")) + nc.Close() + + // Should not, since it requires going through proxy. + _, err = nats.Connect(s.ClientURL(), nats.Nkey(nkPub1, func(nonce []byte) ([]byte, error) { + return nkUsr1.Sign(nonce) + })) + require_True(t, errors.Is(err, nats.ErrAuthorization)) + checkLog() + + // Should connect ok. + nc = natsConnect(t, s.ClientURL(), nats.Nkey(nkPub2, func(nonce []byte) ([]byte, error) { + return nkUsr2.Sign(nonce) + })) + nc.Close() + } + checkClients() + s.Shutdown() + drainLog() + + conf = createConfFile(t, fmt.Appendf(nil, ` + port: -1 + accounts { + A { + users: [ + {user: user1, password: pwd1} + {user: user2, password: pwd2, proxy_required: true} + {user: user3, password: pwd3, proxy_required: false} + {nkey: "%s", proxy_required: true} + {nkey: "%s", proxy_required: false} + ] + } + SYS { users: [ {user:sys, password: pwd} ] } + } + system_account: SYS + `, nkPub1, nkPub2)) + s, _ = RunServerWithConfig(conf) + defer s.Shutdown() + + s.SetLogger(l, true, false) + + snc := natsConnect(t, s.ClientURL(), nats.UserInfo("sys", "pwd")) + defer snc.Close() + sub := natsSubSync(t, snc, "$SYS.SERVER.*.CLIENT.AUTH.ERR") + sub2 := natsSubSync(t, snc, "$SYS.ACCOUNT.A.DISCONNECT") + natsFlush(t, snc) + + checkClients() + + // We should get 2 authentication error messages, saying that + // the reason is "ProxyRequired". + for range 2 { + var de DisconnectEventMsg + msg := natsNexMsg(t, sub, time.Second) + err = json.Unmarshal(msg.Data, &de) + require_NoError(t, err) + require_Equal(t, de.Reason, ProxyRequired.String()) + } + + // We should get 3 disconnect events since only 3 have connected + // ok and then just closed. + for range 3 { + var de DisconnectEventMsg + msg := natsNexMsg(t, sub2, time.Second) + err = json.Unmarshal(msg.Data, &de) + require_NoError(t, err) + require_Equal(t, de.Reason, ClientClosed.String()) + } + // Make sure there is no more. + if msg, err := sub2.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { + t.Fatalf("Should have received only 3 disconnect messages, got another: %s", msg.Data) + } + + s.Shutdown() + drainLog() + + conf = createConfFile(t, []byte(` + port: -1 + leafnodes: { + port: -1 + authorization { + user: user + password: pwd + proxy_required: true + } + } + `)) + s, o := RunServerWithConfig(conf) + defer s.Shutdown() + + s.SetLogger(l, true, false) + + lconf := createConfFile(t, fmt.Appendf(nil, ` + port: -1 + leafnodes { + reconnect_interval: "50ms" + remotes [ {url: "nats://user:pwd@127.0.0.1:%d"} ] + } + `, o.LeafNode.Port)) + leaf, _ := RunServerWithConfig(lconf) + defer leaf.Shutdown() + + time.Sleep(125 * time.Millisecond) + checkLeafNodeConnectedCount(t, leaf, 0) + checkLog() + leaf.Shutdown() + s.Shutdown() + drainLog() + + conf = createConfFile(t, []byte(` + port: -1 + leafnodes: { + port: -1 + reconnect_interval: "50ms" + authorization { + users: [ + {user: user1, password: pwd1} + {user: user2, password: pwd2, proxy_required: true} + {user: user3, password: pwd3, proxy_required: false} + ] + } + } + `)) + s, o = RunServerWithConfig(conf) + defer s.Shutdown() + + s.SetLogger(l, true, false) + + lconf = createConfFile(t, fmt.Appendf(nil, ` + port: -1 + leafnodes { + reconnect_interval: "50ms" + remotes [ {url: "nats://user2:pwd@127.0.0.1:%d"} ] + } + `, o.LeafNode.Port)) + leaf, _ = RunServerWithConfig(lconf) + defer leaf.Shutdown() + + time.Sleep(125 * time.Millisecond) + checkLeafNodeConnectedCount(t, leaf, 0) + checkLog() + leaf.Shutdown() + s.Shutdown() + drainLog() + + conf = createConfFile(t, fmt.Appendf(nil, ` + port: -1 + leafnodes: { + port: -1 + authorization { + nkey: %s + proxy_required: true + } + } + `, nkPub1)) + s, o = RunServerWithConfig(conf) + defer s.Shutdown() + + s.SetLogger(l, true, false) + + nkSeed1, err := nkUsr1.Seed() + require_NoError(t, err) + + lconf = createConfFile(t, fmt.Appendf(nil, ` + port: -1 + leafnodes { + reconnect_interval: "50ms" + remotes [ + { + url: "nats://127.0.0.1:%d" + nkey: "%s" + } + ] + } + `, o.LeafNode.Port, nkSeed1)) + leaf, _ = RunServerWithConfig(lconf) + defer leaf.Shutdown() + + time.Sleep(125 * time.Millisecond) + checkLeafNodeConnectedCount(t, leaf, 0) + checkLog() + leaf.Shutdown() + s.Shutdown() + drainLog() + + // Check with operator mode. + conf = createConfFile(t, []byte(` + port: -1 + server_name: OP + operator = "../test/configs/nkeys/op.jwt" + resolver = MEMORY + listen: "127.0.0.1:-1" + leafnodes { + listen: "127.0.0.1:-1" + } + `)) + s, o = RunServerWithConfig(conf) + defer s.Shutdown() + + s.SetLogger(l, true, false) + + _, akp := createAccount(s) + kp, err := nkeys.CreateUser() + require_NoError(t, err) + pub, err := kp.PublicKey() + require_NoError(t, err) + nuc := jwt.NewUserClaims(pub) + nuc.ProxyRequired = true + ujwt, err := nuc.Encode(akp) + require_NoError(t, err) + + dto := DefaultTestOptions + lopts := &dto + u, err := url.Parse(fmt.Sprintf("nats://%s:%d", o.LeafNode.Host, o.LeafNode.Port)) + require_NoError(t, err) + remote := &RemoteLeafOpts{URLs: []*url.URL{u}} + remote.SignatureCB = func(nonce []byte) (string, []byte, error) { + sig, err := kp.Sign(nonce) + return ujwt, sig, err + } + lopts.LeafNode.Remotes = []*RemoteLeafOpts{remote} + lopts.LeafNode.ReconnectInterval = 100 * time.Millisecond + leaf = RunServer(lopts) + defer leaf.Shutdown() + + time.Sleep(125 * time.Millisecond) + checkLeafNodeConnectedCount(t, leaf, 0) + checkLog() + leaf.Shutdown() + + // Try with an user. + drainLog() + + _, err = nats.Connect(s.ClientURL(), createUserCredsEx(t, nuc, akp)) + require_True(t, errors.Is(err, nats.ErrAuthorization)) + checkLog() + + drainLog() + + // Try with creds file. + kp, err = nkeys.CreateUser() + require_NoError(t, err) + useed, err := kp.Seed() + require_NoError(t, err) + pub, err = kp.PublicKey() + require_NoError(t, err) + nuc = jwt.NewUserClaims(pub) + nuc.ProxyRequired = true + ujwt, err = nuc.Encode(akp) + require_NoError(t, err) + credsFile := genCredsFile(t, ujwt, useed) + + lconf = createConfFile(t, fmt.Appendf(nil, ` + port: -1 + leafnodes { + reconnect_interval: "50ms" + remotes [ + { + url: "nats://127.0.0.1:%d" + credentials: '%s' + } + ] + } + `, o.LeafNode.Port, credsFile)) + leaf, _ = RunServerWithConfig(lconf) + defer leaf.Shutdown() + + time.Sleep(125 * time.Millisecond) + checkLeafNodeConnectedCount(t, leaf, 0) + checkLog() + leaf.Shutdown() + + s.Shutdown() + drainLog() +} + +func TestAuthProxyCheckConnClosedDuringAuth(t *testing.T) { + pkp, err := nkeys.CreateUser() + require_NoError(t, err) + pub, err := pkp.PublicKey() + require_NoError(t, err) + + o := DefaultOptions() + o.Proxies = &ProxiesConfig{Trusted: []*ProxyConfig{{Key: pub}}} + s := RunServer(o) + defer s.Shutdown() + + c, _, _ := newClientForServer(s) + defer c.close() + + // Sign the connection's nonce as the trusted proxy would and store + // the signature in the connect options. + c.mu.Lock() + sig, err := pkp.Sign(c.nonce) + if err == nil { + c.opts.ProxySig = base64.RawURLEncoding.EncodeToString(sig) + } + c.mu.Unlock() + require_NoError(t, err) + + // Simulate the connection being closed concurrently (write error, + // shutdown, kick, etc..) while the readLoop is still authenticating: + // closeConnection invokes removeClient, which tries to remove the + // connection from s.proxiedConns before proxyCheck registered it. + c.client.closeConnection(ClientClosed) + + // Now run proxyCheck as checkAuthentication would on the readLoop. + proxied, ok := s.proxyCheck(c.client, s.getOpts()) + require_True(t, proxied) + require_True(t, ok) + + // Since the connection is already closed and removeClient already ran, + // the client must not have been registered in s.proxiedConns, since + // nothing would ever remove it. + s.mu.RLock() + conns := s.proxiedConns[pub] + s.mu.RUnlock() + require_Len(t, len(conns), 0) +} + +const ( + authViolation = "-ERR 'Authorization Violation'" + authAccepted = "PONG\r\n" +) + +// testAuthWSConnect sends a websocket CONNECT with the given user and password +// followed by a PING, and checks that the first protocol frame the server sends +// back starts with expect. Going through the raw protocol rather than a +// nats.Conn keeps the failure deterministic: the server closes right after +// writing the error, so a client would race it and may surface the closed +// connection instead. +func testAuthWSConnect(t *testing.T, o *Options, user, pass, expect string) { + t.Helper() + wsc, br, _ := testNewWSClient(t, testWSClientOptions{ + host: o.Websocket.Host, + port: o.Websocket.Port, + noTLS: true, + }) + defer wsc.Close() + + proto := fmt.Sprintf("CONNECT {\"verbose\":false,\"protocol\":1,\"user\":%q,\"pass\":%q}\r\nPING\r\n", user, pass) + if _, err := wsc.Write(testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(proto))); err != nil { + t.Fatalf("Error sending message: %v", err) + } + if msg := string(testWSReadFrame(t, br)); !strings.HasPrefix(msg, expect) { + t.Fatalf("Expected %q, got %q", expect, msg) + } +} + +func TestAuthCertMappedUserNotPasswordAuthenticatable(t *testing.T) { + const certUser = "CN=example.com,OU=NATS.io" + + // The client port has to be reachable under a name the test certificates + // carry a SAN for, hence localhost rather than 127.0.0.1. + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: "localhost:-1" + server_name: "S1" + tls { + cert_file: "../test/configs/certs/tlsauth/server.pem" + key_file: "../test/configs/certs/tlsauth/server-key.pem" + ca_file: "../test/configs/certs/tlsauth/ca.pem" + verify_and_map: true + } + leafnodes { listen: "127.0.0.1:-1" } + websocket { listen: "127.0.0.1:-1", no_tls: true } + authorization { + users [ + { user = "CN=other.example.com,OU=NATS.io" } + { user = "%s" } + ] + } + `, certUser))) + s, o := RunServerWithConfig(conf) + defer s.Shutdown() + + for _, test := range []struct { + name string + user string + pass string + }{ + {"empty password", certUser, _EMPTY_}, + {"wrong password", certUser, "wrongpassword"}, + {"unknown user", "CN=attacker.com,OU=NATS.io", _EMPTY_}, + {"anonymous", _EMPTY_, _EMPTY_}, + } { + t.Run(test.name, func(t *testing.T) { + t.Run("leafnode", func(t *testing.T) { + c, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", o.LeafNode.Port)) + require_NoError(t, err) + defer c.Close() + + br := bufio.NewReader(c) + c.SetDeadline(time.Now().Add(2 * time.Second)) + // Consume the INFO the server sends on accept. + _, _, err = br.ReadLine() + require_NoError(t, err) + + _, err = fmt.Fprintf(c, "CONNECT {\"user\":%q,\"pass\":%q}\r\nPING\r\n", test.user, test.pass) + require_NoError(t, err) + + line, _, err := br.ReadLine() + require_NoError(t, err) + require_Contains(t, string(line), authViolation) + }) + + // The mqtt listener is covered by + // TestMQTTCertMappedUserNotPasswordAuthenticatable, its helpers are + // behind the skip_mqtt_tests build tag. + t.Run("websocket", func(t *testing.T) { + testAuthWSConnect(t, o, test.user, test.pass, authViolation) + }) + }) + } + + // The certificate based login on the mapped listener must still work. + nc := natsConnect(t, s.ClientURL(), + nats.ClientCert("../test/configs/certs/tlsauth/client.pem", "../test/configs/certs/tlsauth/client-key.pem"), + nats.RootCAs("../test/configs/certs/tlsauth/ca.pem")) + defer nc.Close() + natsFlush(t, nc) +} + +func TestAuthCertMappedUserWithPassword(t *testing.T) { + const certUser = "CN=example.com,OU=NATS.io" + + conf := createConfFile(t, []byte(` + listen: "localhost:-1" + tls { + cert_file: "../test/configs/certs/tlsauth/server.pem" + key_file: "../test/configs/certs/tlsauth/server-key.pem" + ca_file: "../test/configs/certs/tlsauth/ca.pem" + verify_and_map: true + } + websocket { listen: "127.0.0.1:-1", no_tls: true } + authorization { + users [ + { user = "CN=other.example.com,OU=NATS.io" } + { user = "CN=example.com,OU=NATS.io", password = "s3cr3t" } + ] + } + `)) + s, o := RunServerWithConfig(conf) + defer s.Shutdown() + + certOpts := []nats.Option{ + nats.ClientCert("../test/configs/certs/tlsauth/client.pem", "../test/configs/certs/tlsauth/client-key.pem"), + nats.RootCAs("../test/configs/certs/tlsauth/ca.pem"), + } + + // The certificate on its own does not authenticate: the configured password + // is still compared against what the client sent. + if bad, err := nats.Connect(s.ClientURL(), certOpts...); err == nil { + bad.Close() + t.Fatal("Expected connection without the password to fail, it did not") + } + + // The certificate together with the password does. + nc := natsConnect(t, s.ClientURL(), append(certOpts, nats.UserInfo(_EMPTY_, "s3cr3t"))...) + defer nc.Close() + natsFlush(t, nc) + + // On the listener that does not map, the password is what authenticates. + testAuthWSConnect(t, o, certUser, "s3cr3t", authAccepted) + + // But an empty one still does not. + testAuthWSConnect(t, o, certUser, _EMPTY_, authViolation) +} + +func TestAuthNoAuthUserWithoutPasswordAndCertMapping(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: "127.0.0.1:-1" + tls { + cert_file: "../test/configs/certs/tlsauth/server.pem" + key_file: "../test/configs/certs/tlsauth/server-key.pem" + ca_file: "../test/configs/certs/tlsauth/ca.pem" + verify_and_map: true + } + websocket { listen: "127.0.0.1:-1", no_tls: true } + authorization { + users [ + { user = "CN=example.com,OU=NATS.io" } + { user = "guest" } + ] + } + no_auth_user: "guest" + `)) + s, o := RunServerWithConfig(conf) + defer s.Shutdown() + + // No credentials at all, so the server selects the no_auth_user itself. + testAuthWSConnect(t, o, _EMPTY_, _EMPTY_, authAccepted) + + // Naming the no_auth_user explicitly is equivalent, it is reachable without + // credentials either way. + testAuthWSConnect(t, o, "guest", _EMPTY_, authAccepted) + + // The other passwordless user is a certificate identity, so naming it + // explicitly must not authenticate. + testAuthWSConnect(t, o, "CN=example.com,OU=NATS.io", _EMPTY_, authViolation) +} + +func TestAuthCertMappedLeafNodeUsersScope(t *testing.T) { + const ( + certUser = "CN=example.com,OU=NATS.io" + leafTLS = `tls { + cert_file: "../test/configs/certs/tlsauth/server.pem" + key_file: "../test/configs/certs/tlsauth/server-key.pem" + ca_file: "../test/configs/certs/tlsauth/ca.pem" + verify_and_map: true + }` + ) + + // A leafnode block with its own users never consults the server's users + // table, so its mapping says nothing about the users defined there. A + // passwordless user in the top-level authorization block is not a + // certificate identity here and must still authenticate. + t.Run("leafnode users", func(t *testing.T) { + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: "127.0.0.1:-1" + websocket { listen: "127.0.0.1:-1", no_tls: true } + leafnodes { + listen: "127.0.0.1:-1" + %s + authorization { users = [ { user = %q } ] } + } + authorization { users = [ { user = "telemetry" } ] } + `, leafTLS, certUser))) + s, o := RunServerWithConfig(conf) + defer s.Shutdown() + + testAuthWSConnect(t, o, "telemetry", _EMPTY_, authAccepted) + }) + + // Without a leafnode authorization block the leafnode listener falls back to + // the server's users table, so its mapping does make the passwordless users + // there certificate identities. + t.Run("no leafnode users", func(t *testing.T) { + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: "127.0.0.1:-1" + websocket { listen: "127.0.0.1:-1", no_tls: true } + leafnodes { + listen: "127.0.0.1:-1" + %s + } + authorization { users = [ { user = %q } ] } + `, leafTLS, certUser))) + s, o := RunServerWithConfig(conf) + defer s.Shutdown() + + testAuthWSConnect(t, o, certUser, _EMPTY_, authViolation) + }) +} + +// testAuthConnect performs the same CONNECT/PING exchange as testAuthWSConnect, +// but over the plain client port. +func testAuthConnect(t *testing.T, o *Options, user, pass, expect string) { + t.Helper() + c, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", o.Port)) + require_NoError(t, err) + defer c.Close() + + br := bufio.NewReader(c) + c.SetDeadline(time.Now().Add(5 * time.Second)) + // Consume the INFO the server sends on accept. + _, err = br.ReadString('\n') + require_NoError(t, err) + + _, err = fmt.Fprintf(c, "CONNECT {\"verbose\":false,\"protocol\":1,\"user\":%q,\"pass\":%q}\r\nPING\r\n", user, pass) + require_NoError(t, err) + + msg, err := br.ReadString('\n') + require_NoError(t, err) + if !strings.HasPrefix(msg, expect) { + t.Fatalf("Expected %q, got %q", expect, msg) + } +} + +func TestAuthCertMappedUserConnectionTypeScope(t *testing.T) { + // Only the client listener maps, so a passwordless user is a certificate + // identity only if the mapping lookup there could have selected it, which it + // cannot once allowed_connection_types excludes STANDARD. + for _, test := range []struct { + name string + types string + expect string + }{ + {"unscoped", _EMPTY_, authViolation}, + {"standard and websocket", `, allowed_connection_types: ["STANDARD","WEBSOCKET"]`, authViolation}, + {"websocket only", `, allowed_connection_types: ["WEBSOCKET"]`, authAccepted}, + {"mqtt and websocket", `, allowed_connection_types: ["MQTT","WEBSOCKET"]`, authAccepted}, + } { + t.Run(test.name, func(t *testing.T) { + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: "127.0.0.1:-1" + tls { + cert_file: "../test/configs/certs/tlsauth/server.pem" + key_file: "../test/configs/certs/tlsauth/server-key.pem" + ca_file: "../test/configs/certs/tlsauth/ca.pem" + verify_and_map: true + } + websocket { listen: "127.0.0.1:-1", no_tls: true } + authorization { + users [ + { user = "CN=example.com,OU=NATS.io" } + { user = "scoped"%s } + ] + } + `, test.types))) + s, o := RunServerWithConfig(conf) + defer s.Shutdown() + + testAuthWSConnect(t, o, "scoped", _EMPTY_, test.expect) + }) + } +} + +func TestAuthCertMappedUserOnOtherListeners(t *testing.T) { + const certs = ` + cert_file: "../test/configs/certs/tlsauth/server.pem" + key_file: "../test/configs/certs/tlsauth/server-key.pem" + ca_file: "../test/configs/certs/tlsauth/ca.pem" + verify_and_map: true` + + wsMapped := fmt.Sprintf(`websocket { listen: "127.0.0.1:-1", tls {%s} }`, certs) + mqttMapped := fmt.Sprintf(`mqtt { listen: "127.0.0.1:-1", tls {%s} }`, certs) + + // The same blocks without a listen, so the listener never starts. + wsDisabled := fmt.Sprintf(`websocket { tls {%s} }`, certs) + mqttDisabled := fmt.Sprintf(`mqtt { tls {%s} }`, certs) + leafDisabled := fmt.Sprintf(`leafnodes { tls {%s} }`, certs) + + // The client listener never maps here, so whether the passwordless user is a + // certificate identity depends entirely on the listener that does. + for _, test := range []struct { + name string + listener string + types string + expect string + }{ + {"websocket mapped", wsMapped, _EMPTY_, authViolation}, + {"websocket mapped, standard only", wsMapped, `, allowed_connection_types: ["STANDARD"]`, authAccepted}, + {"mqtt mapped", mqttMapped, _EMPTY_, authViolation}, + {"mqtt mapped, standard only", mqttMapped, `, allowed_connection_types: ["STANDARD"]`, authAccepted}, + // A block without a listener is inert, its mapping can select nobody. + {"websocket mapped but disabled", wsDisabled, _EMPTY_, authAccepted}, + {"mqtt mapped but disabled", mqttDisabled, _EMPTY_, authAccepted}, + {"leafnode mapped but disabled", leafDisabled, _EMPTY_, authAccepted}, + } { + t.Run(test.name, func(t *testing.T) { + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: "127.0.0.1:-1" + jetstream { store_dir: %q } + %s + authorization { + users [ + { user = "CN=example.com,OU=NATS.io" } + { user = "scoped"%s } + ] + } + `, t.TempDir(), test.listener, test.types))) + s, o := RunServerWithConfig(conf) + defer s.Shutdown() + + testAuthConnect(t, o, "scoped", _EMPTY_, test.expect) + }) + } +} + +func TestAuthCertMappedUserOverWebsocketTransport(t *testing.T) { + const certs = ` + cert_file: "../test/configs/certs/tlsauth/server.pem" + key_file: "../test/configs/certs/tlsauth/server-key.pem" + ca_file: "../test/configs/certs/tlsauth/ca.pem"` + + mqttMapped := fmt.Sprintf(`mqtt { listen: "127.0.0.1:-1", tls {%s + verify_and_map: true} }`, certs) + leafMapped := fmt.Sprintf(`leafnodes { listen: "127.0.0.1:-1", tls {%s + verify_and_map: true} }`, certs) + // Only the second websocket listener can collect a client certificate. + wsPlain := `websocket { listen: "127.0.0.1:-1", no_tls: true }` + wsCerts := fmt.Sprintf(`websocket { listen: "127.0.0.1:-1", tls {%s + verify: true} }`, certs) + + // Mqtt and leafnode connections over websocket are mapped with their own + // listener, but can only present a certificate if the websocket listener + // asks for one. + for _, test := range []struct { + name string + mapped string + ws string + types string + expect string + }{ + {"mqtt, plain websocket", mqttMapped, wsPlain, `"STANDARD","MQTT_WS"`, authAccepted}, + {"mqtt, websocket with client certs", mqttMapped, wsCerts, `"STANDARD","MQTT_WS"`, authViolation}, + {"mqtt, direct mqtt allowed", mqttMapped, wsPlain, `"STANDARD","MQTT"`, authViolation}, + {"leafnode, plain websocket", leafMapped, wsPlain, `"STANDARD","LEAFNODE_WS"`, authAccepted}, + {"leafnode, websocket with client certs", leafMapped, wsCerts, `"STANDARD","LEAFNODE_WS"`, authViolation}, + {"leafnode, direct leafnode allowed", leafMapped, wsPlain, `"STANDARD","LEAFNODE"`, authViolation}, + } { + t.Run(test.name, func(t *testing.T) { + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: "127.0.0.1:-1" + jetstream { store_dir: %q } + %s + %s + authorization { + users [ + { user = "CN=example.com,OU=NATS.io" } + { user = "scoped", allowed_connection_types: [%s] } + ] + } + `, t.TempDir(), test.mapped, test.ws, test.types))) + s, o := RunServerWithConfig(conf) + defer s.Shutdown() + + testAuthConnect(t, o, "scoped", _EMPTY_, test.expect) + }) + } +} diff --git a/server/avl/norace_test.go b/server/avl/norace_test.go new file mode 100644 index 0000000..924a27c --- /dev/null +++ b/server/avl/norace_test.go @@ -0,0 +1,198 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !race && !skip_no_race_tests && !skip_no_race_1_tests + +package avl + +import ( + "flag" + "fmt" + "math" + "math/rand/v2" + "runtime" + "runtime/debug" + "testing" + "time" +) + +// Print Results: go test -v --args --results +var printResults = flag.Bool("results", false, "Enable Results Logging") + +// SequenceSet memory tests vs dmaps. +func TestNoRaceSeqSetSizeComparison(t *testing.T) { + // Create 5M random entries (dupes possible but ok for this test) out of 8M range. + num := 5_000_000 + max := 7_000_000 + + seqs := make([]uint64, 0, num) + for i := 0; i < num; i++ { + n := uint64(rand.Int64N(int64(max + 1))) + seqs = append(seqs, n) + } + + runtime.GC() + // Disable to get stable results. + gcp := debug.SetGCPercent(-1) + defer debug.SetGCPercent(gcp) + + mem := runtime.MemStats{} + runtime.ReadMemStats(&mem) + inUseBefore := mem.HeapInuse + + dmap := make(map[uint64]struct{}, num) + for _, n := range seqs { + dmap[n] = struct{}{} + } + runtime.ReadMemStats(&mem) + dmapUse := mem.HeapInuse - inUseBefore + inUseBefore = mem.HeapInuse + + // Now do SequenceSet on same dataset. + var sset SequenceSet + for _, n := range seqs { + sset.Insert(n) + } + + runtime.ReadMemStats(&mem) + seqSetUse := mem.HeapInuse - inUseBefore + + if seqSetUse > 2*1024*1024 { + t.Fatalf("Expected SequenceSet size to be < 2M, got %v", friendlyBytes(seqSetUse)) + } + if seqSetUse*50 > dmapUse { + t.Fatalf("Expected SequenceSet to be at least 50x better then dmap approach: %v vs %v", + friendlyBytes(seqSetUse), + friendlyBytes(dmapUse), + ) + } +} + +func TestNoRaceSeqSetEncodeLarge(t *testing.T) { + num := 2_500_000 + max := 5_000_000 + + dmap := make(map[uint64]struct{}, num) + var ss SequenceSet + for i := 0; i < num; i++ { + n := uint64(rand.Int64N(int64(max + 1))) + ss.Insert(n) + dmap[n] = struct{}{} + } + + // Disable to get stable results. + gcp := debug.SetGCPercent(-1) + defer debug.SetGCPercent(gcp) + + // In general should be about the same, but can see some variability. + expected := time.Millisecond + + start := time.Now() + b := ss.Encode(nil) + + if elapsed := time.Since(start); elapsed > expected { + t.Fatalf("Expected encode of %d items with encoded size %v to take less than %v, got %v", + num, friendlyBytes(len(b)), expected, elapsed) + } else { + logResults("Encode time for %d items was %v, encoded size is %v\n", num, elapsed, friendlyBytes(len(b))) + } + + start = time.Now() + ss2, _, err := Decode(b) + require_NoError(t, err) + if elapsed := time.Since(start); elapsed > expected { + t.Fatalf("Expected decode to take less than %v, got %v", expected, elapsed) + } else { + logResults("Decode time is %v\n", elapsed) + } + require_True(t, ss.Nodes() == ss2.Nodes()) + require_True(t, ss.Size() == ss2.Size()) +} + +func TestNoRaceSeqSetRelativeSpeed(t *testing.T) { + // Create 1M random entries (dupes possible but ok for this test) out of 3M range. + num := 1_000_000 + max := 3_000_000 + + seqs := make([]uint64, 0, num) + for i := 0; i < num; i++ { + n := uint64(rand.Int64N(int64(max + 1))) + seqs = append(seqs, n) + } + + start := time.Now() + // Now do SequenceSet on same dataset. + var sset SequenceSet + for _, n := range seqs { + sset.Insert(n) + } + ssInsertElapsed := time.Since(start) + logResults("Inserts SequenceSet: %v for %d items\n", ssInsertElapsed, num) + + start = time.Now() + for _, n := range seqs { + if ok := sset.Exists(n); !ok { + t.Fatalf("Should exist") + } + } + ssLookupElapsed := time.Since(start) + logResults("Lookups: %v\n", ssLookupElapsed) + + // Now do a map. + dmap := make(map[uint64]struct{}) + start = time.Now() + for _, n := range seqs { + dmap[n] = struct{}{} + } + mapInsertElapsed := time.Since(start) + logResults("Inserts Map[uint64]: %v for %d items\n", mapInsertElapsed, num) + + start = time.Now() + for _, n := range seqs { + if _, ok := dmap[n]; !ok { + t.Fatalf("Should exist") + } + } + mapLookupElapsed := time.Since(start) + logResults("Lookups: %v\n", mapLookupElapsed) + + // In general we are between 1.5 and 1.75 times slower atm then a straight map. + // Let's test an upper bound of 2x for now. + if mapInsertElapsed*2 <= ssInsertElapsed { + t.Fatalf("Expected SequenceSet insert to be no more than 2x slower (%v vs %v)", mapInsertElapsed, ssInsertElapsed) + } + + if mapLookupElapsed*3 <= ssLookupElapsed { + t.Fatalf("Expected SequenceSet lookups to be no more than 3x slower (%v vs %v)", mapLookupElapsed, ssLookupElapsed) + } +} + +// friendlyBytes returns a string with the given bytes int64 +// represented as a size, such as 1KB, 10MB, etc... +func friendlyBytes[T int | uint64 | int64](bytes T) string { + fbytes := float64(bytes) + base := 1024 + pre := []string{"K", "M", "G", "T", "P", "E"} + if fbytes < float64(base) { + return fmt.Sprintf("%v B", fbytes) + } + exp := int(math.Log(fbytes) / math.Log(float64(base))) + index := exp - 1 + return fmt.Sprintf("%.2f %sB", fbytes/math.Pow(float64(base), float64(exp)), pre[index]) +} + +func logResults(format string, args ...any) { + if *printResults { + fmt.Printf(format, args...) + } +} diff --git a/server/avl/seqset.go b/server/avl/seqset.go new file mode 100644 index 0000000..5ce6fc5 --- /dev/null +++ b/server/avl/seqset.go @@ -0,0 +1,715 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package avl + +import ( + "cmp" + "encoding/binary" + "errors" + "math/bits" + "slices" +) + +// SequenceSet is a memory and encoding optimized set for storing unsigned ints. +// +// SequenceSet is ~80-100 times more efficient memory wise than a map[uint64]struct{}. +// SequenceSet is ~1.75 times slower at inserts than the same map. +// SequenceSet is not thread safe. +// +// We use an AVL tree with nodes that hold bitmasks for set membership. +// +// Encoding will convert to a space optimized encoding using bitmasks. +type SequenceSet struct { + root *node // root node + size int // number of items + nodes int // number of nodes + // Having this here vs on the stack in Insert/Delete + // makes a difference in memory usage. + changed bool +} + +// Insert will insert the sequence into the set. +// The tree will be balanced inline. +func (ss *SequenceSet) Insert(seq uint64) { + // If a node covering seq already exists, setting a bit can not change the + // tree shape, so skip the recursive descent and rebalance checks. + for n := ss.root; n != nil; { + if seq < n.base { + n = n.l + } else if seq >= n.base+numEntries { + n = n.r + } else { + n.set(seq, &ss.changed) + if ss.changed { + ss.changed = false + ss.size++ + } + return + } + } + if ss.root = ss.root.insert(seq, &ss.changed, &ss.nodes); ss.changed { + ss.changed = false + ss.size++ + } +} + +// Exists will return true iff the sequence is a member of this set. +func (ss *SequenceSet) Exists(seq uint64) bool { + for n := ss.root; n != nil; { + if seq < n.base { + n = n.l + continue + } else if seq >= n.base+numEntries { + n = n.r + continue + } + return n.exists(seq) + } + return false +} + +// SetInitialMin should be used to set the initial minimum sequence when known. +// This will more effectively utilize space versus self selecting. +// The set should be empty. +func (ss *SequenceSet) SetInitialMin(min uint64) error { + if !ss.IsEmpty() { + return ErrSetNotEmpty + } + ss.root, ss.nodes = &node{base: min, h: 1}, 1 + return nil +} + +// Delete will remove the sequence from the set. +// Will optionally remove nodes and rebalance. +// Returns where the sequence was set. +func (ss *SequenceSet) Delete(seq uint64) bool { + if ss == nil || ss.root == nil { + return false + } + ss.root = ss.root.delete(seq, &ss.changed, &ss.nodes) + if ss.changed { + ss.changed = false + ss.size-- + if ss.size == 0 { + ss.Empty() + } + return true + } + return false +} + +// Size returns the number of items in the set. +func (ss *SequenceSet) Size() int { + return ss.size +} + +// Nodes returns the number of nodes in the tree. +func (ss *SequenceSet) Nodes() int { + return ss.nodes +} + +// Empty will clear all items from a set. +func (ss *SequenceSet) Empty() { + ss.root = nil + ss.size = 0 + ss.nodes = 0 +} + +// IsEmpty is a fast check of the set being empty. +func (ss *SequenceSet) IsEmpty() bool { + if ss == nil || ss.root == nil { + return true + } + return false +} + +// Range will invoke the given function for each item in the set. +// They will range over the set in ascending order. +// If the callback returns false we terminate the iteration. +func (ss *SequenceSet) Range(f func(uint64) bool) { + ss.root.iter(f) +} + +// Heights returns the left and right heights of the tree. +func (ss *SequenceSet) Heights() (l, r int) { + if ss.root == nil { + return 0, 0 + } + if ss.root.l != nil { + l = ss.root.l.h + } + if ss.root.r != nil { + r = ss.root.r.h + } + return l, r +} + +// Returns min, max and number of set items. +func (ss *SequenceSet) State() (min, max, num uint64) { + if ss == nil || ss.root == nil { + return 0, 0, 0 + } + min, max = ss.MinMax() + return min, max, uint64(ss.Size()) +} + +// MinMax will return the minunum and maximum values in the set. +func (ss *SequenceSet) MinMax() (min, max uint64) { + if ss.root == nil { + return 0, 0 + } + for l := ss.root; l != nil; l = l.l { + if l.l == nil { + min = l.min() + } + } + for r := ss.root; r != nil; r = r.r { + if r.r == nil { + max = r.max() + } + } + return min, max +} + +func clone(src *node, target **node) { + if src == nil { + return + } + n := &node{base: src.base, bits: src.bits, h: src.h} + *target = n + clone(src.l, &n.l) + clone(src.r, &n.r) +} + +// Clone will return a clone of the given SequenceSet. +func (ss *SequenceSet) Clone() *SequenceSet { + if ss == nil { + return nil + } + css := &SequenceSet{nodes: ss.nodes, size: ss.size} + clone(ss.root, &css.root) + + return css +} + +// Equal returns whether the two sets contain exactly the same sequences. +func (ss *SequenceSet) Equal(other *SequenceSet) bool { + if ss.IsEmpty() || other.IsEmpty() { + return ss.IsEmpty() && other.IsEmpty() + } + if ss.size != other.size { + return false + } + // Sizes are equal, so a one-way membership check suffices. + equal := true + ss.Range(func(seq uint64) bool { + equal = other.Exists(seq) + return equal + }) + return equal +} + +// Union will union this SequenceSet with ssa. +func (ss *SequenceSet) Union(ssa ...*SequenceSet) { + for _, sa := range ssa { + sa.root.nodeIter(func(n *node) { + for nb, b := range n.bits { + base := n.base + uint64(nb)*bitsPerBucket + for b != 0 { + ss.Insert(base + uint64(bits.TrailingZeros64(b))) + b &= b - 1 + } + } + }) + } +} + +// Union will return a union of all sets. +func Union(ssa ...*SequenceSet) *SequenceSet { + if len(ssa) == 0 { + return nil + } + // Sort so we can clone largest. + slices.SortFunc(ssa, func(i, j *SequenceSet) int { return -cmp.Compare(i.Size(), j.Size()) }) // reverse order + ss := ssa[0].Clone() + + // Insert the rest through range call. + for i := 1; i < len(ssa); i++ { + ssa[i].Range(func(n uint64) bool { + ss.Insert(n) + return true + }) + } + return ss +} + +const ( + // Magic is used to identify the encode binary state.. + magic = uint8(22) + // Version + version = uint8(2) + // hdrLen + hdrLen = 2 + // minimum length of an encoded SequenceSet. + minLen = 2 + 8 // magic + version + num nodes + num entries. +) + +// EncodeLen returns the bytes needed for encoding. +func (ss SequenceSet) EncodeLen() int { + return minLen + (ss.Nodes() * ((numBuckets+1)*8 + 2)) +} + +func (ss SequenceSet) Encode(buf []byte) []byte { + nn, encLen := ss.Nodes(), ss.EncodeLen() + + if cap(buf) < encLen { + buf = make([]byte, encLen) + } else { + buf = buf[:encLen] + } + + // TODO(dlc) - Go 1.19 introduced Append to not have to keep track. + // Once 1.20 is out we could change this over. + // Also binary.Write() is way slower, do not use. + + var le = binary.LittleEndian + buf[0], buf[1] = magic, version + i := hdrLen + le.PutUint32(buf[i:], uint32(nn)) + le.PutUint32(buf[i+4:], uint32(ss.size)) + i += 8 + ss.root.nodeIter(func(n *node) { + le.PutUint64(buf[i:], n.base) + i += 8 + for _, b := range n.bits { + le.PutUint64(buf[i:], b) + i += 8 + } + le.PutUint16(buf[i:], uint16(n.h)) + i += 2 + }) + return buf[:i] +} + +// ErrBadEncoding is returned when we can not decode properly. +var ( + ErrBadEncoding = errors.New("ss: bad encoding") + ErrBadVersion = errors.New("ss: bad version") + ErrSetNotEmpty = errors.New("ss: set not empty") +) + +// Decode returns the sequence set and number of bytes read from the buffer on success. +func Decode(buf []byte) (*SequenceSet, int, error) { + if len(buf) < minLen || buf[0] != magic { + return nil, -1, ErrBadEncoding + } + + switch v := buf[1]; v { + case 1: + return decodev1(buf) + case 2: + return decodev2(buf) + default: + return nil, -1, ErrBadVersion + } +} + +// Helper to decode v2. +func decodev2(buf []byte) (*SequenceSet, int, error) { + var le = binary.LittleEndian + index := 2 + nn := int(le.Uint32(buf[index:])) + sz := int(le.Uint32(buf[index+4:])) + index += 8 + + // nn is decoded as a uint32 but held in an int. On 32-bit builds a value + // above MaxInt32 turns negative and nn*perNode below overflows, so the + // length check would pass for a short buffer and the following make/reads + // run off the end. Compare with division so the bound holds on every arch. + if nn < 0 || nn > (len(buf)-minLen)/((numBuckets+1)*8+2) { + return nil, -1, ErrBadEncoding + } + + ss, nodes := SequenceSet{size: sz}, make([]node, nn) + + for i := 0; i < nn; i++ { + n := &nodes[i] + n.base = le.Uint64(buf[index:]) + index += 8 + for bi := range n.bits { + n.bits[bi] = le.Uint64(buf[index:]) + index += 8 + } + n.h = int(le.Uint16(buf[index:])) + index += 2 + ss.insertNode(n) + } + + return &ss, index, nil +} + +// Helper to decode v1 into v2 which has fixed buckets of 32 vs 64 originally. +func decodev1(buf []byte) (*SequenceSet, int, error) { + var le = binary.LittleEndian + index := 2 + nn := int(le.Uint32(buf[index:])) + sz := int(le.Uint32(buf[index+4:])) + index += 8 + + const v1NumBuckets = 64 + + // See decodev2: guard the node count without overflowing the multiply so + // the bound stays correct on 32-bit builds too. + if nn < 0 || nn > (len(buf)-minLen)/((v1NumBuckets+1)*8+2) { + return nil, -1, ErrBadEncoding + } + + var ss SequenceSet + for i := 0; i < nn; i++ { + base := le.Uint64(buf[index:]) + index += 8 + for nb := uint64(0); nb < v1NumBuckets; nb++ { + n := le.Uint64(buf[index:]) + // Walk all set bits and insert sequences manually for this decode from v1. + for n != 0 { + ss.Insert(base + (nb * uint64(bitsPerBucket)) + uint64(bits.TrailingZeros64(n))) + n &= n - 1 + } + index += 8 + } + // Skip over encoded height. + index += 2 + } + + // Sanity check. + if ss.Size() != sz { + return nil, -1, ErrBadEncoding + } + + return &ss, index, nil + +} + +// insertNode places a decoded node into the tree. +// These should be done in tree order as defined by Encode() +// This allows us to not have to calculate height or do rebalancing. +// So much better performance this way. +func (ss *SequenceSet) insertNode(n *node) { + ss.nodes++ + + if ss.root == nil { + ss.root = n + return + } + // Walk our way to the insertion point. + for p := ss.root; p != nil; { + if n.base < p.base { + if p.l == nil { + p.l = n + return + } + p = p.l + } else { + if p.r == nil { + p.r = n + return + } + p = p.r + } + } +} + +const ( + bitsPerBucket = 64 // bits in uint64 + numBuckets = 32 + numEntries = numBuckets * bitsPerBucket +) + +type node struct { + //v dvalue + base uint64 + bits [numBuckets]uint64 + l *node + r *node + h int +} + +// Set the proper bit. +// seq should have already been qualified and inserted should be non nil. +func (n *node) set(seq uint64, inserted *bool) { + seq -= n.base + i := seq / bitsPerBucket + mask := uint64(1) << (seq % bitsPerBucket) + if (n.bits[i] & mask) == 0 { + n.bits[i] |= mask + *inserted = true + } +} + +func (n *node) insert(seq uint64, inserted *bool, nodes *int) *node { + if n == nil { + base := (seq / numEntries) * numEntries + n := &node{base: base, h: 1} + n.set(seq, inserted) + *nodes++ + return n + } + + if seq < n.base { + n.l = n.l.insert(seq, inserted, nodes) + } else if seq >= n.base+numEntries { + n.r = n.r.insert(seq, inserted, nodes) + } else { + n.set(seq, inserted) + } + + n.h = maxH(n) + 1 + + // Don't make a function, impacts performance. + if bf := balanceF(n); bf > 1 { + // Left unbalanced. + if balanceF(n.l) < 0 { + n.l = n.l.rotateL() + } + return n.rotateR() + } else if bf < -1 { + // Right unbalanced. + if balanceF(n.r) > 0 { + n.r = n.r.rotateR() + } + return n.rotateL() + } + return n +} + +func (n *node) rotateL() *node { + r := n.r + if r != nil { + n.r = r.l + r.l = n + n.h = maxH(n) + 1 + r.h = maxH(r) + 1 + } else { + n.r = nil + n.h = maxH(n) + 1 + } + return r +} + +func (n *node) rotateR() *node { + l := n.l + if l != nil { + n.l = l.r + l.r = n + n.h = maxH(n) + 1 + l.h = maxH(l) + 1 + } else { + n.l = nil + n.h = maxH(n) + 1 + } + return l +} + +func balanceF(n *node) int { + if n == nil { + return 0 + } + var lh, rh int + if n.l != nil { + lh = n.l.h + } + if n.r != nil { + rh = n.r.h + } + return lh - rh +} + +func maxH(n *node) int { + if n == nil { + return 0 + } + var lh, rh int + if n.l != nil { + lh = n.l.h + } + if n.r != nil { + rh = n.r.h + } + if lh > rh { + return lh + } + return rh +} + +// Clear the proper bit. +// seq should have already been qualified and deleted should be non nil. +// Will return true if this node is now empty. +func (n *node) clear(seq uint64, deleted *bool) bool { + seq -= n.base + i := seq / bitsPerBucket + mask := uint64(1) << (seq % bitsPerBucket) + if (n.bits[i] & mask) == 0 { + // Nothing cleared, and nodes in the tree are never empty, + // so no need to scan the buckets. + return false + } + n.bits[i] &^= mask + *deleted = true + for _, b := range n.bits { + if b != 0 { + return false + } + } + return true +} + +func (n *node) delete(seq uint64, deleted *bool, nodes *int) *node { + if n == nil { + return nil + } + + if seq < n.base { + n.l = n.l.delete(seq, deleted, nodes) + } else if seq >= n.base+numEntries { + n.r = n.r.delete(seq, deleted, nodes) + } else if empty := n.clear(seq, deleted); empty { + *nodes-- + if n.l == nil { + n = n.r + } else if n.r == nil { + n = n.l + } else { + // We have both children. + n.r = n.r.insertNodePrev(n.l) + n = n.r + } + } + + if n != nil { + n.h = maxH(n) + 1 + } + + // Check balance. + if bf := balanceF(n); bf > 1 { + // Left unbalanced. + if balanceF(n.l) < 0 { + n.l = n.l.rotateL() + } + return n.rotateR() + } else if bf < -1 { + // right unbalanced. + if balanceF(n.r) > 0 { + n.r = n.r.rotateR() + } + return n.rotateL() + } + + return n +} + +// Will insert nn into the node assuming it is less than all other nodes in n. +// Will re-calculate height and balance. +func (n *node) insertNodePrev(nn *node) *node { + if n.l == nil { + n.l = nn + } else { + n.l = n.l.insertNodePrev(nn) + } + n.h = maxH(n) + 1 + + // Check balance. + if bf := balanceF(n); bf > 1 { + // Left unbalanced. + if balanceF(n.l) < 0 { + n.l = n.l.rotateL() + } + return n.rotateR() + } else if bf < -1 { + // right unbalanced. + if balanceF(n.r) > 0 { + n.r = n.r.rotateR() + } + return n.rotateL() + } + return n +} + +func (n *node) exists(seq uint64) bool { + seq -= n.base + i := seq / bitsPerBucket + mask := uint64(1) << (seq % bitsPerBucket) + return n.bits[i]&mask != 0 +} + +// Return minimum sequence in the set. +// This node can not be empty. +func (n *node) min() uint64 { + for i, b := range n.bits { + if b != 0 { + return n.base + + uint64(i*bitsPerBucket) + + uint64(bits.TrailingZeros64(b)) + } + } + return 0 +} + +// Return maximum sequence in the set. +// This node can not be empty. +func (n *node) max() uint64 { + for i := numBuckets - 1; i >= 0; i-- { + if b := n.bits[i]; b != 0 { + return n.base + + uint64(i*bitsPerBucket) + + uint64(bitsPerBucket-bits.LeadingZeros64(b>>1)) + } + } + return 0 +} + +// This is done in tree order. +func (n *node) nodeIter(f func(n *node)) { + if n == nil { + return + } + f(n) + n.l.nodeIter(f) + n.r.nodeIter(f) +} + +// iter will iterate through the set's items in this node. +// If the supplied function returns false we terminate the iteration. +func (n *node) iter(f func(uint64) bool) bool { + if n == nil { + return true + } + + if ok := n.l.iter(f); !ok { + return false + } + for i, b := range n.bits { + base := n.base + uint64(i)*bitsPerBucket + for b != 0 { + if ok := f(base + uint64(bits.TrailingZeros64(b))); !ok { + return false + } + b &= b - 1 + } + } + if ok := n.r.iter(f); !ok { + return false + } + + return true +} diff --git a/server/avl/seqset_test.go b/server/avl/seqset_test.go new file mode 100644 index 0000000..d1c1c10 --- /dev/null +++ b/server/avl/seqset_test.go @@ -0,0 +1,556 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package avl + +import ( + "encoding/base64" + "encoding/binary" + "math/rand/v2" + "testing" +) + +func TestSeqSetBasics(t *testing.T) { + var ss SequenceSet + + seqs := []uint64{22, 222, 2000, 2, 2, 4} + for _, seq := range seqs { + ss.Insert(seq) + require_True(t, ss.Exists(seq)) + } + + require_True(t, ss.Nodes() == 1) + require_True(t, ss.Size() == len(seqs)-1) // We have one dup in there. + lh, rh := ss.Heights() + require_True(t, lh == 0) + require_True(t, rh == 0) +} + +func TestSeqSetLeftLean(t *testing.T) { + var ss SequenceSet + + for i := uint64(4 * numEntries); i > 0; i-- { + ss.Insert(i) + } + + require_True(t, ss.Nodes() == 5) + require_True(t, ss.Size() == 4*numEntries) + lh, rh := ss.Heights() + require_True(t, lh == 2) + require_True(t, rh == 1) +} + +func TestSeqSetRightLean(t *testing.T) { + var ss SequenceSet + + for i := uint64(0); i < uint64(4*numEntries); i++ { + ss.Insert(i) + } + + require_True(t, ss.Nodes() == 4) + require_True(t, ss.Size() == 4*numEntries) + lh, rh := ss.Heights() + require_True(t, lh == 1) + require_True(t, rh == 2) +} + +func TestSeqSetCorrectness(t *testing.T) { + // Generate 100k sequences across 500k range. + num := 100_000 + max := 500_000 + + set := make(map[uint64]struct{}, num) + var ss SequenceSet + for i := 0; i < num; i++ { + n := uint64(rand.Int64N(int64(max + 1))) + ss.Insert(n) + set[n] = struct{}{} + } + + for i := uint64(0); i <= uint64(max); i++ { + _, exists := set[i] + require_True(t, ss.Exists(i) == exists) + } +} + +func TestSeqSetRange(t *testing.T) { + num := 2*numEntries + 22 + nums := make([]uint64, 0, num) + for i := 0; i < num; i++ { + nums = append(nums, uint64(i)) + } + rand.Shuffle(len(nums), func(i, j int) { nums[i], nums[j] = nums[j], nums[i] }) + + var ss SequenceSet + for _, n := range nums { + ss.Insert(n) + } + + nums = nums[:0] + ss.Range(func(n uint64) bool { + nums = append(nums, n) + return true + }) + require_True(t, len(nums) == num) + for i := uint64(0); i < uint64(num); i++ { + require_True(t, nums[i] == i) + } + + // Test truncating the range call. + nums = nums[:0] + ss.Range(func(n uint64) bool { + if n >= 10 { + return false + } + nums = append(nums, n) + return true + }) + require_True(t, len(nums) == 10) + for i := uint64(0); i < 10; i++ { + require_True(t, nums[i] == i) + } +} + +func TestSeqSetDelete(t *testing.T) { + var ss SequenceSet + + // Simple single node. + seqs := []uint64{22, 222, 2222, 2, 2, 4} + for _, seq := range seqs { + ss.Insert(seq) + } + + for _, seq := range seqs { + ss.Delete(seq) + require_True(t, !ss.Exists(seq)) + } + require_True(t, ss.root == nil) +} + +func TestSeqSetInsertAndDeletePedantic(t *testing.T) { + var ss SequenceSet + + num := 50*numEntries + 22 + nums := make([]uint64, 0, num) + for i := 0; i < num; i++ { + nums = append(nums, uint64(i)) + } + rand.Shuffle(len(nums), func(i, j int) { nums[i], nums[j] = nums[j], nums[i] }) + + // Make sure always balanced. + testBalanced := func() { + t.Helper() + // Check heights. + ss.root.nodeIter(func(n *node) { + if n != nil && n.h != maxH(n)+1 { + t.Fatalf("Node height is wrong: %+v", n) + } + }) + // Check balance factor. + if bf := balanceF(ss.root); bf > 1 || bf < -1 { + t.Fatalf("Unbalanced tree") + } + } + + for _, n := range nums { + ss.Insert(n) + testBalanced() + } + require_True(t, ss.root != nil) + + for _, n := range nums { + ss.Delete(n) + testBalanced() + require_True(t, !ss.Exists(n)) + if ss.Size() > 0 { + require_True(t, ss.root != nil) + } + } + require_True(t, ss.root == nil) +} + +func TestSeqSetMinMax(t *testing.T) { + var ss SequenceSet + + // Simple single node. + seqs := []uint64{22, 222, 2222, 2, 2, 4} + for _, seq := range seqs { + ss.Insert(seq) + } + + min, max := ss.MinMax() + require_True(t, min == 2 && max == 2222) + + // Multi-node + ss.Empty() + + num := 22*numEntries + 22 + nums := make([]uint64, 0, num) + for i := 0; i < num; i++ { + nums = append(nums, uint64(i)) + } + rand.Shuffle(len(nums), func(i, j int) { nums[i], nums[j] = nums[j], nums[i] }) + for _, n := range nums { + ss.Insert(n) + } + + min, max = ss.MinMax() + require_True(t, min == 0 && max == uint64(num-1)) +} + +func TestSeqSetClone(t *testing.T) { + // Generate 100k sequences across 500k range. + num := 100_000 + max := 500_000 + + var ss SequenceSet + for i := 0; i < num; i++ { + ss.Insert(uint64(rand.Int64N(int64(max + 1)))) + } + + ssc := ss.Clone() + require_True(t, ss.Size() == ssc.Size()) + require_True(t, ss.Nodes() == ssc.Nodes()) +} + +func TestSeqSetEqual(t *testing.T) { + var ss1, ss2, ss3 SequenceSet + + // Empty and nil sets are all equal. + var nilSet *SequenceSet + require_True(t, ss1.Equal(&ss2)) + require_True(t, ss1.Equal(nilSet)) + require_True(t, nilSet.Equal(&ss1)) + + // Same state (first, last, size) but different contents. + for _, seq := range []uint64{1, 3, 5} { + ss1.Insert(seq) + } + for _, seq := range []uint64{1, 4, 5} { + ss2.Insert(seq) + } + require_False(t, ss1.Equal(&ss2)) + require_False(t, ss2.Equal(&ss1)) + require_False(t, ss1.Equal(nilSet)) + require_False(t, nilSet.Equal(&ss1)) + + // Identical contents, insertion order should not matter. + for _, seq := range []uint64{5, 1, 3} { + ss3.Insert(seq) + } + require_True(t, ss1.Equal(&ss3)) + require_True(t, ss3.Equal(&ss1)) + + // Differing sizes. + ss3.Insert(7) + require_False(t, ss1.Equal(&ss3)) +} + +func TestSeqSetUnion(t *testing.T) { + var ss1, ss2 SequenceSet + + seqs1 := []uint64{22, 222, 2222, 2, 2, 4} + for _, seq := range seqs1 { + ss1.Insert(seq) + } + + seqs2 := []uint64{33, 333, 3333, 3, 33_333, 333_333} + for _, seq := range seqs2 { + ss2.Insert(seq) + } + + ss := Union(&ss1, &ss2) + require_True(t, ss.Size() == 11) + + seqs := append(seqs1, seqs2...) + for _, n := range seqs { + require_True(t, ss.Exists(n)) + } +} + +func TestSeqSetFirst(t *testing.T) { + var ss SequenceSet + + seqs := []uint64{22, 222, 2222, 222_222} + for _, seq := range seqs { + // Normal case where we pick first/base. + ss.Insert(seq) + require_True(t, ss.root.base == (seq/numEntries)*numEntries) + ss.Empty() + // Where we set the minimum start value. + ss.SetInitialMin(seq) + ss.Insert(seq) + require_True(t, ss.root.base == seq) + ss.Empty() + } +} + +// Test that we can union with nodes vs individual sequence insertion. +func TestSeqSetDistinctUnion(t *testing.T) { + // Distinct sets. + var ss1 SequenceSet + seqs1 := []uint64{1, 10, 100, 200} + for _, seq := range seqs1 { + ss1.Insert(seq) + } + + var ss2 SequenceSet + seqs2 := []uint64{5000, 6100, 6200, 6222} + for _, seq := range seqs2 { + ss2.Insert(seq) + } + + ss := ss1.Clone() + allSeqs := append(seqs1, seqs2...) + + ss.Union(&ss2) + require_True(t, ss.Size() == len(allSeqs)) + for _, seq := range allSeqs { + require_True(t, ss.Exists(seq)) + } +} + +func TestSeqSetDecodeV1(t *testing.T) { + // Encoding from v1 which was 64 buckets. + seqs := []uint64{22, 222, 2222, 222_222, 2_222_222} + encStr := ` +FgEDAAAABQAAAABgAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAADgIQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAA== +` + + enc, err := base64.StdEncoding.DecodeString(encStr) + require_NoError(t, err) + + ss, _, err := Decode(enc) + require_NoError(t, err) + + require_True(t, ss.Size() == len(seqs)) + for _, seq := range seqs { + require_True(t, ss.Exists(seq)) + } +} + +func TestSeqSetDecodeNodeCountOverflow(t *testing.T) { + // A replicated stream state carries an avl-encoded delete map decoded from + // a peer. The node count is read as a uint32 into an int; on 32-bit builds + // a value above MaxInt32 turns negative (and a smaller value can overflow + // the nn*perNode multiply), defeating the length check so the decode then + // panics in make([]node, nn) or reads past the buffer. Both encodings must + // reject such counts on every architecture instead. + le := binary.LittleEndian + for _, ver := range []uint8{1, 2} { + buf := make([]byte, minLen) + buf[0], buf[1] = magic, ver + le.PutUint32(buf[2:], 0xFFFFFFFF) // node count, negative as a 32-bit int + if _, _, err := Decode(buf); err != ErrBadEncoding { + t.Fatalf("v%d: expected ErrBadEncoding for oversized node count, got %v", ver, err) + } + + // Positive on 32-bit but the multiply still overflows for a short buffer. + le.PutUint32(buf[2:], 0x00800000) + if _, _, err := Decode(buf); err != ErrBadEncoding { + t.Fatalf("v%d: expected ErrBadEncoding for overflowing node count, got %v", ver, err) + } + } + + // A well-formed single-node v2 buffer must still decode. + valid := make([]byte, minLen+(numBuckets+1)*8+2) + valid[0], valid[1] = magic, version + le.PutUint32(valid[2:], 1) + if _, _, err := Decode(valid); err != nil { + t.Fatalf("valid single-node buffer should decode, got %v", err) + } +} + +func TestSeqSetInsertExistingNodeFastPath(t *testing.T) { + var ss SequenceSet + + // Build a multi-node tree with only even sequences set. + for i := uint64(0); i < 8*numEntries; i += 2 { + ss.Insert(i) + } + nodes, size := ss.Nodes(), ss.Size() + + // Inserting into existing nodes must not change the tree structure. + for i := uint64(1); i < 8*numEntries; i += 2 { + ss.Insert(i) + require_True(t, ss.Exists(i)) + } + require_True(t, ss.Nodes() == nodes) + require_True(t, ss.Size() == size+4*numEntries) + + // Duplicate inserts must not change the size. + for i := uint64(0); i < 8*numEntries; i++ { + ss.Insert(i) + } + require_True(t, ss.Size() == 8*numEntries) + + // Heights and balance must still be intact. + ss.root.nodeIter(func(n *node) { + if n.h != maxH(n)+1 { + t.Fatalf("Node height is wrong: %+v", n) + } + }) + if bf := balanceF(ss.root); bf > 1 || bf < -1 { + t.Fatalf("Unbalanced tree") + } +} + +func TestSeqSetInsertUnalignedBase(t *testing.T) { + var ss SequenceSet + + // SetInitialMin can create a node whose base is not aligned to numEntries. + require_NoError(t, ss.SetInitialMin(1000)) + for _, seq := range []uint64{1000, 1001, 1000 + numEntries - 1} { + ss.Insert(seq) + require_True(t, ss.Exists(seq)) + } + require_True(t, ss.Nodes() == 1) + require_True(t, ss.Size() == 3) + + // Outside the node's range still creates a new node. + ss.Insert(1000 + numEntries) + require_True(t, ss.Exists(1000+numEntries)) + require_True(t, ss.Nodes() == 2) + require_True(t, ss.Size() == 4) +} + +func TestSeqSetDeleteNotPresent(t *testing.T) { + var ss SequenceSet + + seqs := []uint64{22, 222, 2222, 222_222} + for _, seq := range seqs { + ss.Insert(seq) + } + nodes, size := ss.Nodes(), ss.Size() + + // Deleting sequences that are not set, both inside and outside + // existing node ranges, must be a no-op and report false. + for _, seq := range []uint64{0, 23, 2223, 5000, 100_000, 999_999} { + require_True(t, !ss.Delete(seq)) + } + require_True(t, ss.Nodes() == nodes) + require_True(t, ss.Size() == size) + for _, seq := range seqs { + require_True(t, ss.Exists(seq)) + } + + // Real deletes still work and empty the set. + for _, seq := range seqs { + require_True(t, ss.Delete(seq)) + } + require_True(t, ss.root == nil) +} + +func TestSeqSetRangeSparse(t *testing.T) { + // Cover bucket boundaries, node boundaries and large gaps. + seqs := []uint64{0, 1, 63, 64, 127, 128, 2047, 2048, 4095, 4096, 100_000, 1_000_000} + var ss SequenceSet + for _, seq := range seqs { + ss.Insert(seq) + } + + var got []uint64 + ss.Range(func(n uint64) bool { + got = append(got, n) + return true + }) + require_True(t, len(got) == len(seqs)) + for i, seq := range seqs { + require_True(t, got[i] == seq) + } + + // Terminating mid-bucket should stop the iteration immediately. + got = got[:0] + ss.Range(func(n uint64) bool { + got = append(got, n) + return n != 64 + }) + require_True(t, len(got) == 4) + require_True(t, got[3] == 64) +} + +func BenchmarkSeqSetInsertDense(b *testing.B) { + for i := 0; i < b.N; i++ { + var ss SequenceSet + for s := uint64(0); s < 200_000; s++ { + ss.Insert(s) + } + } +} + +func BenchmarkSeqSetInsertSparse(b *testing.B) { + for i := 0; i < b.N; i++ { + var ss SequenceSet + for s := uint64(0); s < 200_000; s++ { + ss.Insert(s * 7) + } + } +} + +func BenchmarkSeqSetRangeDense(b *testing.B) { + var ss SequenceSet + for s := uint64(0); s < 200_000; s++ { + ss.Insert(s) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + var sum uint64 + ss.Range(func(n uint64) bool { sum += n; return true }) + } +} + +func BenchmarkSeqSetRangeSparse(b *testing.B) { + var ss SequenceSet + for s := uint64(0); s < 10_000; s++ { + ss.Insert(s * 211) // ~10 bits per 2048-entry node + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + var sum uint64 + ss.Range(func(n uint64) bool { sum += n; return true }) + } +} + +func BenchmarkSeqSetDeleteNotPresent(b *testing.B) { + var ss SequenceSet + for s := uint64(0); s < 200_000; s += 2 { + ss.Insert(s) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + for s := uint64(1); s < 200_000; s += 2 { + ss.Delete(s) + } + } +} + +func require_NoError(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatalf("require no error, but got: %v", err) + } +} + +func require_True(t *testing.T, b bool) { + t.Helper() + if !b { + t.Fatalf("require true") + } +} + +func require_False(t *testing.T, b bool) { + t.Helper() + if b { + t.Fatalf("require false") + } +} diff --git a/server/benchmark_publish_test.go b/server/benchmark_publish_test.go new file mode 100644 index 0000000..a0cdf63 --- /dev/null +++ b/server/benchmark_publish_test.go @@ -0,0 +1,185 @@ +// Copyright 2022-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "fmt" + "math/rand/v2" + "sync/atomic" + "testing" + + "github.com/nats-io/nats.go" +) + +func BenchmarkPublish(b *testing.B) { + + const ( + verbose = false + minMessages = 10_000 + subject = "S" + queue = "Q" + ) + + const ( + KB = 1024 + MB = KB * KB + ) + + type SubscriberType string + const ( + Async SubscriberType = "Async" + QueueAsync SubscriberType = "AsyncQueue" + None SubscriberType = "None" + ) + + benchmarksCases := []struct { + messageSize int + }{ + {0}, + {1}, + {32}, + {128}, + {512}, + {4 * KB}, + {32 * KB}, + {128 * KB}, + {512 * KB}, + {1 * MB}, + } + + // All the cases above are run for each of the subscriber cases below + subscribersCases := []struct { + numSubs int + subType SubscriberType + }{ + {0, None}, + {1, Async}, + {1, QueueAsync}, + {10, Async}, + {10, QueueAsync}, + } + + for _, bc := range benchmarksCases { + bcName := fmt.Sprintf( + "MsgSz=%db", + bc.messageSize, + ) + + b.Run( + bcName, + func(b *testing.B) { + + for _, sc := range subscribersCases { + + scName := fmt.Sprintf("Subs=%dx%v", sc.numSubs, sc.subType) + if sc.subType == None { + scName = fmt.Sprintf("Subs=%v", sc.subType) + } + + b.Run( + scName, + func(b *testing.B) { + // Skip short runs, benchmark gets re-executed with a larger N + if b.N < minMessages { + b.ResetTimer() + return + } + + if verbose { + b.Logf("Running %s/%s with %d ops", bcName, scName, b.N) + } + + subErrors := uint64(0) + handleSubError := func(_ *nats.Conn, _ *nats.Subscription, _ error) { + atomic.AddUint64(&subErrors, 1) + } + + // Start single server (no JS) + opts := DefaultTestOptions + opts.Port = -1 + s := RunServer(&opts) + defer s.Shutdown() + + // Create subscribers + for i := 0; i < sc.numSubs; i++ { + subConn, connErr := nats.Connect(s.ClientURL(), nats.ErrorHandler(handleSubError)) + if connErr != nil { + b.Fatalf("Failed to connect: %v", connErr) + } + defer subConn.Close() + + var sub *nats.Subscription + var subErr error + + switch sc.subType { + case None: + // No subscription + case Async: + sub, subErr = subConn.Subscribe(subject, func(*nats.Msg) {}) + case QueueAsync: + sub, subErr = subConn.QueueSubscribe(subject, queue, func(*nats.Msg) {}) + default: + b.Fatalf("Unknow subscribers type: %v", sc.subType) + } + + if subErr != nil { + b.Fatalf("Failed to subscribe: %v", subErr) + } + defer sub.Unsubscribe() + // Do not drop messages due to slow subscribers: + sub.SetPendingLimits(-1, -1) + } + + // Create publisher connection + nc, err := nats.Connect(s.ClientURL()) + if err != nil { + b.Fatalf("Failed to connect: %v", err) + } + defer nc.Close() + + rng := rand.NewChaCha8([32]byte{42}) + message := make([]byte, bc.messageSize) + var published, errors int + + // Benchmark starts here + b.ResetTimer() + + for i := 0; i < b.N; i++ { + rng.Read(message) + pubErr := nc.Publish(subject, message) + if pubErr != nil { + errors++ + } else { + published++ + b.SetBytes(int64(bc.messageSize)) + } + } + + // Benchmark ends here + b.StopTimer() + + if published+errors != b.N { + b.Fatalf("Something doesn't add up: %d + %d != %d", published, errors, b.N) + } else if subErrors > 0 { + b.Fatalf("Subscribers errors: %d", subErrors) + } + + b.ReportMetric(float64(errors)*100/float64(b.N), "%error") + }, + ) + } + }, + ) + } +} diff --git a/server/certidp/certidp.go b/server/certidp/certidp.go new file mode 100644 index 0000000..a266185 --- /dev/null +++ b/server/certidp/certidp.go @@ -0,0 +1,312 @@ +// Copyright 2023-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certidp + +import ( + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/json" + "fmt" + "net/url" + "strings" + "time" + + "golang.org/x/crypto/ocsp" +) + +const ( + DefaultAllowedClockSkew = 30 * time.Second + DefaultOCSPResponderTimeout = 2 * time.Second + DefaultTTLUnsetNextUpdate = 1 * time.Hour +) + +type StatusAssertion int + +var ( + StatusAssertionStrToVal = map[string]StatusAssertion{ + "good": ocsp.Good, + "revoked": ocsp.Revoked, + "unknown": ocsp.Unknown, + } + StatusAssertionValToStr = map[StatusAssertion]string{ + ocsp.Good: "good", + ocsp.Revoked: "revoked", + ocsp.Unknown: "unknown", + } + StatusAssertionIntToVal = map[int]StatusAssertion{ + 0: ocsp.Good, + 1: ocsp.Revoked, + 2: ocsp.Unknown, + } +) + +// GetStatusAssertionStr returns the corresponding string representation of the StatusAssertion. +func GetStatusAssertionStr(sa int) string { + // If the provided status assertion value is not found in the map (StatusAssertionIntToVal), + // the function defaults to "unknown" to avoid defaulting to "good," which is the default iota value + // for the ocsp.StatusAssertion enumeration (https://pkg.go.dev/golang.org/x/crypto/ocsp#pkg-constants). + // This ensures that we don't unintentionally default to "good" when there's no map entry. + v, ok := StatusAssertionIntToVal[sa] + if !ok { + // set unknown as fallback + v = ocsp.Unknown + } + + return StatusAssertionValToStr[v] +} + +func (sa StatusAssertion) MarshalJSON() ([]byte, error) { + // This ensures that we don't unintentionally default to "good" when there's no map entry. + // (see more details in the GetStatusAssertionStr() comment) + str, ok := StatusAssertionValToStr[sa] + if !ok { + // set unknown as fallback + str = StatusAssertionValToStr[ocsp.Unknown] + } + return json.Marshal(str) +} + +func (sa *StatusAssertion) UnmarshalJSON(in []byte) error { + // This ensures that we don't unintentionally default to "good" when there's no map entry. + // (see more details in the GetStatusAssertionStr() comment) + v, ok := StatusAssertionStrToVal[strings.ReplaceAll(string(in), "\"", "")] + if !ok { + // set unknown as fallback + v = StatusAssertionStrToVal["unknown"] + } + *sa = v + return nil +} + +type ChainLink struct { + Leaf *x509.Certificate + Issuer *x509.Certificate + OCSPWebEndpoints *[]*url.URL +} + +// OCSPPeerConfig holds the parsed OCSP peer configuration section of TLS configuration +type OCSPPeerConfig struct { + Verify bool + Timeout float64 + ClockSkew float64 + WarnOnly bool + UnknownIsGood bool + AllowWhenCAUnreachable bool + TTLUnsetNextUpdate float64 +} + +func NewOCSPPeerConfig() *OCSPPeerConfig { + return &OCSPPeerConfig{ + Verify: false, + Timeout: DefaultOCSPResponderTimeout.Seconds(), + ClockSkew: DefaultAllowedClockSkew.Seconds(), + WarnOnly: false, + UnknownIsGood: false, + AllowWhenCAUnreachable: false, + TTLUnsetNextUpdate: DefaultTTLUnsetNextUpdate.Seconds(), + } +} + +// Log is a neutral method of passing server loggers to plugins +type Log struct { + Debugf func(format string, v ...any) + Noticef func(format string, v ...any) + Warnf func(format string, v ...any) + Errorf func(format string, v ...any) + Tracef func(format string, v ...any) +} + +type CertInfo struct { + Subject string `json:"subject,omitempty"` + Issuer string `json:"issuer,omitempty"` + Fingerprint string `json:"fingerprint,omitempty"` + Raw []byte `json:"raw,omitempty"` +} + +var OCSPPeerUsage = ` +For client, leaf spoke (remotes), and leaf hub connections, you may enable OCSP peer validation: + + tls { + ... + # mTLS must be enabled (with exception of Leaf remotes) + verify: true + ... + # short form enables peer verify and takes option defaults + ocsp_peer: true + + # long form includes settable options + ocsp_peer { + # Enable OCSP peer validation (default false) + verify: true + + # OCSP responder timeout in seconds (may be fractional, default 2 seconds) + ca_timeout: 2 + + # Allowed skew between server and OCSP responder time in seconds (may be fractional, default 30 seconds) + allowed_clockskew: 30 + + # Warn-only and never reject connections (default false) + warn_only: false + + # Treat response Unknown status as valid certificate (default false) + unknown_is_good: false + + # Warn-only if no CA response can be obtained and no cached revocation exists (default false) + allow_when_ca_unreachable: false + + # If response NextUpdate unset by CA, set a default cache TTL in seconds from ThisUpdate (default 1 hour) + cache_ttl_when_next_update_unset: 3600 + } + ... + } + +Note: OCSP validation for route and gateway connections is enabled using the 'ocsp' configuration option. +` + +// GenerateFingerprint returns a base64-encoded SHA256 hash of the raw certificate +func GenerateFingerprint(cert *x509.Certificate) string { + data := sha256.Sum256(cert.Raw) + return base64.StdEncoding.EncodeToString(data[:]) +} + +func getWebEndpoints(uris []string) []*url.URL { + var urls []*url.URL + for _, uri := range uris { + endpoint, err := url.ParseRequestURI(uri) + if err != nil { + // skip invalid URLs + continue + } + if endpoint.Scheme != "http" && endpoint.Scheme != "https" { + // skip non-web URLs + continue + } + urls = append(urls, endpoint) + } + return urls +} + +// GetSubjectDNForm returns RDN sequence concatenation of the certificate's subject to be +// used in logs, events, etc. Should never be used for reliable cache matching or other crypto purposes. +func GetSubjectDNForm(cert *x509.Certificate) string { + if cert == nil { + return "" + } + return strings.TrimSuffix(fmt.Sprintf("%s+", cert.Subject.ToRDNSequence()), "+") +} + +// GetIssuerDNForm returns RDN sequence concatenation of the certificate's issuer to be +// used in logs, events, etc. Should never be used for reliable cache matching or other crypto purposes. +func GetIssuerDNForm(cert *x509.Certificate) string { + if cert == nil { + return "" + } + return strings.TrimSuffix(fmt.Sprintf("%s+", cert.Issuer.ToRDNSequence()), "+") +} + +// CertOCSPEligible checks if the certificate's issuer has populated AIA with OCSP responder endpoint(s) +// and is thus eligible for OCSP validation +func CertOCSPEligible(link *ChainLink) bool { + if link == nil || link.Leaf.Raw == nil || len(link.Leaf.Raw) == 0 { + return false + } + if len(link.Leaf.OCSPServer) == 0 { + return false + } + urls := getWebEndpoints(link.Leaf.OCSPServer) + if len(urls) == 0 { + return false + } + link.OCSPWebEndpoints = &urls + return true +} + +// GetLeafIssuerCert returns the issuer certificate of the leaf (positional) certificate in the chain +func GetLeafIssuerCert(chain []*x509.Certificate, leafPos int) *x509.Certificate { + if len(chain) == 0 || leafPos < 0 { + return nil + } + // self-signed certificate or too-big leafPos + if leafPos >= len(chain)-1 { + return nil + } + // returns pointer to issuer cert or nil + return (chain)[leafPos+1] +} + +// OCSPResponseCurrent checks if the OCSP response is current (i.e. not expired and not future effective) +func OCSPResponseCurrent(ocspr *ocsp.Response, opts *OCSPPeerConfig, log *Log) bool { + skew := time.Duration(opts.ClockSkew * float64(time.Second)) + if skew < 0*time.Second { + skew = DefaultAllowedClockSkew + } + now := time.Now().UTC() + // Typical effectivity check based on CA response ThisUpdate and NextUpdate semantics + if !ocspr.NextUpdate.IsZero() && ocspr.NextUpdate.Before(now.Add(-1*skew)) { + t := ocspr.NextUpdate.Format(time.RFC3339Nano) + nt := now.Format(time.RFC3339Nano) + log.Debugf(DbgResponseExpired, t, nt, skew) + return false + } + // CA responder can assert NextUpdate unset, in which case use config option to set a default cache TTL + if ocspr.NextUpdate.IsZero() { + ttl := time.Duration(opts.TTLUnsetNextUpdate * float64(time.Second)) + if ttl < 0*time.Second { + ttl = DefaultTTLUnsetNextUpdate + } + expiryTime := ocspr.ThisUpdate.Add(ttl) + if expiryTime.Before(now.Add(-1 * skew)) { + t := expiryTime.Format(time.RFC3339Nano) + nt := now.Format(time.RFC3339Nano) + log.Debugf(DbgResponseTTLExpired, t, nt, skew) + return false + } + } + if ocspr.ThisUpdate.After(now.Add(skew)) { + t := ocspr.ThisUpdate.Format(time.RFC3339Nano) + nt := now.Format(time.RFC3339Nano) + log.Debugf(DbgResponseFutureDated, t, nt, skew) + return false + } + return true +} + +// ValidDelegationCheck checks if the CA OCSP Response was signed by a valid CA Issuer delegate as per (RFC 6960, section 4.2.2.2) +// If a valid delegate or direct-signed by CA Issuer, true returned. +func ValidDelegationCheck(iss *x509.Certificate, ocspr *ocsp.Response) bool { + // This call assumes prior successful parse and signature validation of the OCSP response + // The Go OCSP library (as of x/crypto/ocsp v0.9) will detect and perform a 1-level delegate signature check but does not + // implement the additional criteria for delegation specified in RFC 6960, section 4.2.2.2. + if iss == nil || ocspr == nil { + return false + } + // not a delegation, no-op + if ocspr.Certificate == nil { + return true + } + // delegate is self-same with CA Issuer, not a delegation although response issued in that form + if ocspr.Certificate.Equal(iss) { + return true + } + // we need to verify CA Issuer stamped id-kp-OCSPSigning on delegate + delegatedSigner := false + for _, keyUseExt := range ocspr.Certificate.ExtKeyUsage { + if keyUseExt == x509.ExtKeyUsageOCSPSigning { + delegatedSigner = true + break + } + } + return delegatedSigner +} diff --git a/server/certidp/certidp_test.go b/server/certidp/certidp_test.go new file mode 100644 index 0000000..ffc3105 --- /dev/null +++ b/server/certidp/certidp_test.go @@ -0,0 +1,56 @@ +// Copyright 2023 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certidp + +import "testing" + +// Checks the return values of the function GetStatusAssertionStr +func TestGetStatusAssertionStr(t *testing.T) { + tests := []struct { + name string + input int + expected string + }{ + { + name: "GoodStatus", + input: 0, + expected: "good", + }, + { + name: "RevokedStatus", + input: 1, + expected: "revoked", + }, + { + name: "UnknownStatus", + input: 2, + expected: "unknown", + }, + // Invalid status assertion value. + { + name: "InvalidStatus", + input: 42, + expected: "unknown", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := GetStatusAssertionStr(tt.input) + if got != tt.expected { + t.Errorf("Expected GetStatusAssertionStr: %v, got %v", tt.expected, got) + } + }) + } +} diff --git a/server/certidp/messages.go b/server/certidp/messages.go new file mode 100644 index 0000000..52a799a --- /dev/null +++ b/server/certidp/messages.go @@ -0,0 +1,106 @@ +// Copyright 2023 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certidp + +var ( + // Returned errors + ErrIllegalPeerOptsConfig = "expected map to define OCSP peer options, got [%T]" + ErrIllegalCacheOptsConfig = "expected map to define OCSP peer cache options, got [%T]" + ErrParsingPeerOptFieldGeneric = "error parsing tls peer config, unknown field [%q]" + ErrParsingPeerOptFieldTypeConversion = "error parsing tls peer config, conversion error: %s" + ErrParsingCacheOptFieldTypeConversion = "error parsing OCSP peer cache config, conversion error: %s" + ErrUnableToPlugTLSEmptyConfig = "unable to plug TLS verify connection, config is nil" + ErrMTLSRequired = "OCSP peer verification for client connections requires TLS verify (mTLS) to be enabled" + ErrUnableToPlugTLSClient = "unable to register client OCSP verification" + ErrUnableToPlugTLSServer = "unable to register server OCSP verification" + ErrCannotWriteCompressed = "error writing to compression writer: %w" + ErrCannotReadCompressed = "error reading compression reader: %w" + ErrTruncatedWrite = "short write on body (%d != %d)" + ErrCannotCloseWriter = "error closing compression writer: %w" + ErrParsingCacheOptFieldGeneric = "error parsing OCSP peer cache config, unknown field [%q]" + ErrUnknownCacheType = "error parsing OCSP peer cache config, unknown type [%s]" + ErrInvalidChainlink = "invalid chain link" + ErrBadResponderHTTPStatus = "bad OCSP responder http status: [%d]" + ErrNoAvailOCSPServers = "no available OCSP servers" + ErrFailedWithAllRequests = "exhausted OCSP responders: %w" + + // Direct logged errors + ErrLoadCacheFail = "Unable to load OCSP peer cache: %s" + ErrSaveCacheFail = "Unable to save OCSP peer cache: %s" + ErrBadCacheTypeConfig = "Unimplemented OCSP peer cache type [%v]" + ErrResponseCompressFail = "Unable to compress OCSP response for key [%s]: %s" + ErrResponseDecompressFail = "Unable to decompress OCSP response for key [%s]: %s" + ErrPeerEmptyNoEvent = "Peer certificate is nil, cannot send OCSP peer reject event" + ErrPeerEmptyAutoReject = "Peer certificate is nil, rejecting OCSP peer" + + // Debug information + DbgPlugTLSForKind = "Plugging TLS OCSP peer for [%s]" + DbgNumServerChains = "Peer OCSP enabled: %d TLS server chain(s) will be evaluated" + DbgNumClientChains = "Peer OCSP enabled: %d TLS client chain(s) will be evaluated" + DbgLinksInChain = "Chain [%d]: %d total link(s)" + DbgSelfSignedValid = "Chain [%d] is self-signed, thus peer is valid" + DbgValidNonOCSPChain = "Chain [%d] has no OCSP eligible links, thus peer is valid" + DbgChainIsOCSPEligible = "Chain [%d] has %d OCSP eligible link(s)" + DbgChainIsOCSPValid = "Chain [%d] is OCSP valid for all eligible links, thus peer is valid" + DbgNoOCSPValidChains = "No OCSP valid chains, thus peer is invalid" + DbgCheckingCacheForCert = "Checking OCSP peer cache for [%s], key [%s]" + DbgCurrentResponseCached = "Cached OCSP response is current, status [%s]" + DbgExpiredResponseCached = "Cached OCSP response is expired, status [%s]" + DbgOCSPValidPeerLink = "OCSP verify pass for [%s]" + DbgCachingResponse = "Caching OCSP response for [%s], key [%s]" + DbgAchievedCompression = "OCSP response compression ratio: [%f]" + DbgCacheHit = "OCSP peer cache hit for key [%s]" + DbgCacheMiss = "OCSP peer cache miss for key [%s]" + DbgPreservedRevocation = "Revoked OCSP response for key [%s] preserved by cache policy" + DbgDeletingCacheResponse = "Deleting OCSP peer cached response for key [%s]" + DbgStartingCache = "Starting OCSP peer cache" + DbgStoppingCache = "Stopping OCSP peer cache" + DbgLoadingCache = "Loading OCSP peer cache [%s]" + DbgNoCacheFound = "No OCSP peer cache found, starting with empty cache" + DbgSavingCache = "Saving OCSP peer cache [%s]" + DbgCacheSaved = "Saved OCSP peer cache successfully (%d bytes)" + DbgMakingCARequest = "Trying OCSP responder url [%s]" + DbgResponseExpired = "OCSP response NextUpdate [%s] is before now [%s] with clockskew [%s]" + DbgResponseTTLExpired = "OCSP response cache expiry [%s] is before now [%s] with clockskew [%s]" + DbgResponseFutureDated = "OCSP response ThisUpdate [%s] is before now [%s] with clockskew [%s]" + DbgCacheSaveTimerExpired = "OCSP peer cache save timer expired" + DbgCacheDirtySave = "OCSP peer cache is dirty, saving" + + // Returned to peer as TLS reject reason + MsgTLSClientRejectConnection = "client not OCSP valid" + MsgTLSServerRejectConnection = "server not OCSP valid" + + // Expected runtime errors (direct logged) + ErrCAResponderCalloutFail = "Attempt to obtain OCSP response from CA responder for [%s] failed: %s" + ErrNewCAResponseNotCurrent = "New OCSP CA response obtained for [%s] but not current" + ErrCAResponseParseFailed = "Could not parse OCSP CA response for [%s]: %s" + ErrOCSPInvalidPeerLink = "OCSP verify fail for [%s] with CA status [%s]" + + // Policy override warnings (direct logged) + MsgAllowWhenCAUnreachableOccurred = "Failed to obtain OCSP CA response for [%s] but AllowWhenCAUnreachable set; no cached revocation so allowing" + MsgAllowWhenCAUnreachableOccurredCachedRevoke = "Failed to obtain OCSP CA response for [%s] but AllowWhenCAUnreachable set; cached revocation exists so rejecting" + MsgAllowWarnOnlyOccurred = "OCSP verify fail for [%s] but WarnOnly is true so allowing" + + // Info (direct logged) + MsgCacheOnline = "OCSP peer cache online, type [%s]" + MsgCacheOffline = "OCSP peer cache offline, type [%s]" + + // OCSP cert invalid reasons (debug and event reasons) + MsgFailedOCSPResponseFetch = "Failed OCSP response fetch" + MsgOCSPResponseNotEffective = "OCSP response not in effectivity window" + MsgFailedOCSPResponseParse = "Failed OCSP response parse" + MsgOCSPResponseInvalidStatus = "Invalid OCSP response status: %s" + MsgOCSPResponseDelegationInvalid = "Invalid OCSP response delegation: %s" + MsgCachedOCSPResponseInvalid = "Invalid cached OCSP response for [%s] with fingerprint [%s]" +) diff --git a/server/certidp/ocsp_responder.go b/server/certidp/ocsp_responder.go new file mode 100644 index 0000000..93d4960 --- /dev/null +++ b/server/certidp/ocsp_responder.go @@ -0,0 +1,92 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certidp + +import ( + "encoding/base64" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/crypto/ocsp" +) + +func FetchOCSPResponse(link *ChainLink, opts *OCSPPeerConfig, log *Log) ([]byte, error) { + if link == nil || link.Leaf == nil || link.Issuer == nil || opts == nil || log == nil { + return nil, errors.New(ErrInvalidChainlink) + } + + timeout := time.Duration(opts.Timeout * float64(time.Second)) + if timeout <= 0*time.Second { + timeout = DefaultOCSPResponderTimeout + } + + getRequestBytes := func(u string, hc *http.Client) ([]byte, error) { + resp, err := hc.Get(u) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf(ErrBadResponderHTTPStatus, resp.StatusCode) + } + return io.ReadAll(resp.Body) + } + + // Request documentation: + // https://tools.ietf.org/html/rfc6960#appendix-A.1 + + reqDER, err := ocsp.CreateRequest(link.Leaf, link.Issuer, nil) + if err != nil { + return nil, err + } + + reqEnc := encodeOCSPRequest(reqDER) + + responders := *link.OCSPWebEndpoints + + if len(responders) == 0 { + return nil, errors.New(ErrNoAvailOCSPServers) + } + + var raw []byte + hc := &http.Client{ + Timeout: timeout, + } + for _, u := range responders { + responderURL := u.String() + log.Debugf(DbgMakingCARequest, responderURL) + responderURL = strings.TrimSuffix(responderURL, "/") + raw, err = getRequestBytes(fmt.Sprintf("%s/%s", responderURL, reqEnc), hc) + if err == nil { + break + } + } + if err != nil { + return nil, fmt.Errorf(ErrFailedWithAllRequests, err) + } + + return raw, nil +} + +// encodeOCSPRequest encodes the OCSP request in base64 and URL-encodes it. +// This is needed to fulfill the OCSP responder's requirements for the request format. (X.690) +func encodeOCSPRequest(reqDER []byte) string { + reqEnc := base64.StdEncoding.EncodeToString(reqDER) + return url.QueryEscape(reqEnc) +} diff --git a/server/certidp/ocsp_responder_test.go b/server/certidp/ocsp_responder_test.go new file mode 100644 index 0000000..826a915 --- /dev/null +++ b/server/certidp/ocsp_responder_test.go @@ -0,0 +1,29 @@ +package certidp + +import ( + "encoding/base64" + "net/url" + "strings" + "testing" +) + +func TestEncodeOCSPRequest(t *testing.T) { + data := []byte("test data for OCSP request") + encoded := encodeOCSPRequest(data) + + if strings.ContainsAny(encoded, "+/=") { + t.Errorf("URL contains unescaped characters: %s", encoded) + } + + decodedURL, err := url.QueryUnescape(encoded) + if err != nil { + t.Errorf("failed to url-decode request: %v", err) + } + decodedData, err := base64.StdEncoding.DecodeString(decodedURL) + if err != nil { + t.Errorf("failed to base64-decode request: %v", err) + } + if string(decodedData) != string(data) { + t.Errorf("Decoded data does not match original: got %s, want %s", decodedData, data) + } +} diff --git a/server/certstore/certstore.go b/server/certstore/certstore.go new file mode 100644 index 0000000..3c8114f --- /dev/null +++ b/server/certstore/certstore.go @@ -0,0 +1,104 @@ +// Copyright 2022-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certstore + +import ( + "crypto" + "crypto/x509" + "io" + "runtime" + "strings" +) + +type StoreType int + +const MATCHBYEMPTY = 0 +const STOREEMPTY = 0 + +const ( + windowsCurrentUser StoreType = iota + 1 + windowsLocalMachine +) + +var StoreMap = map[string]StoreType{ + "windowscurrentuser": windowsCurrentUser, + "windowslocalmachine": windowsLocalMachine, +} + +var StoreOSMap = map[StoreType]string{ + windowsCurrentUser: "windows", + windowsLocalMachine: "windows", +} + +type MatchByType int + +const ( + matchByIssuer MatchByType = iota + 1 + matchBySubject + matchByThumbprint +) + +var MatchByMap = map[string]MatchByType{ + "issuer": matchByIssuer, + "subject": matchBySubject, + "thumbprint": matchByThumbprint, +} + +var Usage = ` +In place of cert_file and key_file you may use the windows certificate store: + + tls { + cert_store: "WindowsCurrentUser" + cert_match_by: "Subject" + cert_match: "MyServer123" + } +` + +func ParseCertStore(certStore string) (StoreType, error) { + certStoreType, exists := StoreMap[strings.ToLower(certStore)] + if !exists { + return 0, ErrBadCertStore + } + validOS, exists := StoreOSMap[certStoreType] + if !exists || validOS != runtime.GOOS { + return 0, ErrOSNotCompatCertStore + } + return certStoreType, nil +} + +func ParseCertMatchBy(certMatchBy string) (MatchByType, error) { + certMatchByType, exists := MatchByMap[strings.ToLower(certMatchBy)] + if !exists { + return 0, ErrBadMatchByType + } + return certMatchByType, nil +} + +func GetLeafIssuer(leaf *x509.Certificate, vOpts x509.VerifyOptions) (issuer *x509.Certificate) { + chains, err := leaf.Verify(vOpts) + if err != nil || len(chains) == 0 { + issuer = nil + } else { + issuer = chains[0][1] + } + return +} + +// credential provides access to a public key and is a crypto.Signer. +type credential interface { + // Public returns the public key corresponding to the leaf certificate. + Public() crypto.PublicKey + // Sign signs digest with the private key. + Sign(rand io.Reader, digest []byte, opts crypto.SignerOpts) (signature []byte, err error) +} diff --git a/server/certstore/certstore_other.go b/server/certstore/certstore_other.go new file mode 100644 index 0000000..185efc6 --- /dev/null +++ b/server/certstore/certstore_other.go @@ -0,0 +1,45 @@ +// Copyright 2022-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !windows + +package certstore + +import ( + "crypto" + "crypto/tls" + "io" +) + +var _ = MATCHBYEMPTY + +// otherKey implements crypto.Signer and crypto.Decrypter to satisfy linter on platforms that don't implement certstore +type otherKey struct{} + +func TLSConfig(_ StoreType, _ MatchByType, _ string, _ []string, _ bool, _ *tls.Config) error { + return ErrOSNotCompatCertStore +} + +// Public always returns nil public key since this is a stub on non-supported platform +func (k otherKey) Public() crypto.PublicKey { + return nil +} + +// Sign always returns a nil signature since this is a stub on non-supported platform +func (k otherKey) Sign(rand io.Reader, digest []byte, opts crypto.SignerOpts) (signature []byte, err error) { + _, _, _ = rand, digest, opts + return nil, nil +} + +// Verify interface conformance. +var _ credential = &otherKey{} diff --git a/server/certstore/certstore_windows.go b/server/certstore/certstore_windows.go new file mode 100644 index 0000000..96eaea8 --- /dev/null +++ b/server/certstore/certstore_windows.go @@ -0,0 +1,965 @@ +// Copyright 2022-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Adapted, updated, and enhanced from CertToStore, https://github.com/google/certtostore/releases/tag/v1.0.2 +// Apache License, Version 2.0, Copyright 2017 Google Inc. + +package certstore + +import ( + "bytes" + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "encoding/binary" + "fmt" + "io" + "math/big" + "reflect" + "sync" + "syscall" + "unicode/utf16" + "unsafe" + + "golang.org/x/crypto/cryptobyte" + "golang.org/x/crypto/cryptobyte/asn1" + "golang.org/x/sys/windows" +) + +const ( + // wincrypt.h constants + winAcquireCached = windows.CRYPT_ACQUIRE_CACHE_FLAG + winAcquireSilent = windows.CRYPT_ACQUIRE_SILENT_FLAG + winAcquireOnlyNCryptKey = windows.CRYPT_ACQUIRE_ONLY_NCRYPT_KEY_FLAG + winEncodingX509ASN = windows.X509_ASN_ENCODING + winEncodingPKCS7 = windows.PKCS_7_ASN_ENCODING + winCertStoreProvSystem = windows.CERT_STORE_PROV_SYSTEM + winCertStoreCurrentUser = windows.CERT_SYSTEM_STORE_CURRENT_USER + winCertStoreLocalMachine = windows.CERT_SYSTEM_STORE_LOCAL_MACHINE + winCertStoreReadOnly = windows.CERT_STORE_READONLY_FLAG + winInfoIssuerFlag = windows.CERT_INFO_ISSUER_FLAG + winInfoSubjectFlag = windows.CERT_INFO_SUBJECT_FLAG + winCompareNameStrW = windows.CERT_COMPARE_NAME_STR_W + winCompareShift = windows.CERT_COMPARE_SHIFT + + // Reference https://learn.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-certfindcertificateinstore + winFindIssuerStr = windows.CERT_FIND_ISSUER_STR_W + winFindSubjectStr = windows.CERT_FIND_SUBJECT_STR_W + winFindHashStr = windows.CERT_FIND_HASH_STR + + winNcryptKeySpec = windows.CERT_NCRYPT_KEY_SPEC + + winBCryptPadPKCS1 uintptr = 0x2 + winBCryptPadPSS uintptr = 0x8 // Modern TLS 1.2+ + winBCryptPadPSSSalt uint32 = 32 // default 20, 32 optimal for typical SHA256 hash + + winRSA1Magic = 0x31415352 // "RSA1" BCRYPT_RSAPUBLIC_MAGIC + + winECS1Magic = 0x31534345 // "ECS1" BCRYPT_ECDSA_PUBLIC_P256_MAGIC + winECS3Magic = 0x33534345 // "ECS3" BCRYPT_ECDSA_PUBLIC_P384_MAGIC + winECS5Magic = 0x35534345 // "ECS5" BCRYPT_ECDSA_PUBLIC_P521_MAGIC + + winECK1Magic = 0x314B4345 // "ECK1" BCRYPT_ECDH_PUBLIC_P256_MAGIC + winECK3Magic = 0x334B4345 // "ECK3" BCRYPT_ECDH_PUBLIC_P384_MAGIC + winECK5Magic = 0x354B4345 // "ECK5" BCRYPT_ECDH_PUBLIC_P521_MAGIC + + winCryptENotFound = windows.CRYPT_E_NOT_FOUND + + providerMSSoftware = "Microsoft Software Key Storage Provider" +) + +var ( + winBCryptRSAPublicBlob = winWide("RSAPUBLICBLOB") + winBCryptECCPublicBlob = winWide("ECCPUBLICBLOB") + + winNCryptAlgorithmGroupProperty = winWide("Algorithm Group") // NCRYPT_ALGORITHM_GROUP_PROPERTY + winNCryptUniqueNameProperty = winWide("Unique Name") // NCRYPT_UNIQUE_NAME_PROPERTY + winNCryptECCCurveNameProperty = winWide("ECCCurveName") // NCRYPT_ECC_CURVE_NAME_PROPERTY + + winCurveIDs = map[uint32]elliptic.Curve{ + winECS1Magic: elliptic.P256(), // BCRYPT_ECDSA_PUBLIC_P256_MAGIC + winECS3Magic: elliptic.P384(), // BCRYPT_ECDSA_PUBLIC_P384_MAGIC + winECS5Magic: elliptic.P521(), // BCRYPT_ECDSA_PUBLIC_P521_MAGIC + winECK1Magic: elliptic.P256(), // BCRYPT_ECDH_PUBLIC_P256_MAGIC + winECK3Magic: elliptic.P384(), // BCRYPT_ECDH_PUBLIC_P384_MAGIC + winECK5Magic: elliptic.P521(), // BCRYPT_ECDH_PUBLIC_P521_MAGIC + } + + winCurveNames = map[string]elliptic.Curve{ + "nistP256": elliptic.P256(), // BCRYPT_ECC_CURVE_NISTP256 + "nistP384": elliptic.P384(), // BCRYPT_ECC_CURVE_NISTP384 + "nistP521": elliptic.P521(), // BCRYPT_ECC_CURVE_NISTP521 + } + + winAlgIDs = map[crypto.Hash]*uint16{ + crypto.SHA1: winWide("SHA1"), // BCRYPT_SHA1_ALGORITHM + crypto.SHA256: winWide("SHA256"), // BCRYPT_SHA256_ALGORITHM + crypto.SHA384: winWide("SHA384"), // BCRYPT_SHA384_ALGORITHM + crypto.SHA512: winWide("SHA512"), // BCRYPT_SHA512_ALGORITHM + } + + // MY is well-known system store on Windows that holds personal certificates. Read + // More about the CA locations here: + // https://learn.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/wcf/certificate-of-clientcertificate-element?redirectedfrom=MSDN + // https://superuser.com/questions/217719/what-are-the-windows-system-certificate-stores + // https://docs.microsoft.com/en-us/windows/win32/seccrypto/certificate-stores + // https://learn.microsoft.com/en-us/windows/win32/seccrypto/system-store-locations + // https://stackoverflow.com/questions/63286085/which-x509-storename-refers-to-the-certificates-stored-beneath-trusted-root-cert#:~:text=4-,StoreName.,is%20%22Intermediate%20Certification%20Authorities%22. + winMyStore = winWide("MY") + winIntermediateCAStore = winWide("CA") + winRootStore = winWide("Root") + winAuthRootStore = winWide("AuthRoot") + + // These DLLs must be available on all Windows hosts + winCrypt32 = windows.NewLazySystemDLL("crypt32.dll") + winNCrypt = windows.NewLazySystemDLL("ncrypt.dll") + + winCertFindCertificateInStore = winCrypt32.NewProc("CertFindCertificateInStore") + winCertVerifyTimeValidity = winCrypt32.NewProc("CertVerifyTimeValidity") + winCryptAcquireCertificatePrivateKey = winCrypt32.NewProc("CryptAcquireCertificatePrivateKey") + winNCryptExportKey = winNCrypt.NewProc("NCryptExportKey") + winNCryptOpenStorageProvider = winNCrypt.NewProc("NCryptOpenStorageProvider") + winNCryptGetProperty = winNCrypt.NewProc("NCryptGetProperty") + winNCryptSignHash = winNCrypt.NewProc("NCryptSignHash") + + winFnGetProperty = winGetProperty +) + +func init() { + for _, d := range []*windows.LazyDLL{ + winCrypt32, winNCrypt, + } { + if err := d.Load(); err != nil { + panic(err) + } + } + for _, p := range []*windows.LazyProc{ + winCertFindCertificateInStore, winCryptAcquireCertificatePrivateKey, + winNCryptExportKey, winNCryptOpenStorageProvider, + winNCryptGetProperty, winNCryptSignHash, + } { + if err := p.Find(); err != nil { + panic(err) + } + } +} + +type winPKCS1PaddingInfo struct { + pszAlgID *uint16 +} + +type winPSSPaddingInfo struct { + pszAlgID *uint16 + cbSalt uint32 +} + +// createCACertsPool generates a CertPool from the Windows certificate store, +// adding all matching certificates from the caCertsMatch array to the pool. +// All matching certificates (vs first) are added to the pool based on a user +// request. If no certificates are found an error is returned. +func createCACertsPool(cs *winCertStore, storeType uint32, caCertsMatch []string, skipInvalid bool) (*x509.CertPool, error) { + var errs []error + caPool := x509.NewCertPool() + for _, s := range caCertsMatch { + lfs, err := cs.caCertsBySubjectMatch(s, storeType, skipInvalid) + if err != nil { + errs = append(errs, err) + } else { + for _, lf := range lfs { + caPool.AddCert(lf) + } + } + } + // If every lookup failed return the errors. + if len(errs) == len(caCertsMatch) { + return nil, fmt.Errorf("unable to match any CA certificate: %v", errs) + } + return caPool, nil +} + +// TLSConfig fulfills the same function as reading cert and key pair from +// pem files but sources the Windows certificate store instead. The +// certMatchBy and certMatch fields search the "MY" certificate location +// for the first certificate that matches the certMatch field. The +// caCertsMatch field is used to search the Trusted Root, Third Party Root, +// and Intermediate Certificate Authority locations for certificates with +// Subjects matching the provided strings. If a match is found, the +// certificate is added to the pool that is used to verify the certificate +// chain. +func TLSConfig(certStore StoreType, certMatchBy MatchByType, certMatch string, caCertsMatch []string, skipInvalid bool, config *tls.Config) error { + var ( + leaf *x509.Certificate + leafCtx *windows.CertContext + pk *winKey + vOpts = x509.VerifyOptions{} + chains [][]*x509.Certificate + chain []*x509.Certificate + rawChain [][]byte + ) + + // By StoreType, open a store + if certStore == windowsCurrentUser || certStore == windowsLocalMachine { + var scope uint32 + cs, err := winOpenCertStore(providerMSSoftware) + if err != nil || cs == nil { + return err + } + if certStore == windowsCurrentUser { + scope = winCertStoreCurrentUser + } + if certStore == windowsLocalMachine { + scope = winCertStoreLocalMachine + } + + // certByIssuer or certBySubject + if certMatchBy == matchBySubject || certMatchBy == MATCHBYEMPTY { + leaf, leafCtx, err = cs.certBySubject(certMatch, scope, skipInvalid) + } else if certMatchBy == matchByIssuer { + leaf, leafCtx, err = cs.certByIssuer(certMatch, scope, skipInvalid) + } else if certMatchBy == matchByThumbprint { + leaf, leafCtx, err = cs.certByThumbprint(certMatch, scope, skipInvalid) + } else { + return ErrBadMatchByType + } + if err != nil { + // pass through error from cert search + return err + } + if leaf == nil || leafCtx == nil { + return ErrFailedCertSearch + } + pk, err = cs.certKey(leafCtx) + if err != nil { + return err + } + if pk == nil { + return ErrNoPrivateKeyStoreRef + } + // Look for CA Certificates + if len(caCertsMatch) != 0 { + caPool, err := createCACertsPool(cs, scope, caCertsMatch, skipInvalid) + if err != nil { + return err + } + config.ClientCAs = caPool + } + } else { + return ErrBadCertStore + } + + // Get intermediates in the cert store for the found leaf IFF there is a full chain of trust in the store + // otherwise just use leaf as the final chain. + // + // Using std lib Verify as a reliable way to get valid chains out of the win store for the leaf; however, + // using empty options since server TLS stanza could be TLS role as server identity or client identity. + chains, err := leaf.Verify(vOpts) + if err != nil || len(chains) == 0 { + chains = append(chains, []*x509.Certificate{leaf}) + } + + // We have at least one verified chain so pop the first chain and remove the self-signed CA cert (if present) + // from the end of the chain + chain = chains[0] + if len(chain) > 1 { + chain = chain[:len(chain)-1] + } + + // For tls.Certificate.Certificate need a [][]byte from []*x509.Certificate + // Approximate capacity for efficiency + rawChain = make([][]byte, 0, len(chain)) + for _, link := range chain { + rawChain = append(rawChain, link.Raw) + } + + tlsCert := tls.Certificate{ + Certificate: rawChain, + PrivateKey: pk, + Leaf: leaf, + } + config.Certificates = []tls.Certificate{tlsCert} + + // note: pk is a windows pointer (not freed by Go) but needs to live the life of the server for Signing. + // The cert context (leafCtx) windows pointer must not be freed underneath the pk so also life of the server. + return nil +} + +// winWide returns a pointer to uint16 representing the equivalent +// to a Windows LPCWSTR. +func winWide(s string) *uint16 { + w := utf16.Encode([]rune(s)) + w = append(w, 0) + return &w[0] +} + +// winOpenProvider gets a provider handle for subsequent calls +func winOpenProvider(provider string) (uintptr, error) { + var hProv uintptr + pname := winWide(provider) + // Open the provider, the last parameter is not used + r, _, err := winNCryptOpenStorageProvider.Call(uintptr(unsafe.Pointer(&hProv)), uintptr(unsafe.Pointer(pname)), 0) + if r == 0 { + return hProv, nil + } + return hProv, fmt.Errorf("NCryptOpenStorageProvider returned %X: %v", r, err) +} + +// winFindCert wraps the CertFindCertificateInStore library call. Note that any cert context passed +// into prev will be freed. If no certificate was found, nil will be returned. +func winFindCert(store windows.Handle, enc, findFlags, findType uint32, para *uint16, prev *windows.CertContext) (*windows.CertContext, error) { + h, _, err := winCertFindCertificateInStore.Call( + uintptr(store), + uintptr(enc), + uintptr(findFlags), + uintptr(findType), + uintptr(unsafe.Pointer(para)), + uintptr(unsafe.Pointer(prev)), + ) + if h == 0 { + // Actual error, or simply not found? + if errno, ok := err.(syscall.Errno); ok && errno == syscall.Errno(winCryptENotFound) { + return nil, ErrFailedCertSearch + } + return nil, ErrFailedCertSearch + } + // nolint:govet + return (*windows.CertContext)(unsafe.Pointer(h)), nil +} + +// winVerifyCertValid wraps the CertVerifyTimeValidity and simply returns true if the certificate is valid +func winVerifyCertValid(timeToVerify *windows.Filetime, certInfo *windows.CertInfo) bool { + // this function does not document returning errors / setting lasterror + r, _, _ := winCertVerifyTimeValidity.Call( + uintptr(unsafe.Pointer(timeToVerify)), + uintptr(unsafe.Pointer(certInfo)), + ) + return r == 0 +} + +// winCertStore is a store implementation for the Windows Certificate Store +type winCertStore struct { + Prov uintptr + ProvName string + stores map[string]*winStoreHandle + mu sync.Mutex +} + +// winOpenCertStore creates a winCertStore +func winOpenCertStore(provider string) (*winCertStore, error) { + cngProv, err := winOpenProvider(provider) + if err != nil { + // pass through error from winOpenProvider + return nil, err + } + + wcs := &winCertStore{ + Prov: cngProv, + ProvName: provider, + stores: make(map[string]*winStoreHandle), + } + + return wcs, nil +} + +// winCertContextToX509 creates an x509.Certificate from a Windows cert context. +func winCertContextToX509(ctx *windows.CertContext) (*x509.Certificate, error) { + var der []byte + slice := (*reflect.SliceHeader)(unsafe.Pointer(&der)) + slice.Data = uintptr(unsafe.Pointer(ctx.EncodedCert)) + slice.Len = int(ctx.Length) + slice.Cap = int(ctx.Length) + return x509.ParseCertificate(der) +} + +// certByIssuer matches and returns the first certificate found by passed issuer. +// CertContext pointer returned allows subsequent key operations like Sign. Caller specifies +// current user's personal certs or local machine's personal certs using storeType. +// See CERT_FIND_ISSUER_STR description at https://learn.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-certfindcertificateinstore +func (w *winCertStore) certByIssuer(issuer string, storeType uint32, skipInvalid bool) (*x509.Certificate, *windows.CertContext, error) { + return w.certSearch(winFindIssuerStr, issuer, winMyStore, storeType, skipInvalid) +} + +// certBySubject matches and returns the first certificate found by passed subject field. +// CertContext pointer returned allows subsequent key operations like Sign. Caller specifies +// current user's personal certs or local machine's personal certs using storeType. +// See CERT_FIND_SUBJECT_STR description at https://learn.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-certfindcertificateinstore +func (w *winCertStore) certBySubject(subject string, storeType uint32, skipInvalid bool) (*x509.Certificate, *windows.CertContext, error) { + return w.certSearch(winFindSubjectStr, subject, winMyStore, storeType, skipInvalid) +} + +// certByThumbprint matches and returns the first certificate found by passed SHA1 thumbprint. +// CertContext pointer returned allows subsequent key operations like Sign. Caller specifies +// current user's personal certs or local machine's personal certs using storeType. +// See CERT_FIND_SUBJECT_STR description at https://learn.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-certfindcertificateinstore +func (w *winCertStore) certByThumbprint(hash string, storeType uint32, skipInvalid bool) (*x509.Certificate, *windows.CertContext, error) { + return w.certSearch(winFindHashStr, hash, winMyStore, storeType, skipInvalid) +} + +// caCertsBySubjectMatch matches and returns all matching certificates of the subject field. +// +// The following locations are searched: +// 1) Root (Trusted Root Certification Authorities) +// 2) AuthRoot (Third-Party Root Certification Authorities) +// 3) CA (Intermediate Certification Authorities) +// +// Caller specifies current user's personal certs or local machine's personal certs using storeType. +// See CERT_FIND_SUBJECT_STR description at https://learn.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-certfindcertificateinstore +func (w *winCertStore) caCertsBySubjectMatch(subject string, storeType uint32, skipInvalid bool) ([]*x509.Certificate, error) { + var ( + leaf *x509.Certificate + searchLocations = [3]*uint16{winRootStore, winAuthRootStore, winIntermediateCAStore} + rv []*x509.Certificate + ) + // surprisingly, an empty string returns a result. We'll treat this as an error. + if subject == "" { + return nil, ErrBadCaCertMatchField + } + for _, sr := range searchLocations { + var err error + if leaf, _, err = w.certSearch(winFindSubjectStr, subject, sr, storeType, skipInvalid); err == nil { + rv = append(rv, leaf) + } else { + // Ignore the failed search from a single location. Errors we catch include + // ErrFailedX509Extract (resulting from a malformed certificate) and errors + // around invalid attributes, unsupported algorithms, etc. These are corner + // cases as certificates with these errors shouldn't have been allowed + // to be added to the store in the first place. + if err != ErrFailedCertSearch { + return nil, err + } + } + } + // Not found anywhere + if len(rv) == 0 { + return nil, ErrFailedCertSearch + } + return rv, nil +} + +// certSearch is a helper function to lookup certificates based on search type and match value. +// store is used to specify which store to perform the lookup in (system or user). +func (w *winCertStore) certSearch(searchType uint32, matchValue string, searchRoot *uint16, store uint32, skipInvalid bool) (*x509.Certificate, *windows.CertContext, error) { + // store handle to "MY" store + h, err := w.storeHandle(store, searchRoot) + if err != nil { + return nil, nil, err + } + + var prev *windows.CertContext + var cert *x509.Certificate + + i, err := windows.UTF16PtrFromString(matchValue) + if err != nil { + return nil, nil, ErrFailedCertSearch + } + + // pass 0 as the third parameter because it is not used + // https://msdn.microsoft.com/en-us/library/windows/desktop/aa376064(v=vs.85).aspx + + for { + nc, err := winFindCert(h, winEncodingX509ASN|winEncodingPKCS7, 0, searchType, i, prev) + if err != nil { + return nil, nil, err + } + if nc != nil { + // certificate found + prev = nc + + var now *windows.Filetime + if skipInvalid && !winVerifyCertValid(now, nc.CertInfo) { + continue + } + + // Extract the DER-encoded certificate from the cert context + xc, err := winCertContextToX509(nc) + if err == nil { + cert = xc + break + } else { + return nil, nil, ErrFailedX509Extract + } + } else { + return nil, nil, ErrFailedCertSearch + } + } + + if cert == nil { + return nil, nil, ErrFailedX509Extract + } + + return cert, prev, nil +} + +type winStoreHandle struct { + handle *windows.Handle +} + +func winNewStoreHandle(provider uint32, store *uint16) (*winStoreHandle, error) { + var s winStoreHandle + if s.handle != nil { + return &s, nil + } + st, err := windows.CertOpenStore( + winCertStoreProvSystem, + 0, + 0, + provider|winCertStoreReadOnly, + uintptr(unsafe.Pointer(store))) + if err != nil { + return nil, ErrBadCryptoStoreProvider + } + s.handle = &st + return &s, nil +} + +// winKey implements crypto.Signer and crypto.Decrypter for key based operations. +type winKey struct { + handle uintptr + pub crypto.PublicKey + Container string + AlgorithmGroup string +} + +// Public exports a public key to implement crypto.Signer +func (k winKey) Public() crypto.PublicKey { + return k.pub +} + +// Sign returns the signature of a hash to implement crypto.Signer +func (k winKey) Sign(_ io.Reader, digest []byte, opts crypto.SignerOpts) ([]byte, error) { + switch k.AlgorithmGroup { + case "ECDSA", "ECDH": + return winSignECDSA(k.handle, digest) + case "RSA": + hf := opts.HashFunc() + algID, ok := winAlgIDs[hf] + if !ok { + return nil, ErrBadRSAHashAlgorithm + } + switch opts.(type) { + case *rsa.PSSOptions: + return winSignRSAPSSPadding(k.handle, digest, algID) + default: + return winSignRSAPKCS1Padding(k.handle, digest, algID) + } + default: + return nil, ErrBadSigningAlgorithm + } +} + +func winSignECDSA(kh uintptr, digest []byte) ([]byte, error) { + var size uint32 + // Obtain the size of the signature + r, _, _ := winNCryptSignHash.Call( + kh, + 0, + uintptr(unsafe.Pointer(&digest[0])), + uintptr(len(digest)), + 0, + 0, + uintptr(unsafe.Pointer(&size)), + 0) + if r != 0 { + return nil, ErrStoreECDSASigningError + } + + // Obtain the signature data + buf := make([]byte, size) + r, _, _ = winNCryptSignHash.Call( + kh, + 0, + uintptr(unsafe.Pointer(&digest[0])), + uintptr(len(digest)), + uintptr(unsafe.Pointer(&buf[0])), + uintptr(size), + uintptr(unsafe.Pointer(&size)), + 0) + if r != 0 { + return nil, ErrStoreECDSASigningError + } + if len(buf) != int(size) { + return nil, ErrStoreECDSASigningError + } + + return winPackECDSASigValue(bytes.NewReader(buf[:size]), int(size/2)) +} + +func winPackECDSASigValue(r io.Reader, digestLength int) ([]byte, error) { + sigR := make([]byte, digestLength) + if _, err := io.ReadFull(r, sigR); err != nil { + return nil, ErrStoreECDSASigningError + } + + sigS := make([]byte, digestLength) + if _, err := io.ReadFull(r, sigS); err != nil { + return nil, ErrStoreECDSASigningError + } + + var b cryptobyte.Builder + b.AddASN1(asn1.SEQUENCE, func(b *cryptobyte.Builder) { + b.AddASN1BigInt(new(big.Int).SetBytes(sigR)) + b.AddASN1BigInt(new(big.Int).SetBytes(sigS)) + }) + return b.Bytes() +} + +func winSignRSAPKCS1Padding(kh uintptr, digest []byte, algID *uint16) ([]byte, error) { + // PKCS#1 v1.5 padding for some TLS 1.2 + padInfo := winPKCS1PaddingInfo{pszAlgID: algID} + var size uint32 + // Obtain the size of the signature + r, _, _ := winNCryptSignHash.Call( + kh, + uintptr(unsafe.Pointer(&padInfo)), + uintptr(unsafe.Pointer(&digest[0])), + uintptr(len(digest)), + 0, + 0, + uintptr(unsafe.Pointer(&size)), + winBCryptPadPKCS1) + if r != 0 { + return nil, ErrStoreRSASigningError + } + + // Obtain the signature data + sig := make([]byte, size) + r, _, _ = winNCryptSignHash.Call( + kh, + uintptr(unsafe.Pointer(&padInfo)), + uintptr(unsafe.Pointer(&digest[0])), + uintptr(len(digest)), + uintptr(unsafe.Pointer(&sig[0])), + uintptr(size), + uintptr(unsafe.Pointer(&size)), + winBCryptPadPKCS1) + if r != 0 { + return nil, ErrStoreRSASigningError + } + + return sig[:size], nil +} + +func winSignRSAPSSPadding(kh uintptr, digest []byte, algID *uint16) ([]byte, error) { + // PSS padding for TLS 1.3 and some TLS 1.2 + padInfo := winPSSPaddingInfo{pszAlgID: algID, cbSalt: winBCryptPadPSSSalt} + + var size uint32 + // Obtain the size of the signature + r, _, _ := winNCryptSignHash.Call( + kh, + uintptr(unsafe.Pointer(&padInfo)), + uintptr(unsafe.Pointer(&digest[0])), + uintptr(len(digest)), + 0, + 0, + uintptr(unsafe.Pointer(&size)), + winBCryptPadPSS) + if r != 0 { + return nil, ErrStoreRSASigningError + } + + // Obtain the signature data + sig := make([]byte, size) + r, _, _ = winNCryptSignHash.Call( + kh, + uintptr(unsafe.Pointer(&padInfo)), + uintptr(unsafe.Pointer(&digest[0])), + uintptr(len(digest)), + uintptr(unsafe.Pointer(&sig[0])), + uintptr(size), + uintptr(unsafe.Pointer(&size)), + winBCryptPadPSS) + if r != 0 { + return nil, ErrStoreRSASigningError + } + + return sig[:size], nil +} + +// certKey wraps CryptAcquireCertificatePrivateKey. It obtains the CNG private +// key of a known certificate and returns a pointer to a winKey which implements +// both crypto.Signer. When a nil cert context is passed +// a nil key is intentionally returned, to model the expected behavior of a +// non-existent cert having no private key. +// https://docs.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-cryptacquirecertificateprivatekey +func (w *winCertStore) certKey(cert *windows.CertContext) (*winKey, error) { + // Return early if a nil cert was passed. + if cert == nil { + return nil, nil + } + var ( + kh uintptr + spec uint32 + mustFree int + ) + r, _, _ := winCryptAcquireCertificatePrivateKey.Call( + uintptr(unsafe.Pointer(cert)), + winAcquireCached|winAcquireSilent|winAcquireOnlyNCryptKey, + 0, // Reserved, must be null. + uintptr(unsafe.Pointer(&kh)), + uintptr(unsafe.Pointer(&spec)), + uintptr(unsafe.Pointer(&mustFree)), + ) + // If the function succeeds, the return value is nonzero (TRUE). + if r == 0 { + return nil, ErrNoPrivateKeyStoreRef + } + if mustFree != 0 { + return nil, ErrNoPrivateKeyStoreRef + } + if spec != winNcryptKeySpec { + return nil, ErrNoPrivateKeyStoreRef + } + + return winKeyMetadata(kh) +} + +func winKeyMetadata(kh uintptr) (*winKey, error) { + // uc is used to populate the unique container name attribute of the private key + uc, err := winGetPropertyStr(kh, winNCryptUniqueNameProperty) + if err != nil { + // unable to determine key unique name + return nil, ErrExtractingPrivateKeyMetadata + } + + alg, err := winGetPropertyStr(kh, winNCryptAlgorithmGroupProperty) + if err != nil { + // unable to determine key algorithm + return nil, ErrExtractingPrivateKeyMetadata + } + + var pub crypto.PublicKey + + switch alg { + case "ECDSA", "ECDH": + buf, err := winExport(kh, winBCryptECCPublicBlob) + if err != nil { + // failed to export ECC public key + return nil, ErrExtractingECCPublicKey + } + pub, err = unmarshalECC(buf, kh) + if err != nil { + return nil, ErrExtractingECCPublicKey + } + case "RSA": + buf, err := winExport(kh, winBCryptRSAPublicBlob) + if err != nil { + return nil, ErrExtractingRSAPublicKey + } + pub, err = winUnmarshalRSA(buf) + if err != nil { + return nil, ErrExtractingRSAPublicKey + } + default: + return nil, ErrBadPublicKeyAlgorithm + } + + return &winKey{handle: kh, pub: pub, Container: uc, AlgorithmGroup: alg}, nil +} + +func winGetProperty(kh uintptr, property *uint16) ([]byte, error) { + var strSize uint32 + r, _, _ := winNCryptGetProperty.Call( + kh, + uintptr(unsafe.Pointer(property)), + 0, + 0, + uintptr(unsafe.Pointer(&strSize)), + 0, + 0) + if r != 0 { + return nil, ErrExtractPropertyFromKey + } + + buf := make([]byte, strSize) + r, _, _ = winNCryptGetProperty.Call( + kh, + uintptr(unsafe.Pointer(property)), + uintptr(unsafe.Pointer(&buf[0])), + uintptr(strSize), + uintptr(unsafe.Pointer(&strSize)), + 0, + 0) + if r != 0 { + return nil, ErrExtractPropertyFromKey + } + + return buf, nil +} + +func winGetPropertyStr(kh uintptr, property *uint16) (string, error) { + buf, err := winFnGetProperty(kh, property) + if err != nil { + return "", ErrExtractPropertyFromKey + } + uc := bytes.ReplaceAll(buf, []byte{0x00}, []byte("")) + return string(uc), nil +} + +func winExport(kh uintptr, blobType *uint16) ([]byte, error) { + var size uint32 + // When obtaining the size of a public key, most parameters are not required + r, _, _ := winNCryptExportKey.Call( + kh, + 0, + uintptr(unsafe.Pointer(blobType)), + 0, + 0, + 0, + uintptr(unsafe.Pointer(&size)), + 0) + if r != 0 { + return nil, ErrExtractingPublicKey + } + + // Place the exported key in buf now that we know the size required + buf := make([]byte, size) + r, _, _ = winNCryptExportKey.Call( + kh, + 0, + uintptr(unsafe.Pointer(blobType)), + 0, + uintptr(unsafe.Pointer(&buf[0])), + uintptr(size), + uintptr(unsafe.Pointer(&size)), + 0) + if r != 0 { + return nil, ErrExtractingPublicKey + } + return buf, nil +} + +func unmarshalECC(buf []byte, kh uintptr) (*ecdsa.PublicKey, error) { + // BCRYPT_ECCKEY_BLOB from bcrypt.h + header := struct { + Magic uint32 + Key uint32 + }{} + + r := bytes.NewReader(buf) + if err := binary.Read(r, binary.LittleEndian, &header); err != nil { + return nil, ErrExtractingECCPublicKey + } + + curve, ok := winCurveIDs[header.Magic] + if !ok { + // Fix for b/185945636, where despite specifying the curve, nCrypt returns + // an incorrect response with BCRYPT_ECDSA_PUBLIC_GENERIC_MAGIC. + var err error + curve, err = winCurveName(kh) + if err != nil { + // unsupported header magic or cannot match the curve by name + return nil, err + } + } + + keyX := make([]byte, header.Key) + if n, err := r.Read(keyX); n != int(header.Key) || err != nil { + // failed to read key X + return nil, ErrExtractingECCPublicKey + } + + keyY := make([]byte, header.Key) + if n, err := r.Read(keyY); n != int(header.Key) || err != nil { + // failed to read key Y + return nil, ErrExtractingECCPublicKey + } + + pub := &ecdsa.PublicKey{ + Curve: curve, + X: new(big.Int).SetBytes(keyX), + Y: new(big.Int).SetBytes(keyY), + } + return pub, nil +} + +// winCurveName reads the curve name property and returns the corresponding curve. +func winCurveName(kh uintptr) (elliptic.Curve, error) { + cn, err := winGetPropertyStr(kh, winNCryptECCCurveNameProperty) + if err != nil { + // unable to determine the curve property name + return nil, ErrExtractPropertyFromKey + } + curve, ok := winCurveNames[cn] + if !ok { + // unknown curve name + return nil, ErrBadECCCurveName + } + return curve, nil +} + +func winUnmarshalRSA(buf []byte) (*rsa.PublicKey, error) { + // BCRYPT_RSA_BLOB from bcrypt.h + header := struct { + Magic uint32 + BitLength uint32 + PublicExpSize uint32 + ModulusSize uint32 + UnusedPrime1 uint32 + UnusedPrime2 uint32 + }{} + + r := bytes.NewReader(buf) + if err := binary.Read(r, binary.LittleEndian, &header); err != nil { + return nil, ErrExtractingRSAPublicKey + } + + if header.Magic != winRSA1Magic { + // invalid header magic + return nil, ErrExtractingRSAPublicKey + } + + if header.PublicExpSize > 8 { + // unsupported public exponent size + return nil, ErrExtractingRSAPublicKey + } + + exp := make([]byte, 8) + if n, err := r.Read(exp[8-header.PublicExpSize:]); n != int(header.PublicExpSize) || err != nil { + // failed to read public exponent + return nil, ErrExtractingRSAPublicKey + } + + mod := make([]byte, header.ModulusSize) + if n, err := r.Read(mod); n != int(header.ModulusSize) || err != nil { + // failed to read modulus + return nil, ErrExtractingRSAPublicKey + } + + pub := &rsa.PublicKey{ + N: new(big.Int).SetBytes(mod), + E: int(binary.BigEndian.Uint64(exp)), + } + return pub, nil +} + +// storeHandle returns a handle to a given cert store, opening the handle as needed. +func (w *winCertStore) storeHandle(provider uint32, store *uint16) (windows.Handle, error) { + w.mu.Lock() + defer w.mu.Unlock() + + key := fmt.Sprintf("%d%s", provider, windows.UTF16PtrToString(store)) + var err error + if w.stores[key] == nil { + w.stores[key], err = winNewStoreHandle(provider, store) + if err != nil { + return 0, ErrBadCryptoStoreProvider + } + } + return *w.stores[key].handle, nil +} + +// Verify interface conformance. +var _ credential = &winKey{} diff --git a/server/certstore/errors.go b/server/certstore/errors.go new file mode 100644 index 0000000..be545cf --- /dev/null +++ b/server/certstore/errors.go @@ -0,0 +1,79 @@ +package certstore + +import ( + "errors" +) + +var ( + // ErrBadCryptoStoreProvider represents inablity to establish link with a certificate store + ErrBadCryptoStoreProvider = errors.New("unable to open certificate store or store not available") + + // ErrBadRSAHashAlgorithm represents a bad or unsupported RSA hash algorithm + ErrBadRSAHashAlgorithm = errors.New("unsupported RSA hash algorithm") + + // ErrBadSigningAlgorithm represents a bad or unsupported signing algorithm + ErrBadSigningAlgorithm = errors.New("unsupported signing algorithm") + + // ErrStoreRSASigningError represents an error returned from store during RSA signature + ErrStoreRSASigningError = errors.New("unable to obtain RSA signature from store") + + // ErrStoreECDSASigningError represents an error returned from store during ECDSA signature + ErrStoreECDSASigningError = errors.New("unable to obtain ECDSA signature from store") + + // ErrNoPrivateKeyStoreRef represents an error getting a handle to a private key in store + ErrNoPrivateKeyStoreRef = errors.New("unable to obtain private key handle from store") + + // ErrExtractingPrivateKeyMetadata represents a family of errors extracting metadata about the private key in store + ErrExtractingPrivateKeyMetadata = errors.New("unable to extract private key metadata") + + // ErrExtractingECCPublicKey represents an error exporting ECC-type public key from store + ErrExtractingECCPublicKey = errors.New("unable to extract ECC public key from store") + + // ErrExtractingRSAPublicKey represents an error exporting RSA-type public key from store + ErrExtractingRSAPublicKey = errors.New("unable to extract RSA public key from store") + + // ErrExtractingPublicKey represents a general error exporting public key from store + ErrExtractingPublicKey = errors.New("unable to extract public key from store") + + // ErrBadPublicKeyAlgorithm represents a bad or unsupported public key algorithm + ErrBadPublicKeyAlgorithm = errors.New("unsupported public key algorithm") + + // ErrExtractPropertyFromKey represents a general failure to extract a metadata property field + ErrExtractPropertyFromKey = errors.New("unable to extract property from key") + + // ErrBadECCCurveName represents an ECC signature curve name that is bad or unsupported + ErrBadECCCurveName = errors.New("unsupported ECC curve name") + + // ErrFailedCertSearch represents not able to find certificate in store + ErrFailedCertSearch = errors.New("unable to find certificate in store") + + // ErrFailedX509Extract represents not being able to extract x509 certificate from found cert in store + ErrFailedX509Extract = errors.New("unable to extract x509 from certificate") + + // ErrBadMatchByType represents unknown CERT_MATCH_BY passed + ErrBadMatchByType = errors.New("cert match by type not implemented") + + // ErrBadCertStore represents unknown CERT_STORE passed + ErrBadCertStore = errors.New("cert store type not implemented") + + // ErrConflictCertFileAndStore represents ambiguous configuration of both file and store + ErrConflictCertFileAndStore = errors.New("'cert_file' and 'cert_store' may not both be configured") + + // ErrBadCertStoreField represents malformed cert_store option + ErrBadCertStoreField = errors.New("expected 'cert_store' to be a valid non-empty string") + + // ErrBadCertMatchByField represents malformed cert_match_by option + ErrBadCertMatchByField = errors.New("expected 'cert_match_by' to be a valid non-empty string") + + // ErrBadCertMatchField represents malformed cert_match option + ErrBadCertMatchField = errors.New("expected 'cert_match' to be a valid non-empty string") + + // ErrBadCaCertMatchField represents malformed cert_match option + ErrBadCaCertMatchField = errors.New("expected 'ca_certs_match' to be a valid non-empty string array") + + // ErrBadCertMatchSkipInvalidField represents malformed cert_match_skip_invalid option + ErrBadCertMatchSkipInvalidField = errors.New("expected 'cert_match_skip_invalid' to be a boolean") + + // ErrOSNotCompatCertStore represents cert_store passed that exists but is not valid on current OS + ErrOSNotCompatCertStore = errors.New("cert_store not compatible with current operating system") +) diff --git a/server/certstore_windows_test.go b/server/certstore_windows_test.go new file mode 100644 index 0000000..6cd50f5 --- /dev/null +++ b/server/certstore_windows_test.go @@ -0,0 +1,350 @@ +// Copyright 2022-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package server + +import ( + "crypto/tls" + "fmt" + "net/url" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +func runPowershellScript(scriptFile string, args []string) error { + psExec, _ := exec.LookPath("powershell.exe") + + execArgs := []string{psExec, "-command", fmt.Sprintf("& '%s'", scriptFile)} + if len(args) > 0 { + execArgs = append(execArgs, args...) + } + + cmdImport := &exec.Cmd{ + Path: psExec, + Args: execArgs, + Stdout: os.Stdout, + Stderr: os.Stderr, + } + return cmdImport.Run() +} + +func runConfiguredLeaf(t *testing.T, hubPort int, certStore string, matchBy string, match string, caMatch string, expectedLeafCount int) { + + // Fire up the leaf + u, err := url.Parse(fmt.Sprintf("nats://localhost:%d", hubPort)) + if err != nil { + t.Fatalf("Error parsing url: %v", err) + } + + configStr := fmt.Sprintf(` + port: -1 + leaf { + remotes [ + { + url: "%s" + tls { + cert_store: "%s" + cert_match_by: "%s" + cert_match: "%s" + ca_certs_match: %s + + # Test settings that succeed should be equivalent to: + # cert_file: "../test/configs/certs/tlsauth/client.pem" + # key_file: "../test/configs/certs/tlsauth/client-key.pem" + # ca_file: "../test/configs/certs/tlsauth/ca.pem" + timeout: 5 + } + } + ] + } + `, u.String(), certStore, matchBy, match, caMatch) + + leafConfig := createConfFile(t, []byte(configStr)) + defer removeFile(t, leafConfig) + leafServer, _ := RunServerWithConfig(leafConfig) + defer leafServer.Shutdown() + + // After client verify, hub will match by SAN email, SAN dns, and Subject (in that order) + // Our test client specifies Subject only so we should match on that... + + // A little settle time + time.Sleep(1 * time.Second) + checkLeafNodeConnectedCount(t, leafServer, expectedLeafCount) +} + +// TestLeafTLSWindowsCertStore tests the topology of two NATS Servers connected as leaf and hub with authentication of +// leaf to hub via mTLS with leaf's certificate and signing key provisioned in the Windows certificate store. +func TestLeafTLSWindowsCertStore(t *testing.T) { + + // Client Identity (client.pem) + // Issuer: O = NATS CA, OU = NATS.io, CN = localhost + // Subject: OU = NATS.io, CN = example.com + + // Make sure windows cert store is reset to avoid conflict with other tests + err := runPowershellScript("../test/configs/certs/tlsauth/certstore/delete-cert-from-store.ps1", nil) + if err != nil { + t.Fatalf("expected powershell cert delete to succeed: %s", err.Error()) + } + + // Provision Windows cert store with client cert and secret + err = runPowershellScript("../test/configs/certs/tlsauth/certstore/import-p12-client.ps1", nil) + if err != nil { + t.Fatalf("expected powershell provision to succeed: %s", err.Error()) + } + + err = runPowershellScript("../test/configs/certs/tlsauth/certstore/import-p12-ca.ps1", nil) + if err != nil { + t.Fatalf("expected powershell provision CA to succeed: %s", err.Error()) + } + + // Fire up the hub + hubConfig := createConfFile(t, []byte(` + port: -1 + leaf { + listen: "127.0.0.1:-1" + tls { + ca_file: "../test/configs/certs/tlsauth/ca.pem" + cert_file: "../test/configs/certs/tlsauth/server.pem" + key_file: "../test/configs/certs/tlsauth/server-key.pem" + timeout: 5 + verify_and_map: true + } + } + + accounts: { + AcctA: { + users: [ {user: "OU = NATS.io, CN = example.com"} ] + }, + AcctB: { + users: [ {user: UserB1} ] + }, + SYS: { + users: [ {user: System} ] + } + } + system_account: "SYS" + `)) + defer removeFile(t, hubConfig) + hubServer, hubOptions := RunServerWithConfig(hubConfig) + defer hubServer.Shutdown() + + testCases := []struct { + certStore string + certMatchBy string + certMatch string + caCertsMatch string + expectedLeafCount int + }{ + // Test subject and issuer + {"WindowsCurrentUser", "Subject", "example.com", "\"NATS CA\"", 1}, + {"WindowsCurrentUser", "Issuer", "NATS CA", "\"NATS CA\"", 1}, + {"WindowsCurrentUser", "Issuer", "Frodo Baggins, Inc.", "\"NATS CA\"", 0}, + {"WindowsCurrentUser", "Thumbprint", "7e44f478114a2e29b98b00beb1b3687d8dc0e481", "\"NATS CA\"", 0}, + // Test CAs, NATS CA is valid, others are missing + {"WindowsCurrentUser", "Subject", "example.com", "[\"NATS CA\"]", 1}, + {"WindowsCurrentUser", "Subject", "example.com", "[\"GlobalSign\"]", 0}, + {"WindowsCurrentUser", "Subject", "example.com", "[\"Missing NATS Cert\"]", 0}, + {"WindowsCurrentUser", "Subject", "example.com", "[\"NATS CA\", \"Missing NATS Cert1\"]", 1}, + {"WindowsCurrentUser", "Subject", "example.com", "[\"Missing Cert2\",\"NATS CA\"]", 1}, + {"WindowsCurrentUser", "Subject", "example.com", "[\"Missing, Cert3\",\"Missing NATS Cert4\"]", 0}, + } + for _, tc := range testCases { + testName := fmt.Sprintf("%s by %s match %s", tc.certStore, tc.certMatchBy, tc.certMatch) + t.Run(fmt.Sprintf(testName, tc.certStore, tc.certMatchBy, tc.certMatch, tc.caCertsMatch), func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + if tc.expectedLeafCount != 0 { + t.Fatalf("did not expect panic: %s", testName) + } else { + if !strings.Contains(fmt.Sprintf("%v", r), "Error processing configuration file") { + t.Fatalf("did not expect unknown panic: %s", testName) + } + } + } + }() + runConfiguredLeaf(t, hubOptions.LeafNode.Port, + tc.certStore, tc.certMatchBy, tc.certMatch, + tc.caCertsMatch, tc.expectedLeafCount) + }) + } +} + +// TestServerTLSWindowsCertStore tests the topology of a NATS server requiring TLS and gettings it own server +// cert identity (as used when accepting NATS client connections and negotiating TLS) from Windows certificate store. +func TestServerTLSWindowsCertStore(t *testing.T) { + + // Server Identity (server.pem) + // Issuer: O = NATS CA, OU = NATS.io, CN = localhost + // Subject: OU = NATS.io Operators, CN = localhost + + // Make sure windows cert store is reset to avoid conflict with other tests + err := runPowershellScript("../test/configs/certs/tlsauth/certstore/delete-cert-from-store.ps1", nil) + if err != nil { + t.Fatalf("expected powershell cert delete to succeed: %s", err.Error()) + } + + // Provision Windows cert store with server cert and secret + err = runPowershellScript("../test/configs/certs/tlsauth/certstore/import-p12-server.ps1", nil) + if err != nil { + t.Fatalf("expected powershell provision to succeed: %s", err.Error()) + } + + err = runPowershellScript("../test/configs/certs/tlsauth/certstore/import-p12-ca.ps1", nil) + if err != nil { + t.Fatalf("expected powershell provision CA to succeed: %s", err.Error()) + } + + // Fire up the server + srvConfig := createConfFile(t, []byte(` + listen: "localhost:-1" + tls { + cert_store: "WindowsCurrentUser" + cert_match_by: "Subject" + cert_match: "NATS.io Operators" + ca_certs_match: ["NATS CA"] + timeout: 5 + } + `)) + defer removeFile(t, srvConfig) + srvServer, _ := RunServerWithConfig(srvConfig) + if srvServer == nil { + t.Fatalf("expected to be able start server with cert store configuration") + } + defer srvServer.Shutdown() + + testCases := []struct { + clientCA string + expect bool + }{ + {"../test/configs/certs/tlsauth/ca.pem", true}, + {"../test/configs/certs/tlsauth/client.pem", false}, + } + for _, tc := range testCases { + t.Run(fmt.Sprintf("Client CA: %s", tc.clientCA), func(t *testing.T) { + nc, _ := nats.Connect(srvServer.clientConnectURLs[0], nats.RootCAs(tc.clientCA)) + err := nc.Publish("foo", []byte("hello TLS server-authenticated server")) + if (err != nil) == tc.expect { + t.Fatalf("expected publish result %v to TLS authenticated server", tc.expect) + } + nc.Close() + + for i := 0; i < 5; i++ { + nc, _ = nats.Connect(srvServer.clientConnectURLs[0], nats.RootCAs(tc.clientCA)) + err = nc.Publish("foo", []byte("hello TLS server-authenticated server")) + if (err != nil) == tc.expect { + t.Fatalf("expected repeated connection result %v to TLS authenticated server", tc.expect) + } + nc.Close() + } + }) + } +} + +// TestServerIgnoreExpiredCerts tests if the server skips expired certificates in configuration, and finds non-expired ones +func TestServerIgnoreExpiredCerts(t *testing.T) { + + // Server Identities: expired.pem; not-expired.pem + // Issuer: OU = NATS.io, CN = localhost + // Subject: OU = NATS.io Operators, CN = localhost + + testCases := []struct { + certFile string + expect bool + }{ + {"expired.p12", false}, + {"not-expired.p12", true}, + } + for _, tc := range testCases { + t.Run(fmt.Sprintf("Server certificate: %s", tc.certFile), func(t *testing.T) { + // Make sure windows cert store is reset to avoid conflict with other tests + err := runPowershellScript("../test/configs/certs/tlsauth/certstore/delete-cert-from-store.ps1", nil) + if err != nil { + t.Fatalf("expected powershell cert delete to succeed: %s", err.Error()) + } + + // Provision Windows cert store with server cert and secret + err = runPowershellScript("../test/configs/certs/tlsauth/certstore/import-p12-server.ps1", []string{tc.certFile}) + if err != nil { + t.Fatalf("expected powershell provision to succeed: %s", err.Error()) + } + // Fire up the server + srvConfig := createConfFile(t, []byte(` + listen: "localhost:-1" + tls { + cert_store: "WindowsCurrentUser" + cert_match_by: "Subject" + cert_match: "NATS.io Operators" + cert_match_skip_invalid: true + timeout: 5 + } + `)) + defer removeFile(t, srvConfig) + cfg, _ := ProcessConfigFile(srvConfig) + if (cfg != nil) == tc.expect { + return + } + if tc.expect == false { + t.Fatalf("expected server start to fail with expired certificate") + } else { + t.Fatalf("expected server to start with non expired certificate") + } + }) + } +} + +func TestWindowsTLS12ECDSA(t *testing.T) { + err := runPowershellScript("../test/configs/certs/tlsauth/certstore/import-p12-server.ps1", []string{"ecdsa_server.pfx"}) + if err != nil { + t.Fatalf("expected powershell provision to succeed: %v", err) + } + + config := createConfFile(t, []byte(` + listen: "localhost:-1" + tls { + cert_store: "WindowsCurrentUser" + cert_match_by: "Thumbprint" + cert_match: "4F8AF21756E5DBBD54619BBB6F3CC5D455ED4468" + cert_match_skip_invalid: true + timeout: 5 + } + `)) + defer removeFile(t, config) + + srv, _ := RunServerWithConfig(config) + if srv == nil { + t.Fatalf("expected to be able start server with cert store configuration") + } + defer srv.Shutdown() + + for name, version := range map[string]uint16{ + "TLS 1.3": tls.VersionTLS13, + "TLS 1.2": tls.VersionTLS12, + } { + t.Run(name, func(t *testing.T) { + tc := &tls.Config{MaxVersion: version, MinVersion: version, InsecureSkipVerify: true} + + if _, err = nats.Connect(srv.clientConnectURLs[0], nats.Secure(tc)); err != nil { + t.Fatalf("connection with %s: %v", name, err) + } + + t.Logf("successful connection with %s", name) + }) + } +} diff --git a/server/ciphersuites.go b/server/ciphersuites.go new file mode 100644 index 0000000..c6f4b04 --- /dev/null +++ b/server/ciphersuites.go @@ -0,0 +1,79 @@ +// Copyright 2016-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "crypto/fips140" + "crypto/tls" +) + +func init() { + for _, cs := range tls.CipherSuites() { + cipherMap[cs.Name] = cs + cipherMapByID[cs.ID] = cs + } + for _, cs := range tls.InsecureCipherSuites() { + cipherMap[cs.Name] = cs + cipherMapByID[cs.ID] = cs + } +} + +var cipherMap = map[string]*tls.CipherSuite{} +var cipherMapByID = map[uint16]*tls.CipherSuite{} + +// Where we maintain available curve preferences +var curvePreferenceMap = map[string]tls.CurveID{ + "SecP384r1MLKEM1024": tls.SecP384r1MLKEM1024, + "SecP256r1MLKEM768": tls.SecP256r1MLKEM768, + "X25519MLKEM768": tls.X25519MLKEM768, + "X25519": tls.X25519, + "CurveP256": tls.CurveP256, + "CurveP384": tls.CurveP384, + "CurveP521": tls.CurveP521, +} + +func defaultCipherSuites() []uint16 { + ciphers := tls.CipherSuites() + defaults := make([]uint16, 0, len(ciphers)) + for _, cs := range ciphers { + defaults = append(defaults, cs.ID) + } + return defaults +} + +// reorder to default to the highest level of security. See: +// https://blog.bracebin.com/achieving-perfect-ssl-labs-score-with-go +func defaultCurvePreferences() []tls.CurveID { + if fips140.Enabled() { + // X25519 is not FIPS-approved by itself, but it is when combined with MLKEM768. + // SecP256r1MLKEM768 and SecP384r1MLKEM1024 are both FIPS-approved hybrids. + return []tls.CurveID{ + tls.X25519MLKEM768, // post-quantum + tls.SecP384r1MLKEM1024, // post-quantum + tls.SecP256r1MLKEM768, // post-quantum + tls.CurveP256, + tls.CurveP384, + tls.CurveP521, + } + } + return []tls.CurveID{ + tls.X25519MLKEM768, // post-quantum + tls.SecP384r1MLKEM1024, // post-quantum + tls.SecP256r1MLKEM768, // post-quantum + tls.X25519, // faster than P256, arguably more secure + tls.CurveP256, + tls.CurveP384, + tls.CurveP521, + } +} diff --git a/server/client_proxyproto.go b/server/client_proxyproto.go new file mode 100644 index 0000000..b9b2bc5 --- /dev/null +++ b/server/client_proxyproto.go @@ -0,0 +1,415 @@ +// Copyright 2025-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "strconv" + "strings" + "time" +) + +// PROXY protocol v2 constants +const ( + // Protocol signature (12 bytes) + proxyProtoV2Sig = "\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A" + + // Version and command byte format: version(4 bits) | command(4 bits) + proxyProtoV2VerMask = 0xF0 + proxyProtoV2Ver = 0x20 // Version 2 + + // Commands + proxyProtoCmdMask = 0x0F + proxyProtoCmdLocal = 0x00 // LOCAL command (health check, use original connection) + proxyProtoCmdProxy = 0x01 // PROXY command (proxied connection) + + // Address family and protocol byte format: family(4 bits) | protocol(4 bits) + proxyProtoFamilyMask = 0xF0 + proxyProtoFamilyUnspec = 0x00 // Unspecified + proxyProtoFamilyInet = 0x10 // IPv4 + proxyProtoFamilyInet6 = 0x20 // IPv6 + proxyProtoFamilyUnix = 0x30 // Unix socket + proxyProtoProtoMask = 0x0F + proxyProtoProtoUnspec = 0x00 // Unspecified + proxyProtoProtoStream = 0x01 // TCP/STREAM + proxyProtoProtoDatagram = 0x02 // UDP/DGRAM + + // Address sizes + proxyProtoAddrSizeIPv4 = 12 // 4 (src IP) + 4 (dst IP) + 2 (src port) + 2 (dst port) + proxyProtoAddrSizeIPv6 = 36 // 16 (src IP) + 16 (dst IP) + 2 (src port) + 2 (dst port) + + // Header sizes + proxyProtoV2HeaderSize = 16 // Fixed header: 12 (sig) + 1 (ver/cmd) + 1 (fam/proto) + 2 (addr len) + + // Timeout for reading PROXY protocol header + proxyProtoReadTimeout = 5 * time.Second +) + +// PROXY protocol v1 constants +const ( + proxyProtoV1Prefix = "PROXY " + proxyProtoV1MaxLineLen = 107 // Maximum line length including CRLF + proxyProtoV1TCP4 = "TCP4" + proxyProtoV1TCP6 = "TCP6" + proxyProtoV1Unknown = "UNKNOWN" +) + +var ( + // Errors + errProxyProtoInvalid = errors.New("invalid PROXY protocol header") + errProxyProtoUnsupported = errors.New("unsupported PROXY protocol feature") + errProxyProtoTimeout = errors.New("timeout reading PROXY protocol header") + errProxyProtoUnrecognized = errors.New("unrecognized PROXY protocol format") +) + +// proxyProtoAddr contains the address information extracted from PROXY protocol header +type proxyProtoAddr struct { + srcIP net.IP + srcPort uint16 + dstIP net.IP + dstPort uint16 +} + +// String implements net.Addr interface +func (p *proxyProtoAddr) String() string { + return net.JoinHostPort(p.srcIP.String(), fmt.Sprintf("%d", p.srcPort)) +} + +// Network implements net.Addr interface +func (p *proxyProtoAddr) Network() string { + if p.srcIP.To4() != nil { + return "tcp4" + } + return "tcp6" +} + +// proxyConn wraps a net.Conn to override RemoteAddr() with the address +// extracted from the PROXY protocol header +type proxyConn struct { + net.Conn + remoteAddr net.Addr +} + +// RemoteAddr returns the original client address extracted from PROXY protocol +func (pc *proxyConn) RemoteAddr() net.Addr { + return pc.remoteAddr +} + +// detectProxyProtoVersion reads the first bytes and determines protocol version. +// Returns 1 for v1, 2 for v2, or error. +// The first 6 bytes read are returned so they can be used by the parser. +func detectProxyProtoVersion(conn net.Conn) (version int, header []byte, err error) { + // Read first 6 bytes to check for "PROXY " or v2 signature + header = make([]byte, 6) + if _, err = io.ReadFull(conn, header); err != nil { + return 0, nil, fmt.Errorf("failed to read protocol version: %w", err) + } + switch bytesToString(header) { + case proxyProtoV1Prefix: + return 1, header, nil + case proxyProtoV2Sig[:6]: + return 2, header, nil + default: + // Return the consumed bytes so the caller can replay them into the + // next protocol layer instead of discarding them. + return 0, header, errProxyProtoUnrecognized + } +} + +// readProxyProtoV1Header parses PROXY protocol v1 text format. +// Expects the "PROXY " prefix (6 bytes) to have already been consumed. +// Returns any bytes that were read past the trailing CRLF so the caller can +// replay them into the next protocol layer. +func readProxyProtoV1Header(conn net.Conn) (*proxyProtoAddr, []byte, error) { + // Read rest of line (max 107 bytes total, already read 6) + maxRemaining := proxyProtoV1MaxLineLen - 6 + + // Read up to maxRemaining bytes at once (more efficient than byte-by-byte) + buf := make([]byte, maxRemaining) + var line []byte + var remaining []byte + + for len(line) < maxRemaining { + // Read available data + n, err := conn.Read(buf[len(line):]) + if err != nil { + return nil, nil, fmt.Errorf("failed to read v1 line: %w", err) + } + + line = buf[:len(line)+n] + + // Look for CRLF in what we've read so far + for i := 0; i < len(line)-1; i++ { + if line[i] == '\r' && line[i+1] == '\n' { + // Found CRLF - keep any over-read bytes for the client parser. + remaining = append(remaining, line[i+2:]...) + line = line[:i] + goto foundCRLF + } + } + } + + // Exceeded max length without finding CRLF + return nil, nil, fmt.Errorf("%w: v1 line too long", errProxyProtoInvalid) + +foundCRLF: + // Get parts from the protocol + parts := strings.Fields(string(line)) + + // Validate format + if len(parts) < 1 { + return nil, nil, fmt.Errorf("%w: invalid v1 format", errProxyProtoInvalid) + } + + // Handle UNKNOWN (health check, like v2 LOCAL) + if parts[0] == proxyProtoV1Unknown { + return nil, remaining, nil + } + + // Must have exactly 5 parts: protocol, src-ip, dst-ip, src-port, dst-port + if len(parts) != 5 { + return nil, nil, fmt.Errorf("%w: invalid v1 format", errProxyProtoInvalid) + } + + protocol := parts[0] + srcIP := net.ParseIP(parts[1]) + dstIP := net.ParseIP(parts[2]) + + if srcIP == nil || dstIP == nil { + return nil, nil, fmt.Errorf("%w: invalid address", errProxyProtoInvalid) + } + + // Parse ports + srcPort, err := strconv.ParseUint(parts[3], 10, 16) + if err != nil { + return nil, nil, fmt.Errorf("invalid source port: %w", err) + } + + dstPort, err := strconv.ParseUint(parts[4], 10, 16) + if err != nil { + return nil, nil, fmt.Errorf("invalid dest port: %w", err) + } + + // Validate protocol matches IP version. The textual form determines the + // family: TCP4 requires dotted-quad addresses, TCP6 requires IPv6 + // addresses. IPv4-mapped IPv6 addresses (e.g. "::ffff:192.0.2.1") are + // valid for TCP6 since dual-stack proxies can emit those for IPv4 + // clients on IPv6 sockets, matching the v2 parser behavior. + srcIsV6 := strings.Contains(parts[1], ":") + dstIsV6 := strings.Contains(parts[2], ":") + if protocol == proxyProtoV1TCP4 && (srcIsV6 || dstIsV6) { + return nil, nil, fmt.Errorf("%w: TCP4 with IPv6 address", errProxyProtoInvalid) + } + if protocol == proxyProtoV1TCP6 && (!srcIsV6 || !dstIsV6) { + return nil, nil, fmt.Errorf("%w: TCP6 with IPv4 address", errProxyProtoInvalid) + } + if protocol != proxyProtoV1TCP4 && protocol != proxyProtoV1TCP6 { + return nil, nil, fmt.Errorf("%w: invalid protocol %s", errProxyProtoInvalid, protocol) + } + + return &proxyProtoAddr{ + srcIP: srcIP, + srcPort: uint16(srcPort), + dstIP: dstIP, + dstPort: uint16(dstPort), + }, remaining, nil +} + +// readProxyProtoHeader reads and parses PROXY protocol (v1 or v2) from the connection. +// Automatically detects version and routes to appropriate parser. +// If the command is LOCAL/UNKNOWN (health check), it returns nil for addr and no error. +// If the command is PROXY, it returns the parsed address information. +// It also returns any bytes that were read past the v1 header terminator so the +// caller can replay them into the normal client parser. +// The connection must be fresh (no data read yet). +func readProxyProtoHeader(conn net.Conn) (*proxyProtoAddr, []byte, error) { + // Set read deadline to prevent hanging on slow/malicious clients + if err := conn.SetReadDeadline(time.Now().Add(proxyProtoReadTimeout)); err != nil { + return nil, nil, err + } + defer conn.SetReadDeadline(time.Time{}) + + // Detect version. + // On errProxyProtoUnrecognized, firstBytes holds the bytes that were + // consumed so the caller can replay them. + version, firstBytes, err := detectProxyProtoVersion(conn) + if err != nil { + return nil, firstBytes, err + } + + switch version { + case 1: + // v1 parser expects "PROXY " prefix already consumed + return readProxyProtoV1Header(conn) + case 2: + // Read rest of v2 signature (bytes 6-11, total 6 more bytes) + remaining := make([]byte, 6) + if _, err := io.ReadFull(conn, remaining); err != nil { + return nil, nil, fmt.Errorf("failed to read v2 signature: %w", err) + } + + // Verify full signature + fullSig := string(firstBytes) + string(remaining) + if fullSig != proxyProtoV2Sig { + return nil, nil, fmt.Errorf("%w: invalid signature", errProxyProtoInvalid) + } + + // Read rest of header: ver/cmd, fam/proto, addr-len (4 bytes) + header := make([]byte, 4) + if _, err := io.ReadFull(conn, header); err != nil { + return nil, nil, fmt.Errorf("failed to read v2 header: %w", err) + } + + // Continue with parsing + addr, err := parseProxyProtoV2Header(conn, header) + return addr, nil, err + default: + return nil, nil, fmt.Errorf("unsupported PROXY protocol version: %d", version) + } +} + +// readProxyProtoV2Header is kept for backward compatibility and direct testing. +// It reads and parses a PROXY protocol v2 header from the connection. +// If the command is LOCAL (health check), it returns nil for addr and no error. +// If the command is PROXY, it returns the parsed address information. +// The connection must be fresh (no data read yet). +func readProxyProtoV2Header(conn net.Conn) (*proxyProtoAddr, error) { + // Set read deadline to prevent hanging on slow/malicious clients + if err := conn.SetReadDeadline(time.Now().Add(proxyProtoReadTimeout)); err != nil { + return nil, err + } + defer conn.SetReadDeadline(time.Time{}) + + // Read fixed header (16 bytes) + header := make([]byte, proxyProtoV2HeaderSize) + if _, err := io.ReadFull(conn, header); err != nil { + if ne, ok := err.(net.Error); ok && ne.Timeout() { + return nil, errProxyProtoTimeout + } + return nil, fmt.Errorf("failed to read PROXY protocol header: %w", err) + } + + // Validate signature (first 12 bytes) + if string(header[:12]) != proxyProtoV2Sig { + return nil, fmt.Errorf("%w: invalid signature", errProxyProtoInvalid) + } + + // Continue with parsing after signature + return parseProxyProtoV2Header(conn, header[12:16]) +} + +// parseProxyProtoV2Header parses v2 protocol after signature has been validated. +// header contains the 4 bytes: ver/cmd, fam/proto, addr-len (2 bytes). +func parseProxyProtoV2Header(conn net.Conn, header []byte) (*proxyProtoAddr, error) { + // Parse version and command + verCmd := header[0] + version := verCmd & proxyProtoV2VerMask + command := verCmd & proxyProtoCmdMask + + if version != proxyProtoV2Ver { + return nil, fmt.Errorf("%w: invalid version 0x%02x", errProxyProtoInvalid, version) + } + + // Parse address family and protocol + famProto := header[1] + family := famProto & proxyProtoFamilyMask + protocol := famProto & proxyProtoProtoMask + + // Parse address length (big-endian uint16) + addrLen := binary.BigEndian.Uint16(header[2:4]) + + // Handle LOCAL command (health check) + if command == proxyProtoCmdLocal { + // For LOCAL, we should skip the address data if any + if addrLen > 0 { + // Discard the address data + if _, err := io.CopyN(io.Discard, conn, int64(addrLen)); err != nil { + return nil, fmt.Errorf("failed to discard LOCAL command address data: %w", err) + } + } + return nil, nil // nil addr indicates LOCAL command + } + + // Handle PROXY command + if command != proxyProtoCmdProxy { + return nil, fmt.Errorf("unknown PROXY protocol command: 0x%02x", command) + } + + // Validate protocol (we only support STREAM/TCP) + if protocol != proxyProtoProtoStream { + return nil, fmt.Errorf("%w: only STREAM protocol supported", errProxyProtoUnsupported) + } + + // Parse address data based on family + var addr *proxyProtoAddr + var err error + switch family { + case proxyProtoFamilyInet: + addr, err = parseIPv4Addr(conn, addrLen) + case proxyProtoFamilyInet6: + addr, err = parseIPv6Addr(conn, addrLen) + case proxyProtoFamilyUnspec: + // UNSPEC family with PROXY command is valid but rare + // Just skip the address data + if addrLen > 0 { + if _, err := io.CopyN(io.Discard, conn, int64(addrLen)); err != nil { + return nil, fmt.Errorf("failed to discard UNSPEC address address data: %w", err) + } + } + return nil, nil + default: + return nil, fmt.Errorf("%w: unsupported address family 0x%02x", errProxyProtoUnsupported, family) + } + return addr, err +} + +// parseIPv4Addr parses IPv4 address data from PROXY protocol header +func parseIPv4Addr(conn net.Conn, addrLen uint16) (*proxyProtoAddr, error) { + // IPv4: 4 (src IP) + 4 (dst IP) + 2 (src port) + 2 (dst port) = 12 bytes minimum + if addrLen < proxyProtoAddrSizeIPv4 { + return nil, fmt.Errorf("IPv4 address data too short: %d bytes", addrLen) + } + addrData := make([]byte, addrLen) + if _, err := io.ReadFull(conn, addrData); err != nil { + return nil, fmt.Errorf("failed to read IPv4 address data: %w", err) + } + return &proxyProtoAddr{ + srcIP: net.IP(addrData[0:4]), + dstIP: net.IP(addrData[4:8]), + srcPort: binary.BigEndian.Uint16(addrData[8:10]), + dstPort: binary.BigEndian.Uint16(addrData[10:12]), + }, nil +} + +// parseIPv6Addr parses IPv6 address data from PROXY protocol header +func parseIPv6Addr(conn net.Conn, addrLen uint16) (*proxyProtoAddr, error) { + // IPv6: 16 (src IP) + 16 (dst IP) + 2 (src port) + 2 (dst port) = 36 bytes minimum + if addrLen < proxyProtoAddrSizeIPv6 { + return nil, fmt.Errorf("IPv6 address data too short: %d bytes", addrLen) + } + addrData := make([]byte, addrLen) + if _, err := io.ReadFull(conn, addrData); err != nil { + return nil, fmt.Errorf("failed to read IPv6 address data: %w", err) + } + return &proxyProtoAddr{ + srcIP: net.IP(addrData[0:16]), + dstIP: net.IP(addrData[16:32]), + srcPort: binary.BigEndian.Uint16(addrData[32:34]), + dstPort: binary.BigEndian.Uint16(addrData[34:36]), + }, nil +} diff --git a/server/client_proxyproto_test.go b/server/client_proxyproto_test.go new file mode 100644 index 0000000..c881487 --- /dev/null +++ b/server/client_proxyproto_test.go @@ -0,0 +1,925 @@ +// Copyright 2025-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bufio" + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/binary" + "fmt" + "io" + "net" + "os" + "strings" + "testing" + "time" +) + +// mockConn is a mock net.Conn for testing +type mockConn struct { + net.Conn + readBuf *bytes.Buffer + writeBuf *bytes.Buffer + closed bool + deadline time.Time +} + +func newMockConn(data []byte) *mockConn { + return &mockConn{ + readBuf: bytes.NewBuffer(data), + writeBuf: &bytes.Buffer{}, + } +} + +func (m *mockConn) Read(b []byte) (int, error) { + if m.closed { + return 0, fmt.Errorf("connection closed") + } + if !m.deadline.IsZero() && time.Now().After(m.deadline) { + return 0, &net.OpError{Op: "read", Err: fmt.Errorf("timeout")} + } + return m.readBuf.Read(b) +} + +func (m *mockConn) Write(b []byte) (int, error) { + if m.closed { + return 0, fmt.Errorf("connection closed") + } + return m.writeBuf.Write(b) +} + +func (m *mockConn) Close() error { + m.closed = true + return nil +} + +func (m *mockConn) LocalAddr() net.Addr { + return &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 4222} +} + +func (m *mockConn) RemoteAddr() net.Addr { + return &net.TCPAddr{IP: net.ParseIP("192.168.1.100"), Port: 54321} +} + +func (m *mockConn) SetDeadline(t time.Time) error { + m.deadline = t + return nil +} + +func (m *mockConn) SetReadDeadline(t time.Time) error { + m.deadline = t + return nil +} + +func (m *mockConn) SetWriteDeadline(t time.Time) error { + return nil +} + +// buildProxyV2Header builds a valid PROXY protocol v2 header +func buildProxyV2Header(t *testing.T, srcIP, dstIP string, srcPort, dstPort uint16, family byte) []byte { + t.Helper() + + buf := &bytes.Buffer{} + buf.WriteString(proxyProtoV2Sig) // Write signature + buf.WriteByte(proxyProtoV2Ver | proxyProtoCmdProxy) // Write version and command (version 2, PROXY command) + buf.WriteByte(family | proxyProtoProtoStream) // Write family and protocol + + src := net.ParseIP(srcIP) + dst := net.ParseIP(dstIP) + var addrData []byte + switch family { + case proxyProtoFamilyInet: + // IPv4: 12 bytes + addrData = make([]byte, proxyProtoAddrSizeIPv4) + copy(addrData[0:4], src.To4()) + copy(addrData[4:8], dst.To4()) + binary.BigEndian.PutUint16(addrData[8:10], srcPort) + binary.BigEndian.PutUint16(addrData[10:12], dstPort) + case proxyProtoFamilyInet6: + // IPv6: 36 bytes + addrData = make([]byte, proxyProtoAddrSizeIPv6) + copy(addrData[0:16], src.To16()) + copy(addrData[16:32], dst.To16()) + binary.BigEndian.PutUint16(addrData[32:34], srcPort) + binary.BigEndian.PutUint16(addrData[34:36], dstPort) + default: + t.Fatalf("unsupported address family: %d", family) + } + + addrLen := make([]byte, 2) + binary.BigEndian.PutUint16(addrLen, uint16(len(addrData))) + buf.Write(addrLen) + buf.Write(addrData) + + return buf.Bytes() +} + +// buildProxyV2LocalHeader builds a PROXY protocol v2 LOCAL command header +func buildProxyV2LocalHeader() []byte { + buf := &bytes.Buffer{} + buf.WriteString(proxyProtoV2Sig) // // Write signature + buf.WriteByte(proxyProtoV2Ver | proxyProtoCmdLocal) // // Write version and command (version 2, LOCAL command) + buf.WriteByte(proxyProtoFamilyUnspec | proxyProtoProtoUnspec) // // Write family and protocol (UNSPEC) + buf.WriteByte(0) + buf.WriteByte(0) + return buf.Bytes() +} + +// buildProxyV1Header builds a valid PROXY protocol v1 header (text format) +func buildProxyV1Header(t *testing.T, protocol, srcIP, dstIP string, srcPort, dstPort uint16) []byte { + t.Helper() + + if protocol != "TCP4" && protocol != "TCP6" && protocol != "UNKNOWN" { + t.Fatalf("invalid protocol: %s", protocol) + } + + var line string + if protocol == "UNKNOWN" { + line = "PROXY UNKNOWN\r\n" + } else { + line = fmt.Sprintf("PROXY %s %s %s %d %d\r\n", protocol, srcIP, dstIP, srcPort, dstPort) + } + + return []byte(line) +} + +func TestClientProxyProtoV2ParseIPv4(t *testing.T) { + header := buildProxyV2Header(t, "192.168.1.50", "10.0.0.1", 12345, 4222, proxyProtoFamilyInet) + conn := newMockConn(header) + + addr, err := readProxyProtoV2Header(conn) + require_NoError(t, err) + require_NotNil(t, addr) + + require_Equal(t, addr.srcIP.String(), "192.168.1.50") + require_Equal(t, addr.srcPort, 12345) + + require_Equal(t, addr.dstIP.String(), "10.0.0.1") + require_Equal(t, addr.dstPort, 4222) + + // Test String() and Network() methods + require_Equal(t, addr.String(), "192.168.1.50:12345") + require_Equal(t, addr.Network(), "tcp4") +} + +func TestClientProxyProtoV2ParseIPv6(t *testing.T) { + header := buildProxyV2Header(t, "2001:db8::1", "2001:db8::2", 54321, 4222, proxyProtoFamilyInet6) + conn := newMockConn(header) + + addr, err := readProxyProtoV2Header(conn) + require_NoError(t, err) + require_NotNil(t, addr) + + require_Equal(t, addr.srcIP.String(), "2001:db8::1") + require_Equal(t, addr.srcPort, 54321) + + require_Equal(t, addr.dstIP.String(), "2001:db8::2") + require_Equal(t, addr.dstPort, 4222) + + // Test Network() method for IPv6 + require_Equal(t, addr.String(), "[2001:db8::1]:54321") + require_Equal(t, addr.Network(), "tcp6") +} + +func TestClientProxyProtoV2ParseLocalCommand(t *testing.T) { + header := buildProxyV2LocalHeader() + conn := newMockConn(header) + + addr, err := readProxyProtoV2Header(conn) + require_NoError(t, err) + require_True(t, addr == nil) +} + +func TestClientProxyProtoV2InvalidSignature(t *testing.T) { + // Create invalid signature + header := []byte("INVALID_SIG_") + header = append(header, []byte{0x20, 0x11, 0x00, 0x0C}...) + conn := newMockConn(header) + + _, err := readProxyProtoV2Header(conn) + require_Error(t, err, errProxyProtoInvalid) +} + +func TestClientProxyProtoV2InvalidVersion(t *testing.T) { + buf := &bytes.Buffer{} + buf.WriteString(proxyProtoV2Sig) + buf.WriteByte(0x10 | proxyProtoCmdProxy) // Version 1 instead of 2 + buf.WriteByte(proxyProtoFamilyInet | proxyProtoProtoStream) + buf.WriteByte(0) + buf.WriteByte(0) + + conn := newMockConn(buf.Bytes()) + + _, err := readProxyProtoV2Header(conn) + require_Error(t, err, errProxyProtoInvalid) +} + +func TestClientProxyProtoV2UnsupportedFamily(t *testing.T) { + buf := &bytes.Buffer{} + buf.WriteString(proxyProtoV2Sig) + buf.WriteByte(proxyProtoV2Ver | proxyProtoCmdProxy) + buf.WriteByte(proxyProtoFamilyUnix | proxyProtoProtoStream) // Unix socket family + buf.WriteByte(0) + buf.WriteByte(0) + + conn := newMockConn(buf.Bytes()) + + _, err := readProxyProtoV2Header(conn) + require_Error(t, err, errProxyProtoUnsupported) +} + +func TestClientProxyProtoV2UnsupportedProtocol(t *testing.T) { + buf := &bytes.Buffer{} + buf.WriteString(proxyProtoV2Sig) + buf.WriteByte(proxyProtoV2Ver | proxyProtoCmdProxy) + buf.WriteByte(proxyProtoFamilyInet | proxyProtoProtoDatagram) // UDP instead of TCP + buf.WriteByte(0) + buf.WriteByte(12) + + conn := newMockConn(buf.Bytes()) + + _, err := readProxyProtoV2Header(conn) + require_Error(t, err, errProxyProtoUnsupported) +} + +func TestClientProxyProtoV2TruncatedHeader(t *testing.T) { + header := buildProxyV2Header(t, "192.168.1.50", "10.0.0.1", 12345, 4222, proxyProtoFamilyInet) + // Only send first 10 bytes (incomplete header) + conn := newMockConn(header[:10]) + + _, err := readProxyProtoV2Header(conn) + require_Error(t, err, io.ErrUnexpectedEOF) +} + +func TestClientProxyProtoV2ShortAddressData(t *testing.T) { + buf := &bytes.Buffer{} + buf.WriteString(proxyProtoV2Sig) + buf.WriteByte(proxyProtoV2Ver | proxyProtoCmdProxy) + buf.WriteByte(proxyProtoFamilyInet | proxyProtoProtoStream) + // Set address length to 12 but don't provide data + buf.WriteByte(0) + buf.WriteByte(12) + // Only provide 5 bytes instead of 12 + buf.Write([]byte{1, 2, 3, 4, 5}) + + conn := newMockConn(buf.Bytes()) + + _, err := readProxyProtoV2Header(conn) + require_Error(t, err, io.ErrUnexpectedEOF) +} + +func TestProxyConnRemoteAddr(t *testing.T) { + // Create a real TCP connection for testing + originalAddr := &net.TCPAddr{IP: net.ParseIP("192.168.1.100"), Port: 54321} + + // Create proxy address + proxyAddr := &proxyProtoAddr{ + srcIP: net.ParseIP("10.0.0.50"), + srcPort: 12345, + dstIP: net.ParseIP("10.0.0.1"), + dstPort: 4222, + } + + mockConn := newMockConn(nil) + wrapped := &proxyConn{ + Conn: mockConn, + remoteAddr: proxyAddr, + } + + // Verify RemoteAddr returns the proxied address + addr := wrapped.RemoteAddr() + require_Equal(t, addr.String(), "10.0.0.50:12345") + require_Equal(t, mockConn.RemoteAddr().String(), originalAddr.String()) +} + +func TestClientProxyProtoV2EndToEnd(t *testing.T) { + // Start a test server with PROXY protocol enabled + opts := DefaultOptions() + opts.Port = -1 // Random port + opts.ProxyProtocol = true + + s := RunServer(opts) + defer s.Shutdown() + + // Get the server's listening port + addr := s.Addr().String() + + // Connect to the server + conn, err := net.Dial("tcp", addr) + require_NoError(t, err) + defer conn.Close() + + // Send PROXY protocol header + clientIP := "203.0.113.50" + clientPort := uint16(54321) + header := buildProxyV2Header(t, clientIP, "127.0.0.1", clientPort, 4222, proxyProtoFamilyInet) + + _, err = conn.Write(header) + require_NoError(t, err) + + // Send CONNECT message + connectMsg := "CONNECT {\"verbose\":false,\"pedantic\":false,\"protocol\":1}\r\n" + _, err = conn.Write([]byte(connectMsg)) + require_NoError(t, err) + + // Read INFO and +OK + buf := make([]byte, 4096) + n, err := conn.Read(buf) + require_NoError(t, err) + + response := string(buf[:n]) + require_True(t, strings.Contains(response, "INFO")) + + // Give server time to process + time.Sleep(100 * time.Millisecond) + + // Check server's client list to verify the IP was extracted correctly + s.mu.Lock() + clients := s.clients + s.mu.Unlock() + require_True(t, len(clients) != 0) + + // Find our client + var foundClient *client + for _, c := range clients { + c.mu.Lock() + if c.host == clientIP && c.port == clientPort { + foundClient = c + } + c.mu.Unlock() + if foundClient != nil { + break + } + } + require_NotNil(t, foundClient) +} + +func TestClientProxyProtoV2LocalCommandEndToEnd(t *testing.T) { + // Start a test server with PROXY protocol enabled + opts := DefaultOptions() + opts.Port = -1 // Random port + opts.ProxyProtocol = true + + s := RunServer(opts) + defer s.Shutdown() + + // Get the server's listening port + addr := s.Addr().String() + + // Connect to the server + conn, err := net.Dial("tcp", addr) + require_NoError(t, err) + defer conn.Close() + + // Send PROXY protocol LOCAL header (health check) + header := buildProxyV2LocalHeader() + + _, err = conn.Write(header) + require_NoError(t, err) + + // Send CONNECT message + connectMsg := "CONNECT {\"verbose\":false,\"pedantic\":false,\"protocol\":1}\r\n" + _, err = conn.Write([]byte(connectMsg)) + require_NoError(t, err) + + // Read INFO and +OK + buf := make([]byte, 4096) + n, err := conn.Read(buf) + require_NoError(t, err) + + response := string(buf[:n]) + require_True(t, strings.Contains(response, "INFO")) + + // Connection should work normally with LOCAL command + time.Sleep(100 * time.Millisecond) + + // Verify at least one client is connected + s.mu.Lock() + numClients := len(s.clients) + s.mu.Unlock() + require_NotEqual(t, numClients, 0) +} + +// ============================================================================ +// PROXY Protocol v1 Tests +// ============================================================================ + +func TestClientProxyProtoV1ParseTCP4(t *testing.T) { + header := buildProxyV1Header(t, "TCP4", "192.168.1.50", "10.0.0.1", 12345, 4222) + conn := newMockConn(header) + + addr, _, err := readProxyProtoHeader(conn) + require_NoError(t, err) + require_NotNil(t, addr) + + require_Equal(t, addr.srcIP.String(), "192.168.1.50") + require_Equal(t, addr.srcPort, 12345) + + require_Equal(t, addr.dstIP.String(), "10.0.0.1") + require_Equal(t, addr.dstPort, 4222) +} + +func TestClientProxyProtoV1ParseTCP6(t *testing.T) { + header := buildProxyV1Header(t, "TCP6", "2001:db8::1", "2001:db8::2", 54321, 4222) + conn := newMockConn(header) + + addr, _, err := readProxyProtoHeader(conn) + require_NoError(t, err) + require_NotNil(t, addr) + + require_Equal(t, addr.srcIP.String(), "2001:db8::1") + require_Equal(t, addr.srcPort, 54321) + + require_Equal(t, addr.dstIP.String(), "2001:db8::2") + require_Equal(t, addr.dstPort, 4222) +} + +func TestClientProxyProtoV1ParseTCP6IPv4Mapped(t *testing.T) { + // Dual-stack proxies can emit IPv4-mapped IPv6 addresses for TCP6 + // (an IPv4 client connecting to an IPv6 socket). These are valid IPv6 + // textual addresses and must be accepted, matching the v2 parser. + header := buildProxyV1Header(t, "TCP6", "::ffff:192.0.2.1", "::ffff:10.0.0.1", 1234, 4222) + conn := newMockConn(header) + + addr, _, err := readProxyProtoHeader(conn) + require_NoError(t, err) + require_NotNil(t, addr) + + require_Equal(t, addr.srcIP.String(), "192.0.2.1") + require_Equal(t, addr.srcPort, 1234) + + require_Equal(t, addr.dstIP.String(), "10.0.0.1") + require_Equal(t, addr.dstPort, 4222) +} + +func TestClientProxyProtoV1MismatchedDestProtocol(t *testing.T) { + // TCP4 with IPv6 destination address + header := buildProxyV1Header(t, "TCP4", "192.168.1.1", "2001:db8::2", 12345, 443) + conn := newMockConn(header) + + _, _, err := readProxyProtoHeader(conn) + require_Error(t, err, errProxyProtoInvalid) + + // TCP6 with IPv4 destination address + header2 := buildProxyV1Header(t, "TCP6", "2001:db8::1", "10.0.0.1", 12345, 443) + conn2 := newMockConn(header2) + + _, _, err = readProxyProtoHeader(conn2) + require_Error(t, err, errProxyProtoInvalid) +} + +func TestClientProxyProtoV1ParseUnknown(t *testing.T) { + header := buildProxyV1Header(t, "UNKNOWN", "", "", 0, 0) + conn := newMockConn(header) + + addr, _, err := readProxyProtoHeader(conn) + require_NoError(t, err) + require_True(t, addr == nil) +} + +func TestClientProxyProtoV1PreservesCoalescedClientBytes(t *testing.T) { + header := buildProxyV1Header(t, "TCP4", "192.168.1.50", "10.0.0.1", 12345, 4222) + connect := []byte("CONNECT {\"verbose\":false,\"pedantic\":false,\"protocol\":1}\r\n") + conn := newMockConn(append(header, connect...)) + + addr, remaining, err := readProxyProtoHeader(conn) + require_NoError(t, err) + require_NotNil(t, addr) + require_Equal(t, string(remaining), string(connect)) + + rest, err := io.ReadAll(conn) + require_NoError(t, err) + require_Equal(t, len(rest), 0) +} + +func TestClientProxyProtoV1InvalidFormat(t *testing.T) { + // Missing fields + header := []byte("PROXY TCP4 192.168.1.1\r\n") + conn := newMockConn(header) + + _, _, err := readProxyProtoHeader(conn) + require_Error(t, err, errProxyProtoInvalid) +} + +func TestClientProxyProtoV1LineTooLong(t *testing.T) { + // Create a line longer than 107 bytes + longIP := strings.Repeat("1234567890", 12) // 120 chars + header := fmt.Appendf(nil, "PROXY TCP4 %s 10.0.0.1 12345 443\r\n", longIP) + conn := newMockConn(header) + + _, _, err := readProxyProtoHeader(conn) + require_Error(t, err, errProxyProtoInvalid) +} + +func TestClientProxyProtoV1InvalidIP(t *testing.T) { + header := []byte("PROXY TCP4 not.an.ip.addr 10.0.0.1 12345 443\r\n") + conn := newMockConn(header) + + _, _, err := readProxyProtoHeader(conn) + require_Error(t, err, errProxyProtoInvalid) +} + +func TestClientProxyProtoV1MismatchedProtocol(t *testing.T) { + // TCP4 with IPv6 address + header := buildProxyV1Header(t, "TCP4", "2001:db8::1", "2001:db8::2", 12345, 443) + conn := newMockConn(header) + + _, _, err := readProxyProtoHeader(conn) + require_Error(t, err, errProxyProtoInvalid) + + // TCP6 with IPv4 address + header2 := buildProxyV1Header(t, "TCP6", "192.168.1.1", "10.0.0.1", 12345, 443) + conn2 := newMockConn(header2) + + _, _, err = readProxyProtoHeader(conn2) + require_Error(t, err, errProxyProtoInvalid) +} + +func TestClientProxyProtoV1InvalidPort(t *testing.T) { + header := []byte("PROXY TCP4 192.168.1.1 10.0.0.1 99999 443\r\n") + conn := newMockConn(header) + + _, _, err := readProxyProtoHeader(conn) + require_True(t, err != nil) +} + +func TestClientProxyProtoV1EndToEnd(t *testing.T) { + // Start a test server with PROXY protocol enabled + opts := DefaultOptions() + opts.Port = -1 // Random port + opts.ProxyProtocol = true + + s := RunServer(opts) + defer s.Shutdown() + + // Get the server's listening port + addr := s.Addr().String() + + // Connect to the server + conn, err := net.Dial("tcp", addr) + require_NoError(t, err) + defer conn.Close() + + // Send the PROXY protocol header and first client commands in one write so + // the parser must preserve bytes read past the end of the v1 header. + clientIP, clientPort := "203.0.113.50", uint16(54321) + header := buildProxyV1Header(t, "TCP4", clientIP, "127.0.0.1", clientPort, 4222) + payload := append(header, []byte("CONNECT {\"verbose\":true,\"pedantic\":false,\"protocol\":1}\r\nPING\r\n")...) + + _, err = conn.Write(payload) + require_NoError(t, err) + + require_NoError(t, conn.SetReadDeadline(time.Now().Add(2*time.Second))) + cr := bufio.NewReader(conn) + + line, err := cr.ReadString('\n') + require_NoError(t, err) + require_True(t, strings.HasPrefix(line, "INFO ")) + + line, err = cr.ReadString('\n') + require_NoError(t, err) + require_True(t, strings.HasPrefix(line, "+OK")) + + expectPong(t, cr) + + // Give server time to process + time.Sleep(100 * time.Millisecond) + + // Check server's client list to verify the IP was extracted correctly + s.mu.Lock() + clients := s.clients + var foundClient *client + for _, c := range clients { + c.mu.Lock() + if c.host == clientIP && c.port == clientPort { + foundClient = c + } + c.mu.Unlock() + if foundClient != nil { + break + } + } + s.mu.Unlock() + require_NotNil(t, foundClient) +} + +// ============================================================================ +// Mixed Protocol Version Tests +// ============================================================================ + +func TestClientProxyProtoVersionDetection(t *testing.T) { + // Test v1 detection + v1Header := buildProxyV1Header(t, "TCP4", "192.168.1.1", "10.0.0.1", 12345, 443) + conn1 := newMockConn(v1Header) + + addr1, _, err := readProxyProtoHeader(conn1) + require_NoError(t, err) + require_NotNil(t, addr1) + require_Equal(t, addr1.srcIP.String(), "192.168.1.1") + + // Test v2 detection + v2Header := buildProxyV2Header(t, "192.168.1.2", "10.0.0.1", 54321, 443, proxyProtoFamilyInet) + conn2 := newMockConn(v2Header) + + addr2, _, err := readProxyProtoHeader(conn2) + require_NoError(t, err) + require_NotNil(t, addr2) + require_Equal(t, addr2.srcIP.String(), "192.168.1.2") +} + +func TestClientProxyProtoUnrecognizedVersion(t *testing.T) { + // Invalid header that doesn't match v1 or v2 + header := []byte("HELLO WORLD\r\n") + conn := newMockConn(header) + + _, _, err := readProxyProtoHeader(conn) + require_Error(t, err, errProxyProtoUnrecognized) +} + +// ============================================================================ +// PROXY Protocol + TLS Tests (regression for issue #7571) +// ============================================================================ + +// runProxyProtoTLSServer starts a server with proxy_protocol and a required +// TLS listener (no allow_non_tls, no tls_first). +func runProxyProtoTLSServer(t *testing.T) *Server { + t.Helper() + tc := &TLSConfigOpts{ + CertFile: "../test/configs/certs/server-cert.pem", + KeyFile: "../test/configs/certs/server-key.pem", + CaFile: "../test/configs/certs/ca.pem", + } + tlsConfig, err := GenTLSConfig(tc) + require_NoError(t, err) + + opts := DefaultOptions() + opts.Port = -1 + opts.ProxyProtocol = true + opts.TLSConfig = tlsConfig + opts.TLSTimeout = 2 + return RunServer(opts) +} + +// proxyProtoTLSClientConfig builds a tls.Config trusting the test CA. The +// server cert is issued for "localhost" / 127.0.0.1, so ServerName must match. +func proxyProtoTLSClientConfig(t *testing.T) *tls.Config { + t.Helper() + caPEM, err := os.ReadFile("../test/configs/certs/ca.pem") + require_NoError(t, err) + pool := x509.NewCertPool() + require_True(t, pool.AppendCertsFromPEM(caPEM)) + return &tls.Config{ + RootCAs: pool, + ServerName: "localhost", + } +} + +// findProxyProtoClient finds a connected client whose host/port matches what +// was advertised in the PROXY header. +func findProxyProtoClient(t *testing.T, s *Server, clientIP string, clientPort uint16) *client { + t.Helper() + s.mu.Lock() + clients := make([]*client, 0, len(s.clients)) + for _, c := range s.clients { + clients = append(clients, c) + } + s.mu.Unlock() + + for _, c := range clients { + c.mu.Lock() + host, port := c.host, c.port + c.mu.Unlock() + if host == clientIP && port == clientPort { + return c + } + } + return nil +} + +// connectProxyProtoTLS mirrors what an L4 proxy and a TLS-enabled NATS client +// do over a single TCP connection: prepend the PROXY header, read the initial +// plaintext INFO, then upgrade to TLS. Returns the buffered reader and the +// TLS-wrapped connection for subsequent protocol writes. +func connectProxyProtoTLS(t *testing.T, addr string, header []byte) (*bufio.Reader, *tls.Conn) { + t.Helper() + rawConn, err := net.Dial("tcp", addr) + require_NoError(t, err) + require_NoError(t, rawConn.SetDeadline(time.Now().Add(5*time.Second))) + + _, err = rawConn.Write(header) + require_NoError(t, err) + + // Server emits INFO over plaintext before the TLS upgrade. Read it now + // so the bytes don't end up as input to the TLS handshake. + plainReader := bufio.NewReader(rawConn) + infoLine, err := plainReader.ReadString('\n') + require_NoError(t, err) + require_True(t, strings.HasPrefix(infoLine, "INFO ")) + require_Equal(t, plainReader.Buffered(), 0) + + tlsConn := tls.Client(rawConn, proxyProtoTLSClientConfig(t)) + require_NoError(t, tlsConn.Handshake()) + return bufio.NewReader(tlsConn), tlsConn +} + +// TestClientProxyProtoV1WithRequiredTLS verifies that with proxy_protocol and +// required TLS, a client may send the PROXY v1 header followed by the TLS +// upgrade on a single TCP connection. Regression for issue #7571. +func TestClientProxyProtoV1WithRequiredTLS(t *testing.T) { + s := runProxyProtoTLSServer(t) + defer s.Shutdown() + + clientIP, clientPort := "203.0.113.50", uint16(54321) + header := buildProxyV1Header(t, "TCP4", clientIP, "127.0.0.1", clientPort, 4222) + cr, tlsConn := connectProxyProtoTLS(t, s.Addr().String(), header) + defer tlsConn.Close() + + _, err := tlsConn.Write([]byte("CONNECT {\"verbose\":true,\"pedantic\":false,\"protocol\":1}\r\nPING\r\n")) + require_NoError(t, err) + + line, err := cr.ReadString('\n') + require_NoError(t, err) + require_True(t, strings.HasPrefix(line, "+OK")) + expectPong(t, cr) + + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if findProxyProtoClient(t, s, clientIP, clientPort) == nil { + return fmt.Errorf("no client with PROXY-supplied host/port yet") + } + return nil + }) +} + +// TestClientProxyProtoV2WithRequiredTLS is the v2 counterpart to +// TestClientProxyProtoV1WithRequiredTLS. +func TestClientProxyProtoV2WithRequiredTLS(t *testing.T) { + s := runProxyProtoTLSServer(t) + defer s.Shutdown() + + clientIP, clientPort := "203.0.113.51", uint16(54322) + header := buildProxyV2Header(t, clientIP, "127.0.0.1", clientPort, 4222, proxyProtoFamilyInet) + cr, tlsConn := connectProxyProtoTLS(t, s.Addr().String(), header) + defer tlsConn.Close() + + _, err := tlsConn.Write([]byte("CONNECT {\"verbose\":true,\"pedantic\":false,\"protocol\":1}\r\nPING\r\n")) + require_NoError(t, err) + + line, err := cr.ReadString('\n') + require_NoError(t, err) + require_True(t, strings.HasPrefix(line, "+OK")) + expectPong(t, cr) + + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if findProxyProtoClient(t, s, clientIP, clientPort) == nil { + return fmt.Errorf("no client with PROXY-supplied host/port yet") + } + return nil + }) +} + +func TestClientProxyProtoNonProxiedTLSFirstFallback(t *testing.T) { + tc := &TLSConfigOpts{ + CertFile: "../test/configs/certs/server-cert.pem", + KeyFile: "../test/configs/certs/server-key.pem", + CaFile: "../test/configs/certs/ca.pem", + } + tlsConfig, err := GenTLSConfig(tc) + require_NoError(t, err) + + opts := DefaultOptions() + opts.Port = -1 + opts.ProxyProtocol = true + opts.TLSConfig = tlsConfig + opts.TLSTimeout = 2 + opts.TLSHandshakeFirst = true + opts.TLSHandshakeFirstFallback = 250 * time.Millisecond + s := RunServer(opts) + defer s.Shutdown() + + rawConn, err := net.Dial("tcp", s.Addr().String()) + require_NoError(t, err) + defer rawConn.Close() + require_NoError(t, rawConn.SetDeadline(time.Now().Add(5*time.Second))) + + // Direct TLS-first client: start the handshake right away (within the + // fallback delay), no PROXY header. + tlsConn := tls.Client(rawConn, proxyProtoTLSClientConfig(t)) + require_NoError(t, tlsConn.Handshake()) + + cr := bufio.NewReader(tlsConn) + infoLine, err := cr.ReadString('\n') + require_NoError(t, err) + require_True(t, strings.HasPrefix(infoLine, "INFO ")) + + _, err = tlsConn.Write([]byte("CONNECT {\"verbose\":true,\"pedantic\":false,\"protocol\":1}\r\nPING\r\n")) + require_NoError(t, err) + + line, err := cr.ReadString('\n') + require_NoError(t, err) + require_True(t, strings.HasPrefix(line, "+OK")) + expectPong(t, cr) +} + +// runProxyProtoAllowNonTLSServer starts a server with proxy_protocol and a +// TLS listener that also allows non-TLS clients (allow_non_tls). +func runProxyProtoAllowNonTLSServer(t *testing.T) *Server { + t.Helper() + tc := &TLSConfigOpts{ + CertFile: "../test/configs/certs/server-cert.pem", + KeyFile: "../test/configs/certs/server-key.pem", + CaFile: "../test/configs/certs/ca.pem", + } + tlsConfig, err := GenTLSConfig(tc) + require_NoError(t, err) + + opts := DefaultOptions() + opts.Port = -1 + opts.ProxyProtocol = true + opts.TLSConfig = tlsConfig + opts.TLSTimeout = 2 + opts.AllowNonTLS = true + return RunServer(opts) +} + +// testClientProxyProtoAllowNonTLS verifies that with proxy_protocol and +// allow_non_tls, the TLS-vs-plaintext detection is performed on the first +// byte of the client's actual traffic, not on the first byte of the PROXY +// header. Both a TLS client and a plaintext client behind the proxy must +// be able to connect on the same listener. +func testClientProxyProtoAllowNonTLS(t *testing.T, header []byte, clientIP string, clientPort uint16) { + t.Helper() + s := runProxyProtoAllowNonTLSServer(t) + defer s.Shutdown() + + // TLS client behind the proxy: the sniff must detect the TLS + // ClientHello that follows the PROXY header. + cr, tlsConn := connectProxyProtoTLS(t, s.Addr().String(), header) + defer tlsConn.Close() + + _, err := tlsConn.Write([]byte("CONNECT {\"verbose\":true,\"pedantic\":false,\"protocol\":1}\r\nPING\r\n")) + require_NoError(t, err) + + line, err := cr.ReadString('\n') + require_NoError(t, err) + require_True(t, strings.HasPrefix(line, "+OK")) + expectPong(t, cr) + + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if findProxyProtoClient(t, s, clientIP, clientPort) == nil { + return fmt.Errorf("no client with PROXY-supplied host/port yet") + } + return nil + }) + + // Plaintext client behind the proxy must still be allowed. + rawConn, err := net.Dial("tcp", s.Addr().String()) + require_NoError(t, err) + defer rawConn.Close() + require_NoError(t, rawConn.SetDeadline(time.Now().Add(5*time.Second))) + + _, err = rawConn.Write(header) + require_NoError(t, err) + + plainReader := bufio.NewReader(rawConn) + infoLine, err := plainReader.ReadString('\n') + require_NoError(t, err) + require_True(t, strings.HasPrefix(infoLine, "INFO ")) + + _, err = rawConn.Write([]byte("CONNECT {\"verbose\":true,\"pedantic\":false,\"protocol\":1}\r\nPING\r\n")) + require_NoError(t, err) + + line, err = plainReader.ReadString('\n') + require_NoError(t, err) + require_True(t, strings.HasPrefix(line, "+OK")) + expectPong(t, plainReader) +} + +// TestClientProxyProtoV1WithAllowNonTLS verifies that a proxied TLS client is +// detected as TLS (and a proxied plaintext client as non-TLS) when the +// listener has both tls and allow_non_tls configured, using a PROXY v1 header. +func TestClientProxyProtoV1WithAllowNonTLS(t *testing.T) { + clientIP, clientPort := "203.0.113.60", uint16(54331) + header := buildProxyV1Header(t, "TCP4", clientIP, "127.0.0.1", clientPort, 4222) + testClientProxyProtoAllowNonTLS(t, header, clientIP, clientPort) +} + +// TestClientProxyProtoV2WithAllowNonTLS is the v2 counterpart to +// TestClientProxyProtoV1WithAllowNonTLS. +func TestClientProxyProtoV2WithAllowNonTLS(t *testing.T) { + clientIP, clientPort := "203.0.113.61", uint16(54332) + header := buildProxyV2Header(t, clientIP, "127.0.0.1", clientPort, 4222, proxyProtoFamilyInet) + testClientProxyProtoAllowNonTLS(t, header, clientIP, clientPort) +} diff --git a/server/closed_conns_test.go b/server/closed_conns_test.go new file mode 100644 index 0000000..aea061f --- /dev/null +++ b/server/closed_conns_test.go @@ -0,0 +1,270 @@ +// Copyright 2018-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "fmt" + "net" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +func checkClosedConns(t *testing.T, s *Server, num int, wait time.Duration) { + t.Helper() + checkFor(t, wait, 5*time.Millisecond, func() error { + if nc := s.closed.len(); nc != num { + return fmt.Errorf("Closed conns expected to be %v, got %v", num, nc) + } + return nil + }) +} + +func checkTotalClosedConns(t *testing.T, s *Server, num uint64, wait time.Duration) { + t.Helper() + checkFor(t, wait, 5*time.Millisecond, func() error { + if nc := s.closed.totalConns(); nc != num { + return fmt.Errorf("Total closed conns expected to be %v, got %v", num, nc) + } + return nil + }) +} + +func TestClosedConnsAccounting(t *testing.T) { + opts := DefaultOptions() + opts.MaxClosedClients = 10 + opts.NoSystemAccount = true + + s := RunServer(opts) + defer s.Shutdown() + + wait := time.Second + + nc, err := nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + id, _ := nc.GetClientID() + nc.Close() + + checkClosedConns(t, s, 1, wait) + + conns := s.closed.closedClients() + if lc := len(conns); lc != 1 { + t.Fatalf("len(conns) expected to be %d, got %d\n", 1, lc) + } + if conns[0].Cid != id { + t.Fatalf("Expected CID to be %d, got %d\n", id, conns[0].Cid) + } + + // Now create 21 more + for i := 0; i < 21; i++ { + nc, err = nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + nc.Close() + checkTotalClosedConns(t, s, uint64(i+2), wait) + } + + checkClosedConns(t, s, opts.MaxClosedClients, wait) + checkTotalClosedConns(t, s, 22, wait) + + conns = s.closed.closedClients() + if lc := len(conns); lc != opts.MaxClosedClients { + t.Fatalf("len(conns) expected to be %d, got %d\n", + opts.MaxClosedClients, lc) + } + + // Set it to the start after overflow. + cid := uint64(22 - opts.MaxClosedClients) + for _, ci := range conns { + cid++ + if ci.Cid != cid { + t.Fatalf("Expected cid of %d, got %d\n", cid, ci.Cid) + } + } +} + +func TestClosedConnsSubsAccounting(t *testing.T) { + opts := DefaultOptions() + s := RunServer(opts) + defer s.Shutdown() + + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + + nc, err := nats.Connect(url) + if err != nil { + t.Fatalf("Error on subscribe: %v", err) + } + defer nc.Close() + + // Now create some subscriptions + numSubs := 10 + for i := 0; i < numSubs; i++ { + subj := fmt.Sprintf("foo.%d", i) + nc.Subscribe(subj, func(m *nats.Msg) {}) + } + nc.Flush() + nc.Close() + + checkClosedConns(t, s, 1, 20*time.Millisecond) + conns := s.closed.closedClients() + if lc := len(conns); lc != 1 { + t.Fatalf("len(conns) expected to be 1, got %d\n", lc) + } + ci := conns[0] + + if len(ci.subs) != numSubs { + t.Fatalf("Expected number of Subs to be %d, got %d\n", numSubs, len(ci.subs)) + } +} + +func checkReason(t *testing.T, reason string, expected ClosedState) { + if !strings.Contains(reason, expected.String()) { + t.Fatalf("Expected closed connection with `%s` state, got `%s`\n", + expected, reason) + } +} + +func TestClosedAuthorizationTimeout(t *testing.T) { + serverOptions := DefaultOptions() + serverOptions.Authorization = "my_token" + serverOptions.AuthTimeout = 0.4 + s := RunServer(serverOptions) + defer s.Shutdown() + + conn, err := net.Dial("tcp", net.JoinHostPort(serverOptions.Host, fmt.Sprintf("%d", serverOptions.Port))) + if err != nil { + t.Fatalf("Error dialing server: %v\n", err) + } + defer conn.Close() + + checkClosedConns(t, s, 1, 2*time.Second) + conns := s.closed.closedClients() + if lc := len(conns); lc != 1 { + t.Fatalf("len(conns) expected to be %d, got %d\n", 1, lc) + } + checkReason(t, conns[0].Reason, AuthenticationTimeout) +} + +func TestClosedAuthorizationViolation(t *testing.T) { + serverOptions := DefaultOptions() + serverOptions.Authorization = "my_token" + s := RunServer(serverOptions) + defer s.Shutdown() + + opts := s.getOpts() + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + + nc, err := nats.Connect(url) + if err == nil { + nc.Close() + t.Fatal("Expected failure for connection") + } + + checkClosedConns(t, s, 1, 2*time.Second) + conns := s.closed.closedClients() + if lc := len(conns); lc != 1 { + t.Fatalf("len(conns) expected to be %d, got %d\n", 1, lc) + } + checkReason(t, conns[0].Reason, AuthenticationViolation) +} + +func TestClosedUPAuthorizationViolation(t *testing.T) { + serverOptions := DefaultOptions() + serverOptions.Username = "my_user" + serverOptions.Password = "my_secret" + s := RunServer(serverOptions) + defer s.Shutdown() + + opts := s.getOpts() + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + + nc, err := nats.Connect(url) + if err == nil { + nc.Close() + t.Fatal("Expected failure for connection") + } + + url2 := fmt.Sprintf("nats://my_user:wrong_pass@%s:%d", opts.Host, opts.Port) + nc, err = nats.Connect(url2) + if err == nil { + nc.Close() + t.Fatal("Expected failure for connection") + } + + checkClosedConns(t, s, 2, 2*time.Second) + conns := s.closed.closedClients() + if lc := len(conns); lc != 2 { + t.Fatalf("len(conns) expected to be %d, got %d\n", 2, lc) + } + checkReason(t, conns[0].Reason, AuthenticationViolation) + checkReason(t, conns[1].Reason, AuthenticationViolation) +} + +func TestClosedMaxPayload(t *testing.T) { + serverOptions := DefaultOptions() + serverOptions.MaxPayload = 100 + + s := RunServer(serverOptions) + defer s.Shutdown() + + opts := s.getOpts() + endpoint := net.JoinHostPort(opts.Host, fmt.Sprintf("%d", opts.Port)) + + conn, err := net.DialTimeout("tcp", endpoint, time.Second) + if err != nil { + t.Fatalf("Could not make a raw connection to the server: %v", err) + } + defer conn.Close() + + // This should trigger it. + pub := "PUB foo.bar 1024\r\n" + conn.Write([]byte(pub)) + + checkClosedConns(t, s, 1, 2*time.Second) + conns := s.closed.closedClients() + if lc := len(conns); lc != 1 { + t.Fatalf("len(conns) expected to be %d, got %d\n", 1, lc) + } + checkReason(t, conns[0].Reason, MaxPayloadExceeded) +} + +func TestClosedTLSHandshake(t *testing.T) { + opts, err := ProcessConfigFile("./configs/tls.conf") + if err != nil { + t.Fatalf("Error processing config file: %v", err) + } + opts.TLSVerify = true + opts.NoLog = true + opts.NoSigs = true + s := RunServer(opts) + defer s.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port)) + if err == nil { + nc.Close() + t.Fatal("Expected failure for connection") + } + + checkClosedConns(t, s, 1, 2*time.Second) + conns := s.closed.closedClients() + if lc := len(conns); lc != 1 { + t.Fatalf("len(conns) expected to be %d, got %d\n", 1, lc) + } + checkReason(t, conns[0].Reason, TLSHandshakeError) +} diff --git a/server/config_check_test.go b/server/config_check_test.go new file mode 100644 index 0000000..cb39189 --- /dev/null +++ b/server/config_check_test.go @@ -0,0 +1,2636 @@ +// Copyright 2018-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestConfigCheck(t *testing.T) { + tests := []struct { + // name is the name of the test. + name string + + // config is content of the configuration file. + config string + + // warningErr is an error that does not prevent server from starting. + warningErr error + + // errorLine is the location of the error. + errorLine int + + // errorPos is the position of the error. + errorPos int + + // warning errors also include a reason optionally. + reason string + + // newDefaultErr is a configuration error that includes source of error. + err error + }{ + { + name: "when unknown field is used at top level", + config: ` + monitor = "127.0.0.1:4442" + `, + err: errors.New(`unknown field "monitor"`), + errorLine: 2, + errorPos: 17, + }, + { + name: "when default permissions are used at top level", + config: ` + "default_permissions" { + publish = ["_SANDBOX.>"] + subscribe = ["_SANDBOX.>"] + } + `, + err: errors.New(`unknown field "default_permissions"`), + errorLine: 2, + errorPos: 18, + }, + { + name: "when authorization config is empty", + config: ` + authorization = { + } + `, + err: nil, + }, + { + name: "when authorization config has unknown fields", + config: ` + authorization = { + foo = "bar" + } + `, + err: errors.New(`unknown field "foo"`), + errorLine: 3, + errorPos: 5, + }, + { + name: "when authorization config has unknown fields", + config: ` + port = 4222 + + authorization = { + user = "hello" + foo = "bar" + password = "world" + } + + `, + err: errors.New(`unknown field "foo"`), + errorLine: 6, + errorPos: 5, + }, + { + name: "when user authorization config has unknown fields", + config: ` + authorization = { + users = [ + { + user = "foo" + pass = "bar" + token = "quux" + } + ] + } + `, + err: errors.New(`unknown field "token"`), + errorLine: 7, + errorPos: 9, + }, + { + name: "when user authorization permissions config has unknown fields", + config: ` + authorization { + permissions { + subscribe = {} + inboxes = {} + publish = {} + } + } + `, + err: errors.New(`Unknown field "inboxes" parsing permissions`), + errorLine: 5, + errorPos: 7, + }, + { + name: "when user authorization permissions config has unknown fields within allow or deny", + config: ` + authorization { + permissions { + subscribe = { + allow = ["hello", "world"] + deny = ["foo", "bar"] + denied = "_INBOX.>" + } + publish = {} + } + } + `, + err: errors.New(`Unknown field name "denied" parsing subject permissions, only 'allow' or 'deny' are permitted`), + errorLine: 7, + errorPos: 9, + }, + { + name: "when user authorization permissions config has unknown fields within allow or deny", + config: ` + authorization { + permissions { + publish = { + allow = ["hello", "world"] + deny = ["foo", "bar"] + allowed = "_INBOX.>" + } + subscribe = {} + } + } + `, + err: errors.New(`Unknown field name "allowed" parsing subject permissions, only 'allow' or 'deny' are permitted`), + errorLine: 7, + errorPos: 9, + }, + { + name: "when user authorization permissions config has unknown fields using arrays", + config: ` + authorization { + + default_permissions { + subscribe = ["a"] + publish = ["b"] + inboxes = ["c"] + } + + users = [ + { + user = "foo" + pass = "bar" + } + ] + } + `, + err: errors.New(`Unknown field "inboxes" parsing permissions`), + errorLine: 7, + errorPos: 6, + }, + { + name: "when user authorization permissions config has unknown fields using strings", + config: ` + authorization { + + default_permissions { + subscribe = "a" + requests = "b" + publish = "c" + } + + users = [ + { + user = "foo" + pass = "bar" + } + ] + } + `, + err: errors.New(`Unknown field "requests" parsing permissions`), + errorLine: 6, + errorPos: 6, + }, + { + name: "when user authorization permissions config is empty", + config: ` + authorization = { + users = [ + { + user = "foo", pass = "bar", permissions = { + } + } + ] + } + `, + err: nil, + }, + { + name: "when unknown permissions are included in user config", + config: ` + authorization = { + users = [ + { + user = "foo", pass = "bar", permissions { + inboxes = true + } + } + ] + } + `, + err: errors.New(`Unknown field "inboxes" parsing permissions`), + errorLine: 6, + errorPos: 11, + }, + { + name: "when clustering config is empty", + config: ` + cluster = { + } + `, + + err: nil, + }, + { + name: "when unknown option is in clustering config", + config: ` + # NATS Server Configuration + port = 4222 + + cluster = { + + port = 6222 + + foo = "bar" + + authorization { + user = "hello" + pass = "world" + } + + } + `, + + err: errors.New(`unknown field "foo"`), + errorLine: 9, + errorPos: 5, + }, + { + name: "when unknown option is in clustering authorization config", + config: ` + cluster = { + authorization { + foo = "bar" + } + } + `, + + err: errors.New(`unknown field "foo"`), + errorLine: 4, + errorPos: 7, + }, + { + name: "when unknown option is in tls config", + config: ` + tls = { + hello = "world" + } + `, + err: errors.New(`error parsing tls config, unknown field "hello"`), + errorLine: 3, + errorPos: 5, + }, + { + name: "when unknown option is in cluster tls config", + config: ` + cluster { + tls = { + foo = "bar" + } + } + `, + err: errors.New(`error parsing tls config, unknown field "foo"`), + errorLine: 4, + errorPos: 7, + }, + { + name: "when using cipher suites in the TLS config", + config: ` + tls = { + cipher_suites: [ + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" + ] + preferences = [] + } + `, + err: errors.New(`error parsing tls config, unknown field "preferences"`), + errorLine: 7, + errorPos: 7, + }, + { + name: "when using curve preferences in the TLS config", + config: ` + tls = { + curve_preferences: [ + "CurveP256", + "CurveP384", + "CurveP521" + ] + suites = [] + } + `, + err: errors.New(`error parsing tls config, unknown field "suites"`), + errorLine: 8, + errorPos: 7, + }, + { + name: "when using curve preferences in the TLS config", + config: ` + tls = { + curve_preferences: [ + "CurveP5210000" + ] + } + `, + err: errors.New(`unrecognized curve preference CurveP5210000`), + errorLine: 4, + errorPos: 5, + }, + { + name: "verify_cert_and_check_known_urls not support for clients", + config: ` + tls = { + cert_file: "configs/certs/server.pem" + key_file: "configs/certs/key.pem" + verify_cert_and_check_known_urls: true + } + `, + err: errors.New("verify_cert_and_check_known_urls not supported in this context"), + errorLine: 5, + errorPos: 10, + }, + { + name: "when unknown option is in cluster config with defined routes", + config: ` + cluster { + port = 6222 + routes = [ + nats://127.0.0.1:6222 + ] + peers = [] + } + `, + err: errors.New(`unknown field "peers"`), + errorLine: 7, + errorPos: 5, + }, + { + name: "when used as variable in authorization block it should not be considered as unknown field", + config: ` + # listen: 127.0.0.1:-1 + listen: 127.0.0.1:4222 + + authorization { + # Superuser can do anything. + super_user = { + publish = ">" + subscribe = ">" + } + + # Can do requests on foo or bar, and subscribe to anything + # that is a response to an _INBOX. + # + # Notice that authorization filters can be singletons or arrays. + req_pub_user = { + publish = ["req.foo", "req.bar"] + subscribe = "_INBOX.>" + } + + # Setup a default user that can subscribe to anything, but has + # no publish capabilities. + default_user = { + subscribe = "PUBLIC.>" + } + + unused = "hello" + + # Default permissions if none presented. e.g. susan below. + default_permissions: $default_user + + # Users listed with persmissions. + users = [ + {user: alice, password: foo, permissions: $super_user} + {user: bob, password: bar, permissions: $req_pub_user} + {user: susan, password: baz} + ] + } + `, + err: errors.New(`unknown field "unused"`), + errorLine: 27, + errorPos: 5, + }, + { + name: "when used as variable in top level config it should not be considered as unknown field", + config: ` + monitoring_port = 8222 + + http_port = $monitoring_port + + port = 4222 + `, + err: nil, + }, + { + name: "when used as variable in cluster config it should not be considered as unknown field", + config: ` + cluster { + clustering_port = 6222 + port = $clustering_port + } + `, + err: nil, + }, + { + name: "when setting permissions within cluster authorization block", + config: ` + cluster { + authorization { + permissions = { + publish = { allow = ["foo", "bar"] } + } + } + + permissions = { + publish = { deny = ["foo", "bar"] } + } + } + `, + warningErr: errors.New(`invalid use of field "authorization"`), + errorLine: 3, + errorPos: 5, + reason: `setting "permissions" within cluster authorization block is deprecated`, + }, + { + name: "when write deadline is used with deprecated usage", + config: ` + write_deadline = 100 + `, + warningErr: errors.New(`invalid use of field "write_deadline"`), + errorLine: 2, + errorPos: 17, + reason: `write_deadline should be converted to a duration`, + }, + ///////////////////// + // ACCOUNTS // + ///////////////////// + { + name: "when accounts block is correctly configured", + config: ` + http_port = 8222 + + accounts { + + # + # synadia > nats.io, cncf + # + synadia { + # SAADJL5XAEM6BDYSWDTGVILJVY54CQXZM5ZLG4FRUAKB62HWRTPNSGXOHA + nkey = "AC5GRL36RQV7MJ2GT6WQSCKDKJKYTK4T2LGLWJ2SEJKRDHFOQQWGGFQL" + + users [ + { + # SUAEL6RU3BSDAFKOHNTEOK5Q6FTM5FTAMWVIKBET6FHPO4JRII3CYELVNM + nkey = "UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3" + } + ] + + exports = [ + { service: "synadia.requests", accounts: [nats, cncf] } + ] + } + + # + # nats < synadia + # + nats { + # SUAJTM55JH4BNYDA22DMDZJSRBRKVDGSLYK2HDIOCM3LPWCDXIDV5Q4CIE + nkey = "ADRZ42QBM7SXQDXXTSVWT2WLLFYOQGAFC4TO6WOAXHEKQHIXR4HFYJDS" + + users [ + { + # SUADZTYQAKTY5NQM7XRB5XR3C24M6ROGZLBZ6P5HJJSSOFUGC5YXOOECOM + nkey = "UD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI" + } + ] + + imports = [ + # This account has to send requests to 'nats.requests' subject + { service: { account: "synadia", subject: "synadia.requests" }, to: "nats.requests" } + ] + } + + # + # cncf < synadia + # + cncf { + # SAAFHDZX7SGZ2SWHPS22JRPPK5WX44NPLNXQHR5C5RIF6QRI3U65VFY6C4 + nkey = "AD4YRVUJF2KASKPGRMNXTYKIYSCB3IHHB4Y2ME6B2PDIV5QJ23C2ZRIT" + + users [ + { + # SUAKINP3Z2BPUXWOFSW2FZC7TFJCMMU7DHKP2C62IJQUDASOCDSTDTRMJQ + nkey = "UB57IEMPG4KOTPFV5A66QKE2HZ3XBXFHVRCCVMJEWKECMVN2HSH3VTSJ" + } + ] + + imports = [ + # This account has to send requests to 'synadia.requests' subject + { service: { account: "synadia", subject: "synadia.requests" } } + ] + } + } + `, + err: nil, + }, + { + name: "when nkey is invalid within accounts block", + config: ` + accounts { + + # + # synadia > nats.io, cncf + # + synadia { + # SAADJL5XAEM6BDYSWDTGVILJVY54CQXZM5ZLG4FRUAKB62HWRTPNSGXOHA + nkey = "AC5GRL36RQV7MJ2GT6WQSCKDKJKYTK4T2LGLWJ2SEJKRDHFOQQWGGFQL" + + users [ + { + # SUAEL6RU3BSDAFKOHNTEOK5Q6FTM5FTAMWVIKBET6FHPO4JRII3CYELVNM + nkey = "SCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3" + } + ] + + exports = [ + { service: "synadia.requests", accounts: [nats, cncf] } + ] + } + + # + # nats < synadia + # + nats { + # SUAJTM55JH4BNYDA22DMDZJSRBRKVDGSLYK2HDIOCM3LPWCDXIDV5Q4CIE + nkey = "ADRZ42QBM7SXQDXXTSVWT2WLLFYOQGAFC4TO6WOAXHEKQHIXR4HFYJDS" + + users [ + { + # SUADZTYQAKTY5NQM7XRB5XR3C24M6ROGZLBZ6P5HJJSSOFUGC5YXOOECOM + nkey = "UD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI" + } + ] + + imports = [ + # This account has to send requests to 'nats.requests' subject + { service: { account: "synadia", subject: "synadia.requests" }, to: "nats.requests" } + ] + } + + # + # cncf < synadia + # + cncf { + # SAAFHDZX7SGZ2SWHPS22JRPPK5WX44NPLNXQHR5C5RIF6QRI3U65VFY6C4 + nkey = "AD4YRVUJF2KASKPGRMNXTYKIYSCB3IHHB4Y2ME6B2PDIV5QJ23C2ZRIT" + + users [ + { + # SUAKINP3Z2BPUXWOFSW2FZC7TFJCMMU7DHKP2C62IJQUDASOCDSTDTRMJQ + nkey = "UB57IEMPG4KOTPFV5A66QKE2HZ3XBXFHVRCCVMJEWKECMVN2HSH3VTSJ" + } + ] + + imports = [ + # This account has to send requests to 'synadia.requests' subject + { service: { account: "synadia", subject: "synadia.requests" } } + ] + } + } + `, + err: errors.New(`Not a valid public nkey for a user`), + errorLine: 14, + errorPos: 11, + }, + { + name: "when accounts block has unknown fields", + config: ` + http_port = 8222 + + accounts { + foo = "bar" + }`, + err: errors.New(`Expected map entries for accounts`), + errorLine: 5, + errorPos: 19, + }, + { + name: "when accounts has a referenced config variable within same block", + config: ` + accounts { + PERMISSIONS = { + publish = { + allow = ["foo","bar"] + deny = ["quux"] + } + } + + synadia { + nkey = "AC5GRL36RQV7MJ2GT6WQSCKDKJKYTK4T2LGLWJ2SEJKRDHFOQQWGGFQL" + + users [ + { + nkey = "UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3" + permissions = $PERMISSIONS + } + ] + exports = [ + { stream: "synadia.>" } + ] + } + }`, + err: nil, + }, + { + name: "when accounts has an unreferenced config variables within same block", + config: ` + accounts { + PERMISSIONS = { + publish = { + allow = ["foo","bar"] + deny = ["quux"] + } + } + + synadia { + nkey = "AC5GRL36RQV7MJ2GT6WQSCKDKJKYTK4T2LGLWJ2SEJKRDHFOQQWGGFQL" + + users [ + { + nkey = "UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3" + } + ] + exports = [ + { stream: "synadia.>" } + ] + } + }`, + err: errors.New(`unknown field "publish"`), + errorLine: 4, + errorPos: 5, + }, + { + name: "when accounts block defines a global account", + config: ` + http_port = 8222 + + accounts { + $G = { + } + } + `, + err: errors.New(`"$G" is a Reserved Account`), + errorLine: 5, + errorPos: 19, + }, + { + name: "when accounts block uses an invalid public key", + config: ` + accounts { + synadia = { + nkey = "invalid" + } + } + `, + err: errors.New(`Not a valid public nkey for an account: "invalid"`), + errorLine: 4, + errorPos: 21, + }, + { + name: "when accounts list includes reserved account", + config: ` + port = 4222 + + accounts = [foo, bar, "$G"] + + http_port = 8222 + `, + err: errors.New(`"$G" is a Reserved Account`), + errorLine: 4, + errorPos: 26, + }, + { + name: "when accounts list includes a dupe entry", + config: ` + port = 4222 + + accounts = [foo, bar, bar] + + http_port = 8222 + `, + err: errors.New(`Duplicate Account Entry: bar`), + errorLine: 4, + errorPos: 25, + }, + { + name: "when accounts block includes a dupe user", + config: ` + port = 4222 + + accounts = { + nats { + users = [ + { user: "foo", pass: "bar" }, + { user: "hello", pass: "world" }, + { user: "foo", pass: "bar" } + ] + } + } + + http_port = 8222 + `, + err: errors.New(`Duplicate user "foo" detected`), + errorLine: 6, + errorPos: 21, + }, + { + name: "when accounts block imports are not a list", + config: ` + port = 4222 + + accounts = { + nats { + imports = true + } + } + + http_port = 8222 + `, + err: errors.New(`Imports should be an array, got bool`), + errorLine: 6, + errorPos: 21, + }, + { + name: "when accounts block exports are not a list", + config: ` + port = 4222 + + accounts = { + nats { + exports = true + } + } + + http_port = 8222 + `, + err: errors.New(`Exports should be an array, got bool`), + errorLine: 6, + errorPos: 21, + }, + { + name: "when accounts block imports items are not a map", + config: ` + port = 4222 + + accounts = { + nats { + imports = [ + false + ] + } + } + + http_port = 8222 + `, + err: errors.New(`Import Items should be a map with type entry, got bool`), + errorLine: 7, + errorPos: 23, + }, + { + name: "when accounts block export items are not a map", + config: ` + port = 4222 + + accounts = { + nats { + exports = [ + false + ] + } + } + + http_port = 8222 + `, + err: errors.New(`Export Items should be a map with type entry, got bool`), + errorLine: 7, + errorPos: 23, + }, + { + name: "when accounts exports has a stream name that is not a string", + config: ` + port = 4222 + + accounts = { + nats { + exports = [ + { + stream: false + } + ] + } + } + + http_port = 8222 + `, + err: errors.New(`Expected stream name to be string, got bool`), + errorLine: 8, + errorPos: 25, + }, + { + name: "when accounts exports has a service name that is not a string", + config: ` + accounts = { + nats { + exports = [ + { + service: false + } + ] + } + } + `, + err: errors.New(`Expected service name to be string, got bool`), + errorLine: 6, + errorPos: 25, + }, + { + name: "when accounts imports stream without name", + config: ` + port = 4222 + + accounts = { + nats { + imports = [ + { stream: { }} + ] + } + } + + http_port = 8222 + `, + err: errors.New(`Expect an account name and a subject`), + errorLine: 7, + errorPos: 25, + }, + { + name: "when accounts imports service without name", + config: ` + port = 4222 + + accounts = { + nats { + imports = [ + { service: { }} + ] + } + } + + http_port = 8222 + `, + err: errors.New(`Expect an account name and a subject`), + errorLine: 7, + errorPos: 25, + }, + { + name: "when account trace destination is of the wrong type", + config: ` + accounts { + A { trace_dest: 123 } + } + `, + err: errors.New(`Expected account message trace "trace_dest" to be a string or a map/struct, got int64`), + errorLine: 3, + errorPos: 23, + }, + { + name: "when account trace destination is not a valid destination", + config: ` + accounts { + A { trace_dest: "invalid..dest" } + } + `, + err: errors.New(`Trace destination "invalid..dest" is not valid`), + errorLine: 3, + errorPos: 23, + }, + { + name: "when account trace destination is not a valid publish subject", + config: ` + accounts { + A { trace_dest: "invalid.publish.*.subject" } + } + `, + err: errors.New(`Trace destination "invalid.publish.*.subject" is not valid`), + errorLine: 3, + errorPos: 23, + }, + { + name: "when account message trace dest is wrong type", + config: ` + accounts { + A { msg_trace: {dest: 123} } + } + `, + err: errors.New(`Field "dest" should be a string, got int64`), + errorLine: 3, + errorPos: 35, + }, + { + name: "when account message trace dest is invalid", + config: ` + accounts { + A { msg_trace: {dest: "invalid..dest"} } + } + `, + err: errors.New(`Trace destination "invalid..dest" is not valid`), + errorLine: 3, + errorPos: 35, + }, + { + name: "when account message trace sampling is wrong type", + config: ` + accounts { + A { msg_trace: {dest: "acc.dest", sampling: {wront: "type"}} } + } + `, + err: errors.New(`Trace destination sampling field "sampling" should be an integer or a percentage, got map[string]interface {}`), + errorLine: 3, + errorPos: 53, + }, + { + name: "when account message trace sampling is wrong string", + config: ` + accounts { + A { msg_trace: {dest: "acc.dest", sampling: abc%} } + } + `, + err: errors.New(`Invalid trace destination sampling value "abc%"`), + errorLine: 3, + errorPos: 53, + }, + { + name: "when account message trace sampling is negative", + config: ` + accounts { + A { msg_trace: {dest: "acc.dest", sampling: -1} } + } + `, + err: errors.New(`Ttrace destination sampling value -1 is invalid, needs to be [1..100]`), + errorLine: 3, + errorPos: 53, + }, + { + name: "when account message trace sampling is zero", + config: ` + accounts { + A { msg_trace: {dest: "acc.dest", sampling: 0} } + } + `, + err: errors.New(`Ttrace destination sampling value 0 is invalid, needs to be [1..100]`), + errorLine: 3, + errorPos: 53, + }, + { + name: "when account message trace sampling is more than 100", + config: ` + accounts { + A { msg_trace: {dest: "acc.dest", sampling: 101} } + } + `, + err: errors.New(`Ttrace destination sampling value 101 is invalid, needs to be [1..100]`), + errorLine: 3, + errorPos: 53, + }, + { + name: "when account message trace has unknown field", + config: ` + accounts { + A { msg_trace: {wrong: "field"} } + } + `, + err: errors.New(`Unknown field "wrong" parsing account message trace map/struct "msg_trace"`), + errorLine: 3, + errorPos: 35, + }, + { + name: "when user authorization config has both token and users", + config: ` + authorization = { + token = "s3cr3t" + users = [ + { + user = "foo" + pass = "bar" + } + ] + } + `, + err: errors.New(`Can not have a token and a users array`), + errorLine: 2, + errorPos: 3, + }, + { + name: "when user authorization config has both token and user", + config: ` + authorization = { + user = "foo" + pass = "bar" + token = "baz" + } + `, + err: errors.New(`Cannot have a user/pass and token`), + errorLine: 2, + errorPos: 3, + }, + { + name: "when user authorization config has both user and users array", + config: ` + authorization = { + user = "user1" + pass = "pwd1" + users = [ + { + user = "user2" + pass = "pwd2" + } + ] + } + `, + err: errors.New(`Can not have a single user/pass and a users array`), + errorLine: 2, + errorPos: 3, + }, + { + name: "when user authorization has duplicate users", + config: ` + authorization = { + users = [ + {user: "user1", pass: "pwd"} + {user: "user2", pass: "pwd"} + {user: "user1", pass: "pwd"} + ] + } + `, + err: fmt.Errorf(`Duplicate user %q detected`, "user1"), + errorLine: 2, + errorPos: 3, + }, + { + name: "when user authorization has duplicate nkeys", + config: ` + authorization = { + users = [ + {nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX } + {nkey: UBAAQWTW6CG2G6ANGNKB5U2B7HRWHSGMZEZX3AQSAJOQDAUGJD46LD2E } + {nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX } + ] + } + `, + err: fmt.Errorf(`Duplicate nkey %q detected`, "UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX"), + errorLine: 2, + errorPos: 3, + }, + { + name: "when user authorization config has users not as a list", + config: ` + authorization = { + users = false + } + `, + err: errors.New(`Expected users field to be an array, got false`), + errorLine: 3, + errorPos: 5, + }, + { + name: "when user authorization config has users not as a map", + config: ` + authorization = { + users = [false] + } + `, + err: errors.New(`Expected user entry to be a map/struct, got false`), + errorLine: 3, + errorPos: 14, + }, + { + name: "when user authorization config has permissions not as a map", + config: ` + authorization = { + users = [{user: hello, pass: world}] + permissions = false + } + `, + err: errors.New(`Expected permissions to be a map/struct, got false`), + errorLine: 4, + errorPos: 19, + }, + { + name: "when user authorization permissions config has invalid fields within allow", + config: ` + authorization { + permissions { + publish = { + allow = [false, "hello", "world"] + deny = ["foo", "bar"] + } + subscribe = {} + } + } + `, + err: errors.New(`Subject in permissions array cannot be cast to string`), + errorLine: 5, + errorPos: 18, + }, + { + name: "when user authorization permissions config has invalid fields within deny", + config: ` + authorization { + permissions { + publish = { + allow = ["hello", "world"] + deny = [true, "foo", "bar"] + } + subscribe = {} + } + } + `, + err: errors.New(`Subject in permissions array cannot be cast to string`), + errorLine: 6, + errorPos: 17, + }, + { + name: "when user authorization permissions config has invalid type", + config: ` + authorization { + permissions { + publish = { + allow = false + } + subscribe = {} + } + } + `, + err: errors.New(`Expected subject permissions to be a subject, or array of subjects, got bool`), + errorLine: 5, + errorPos: 9, + }, + { + name: "when user authorization permissions subject is invalid", + config: ` + authorization { + permissions { + publish = { + allow = ["foo..bar"] + } + subscribe = {} + } + } + `, + err: errors.New(`subject "foo..bar" is not a valid subject`), + errorLine: 5, + errorPos: 9, + }, + { + name: "when cluster config listen is invalid", + config: ` + cluster { + listen = "0.0.0.0:XXXX" + } + `, + err: errors.New(`could not parse port "XXXX"`), + errorLine: 3, + errorPos: 5, + }, + { + name: "when cluster config includes multiple users", + config: ` + cluster { + authorization { + users = [] + } + } + `, + err: errors.New(`Cluster authorization does not allow multiple users`), + errorLine: 3, + errorPos: 5, + }, + { + name: "when cluster routes are invalid", + config: ` + cluster { + routes = [ + "0.0.0.0:XXXX" + # "0.0.0.0:YYYY" + # "0.0.0.0:ZZZZ" + ] + } + `, + err: errors.New(`error parsing route url ["0.0.0.0:XXXX"]`), + errorLine: 4, + errorPos: 22, + }, + { + name: "when setting invalid TLS config within cluster block", + config: ` + cluster { + tls { + } + } + `, + err: nil, + errorLine: 0, + errorPos: 0, + }, + { + name: "invalid lame_duck_duration type", + config: ` + lame_duck_duration: abc + `, + err: errors.New(`error parsing lame_duck_duration: time: invalid duration`), + errorLine: 2, + errorPos: 5, + }, + { + name: "lame_duck_duration too small", + config: ` + lame_duck_duration: "5s" + `, + err: errors.New(`invalid lame_duck_duration of 5s, minimum is 30 seconds`), + errorLine: 2, + errorPos: 5, + }, + { + name: "invalid lame_duck_grace_period type", + config: ` + lame_duck_grace_period: abc + `, + err: errors.New(`error parsing lame_duck_grace_period: time: invalid duration`), + errorLine: 2, + errorPos: 5, + }, + { + name: "lame_duck_grace_period should be positive", + config: ` + lame_duck_grace_period: "-5s" + `, + err: errors.New(`invalid lame_duck_grace_period, needs to be positive`), + errorLine: 2, + errorPos: 5, + }, + { + name: "when only setting TLS timeout for a leafnode remote", + config: ` + leafnodes { + remotes = [ + { + url: "tls://nats:7422" + tls { + timeout: 0.01 + } + } + ] + }`, + err: nil, + errorLine: 0, + errorPos: 0, + }, + { + name: "verify_cert_and_check_known_urls do not work for leaf nodes", + config: ` + leafnodes { + remotes = [ + { + url: "tls://nats:7422" + tls { + timeout: 0.01 + verify_cert_and_check_known_urls: true + } + } + ] + }`, + //Unexpected error after processing config: /var/folders/9h/6g_c9l6n6bb8gp331d_9y0_w0000gn/T/057996446:8:5: + err: errors.New("verify_cert_and_check_known_urls not supported in this context"), + errorLine: 8, + errorPos: 5, + }, + { + name: "when leafnode remotes use wrong type", + config: ` + leafnodes { + remotes: { + url: "tls://nats:7422" + } + }`, + err: errors.New(`Expected remotes field to be an array, got map[string]interface {}`), + errorLine: 3, + errorPos: 5, + }, + { + name: "when leafnode remotes url uses wrong type", + config: ` + leafnodes { + remotes: [ + { urls: 1234 } + ] + }`, + err: errors.New(`Expected remote leafnode url to be an array or string, got 1234`), + errorLine: 4, + errorPos: 18, + }, + { + name: "when leafnode min_version is wrong type", + config: ` + leafnodes { + port: -1 + min_version = 123 + }`, + err: errors.New(`interface conversion: interface {} is int64, not string`), + errorLine: 4, + errorPos: 6, + }, + { + name: "when leafnode min_version has parsing error", + config: ` + leafnodes { + port: -1 + min_version = bad.version + }`, + err: errors.New(`invalid leafnode's minimum version: invalid semver`), + errorLine: 4, + errorPos: 6, + }, + { + name: "when leafnode min_version is too low", + config: ` + leafnodes { + port: -1 + min_version = 2.7.9 + }`, + err: errors.New(`the minimum version should be at least 2.8.0`), + errorLine: 4, + errorPos: 6, + }, + { + name: "when setting latency tracking with a system account", + config: ` + system_account: sys + + accounts { + sys { users = [ {user: sys, pass: "" } ] } + + nats.io: { + users = [ { user : bar, pass: "" } ] + + exports = [ + { service: "nats.add" + response: singleton + latency: { + sampling: 100% + subject: "latency.tracking.add" + } + } + + ] + } + } + `, + err: nil, + errorLine: 0, + errorPos: 0, + }, + { + name: "when setting latency tracking with an invalid publish subject", + config: ` + system_account = sys + accounts { + sys { users = [ {user: sys, pass: "" } ] } + + nats.io: { + users = [ { user : bar, pass: "" } ] + + exports = [ + { service: "nats.add" + response: singleton + latency: "*" + } + ] + } + } + `, + err: errors.New(`Error adding service latency sampling for "nats.add" on subject "*": invalid publish subject`), + errorLine: 3, + errorPos: 17, + }, + { + name: "when setting latency tracking on a stream", + config: ` + system_account = sys + accounts { + sys { users = [ {user: sys, pass: "" } ] } + + nats.io: { + users = [ { user : bar, pass: "" } ] + + exports = [ + { stream: "nats.add" + latency: "foo" + } + ] + } + } + `, + err: errors.New(`Detected latency directive on non-service`), + errorLine: 11, + errorPos: 25, + }, + { + name: "when setting allow_trace on a stream export (after)", + config: ` + system_account = sys + accounts { + sys { users = [ {user: sys, pass: "" } ] } + + nats.io: { + users = [ { user : bar, pass: "" } ] + exports = [ { stream: "nats.add", allow_trace: true } ] + } + } + `, + err: errors.New(`Detected allow_trace directive on non-service`), + errorLine: 8, + errorPos: 55, + }, + { + name: "when setting allow_trace on a stream export (before)", + config: ` + system_account = sys + accounts { + sys { users = [ {user: sys, pass: "" } ] } + + nats.io: { + users = [ { user : bar, pass: "" } ] + exports = [ { allow_trace: true, stream: "nats.add" } ] + } + } + `, + err: errors.New(`Detected allow_trace directive on non-service`), + errorLine: 8, + errorPos: 35, + }, + { + name: "when setting allow_trace on a service import (after)", + config: ` + accounts { + A: { + users = [ {user: user1, pass: ""} ] + exports = [{service: "foo"}] + } + B: { + users = [ {user: user2, pass: ""} ] + imports = [ { service: {account: "A", subject: "foo"}, allow_trace: true } ] + } + } + `, + err: errors.New(`Detected allow_trace directive on a non-stream`), + errorLine: 9, + errorPos: 76, + }, + { + name: "when setting allow_trace on a service import (before)", + config: ` + accounts { + A: { + users = [ {user: user1, pass: ""} ] + exports = [{service: "foo"}] + } + B: { + users = [ {user: user2, pass: ""} ] + imports = [ { allow_trace: true, service: {account: "A", subject: "foo"} } ] + } + } + `, + err: errors.New(`Detected allow_trace directive on a non-stream`), + errorLine: 9, + errorPos: 35, + }, + { + name: "when using duplicate service import subject", + config: ` + accounts { + A: { + users = [ {user: user1, pass: ""} ] + exports = [ + {service: "remote1"} + {service: "remote2"} + ] + } + B: { + users = [ {user: user2, pass: ""} ] + imports = [ + {service: {account: "A", subject: "remote1"}, to: "local"} + {service: {account: "A", subject: "remote2"}, to: "local"} + ] + } + } + `, + err: errors.New(`Duplicate service import subject "local", previously used in import for account "A", subject "remote1"`), + errorLine: 14, + errorPos: 71, + }, + { + name: "mixing single and multi users in leafnode authorization", + config: ` + leafnodes { + authorization { + user: user1 + password: pwd + users = [{user: user2, password: pwd}] + } + } + `, + err: errors.New("can not have a single user/pass and a users array"), + errorLine: 3, + errorPos: 20, + }, + { + name: "duplicate usernames in leafnode authorization", + config: ` + leafnodes { + authorization { + users = [ + {user: user, password: pwd} + {user: user, password: pwd} + ] + } + } + `, + err: errors.New(`duplicate user "user" detected in leafnode authorization`), + errorLine: 3, + errorPos: 21, + }, + { + name: "mqtt bad type", + config: ` + mqtt [ + "wrong" + ] + `, + err: errors.New(`Expected mqtt to be a map, got []interface {}`), + errorLine: 2, + errorPos: 17, + }, + { + name: "mqtt bad listen", + config: ` + mqtt { + listen: "xxxxxxxx" + } + `, + err: errors.New(`could not parse address string "xxxxxxxx"`), + errorLine: 3, + errorPos: 21, + }, + { + name: "mqtt bad host", + config: ` + mqtt { + host: 1234 + } + `, + err: errors.New(`interface conversion: interface {} is int64, not string`), + errorLine: 3, + errorPos: 21, + }, + { + name: "mqtt bad port", + config: ` + mqtt { + port: "abc" + } + `, + err: errors.New(`interface conversion: interface {} is string, not int64`), + errorLine: 3, + errorPos: 21, + }, + { + name: "mqtt bad TLS", + config: ` + mqtt { + port: -1 + tls { + cert_file: "./configs/certs/server.pem" + } + } + `, + err: errors.New(`missing 'key_file' in TLS configuration`), + errorLine: 4, + errorPos: 21, + }, + { + name: "connection types wrong type", + config: ` + authorization { + users [ + {user: a, password: pwd, allowed_connection_types: 123} + ] + } + `, + err: errors.New(`error parsing allowed connection types: unsupported type int64`), + errorLine: 4, + errorPos: 53, + }, + { + name: "connection types content wrong type", + config: ` + authorization { + users [ + {user: a, password: pwd, allowed_connection_types: [ + 123 + WEBSOCKET + ]} + ] + } + `, + err: errors.New(`error parsing allowed connection types: unsupported type in array int64`), + errorLine: 5, + errorPos: 32, + }, + { + name: "connection types type unknown", + config: ` + authorization { + users [ + {user: a, password: pwd, allowed_connection_types: [ "UNKNOWN" ]} + ] + } + `, + err: fmt.Errorf("invalid connection types [%q]", "UNKNOWN"), + errorLine: 4, + errorPos: 53, + }, + { + name: "websocket auth unknown var", + config: ` + websocket { + authorization { + unknown: "field" + } + } + `, + err: fmt.Errorf("unknown field %q", "unknown"), + errorLine: 4, + errorPos: 25, + }, + { + name: "websocket bad tls", + config: ` + websocket { + tls { + cert_file: "configs/certs/server.pem" + } + } + `, + err: fmt.Errorf("missing 'key_file' in TLS configuration"), + errorLine: 3, + errorPos: 21, + }, + { + name: "verify_cert_and_check_known_urls not support for websockets", + config: ` + websocket { + tls { + cert_file: "configs/certs/server.pem" + key_file: "configs/certs/key.pem" + verify_cert_and_check_known_urls: true + } + } + `, + err: fmt.Errorf("verify_cert_and_check_known_urls not supported in this context"), + errorLine: 6, + errorPos: 10, + }, + { + name: "ambiguous store dir", + config: ` + store_dir: "foo" + jetstream { + store_dir: "bar" + } + `, + err: fmt.Errorf(`Duplicate 'store_dir' configuration`), + }, + { + name: "token not supported in cluster", + config: ` + cluster { + port: -1 + authorization { + token: "my_token" + } + } + `, + err: fmt.Errorf("Cluster authorization does not support tokens"), + errorLine: 4, + errorPos: 6, + }, + { + name: "token not supported in gateway", + config: ` + gateway { + port: -1 + name: "A" + authorization { + token: "my_token" + } + } + `, + err: fmt.Errorf("Gateway authorization does not support tokens"), + errorLine: 5, + errorPos: 6, + }, + { + name: "wrong type for cluster pool size", + config: ` + cluster { + port: -1 + pool_size: "abc" + } + `, + err: fmt.Errorf("interface conversion: interface {} is string, not int64"), + errorLine: 4, + errorPos: 6, + }, + { + name: "wrong type for cluster accounts", + config: ` + cluster { + port: -1 + accounts: 123 + } + `, + err: fmt.Errorf("error parsing accounts: unsupported type int64"), + errorLine: 4, + errorPos: 6, + }, + { + name: "wrong type for cluster compression", + config: ` + cluster { + port: -1 + compression: 123 + } + `, + err: fmt.Errorf("field %q should be a boolean or a structure, got int64", "compression"), + errorLine: 4, + errorPos: 6, + }, + { + name: "wrong type for cluster compression mode", + config: ` + cluster { + port: -1 + compression: { + mode: 123 + } + } + `, + err: fmt.Errorf("interface conversion: interface {} is int64, not string"), + errorLine: 5, + errorPos: 7, + }, + { + name: "wrong type for cluster compression rtt thresholds", + config: ` + cluster { + port: -1 + compression: { + mode: "s2_auto" + rtt_thresholds: 123 + } + } + `, + err: fmt.Errorf("interface conversion: interface {} is int64, not []interface {}"), + errorLine: 6, + errorPos: 7, + }, + { + name: "invalid durations for cluster compression rtt thresholds", + config: ` + cluster { + port: -1 + compression: { + mode: "s2_auto" + rtt_thresholds: [abc] + } + } + `, + err: fmt.Errorf("time: invalid duration %q", "abc"), + errorLine: 6, + errorPos: 7, + }, + { + name: "invalid durations for cluster ping interval", + config: ` + cluster { + port: -1 + ping_interval: -1 + ping_max: 6 + } + `, + err: fmt.Errorf(`invalid use of field "ping_interval": ping_interval should be converted to a duration`), + errorLine: 4, + errorPos: 6, + }, + { + name: "invalid durations for cluster ping interval", + config: ` + cluster { + port: -1 + ping_interval: '2m' + ping_max: 6 + } + `, + warningErr: fmt.Errorf(`Cluster 'ping_interval' will reset to 30s which is the max for routes`), + errorLine: 4, + errorPos: 6, + }, + { + name: "wrong type for leafnodes compression", + config: ` + leafnodes { + port: -1 + compression: 123 + } + `, + err: fmt.Errorf("field %q should be a boolean or a structure, got int64", "compression"), + errorLine: 4, + errorPos: 6, + }, + { + name: "wrong type for leafnodes compression mode", + config: ` + leafnodes { + port: -1 + compression: { + mode: 123 + } + } + `, + err: fmt.Errorf("interface conversion: interface {} is int64, not string"), + errorLine: 5, + errorPos: 7, + }, + { + name: "wrong type for leafnodes compression rtt thresholds", + config: ` + leafnodes { + port: -1 + compression: { + mode: "s2_auto" + rtt_thresholds: 123 + } + } + `, + err: fmt.Errorf("interface conversion: interface {} is int64, not []interface {}"), + errorLine: 6, + errorPos: 7, + }, + { + name: "invalid durations for leafnodes compression rtt thresholds", + config: ` + leafnodes { + port: -1 + compression: { + mode: "s2_auto" + rtt_thresholds: [abc] + } + } + `, + err: fmt.Errorf("time: invalid duration %q", "abc"), + errorLine: 6, + errorPos: 7, + }, + { + name: "wrong type for remote leafnodes compression", + config: ` + leafnodes { + port: -1 + remotes [ + { + url: "nats://127.0.0.1:123" + compression: 123 + } + ] + } + `, + err: fmt.Errorf("field %q should be a boolean or a structure, got int64", "compression"), + errorLine: 7, + errorPos: 8, + }, + { + name: "wrong type for remote leafnodes compression mode", + config: ` + leafnodes { + port: -1 + remotes [ + { + url: "nats://127.0.0.1:123" + compression: { + mode: 123 + } + } + ] + } + `, + err: fmt.Errorf("interface conversion: interface {} is int64, not string"), + errorLine: 8, + errorPos: 9, + }, + { + name: "wrong type for remote leafnodes compression rtt thresholds", + config: ` + leafnodes { + port: -1 + remotes [ + { + url: "nats://127.0.0.1:123" + compression: { + mode: "s2_auto" + rtt_thresholds: 123 + } + } + ] + } + `, + err: fmt.Errorf("interface conversion: interface {} is int64, not []interface {}"), + errorLine: 9, + errorPos: 9, + }, + { + name: "invalid durations for remote leafnodes compression rtt thresholds", + config: ` + leafnodes { + port: -1 + remotes [ + { + url: "nats://127.0.0.1:123" + compression: { + mode: "s2_auto" + rtt_thresholds: [abc] + } + } + ] + } + `, + err: fmt.Errorf("time: invalid duration %q", "abc"), + errorLine: 9, + errorPos: 9, + }, + { + name: "invalid duration for remote leafnode first info timeout", + config: ` + leafnodes { + port: -1 + remotes [ + { + url: "nats://127.0.0.1:123" + first_info_timeout: abc + } + ] + } + `, + err: fmt.Errorf("error parsing first_info_timeout: time: invalid duration %q", "abc"), + errorLine: 7, + errorPos: 8, + }, + { + name: "duplicate remote leafnode", + config: ` + leafnodes { + port: -1 + remotes [ + { url: "nats://user1:pwd@127.0.0.1:1234" } + { url: "nats://user2:pwd@127.0.0.1:1235" } + { url: "nats://user1:pwd@127.0.0.1:1234" } + ] + } + `, + err: fmt.Errorf("duplicate remote"), + errorLine: 7, + errorPos: 9, + }, + { + name: "show warnings on empty configs without values", + config: ``, + warningErr: errors.New(`config has no values or is empty`), + errorLine: 0, + errorPos: 0, + reason: "", + }, + { + name: "show warnings on empty configs without values and only comments", + config: `# Valid file but has no usable values. + `, + warningErr: errors.New(`config has no values or is empty`), + errorLine: 0, + errorPos: 0, + reason: "", + }, + { + name: "TLS handshake first, wrong type", + config: ` + port: -1 + tls { + first: 123 + } + `, + err: fmt.Errorf("field %q should be a boolean or a string, got int64", "first"), + errorLine: 4, + errorPos: 6, + }, + { + name: "TLS handshake first, wrong value", + config: ` + port: -1 + tls { + first: "123" + } + `, + err: fmt.Errorf("field %q's value %q is invalid", "first", "123"), + errorLine: 4, + errorPos: 6, + }, + { + name: "TLS multiple certs", + config: ` + port: -1 + tls { + certs: [ + { cert_file: "configs/certs/server.pem", key_file: "configs/certs/key.pem"}, + { cert_file: "configs/certs/cert.new.pem", key_file: "configs/certs/key.new.pem"}, + ] + } + `, + err: nil, + }, + { + name: "TLS multiple certs, bad type", + config: ` + port: -1 + tls { + certs: [ + { cert_file: "configs/certs/server.pem", key_file: 123 }, + { cert_file: "configs/certs/cert.new.pem", key_file: "configs/certs/key.new.pem"}, + ] + } + `, + err: fmt.Errorf("error parsing certificates config: unsupported type int64"), + errorLine: 5, + errorPos: 49, + }, + { + name: "TLS multiple certs, missing key_file", + config: ` + port: -1 + tls { + certs: [ + { cert_file: "configs/certs/server.pem" } + { cert_file: "configs/certs/cert.new.pem", key_file: "configs/certs/key.new.pem"} + ] + } + `, + err: fmt.Errorf("error parsing certificates config: both 'cert_file' and 'cert_key' options are required"), + errorLine: 5, + errorPos: 10, + }, + { + name: "TLS multiple certs and single cert options at the same time", + config: ` + port: -1 + tls { + cert_file: "configs/certs/server.pem" + key_file: "configs/certs/key.pem" + certs: [ + { cert_file: "configs/certs/server.pem", key_file: "configs/certs/key.pem"}, + { cert_file: "configs/certs/cert.new.pem", key_file: "configs/certs/key.new.pem"}, + ] + } + `, + err: fmt.Errorf("error parsing tls config, cannot combine 'cert_file' option with 'certs' option"), + errorLine: 3, + errorPos: 5, + }, + { + name: "TLS multiple certs used but not configured, but cert_file configured", + config: ` + port: -1 + tls { + cert_file: "configs/certs/server.pem" + key_file: "configs/certs/key.pem" + certs: [] + } + `, + err: nil, + }, + { + name: "TLS multiple certs, missing bad path", + config: ` + port: -1 + tls { + certs: [ + { cert_file: "configs/certs/cert.new.pem", key_file: "configs/certs/key.new.pem"} + { cert_file: "configs/certs/server.pem", key_file: "configs/certs/key.new.pom" } + ] + } + `, + err: fmt.Errorf("error parsing X509 certificate/key pair 2/2: open configs/certs/key.new.pom: no such file or directory"), + errorLine: 3, + errorPos: 5, + }, + { + name: "Proxies wrong type", + config: ` + port: -1 + proxies: 123 + `, + err: errors.New("expected proxies to be a map/struct, got int64"), + errorLine: 3, + errorPos: 5, + }, + { + name: "Proxies unknown field", + config: ` + port: -1 + proxies: { + field1: 123 + } + `, + err: errors.New("unknown field"), + errorLine: 4, + errorPos: 6, + }, + { + name: "Proxies trusted wrong type", + config: ` + port: -1 + proxies: { + trusted: 123 + } + `, + err: errors.New("expected proxies' trusted field to be an array, got int64"), + errorLine: 4, + errorPos: 6, + }, + { + name: "Proxies trusted key wrong type", + config: ` + port: -1 + proxies: { + trusted: [ + "abc", + "def" + ] + } + `, + err: errors.New("expected proxies' trusted entry to be a map/struct, got string"), + errorLine: 5, + errorPos: 8, + }, + { + name: "Proxies trusted unknown field", + config: ` + port: -1 + proxies: { + trusted: [ + { + key: "UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3" + field1: 123 + } + ] + } + `, + err: errors.New("unknown field"), + errorLine: 7, + errorPos: 8, + }, + { + name: "Proxies trusted key invalid", + config: ` + port: -1 + proxies: { + trusted: [ + {key: "UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3"} + {key: "bad1"} + {key: "UD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI"} + {key: "bad2"} + ] + } + `, + err: errors.New("invalid proxy key"), + errorLine: 6, + errorPos: 8, + }, + { + name: "Auth user require proxied wrong type", + config: ` + port: -1 + authorization { + user: user + password: pwd + proxy_required: 123 + } + `, + err: errors.New("interface conversion"), + errorLine: 6, + errorPos: 6, + }, + { + name: "Auth users require proxied wrong type", + config: ` + port: -1 + authorization { + users: [ + { + user: user + password: pwd + proxy_required: 123 + } + ] + } + `, + err: errors.New("interface conversion"), + errorLine: 8, + errorPos: 8, + }, + { + name: "Auth nkey users require proxied wrong type", + config: ` + port: -1 + authorization { + users: [ + { + nkey: "UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3" + proxy_required: 123 + } + ] + } + `, + err: errors.New("interface conversion"), + errorLine: 7, + errorPos: 8, + }, + { + name: "Account users require proxied wrong type", + config: ` + port: -1 + accounts { + A: { + users: [ + { + user: user + password: pwd + proxy_required: 123 + } + ] + } + } + `, + err: errors.New("interface conversion"), + errorLine: 9, + errorPos: 9, + }, + { + name: "Account nkey users require proxied wrong type", + config: ` + port: -1 + accounts { + A: { + users: [ + { + nkey: "UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3" + proxy_required: 123 + } + ] + } + } + `, + err: errors.New("interface conversion"), + errorLine: 8, + errorPos: 9, + }, + { + name: "Leafnode user require proxied wrong type", + config: ` + port: -1 + leafnodes { + port: -1 + authorization { + user: user + password: pwd + proxy_required: 123 + } + } + `, + err: errors.New("interface conversion"), + errorLine: 8, + errorPos: 7, + }, + { + name: "Leafnode neky user require proxied wrong type", + config: ` + port: -1 + leafnodes { + port: -1 + authorization { + nkey: "UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3" + proxy_required: 123 + } + } + `, + err: errors.New("interface conversion"), + errorLine: 7, + errorPos: 7, + }, + { + name: "Leafnode users require proxied wrong type", + config: ` + port: -1 + leafnodes { + port: -1 + authorization { + users: [ + { + user: user + password: pwd + proxy_required: 123 + } + ] + } + } + `, + err: errors.New("interface conversion"), + errorLine: 10, + errorPos: 9, + }, + { + name: "Leafnode nkey users require proxied wrong type", + config: ` + port: -1 + leafnodes { + port: -1 + authorization { + users: [ + { + nkey: "UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3" + proxy_required: 123 + } + ] + } + } + `, + err: errors.New("interface conversion"), + errorLine: 9, + errorPos: 9, + }, + { + name: "leafnode proxy with unsupported scheme", + config: ` + leafnodes { + remotes = [ + { + url: "ws://127.0.0.1:7422" + proxy { + url: "ftp://proxy.example.com:8080" + } + } + ] + } + `, + err: errors.New("proxy URL scheme must be http or https, got: ftp"), + errorLine: 6, + errorPos: 8, + }, + { + name: "leafnode proxy with missing host", + config: ` + leafnodes { + remotes = [ + { + url: "ws://127.0.0.1:7422" + proxy { + url: "http://" + } + } + ] + } + `, + err: errors.New("proxy URL must specify a host"), + errorLine: 6, + errorPos: 8, + }, + { + name: "leafnode proxy with username but no password", + config: ` + leafnodes { + remotes = [ + { + url: "ws://127.0.0.1:7422" + proxy { + url: "http://proxy.example.com:8080" + username: "testuser" + } + } + ] + } + `, + err: errors.New("proxy username and password must both be specified or both be empty"), + errorLine: 6, + errorPos: 8, + }, + { + name: "leafnode proxy with password but no username", + config: ` + leafnodes { + remotes = [ + { + url: "ws://127.0.0.1:7422" + proxy { + url: "http://proxy.example.com:8080" + password: "testpass" + } + } + ] + } + `, + err: errors.New("proxy username and password must both be specified or both be empty"), + errorLine: 6, + errorPos: 8, + }, + { + name: "leafnode proxy with WSS URL but no TLS config", + config: ` + leafnodes { + remotes = [ + { + url: "wss://127.0.0.1:7422" + proxy { + url: "http://proxy.example.com:8080" + } + } + ] + } + `, + err: errors.New("proxy is configured but remote URL wss://127.0.0.1:7422 requires TLS and no TLS configuration is provided"), + errorLine: 6, + errorPos: 8, + }, + } + + checkConfig := func(config string) error { + opts := &Options{ + CheckConfig: true, + } + return opts.ProcessConfigFile(config) + } + + checkErr := func(t *testing.T, err, expectedErr error) { + t.Helper() + switch { + case err == nil && expectedErr == nil: + // OK + case err != nil && expectedErr == nil: + t.Errorf("Unexpected error after processing config: %s", err) + case err == nil && expectedErr != nil: + t.Errorf("Expected %q error after processing invalid config but got nothing", expectedErr) + } + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + conf := createConfFile(t, []byte(test.config)) + err := checkConfig(conf) + var expectedErr error + + // Check for either warnings or errors. + if test.err != nil { + expectedErr = test.err + } else if test.warningErr != nil { + expectedErr = test.warningErr + } + + if err != nil && expectedErr != nil { + var msg string + + if test.errorPos > 0 { + msg = fmt.Sprintf("%s:%d:%d: %s", conf, test.errorLine, test.errorPos, expectedErr.Error()) + if test.reason != "" { + msg += ": " + test.reason + } + } else if test.warningErr != nil { + msg = expectedErr.Error() + } else { + msg = test.reason + } + + if !strings.Contains(err.Error(), msg) { + t.Errorf("Expected:\n%q\ngot:\n%q", msg, err.Error()) + } + } + + checkErr(t, err, expectedErr) + }) + } +} + +func TestConfigCheckIncludes(t *testing.T) { + // Check happy path first. + opts := &Options{ + CheckConfig: true, + } + err := opts.ProcessConfigFile("./configs/include_conf_check_a.conf") + if err != nil { + t.Errorf("Unexpected error processing include files with configuration check enabled: %v", err) + } + + opts = &Options{ + CheckConfig: true, + } + err = opts.ProcessConfigFile("./configs/include_bad_conf_check_a.conf") + if err == nil { + t.Errorf("Expected error processing include files with configuration check enabled: %v", err) + } + expectedErr := `include_bad_conf_check_b.conf:10:19: unknown field "monitoring_port"` + "\n" + if err != nil && !strings.HasSuffix(err.Error(), expectedErr) { + t.Errorf("Expected: \n%q, got\n: %q", expectedErr, err.Error()) + } +} + +func TestConfigCheckMultipleErrors(t *testing.T) { + opts := &Options{ + CheckConfig: true, + } + err := opts.ProcessConfigFile("./configs/multiple_errors.conf") + if err == nil { + t.Errorf("Expected error processing config files with multiple errors check enabled: %v", err) + } + cerr, ok := err.(*processConfigErr) + if !ok { + t.Fatalf("Expected a configuration process error") + } + got := len(cerr.Warnings()) + expected := 1 + if got != expected { + t.Errorf("Expected a %d warning, got: %d", expected, got) + } + got = len(cerr.Errors()) + // Could be 7 or 8 errors depending on internal ordering of the parsing. + if got != 7 && got != 8 { + t.Errorf("Expected 7 or 8 errors, got: %d", got) + } + + errMsg := err.Error() + + errs := []string{ + `./configs/multiple_errors.conf:12:1: invalid use of field "write_deadline": write_deadline should be converted to a duration`, + `./configs/multiple_errors.conf:2:1: Cannot have a user/pass and token`, + `./configs/multiple_errors.conf:10:1: unknown field "monitoring"`, + `./configs/multiple_errors.conf:67:3: Cluster authorization does not allow multiple users`, + `./configs/multiple_errors.conf:21:5: Not a valid public nkey for an account: "OC5GRL36RQV7MJ2GT6WQSCKDKJKYTK4T2LGLWJ2SEJKRDHFOQQWGGFQL"`, + `./configs/multiple_errors.conf:26:9: Not a valid public nkey for a user`, + `./configs/multiple_errors.conf:36:5: Not a valid public nkey for an account: "ODRZ42QBM7SXQDXXTSVWT2WLLFYOQGAFC4TO6WOAXHEKQHIXR4HFYJDS"`, + `./configs/multiple_errors.conf:41:9: Not a valid public nkey for a user`, + } + for _, msg := range errs { + found := strings.Contains(errMsg, msg) + if !found { + t.Fatalf("Expected to find error %q", msg) + } + } + if got == 8 { + extra := "./configs/multiple_errors.conf:54:5: Can not have a single user/pass and accounts" + if !strings.Contains(errMsg, extra) { + t.Fatalf("Expected to find error %q (%s)", extra, errMsg) + } + } +} diff --git a/server/configs/certs/cert.new.pem b/server/configs/certs/cert.new.pem new file mode 100644 index 0000000..e4c2962 --- /dev/null +++ b/server/configs/certs/cert.new.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDnjCCAoagAwIBAgIJAM/HacKKaH7zMA0GCSqGSIb3DQEBCwUAMIGDMQswCQYD +VQQGEwJVUzELMAkGA1UECAwCQ0ExFDASBgNVBAcMC0xvcyBBbmdlbGVzMSQwIgYD +VQQKDBtTeW5hZGlhIENvbW11bmljYXRpb25zIEluYy4xKzApBgNVBAMMIm5hdHMu +aW8vZW1haWxBZGRyZXNzPWRlcmVrQG5hdHMuaW8wHhcNMTkxMDE3MTIyODIxWhcN +MjkxMDE0MTIyODIxWjCBgzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRQwEgYD +VQQHDAtMb3MgQW5nZWxlczEkMCIGA1UECgwbU3luYWRpYSBDb21tdW5pY2F0aW9u +cyBJbmMuMSswKQYDVQQDDCJuYXRzLmlvL2VtYWlsQWRkcmVzcz1kZXJla0BuYXRz +LmlvMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvBIAVUr4gCEBRVbA +HSweCLExOrMZmII4AdvgIqT+svjBkJd+vdkbd5b/SC1HQx1E14kiRJ/JrZIZoMqi ++7pK3kFM63Fkhkg8rWOxn0tQznSymKTpha5NdDWxnB0dXlXFCQG1e/cuDalR7UhF +LPHiuK42gAvhivBcymDPV0hTYt4rHb71SQ1DwfCYzcLkDvDFA/W7kronaEhRyWn6 +uvZvHkdvScoubdzoW/kNBH4JYZw5svLzGz3z20rUGeLttF4ge5SCAz9unZk96HUO +EFmUDvFmxdnTXrYINjraNvgu7fPFmVzupWrWPA/7U+cxOvm3qBMdqxFxNr7bq+Md +UPKi3QIDAQABoxMwETAPBgNVHREECDAGhwR/AAABMA0GCSqGSIb3DQEBCwUAA4IB +AQCCgFKP/bMi5cvasJOzXKLpwOneW+oItL2t/5NxPumIBMDo5NnShzsFLfGmujYw +fjcMharynkkbz/oeDAm8h1mySAJrnqtabiWgaW8zbNJfJOrAq4Jvs9COMAzjJclL ++h9GWtvVylDnsNtd18n1gA5OYv0A6YjuSrWINL8Sp5QvTF/5tT8jFrDOIjZl7m50 +lX4R70N9GLt2jIlKro+qdsi6qUZccuJmoQxUpG1iQcRNFHtWfDPr5KFEXaO6IoYt +D1kYWmo/A3WQm7nbXeZw/zaSGSS0t6/hKZwm+gPCL6TEdDrjhjxpCZQTwaMd6jj2 +bvT2OA0ZpzUWyxnaX6u+cDfM +-----END CERTIFICATE----- diff --git a/server/configs/certs/key.new.pem b/server/configs/certs/key.new.pem new file mode 100644 index 0000000..532c8e4 --- /dev/null +++ b/server/configs/certs/key.new.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC8EgBVSviAIQFF +VsAdLB4IsTE6sxmYgjgB2+AipP6y+MGQl3692Rt3lv9ILUdDHUTXiSJEn8mtkhmg +yqL7ukreQUzrcWSGSDytY7GfS1DOdLKYpOmFrk10NbGcHR1eVcUJAbV79y4NqVHt +SEUs8eK4rjaAC+GK8FzKYM9XSFNi3isdvvVJDUPB8JjNwuQO8MUD9buSuidoSFHJ +afq69m8eR29Jyi5t3Ohb+Q0EfglhnDmy8vMbPfPbStQZ4u20XiB7lIIDP26dmT3o +dQ4QWZQO8WbF2dNetgg2Oto2+C7t88WZXO6latY8D/tT5zE6+beoEx2rEXE2vtur +4x1Q8qLdAgMBAAECggEBAI0zJrTi2Le6D+4zN4Gvgc0c280qcmkiO1KJ9jmMjv1n +7kvUsf9vZUPgRkG2XO5ypyD7gJLtNMnwCvXBraQ5NcSwWkPampKG4ad8Vfs23LBk +xUH9bqZDOzuopHSFF2ugEZK1icBM0HLJUQ1JWUZpRMNLaPex8+AQnloDXSg20QRE +Z53CDYZVEVaLAYIkfgZ6QgBSbJ0ob/a+ZJkChWs3f6cpNHige4tR2RQ0aJFwvlwu +eC8G3RDNzUMhPf4TSf/XUlHXngyHKa7MS3feEtuKIQ4gb8hjESDcZznJUGG9qyMT +7445mapc60t8z46o9iVeVJQmQxmjFUDkLcZ9ckcWfzECgYEA4OiolNUiTnXbvP0H +3rxAnhx9GlyHiRaIZvgsAHH/6aMbJIXnH89cAR38PL79OaQpWZqK8Z1vrCHC8q7s +Wb1RehgnwPxtTl4RTZ0wj3Ri5BIq32nzgla0w/5RAmAw9HrYXjp4TR4Quc+bHLcr +ZM2gA8nZ4PG73rYvd/Mu3lCQiS8CgYEA1hGoaduGG7eHTMCct/kCupFfYXgEuUDh +V/nO1SYJbIkMuSONNaQy1oUc5o0BB/JTcslZWqtuGa9snKPVd3/rJz75YGt1QR6u +MukCYjbWGAWXfDh8G8aGSr3k1k3JmKNE7fI7L5DuvKCGoGjzzC+kclyoij8jwXk0 +zghaYPvM+bMCgYEA3/fwgR3p5vZJF19mqfEP7CP0lP7V3bd5qAi1UNA1h4VsrydF +LRFCzr38hMWwx+jpYJicitU78s9AIon9RbRY4dwSIoV9mE/mrUK+q+y72eEZnpgU +7ZPIuXCVXWdK+PsoYlWZnTo2b8ME9UiWxvBZy8wD05UGgFcu2CVsY+kYtfMCgYAs +Vn/xXPyL5Rlq9kH/gN3l0pJU18zyqdOCq0UBtN0i08gE2K44vAejkvKHdhEOmkxa +bAXL19H4E/OFBhICrEYCXPK928PvdvFRrh1GRmFVnGLh4bki752FAYvSL05gBQET +36YOlhA8lWsM8m/8jKmc3kAyUh2PxxD+05AUolK0LQKBgHqPgMj5rfiR7MoVIaXr +K8h7g+HejppmCxzENc98hvIQIO2aWeC4Hi/EMe+XYSN/Gl6d5+aw80JYRgIv6dFV +hDEEt8lEI4sFhKD0oG1dQo+BdDNXOLbpHq+RQRPHa8J/pmSjI5Joqb2mCaAcm/KJ +nYE20ox06YMSb+ZrjP2p4qw3 +-----END PRIVATE KEY----- diff --git a/server/configs/certs/key.pem b/server/configs/certs/key.pem new file mode 100644 index 0000000..64b9bea --- /dev/null +++ b/server/configs/certs/key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDc8YaxliKNsa8M +idwQsDXS0MI4z+ZP1UMW9U/Vt2qTienYs3L6KqezRfwEH2AoE7XyvQdzePV3aN1g +A0L3mAKiS2FN3DhxR6Y8D/JNb5YA6Pl0flaMKnWktma/SMOf7o7eS7Zt6ZmIvtBg +p1ZtRSZSH64KSqC467eGByHnXm25Z7Uf/2rO8K1jsmRwgDEJoJL5Z4va4pypX5Kq +7OxWk8Za50Kuk6DtdRfIcnUjoDadsYzGHzF7kp1/31elQ2aUPp9e6X3aWu++gJv5 +brBMsWeQ+ETmjCMPqxoXDM2AJqwP73irELU7POUR/q3gooqy16BvTp7TaLvAtKdP +SK7kl1QVAgMBAAECggEAKVT1gZs4c5Fd0daoWGbeasWqz730s9KNbmoNuUlKorcW +C/fl9m6sWJkrAApujpDBZNt/3VYvdVskrqVtjaveMkYvucUMugRRUMMa7OmHAjyT +DfvkbOw0fc+xgO83yV9zUpqPUhh1JGOtz6b+QArDlHFguYQkdPODrYsACKxdkmU/ +A+ZJkC2Sdn0L7ARmROxvoelxV3HNbC+p8lqrIVarpni8oeDF2JxQfyJETl3Df099 +/KA/IQvXytEQBJpguh0j7sK81cI40TKO3kNWsCe46S2nkCuEeEums7BqAspmdNMS +gBW0rpFA9Ep5AyT4+A2XhWFQAwf/PI26JUVDhbClAQKBgQD/XlONHf5tNiOEtW88 +Whd9ha7oLgoBhr4gFOcPM4MUStRDftRXY1ty0IGseV0Rj1xQRUp31Pl6Kep/udy1 +r/6ML/IQNaht1VJOf2+2v7oZFoLRGLvR3qSRFNVtfLWp1Dg5OPgyi00OpFnxz/6o +iuWlh1EWwSm9vPl91HcWPPcwBQKBgQDdfWfDfy8q9jExj8tVNh5++haeemprkZxk +QuduIkahlbQ3V+SLaSWQrwzcB4PWL+45fnIWwK3Vmm2POBv+OWfKGjJniSd4Zrrf +H+lSg4+hpivvoaYf1TsCTHy/af/5pYIkEUmbPM0EqwubyoW6oMAgGIAMyH6XBLaY +a4vWZo6g0QKBgFXDXhgUrLAM8JzPOk5wi1cSoI1FeQLON+gaXQdT63/TKbqJS9MV +gU7sC8Da+ZC+LuiefMYF9ss6bJD84Mz8EGcQayFag/hvHjdSwTgE6AEo+EI1Jk7z +kR1Qe+VLbs9cgI1nPqPq+LQkKjj1+7aq/zk6WtdjhBs+7iN+SYhkbTfpAoGAeCbv +gz4beFLVnO1EgJU7Nea1HoOJ95CmJj2lDjnJ1x4/BUbI1FfV6QcNEs+A1VBrCwVt +HqqnopiDlo35oY/CngBYF5JvtwEDnsbQ69IyuJ5Md1JZrCsgN78GbVAzbFo8nxRB +udTh4wZm5byXjwZlMwQXctfQ1FvaMhUlZsl71gECgYA82osmLHxZZOiQrYJ7z302 +CmPohwcTtUASqeA0kQOX/OdFJ1IxWIFpzq2ffMipLTfzk7OeFS2hqvNo/NE8NHlR +/79IycYwlO99T3e2yQhYKW5ZMT3Q8YtyLLPtBzPPtO/Tc8yKhjX8wc9w9sxq9kqg +i5Z+6xfO+l/NBT/JHF4yEw== +-----END PRIVATE KEY----- diff --git a/server/configs/certs/server.pem b/server/configs/certs/server.pem new file mode 100644 index 0000000..722f6a2 --- /dev/null +++ b/server/configs/certs/server.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDnjCCAoagAwIBAgIJAOXpOWk41Pd7MA0GCSqGSIb3DQEBCwUAMIGDMQswCQYD +VQQGEwJVUzELMAkGA1UECAwCQ0ExFDASBgNVBAcMC0xvcyBBbmdlbGVzMSQwIgYD +VQQKDBtTeW5hZGlhIENvbW11bmljYXRpb25zIEluYy4xKzApBgNVBAMMIm5hdHMu +aW8vZW1haWxBZGRyZXNzPWRlcmVrQG5hdHMuaW8wHhcNMTkxMDE3MTIzMjQ5WhcN +MjkxMDE0MTIzMjQ5WjCBgzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRQwEgYD +VQQHDAtMb3MgQW5nZWxlczEkMCIGA1UECgwbU3luYWRpYSBDb21tdW5pY2F0aW9u +cyBJbmMuMSswKQYDVQQDDCJuYXRzLmlvL2VtYWlsQWRkcmVzcz1kZXJla0BuYXRz +LmlvMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3PGGsZYijbGvDInc +ELA10tDCOM/mT9VDFvVP1bdqk4np2LNy+iqns0X8BB9gKBO18r0Hc3j1d2jdYANC +95gCokthTdw4cUemPA/yTW+WAOj5dH5WjCp1pLZmv0jDn+6O3ku2bemZiL7QYKdW +bUUmUh+uCkqguOu3hgch515tuWe1H/9qzvCtY7JkcIAxCaCS+WeL2uKcqV+Squzs +VpPGWudCrpOg7XUXyHJ1I6A2nbGMxh8xe5Kdf99XpUNmlD6fXul92lrvvoCb+W6w +TLFnkPhE5owjD6saFwzNgCasD+94qxC1OzzlEf6t4KKKstegb06e02i7wLSnT0iu +5JdUFQIDAQABoxMwETAPBgNVHREECDAGhwR/AAABMA0GCSqGSIb3DQEBCwUAA4IB +AQDIMNgMqgGdXwZTIDxz4iq76iHdSHxZK4YwJr/4xRSq1uEddxmUfAQzo+gBboA5 +c0XxYxc0xViuueThnqdLhHmEyCs8uGFRLaQAI5Bq1PdMQP12m5fCWRAyKrCxdtli +zm8ByDq7mWpkfMTd/rJGR4wCR9qI9Y5Bp6p4FBKZ3pzEanFXMV9IHhkm1BGh9tbe +l6GQyBptEpfTiRwNCC/ympeiL3G8hfDCPkcLed5sQ+OhPe5iWMVPncZh/qehnUJK +B5CIXcagcROFutsDYPCurKcfQOsfqulu0q95h7FQUOsIeU7jIlcLxIii19qjlTZh +sFjsul5G7qqMEgIUsx3U985v +-----END CERTIFICATE----- diff --git a/server/configs/certs/tls/benchmark-ca-cert.pem b/server/configs/certs/tls/benchmark-ca-cert.pem new file mode 100644 index 0000000..f91985a --- /dev/null +++ b/server/configs/certs/tls/benchmark-ca-cert.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIID2zCCAsOgAwIBAgIUZj0PngA93uUSShcRndTQju/J88YwDQYJKoZIhvcNAQEL +BQAwfDETMBEGA1UEAwwKbmF0cy5pby5DQTEiMCAGA1UECwwZUGVyZm9ybWFuY2VB +bmRSZWxpYWJpbGl0eTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UEBwwHVG9yb250 +bzEQMA4GA1UECAwHT250YXJpbzELMAkGA1UEBhMCQ0EwIBcNMjMwODE0MTUyNzU3 +WhgPMjEyMzA3MjExNTI3NTdaMHwxEzARBgNVBAMMCm5hdHMuaW8uQ0ExIjAgBgNV +BAsMGVBlcmZvcm1hbmNlQW5kUmVsaWFiaWxpdHkxEDAOBgNVBAoMB1N5bmFkaWEx +EDAOBgNVBAcMB1Rvcm9udG8xEDAOBgNVBAgMB09udGFyaW8xCzAJBgNVBAYTAkNB +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2cCyJL+DExUyZto2eFLm +MBRSkQLxM9pOWB9O8TecHlPcc/SPGq/x9lpguJ/IiaUj+VffVWy236KW2JL5Xj83 +PZwhXi1yZzxlIBsKAgAUeNfWuTAc0K0Qm9pR5Wjv5eNcT0mw6JX0SPgUQAl9BSwU +WvtMOTxOt0hBjHmZaEamp7nLmwogpvgPsrubD6U4O/vUQm3JTsbp2rFQxXPpkG19 +69PGsT37r0/w9Zv0xNAcB/zCWdNBXCTA2ACV2IpJedWm8Jrjcn3Kp4Fv3TKTsCZl +eWtfxCdljndk88+NFK7cEw7b9Bs5R5Zhu20C+Ne8vmMWhYbVBFYws5/jGzPBkVTD +7wIDAQABo1MwUTAdBgNVHQ4EFgQUEqfeAemfeIp4MM4C7H1bJS+mra4wHwYDVR0j +BBgwFoAUEqfeAemfeIp4MM4C7H1bJS+mra4wDwYDVR0TAQH/BAUwAwEB/zANBgkq +hkiG9w0BAQsFAAOCAQEAiamiPxOlZ0pwOJvv0ylDreHVk2kast67YlhAOcZoSMvi +e2jbKL98U3+ZznGj21AKqEaOkO7UmKoJ/3QlrjgElXzcMUrUrJ1WNowlbXPlAhyL +KhNthLKUr72Tv6wv5GZdAR6DaAwq3iYTbpnLq4oCnFHiXgDWgWyJDLsTGulWve/K +GGM2JMcnacNgNC18uki440Wcfp0vGj9HhO6I/u63oGewZnIK87GQMQCt3JLFyiUc +hrn9nWoixFWcJfCjBcMlwZXMIAlDdelU1/hWtSknKCs57GvZuACcicAYiYIkWCkd +p1pF4G0Ic6irAnLTqhdGwL4+5pjNd1Ih0Gezn9hJLg== +-----END CERTIFICATE----- diff --git a/server/configs/certs/tls/benchmark-ca-key.pem b/server/configs/certs/tls/benchmark-ca-key.pem new file mode 100644 index 0000000..69bbac4 --- /dev/null +++ b/server/configs/certs/tls/benchmark-ca-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDZwLIkv4MTFTJm +2jZ4UuYwFFKRAvEz2k5YH07xN5weU9xz9I8ar/H2WmC4n8iJpSP5V99VbLbfopbY +kvlePzc9nCFeLXJnPGUgGwoCABR419a5MBzQrRCb2lHlaO/l41xPSbDolfRI+BRA +CX0FLBRa+0w5PE63SEGMeZloRqanucubCiCm+A+yu5sPpTg7+9RCbclOxunasVDF +c+mQbX3r08axPfuvT/D1m/TE0BwH/MJZ00FcJMDYAJXYikl51abwmuNyfcqngW/d +MpOwJmV5a1/EJ2WOd2Tzz40UrtwTDtv0GzlHlmG7bQL417y+YxaFhtUEVjCzn+Mb +M8GRVMPvAgMBAAECggEASvFWfm5JMtqjP6HPiGbjoV2FMzJjiFiUiSCxXzSn7wom +v+PGEsXGTWa6jiAz+SeUc38KNtDVOa+wIfani4fPP82J8GtMyfoPxdZ4gcq8QQDr +/k1wRWOi6Tjg4cdVdXXkMcencs0VR73V3lpFpG+Qy+VcTQCUCOF96dZ59VkHh4a4 +CHX6PegOWwr0TSaCUacwhua+rPmCar/btAYv7Wp7+c+Zf7Rn2WYTV7ol4sYXR4ZQ +Sy/ROijeFTMkYNpaW/KOO2/pn3OJA8ycYH8UWZpsenPfIajC0Eka7BoEQpw6M8HR +wRWrKwssBEs0psFiq9s8J+6resPgXfU/9pf+mTkTqQKBgQD8kktUqN+vYc6t22DE +tSkg8y8OsGh9VTYfp7x5hu9qEC9t4mAKjqA/rRLiTXze/wInntreRTjjMb/NlqMy +PvI0Z+dM1UuqcF1axgKrIYsgnLJWuunOhaj5K3LhiNcHznlCtN9601dbccwLlQhL +5jdjnOuJ0i+Nh9v5oiu37SfldwKBgQDctWdbF4hJrBPnS6CcojuQj6ha10wYYe46 +ZVcxKe5hFBs1q975YCHnEntyCDvXGfOTRgbKeZbwNhMvAc7Pp6eGMR/9SpiRwTt4 +567hUz56WXVmp4gSvxoNuYRlWiMI8rZkyKJ8KFipvHgRa8nuamh0QBB4ShEJiVk8 +fhaUiZeTSQKBgF0hAD/OKPR9Jv06J9tARVMN+Cr9ZvnXwqY3biqNU5gTMbndv7YE +0xfHlG/3THTZKI09aMyOT6SOQn/m7HPpe9tQ+Jt/BnBpEDMZUgCR1MAIp0WNlAp/ +hEej+q8oiskpG9M56DFc3hgsxKT8pdt+nqvPP5ZI9xnDn5vTbTVbb9uPAoGAFRvU +csXhZwpqLOjyx4hMohrbQzTsNjjHjBY9LJqSDf7aS1vQy5ECLRN7cwCOmJgGz8MW +yy6t3POPCiPmH74tK4xvPs5voSEWCw49j5dillkP/W1wejqEx2NC4l6okyaDg0gd +IjrJoBJCeYgRnBfZPaUS7i5HSt40BrEYf8RZFuECgYAjSnvY8nYFRudsylYOu3TL +AcGbAdpDfL2H4G9z7qEQC6t1LqqGNcfMan/qS4J/n5GCOedVWHcfCraROOMil52v +3ZDXyyjGEO08XgKnoa2ZL/z2a6s077+hAAnbcywyi2Qz8Yfi6GnwYfU1u8SN0APd +T71HPNsWkU4zkxmP4S0Olg== +-----END PRIVATE KEY----- diff --git a/server/configs/certs/tls/benchmark-server-cert-ed25519.pem b/server/configs/certs/tls/benchmark-server-cert-ed25519.pem new file mode 100644 index 0000000..bc9cb2d --- /dev/null +++ b/server/configs/certs/tls/benchmark-server-cert-ed25519.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDHTCCAgWgAwIBAgIUc31LCktokAIQGqLsSC2BlsLCTsYwDQYJKoZIhvcNAQEL +BQAwfDETMBEGA1UEAwwKbmF0cy5pby5DQTEiMCAGA1UECwwZUGVyZm9ybWFuY2VB +bmRSZWxpYWJpbGl0eTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UEBwwHVG9yb250 +bzEQMA4GA1UECAwHT250YXJpbzELMAkGA1UEBhMCQ0EwIBcNMjMwODE0MTUyNzU3 +WhgPMjEyMzA3MjExNTI3NTdaMHkxEDAOBgNVBAMMB25hdHMuaW8xIjAgBgNVBAsM +GVBlcmZvcm1hbmNlQW5kUmVsaWFiaWxpdHkxEDAOBgNVBAoMB1N5bmFkaWExEDAO +BgNVBAcMB1Rvcm9udG8xEDAOBgNVBAgMB09udGFyaW8xCzAJBgNVBAYTAkNBMCow +BQYDK2VwAyEAyyc9y9iZgWWSsPRahbeGxF6XN3VOFPZBvD/HQps6jr6jgZEwgY4w +CwYDVR0PBAQDAgQwMBMGA1UdJQQMMAoGCCsGAQUFBwMBMCoGA1UdEQQjMCGCDnJl +dWJlbi5uYXRzLmlvgg9yZXViZW4ubmF0cy5jb20wHQYDVR0OBBYEFBwkwMU8xuQO +FN1Ck5o2qQ4Dz87ZMB8GA1UdIwQYMBaAFBKn3gHpn3iKeDDOAux9WyUvpq2uMA0G +CSqGSIb3DQEBCwUAA4IBAQALjCynuxEobk1MYQAFhkrfAD29H6yRpOcKigHCZjTJ +Dnpupip1xvaFPPvhi4nxtuWcXgKpWEfd1jOPaiNV6lrefahitZpzcflD7wNOxqvx +Hau2U3lFnjnGaC0ppp66x26cQznp6YcTdxrJ1QF4vkOejxqNvaTzmiwzSPIIYm7+ +iKVWT+Z86WKof3vAdsX/f148YH1YSPk0ykiBzlbLScbyWebbaydrAIpU01IkSvMo +qDYu+Fba0tpONLe1BUklc608riwQjw9HiJJ2zJIAOBAUev5+48RP91/K111Ix1bl +fGPT8/1TJbyGG2jeJwyLoSIu72aDnnIBfqGkVunRTmeg +-----END CERTIFICATE----- diff --git a/server/configs/certs/tls/benchmark-server-cert-rsa-1024.pem b/server/configs/certs/tls/benchmark-server-cert-rsa-1024.pem new file mode 100644 index 0000000..be9bbaf --- /dev/null +++ b/server/configs/certs/tls/benchmark-server-cert-rsa-1024.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDkzCCAnugAwIBAgIUc31LCktokAIQGqLsSC2BlsLCTsMwDQYJKoZIhvcNAQEL +BQAwfDETMBEGA1UEAwwKbmF0cy5pby5DQTEiMCAGA1UECwwZUGVyZm9ybWFuY2VB +bmRSZWxpYWJpbGl0eTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UEBwwHVG9yb250 +bzEQMA4GA1UECAwHT250YXJpbzELMAkGA1UEBhMCQ0EwIBcNMjMwODE0MTUyNzU3 +WhgPMjEyMzA3MjExNTI3NTdaMHkxEDAOBgNVBAMMB25hdHMuaW8xIjAgBgNVBAsM +GVBlcmZvcm1hbmNlQW5kUmVsaWFiaWxpdHkxEDAOBgNVBAoMB1N5bmFkaWExEDAO +BgNVBAcMB1Rvcm9udG8xEDAOBgNVBAgMB09udGFyaW8xCzAJBgNVBAYTAkNBMIGf +MA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCyHHaVHinB3jBsicR4hp7uopz0u3+O +kUicIUSQXDcWiPzdvE+7YZ/s4+Ud4aw4g9q0wHzkZSaMg8nil4tCKmTrUKolVTVj +CCCBmtqq3LwzNLapyoDJRyXsWqHt5TWYSxaf/UQT6sWOgqHOLrbd4J8F0sjxEniB +GDHR1ZXpJCBaIQIDAQABo4GRMIGOMAsGA1UdDwQEAwIEMDATBgNVHSUEDDAKBggr +BgEFBQcDATAqBgNVHREEIzAhgg5yZXViZW4ubmF0cy5pb4IPcmV1YmVuLm5hdHMu +Y29tMB0GA1UdDgQWBBQk5kWOUcUNn7FppddLANe3droUlzAfBgNVHSMEGDAWgBQS +p94B6Z94ingwzgLsfVslL6atrjANBgkqhkiG9w0BAQsFAAOCAQEA2Njy2f1PUZRf +G1/oZ0El7J8L6Ql1HmEC7tOTzbORg7U9uMHKqIFL/IXXAdAlE/EjFEA2riPO8cu/ +bvL2A4CapYzt2kDD9PPYfVtniRr7mv0EVntPwEvfiySMAEeZuW/M2liPfgPpQkhL +fzwPeCOfqM8AjpyDab8NEGX5Bbf421oQorlENpm4PKQCXoUN5cWpBwuwWxj7yndj +256MevLDKKe/ALSLQEo/2Jgpnmp7Qol0GtomCzsLgZ+ASuVtCsGTFmaRrsqVPspJ +oOl6qby5gYwN9TR8zfRYL1m1sbYROz+5+ofEoiTnaOoOSjiBIoYoMeSC/jvJQTPT +VdD8QeQ6Og== +-----END CERTIFICATE----- diff --git a/server/configs/certs/tls/benchmark-server-cert-rsa-2048.pem b/server/configs/certs/tls/benchmark-server-cert-rsa-2048.pem new file mode 100644 index 0000000..9fb03ac --- /dev/null +++ b/server/configs/certs/tls/benchmark-server-cert-rsa-2048.pem @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIIEFzCCAv+gAwIBAgIUc31LCktokAIQGqLsSC2BlsLCTsQwDQYJKoZIhvcNAQEL +BQAwfDETMBEGA1UEAwwKbmF0cy5pby5DQTEiMCAGA1UECwwZUGVyZm9ybWFuY2VB +bmRSZWxpYWJpbGl0eTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UEBwwHVG9yb250 +bzEQMA4GA1UECAwHT250YXJpbzELMAkGA1UEBhMCQ0EwIBcNMjMwODE0MTUyNzU3 +WhgPMjEyMzA3MjExNTI3NTdaMHkxEDAOBgNVBAMMB25hdHMuaW8xIjAgBgNVBAsM +GVBlcmZvcm1hbmNlQW5kUmVsaWFiaWxpdHkxEDAOBgNVBAoMB1N5bmFkaWExEDAO +BgNVBAcMB1Rvcm9udG8xEDAOBgNVBAgMB09udGFyaW8xCzAJBgNVBAYTAkNBMIIB +IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyx3O+Z6u8Y1SiuHu3szWbLvL +WrZpSpEiZkll+wk5205S1FRcQLccfr4ubdtjOBdi+RzILCtkflUI01Dbqu6cV7/2 +yfLthxBeNDiXMhjyOFkYLwwE4w7CdTwWWsmW31oUH1rYXIDPoeb7WPF7w3NwaUJu +ZXnqM98LRgWDTmh+nsqDDW/bz1fYIdxcO9az6iBOnJ2AGWI2ur5GzWc4+gNMOZiZ +Xj657g0MbyVM4Gzyc4Au22hShZ/YorLP8NAiwNJamlrCFzlnZN/ePjuQPcI6glnb +oO9IAGfPdAOJptfayuPAZgUngzewB38yY0Q/rKG1GJKSkQ8X6/lXiWaRPZJjYwID +AQABo4GRMIGOMAsGA1UdDwQEAwIEMDATBgNVHSUEDDAKBggrBgEFBQcDATAqBgNV +HREEIzAhgg5yZXViZW4ubmF0cy5pb4IPcmV1YmVuLm5hdHMuY29tMB0GA1UdDgQW +BBRtanJZScdSlsPsi58lBcpdj+bV/zAfBgNVHSMEGDAWgBQSp94B6Z94ingwzgLs +fVslL6atrjANBgkqhkiG9w0BAQsFAAOCAQEAV4TZ3b8cYO7ZeRyoCQtCBAab9gNe +kbQpWqICvkVQOk5Anq3opwAWk2FuIRs5KoT7ssckHpXwTwWLs+KuIVo+Fet19IH6 +BQfck1jwhzM04MA6zLO/F2j548XlrJy3IzViPM/VxwMMTt5YSoogrz/3TzzJPIe0 +eQomf5HbpVgrf08pMVkdaI7PCd7N/CxeWiD5zEWqBu9FqofO188Kb/umx0VwgBju +dX46MKO5TyUc91UrG3M35/r4Z7fd52SWWWFQiI7UBOl2L27samjHlJsKjyFoBF3Z +alvnoUVzo7zwAYmhEdPYDNVceF4KtAFpGipoQPRMg83G87LgYBA4Sa6uKw== +-----END CERTIFICATE----- diff --git a/server/configs/certs/tls/benchmark-server-cert-rsa-4096.pem b/server/configs/certs/tls/benchmark-server-cert-rsa-4096.pem new file mode 100644 index 0000000..ed231a1 --- /dev/null +++ b/server/configs/certs/tls/benchmark-server-cert-rsa-4096.pem @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFFzCCA/+gAwIBAgIUc31LCktokAIQGqLsSC2BlsLCTsUwDQYJKoZIhvcNAQEL +BQAwfDETMBEGA1UEAwwKbmF0cy5pby5DQTEiMCAGA1UECwwZUGVyZm9ybWFuY2VB +bmRSZWxpYWJpbGl0eTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UEBwwHVG9yb250 +bzEQMA4GA1UECAwHT250YXJpbzELMAkGA1UEBhMCQ0EwIBcNMjMwODE0MTUyNzU3 +WhgPMjEyMzA3MjExNTI3NTdaMHkxEDAOBgNVBAMMB25hdHMuaW8xIjAgBgNVBAsM +GVBlcmZvcm1hbmNlQW5kUmVsaWFiaWxpdHkxEDAOBgNVBAoMB1N5bmFkaWExEDAO +BgNVBAcMB1Rvcm9udG8xEDAOBgNVBAgMB09udGFyaW8xCzAJBgNVBAYTAkNBMIIC +IjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAj/rw0WpCnizUSs06NxSsjuYb +6sWtYQjLd6O6SCoQwrPSY2Zv0u28RywBJkSiIbv12kWxrhqv+kDqhmt7tfwEdHAE +gOl6E2P/CFDnyAJ4it5qgFRHcEItGp1Ap4ZrZ3OTl41DDe28giflHb7+VAzxUV6r +zkHmrqkaJd41YrJoKyZ3u+/FHs6CudO+jGXC2ubH1i71ARXHn6tkmOTwFf1N49wB +Hr144xBmPyvH/A/elgXBjR28/7x64MO/qBZsr/lbTaS4YwN1+rod/HRc2GIiJxpS +tB1bh/dXZCfa0QyhTzCNLG/j7IzetDrPBGzpw2WjufhkSuaxoMDWlDkkfqmQxtHJ +5L+PqIiPT69tzEbfuS9Ogz7DW10CcpSdXW13sWCdSNCGEvPqLFka36q9B82V0GHz +tmx8VqdfWSu4qMyVmTsxxzLTTxpQU4X1Q2RnT/1igbsOM620LuvrvnXlh5Rdg7tp +T++QJ/b4xCCg62tv4VwORe27xHYXMeDn4aoRdyoI45/+ZK6yqwNAOrKKpse/M8uz +mJK2i8pfEFmitIKoNYn3MR2dFrCqifZkFf9rX9A/1Ym+WKAPLmWdGp13fvTdzxQG +Y44f9tBL2RWsoGX++01XEwIiWz7kqObC0L8fz3EdIPaULX7MZiQrxzzhRCcJhyn/ +aOrJfLYj0GAmIaHElHkCAwEAAaOBkTCBjjALBgNVHQ8EBAMCBDAwEwYDVR0lBAww +CgYIKwYBBQUHAwEwKgYDVR0RBCMwIYIOcmV1YmVuLm5hdHMuaW+CD3JldWJlbi5u +YXRzLmNvbTAdBgNVHQ4EFgQUbJSM8LNWmc9IgLe0X53yE3c3h1cwHwYDVR0jBBgw +FoAUEqfeAemfeIp4MM4C7H1bJS+mra4wDQYJKoZIhvcNAQELBQADggEBAEGmLvEE ++MTE1bHMbl/5QG+/xusmervIuxkfAfId0H+8TWB75y+yhVZpEdM7knfl+lexmtGQ +Gr4HNGTZhAZ3NYFaBw7nfeqO48He7gHUKfJA/zv7FREF3Fy+Qe/hydDJQzBzZfaU +64XqhY6jOurpZhTAoOXfjpYzZaLi7+rdpTAbfxHCCAC8SxZD3++Q97ZeoT6en47O +8SQQ7FIzWxs15k88oYalw51vZujxX7dz4l+LxsLXtlYW7ZM1163cgU7lF/jQqDcN +z8X8jk1AjQY7AuFPuOzQ1hLXcZySm8rUG5pPgHrZ1QKmkFFWRaeCiO2hU794wI5C +vIGR8lIhkNwEJnQ= +-----END CERTIFICATE----- diff --git a/server/configs/certs/tls/benchmark-server-key-ed25519.pem b/server/configs/certs/tls/benchmark-server-key-ed25519.pem new file mode 100644 index 0000000..edb1732 --- /dev/null +++ b/server/configs/certs/tls/benchmark-server-key-ed25519.pem @@ -0,0 +1,3 @@ +-----BEGIN PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIJRCtUNxUuutNs9j8OtcwFw1xkbs+zxjHhpAqVuqDNo5 +-----END PRIVATE KEY----- diff --git a/server/configs/certs/tls/benchmark-server-key-rsa-1024.pem b/server/configs/certs/tls/benchmark-server-key-rsa-1024.pem new file mode 100644 index 0000000..4847c5a --- /dev/null +++ b/server/configs/certs/tls/benchmark-server-key-rsa-1024.pem @@ -0,0 +1,16 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBALIcdpUeKcHeMGyJ +xHiGnu6inPS7f46RSJwhRJBcNxaI/N28T7thn+zj5R3hrDiD2rTAfORlJoyDyeKX +i0IqZOtQqiVVNWMIIIGa2qrcvDM0tqnKgMlHJexaoe3lNZhLFp/9RBPqxY6Coc4u +tt3gnwXSyPESeIEYMdHVlekkIFohAgMBAAECgYAwt8RfyV5WnvXT2mMZLIlwcJ5J ++rdLQcYAnsDoU7DlwxaXeBi/AlcCLtvOrpmy464A3t3KgzhmGu4vwo/ey0XK+nTQ +tzORP/PXTaVC8DzJ8PnJmUaB7l+H7a88OSPLbjgnbpw4SyvDpKUHiiw0EDYC7L6Y +1vvCOlnprptXbE5eeQJBAOpjwdBVWkVtmStjsxbxZsUTI7XKxS2VZinRLH0l5/hI +hIHRxwy9oRbeNrf5815lGolTUD0mq+N0dJRlMop1yKsCQQDCiGDkH/pQqhB8ibmD +0XNw0EzxJmPFACO/x49VCfCPE5p1FQhpyIl6JkyAFNN7Xs4HX8jMHTuvNgJVti61 +O0BjAkEAj0wr2vXDubyWrztF61nszcG0zFjKkeLL0fcLLvv0xQt4z3F0MyrgCH4U +kAflLSm8voZMAQbagbXZ7DuuWY5G/wJAWyKnOdidXZL+3ElthwrmKVD86vEQRqe1 +F9C3HqDkeTM25mkvItfXSEmPB2Y6WY7luOCv4qhDYOdNmrgaE7+pfwJAcbV5ZVJW +OZvH1ofsJVvUA8J58tzv1+KPb96pI3YRAu8xbMC0mzezPsYjg2wjaRgJ2C+7On27 +BaArNo75B20AkA== +-----END PRIVATE KEY----- diff --git a/server/configs/certs/tls/benchmark-server-key-rsa-2048.pem b/server/configs/certs/tls/benchmark-server-key-rsa-2048.pem new file mode 100644 index 0000000..60bd945 --- /dev/null +++ b/server/configs/certs/tls/benchmark-server-key-rsa-2048.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDLHc75nq7xjVKK +4e7ezNZsu8tatmlKkSJmSWX7CTnbTlLUVFxAtxx+vi5t22M4F2L5HMgsK2R+VQjT +UNuq7pxXv/bJ8u2HEF40OJcyGPI4WRgvDATjDsJ1PBZayZbfWhQfWthcgM+h5vtY +8XvDc3BpQm5leeoz3wtGBYNOaH6eyoMNb9vPV9gh3Fw71rPqIE6cnYAZYja6vkbN +Zzj6A0w5mJlePrnuDQxvJUzgbPJzgC7baFKFn9iiss/w0CLA0lqaWsIXOWdk394+ +O5A9wjqCWdug70gAZ890A4mm19rK48BmBSeDN7AHfzJjRD+sobUYkpKRDxfr+VeJ +ZpE9kmNjAgMBAAECggEADicdEWKfsQAWZMv6X3bpZ/kr4b29F2+GdJcfrn7Fk8Tg +25+nL/EyYJhy1r/HKZTjlhUN05oQbgcRztue+smWhjy/fvHY4CThU4Uv79MyKX/3 +wet1+bZBEXcm3ZuXUifOKCMl2Ug2b4MPN3LYG1XTWtpAo/x7N7MOb4oZzKBWVk0J +GEfxvJVZyVe3libIN2WWZFQJ1620AxaWP2jZ83PhRR/TJIBiq+pI/lBIyt+nu8Z3 +3vzB020R2uvENOhnaDjzNVfnJxSvlAAQkN47zo2Tf4UnB0792Y7DEEjFKX4XCFVd +zxmjmw7VcRcnyruDhCoRC6mNraaAHuMqPwuBoC6GtQKBgQD/7CV8XKzcPPfFJY9e +aHPzgXJwK+5u3jq7tNYUksVfv0s2lLQnRdbqAhHzxNYLQjdVd7J6t55h2z8scYaP +oB7TTwszhKZS2sQ/lcpfOFoFN6KjN0iOnXVFucGoQ36gexNqPw894NFKWX/RHrnZ +UfL/OnOUPpra3w+WMxjUYQ5ivQKBgQDLLZDIpM8fSRKeqZzjv/eoKnoDhqmzPipj +bvNXAkIr/nWfUHL9YRnpxX7PW8DqFWIYoM8b0uOKO4vLJnQUMyc2jPq1rPXEBrjk +w+xKWCipKdPrqwttiLKme+ZArRT5CJ9qcqxSX0yNp69xwXtyHe+zY18F/a4+70wT +5wJwYmhQnwKBgHpICTk8xtOMxg6K/c/sNMr65QU32Htc789Ufp3h6zDupC92KgZB +1oiFaLKDMIq8ntfVk5ATQDgdnDfOHq9toIzyzbVWAmrAYNjI56NLt6eah7lY5vBN +yAUC1sdhSJXBeOthKhU04IuX6/yto7t07piJA0SoDTHbNwVbcNe5cDg5AoGBALjR +jxVlDd+4mc5oHYXy1rZLRUg10+JvlyFyCLrKHCVmx9oO1Tr1fBvxggPfw+FraBtd +FGiL8l2JAwXdydOiIHZ30Ys3dSxGrSOzsRqDjSEsIlEK+088/L2CkRWeHCjYliK/ +g08+zyVANtC0nrVU0/mLWCHb/AfVp4+nIMnYSmmjAoGAEMyBqq2AyUmx1xAmhw36 +LqgKy+vgHEAFRFPD8IttHFLOlUdXlvxoDq4xW2a7bJsJrs9ZrluRFKVh7QnSckmP +Jt/Plg+XYB3B2exD5Xyh9xNYNVW/Aqvg+NuiWeCGK/o7mUfGWd9qWrD2aw51m+X3 +Svtkgck1kulqPoUFG1b3R4k= +-----END PRIVATE KEY----- diff --git a/server/configs/certs/tls/benchmark-server-key-rsa-4096.pem b/server/configs/certs/tls/benchmark-server-key-rsa-4096.pem new file mode 100644 index 0000000..daf897f --- /dev/null +++ b/server/configs/certs/tls/benchmark-server-key-rsa-4096.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCP+vDRakKeLNRK +zTo3FKyO5hvqxa1hCMt3o7pIKhDCs9JjZm/S7bxHLAEmRKIhu/XaRbGuGq/6QOqG +a3u1/AR0cASA6XoTY/8IUOfIAniK3mqAVEdwQi0anUCnhmtnc5OXjUMN7byCJ+Ud +vv5UDPFRXqvOQeauqRol3jVismgrJne778UezoK5076MZcLa5sfWLvUBFcefq2SY +5PAV/U3j3AEevXjjEGY/K8f8D96WBcGNHbz/vHrgw7+oFmyv+VtNpLhjA3X6uh38 +dFzYYiInGlK0HVuH91dkJ9rRDKFPMI0sb+PsjN60Os8EbOnDZaO5+GRK5rGgwNaU +OSR+qZDG0cnkv4+oiI9Pr23MRt+5L06DPsNbXQJylJ1dbXexYJ1I0IYS8+osWRrf +qr0HzZXQYfO2bHxWp19ZK7iozJWZOzHHMtNPGlBThfVDZGdP/WKBuw4zrbQu6+u+ +deWHlF2Du2lP75An9vjEIKDra2/hXA5F7bvEdhcx4OfhqhF3Kgjjn/5krrKrA0A6 +soqmx78zy7OYkraLyl8QWaK0gqg1ifcxHZ0WsKqJ9mQV/2tf0D/Vib5YoA8uZZ0a +nXd+9N3PFAZjjh/20EvZFaygZf77TVcTAiJbPuSo5sLQvx/PcR0g9pQtfsxmJCvH +POFEJwmHKf9o6sl8tiPQYCYhocSUeQIDAQABAoICAAEYUQ4KqeyJiJN0DvIIdc0q +v1dwVEKlaOS6nvSkYJcWe9kL2E8JRfy1lIxTCneethPdoqhhZWljZn/fX+QRAVir +BGxq5NCHxWgDCNble+oJydNlhgXldEcxnyJXBvM/trA49Q5X+pkeeWMvRsBiuSVx +BqCi3CtZDQzw18TtzessF6OF2IuE2bYqP9anVyLH4rOvN5JKn9zH69PaLorq76rL +YMp4JEFNKIs+R5RUFjwxBC6Q+r+9fM1qTQdtJOZMC293pfusylAoll315ZczXIah +xYi/ThOmjtMrwWyEan1PcCIVzMI0X7o3q0eMQNU0ApmBY9k0+sYEnsJ8Um8l1icb +WE/KBDzQk2mSoIzEdo99gYiuLAhi5WTEcQzeovZKHKkhRxLn3GUK/ZznpTuI0Voh +1Qbfw05wf62yh8/nc747n5ULPgf8/nUH5G8jUeSis9gMGtFloqIxUEBphzIHwj7W +FUIKoKpGkHL4Jkg9wXIe7lg5Qp+cdCrkWKKvQc57EbWxsM1Pjktw6X5ctNQclWG0 +WQJUw+BJhq6InKAV4aDW4/QlCOjvGtDDkYvwghE+ZgIzm9BARM4+DbeoPTmAkwwz +6NTM8DYWvdcYVpgchEwphS6OFkOD1Dc6Wg2OUUtLjvl4NVf/NDYN8wIWNwmMm4aj ++Is12NwoUqZ44E4pSfohAoIBAQDLloatDuObuLzInBqER2tByPNnDV9AVTR2rm6w +TgBmslFHBCf/uSzq0ivh1oZmFfDhcqdrtheTL0BWXbFxR/GelDm9N4d/LqOGRyWV +Qmy5dGp2wE6gsZGZuvsbVst6eFO+zgDLvDMLrVqP+577g/QbrSR8gPz39WCgLpvf +RcNpM2rHeKkm09IdUe0G+YEnW3fD7rxtMS+j8J57yuuxLO+SemGJuJQqGHYOukxF +KMZxX2wMs1YMNwKEXY0/9M+LAtW9J3azkRp5xd+E2kzXGoWcl2nS/3+J6nh1+HWq +SZE7SDWSimzut/aPTOM7YxLVqtANWHzVBBNpTbOMSrSZTZQJAoIBAQC1C/afOROb +GQMb1LOt95AxvGyYo62mOkcPxhxQGojVIi+Yz8Q1ZWTuT5qyyWSofR3bE9mQwQGW +KQ/znJKiwA48BS0osnmQi3Bi6eVD73NfwxMkL6zrGFxf336DNQi9mbidJvOaHNsT +wXGPqRQXDcLa3s3WqerDzKJ2gxG22rtkoC0uCw+J9NlkJTTI9bJ9rjDZIh6mE/M0 +3ye19IgkBdbMv7FSjGVpovdqWZ2HELDYAXJftzuMPUO1GNYbyMHAHnWH0a1TQRM8 +ELHzmPRaFBeegmpBVaenlxKoi72iwqrVFrEr5FLAKxLq/9RwizJ8FzySIpH/5+hU +Ky4mG93lmHjxAoIBAQDLOXs+jTpPW923M3yUxuYeSQYPvJ10jplMT1tWysZDvYS8 +qz1yW9qmnR4I1ihbB1Po+JZ/QsnNtsE2dViHiBV9AuGQLDopjtjVVXgCwsfdaIRN +/jF+30JEfw3igIWlvy95rBHHThp2cZmRWM+eql2msvNVBT2AF4VY4K3f9rfV7+mL +LLtNcuyvL/S3naB7NSccgte89/hiYfMSB8G2nvCW+2saGxJr4vcWRImWD9nnmiU1 +mF8w2ki88NXrHel/Dlll9FrdbN9M52T0LSW/I050vgB5C2q4tUGCIX7zeXRsBOzV +VzDeKu0Ipuu9gGxwtY3xhH839FWcLGAqjvgwf+xhAoIBAQCuvWs9ZoNr0QpVFEiQ +Aj9kIa7W7DOwGtN3gAjXr6Sdwa8a2H1R5Bk0ghSXtxW2IXxtdI0qz35OhjdlM5u8 +BY43k+9wNkJqporEjWfA2B4NMWUKKhHFnu+ZgUbEMK3NAc9TrsKz3mH8gVqwA8rm +LVwCj8UwCTQT4zBzHjI8wITZrFeu9vH6fx5LMDXwOGQcNcHj8LCQLvUv9KqJTgkQ +a6pUWDg3qlY/TRFrzi7iq9NjyJGxnFKXGpJ8+gm9K1kFquBZRKD7l/WOpbZ7nQdK +4dWiIdGYWanFcWSK1MUlkKn9nTdHW8oau/g4ZM+QCGmjp3HIwiEUU6rDgiG6mm7j +KPShAoIBAG1sFn8X841038Z/sp4JzYypQjj7g9UOTkghHcJHIZzRXLiTqW3lKxbt +Gz93DjWRxXD404hem8dOUf22VgfjB4z1mSrV5SWtLjLf5wD8gl0eUNKP2lZFLggz +O6nHCLLzlKs2RH5pDo3c8qyjLRIfCXy0YGnr+9RErVJG+TVB/MOSgHagmkdVOYCH +phw4EiwJ+rPFy/xm5D6+BuOyt7hw7boQsw3EHpZTyQHWObcIggjlopTX2VSG8Dx+ +/iQRTuVRVyNAhYwCuNtSh27zawWr+A40acFJsJpvkFkbZBEH1IqoCteUaiEVU1qm +51lKgt3ZVAXuecJ1U/0u4HtC0QdEBGE= +-----END PRIVATE KEY----- diff --git a/server/configs/one.creds b/server/configs/one.creds new file mode 100644 index 0000000..9582ee1 --- /dev/null +++ b/server/configs/one.creds @@ -0,0 +1,13 @@ +-----BEGIN NATS USER JWT----- +eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJHRUNQVEpISE1TM01DTUtMVFBHWUdBTzQ1R1E2TjZRUFlXUTRHUExBRUIzM1ZDUkpOUlZRIiwiaWF0IjoxNjE2MjQ3MjMyLCJpc3MiOiJBQlZSWktKNlo3TklNUElZSlJDSEVZRlJVTzdFTk42TldPS1FERkxGREZWUFNNMzZVUFgyVUNQUCIsIm5hbWUiOiJvbmUiLCJzdWIiOiJVRENJQkdHR0hDSkJRUE9PNFNDSkpCSFpEUjM3TlFHR0NJV01ORzJEREFLSjZXTUtCTUFLWElNTyIsIm5hdHMiOnsicHViIjp7fSwic3ViIjp7fSwic3VicyI6LTEsImRhdGEiOi0xLCJwYXlsb2FkIjotMSwidHlwZSI6InVzZXIiLCJ2ZXJzaW9uIjoyfX0.VLhSDtGZEF_jdvgmhgkdISXAt5wFMMZxxwm5w8UrsnlM1hkUvtxBlTe4IP0xJIf4xf8JOR2Bmf73xUGJKUZECQ +------END NATS USER JWT------ + +************************* IMPORTANT ************************* +NKEY Seed printed below can be used to sign and prove identity. +NKEYs are sensitive and should be treated as secrets. + +-----BEGIN USER NKEY SEED----- +SUAPCDMU5TSHHLWUUZSOUABJXP2GXRCZVEOVWPSVM5XRSXYGQMRRFDYNMY +------END USER NKEY SEED------ + +************************************************************* diff --git a/server/const.go b/server/const.go new file mode 100644 index 0000000..1243418 --- /dev/null +++ b/server/const.go @@ -0,0 +1,251 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "regexp" + "runtime/debug" + "time" +) + +// Command is a signal used to control a running nats-server process. +type Command string + +// Valid Command values. +const ( + CommandStop = Command("stop") + CommandQuit = Command("quit") + CommandReopen = Command("reopen") + CommandReload = Command("reload") + + // private for now + commandLDMode = Command("ldm") + commandTerm = Command("term") +) + +var ( + // gitCommit and serverVersion injected at build. + gitCommit, serverVersion string + // trustedKeys is a whitespace separated array of trusted operator's public nkeys. + trustedKeys string + // SemVer regexp to validate the VERSION. + semVerRe = regexp.MustCompile(`^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`) +) + +// formatRevision formats a VCS revision string for display. +func formatRevision(revision string) string { + if len(revision) >= 7 { + return revision[:7] + } + return revision +} + +func init() { + // Use build info if present, it would be if building using 'go build .' + // or when using a release. + if info, ok := debug.ReadBuildInfo(); ok { + for _, setting := range info.Settings { + switch setting.Key { + case "vcs.revision": + gitCommit = formatRevision(setting.Value) + } + } + } +} + +const ( + // VERSION is the current version for the server. + VERSION = "2.15.0-dev" + + // PROTO is the currently supported protocol. + // 0 was the original + // 1 maintains proto 0, adds echo abilities for CONNECT from the client. Clients + // should not send echo unless proto in INFO is >= 1. + PROTO = 1 + + // DEFAULT_PORT is the default port for client connections. + DEFAULT_PORT = 4222 + + // RANDOM_PORT is the value for port that, when supplied, will cause the + // server to listen on a randomly-chosen available port. The resolved port + // is available via the Addr() method. + RANDOM_PORT = -1 + + // DEFAULT_HOST defaults to all interfaces. + DEFAULT_HOST = "0.0.0.0" + + // MAX_CONTROL_LINE_SIZE is the maximum allowed protocol control line size. + // 4k should be plenty since payloads sans connect/info string are separate. + MAX_CONTROL_LINE_SIZE = 4096 + + // MAX_PAYLOAD_SIZE is the maximum allowed payload size. Should be using + // something different if > 1MB payloads are needed. + MAX_PAYLOAD_SIZE = (1024 * 1024) + + // MAX_PAYLOAD_MAX_SIZE is the size at which the server will warn about + // max_payload being too high. In the future, the server may enforce/reject + // max_payload above this value. + MAX_PAYLOAD_MAX_SIZE = (8 * 1024 * 1024) + + // MAX_PENDING_SIZE is the maximum outbound pending bytes per client. + MAX_PENDING_SIZE = (64 * 1024 * 1024) + + // DEFAULT_MAX_CONNECTIONS is the default maximum connections allowed. + DEFAULT_MAX_CONNECTIONS = (64 * 1024) + + // TLS_TIMEOUT is the TLS wait time. + TLS_TIMEOUT = 2 * time.Second + + // DEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY is the default amount of + // time for the server to wait for the TLS handshake with a client to + // be initiated before falling back to sending the INFO protocol first. + // See TLSHandshakeFirst and TLSHandshakeFirstFallback options. + DEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY = 50 * time.Millisecond + + // AUTH_TIMEOUT is the authorization wait time. + AUTH_TIMEOUT = 2 * time.Second + + // DEFAULT_PING_INTERVAL is how often pings are sent to clients, etc... + DEFAULT_PING_INTERVAL = 2 * time.Minute + + // DEFAULT_PING_MAX_OUT is maximum allowed pings outstanding before disconnect. + DEFAULT_PING_MAX_OUT = 2 + + // CR_LF string + CR_LF = "\r\n" + + // LEN_CR_LF hold onto the computed size. + LEN_CR_LF = len(CR_LF) + + // DEFAULT_FLUSH_DEADLINE is the write/flush deadlines. + DEFAULT_FLUSH_DEADLINE = 10 * time.Second + + // DEFAULT_HTTP_PORT is the default monitoring port. + DEFAULT_HTTP_PORT = 8222 + + // DEFAULT_HTTP_BASE_PATH is the default base path for monitoring. + DEFAULT_HTTP_BASE_PATH = "/" + + // ACCEPT_MIN_SLEEP is the minimum acceptable sleep times on temporary errors. + ACCEPT_MIN_SLEEP = 10 * time.Millisecond + + // ACCEPT_MAX_SLEEP is the maximum acceptable sleep times on temporary errors + ACCEPT_MAX_SLEEP = 1 * time.Second + + // DEFAULT_ROUTE_CONNECT Route solicitation intervals. + DEFAULT_ROUTE_CONNECT = 1 * time.Second + + // DEFAULT_ROUTE_CONNECT_MAX Route solicitation intervals (max). + DEFAULT_ROUTE_CONNECT_MAX = 30 * time.Second + + // DEFAULT_ROUTE_RECONNECT Route reconnect delay. + DEFAULT_ROUTE_RECONNECT = 1 * time.Second + + // DEFAULT_ROUTE_DIAL Route dial timeout. + DEFAULT_ROUTE_DIAL = 1 * time.Second + + // DEFAULT_ROUTE_POOL_SIZE Route default pool size + DEFAULT_ROUTE_POOL_SIZE = 3 + + // DEFAULT_LEAF_NODE_RECONNECT LeafNode reconnect interval. + DEFAULT_LEAF_NODE_RECONNECT = time.Second + + // DEFAULT_LEAF_TLS_TIMEOUT TLS timeout for LeafNodes + DEFAULT_LEAF_TLS_TIMEOUT = 2 * time.Second + + // PROTO_SNIPPET_SIZE is the default size of proto to print on parse errors. + PROTO_SNIPPET_SIZE = 32 + + // MAX_CONTROL_LINE_SNIPPET_SIZE is the default size of proto to print on max control line errors. + MAX_CONTROL_LINE_SNIPPET_SIZE = 128 + + // MAX_MSG_ARGS Maximum possible number of arguments from MSG proto. + MAX_MSG_ARGS = 4 + + // MAX_RMSG_ARGS Maximum possible number of arguments from RMSG proto. + MAX_RMSG_ARGS = 6 + + // MAX_HMSG_ARGS Maximum possible number of arguments from HMSG proto. + MAX_HMSG_ARGS = 7 + + // MAX_PUB_ARGS Maximum possible number of arguments from PUB proto. + MAX_PUB_ARGS = 3 + + // MAX_HPUB_ARGS Maximum possible number of arguments from HPUB proto. + MAX_HPUB_ARGS = 4 + + // MAX_RSUB_ARGS Maximum possible number of arguments from a RS+/LS+ proto. + MAX_RSUB_ARGS = 6 + + // DEFAULT_MAX_CLOSED_CLIENTS is the maximum number of closed connections we hold onto. + DEFAULT_MAX_CLOSED_CLIENTS = 10000 + + // DEFAULT_LAME_DUCK_DURATION is the time in which the server spreads + // the closing of clients when signaled to go in lame duck mode. + DEFAULT_LAME_DUCK_DURATION = 2 * time.Minute + + // DEFAULT_LAME_DUCK_GRACE_PERIOD is the duration the server waits, after entering + // lame duck mode, before starting closing client connections. + DEFAULT_LAME_DUCK_GRACE_PERIOD = 10 * time.Second + + // DEFAULT_LEAFNODE_INFO_WAIT Route dial timeout. + DEFAULT_LEAFNODE_INFO_WAIT = 1 * time.Second + + // DEFAULT_LEAFNODE_PORT is the default port for remote leafnode connections. + DEFAULT_LEAFNODE_PORT = 7422 + + // DEFAULT_CONNECT_ERROR_REPORTS is the number of attempts at which a + // repeated failed route, gateway or leaf node connection is reported. + // This is used for initial connection, that is, when the server has + // never had a connection to the given endpoint. Once connected, and + // if a disconnect occurs, DEFAULT_RECONNECT_ERROR_REPORTS is used + // instead. + // The default is to report every 3600 attempts (roughly every hour). + DEFAULT_CONNECT_ERROR_REPORTS = 3600 + + // DEFAULT_RECONNECT_ERROR_REPORTS is the default number of failed + // attempt to reconnect a route, gateway or leaf node connection. + // The default is to report every attempt. + DEFAULT_RECONNECT_ERROR_REPORTS = 1 + + // DEFAULT_RTT_MEASUREMENT_INTERVAL is how often we want to measure RTT from + // this server to clients, routes, gateways or leafnode connections. + DEFAULT_RTT_MEASUREMENT_INTERVAL = time.Hour + + // DEFAULT_ALLOW_RESPONSE_MAX_MSGS is the default number of responses allowed + // for a reply subject. + DEFAULT_ALLOW_RESPONSE_MAX_MSGS = 1 + + // DEFAULT_ALLOW_RESPONSE_EXPIRATION is the default time allowed for a given + // dynamic response permission. + DEFAULT_ALLOW_RESPONSE_EXPIRATION = 2 * time.Minute + + // DEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD is the default time that the system will + // expect a service export response to be delivered. This is used in corner cases for + // time based cleanup of reverse mapping structures. + DEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD = 2 * time.Minute + + // DEFAULT_SERVICE_LATENCY_SAMPLING is the default sampling rate for service + // latency metrics + DEFAULT_SERVICE_LATENCY_SAMPLING = 100 + + // DEFAULT_SYSTEM_ACCOUNT + DEFAULT_SYSTEM_ACCOUNT = "$SYS" + + // DEFAULT GLOBAL_ACCOUNT + DEFAULT_GLOBAL_ACCOUNT = "$G" + + // DEFAULT_FETCH_TIMEOUT is the default time that the system will wait for an account fetch to return. + DEFAULT_ACCOUNT_FETCH_TIMEOUT = 1900 * time.Millisecond +) diff --git a/server/core_benchmarks_test.go b/server/core_benchmarks_test.go new file mode 100644 index 0000000..d98adf7 --- /dev/null +++ b/server/core_benchmarks_test.go @@ -0,0 +1,607 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "crypto/tls" + "errors" + "fmt" + "math" + "math/rand/v2" + "os" + "strconv" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +func BenchmarkCoreRequestReply(b *testing.B) { + const ( + subject = "test-subject" + ) + + messageSizes := []int64{ + 1024, // 1kb + 4096, // 4kb + 40960, // 40kb + 409600, // 400kb + } + + for _, messageSize := range messageSizes { + b.Run(fmt.Sprintf("msgSz=%db", messageSize), func(b *testing.B) { + // Start server + serverOpts := DefaultOptions() + server := RunServer(serverOpts) + defer server.Shutdown() + + clientUrl := server.ClientURL() + + // Create "echo" subscriber + ncSub, err := nats.Connect(clientUrl) + if err != nil { + b.Fatal(err) + } + defer ncSub.Close() + sub, err := ncSub.Subscribe(subject, func(msg *nats.Msg) { + // Responder echoes the request payload as-is + msg.Respond(msg.Data) + }) + defer sub.Unsubscribe() + if err != nil { + b.Fatal(err) + } + + // Create publisher + ncPub, err := nats.Connect(clientUrl) + if err != nil { + b.Fatal(err) + } + defer ncPub.Close() + + errors := 0 + + // Create message + messageData := make([]byte, messageSize) + rand.NewChaCha8([32]byte{42}).Read(messageData) + + b.SetBytes(messageSize) + + // Benchmark + b.ResetTimer() + for i := 0; i < b.N; i++ { + fastRandomMutation(messageData, 10) + + _, err := ncPub.Request(subject, messageData, time.Second) + if err != nil { + errors++ + } + } + b.StopTimer() + + b.ReportMetric(float64(errors), "errors") + }) + } +} + +func BenchmarkCoreTLSFanOut(b *testing.B) { + const ( + subject = "test-subject" + configsBasePath = "./configs/tls" + maxPendingMessages = 25 + maxPendingBytes = 15 * 1024 * 1024 // 15MiB + ) + + keyTypeCases := []string{ + "none", + "ed25519", + "rsa-1024", + "rsa-2048", + "rsa-4096", + } + messageSizeCases := []int64{ + 512 * 1024, // 512Kib + } + numSubsCases := []int{ + 5, + } + + // Custom error handler that ignores ErrSlowConsumer. + // Lots of them are expected in this benchmark which indiscriminately publishes at a rate higher + // than what the server can relay to subscribers. + ignoreSlowConsumerErrorHandler := func(conn *nats.Conn, s *nats.Subscription, err error) { + if errors.Is(err, nats.ErrSlowConsumer) { + // Swallow this error + } else { + _, _ = fmt.Fprintf(os.Stderr, "Warning: %s\n", err) + } + } + + for _, keyType := range keyTypeCases { + b.Run( + fmt.Sprintf("keyType=%s", keyType), + func(b *testing.B) { + for _, messageSize := range messageSizeCases { + b.Run( + fmt.Sprintf("msgSz=%db", messageSize), + func(b *testing.B) { + for _, numSubs := range numSubsCases { + b.Run( + fmt.Sprintf("subs=%d", numSubs), + func(b *testing.B) { + // Start server + configPath := fmt.Sprintf("%s/tls-%s.conf", configsBasePath, keyType) + server, _ := RunServerWithConfig(configPath) + defer server.Shutdown() + + opts := []nats.Option{ + nats.MaxReconnects(-1), + nats.ReconnectWait(0), + nats.ErrorHandler(ignoreSlowConsumerErrorHandler), + } + + if keyType != "none" { + opts = append(opts, nats.Secure(&tls.Config{ + InsecureSkipVerify: true, + })) + } + + clientUrl := server.ClientURL() + + // Count of messages received for by each subscriber + counters := make([]int, numSubs) + + // Wait group for subscribers to signal they received b.N messages + var wg sync.WaitGroup + wg.Add(numSubs) + + // Create subscribers + for i := 0; i < numSubs; i++ { + subIndex := i + ncSub, err := nats.Connect(clientUrl, opts...) + if err != nil { + b.Fatal(err) + } + defer ncSub.Close() + sub, err := ncSub.Subscribe(subject, func(msg *nats.Msg) { + counters[subIndex] += 1 + if counters[subIndex] == b.N { + wg.Done() + } + }) + if err != nil { + b.Fatalf("failed to subscribe: %s", err) + } + err = sub.SetPendingLimits(maxPendingMessages, maxPendingBytes) + if err != nil { + b.Fatalf("failed to set pending limits: %s", err) + } + defer sub.Unsubscribe() + if err != nil { + b.Fatal(err) + } + } + + // publisher + ncPub, err := nats.Connect(clientUrl, opts...) + if err != nil { + b.Fatal(err) + } + defer ncPub.Close() + + errorCount := 0 + + // random bytes as payload + messageData := make([]byte, messageSize) + rand.NewChaCha8([32]byte{42}).Read(messageData) + + quitCh := make(chan bool, 1) + + publish := func() { + for { + select { + case <-quitCh: + return + default: + // continue publishing + } + + fastRandomMutation(messageData, 10) + err := ncPub.Publish(subject, messageData) + if err != nil { + errorCount += 1 + } + } + } + + // Set bytes per operation + b.SetBytes(messageSize) + // Start the clock + b.ResetTimer() + // Start publishing as fast as the server allows + go publish() + // Wait for all subscribers to have delivered b.N messages + wg.Wait() + // Stop the clock + b.StopTimer() + + // Stop publisher + quitCh <- true + + b.ReportMetric(float64(errorCount), "errors") + }, + ) + } + }, + ) + } + }, + ) + } +} + +func BenchmarkCoreFanOut(b *testing.B) { + const ( + subject = "test-subject" + maxPendingMessages = 25 + maxPendingBytes = 15 * 1024 * 1024 // 15MiB + ) + + messageSizeCases := []int64{ + 100, // 100B + 1024, // 1KiB + 10240, // 10KiB + 512 * 1024, // 512KiB + } + numSubsCases := []int{ + 3, + 5, + 10, + } + + // Custom error handler that ignores ErrSlowConsumer. + // Lots of them are expected in this benchmark which indiscriminately publishes at a rate higher + // than what the server can relay to subscribers. + ignoreSlowConsumerErrorHandler := func(_ *nats.Conn, _ *nats.Subscription, err error) { + if errors.Is(err, nats.ErrSlowConsumer) { + // Swallow this error + } else { + _, _ = fmt.Fprintf(os.Stderr, "Warning: %s\n", err) + } + } + + for _, messageSize := range messageSizeCases { + b.Run( + fmt.Sprintf("msgSz=%db", messageSize), + func(b *testing.B) { + for _, numSubs := range numSubsCases { + b.Run( + fmt.Sprintf("subs=%d", numSubs), + func(b *testing.B) { + // Start server + defaultOpts := DefaultOptions() + server := RunServer(defaultOpts) + defer server.Shutdown() + + opts := []nats.Option{ + nats.MaxReconnects(-1), + nats.ReconnectWait(0), + nats.ErrorHandler(ignoreSlowConsumerErrorHandler), + } + + clientUrl := server.ClientURL() + + // Count of messages received for by each subscriber + counters := make([]int, numSubs) + + // Wait group for subscribers to signal they received b.N messages + var wg sync.WaitGroup + wg.Add(numSubs) + + // Create subscribers + for i := 0; i < numSubs; i++ { + subIndex := i + ncSub, err := nats.Connect(clientUrl, opts...) + if err != nil { + b.Fatal(err) + } + defer ncSub.Close() + sub, err := ncSub.Subscribe(subject, func(_ *nats.Msg) { + counters[subIndex] += 1 + if counters[subIndex] == b.N { + wg.Done() + } + }) + if err != nil { + b.Fatalf("failed to subscribe: %s", err) + } + err = sub.SetPendingLimits(maxPendingMessages, maxPendingBytes) + if err != nil { + b.Fatalf("failed to set pending limits: %s", err) + } + defer sub.Unsubscribe() + } + + // publisher + ncPub, err := nats.Connect(clientUrl, opts...) + if err != nil { + b.Fatal(err) + } + defer ncPub.Close() + + errorCount := 0 + + // random bytes as payload + messageData := make([]byte, messageSize) + rand.NewChaCha8([32]byte{42}).Read(messageData) + + quitCh := make(chan bool, 1) + + publish := func() { + for { + select { + case <-quitCh: + return + default: + // continue publishing + } + + fastRandomMutation(messageData, 10) + err := ncPub.Publish(subject, messageData) + if err != nil { + errorCount += 1 + } + } + } + + // Set bytes per operation + b.SetBytes(messageSize) + // Start the clock + b.ResetTimer() + // Start publishing as fast as the server allows + go publish() + // Wait for all subscribers to have delivered b.N messages + wg.Wait() + // Stop the clock + b.StopTimer() + + // Stop publisher + quitCh <- true + + b.ReportMetric(100*float64(errorCount)/float64(b.N), "%error") + }, + ) + } + }, + ) + } +} + +func BenchmarkCoreFanIn(b *testing.B) { + type BenchPublisher struct { + // nats connection for this publisher + conn *nats.Conn + // number of publishing errors encountered + publishErrors int + // number of messages published + publishCounter int + // quit channel which will terminate publishing + quitCh chan bool + // message data buffer + messageData []byte + } + + const subjectBaseName = "test-subject" + + messageSizeCases := []int64{ + 100, // 100B + 1024, // 1KiB + 10240, // 10KiB + 512 * 1024, // 512KiB + } + numPubsCases := []int{ + 3, + 5, + 10, + } + + // Custom error handler that ignores ErrSlowConsumer. + // Lots of them are expected in this benchmark which indiscriminately publishes at a rate higher + // than what the server can relay to subscribers. + ignoreSlowConsumerErrorHandler := func(_ *nats.Conn, _ *nats.Subscription, err error) { + if errors.Is(err, nats.ErrSlowConsumer) { + // Swallow this error + } else { + _, _ = fmt.Fprintf(os.Stderr, "Warning: %s\n", err) + } + } + + workload := func(b *testing.B, clientUrl string, numPubs int, messageSize int64) { + // connection options + opts := []nats.Option{ + nats.MaxReconnects(-1), + nats.ReconnectWait(0), + nats.ErrorHandler(ignoreSlowConsumerErrorHandler), + } + + // waits for all publishers sub-routines and for main thread to be ready + var publishersReadyWg sync.WaitGroup + publishersReadyWg.Add(numPubs + 1) + + // wait group to ensure all publishers have been torn down + var finishedPublishersWg sync.WaitGroup + finishedPublishersWg.Add(numPubs) + + publishers := make([]BenchPublisher, numPubs) + // create N publishers + for i := range publishers { + // create publisher connection + ncPub, err := nats.Connect(clientUrl, opts...) + if err != nil { + b.Fatal(err) + } + defer ncPub.Close() + + // create bench publisher object + publisher := BenchPublisher{ + conn: ncPub, + publishErrors: 0, + publishCounter: 0, + quitCh: make(chan bool, 1), + messageData: make([]byte, messageSize), + } + rand.NewChaCha8([32]byte{byte(i)}).Read(publisher.messageData) + publishers[i] = publisher + } + + // total number of publishers that have published b.N to the subscriber successfully + completedPublishersCount := 0 + + // wait group blocks main thread until publish workload is completed, it is decremented after subscriber receives b.N messages from all publishers + var benchCompleteWg sync.WaitGroup + benchCompleteWg.Add(1) + + // start subscriber + ncSub, err := nats.Connect(clientUrl, opts...) + if err != nil { + b.Fatal(err) + } + defer ncSub.Close() + + // subscriber + ncSub.Subscribe(fmt.Sprintf("%s.*", subjectBaseName), func(msg *nats.Msg) { + // get the publisher id from subject + pubIdx, err := strconv.Atoi(msg.Subject[len(subjectBaseName)+1:]) + if err != nil { + b.Fatal(err) + } + + // message successfully received from publisher + publishers[pubIdx].publishCounter += 1 + + // subscriber has received a total of b.N messages from this publisher + if publishers[pubIdx].publishCounter == b.N { + completedPublishersCount++ + // every publisher has successfully sent b.N messages to subscriber + if completedPublishersCount == numPubs { + benchCompleteWg.Done() + } + } + }) + + // start publisher sub-routines + for i := range publishers { + go func(pubId int) { + // publisher sub-routine initialized + publishersReadyWg.Done() + + publisher := publishers[pubId] + subject := fmt.Sprintf("%s.%d", subjectBaseName, pubId) + + // signal that this publisher has been torn down + defer finishedPublishersWg.Done() + + // wait till all other publishers are ready to start workload + publishersReadyWg.Wait() + + // publish until quitCh is closed + for { + select { + case <-publisher.quitCh: + return + default: + // continue publishing + } + fastRandomMutation(publisher.messageData, 10) + err := publisher.conn.Publish(subject, publisher.messageData) + if err != nil { + publisher.publishErrors += 1 + } + } + }(i) + } + + // set bytes per operation + b.SetBytes(messageSize) + // main thread is ready + publishersReadyWg.Done() + // wait till publishers are ready + publishersReadyWg.Wait() + + // start the clock + b.ResetTimer() + // wait till termination cond reached + benchCompleteWg.Wait() + // stop the clock + b.StopTimer() + + // send quit signal to all publishers + for i := range publishers { + publishers[i].quitCh <- true + } + // wait for all publishers to shutdown + finishedPublishersWg.Wait() + + // sum errors from all publishers + totalErrors := 0 + for _, publisher := range publishers { + totalErrors += publisher.publishErrors + } + // sum total messages sent from all publishers + totalMessages := 0 + for _, publisher := range publishers { + totalMessages += publisher.publishCounter + } + errorRate := 100 * float64(totalErrors) / float64(totalMessages) + + // report error rate + b.ReportMetric(errorRate, "%error") + } + + // benchmark case matrix + for _, messageSize := range messageSizeCases { + b.Run( + fmt.Sprintf("msgSz=%db", messageSize), + func(b *testing.B) { + for _, numPubs := range numPubsCases { + b.Run( + fmt.Sprintf("pubs=%d", numPubs), + func(b *testing.B) { + // start server + defaultOpts := DefaultOptions() + server := RunServer(defaultOpts) + defer server.Shutdown() + + // get connection string + clientUrl := server.ClientURL() + + // run fan-in workload + workload(b, clientUrl, numPubs, messageSize) + }, + ) + } + }, + ) + } +} + +// fastRandomMutation performs a minor in-place mutation to the given buffer. +// This is useful in benchmark to avoid sending the same payload every time (which could result in some optimizations +// we do not want to measure), while not slowing down the benchmark with a full payload generated for each operation. +func fastRandomMutation(data []byte, mutations int) { + for i := 0; i < mutations; i++ { + data[rand.Uint32N(uint32(len(data)))] = byte(rand.Uint32() % math.MaxUint8) + } +} diff --git a/server/cron.go b/server/cron.go new file mode 100644 index 0000000..74286d5 --- /dev/null +++ b/server/cron.go @@ -0,0 +1,327 @@ +// Copyright 2025-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Based on code from https://github.com/robfig/cron +// Copyright (C) 2012 Rob Figueiredo +// All Rights Reserved. +// +// MIT LICENSE +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +package server + +import ( + "errors" + "fmt" + "math" + "strconv" + "strings" + "time" +) + +// parseCron parses the given cron pattern and returns the next time it will fire based on the provided ts. +func parseCron(pattern string, loc *time.Location, ts int64) (time.Time, error) { + fields := strings.Fields(pattern) + if len(fields) != 6 { + return time.Time{}, fmt.Errorf("pattern requires 6 fields, got %d", len(fields)) + } + + // If no time zone is passed, default to UTC. + if loc == nil { + loc = time.UTC + } + + // Parse each field. + var err error + var second, minute, hour, dayOfMonth, month, dayOfWeek uint64 + if second, err = getField(fields[0], seconds); err != nil { + return time.Time{}, err + } + if minute, err = getField(fields[1], minutes); err != nil { + return time.Time{}, err + } + if hour, err = getField(fields[2], hours); err != nil { + return time.Time{}, err + } + if dayOfMonth, err = getField(fields[3], dom); err != nil { + return time.Time{}, err + } + if month, err = getField(fields[4], months); err != nil { + return time.Time{}, err + } + if dayOfWeek, err = getField(fields[5], dow); err != nil { + return time.Time{}, err + } + + // General approach + // + // For Month, Day, Hour, Minute, Second: + // Check if the time value matches. If yes, continue to the next field. + // If the field doesn't match the schedule, then increment the field until it matches. + // While incrementing the field, a wrap-around brings it back to the beginning + // of the field list (since it is necessary to re-verify previous field values) + next := time.Unix(0, ts).In(loc) + + // Start at the earliest possible time (the upcoming second). + next = next.Truncate(time.Second).Add(time.Second) + + // This flag indicates whether a field has been truncated at one point. + truncated := false + + // If no time is found within five years, return error. + yearLimit := next.Year() + 5 + +WRAP: + if next.Year() > yearLimit { + return time.Time{}, errors.New("pattern exceeds maximum range") + } + for 1< 1 { + extra = 0 + } + default: + return 0, fmt.Errorf("too many slashes: %s", expr) + } + + if start < r.min { + return 0, fmt.Errorf("beginning of range (%d) below minimum (%d): %s", start, r.min, expr) + } + if end > r.max { + return 0, fmt.Errorf("end of range (%d) above maximum (%d): %s", end, r.max, expr) + } + if start > end { + return 0, fmt.Errorf("beginning of range (%d) beyond end of range (%d): %s", start, end, expr) + } + if step == 0 { + return 0, fmt.Errorf("step of range should be a positive number: %s", expr) + } + return getBits(start, end, step) | extra, nil +} + +// parseIntOrName returns the (possibly-named) integer contained in expr. +func parseIntOrName(expr string, names map[string]uint) (uint, error) { + if names != nil { + if namedInt, ok := names[strings.ToLower(expr)]; ok { + return namedInt, nil + } + } + return mustParseInt(expr) +} + +// mustParseInt parses the given expression as an int or returns an error. +func mustParseInt(expr string) (uint, error) { + num, err := strconv.Atoi(expr) + if err != nil { + return 0, fmt.Errorf("failed to parse int from %s: %s", expr, err) + } + if num < 0 { + return 0, fmt.Errorf("negative number (%d) not allowed: %s", num, expr) + } + return uint(num), nil +} + +// getBits sets all bits in the range [min, max], modulo the given step size. +func getBits(min, max, step uint) uint64 { + var bits uint64 + + // If step is 1, use shifts. + if step == 1 { + return ^(math.MaxUint64 << (max + 1)) & (math.MaxUint64 << min) + } + + // Else, use a simple loop. + for i := min; i <= max; i += step { + bits |= 1 << i + } + return bits +} + +// bounds provides a range of acceptable values (plus a map of name to value). +type bounds struct { + min, max uint + names map[string]uint +} + +// The bounds for each field. +var ( + seconds = bounds{0, 59, nil} + minutes = bounds{0, 59, nil} + hours = bounds{0, 23, nil} + dom = bounds{1, 31, nil} + months = bounds{1, 12, map[string]uint{ + "jan": 1, + "feb": 2, + "mar": 3, + "apr": 4, + "may": 5, + "jun": 6, + "jul": 7, + "aug": 8, + "sep": 9, + "oct": 10, + "nov": 11, + "dec": 12, + }} + dow = bounds{0, 6, map[string]uint{ + "sun": 0, + "mon": 1, + "tue": 2, + "wed": 3, + "thu": 4, + "fri": 5, + "sat": 6, + }} +) + +const ( + // Set the top bit if a star was included in the expression. + starBit = 1 << 63 +) + +// dayMatches returns true if the schedule's day-of-week and day-of-month +// restrictions are satisfied by the given time. +func dayMatches(dayOfMonth, dayOfWeek uint64, t time.Time) bool { + var ( + domMatch = 1< 0 + dowMatch = 1< 0 + ) + if dayOfMonth&starBit > 0 || dayOfWeek&starBit > 0 { + return domMatch && dowMatch + } + return domMatch || dowMatch +} diff --git a/server/dios.go b/server/dios.go new file mode 100644 index 0000000..8ad3aff --- /dev/null +++ b/server/dios.go @@ -0,0 +1,85 @@ +// Copyright 2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "sync/atomic" + "time" +) + +const defaultConcurrentIOs = 4096 +const minConcurrentIOs = 4 +const maxConcurrentIOs = 8192 + +// Used to limit number of disk IO calls in flight since they could all be blocking an OS thread. +// https://github.com/nats-io/nats-server/issues/2742 +type diskIOSemaphore struct { + ch chan struct{} + waiters atomic.Int64 + waits atomic.Uint64 + waitNanos atomic.Uint64 + maxWaitNanos atomic.Uint64 +} + +func newDiskIOSemaphore(n int) *diskIOSemaphore { + n = max(minConcurrentIOs, min(n, maxConcurrentIOs)) + d := &diskIOSemaphore{ch: make(chan struct{}, n)} + for range n { + d.ch <- struct{}{} + } + return d +} + +func defaultDiskIOSemaphore() *diskIOSemaphore { + // The disk IO semaphore used to be sized based on the number + // of CPU cores. That policy led to poor use of devices that + // can handle many requests in parallel, so simply cap the + // number of concurrent IO requests handed to the Go runtime. + return newDiskIOSemaphore(defaultConcurrentIOs) +} + +func (d *diskIOSemaphore) acquire() { + select { + case <-d.ch: + return + default: + // No slot available, count this + // waiter before blocking. + d.waiters.Add(1) + start := time.Now() + <-d.ch + waited := time.Since(start) + d.waiters.Add(-1) + d.countWait(uint64(waited.Nanoseconds())) + } +} + +func (d *diskIOSemaphore) countWait(ns uint64) { + d.waits.Add(1) + d.waitNanos.Add(ns) + for { + cur := d.maxWaitNanos.Load() + if ns <= cur || d.maxWaitNanos.CompareAndSwap(cur, ns) { + return + } + } +} + +func (d *diskIOSemaphore) release() { + d.ch <- struct{}{} +} + +func (d *diskIOSemaphore) cap() int { + return cap(d.ch) +} diff --git a/server/dirstore.go b/server/dirstore.go new file mode 100644 index 0000000..e19a732 --- /dev/null +++ b/server/dirstore.go @@ -0,0 +1,727 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "container/heap" + "container/list" + "crypto/sha256" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/nats-io/nkeys" + + "github.com/nats-io/jwt/v2" // only used to decode, not for storage +) + +const ( + fileExtension = ".jwt" +) + +// validatePathExists checks that the provided path exists and is a dir if requested +func validatePathExists(path string, dir bool) (string, error) { + if path == _EMPTY_ { + return _EMPTY_, errors.New("path is not specified") + } + + abs, err := filepath.Abs(path) + if err != nil { + return _EMPTY_, fmt.Errorf("error parsing path [%s]: %v", abs, err) + } + + var finfo os.FileInfo + if finfo, err = os.Stat(abs); err != nil { + if os.IsNotExist(err) { + return _EMPTY_, fmt.Errorf("the path [%s] doesn't exist", abs) + } + return _EMPTY_, fmt.Errorf("error accessing path [%s]: %v", abs, err) + } + + mode := finfo.Mode() + if dir && mode.IsRegular() { + return _EMPTY_, fmt.Errorf("the path [%s] is not a directory", abs) + } + + if !dir && mode.IsDir() { + return _EMPTY_, fmt.Errorf("the path [%s] is not a file", abs) + } + + return abs, nil +} + +// ValidateDirPath checks that the provided path exists and is a dir +func validateDirPath(path string) (string, error) { + return validatePathExists(path, true) +} + +// JWTChanged functions are called when the store file watcher notices a JWT changed +type JWTChanged func(publicKey string) + +// DirJWTStore implements the JWT Store interface, keeping JWTs in an optionally sharded +// directory structure +type DirJWTStore struct { + sync.Mutex + directory string + shard bool + readonly bool + deleteType deleteType + operator map[string]struct{} + expiration *expirationTracker + changed JWTChanged + deleted JWTChanged +} + +func newDir(dirPath string, create bool) (string, error) { + fullPath, err := validateDirPath(dirPath) + if err != nil { + if !create { + return _EMPTY_, err + } + if err = os.MkdirAll(dirPath, defaultDirPerms); err != nil { + return _EMPTY_, err + } + if fullPath, err = validateDirPath(dirPath); err != nil { + return _EMPTY_, err + } + } + return fullPath, nil +} + +// future proofing in case new options will be added +type dirJWTStoreOption any + +// Creates a directory based jwt store. +// Reads files only, does NOT watch directories and files. +func NewImmutableDirJWTStore(dirPath string, shard bool, _ ...dirJWTStoreOption) (*DirJWTStore, error) { + theStore, err := NewDirJWTStore(dirPath, shard, false, nil) + if err != nil { + return nil, err + } + theStore.readonly = true + return theStore, nil +} + +// Creates a directory based jwt store. +// Operates on files only, does NOT watch directories and files. +func NewDirJWTStore(dirPath string, shard bool, create bool, _ ...dirJWTStoreOption) (*DirJWTStore, error) { + fullPath, err := newDir(dirPath, create) + if err != nil { + return nil, err + } + theStore := &DirJWTStore{ + directory: fullPath, + shard: shard, + } + return theStore, nil +} + +type deleteType int + +const ( + NoDelete deleteType = iota + RenameDeleted + HardDelete +) + +// Creates a directory based jwt store. +// +// When ttl is set deletion of file is based on it and not on the jwt expiration +// To completely disable expiration (including expiration in jwt) set ttl to max duration time.Duration(math.MaxInt64) +// +// limit defines how many files are allowed at any given time. Set to math.MaxInt64 to disable. +// evictOnLimit determines the behavior once limit is reached. +// * true - Evict based on lru strategy +// * false - return an error +func NewExpiringDirJWTStore(dirPath string, shard bool, create bool, delete deleteType, expireCheck time.Duration, limit int64, + evictOnLimit bool, ttl time.Duration, changeNotification JWTChanged, _ ...dirJWTStoreOption) (*DirJWTStore, error) { + fullPath, err := newDir(dirPath, create) + if err != nil { + return nil, err + } + theStore := &DirJWTStore{ + directory: fullPath, + shard: shard, + deleteType: delete, + changed: changeNotification, + } + if expireCheck <= 0 { + if ttl != 0 { + expireCheck = ttl / 2 + } + if expireCheck == 0 || expireCheck > time.Minute { + expireCheck = time.Minute + } + } + if limit <= 0 { + limit = math.MaxInt64 + } + theStore.startExpiring(expireCheck, limit, evictOnLimit, ttl) + theStore.Lock() + err = filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { + if strings.HasSuffix(path, fileExtension) { + if theJwt, err := os.ReadFile(path); err == nil { + hash := sha256.Sum256(theJwt) + _, file := filepath.Split(path) + theStore.expiration.track(strings.TrimSuffix(file, fileExtension), &hash, string(theJwt)) + } + } + return nil + }) + theStore.Unlock() + if err != nil { + theStore.Close() + return nil, err + } + return theStore, err +} + +func (store *DirJWTStore) IsReadOnly() bool { + return store.readonly +} + +func (store *DirJWTStore) LoadAcc(publicKey string) (string, error) { + return store.load(publicKey) +} + +func (store *DirJWTStore) SaveAcc(publicKey string, theJWT string) error { + return store.save(publicKey, theJWT) +} + +func (store *DirJWTStore) LoadAct(hash string) (string, error) { + return store.load(hash) +} + +func (store *DirJWTStore) SaveAct(hash string, theJWT string) error { + return store.save(hash, theJWT) +} + +func (store *DirJWTStore) Close() { + store.Lock() + defer store.Unlock() + if store.expiration != nil { + store.expiration.close() + store.expiration = nil + } +} + +// Pack up to maxJWTs into a package +func (store *DirJWTStore) Pack(maxJWTs int) (string, error) { + count := 0 + var pack []string + if maxJWTs > 0 { + pack = make([]string, 0, maxJWTs) + } else { + pack = []string{} + } + store.Lock() + err := filepath.Walk(store.directory, func(path string, info os.FileInfo, err error) error { + if info != nil && !info.IsDir() && strings.HasSuffix(path, fileExtension) { // this is a JWT + if count == maxJWTs { // won't match negative + return nil + } + pubKey := strings.TrimSuffix(filepath.Base(path), fileExtension) + if store.expiration != nil { + if _, ok := store.expiration.idx[pubKey]; !ok { + return nil // only include indexed files + } + } + jwtBytes, err := os.ReadFile(path) + if err != nil { + return err + } + if store.expiration != nil { + claim, err := jwt.DecodeGeneric(string(jwtBytes)) + if err == nil && claim.Expires > 0 && claim.Expires < time.Now().Unix() { + return nil + } + } + pack = append(pack, fmt.Sprintf("%s|%s", pubKey, string(jwtBytes))) + count++ + } + return nil + }) + store.Unlock() + if err != nil { + return _EMPTY_, err + } else { + return strings.Join(pack, "\n"), nil + } +} + +// Pack up to maxJWTs into a message and invoke callback with it +func (store *DirJWTStore) PackWalk(maxJWTs int, cb func(partialPackMsg string)) error { + if maxJWTs <= 0 || cb == nil { + return errors.New("bad arguments to PackWalk") + } + var packMsg []string + store.Lock() + dir := store.directory + exp := store.expiration + store.Unlock() + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if info != nil && !info.IsDir() && strings.HasSuffix(path, fileExtension) { // this is a JWT + pubKey := strings.TrimSuffix(filepath.Base(path), fileExtension) + store.Lock() + if exp != nil { + if _, ok := exp.idx[pubKey]; !ok { + store.Unlock() + return nil // only include indexed files + } + } + store.Unlock() + jwtBytes, err := os.ReadFile(path) + if err != nil { + return err + } + if len(jwtBytes) == 0 { + // Skip if no contents in the JWT. + return nil + } + if exp != nil { + claim, err := jwt.DecodeGeneric(string(jwtBytes)) + if err == nil && claim.Expires > 0 && claim.Expires < time.Now().Unix() { + return nil + } + } + packMsg = append(packMsg, fmt.Sprintf("%s|%s", pubKey, string(jwtBytes))) + if len(packMsg) == maxJWTs { // won't match negative + cb(strings.Join(packMsg, "\n")) + packMsg = nil + } + } + return nil + }) + if packMsg != nil { + cb(strings.Join(packMsg, "\n")) + } + return err +} + +// Merge takes the JWTs from package and adds them to the store +// Merge is destructive in the sense that it doesn't check if the JWT +// is newer or anything like that. +func (store *DirJWTStore) Merge(pack string) error { + newJWTs := strings.Split(pack, "\n") + for _, line := range newJWTs { + if line == _EMPTY_ { // ignore blank lines + continue + } + split := strings.Split(line, "|") + if len(split) != 2 { + return fmt.Errorf("line in package didn't contain 2 entries: %q", line) + } + pubKey := split[0] + if !nkeys.IsValidPublicAccountKey(pubKey) { + return fmt.Errorf("key to merge is not a valid public account key") + } + if err := store.saveIfNewer(pubKey, split[1]); err != nil { + return err + } + } + return nil +} + +func (store *DirJWTStore) Reload() error { + store.Lock() + exp := store.expiration + if exp == nil || store.readonly { + store.Unlock() + return nil + } + idx := exp.idx + changed := store.changed + isCache := store.expiration.evictOnLimit + // clear out indexing data structures + exp.heap = make([]*jwtItem, 0, len(exp.heap)) + exp.idx = make(map[string]*list.Element) + exp.lru = list.New() + exp.hash = [sha256.Size]byte{} + store.Unlock() + return filepath.Walk(store.directory, func(path string, info os.FileInfo, err error) error { + if strings.HasSuffix(path, fileExtension) { + if theJwt, err := os.ReadFile(path); err == nil { + hash := sha256.Sum256(theJwt) + _, file := filepath.Split(path) + pkey := strings.TrimSuffix(file, fileExtension) + notify := isCache // for cache, issue cb even when file not present (may have been evicted) + if i, ok := idx[pkey]; ok { + notify = !bytes.Equal(i.Value.(*jwtItem).hash[:], hash[:]) + } + store.Lock() + exp.track(pkey, &hash, string(theJwt)) + store.Unlock() + if notify && changed != nil { + changed(pkey) + } + } + } + return nil + }) +} + +func (store *DirJWTStore) pathForKey(publicKey string) string { + if len(publicKey) < 2 { + return _EMPTY_ + } + if !nkeys.IsValidPublicKey(publicKey) { + return _EMPTY_ + } + fileName := fmt.Sprintf("%s%s", publicKey, fileExtension) + if store.shard { + last := publicKey[len(publicKey)-2:] + return filepath.Join(store.directory, last, fileName) + } else { + return filepath.Join(store.directory, fileName) + } +} + +// Load checks the memory store and returns the matching JWT or an error +// Assumes lock is NOT held +func (store *DirJWTStore) load(publicKey string) (string, error) { + store.Lock() + defer store.Unlock() + if path := store.pathForKey(publicKey); path == _EMPTY_ { + return _EMPTY_, fmt.Errorf("invalid public key") + } else if data, err := os.ReadFile(path); err != nil { + return _EMPTY_, err + } else { + if store.expiration != nil { + store.expiration.updateTrack(publicKey) + } + return string(data), nil + } +} + +// write that keeps hash of all jwt in sync +// Assumes the lock is held. Does return true or an error never both. +func (store *DirJWTStore) write(path string, publicKey string, theJWT string) (bool, error) { + if len(theJWT) == 0 { + return false, fmt.Errorf("invalid JWT") + } + var newHash *[sha256.Size]byte + if store.expiration != nil { + h := sha256.Sum256([]byte(theJWT)) + newHash = &h + if v, ok := store.expiration.idx[publicKey]; ok { + store.expiration.updateTrack(publicKey) + // this write is an update, move to back + it := v.Value.(*jwtItem) + oldHash := it.hash[:] + if bytes.Equal(oldHash, newHash[:]) { + return false, nil + } + } else if int64(store.expiration.Len()) >= store.expiration.limit { + if !store.expiration.evictOnLimit { + return false, errors.New("jwt store is full") + } + // this write is an add, pick the least recently used value for removal + i := store.expiration.lru.Front().Value.(*jwtItem) + if err := os.Remove(store.pathForKey(i.publicKey)); err != nil { + return false, err + } else { + store.expiration.unTrack(i.publicKey) + } + } + } + if err := os.WriteFile(path, []byte(theJWT), defaultFilePerms); err != nil { + return false, err + } else if store.expiration != nil { + store.expiration.track(publicKey, newHash, theJWT) + } + return true, nil +} + +func (store *DirJWTStore) delete(publicKey string) error { + if store.readonly { + return fmt.Errorf("store is read-only") + } else if store.deleteType == NoDelete { + return fmt.Errorf("store is not set up to for delete") + } + store.Lock() + defer store.Unlock() + name := store.pathForKey(publicKey) + if store.deleteType == RenameDeleted { + if err := os.Rename(name, name+".deleted"); err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + } else if err := os.Remove(name); err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + store.expiration.unTrack(publicKey) + store.deleted(publicKey) + return nil +} + +// Save puts the JWT in a map by public key and performs update callbacks +// Assumes lock is NOT held +func (store *DirJWTStore) save(publicKey string, theJWT string) error { + if store.readonly { + return fmt.Errorf("store is read-only") + } + store.Lock() + path := store.pathForKey(publicKey) + if path == _EMPTY_ { + store.Unlock() + return fmt.Errorf("invalid public key") + } + dirPath := filepath.Dir(path) + if _, err := validateDirPath(dirPath); err != nil { + if err := os.MkdirAll(dirPath, defaultDirPerms); err != nil { + store.Unlock() + return err + } + } + changed, err := store.write(path, publicKey, theJWT) + cb := store.changed + store.Unlock() + if changed && cb != nil { + cb(publicKey) + } + return err +} + +// Assumes the lock is NOT held, and only updates if the jwt is new, or the one on disk is older +// When changed, invokes jwt changed callback +func (store *DirJWTStore) saveIfNewer(publicKey string, theJWT string) error { + if store.readonly { + return fmt.Errorf("store is read-only") + } + path := store.pathForKey(publicKey) + if path == _EMPTY_ { + return fmt.Errorf("invalid public key") + } + dirPath := filepath.Dir(path) + if _, err := validateDirPath(dirPath); err != nil { + if err := os.MkdirAll(dirPath, defaultDirPerms); err != nil { + return err + } + } + if _, err := os.Stat(path); err == nil { + if newJWT, err := jwt.DecodeGeneric(theJWT); err != nil { + return err + } else if existing, err := os.ReadFile(path); err != nil { + return err + } else if existingJWT, err := jwt.DecodeGeneric(string(existing)); err != nil { + // skip if it can't be decoded + } else if existingJWT.ID == newJWT.ID { + return nil + } else if existingJWT.IssuedAt > newJWT.IssuedAt { + return nil + } else if newJWT.Subject != publicKey { + return fmt.Errorf("jwt subject nkey and provided nkey do not match") + } else if existingJWT.Subject != newJWT.Subject { + return fmt.Errorf("subject of existing and new jwt do not match") + } + } + store.Lock() + cb := store.changed + changed, err := store.write(path, publicKey, theJWT) + store.Unlock() + if err != nil { + return err + } else if changed && cb != nil { + cb(publicKey) + } + return nil +} + +func xorAssign(lVal *[sha256.Size]byte, rVal [sha256.Size]byte) { + for i := range rVal { + (*lVal)[i] ^= rVal[i] + } +} + +// returns a hash representing all indexed jwt +func (store *DirJWTStore) Hash() [sha256.Size]byte { + store.Lock() + defer store.Unlock() + if store.expiration == nil { + return [sha256.Size]byte{} + } else { + return store.expiration.hash + } +} + +// An jwtItem is something managed by the priority queue +type jwtItem struct { + index int + publicKey string + expiration int64 // consists of unix time of expiration (ttl when set or jwt expiration) in seconds + hash [sha256.Size]byte +} + +// A expirationTracker implements heap.Interface and holds Items. +type expirationTracker struct { + heap []*jwtItem // sorted by jwtItem.expiration + idx map[string]*list.Element + lru *list.List // keep which jwt are least used + limit int64 // limit how many jwt are being tracked + evictOnLimit bool // when limit is hit, error or evict using lru + ttl time.Duration + hash [sha256.Size]byte // xor of all jwtItem.hash in idx + quit chan struct{} + wg sync.WaitGroup +} + +func (q *expirationTracker) Len() int { return len(q.heap) } + +func (q *expirationTracker) Less(i, j int) bool { + pq := q.heap + return pq[i].expiration < pq[j].expiration +} + +func (q *expirationTracker) Swap(i, j int) { + pq := q.heap + pq[i], pq[j] = pq[j], pq[i] + pq[i].index = i + pq[j].index = j +} + +func (q *expirationTracker) Push(x any) { + n := len(q.heap) + item := x.(*jwtItem) + item.index = n + q.heap = append(q.heap, item) + q.idx[item.publicKey] = q.lru.PushBack(item) +} + +func (q *expirationTracker) Pop() any { + old := q.heap + n := len(old) + item := old[n-1] + old[n-1] = nil // avoid memory leak + item.index = -1 + q.heap = old[0 : n-1] + q.lru.Remove(q.idx[item.publicKey]) + delete(q.idx, item.publicKey) + return item +} + +func (pq *expirationTracker) updateTrack(publicKey string) { + if e, ok := pq.idx[publicKey]; ok { + i := e.Value.(*jwtItem) + if pq.ttl != 0 { + // only update expiration when set + i.expiration = time.Now().Add(pq.ttl).UnixNano() + heap.Fix(pq, i.index) + } + if pq.evictOnLimit { + pq.lru.MoveToBack(e) + } + } +} + +func (pq *expirationTracker) unTrack(publicKey string) { + if it, ok := pq.idx[publicKey]; ok { + xorAssign(&pq.hash, it.Value.(*jwtItem).hash) + heap.Remove(pq, it.Value.(*jwtItem).index) + delete(pq.idx, publicKey) + } +} + +func (pq *expirationTracker) track(publicKey string, hash *[sha256.Size]byte, theJWT string) { + var exp int64 + // prioritize ttl over expiration + if pq.ttl != 0 { + if pq.ttl == time.Duration(math.MaxInt64) { + exp = math.MaxInt64 + } else { + exp = time.Now().Add(pq.ttl).UnixNano() + } + } else { + if g, err := jwt.DecodeGeneric(theJWT); err == nil { + exp = time.Unix(g.Expires, 0).UnixNano() + } + if exp == 0 { + exp = math.MaxInt64 // default to indefinite + } + } + if e, ok := pq.idx[publicKey]; ok { + i := e.Value.(*jwtItem) + xorAssign(&pq.hash, i.hash) // remove old hash + i.expiration = exp + i.hash = *hash + heap.Fix(pq, i.index) + } else { + heap.Push(pq, &jwtItem{-1, publicKey, exp, *hash}) + } + xorAssign(&pq.hash, *hash) // add in new hash +} + +func (pq *expirationTracker) close() { + if pq == nil || pq.quit == nil { + return + } + close(pq.quit) + pq.quit = nil +} + +func (store *DirJWTStore) startExpiring(reCheck time.Duration, limit int64, evictOnLimit bool, ttl time.Duration) { + store.Lock() + defer store.Unlock() + quit := make(chan struct{}) + pq := &expirationTracker{ + make([]*jwtItem, 0, 10), + make(map[string]*list.Element), + list.New(), + limit, + evictOnLimit, + ttl, + [sha256.Size]byte{}, + quit, + sync.WaitGroup{}, + } + store.expiration = pq + pq.wg.Add(1) + go func() { + t := time.NewTicker(reCheck) + defer t.Stop() + defer pq.wg.Done() + for { + now := time.Now().UnixNano() + store.Lock() + if pq.Len() > 0 { + if it := pq.heap[0]; it.expiration <= now { + path := store.pathForKey(it.publicKey) + if err := os.Remove(path); err == nil { + heap.Pop(pq) + pq.unTrack(it.publicKey) + xorAssign(&pq.hash, it.hash) + store.Unlock() + continue // we removed an entry, check next one right away + } + } + } + store.Unlock() + select { + case <-t.C: + case <-quit: + return + } + } + }() +} diff --git a/server/dirstore_test.go b/server/dirstore_test.go new file mode 100644 index 0000000..7ac5e4b --- /dev/null +++ b/server/dirstore_test.go @@ -0,0 +1,1020 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "crypto/sha256" + "fmt" + "math" + "math/rand/v2" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/nats-io/jwt/v2" + "github.com/nats-io/nkeys" +) + +var ( + one, two, three, four = "", "", "", "" + jwt1, jwt2, jwt3, jwt4 = "", "", "", "" + op nkeys.KeyPair +) + +func init() { + op, _ = nkeys.CreateOperator() + + nkone, _ := nkeys.CreateAccount() + pub, _ := nkone.PublicKey() + one = pub + ac := jwt.NewAccountClaims(pub) + jwt1, _ = ac.Encode(op) + + nktwo, _ := nkeys.CreateAccount() + pub, _ = nktwo.PublicKey() + two = pub + ac = jwt.NewAccountClaims(pub) + jwt2, _ = ac.Encode(op) + + nkthree, _ := nkeys.CreateAccount() + pub, _ = nkthree.PublicKey() + three = pub + ac = jwt.NewAccountClaims(pub) + jwt3, _ = ac.Encode(op) + + nkfour, _ := nkeys.CreateAccount() + pub, _ = nkfour.PublicKey() + four = pub + ac = jwt.NewAccountClaims(pub) + jwt4, _ = ac.Encode(op) +} + +func TestShardedDirStoreWriteAndReadonly(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + store, err := NewDirJWTStore(dir, true, false) + require_NoError(t, err) + + expected := map[string]string{ + one: "alpha", + two: "beta", + three: "gamma", + four: "delta", + } + + for k, v := range expected { + store.SaveAcc(k, v) + } + + for k, v := range expected { + got, err := store.LoadAcc(k) + require_NoError(t, err) + require_Equal(t, v, got) + } + + got, err := store.LoadAcc("random") + require_Error(t, err) + require_Equal(t, "", got) + + got, err = store.LoadAcc("") + require_Error(t, err) + require_Equal(t, "", got) + + err = store.SaveAcc("", "onetwothree") + require_Error(t, err) + store.Close() + + // re-use the folder for readonly mode + store, err = NewImmutableDirJWTStore(dir, true) + require_NoError(t, err) + + require_True(t, store.IsReadOnly()) + + err = store.SaveAcc("five", "omega") + require_Error(t, err) + + for k, v := range expected { + got, err := store.LoadAcc(k) + require_NoError(t, err) + require_Equal(t, v, got) + } + store.Close() +} + +func TestUnshardedDirStoreWriteAndReadonly(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + store, err := NewDirJWTStore(dir, false, false) + require_NoError(t, err) + + expected := map[string]string{ + one: "alpha", + two: "beta", + three: "gamma", + four: "delta", + } + + require_False(t, store.IsReadOnly()) + + for k, v := range expected { + store.SaveAcc(k, v) + } + + for k, v := range expected { + got, err := store.LoadAcc(k) + require_NoError(t, err) + require_Equal(t, v, got) + } + + got, err := store.LoadAcc("random") + require_Error(t, err) + require_Equal(t, "", got) + + got, err = store.LoadAcc("") + require_Error(t, err) + require_Equal(t, "", got) + + err = store.SaveAcc("", "onetwothree") + require_Error(t, err) + store.Close() + + // re-use the folder for readonly mode + store, err = NewImmutableDirJWTStore(dir, false) + require_NoError(t, err) + + require_True(t, store.IsReadOnly()) + + err = store.SaveAcc("five", "omega") + require_Error(t, err) + + for k, v := range expected { + got, err := store.LoadAcc(k) + require_NoError(t, err) + require_Equal(t, v, got) + } + store.Close() +} + +func TestNoCreateRequiresDir(t *testing.T) { + t.Parallel() + _, err := NewDirJWTStore("/a/b/c", true, false) + require_Error(t, err) +} + +func TestCreateMakesDir(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + fullPath := filepath.Join(dir, "a/b") + + _, err := os.Stat(fullPath) + require_Error(t, err) + require_True(t, os.IsNotExist(err)) + + s, err := NewDirJWTStore(fullPath, false, true) + require_NoError(t, err) + s.Close() + + _, err = os.Stat(fullPath) + require_NoError(t, err) +} + +func TestDirStoreInaccessibleParent(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("directory permission bits do not block traversal on Windows") + } + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permission checks") + } + t.Parallel() + + dir := t.TempDir() + parent := filepath.Join(dir, "parent") + require_NoError(t, os.Mkdir(parent, 0o755)) + child := filepath.Join(parent, "child") + require_NoError(t, os.Mkdir(child, 0o755)) + + // Remove the search bit on the parent so stat of child returns EACCES + // rather than IsNotExist. Previously this panicked with a nil pointer + // dereference instead of returning an error. + require_NoError(t, os.Chmod(parent, 0o000)) + t.Cleanup(func() { os.Chmod(parent, 0o755) }) + + _, err := NewDirJWTStore(child, false, true) + require_Error(t, err) +} + +func TestShardedDirStorePackMerge(t *testing.T) { + t.Parallel() + dir := t.TempDir() + dir2 := t.TempDir() + dir3 := t.TempDir() + + store, err := NewDirJWTStore(dir, true, false) + require_NoError(t, err) + + expected := map[string]string{ + one: "alpha", + two: "beta", + three: "gamma", + four: "delta", + } + + require_False(t, store.IsReadOnly()) + + for k, v := range expected { + store.SaveAcc(k, v) + } + + for k, v := range expected { + got, err := store.LoadAcc(k) + require_NoError(t, err) + require_Equal(t, v, got) + } + + got, err := store.LoadAcc("random") + require_Error(t, err) + require_Equal(t, "", got) + + pack, err := store.Pack(-1) + require_NoError(t, err) + + inc, err := NewDirJWTStore(dir2, true, false) + require_NoError(t, err) + + inc.Merge(pack) + + for k, v := range expected { + got, err := inc.LoadAcc(k) + require_NoError(t, err) + require_Equal(t, v, got) + } + + got, err = inc.LoadAcc("random") + require_Error(t, err) + require_Equal(t, "", got) + + limitedPack, err := inc.Pack(1) + require_NoError(t, err) + + limited, err := NewDirJWTStore(dir3, true, false) + + require_NoError(t, err) + + limited.Merge(limitedPack) + + count := 0 + for k, v := range expected { + got, err := limited.LoadAcc(k) + if err == nil { + count++ + require_Equal(t, v, got) + } + } + + require_Len(t, 1, count) + + got, err = inc.LoadAcc("random") + require_Error(t, err) + require_Equal(t, "", got) +} + +func TestShardedToUnsharedDirStorePackMerge(t *testing.T) { + t.Parallel() + dir := t.TempDir() + dir2 := t.TempDir() + + store, err := NewDirJWTStore(dir, true, false) + require_NoError(t, err) + + expected := map[string]string{ + one: "alpha", + two: "beta", + three: "gamma", + four: "delta", + } + + require_False(t, store.IsReadOnly()) + + for k, v := range expected { + store.SaveAcc(k, v) + } + + for k, v := range expected { + got, err := store.LoadAcc(k) + require_NoError(t, err) + require_Equal(t, v, got) + } + + got, err := store.LoadAcc("random") + require_Error(t, err) + require_Equal(t, "", got) + + pack, err := store.Pack(-1) + require_NoError(t, err) + + inc, err := NewDirJWTStore(dir2, false, false) + require_NoError(t, err) + + inc.Merge(pack) + + for k, v := range expected { + got, err := inc.LoadAcc(k) + require_NoError(t, err) + require_Equal(t, v, got) + } + + got, err = inc.LoadAcc("random") + require_Error(t, err) + require_Equal(t, "", got) + + err = store.Merge("foo") + require_Error(t, err) + + err = store.Merge("") // will skip it + require_NoError(t, err) + + err = store.Merge("a|something") // should fail on a for sharding + require_Error(t, err) +} + +func TestMergeOnlyOnNewer(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + dirStore, err := NewDirJWTStore(dir, true, false) + require_NoError(t, err) + + accountKey, err := nkeys.CreateAccount() + require_NoError(t, err) + + pubKey, err := accountKey.PublicKey() + require_NoError(t, err) + + account := jwt.NewAccountClaims(pubKey) + account.Name = "old" + olderJWT, err := account.Encode(accountKey) + require_NoError(t, err) + + time.Sleep(2 * time.Second) + + account.Name = "new" + newerJWT, err := account.Encode(accountKey) + require_NoError(t, err) + + // Should work + err = dirStore.SaveAcc(pubKey, olderJWT) + require_NoError(t, err) + fromStore, err := dirStore.LoadAcc(pubKey) + require_NoError(t, err) + require_Equal(t, olderJWT, fromStore) + + // should replace + err = dirStore.saveIfNewer(pubKey, newerJWT) + require_NoError(t, err) + fromStore, err = dirStore.LoadAcc(pubKey) + require_NoError(t, err) + require_Equal(t, newerJWT, fromStore) + + // should fail + err = dirStore.saveIfNewer(pubKey, olderJWT) + require_NoError(t, err) + fromStore, err = dirStore.LoadAcc(pubKey) + require_NoError(t, err) + require_Equal(t, newerJWT, fromStore) +} + +func createTestAccount(t *testing.T, dirStore *DirJWTStore, expSec int, accKey nkeys.KeyPair) string { + t.Helper() + pubKey, err := accKey.PublicKey() + require_NoError(t, err) + account := jwt.NewAccountClaims(pubKey) + if expSec > 0 { + account.Expires = time.Now().Round(time.Second).Add(time.Second * time.Duration(expSec)).Unix() + } + jwt, err := account.Encode(accKey) + require_NoError(t, err) + err = dirStore.SaveAcc(pubKey, jwt) + require_NoError(t, err) + return jwt +} + +func assertStoreSize(t *testing.T, dirStore *DirJWTStore, length int) { + t.Helper() + // Count actual JWT files recursively. In shard mode files are nested in + // subdirectories named by the key's last two characters, so a plain + // os.ReadDir of the top-level directory would miscount when two keys share + // the same shard suffix. + var fileCnt int + err := filepath.Walk(dirStore.directory, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(path, fileExtension) { + fileCnt++ + } + return nil + }) + require_NoError(t, err) + require_Len(t, fileCnt, length) + dirStore.Lock() + require_Len(t, len(dirStore.expiration.idx), length) + require_Len(t, dirStore.expiration.lru.Len(), length) + require_Len(t, len(dirStore.expiration.heap), length) + dirStore.Unlock() +} + +func TestExpiration(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*50, 10, true, 0, nil) + require_NoError(t, err) + defer dirStore.Close() + + account := func(expSec int) { + accountKey, err := nkeys.CreateAccount() + require_NoError(t, err) + createTestAccount(t, dirStore, expSec, accountKey) + } + + hBegin := dirStore.Hash() + account(100) + hNoExp := dirStore.Hash() + require_NotEqual(t, hBegin, hNoExp) + account(1) + nh2 := dirStore.Hash() + require_NotEqual(t, hNoExp, nh2) + assertStoreSize(t, dirStore, 2) + + failAt := time.Now().Add(4 * time.Second) + for time.Now().Before(failAt) { + time.Sleep(100 * time.Millisecond) + f, err := os.ReadDir(dir) + require_NoError(t, err) + if len(f) == 1 { + lh := dirStore.Hash() + require_Equal(t, string(hNoExp[:]), string(lh[:])) + return + } + } + t.Fatalf("Waited more than 4 seconds for the file with expiration 1 second to expire") +} + +func TestLimit(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*100, 5, true, 0, nil) + require_NoError(t, err) + defer dirStore.Close() + + account := func(expSec int) { + accountKey, err := nkeys.CreateAccount() + require_NoError(t, err) + createTestAccount(t, dirStore, expSec, accountKey) + } + + h := dirStore.Hash() + + accountKey, err := nkeys.CreateAccount() + require_NoError(t, err) + // update first account + for i := 0; i < 10; i++ { + createTestAccount(t, dirStore, 50, accountKey) + assertStoreSize(t, dirStore, 1) + } + // new accounts + for i := 0; i < 10; i++ { + account(i) + nh := dirStore.Hash() + require_NotEqual(t, h, nh) + h = nh + } + // first account should be gone now accountKey.PublicKey() + key, _ := accountKey.PublicKey() + _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, key)) + require_True(t, os.IsNotExist(err)) + + // update first account + for i := 0; i < 10; i++ { + createTestAccount(t, dirStore, 50, accountKey) + assertStoreSize(t, dirStore, 5) + } +} + +func TestLimitNoEvict(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*50, 2, false, 0, nil) + require_NoError(t, err) + defer dirStore.Close() + + accountKey1, err := nkeys.CreateAccount() + require_NoError(t, err) + pKey1, err := accountKey1.PublicKey() + require_NoError(t, err) + accountKey2, err := nkeys.CreateAccount() + require_NoError(t, err) + accountKey3, err := nkeys.CreateAccount() + require_NoError(t, err) + pKey3, err := accountKey3.PublicKey() + require_NoError(t, err) + + createTestAccount(t, dirStore, 100, accountKey1) + assertStoreSize(t, dirStore, 1) + createTestAccount(t, dirStore, 1, accountKey2) + assertStoreSize(t, dirStore, 2) + + hBefore := dirStore.Hash() + // 2 jwt are already stored. third must result in an error + pubKey, err := accountKey3.PublicKey() + require_NoError(t, err) + account := jwt.NewAccountClaims(pubKey) + jwt, err := account.Encode(accountKey3) + require_NoError(t, err) + err = dirStore.SaveAcc(pubKey, jwt) + require_Error(t, err) + assertStoreSize(t, dirStore, 2) + _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey1)) + require_NoError(t, err) + _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey3)) + require_True(t, os.IsNotExist(err)) + // check that the hash did not change + hAfter := dirStore.Hash() + require_True(t, bytes.Equal(hBefore[:], hAfter[:])) + // wait for expiration of account2 + time.Sleep(2200 * time.Millisecond) + err = dirStore.SaveAcc(pubKey, jwt) + require_NoError(t, err) + assertStoreSize(t, dirStore, 2) + _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey1)) + require_NoError(t, err) + _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey3)) + require_NoError(t, err) +} + +func TestLruLoad(t *testing.T) { + t.Parallel() + dir := t.TempDir() + dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*100, 2, true, 0, nil) + require_NoError(t, err) + defer dirStore.Close() + + accountKey1, err := nkeys.CreateAccount() + require_NoError(t, err) + pKey1, err := accountKey1.PublicKey() + require_NoError(t, err) + accountKey2, err := nkeys.CreateAccount() + require_NoError(t, err) + accountKey3, err := nkeys.CreateAccount() + require_NoError(t, err) + pKey3, err := accountKey3.PublicKey() + require_NoError(t, err) + + createTestAccount(t, dirStore, 10, accountKey1) + assertStoreSize(t, dirStore, 1) + createTestAccount(t, dirStore, 10, accountKey2) + assertStoreSize(t, dirStore, 2) + dirStore.LoadAcc(pKey1) // will reorder 1/2 + createTestAccount(t, dirStore, 10, accountKey3) + assertStoreSize(t, dirStore, 2) + + _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey1)) + require_NoError(t, err) + _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey3)) + require_NoError(t, err) +} + +func TestLruVolume(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*50, 2, true, 0, nil) + require_NoError(t, err) + defer dirStore.Close() + replaceCnt := 500 // needs to be bigger than 2 due to loop unrolling + keys := make([]string, replaceCnt) + + key, err := nkeys.CreateAccount() + require_NoError(t, err) + keys[0], err = key.PublicKey() + require_NoError(t, err) + createTestAccount(t, dirStore, 10000, key) // not intended to expire + assertStoreSize(t, dirStore, 1) + + key, err = nkeys.CreateAccount() + require_NoError(t, err) + keys[1], err = key.PublicKey() + require_NoError(t, err) + createTestAccount(t, dirStore, 10000, key) + assertStoreSize(t, dirStore, 2) + + for i := 2; i < replaceCnt; i++ { + k, err := nkeys.CreateAccount() + require_NoError(t, err) + keys[i], err = k.PublicKey() + require_NoError(t, err) + + createTestAccount(t, dirStore, 10000+rand.IntN(10000), k) // not intended to expire + assertStoreSize(t, dirStore, 2) + _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, keys[i-2])) + require_Error(t, err) + require_True(t, os.IsNotExist(err)) + _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, keys[i-1])) + require_NoError(t, err) + _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, keys[i])) + require_NoError(t, err) + } +} + +func TestLru(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*50, 2, true, 0, nil) + require_NoError(t, err) + defer dirStore.Close() + + accountKey1, err := nkeys.CreateAccount() + require_NoError(t, err) + pKey1, err := accountKey1.PublicKey() + require_NoError(t, err) + accountKey2, err := nkeys.CreateAccount() + require_NoError(t, err) + accountKey3, err := nkeys.CreateAccount() + require_NoError(t, err) + pKey3, err := accountKey3.PublicKey() + require_NoError(t, err) + + createTestAccount(t, dirStore, 1000, accountKey1) + assertStoreSize(t, dirStore, 1) + createTestAccount(t, dirStore, 1000, accountKey2) + assertStoreSize(t, dirStore, 2) + createTestAccount(t, dirStore, 1000, accountKey3) + assertStoreSize(t, dirStore, 2) + _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey1)) + require_Error(t, err) + require_True(t, os.IsNotExist(err)) + _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey3)) + require_NoError(t, err) + + // update -> will change this keys position for eviction + createTestAccount(t, dirStore, 1000, accountKey2) + assertStoreSize(t, dirStore, 2) + // recreate -> will evict 3 + createTestAccount(t, dirStore, 1, accountKey1) + assertStoreSize(t, dirStore, 2) + _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey3)) + require_True(t, os.IsNotExist(err)) + // let key1 expire. sleep expSec=1 + 1 for rounding + time.Sleep(2200 * time.Millisecond) + assertStoreSize(t, dirStore, 1) + _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey1)) + require_True(t, os.IsNotExist(err)) + // recreate key3 - no eviction + createTestAccount(t, dirStore, 1000, accountKey3) + assertStoreSize(t, dirStore, 2) +} + +func TestReload(t *testing.T) { + t.Parallel() + dir := t.TempDir() + notificationChan := make(chan struct{}, 5) + dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*100, 2, true, 0, func(publicKey string) { + notificationChan <- struct{}{} + }) + require_NoError(t, err) + defer dirStore.Close() + newAccount := func() string { + t.Helper() + accKey, err := nkeys.CreateAccount() + require_NoError(t, err) + pKey, err := accKey.PublicKey() + require_NoError(t, err) + pubKey, err := accKey.PublicKey() + require_NoError(t, err) + account := jwt.NewAccountClaims(pubKey) + jwt, err := account.Encode(accKey) + require_NoError(t, err) + file := fmt.Sprintf("%s/%s.jwt", dir, pKey) + err = os.WriteFile(file, []byte(jwt), 0644) + require_NoError(t, err) + return file + } + files := make(map[string]struct{}) + assertStoreSize(t, dirStore, 0) + hash := dirStore.Hash() + emptyHash := [sha256.Size]byte{} + require_True(t, bytes.Equal(hash[:], emptyHash[:])) + for i := 0; i < 5; i++ { + files[newAccount()] = struct{}{} + err = dirStore.Reload() + require_NoError(t, err) + <-notificationChan + assertStoreSize(t, dirStore, i+1) + hash = dirStore.Hash() + require_False(t, bytes.Equal(hash[:], emptyHash[:])) + msg, err := dirStore.Pack(-1) + require_NoError(t, err) + require_Len(t, len(strings.Split(msg, "\n")), len(files)) + } + for k := range files { + hash = dirStore.Hash() + require_False(t, bytes.Equal(hash[:], emptyHash[:])) + removeFile(t, k) + err = dirStore.Reload() + require_NoError(t, err) + assertStoreSize(t, dirStore, len(files)-1) + delete(files, k) + msg, err := dirStore.Pack(-1) + require_NoError(t, err) + if len(files) != 0 { // when len is 0, we have an empty line + require_Len(t, len(strings.Split(msg, "\n")), len(files)) + } + } + require_True(t, len(notificationChan) == 0) + hash = dirStore.Hash() + require_True(t, bytes.Equal(hash[:], emptyHash[:])) +} + +func TestExpirationUpdate(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*50, 10, true, 0, nil) + require_NoError(t, err) + defer dirStore.Close() + + accountKey, err := nkeys.CreateAccount() + require_NoError(t, err) + + h := dirStore.Hash() + + createTestAccount(t, dirStore, 0, accountKey) + nh := dirStore.Hash() + require_NotEqual(t, h, nh) + h = nh + + time.Sleep(1500 * time.Millisecond) + f, err := os.ReadDir(dir) + require_NoError(t, err) + require_Len(t, len(f), 1) + + createTestAccount(t, dirStore, 2, accountKey) + nh = dirStore.Hash() + require_NotEqual(t, h, nh) + h = nh + + time.Sleep(1500 * time.Millisecond) + f, err = os.ReadDir(dir) + require_NoError(t, err) + require_Len(t, len(f), 1) + + createTestAccount(t, dirStore, 0, accountKey) + nh = dirStore.Hash() + require_NotEqual(t, h, nh) + h = nh + + time.Sleep(1500 * time.Millisecond) + f, err = os.ReadDir(dir) + require_NoError(t, err) + require_Len(t, len(f), 1) + + createTestAccount(t, dirStore, 1, accountKey) + nh = dirStore.Hash() + require_NotEqual(t, h, nh) + + time.Sleep(1500 * time.Millisecond) + f, err = os.ReadDir(dir) + require_NoError(t, err) + require_Len(t, len(f), 0) + + empty := [32]byte{} + h = dirStore.Hash() + require_Equal(t, string(h[:]), string(empty[:])) +} + +func TestTTL(t *testing.T) { + t.Parallel() + dir := t.TempDir() + require_OneJWT := func() { + t.Helper() + f, err := os.ReadDir(dir) + require_NoError(t, err) + require_Len(t, len(f), 1) + } + test := func(op func(store *DirJWTStore, accountKey nkeys.KeyPair, accountPubKey string, jwt string)) { + dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, 50*time.Millisecond, 10, true, 200*time.Millisecond, nil) + require_NoError(t, err) + defer dirStore.Close() + + accountKey, err := nkeys.CreateAccount() + require_NoError(t, err) + pubKey, err := accountKey.PublicKey() + require_NoError(t, err) + jwt := createTestAccount(t, dirStore, 0, accountKey) + require_OneJWT() + // observe non expiration due to activity + for i := 0; i < 4; i++ { + time.Sleep(110 * time.Millisecond) + op(dirStore, accountKey, pubKey, jwt) + require_OneJWT() + } + // observe expiration + for i := 0; i < 40; i++ { + time.Sleep(50 * time.Millisecond) + f, err := os.ReadDir(dir) + require_NoError(t, err) + if len(f) == 0 { + return + } + } + t.Fatalf("jwt should have expired by now") + } + t.Run("no expiration due to load", func(t *testing.T) { + test(func(store *DirJWTStore, accountKey nkeys.KeyPair, pubKey string, jwt string) { + store.LoadAcc(pubKey) + }) + }) + t.Run("no expiration due to store", func(t *testing.T) { + test(func(store *DirJWTStore, accountKey nkeys.KeyPair, pubKey string, jwt string) { + store.SaveAcc(pubKey, jwt) + }) + }) + t.Run("no expiration due to overwrite", func(t *testing.T) { + test(func(store *DirJWTStore, accountKey nkeys.KeyPair, pubKey string, jwt string) { + createTestAccount(t, store, 0, accountKey) + }) + }) +} + +func TestRemove(t *testing.T) { + for deleteType, test := range map[deleteType]struct { + expected int + moved int + }{ + HardDelete: {0, 0}, + RenameDeleted: {0, 1}, + NoDelete: {1, 0}, + } { + deleteType, test := deleteType, test // fixes govet capturing loop variables + t.Run("", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + require_OneJWT := func() { + t.Helper() + f, err := os.ReadDir(dir) + require_NoError(t, err) + require_Len(t, len(f), 1) + } + dirStore, err := NewExpiringDirJWTStore(dir, false, false, deleteType, 0, 10, true, 0, nil) + delPubKey := "" + dirStore.deleted = func(publicKey string) { + delPubKey = publicKey + } + require_NoError(t, err) + defer dirStore.Close() + accountKey, err := nkeys.CreateAccount() + require_NoError(t, err) + pubKey, err := accountKey.PublicKey() + require_NoError(t, err) + createTestAccount(t, dirStore, 0, accountKey) + require_OneJWT() + dirStore.delete(pubKey) + if deleteType == NoDelete { + require_True(t, delPubKey == "") + } else { + require_True(t, delPubKey == pubKey) + } + f, err := filepath.Glob(dir + string(os.PathSeparator) + "/*.jwt") + require_NoError(t, err) + require_Len(t, len(f), test.expected) + f, err = filepath.Glob(dir + string(os.PathSeparator) + "/*.jwt.deleted") + require_NoError(t, err) + require_Len(t, len(f), test.moved) + }) + } +} + +const infDur = time.Duration(math.MaxInt64) + +func TestNotificationOnPack(t *testing.T) { + t.Parallel() + jwts := map[string]string{ + one: jwt1, + two: jwt2, + three: jwt3, + four: jwt4, + } + notificationChan := make(chan struct{}, len(jwts)) // set to same len so all extra will block + notification := func(pubKey string) { + if _, ok := jwts[pubKey]; !ok { + t.Fatalf("Key not found: %s", pubKey) + } + notificationChan <- struct{}{} + } + dirPack := t.TempDir() + packStore, err := NewExpiringDirJWTStore(dirPack, false, false, NoDelete, infDur, 0, true, 0, notification) + require_NoError(t, err) + // prefill the store with data + for k, v := range jwts { + require_NoError(t, packStore.SaveAcc(k, v)) + } + for i := 0; i < len(jwts); i++ { + <-notificationChan + } + msg, err := packStore.Pack(-1) + require_NoError(t, err) + packStore.Close() + hash := packStore.Hash() + for _, shard := range []bool{true, false, true, false} { + dirMerge := t.TempDir() + mergeStore, err := NewExpiringDirJWTStore(dirMerge, shard, false, NoDelete, infDur, 0, true, 0, notification) + require_NoError(t, err) + // set + err = mergeStore.Merge(msg) + require_NoError(t, err) + assertStoreSize(t, mergeStore, len(jwts)) + hash1 := packStore.Hash() + require_True(t, bytes.Equal(hash[:], hash1[:])) + for i := 0; i < len(jwts); i++ { + <-notificationChan + } + // overwrite - assure + err = mergeStore.Merge(msg) + require_NoError(t, err) + assertStoreSize(t, mergeStore, len(jwts)) + hash2 := packStore.Hash() + require_True(t, bytes.Equal(hash1[:], hash2[:])) + + hash = hash1 + msg, err = mergeStore.Pack(-1) + require_NoError(t, err) + mergeStore.Close() + require_True(t, len(notificationChan) == 0) + + for k, v := range jwts { + j, err := packStore.LoadAcc(k) + require_NoError(t, err) + require_Equal(t, j, v) + } + } +} + +func TestNotificationOnPackWalk(t *testing.T) { + t.Parallel() + const storeCnt = 5 + const keyCnt = 50 + const iterCnt = 8 + store := [storeCnt]*DirJWTStore{} + for i := 0; i < storeCnt; i++ { + dirMerge := t.TempDir() + mergeStore, err := NewExpiringDirJWTStore(dirMerge, true, false, NoDelete, infDur, 0, true, 0, nil) + require_NoError(t, err) + store[i] = mergeStore + } + for i := 0; i < iterCnt; i++ { //iterations + jwts := make(map[string]string) + for j := 0; j < keyCnt; j++ { + kp, _ := nkeys.CreateAccount() + key, _ := kp.PublicKey() + ac := jwt.NewAccountClaims(key) + jwts[key], _ = ac.Encode(op) + require_NoError(t, store[0].SaveAcc(key, jwts[key])) + } + for j := 0; j < storeCnt-1; j++ { // stores + err := store[j].PackWalk(3, func(partialPackMsg string) { + err := store[j+1].Merge(partialPackMsg) + require_NoError(t, err) + }) + require_NoError(t, err) + } + for i := 0; i < storeCnt-1; i++ { + h1 := store[i].Hash() + h2 := store[i+1].Hash() + require_True(t, bytes.Equal(h1[:], h2[:])) + } + } + for i := 0; i < storeCnt; i++ { + store[i].Close() + } +} diff --git a/server/disk_avail.go b/server/disk_avail.go new file mode 100644 index 0000000..e27c1a7 --- /dev/null +++ b/server/disk_avail.go @@ -0,0 +1,37 @@ +// Copyright 2020-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !windows && !openbsd && !netbsd && !wasm && !illumos && !solaris + +package server + +import ( + "os" + "syscall" +) + +func diskAvailable(storeDir string) int64 { + var ba int64 + if _, err := os.Stat(storeDir); os.IsNotExist(err) { + os.MkdirAll(storeDir, defaultDirPerms) + } + var fs syscall.Statfs_t + if err := syscall.Statfs(storeDir, &fs); err == nil { + // Estimate 75% of available storage. + ba = int64(uint64(fs.Bavail) * uint64(fs.Bsize) / 4 * 3) + } else { + // Used 1TB default as a guess if all else fails. + ba = JetStreamMaxStoreDefault + } + return ba +} diff --git a/server/disk_avail_netbsd.go b/server/disk_avail_netbsd.go new file mode 100644 index 0000000..dd81a1b --- /dev/null +++ b/server/disk_avail_netbsd.go @@ -0,0 +1,21 @@ +// Copyright 2022-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build netbsd + +package server + +// TODO - See if there is a version of this for NetBSD. +func diskAvailable(storeDir string) int64 { + return JetStreamMaxStoreDefault +} diff --git a/server/disk_avail_openbsd.go b/server/disk_avail_openbsd.go new file mode 100644 index 0000000..8ffdf12 --- /dev/null +++ b/server/disk_avail_openbsd.go @@ -0,0 +1,37 @@ +// Copyright 2021-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build openbsd + +package server + +import ( + "os" + "syscall" +) + +func diskAvailable(storeDir string) int64 { + var ba int64 + if _, err := os.Stat(storeDir); os.IsNotExist(err) { + os.MkdirAll(storeDir, defaultDirPerms) + } + var fs syscall.Statfs_t + if err := syscall.Statfs(storeDir, &fs); err == nil { + // Estimate 75% of available storage. + ba = int64(uint64(fs.F_bavail) * uint64(fs.F_bsize) / 4 * 3) + } else { + // Used 1TB default as a guess if all else fails. + ba = JetStreamMaxStoreDefault + } + return ba +} diff --git a/server/disk_avail_solaris.go b/server/disk_avail_solaris.go new file mode 100644 index 0000000..e0febd1 --- /dev/null +++ b/server/disk_avail_solaris.go @@ -0,0 +1,38 @@ +// Copyright 2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build illumos || solaris + +package server + +import ( + "os" + "golang.org/x/sys/unix" +) + +func diskAvailable(storeDir string) int64 { + var ba int64 + if _, err := os.Stat(storeDir); os.IsNotExist(err) { + os.MkdirAll(storeDir, defaultDirPerms) + } + var fs unix.Statvfs_t + if err := unix.Statvfs(storeDir, &fs); err == nil { + // Estimate 75% of available storage. + ba = int64(uint64(fs.Frsize) * uint64(fs.Bavail) / 4 * 3) + } else { + // Used 1TB default as a guess if all else fails. + ba = JetStreamMaxStoreDefault + } + return ba +} + diff --git a/server/disk_avail_wasm.go b/server/disk_avail_wasm.go new file mode 100644 index 0000000..1946870 --- /dev/null +++ b/server/disk_avail_wasm.go @@ -0,0 +1,20 @@ +// Copyright 2022-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build wasm + +package server + +func diskAvailable(storeDir string) int64 { + return JetStreamMaxStoreDefault +} diff --git a/server/disk_avail_windows.go b/server/disk_avail_windows.go new file mode 100644 index 0000000..c79c506 --- /dev/null +++ b/server/disk_avail_windows.go @@ -0,0 +1,21 @@ +// Copyright 2020-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package server + +// TODO(dlc) - See if there is a version of this for windows. +func diskAvailable(storeDir string) int64 { + return JetStreamMaxStoreDefault +} diff --git a/server/elastic/elastic.go b/server/elastic/elastic.go new file mode 100644 index 0000000..d764cf1 --- /dev/null +++ b/server/elastic/elastic.go @@ -0,0 +1,60 @@ +// Copyright 2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package elastic + +import ( + "weak" +) + +func Make[T any](ptr *T) *Pointer[T] { + return &Pointer[T]{ + weak: weak.Make(ptr), + } +} + +type Pointer[T any] struct { + weak weak.Pointer[T] + strong *T +} + +func (e *Pointer[T]) Set(ptr *T) { + e.weak = weak.Make(ptr) + if e.strong != nil { + e.strong = ptr + } +} + +func (e *Pointer[T]) Strengthen() { + if e == nil || e.strong != nil { + return + } + e.strong = e.weak.Value() +} + +func (e *Pointer[T]) Weaken() { + if e == nil || e.strong == nil { + return + } + e.strong = nil +} + +func (e *Pointer[T]) Value() *T { + if e == nil { + return nil + } + if e.strong != nil { + return e.strong + } + return e.weak.Value() +} diff --git a/server/errors.go b/server/errors.go new file mode 100644 index 0000000..7f517c5 --- /dev/null +++ b/server/errors.go @@ -0,0 +1,413 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "errors" + "fmt" +) + +var ( + // ErrConnectionClosed represents an error condition on a closed connection. + ErrConnectionClosed = errors.New("connection closed") + + // ErrAuthentication represents an error condition on failed authentication. + ErrAuthentication = errors.New("authentication error") + + // ErrAuthTimeout represents an error condition on failed authorization due to timeout. + ErrAuthTimeout = errors.New("authentication timeout") + + // ErrAuthExpired represents an expired authorization due to timeout. + ErrAuthExpired = errors.New("authentication expired") + + // ErrAuthProxyNotTrusted represents an error condition on failed authentication + // due to a connection from a proxy not in the list of trusted proxies. + ErrAuthProxyNotTrusted = errors.New("proxy is not trusted") + + // ErrAuthProxyRequired represents an error condition on failed authentication + // due to a connection not coming from a proxy. + ErrAuthProxyRequired = errors.New("proxy connection required") + + // ErrMaxPayload represents an error condition when the payload is too big. + ErrMaxPayload = errors.New("maximum payload exceeded") + + // ErrMaxControlLine represents an error condition when the control line is too big. + ErrMaxControlLine = errors.New("maximum control line exceeded") + + // ErrReservedPublishSubject represents an error condition when sending to a reserved subject, e.g. _SYS.> + ErrReservedPublishSubject = errors.New("reserved internal subject") + + // ErrBadPublishSubject represents an error condition for an invalid publish subject. + ErrBadPublishSubject = errors.New("invalid publish subject") + + // ErrBadSubject represents an error condition for an invalid subject. + ErrBadSubject = errors.New("invalid subject") + + // ErrBadQualifier is used to error on a bad qualifier for a transform. + ErrBadQualifier = errors.New("bad qualifier") + + // ErrBadClientProtocol signals a client requested an invalid client protocol. + ErrBadClientProtocol = errors.New("invalid client protocol") + + // ErrTooManyConnections signals a client that the maximum number of connections supported by the + // server has been reached. + ErrTooManyConnections = errors.New("maximum connections exceeded") + + // ErrTooManyAccountConnections signals that an account has reached its maximum number of active + // connections. + ErrTooManyAccountConnections = errors.New("maximum account active connections exceeded") + + // ErrLeafNodeLoop signals a leafnode is trying to register for a cluster we already have registered. + ErrLeafNodeLoop = errors.New("leafnode loop detected") + + // ErrTooManySubs signals a client that the maximum number of subscriptions per connection + // has been reached. + ErrTooManySubs = errors.New("maximum subscriptions exceeded") + + // ErrTooManySubTokens signals a client that the subject has too many tokens. + ErrTooManySubTokens = errors.New("subject has exceeded number of tokens limit") + + // ErrClientConnectedToRoutePort represents an error condition when a client + // attempted to connect to the route listen port. + ErrClientConnectedToRoutePort = errors.New("attempted to connect to route port") + + // ErrClientConnectedToLeafNodePort represents an error condition when a client + // attempted to connect to the leaf node listen port. + ErrClientConnectedToLeafNodePort = errors.New("attempted to connect to leaf node port") + + // ErrLeafNodeHasSameClusterName represents an error condition when a leafnode is a cluster + // and it has the same cluster name as the hub cluster. + ErrLeafNodeHasSameClusterName = errors.New("remote leafnode has same cluster name") + + // ErrLeafNodeDisabled is when we disable leafnodes. + ErrLeafNodeDisabled = errors.New("leafnodes disabled") + + // ErrConnectedToWrongPort represents an error condition when a connection is attempted + // to the wrong listen port (for instance a LeafNode to a client port, etc...) + ErrConnectedToWrongPort = errors.New("attempted to connect to wrong port") + + // ErrAccountExists is returned when an account is attempted to be registered + // but already exists. + ErrAccountExists = errors.New("account exists") + + // ErrBadAccount represents a malformed or incorrect account. + ErrBadAccount = errors.New("bad account") + + // ErrReservedAccount represents a reserved account that can not be created. + ErrReservedAccount = errors.New("reserved account") + + // ErrMissingAccount is returned when an account does not exist. + ErrMissingAccount = errors.New("account missing") + + // ErrMissingService is returned when an account does not have an exported service. + ErrMissingService = errors.New("service missing") + + // ErrBadServiceType is returned when latency tracking is being applied to non-singleton response types. + ErrBadServiceType = errors.New("bad service response type") + + // ErrBadSampling is returned when the sampling for latency tracking is not 1 >= sample <= 100. + ErrBadSampling = errors.New("bad sampling percentage, should be 1-100") + + // ErrAccountValidation is returned when an account has failed validation. + ErrAccountValidation = errors.New("account validation failed") + + // ErrAccountExpired is returned when an account has expired. + ErrAccountExpired = errors.New("account expired") + + // ErrNoAccountResolver is returned when we attempt an update but do not have an account resolver. + ErrNoAccountResolver = errors.New("account resolver missing") + + // ErrAccountResolverUpdateTooSoon is returned when we attempt an update too soon to last request. + ErrAccountResolverUpdateTooSoon = errors.New("account resolver update too soon") + + // ErrAccountResolverSameClaims is returned when same claims have been fetched. + ErrAccountResolverSameClaims = errors.New("account resolver no new claims") + + // ErrStreamImportAuthorization is returned when a stream import is not authorized. + ErrStreamImportAuthorization = errors.New("stream import not authorized") + + // ErrStreamImportBadPrefix is returned when a stream import prefix contains wildcards. + ErrStreamImportBadPrefix = errors.New("stream import prefix can not contain wildcard tokens") + + // ErrStreamImportDuplicate is returned when a stream import is a duplicate of one that already exists. + ErrStreamImportDuplicate = errors.New("stream import already exists") + + // ErrServiceImportAuthorization is returned when a service import is not authorized. + ErrServiceImportAuthorization = errors.New("service import not authorized") + + // ErrImportFormsCycle is returned when an import would form a cycle. + ErrImportFormsCycle = errors.New("import forms a cycle") + + // ErrCycleSearchDepth is returned when we have exceeded our maximum search depth.. + ErrCycleSearchDepth = errors.New("search cycle depth exhausted") + + // ErrClientOrRouteConnectedToGatewayPort represents an error condition when + // a client or route attempted to connect to the Gateway port. + ErrClientOrRouteConnectedToGatewayPort = errors.New("attempted to connect to gateway port") + + // ErrWrongGateway represents an error condition when a server receives a connect + // request from a remote Gateway with a destination name that does not match the server's + // Gateway's name. + ErrWrongGateway = errors.New("wrong gateway") + + // ErrGatewayNameHasSpaces signals that the gateway name contains spaces, which is not allowed. + ErrGatewayNameHasSpaces = errors.New("gateway name cannot contain spaces") + + // ErrNoSysAccount is returned when an attempt to publish or subscribe is made + // when there is no internal system account defined. + ErrNoSysAccount = errors.New("system account not setup") + + // ErrRevocation is returned when a credential has been revoked. + ErrRevocation = errors.New("credentials have been revoked") + + // ErrServerNotRunning is used to signal an error that a server is not running. + ErrServerNotRunning = errors.New("server is not running") + + // ErrServerNameHasSpaces signals that the server name contains spaces, which is not allowed. + ErrServerNameHasSpaces = errors.New("server name cannot contain spaces") + + // ErrBadMsgHeader signals the parser detected a bad message header + ErrBadMsgHeader = errors.New("bad message header detected") + + // ErrMsgHeadersNotSupported signals the parser detected a message header + // but they are not supported on this server. + ErrMsgHeadersNotSupported = errors.New("message headers not supported") + + // ErrNoRespondersRequiresHeaders signals that a client needs to have headers + // on if they want no responders behavior. + ErrNoRespondersRequiresHeaders = errors.New("no responders requires headers support") + + // ErrClusterNameConfigConflict signals that the options for cluster name in cluster and gateway are in conflict. + ErrClusterNameConfigConflict = errors.New("cluster name conflicts between cluster and gateway definitions") + + // ErrClusterNameRemoteConflict signals that a remote server has a different cluster name. + ErrClusterNameRemoteConflict = errors.New("cluster name from remote server conflicts") + + // ErrClusterNameHasSpaces signals that the cluster name contains spaces, which is not allowed. + ErrClusterNameHasSpaces = errors.New("cluster name cannot contain spaces") + + // ErrClusterNameReserved signals that the cluster name is reserved for internal protocol use. + ErrClusterNameReserved = errors.New("cluster name is reserved") + + // ErrMalformedSubject is returned when a subscription is made with a subject that does not conform to subject rules. + ErrMalformedSubject = errors.New("malformed subject") + + // ErrSubscribePermissionViolation is returned when processing of a subscription fails due to permissions. + ErrSubscribePermissionViolation = errors.New("subscribe permission violation") + + // ErrNoTransforms signals no subject transforms are available to map this subject. + ErrNoTransforms = errors.New("no matching transforms available") + + // ErrCertNotPinned is returned when pinned certs are set and the certificate is not in it + ErrCertNotPinned = errors.New("certificate not pinned") + + // ErrDuplicateServerName is returned when processing a server remote connection and + // the server reports that this server name is already used in the cluster. + ErrDuplicateServerName = errors.New("duplicate server name") + + // ErrMinimumVersionRequired is returned when a connection is not at the minimum version required. + ErrMinimumVersionRequired = errors.New("minimum version required") + // ErrLeafNodeMinVersionRejected is the leafnode protocol error prefix used + // when rejecting a remote due to leafnodes.min_version. + ErrLeafNodeMinVersionRejected = errors.New("connection rejected since minimum version required is") + + // ErrInvalidMappingDestination is used for all subject mapping destination errors + ErrInvalidMappingDestination = errors.New("invalid mapping destination") + + // ErrInvalidMappingDestinationSubject is used to error on a bad transform destination mapping + ErrInvalidMappingDestinationSubject = fmt.Errorf("%w: invalid transform", ErrInvalidMappingDestination) + + // ErrMappingDestinationNotUsingAllWildcards is used to error on a transform destination not using all of the token wildcards + ErrMappingDestinationNotUsingAllWildcards = fmt.Errorf("%w: not using all of the token wildcard(s)", ErrInvalidMappingDestination) + + // ErrUnknownMappingDestinationFunction is returned when a subject mapping destination contains an unknown mustache-escaped mapping function. + ErrUnknownMappingDestinationFunction = fmt.Errorf("%w: unknown function", ErrInvalidMappingDestination) + + // ErrMappingDestinationIndexOutOfRange is returned when the mapping destination function is passed an out of range wildcard index value for one of it's arguments + ErrMappingDestinationIndexOutOfRange = fmt.Errorf("%w: wildcard index out of range", ErrInvalidMappingDestination) + + // ErrMappingDestinationNotEnoughArgs is returned when the mapping destination function is not passed enough arguments + ErrMappingDestinationNotEnoughArgs = fmt.Errorf("%w: not enough arguments passed to the function", ErrInvalidMappingDestination) + + // ErrMappingDestinationInvalidArg is returned when the mapping destination function is passed and invalid argument + ErrMappingDestinationInvalidArg = fmt.Errorf("%w: function argument is invalid or in the wrong format", ErrInvalidMappingDestination) + + // ErrMappingDestinationTooManyArgs is returned when the mapping destination function is passed too many arguments + ErrMappingDestinationTooManyArgs = fmt.Errorf("%w: too many arguments passed to the function", ErrInvalidMappingDestination) + + // ErrMappingDestinationNotSupportedForImport is returned when you try to use a mapping function other than wildcard in a transform that needs to be reversible (i.e. an import) + ErrMappingDestinationNotSupportedForImport = fmt.Errorf("%w: the only mapping function allowed for import transforms is {{Wildcard()}}", ErrInvalidMappingDestination) +) + +// mappingDestinationErr is a type of subject mapping destination error +type mappingDestinationErr struct { + token string + err error +} + +func (e *mappingDestinationErr) Error() string { + if e.token == _EMPTY_ { + return e.err.Error() + } + return fmt.Sprintf("%s in %s", e.err, e.token) +} + +func (e *mappingDestinationErr) Is(target error) bool { + return target == ErrInvalidMappingDestination +} + +// configErr is a configuration error. +type configErr struct { + token token + reason string +} + +// Source reports the location of a configuration error. +func (e *configErr) Source() string { + return fmt.Sprintf("%s:%d:%d", e.token.SourceFile(), e.token.Line(), e.token.Position()) +} + +// Error reports the location and reason from a configuration error. +func (e *configErr) Error() string { + if e.token != nil { + return fmt.Sprintf("%s: %s", e.Source(), e.reason) + } + return e.reason +} + +// unknownConfigFieldErr is an error reported in pedantic mode. +type unknownConfigFieldErr struct { + configErr + field string +} + +// Error reports that an unknown field was in the configuration. +func (e *unknownConfigFieldErr) Error() string { + return fmt.Sprintf("%s: unknown field %q", e.Source(), e.field) +} + +// configWarningErr is an error reported in pedantic mode. +type configWarningErr struct { + configErr + field string +} + +// Error reports a configuration warning. +func (e *configWarningErr) Error() string { + return fmt.Sprintf("%s: invalid use of field %q: %s", e.Source(), e.field, e.reason) +} + +// processConfigErr is the result of processing the configuration from the server. +type processConfigErr struct { + errors []error + warnings []error +} + +// Error returns the collection of errors separated by new lines, +// warnings appear first then hard errors. +func (e *processConfigErr) Error() string { + var msg string + for _, err := range e.Warnings() { + msg += err.Error() + "\n" + } + for _, err := range e.Errors() { + msg += err.Error() + "\n" + } + return msg +} + +// Warnings returns the list of warnings. +func (e *processConfigErr) Warnings() []error { + return e.warnings +} + +// Errors returns the list of errors. +func (e *processConfigErr) Errors() []error { + return e.errors +} + +// errCtx wraps an error and stores additional ctx information for tracing. +// Does not print or return it unless explicitly requested. +type errCtx struct { + error + ctx string +} + +func NewErrorCtx(err error, format string, args ...any) error { + return &errCtx{err, fmt.Sprintf(format, args...)} +} + +// Unwrap implement to work with errors.Is and errors.As +func (e *errCtx) Unwrap() error { + if e == nil { + return nil + } + return e.error +} + +// Context for error +func (e *errCtx) Context() string { + if e == nil { + return "" + } + return e.ctx +} + +// UnpackIfErrorCtx return Error or, if type is right error and context +func UnpackIfErrorCtx(err error) string { + if e, ok := err.(*errCtx); ok { + if _, ok := e.error.(*errCtx); ok { + return fmt.Sprint(UnpackIfErrorCtx(e.error), ": ", e.Context()) + } + return fmt.Sprint(e.Error(), ": ", e.Context()) + } + return err.Error() +} + +// implements: go 1.13 errors.Unwrap(err error) error +// TODO replace with native code once we no longer support go1.12 +func errorsUnwrap(err error) error { + u, ok := err.(interface { + Unwrap() error + }) + if !ok { + return nil + } + return u.Unwrap() +} + +// ErrorIs implements: go 1.13 errors.Is(err, target error) bool +// TODO replace with native code once we no longer support go1.12 +func ErrorIs(err, target error) bool { + // this is an outright copy of go 1.13 errors.Is(err, target error) bool + // removed isComparable + if err == nil || target == nil { + return err == target + } + + for { + if err == target { + return true + } + if x, ok := err.(interface{ Is(error) bool }); ok && x.Is(target) { + return true + } + // TODO: consider supporing target.Is(err). This would allow + // user-definable predicates, but also may allow for coping with sloppy + // APIs, thereby making it easier to get away with them. + if err = errorsUnwrap(err); err == nil { + return false + } + } +} diff --git a/server/errors.json b/server/errors.json new file mode 100644 index 0000000..63c217f --- /dev/null +++ b/server/errors.json @@ -0,0 +1,2262 @@ +[ + { + "constant": "JSClusterPeerNotMemberErr", + "code": 400, + "error_code": 10040, + "description": "peer not a member", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerEphemeralWithDurableInSubjectErr", + "code": 400, + "error_code": 10019, + "description": "consumer expected to be ephemeral but detected a durable name set in subject", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamExternalDelPrefixOverlapsErrF", + "code": 400, + "error_code": 10022, + "description": "stream external delivery prefix {prefix} overlaps with stream subject {subject}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSAccountResourcesExceededErr", + "code": 400, + "error_code": 10002, + "description": "resource limits exceeded for account", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSClusterNotAvailErr", + "code": 503, + "error_code": 10008, + "description": "JetStream system temporarily unavailable", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamSubjectOverlapErr", + "code": 400, + "error_code": 10065, + "description": "subjects overlap with an existing stream", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamWrongLastSequenceErrF", + "code": 400, + "error_code": 10071, + "description": "wrong last sequence: {seq}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSTemplateNameNotMatchSubjectErr", + "code": 400, + "error_code": 10073, + "description": "template name in subject does not match request", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSClusterNoPeersErrF", + "code": 400, + "error_code": 10005, + "description": "{err}", + "comment": "Error causing no peers to be available", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerEphemeralWithDurableNameErr", + "code": 400, + "error_code": 10020, + "description": "consumer expected to be ephemeral but a durable name was set in request", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSInsufficientResourcesErr", + "code": 503, + "error_code": 10023, + "description": "insufficient resources", + "comment": "", + "help": "", + "url": "", + "deprecates": "ErrJetStreamResourcesExceeded" + }, + { + "constant": "JSMirrorMaxMessageSizeTooBigErr", + "code": 400, + "error_code": 10030, + "description": "stream mirror must have max message size \u003e= source", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamTemplateDeleteErrF", + "code": 500, + "error_code": 10067, + "description": "{err}", + "comment": "Generic stream template deletion failed error string", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSBadRequestErr", + "code": 400, + "error_code": 10003, + "description": "bad request", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSClusterUnSupportFeatureErr", + "code": 503, + "error_code": 10036, + "description": "not currently supported in clustered mode", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerNotFoundErr", + "code": 404, + "error_code": 10014, + "description": "consumer not found", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSSourceMaxMessageSizeTooBigErr", + "code": 400, + "error_code": 10046, + "description": "stream source must have max message size \u003e= target", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamAssignmentErrF", + "code": 500, + "error_code": 10048, + "description": "{err}", + "comment": "Generic stream assignment error string", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamMessageExceedsMaximumErr", + "code": 400, + "error_code": 10054, + "description": "message size exceeds maximum allowed", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamTemplateCreateErrF", + "code": 500, + "error_code": 10066, + "description": "{err}", + "comment": "Generic template creation failed string", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSInvalidJSONErr", + "code": 400, + "error_code": 10025, + "description": "invalid JSON: {err}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamInvalidExternalDeliverySubjErrF", + "code": 400, + "error_code": 10024, + "description": "stream external delivery prefix {prefix} must not contain wildcards", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamRestoreErrF", + "code": 500, + "error_code": 10062, + "description": "restore failed: {err}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSClusterIncompleteErr", + "code": 503, + "error_code": 10004, + "description": "incomplete results", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSNoAccountErr", + "code": 503, + "error_code": 10035, + "description": "account not found", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSRaftGeneralErrF", + "code": 500, + "error_code": 10041, + "description": "{err}", + "comment": "General RAFT error string", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSRestoreSubscribeFailedErrF", + "code": 500, + "error_code": 10042, + "description": "JetStream unable to subscribe to restore snapshot {subject}: {err}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamDeleteErrF", + "code": 500, + "error_code": 10050, + "description": "{err}", + "comment": "General stream deletion error string", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamExternalApiOverlapErrF", + "code": 400, + "error_code": 10021, + "description": "stream external api prefix {prefix} must not overlap with {subject}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMirrorWithSubjectsErr", + "code": 400, + "error_code": 10034, + "description": "stream mirrors can not contain subjects", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMirrorWithFirstSeqErr", + "code": 400, + "error_code": 10143, + "description": "stream mirrors can not have first sequence configured", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSNotEnabledErr", + "code": 503, + "error_code": 10076, + "description": "JetStream not enabled", + "comment": "", + "help": "This error indicates that JetStream is not enabled at a global level", + "url": "", + "deprecates": "ErrJetStreamNotEnabled" + }, + { + "constant": "JSNotEnabledForAccountErr", + "code": 503, + "error_code": 10039, + "description": "JetStream not enabled for account", + "comment": "", + "help": "This error indicates that JetStream is not enabled for an account account level", + "url": "", + "deprecates": "" + }, + { + "constant": "JSSequenceNotFoundErrF", + "code": 400, + "error_code": 10043, + "description": "sequence {seq} not found", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamMirrorNotUpdatableErr", + "code": 400, + "error_code": 10055, + "description": "stream mirror configuration can not be updated", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamSequenceNotMatchErr", + "code": 503, + "error_code": 10063, + "description": "expected stream sequence does not match", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamWrongLastMsgIDErrF", + "code": 400, + "error_code": 10070, + "description": "wrong last msg ID: {id}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSTempStorageFailedErr", + "code": 500, + "error_code": 10072, + "description": "JetStream unable to open temp storage for restore", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStorageResourcesExceededErr", + "code": 500, + "error_code": 10047, + "description": "insufficient storage resources available", + "comment": "", + "help": "", + "url": "", + "deprecates": "ErrStorageResourcesExceeded" + }, + { + "constant": "JSStreamMismatchErr", + "code": 400, + "error_code": 10056, + "description": "stream name in subject does not match request", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamNotMatchErr", + "code": 400, + "error_code": 10060, + "description": "expected stream does not match", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMirrorConsumerSetupFailedErrF", + "code": 500, + "error_code": 10029, + "description": "{err}", + "comment": "generic mirror consumer setup failure string", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSNotEmptyRequestErr", + "code": 400, + "error_code": 10038, + "description": "expected an empty request payload", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamNameExistErr", + "code": 400, + "error_code": 10058, + "description": "stream name already in use with a different configuration", + "comment": "", + "help": "", + "url": "", + "deprecates": "ErrJetStreamStreamAlreadyUsed" + }, + { + "constant": "JSClusterTagsErr", + "code": 400, + "error_code": 10011, + "description": "tags placement not supported for operation", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMaximumConsumersLimitErr", + "code": 400, + "error_code": 10026, + "description": "maximum consumers limit reached", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSSourceConsumerSetupFailedErrF", + "code": 500, + "error_code": 10045, + "description": "{err}", + "comment": "General source consumer setup failure string", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerCreateErrF", + "code": 500, + "error_code": 10012, + "description": "{err}", + "comment": "General consumer creation failure string", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerDurableNameNotInSubjectErr", + "code": 400, + "error_code": 10016, + "description": "consumer expected to be durable but no durable name set in subject", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamLimitsErrF", + "code": 500, + "error_code": 10053, + "description": "{err}", + "comment": "General stream limits exceeded error string", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamReplicasNotUpdatableErr", + "code": 400, + "error_code": 10061, + "description": "Replicas configuration can not be updated", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamTemplateNotFoundErr", + "code": 404, + "error_code": 10068, + "description": "template not found", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSClusterNotAssignedErr", + "code": 500, + "error_code": 10007, + "description": "JetStream cluster not assigned to this server", + "comment": "", + "help": "", + "url": "", + "deprecates": "ErrJetStreamNotAssigned" + }, + { + "constant": "JSClusterNotLeaderErr", + "code": 500, + "error_code": 10009, + "description": "JetStream cluster can not handle request", + "comment": "", + "help": "", + "url": "", + "deprecates": "ErrJetStreamNotLeader" + }, + { + "constant": "JSConsumerNameExistErr", + "code": 400, + "error_code": 10013, + "description": "consumer name already in use", + "comment": "", + "help": "", + "url": "", + "deprecates": "ErrJetStreamConsumerAlreadyUsed" + }, + { + "constant": "JSMirrorWithSourcesErr", + "code": 400, + "error_code": 10031, + "description": "stream mirrors can not also contain other sources", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamNotFoundErr", + "code": 404, + "error_code": 10059, + "description": "stream not found", + "comment": "", + "help": "", + "url": "", + "deprecates": "ErrJetStreamStreamNotFound" + }, + { + "constant": "JSClusterRequiredErr", + "code": 503, + "error_code": 10010, + "description": "JetStream clustering support required", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerDurableNameNotSetErr", + "code": 400, + "error_code": 10018, + "description": "consumer expected to be durable but a durable name was not set", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMaximumStreamsLimitErr", + "code": 400, + "error_code": 10027, + "description": "maximum number of streams reached", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMirrorWithStartSeqAndTimeErr", + "code": 400, + "error_code": 10032, + "description": "stream mirrors can not have both start seq and start time configured", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamSnapshotErrF", + "code": 500, + "error_code": 10064, + "description": "snapshot failed: {err}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamUpdateErrF", + "code": 500, + "error_code": 10069, + "description": "{err}", + "comment": "Generic stream update error string", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSClusterNotActiveErr", + "code": 500, + "error_code": 10006, + "description": "JetStream not in clustered mode", + "comment": "", + "help": "", + "url": "", + "deprecates": "ErrJetStreamNotClustered" + }, + { + "constant": "JSConsumerDurableNameNotMatchSubjectErr", + "code": 400, + "error_code": 10017, + "description": "consumer name in subject does not match durable name in request", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMemoryResourcesExceededErr", + "code": 500, + "error_code": 10028, + "description": "insufficient memory resources available", + "comment": "", + "help": "", + "url": "", + "deprecates": "ErrMemoryResourcesExceeded" + }, + { + "constant": "JSMirrorWithSubjectFiltersErr", + "code": 400, + "error_code": 10033, + "description": "stream mirrors can not contain filtered subjects", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamCreateErrF", + "code": 500, + "error_code": 10049, + "description": "{err}", + "comment": "Generic stream creation error string", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSClusterServerNotMemberErr", + "code": 400, + "error_code": 10044, + "description": "server is not a member of the cluster", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSNoMessageFoundErr", + "code": 404, + "error_code": 10037, + "description": "no message found", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSSnapshotDeliverSubjectInvalidErr", + "code": 400, + "error_code": 10015, + "description": "deliver subject not valid", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamGeneralErrorF", + "code": 500, + "error_code": 10051, + "description": "{err}", + "comment": "General stream failure string", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamInvalidConfigF", + "code": 500, + "error_code": 10052, + "description": "{err}", + "comment": "Stream configuration validation error string", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamReplicasNotSupportedErr", + "code": 500, + "error_code": 10074, + "description": "replicas \u003e 1 not supported in non-clustered mode", + "comment": "", + "help": "", + "url": "", + "deprecates": "ErrReplicasNotSupported" + }, + { + "constant": "JSStreamMsgDeleteFailedF", + "code": 500, + "error_code": 10057, + "description": "{err}", + "comment": "Generic message deletion failure error string", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSPeerRemapErr", + "code": 503, + "error_code": 10075, + "description": "peer remap failed", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamStoreFailedF", + "code": 503, + "error_code": 10077, + "description": "{err}", + "comment": "Generic error when storing a message failed", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerConfigRequiredErr", + "code": 400, + "error_code": 10078, + "description": "consumer config required", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerDeliverToWildcardsErr", + "code": 400, + "error_code": 10079, + "description": "consumer deliver subject has wildcards", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerPushMaxWaitingErr", + "code": 400, + "error_code": 10080, + "description": "consumer in push mode can not set max waiting", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerDeliverCycleErr", + "code": 400, + "error_code": 10081, + "description": "consumer deliver subject forms a cycle", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerMaxPendingAckPolicyRequiredErr", + "code": 400, + "error_code": 10082, + "description": "consumer requires ack policy for max ack pending", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerSmallHeartbeatErr", + "code": 400, + "error_code": 10083, + "description": "consumer idle heartbeat needs to be \u003e= 100ms", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerPullRequiresAckErr", + "code": 400, + "error_code": 10084, + "description": "consumer in pull mode requires explicit ack policy on workqueue stream", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerPullNotDurableErr", + "code": 400, + "error_code": 10085, + "description": "consumer in pull mode requires a durable name", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerPullWithRateLimitErr", + "code": 400, + "error_code": 10086, + "description": "consumer in pull mode can not have rate limit set", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerMaxWaitingNegativeErr", + "code": 400, + "error_code": 10087, + "description": "consumer max waiting needs to be positive", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerHBRequiresPushErr", + "code": 400, + "error_code": 10088, + "description": "consumer idle heartbeat requires a push based consumer", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerFCRequiresPushErr", + "code": 400, + "error_code": 10089, + "description": "consumer flow control requires a push based consumer", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerDirectRequiresPushErr", + "code": 400, + "error_code": 10090, + "description": "consumer direct requires a push based consumer", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerDirectRequiresEphemeralErr", + "code": 400, + "error_code": 10091, + "description": "consumer direct requires an ephemeral consumer", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerOnMappedErr", + "code": 400, + "error_code": 10092, + "description": "consumer direct on a mapped consumer", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerFilterNotSubsetErr", + "code": 400, + "error_code": 10093, + "description": "consumer filter subject is not a valid subset of the interest subjects", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerInvalidPolicyErrF", + "code": 400, + "error_code": 10094, + "description": "{err}", + "comment": "Generic delivery policy error", + "help": "Error returned for impossible deliver policies when combined with start sequences etc", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerInvalidSamplingErrF", + "code": 400, + "error_code": 10095, + "description": "failed to parse consumer sampling configuration: {err}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamInvalidErr", + "code": 500, + "error_code": 10096, + "description": "stream not valid", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerWQRequiresExplicitAckErr", + "code": 400, + "error_code": 10098, + "description": "workqueue stream requires explicit ack", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerWQMultipleUnfilteredErr", + "code": 400, + "error_code": 10099, + "description": "multiple non-filtered consumers not allowed on workqueue stream", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerWQConsumerNotUniqueErr", + "code": 400, + "error_code": 10100, + "description": "filtered consumer not unique on workqueue stream", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerWQConsumerNotDeliverAllErr", + "code": 400, + "error_code": 10101, + "description": "consumer must be deliver all on workqueue stream", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerNameTooLongErrF", + "code": 400, + "error_code": 10102, + "description": "consumer name is too long, maximum allowed is {max}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerBadDurableNameErr", + "code": 400, + "error_code": 10103, + "description": "durable name can not contain '.', '*', '\u003e'", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerStoreFailedErrF", + "code": 500, + "error_code": 10104, + "description": "error creating store for consumer: {err}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerExistingActiveErr", + "code": 400, + "error_code": 10105, + "description": "consumer already exists and is still active", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerReplacementWithDifferentNameErr", + "code": 400, + "error_code": 10106, + "description": "consumer replacement durable config not the same", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerDescriptionTooLongErrF", + "code": 400, + "error_code": 10107, + "description": "consumer description is too long, maximum allowed is {max}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamHeaderExceedsMaximumErr", + "code": 400, + "error_code": 10097, + "description": "header size exceeds maximum allowed of 64k", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerWithFlowControlNeedsHeartbeats", + "code": 400, + "error_code": 10108, + "description": "consumer with flow control also needs heartbeats", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamSealedErr", + "code": 400, + "error_code": 10109, + "description": "invalid operation on sealed stream", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamPurgeFailedF", + "code": 500, + "error_code": 10110, + "description": "{err}", + "comment": "Generic stream purge failure error string", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamRollupFailedF", + "code": 500, + "error_code": 10111, + "description": "{err}", + "comment": "Generic stream rollup failure error string", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerInvalidDeliverSubject", + "code": 400, + "error_code": 10112, + "description": "invalid push consumer deliver subject", + "comment": "", + "help": "Returned when the delivery subject on a Push Consumer is not a valid NATS Subject", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamMaxBytesRequired", + "code": 400, + "error_code": 10113, + "description": "account requires a stream config to have max bytes set", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerMaxRequestBatchNegativeErr", + "code": 400, + "error_code": 10114, + "description": "consumer max request batch needs to be \u003e 0", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerMaxRequestExpiresTooSmall", + "code": 400, + "error_code": 10115, + "description": "consumer max request expires needs to be \u003e= 1ms", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerMaxDeliverBackoffErr", + "code": 400, + "error_code": 10116, + "description": "max deliver is required to be \u003e length of backoff values", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamInfoMaxSubjectsErr", + "code": 500, + "error_code": 10117, + "description": "subject details would exceed maximum allowed", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamOfflineErr", + "code": 500, + "error_code": 10118, + "description": "stream is offline", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerOfflineErr", + "code": 500, + "error_code": 10119, + "description": "consumer is offline", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSNoLimitsErr", + "code": 400, + "error_code": 10120, + "description": "no JetStream default or applicable tiered limit present", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerMaxPendingAckExcessErrF", + "code": 400, + "error_code": 10121, + "description": "consumer max ack pending exceeds system limit of {limit}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamMaxStreamBytesExceeded", + "code": 400, + "error_code": 10122, + "description": "stream max bytes exceeds account limit max stream bytes", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamMoveAndScaleErr", + "code": 400, + "error_code": 10123, + "description": "can not move and scale a stream in a single update", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamMoveInProgressErr", + "code": 400, + "error_code": 10124, + "description": "stream move already in progress", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerMaxRequestBatchExceededF", + "code": 400, + "error_code": 10125, + "description": "consumer max request batch exceeds server limit of {limit}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerReplicasExceedsStream", + "code": 400, + "error_code": 10126, + "description": "consumer config replica count exceeds parent stream", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerNameContainsPathSeparatorsErr", + "code": 400, + "error_code": 10127, + "description": "Consumer name can not contain path separators", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamNameContainsPathSeparatorsErr", + "code": 400, + "error_code": 10128, + "description": "Stream name can not contain path separators", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamReconfigureNotInProgressErr", + "code": 400, + "error_code": 10129, + "description": "stream reconfiguration not in progress", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamNameExistRestoreFailedErr", + "code": 400, + "error_code": 10130, + "description": "stream name already in use, cannot restore", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerCreateFilterSubjectMismatchErr", + "code": 400, + "error_code": 10131, + "description": "Consumer create request did not match filtered subject from create subject", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerCreateDurableAndNameMismatch", + "code": 400, + "error_code": 10132, + "description": "Consumer Durable and Name have to be equal if both are provided", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSReplicasCountCannotBeNegative", + "code": 400, + "error_code": 10133, + "description": "replicas count cannot be negative", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerReplicasShouldMatchStream", + "code": 400, + "error_code": 10134, + "description": "consumer config replicas must match interest retention stream's replicas", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerMetadataLengthErrF", + "code": 400, + "error_code": 10135, + "description": "consumer metadata exceeds maximum size of {limit}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerDuplicateFilterSubjects", + "code": 400, + "error_code": 10136, + "description": "consumer cannot have both FilterSubject and FilterSubjects specified", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerMultipleFiltersNotAllowed", + "code": 400, + "error_code": 10137, + "description": "consumer with multiple subject filters cannot use subject based API", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerOverlappingSubjectFilters", + "code": 400, + "error_code": 10138, + "description": "consumer subject filters cannot overlap", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerEmptyFilter", + "code": 400, + "error_code": 10139, + "description": "consumer filter in FilterSubjects cannot be empty", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSSourceDuplicateDetected", + "code": 400, + "error_code": 10140, + "description": "duplicate source configuration detected", + "comment": "source stream, filter and transform (plus external if present) must form a unique combination", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSSourceInvalidStreamName", + "code": 400, + "error_code": 10141, + "description": "sourced stream name is invalid", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMirrorInvalidStreamName", + "code": 400, + "error_code": 10142, + "description": "mirrored stream name is invalid", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSSourceMultipleFiltersNotAllowed", + "code": 400, + "error_code": 10144, + "description": "source with multiple subject transforms cannot also have a single subject filter", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSSourceInvalidSubjectFilter", + "code": 400, + "error_code": 10145, + "description": "source transform source: {err}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSSourceInvalidTransformDestination", + "code": 400, + "error_code": 10146, + "description": "source transform: {err}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSSourceOverlappingSubjectFilters", + "code": 400, + "error_code": 10147, + "description": "source filters can not overlap", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerAlreadyExists", + "code": 400, + "error_code": 10148, + "description": "consumer already exists", + "comment": "action CREATE is used for a existing consumer with a different config", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerDoesNotExist", + "code": 400, + "error_code": 10149, + "description": "consumer does not exist", + "comment": "action UPDATE is used for a nonexisting consumer", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMirrorMultipleFiltersNotAllowed", + "code": 400, + "error_code": 10150, + "description": "mirror with multiple subject transforms cannot also have a single subject filter", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMirrorInvalidSubjectFilter", + "code": 400, + "error_code": 10151, + "description": "mirror transform source: {err}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMirrorOverlappingSubjectFilters", + "code": 400, + "error_code": 10152, + "description": "mirror subject filters can not overlap", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerInactiveThresholdExcess", + "code": 400, + "error_code": 10153, + "description": "consumer inactive threshold exceeds system limit of {limit}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMirrorInvalidTransformDestination", + "code": 400, + "error_code": 10154, + "description": "mirror transform: {err}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamTransformInvalidSource", + "code": 400, + "error_code": 10155, + "description": "stream transform source: {err}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamTransformInvalidDestination", + "code": 400, + "error_code": 10156, + "description": "stream transform: {err}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSPedanticErrF", + "code": 400, + "error_code": 10157, + "description": "pedantic mode: {err}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamDuplicateMessageConflict", + "code": 409, + "error_code": 10158, + "description": "duplicate message id is in process", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerPriorityPolicyWithoutGroup", + "code": 400, + "error_code": 10159, + "description": "Setting PriorityPolicy requires at least one PriorityGroup to be set", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerInvalidPriorityGroupErr", + "code": 400, + "error_code": 10160, + "description": "Provided priority group does not exist for this consumer", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerEmptyGroupName", + "code": 400, + "error_code": 10161, + "description": "Group name cannot be an empty string", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerInvalidGroupNameErr", + "code": 400, + "error_code": 10162, + "description": "Valid priority group name must match A-Z, a-z, 0-9, -_/=)+ and may not exceed 16 characters", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamExpectedLastSeqPerSubjectNotReady", + "code": 503, + "error_code": 10163, + "description": "expected last sequence per subject temporarily unavailable", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamWrongLastSequenceConstantErr", + "code": 400, + "error_code": 10164, + "description": "wrong last sequence", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMessageTTLInvalidErr", + "code": 400, + "error_code": 10165, + "description": "invalid per-message TTL", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMessageTTLDisabledErr", + "code": 400, + "error_code": 10166, + "description": "per-message TTL is disabled", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamTooManyRequests", + "code": 429, + "error_code": 10167, + "description": "too many requests", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMessageIncrDisabledErr", + "code": 400, + "error_code": 10168, + "description": "message counters is disabled", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMessageIncrMissingErr", + "code": 400, + "error_code": 10169, + "description": "message counter increment is missing", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMessageIncrPayloadErr", + "code": 400, + "error_code": 10170, + "description": "message counter has payload", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMessageIncrInvalidErr", + "code": 400, + "error_code": 10171, + "description": "message counter increment is invalid", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMessageCounterBrokenErr", + "code": 400, + "error_code": 10172, + "description": "message counter is broken", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMirrorWithCountersErr", + "code": 400, + "error_code": 10173, + "description": "stream mirrors can not also calculate counters", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSAtomicPublishDisabledErr", + "code": 400, + "error_code": 10174, + "description": "atomic publish is disabled", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSAtomicPublishMissingSeqErr", + "code": 400, + "error_code": 10175, + "description": "atomic publish sequence is missing", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSAtomicPublishIncompleteBatchErr", + "code": 400, + "error_code": 10176, + "description": "atomic publish batch is incomplete", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSAtomicPublishUnsupportedHeaderBatchErr", + "code": 400, + "error_code": 10177, + "description": "atomic publish unsupported header used: {header}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerPushWithPriorityGroupErr", + "code": 400, + "error_code": 10178, + "description": "priority groups can not be used with push consumers", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSAtomicPublishInvalidBatchIDErr", + "code": 400, + "error_code": 10179, + "description": "atomic publish batch ID is invalid", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamMinLastSeqErr", + "code": 412, + "error_code": 10180, + "description": "min last sequence", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerAckPolicyInvalidErr", + "code": 400, + "error_code": 10181, + "description": "consumer ack policy invalid", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerReplayPolicyInvalidErr", + "code": 400, + "error_code": 10182, + "description": "consumer replay policy invalid", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerAckWaitNegativeErr", + "code": 400, + "error_code": 10183, + "description": "consumer ack wait needs to be positive", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerBackOffNegativeErr", + "code": 400, + "error_code": 10184, + "description": "consumer backoff needs to be positive", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSRequiredApiLevelErr", + "code": 412, + "error_code": 10185, + "description": "JetStream minimum api level required", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMirrorWithMsgSchedulesErr", + "code": 400, + "error_code": 10186, + "description": "stream mirrors can not also schedule messages", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSSourceWithMsgSchedulesErr", + "code": 400, + "error_code": 10187, + "description": "stream source can not also schedule messages", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMessageSchedulesDisabledErr", + "code": 400, + "error_code": 10188, + "description": "message schedules is disabled", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMessageSchedulesPatternInvalidErr", + "code": 400, + "error_code": 10189, + "description": "message schedules pattern is invalid", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMessageSchedulesTargetInvalidErr", + "code": 400, + "error_code": 10190, + "description": "message schedules target is invalid", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMessageSchedulesTTLInvalidErr", + "code": 400, + "error_code": 10191, + "description": "message schedules invalid per-message TTL", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMessageSchedulesRollupInvalidErr", + "code": 400, + "error_code": 10192, + "description": "message schedules invalid rollup", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamExpectedLastSeqPerSubjectInvalid", + "code": 400, + "error_code": 10193, + "description": "missing sequence for expected last sequence per subject", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamOfflineReasonErrF", + "code": 500, + "error_code": 10194, + "description": "stream is offline: {err}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerOfflineReasonErrF", + "code": 500, + "error_code": 10195, + "description": "consumer is offline: {err}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerPriorityGroupWithPolicyNone", + "code": 400, + "error_code": 10196, + "description": "consumer can not have priority groups when policy is none", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerPinnedTTLWithoutPriorityPolicyNone", + "code": 400, + "error_code": 10197, + "description": "PinnedTTL cannot be set when PriorityPolicy is none", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMirrorWithAtomicPublishErr", + "code": 400, + "error_code": 10198, + "description": "stream mirrors can not also use atomic publishing", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSAtomicPublishTooLargeBatchErrF", + "code": 400, + "error_code": 10199, + "description": "atomic publish batch is too large: {size}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSAtomicPublishInvalidBatchCommitErr", + "code": 400, + "error_code": 10200, + "description": "atomic publish batch commit is invalid", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSAtomicPublishContainsDuplicateMessageErr", + "code": 400, + "error_code": 10201, + "description": "atomic publish batch contains duplicate message id", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSClusterServerMemberChangeInflightErr", + "code": 400, + "error_code": 10202, + "description": "cluster member change is in progress", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMessageSchedulesSourceInvalidErr", + "code": 400, + "error_code": 10203, + "description": "message schedules source is invalid", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerInvalidResetErr", + "code": 400, + "error_code": 10204, + "description": "invalid reset: {err}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSBatchPublishDisabledErr", + "code": 400, + "error_code": 10205, + "description": "batch publish is disabled", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSBatchPublishInvalidPatternErr", + "code": 400, + "error_code": 10206, + "description": "batch publish pattern is invalid", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSBatchPublishInvalidBatchIDErr", + "code": 400, + "error_code": 10207, + "description": "batch publish ID is invalid", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSBatchPublishUnknownBatchIDErr", + "code": 400, + "error_code": 10208, + "description": "batch publish ID unknown", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMirrorWithBatchPublishErr", + "code": 400, + "error_code": 10209, + "description": "stream mirrors can not also use batch publishing", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSAtomicPublishTooManyInflight", + "code": 429, + "error_code": 10210, + "description": "atomic publish too many inflight", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSBatchPublishTooManyInflight", + "code": 429, + "error_code": 10211, + "description": "batch publish too many inflight", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMessageSchedulesSchedulerInvalidErr", + "code": 400, + "error_code": 10212, + "description": "message schedules invalid scheduler", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMirrorDurableConsumerCfgInvalid", + "code": 400, + "error_code": 10213, + "description": "stream mirror consumer config is invalid", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMirrorConsumerRequiresAckFCErr", + "code": 400, + "error_code": 10214, + "description": "stream mirror consumer requires flow control ack policy", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSSourceDurableConsumerCfgInvalid", + "code": 400, + "error_code": 10215, + "description": "stream source consumer config is invalid", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSSourceDurableConsumerDuplicateDetected", + "code": 400, + "error_code": 10216, + "description": "duplicate stream source consumer detected", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSSourceConsumerRequiresAckFCErr", + "code": 400, + "error_code": 10217, + "description": "stream source consumer requires flow control ack policy", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerAckFCRequiresPushErr", + "code": 400, + "error_code": 10218, + "description": "flow control ack policy requires a push based consumer", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerAckFCRequiresFCErr", + "code": 400, + "error_code": 10219, + "description": "flow control ack policy requires flow control", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerAckFCRequiresMaxAckPendingErr", + "code": 400, + "error_code": 10220, + "description": "flow control ack policy requires max ack pending", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerAckFCRequiresNoAckWaitErr", + "code": 400, + "error_code": 10221, + "description": "flow control ack policy requires unset ack wait", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerAckFCRequiresNoMaxDeliverErr", + "code": 400, + "error_code": 10222, + "description": "flow control ack policy requires unset max deliver", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMessageSchedulesTimeZoneInvalidErr", + "code": 400, + "error_code": 10223, + "description": "message schedules time zone is invalid", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSClusterRescueErr", + "code": 400, + "error_code": 10224, + "description": "JetStream system rescue not applied: {err}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSConsumerStreamIdentityMismatchF", + "code": 400, + "error_code": 10225, + "description": "consumer's stream identity does not match: {msg}", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSStreamReconfigureInProgressErr", + "code": 400, + "error_code": 10226, + "description": "stream reconfiguration already in progress", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + }, + { + "constant": "JSMessageSourceHdrNotAllowedErr", + "code": 400, + "error_code": 10227, + "description": "message stream source header is not allowed", + "comment": "", + "help": "", + "url": "", + "deprecates": "" + } +] \ No newline at end of file diff --git a/server/errors_gen.go b/server/errors_gen.go new file mode 100644 index 0000000..675c405 --- /dev/null +++ b/server/errors_gen.go @@ -0,0 +1,238 @@ +//go:build ignore + +package main + +import ( + "encoding/json" + "fmt" + "log" + "os" + "os/exec" + "regexp" + "sort" + "strings" + "text/template" + + "github.com/nats-io/nats-server/v2/server" +) + +var tagRe = regexp.MustCompile("\\{(.+?)}") + +var templ = ` +// Generated code, do not edit. See errors.json and run go generate to update + +package server + +import "strings" + +const ( +{{- range $i, $error := . }} +{{- if .Comment }} + // {{ .Constant }} {{ .Comment }} ({{ .Description | print }}) +{{- else }} + // {{ .Constant }} {{ .Description | print }} +{{- end }} + {{ .Constant }} ErrorIdentifier = {{ .ErrCode }} +{{ end }} +) + +var ( + ApiErrors = map[ErrorIdentifier]*ApiError{ +{{- range $i, $error := . }} + {{ .Constant }}: {Code: {{ .Code }},ErrCode: {{ .ErrCode }},Description: {{ .Description | printf "%q" }}},{{- end }} + } + +{{- range $i, $error := . }} +{{- if .Deprecates }} +// {{ .Deprecates }} Deprecated by {{ .Constant }} ApiError, use IsNatsError() for comparisons +{{ .Deprecates }} = ApiErrors[{{ .Constant }}] +{{- end }} +{{- end }} +) + +{{- range $i, $error := . }} +// {{ .Constant | funcNameForConstant }} creates a new {{ .Constant }} error: {{ .Description | printf "%q" }} +func {{ .Constant | funcNameForConstant }}({{ .Description | funcArgsForTags }}) *ApiError { + eopts := parseOpts(opts) + if ae, ok := eopts.err.(*ApiError); ok { + return ae + } +{{ if .Description | hasTags }} + e:=ApiErrors[{{.Constant}}] + args:=e.toReplacerArgs([]interface{}{ {{.Description | replacerArgsForTags }} }) + return &ApiError{ + Code: e.Code, + ErrCode: e.ErrCode, + Description: strings.NewReplacer(args...).Replace(e.Description), + } +{{- else }} + return ApiErrors[{{.Constant}}] +{{- end }} +} + +{{- end }} +` + +func panicIfErr(err error) { + if err == nil { + return + } + panic(err) +} + +func goFmt(file string) error { + c := exec.Command("go", "fmt", file) + out, err := c.CombinedOutput() + if err != nil { + log.Printf("go fmt failed: %s", string(out)) + } + + return err +} + +func checkIncrements(errs []server.ErrorsData) error { + sort.Slice(errs, func(i, j int) bool { + return errs[i].ErrCode < errs[j].ErrCode + }) + + last := errs[0].ErrCode + gaps := []uint16{} + + for i := 1; i < len(errs); i++ { + if errs[i].ErrCode != last+1 { + gaps = append(gaps, last) + } + last = errs[i].ErrCode + } + + if len(gaps) > 0 { + return fmt.Errorf("gaps found in sequences: %v", gaps) + } + + return nil +} + +func checkDupes(errs []server.ErrorsData) error { + codes := []uint16{} + highest := uint16(0) + for _, err := range errs { + codes = append(codes, err.ErrCode) + if highest < err.ErrCode { + highest = err.ErrCode + } + } + + codeKeys := make(map[uint16]bool) + constKeys := make(map[string]bool) + + for _, entry := range errs { + if _, found := codeKeys[entry.ErrCode]; found { + return fmt.Errorf("duplicate error code %+v, highest code is %d", entry, highest) + } + + if _, found := constKeys[entry.Constant]; found { + return fmt.Errorf("duplicate error constant %+v", entry) + } + + codeKeys[entry.ErrCode] = true + constKeys[entry.Constant] = true + } + + return nil +} + +func findTags(d string) []string { + tags := []string{} + for _, tag := range tagRe.FindAllStringSubmatch(d, -1) { + if len(tag) != 2 { + continue + } + + tags = append(tags, tag[1]) + } + + sort.Strings(tags) + + return tags +} + +func main() { + ej, err := os.ReadFile("server/errors.json") + panicIfErr(err) + + errs := []server.ErrorsData{} + panicIfErr(json.Unmarshal(ej, &errs)) + + panicIfErr(checkDupes(errs)) + panicIfErr(checkIncrements(errs)) + + sort.Slice(errs, func(i, j int) bool { + return errs[i].Constant < errs[j].Constant + }) + + t := template.New("errors").Funcs( + template.FuncMap{ + "inc": func(i int) int { return i + 1 }, + "hasTags": func(d string) bool { + return strings.Contains(d, "{") && strings.Contains(d, "}") + }, + "replacerArgsForTags": func(d string) string { + res := []string{} + for _, tag := range findTags(d) { + res = append(res, `"{`+tag+`}"`) + res = append(res, tag) + } + + return strings.Join(res, ", ") + }, + "funcArgsForTags": func(d string) string { + res := []string{} + for _, tag := range findTags(d) { + if tag == "err" { + res = append(res, "err error") + } else if tag == "seq" { + res = append(res, "seq uint64") + } else { + res = append(res, fmt.Sprintf("%s interface{}", tag)) + } + } + + res = append(res, "opts ...ErrorOption") + + return strings.Join(res, ", ") + }, + "funcNameForConstant": func(c string) string { + res := "" + + switch { + case strings.HasSuffix(c, "ErrF"): + res = fmt.Sprintf("New%sError", strings.TrimSuffix(c, "ErrF")) + case strings.HasSuffix(c, "Err"): + res = fmt.Sprintf("New%sError", strings.TrimSuffix(c, "Err")) + case strings.HasSuffix(c, "ErrorF"): + res = fmt.Sprintf("New%s", strings.TrimSuffix(c, "F")) + case strings.HasSuffix(c, "F"): + res = fmt.Sprintf("New%sError", strings.TrimSuffix(c, "F")) + default: + res = fmt.Sprintf("New%s", c) + } + + if !strings.HasSuffix(res, "Error") { + res = fmt.Sprintf("%sError", res) + } + + return res + }, + }) + p, err := t.Parse(templ) + panicIfErr(err) + + tf, err := os.CreateTemp("", "") + panicIfErr(err) + defer tf.Close() + + panicIfErr(p.Execute(tf, errs)) + + panicIfErr(os.Rename(tf.Name(), "server/jetstream_errors_generated.go")) + panicIfErr(goFmt("server/jetstream_errors_generated.go")) +} diff --git a/server/errors_test.go b/server/errors_test.go new file mode 100644 index 0000000..fd62994 --- /dev/null +++ b/server/errors_test.go @@ -0,0 +1,74 @@ +// Copyright 2020-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "strings" + "testing" +) + +func TestErrCtx(t *testing.T) { + ctx := "Extra context information" + e := NewErrorCtx(ErrWrongGateway, "%s", ctx) + + if e.Error() != ErrWrongGateway.Error() { + t.Fatalf("%v and %v are supposed to be identical", e, ErrWrongGateway) + } + if e == ErrWrongGateway { + t.Fatalf("%v and %v can't be compared this way", e, ErrWrongGateway) + } + if !ErrorIs(e, ErrWrongGateway) { + t.Fatalf("%s and %s ", e, ErrWrongGateway) + } + if UnpackIfErrorCtx(ErrWrongGateway) != ErrWrongGateway.Error() { + t.Fatalf("Error of different type should be processed unchanged") + } + trace := UnpackIfErrorCtx(e) + if !strings.HasPrefix(trace, ErrWrongGateway.Error()) { + t.Fatalf("original error needs to remain") + } + if !strings.HasSuffix(trace, ctx) { + t.Fatalf("ctx needs to be added") + } +} + +func TestErrCtxWrapped(t *testing.T) { + ctxO := "Original Ctx" + eO := NewErrorCtx(ErrWrongGateway, "%s", ctxO) + ctx := "Extra context information" + e := NewErrorCtx(eO, "%s", ctx) + + if e.Error() != ErrWrongGateway.Error() { + t.Fatalf("%v and %v are supposed to be identical", e, ErrWrongGateway) + } + if e == ErrWrongGateway { + t.Fatalf("%v and %v can't be compared this way", e, ErrWrongGateway) + } + if !ErrorIs(e, ErrWrongGateway) { + t.Fatalf("%s and %s ", e, ErrWrongGateway) + } + if UnpackIfErrorCtx(ErrWrongGateway) != ErrWrongGateway.Error() { + t.Fatalf("Error of different type should be processed unchanged") + } + trace := UnpackIfErrorCtx(e) + if !strings.HasPrefix(trace, ErrWrongGateway.Error()) { + t.Fatalf("original error needs to remain") + } + if !strings.HasSuffix(trace, ctx) { + t.Fatalf("ctx needs to be added") + } + if !strings.Contains(trace, ctxO) { + t.Fatalf("Needs to contain every context") + } +} diff --git a/server/events.go b/server/events.go new file mode 100644 index 0000000..4af4f35 --- /dev/null +++ b/server/events.go @@ -0,0 +1,3389 @@ +// Copyright 2018-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "compress/gzip" + "crypto/sha256" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "math" + "math/rand/v2" + "net/http" + "runtime" + "runtime/debug" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/klauspost/compress/s2" + "github.com/nats-io/jwt/v2" + "github.com/nats-io/nats-server/v2/server/certidp" + "github.com/nats-io/nats-server/v2/server/pse" +) + +const ( + accLookupReqTokens = 6 + accLookupReqSubj = "$SYS.REQ.ACCOUNT.%s.CLAIMS.LOOKUP" + accPackReqSubj = "$SYS.REQ.CLAIMS.PACK" + accListReqSubj = "$SYS.REQ.CLAIMS.LIST" + accClaimsReqSubj = "$SYS.REQ.CLAIMS.UPDATE" + accDeleteReqSubj = "$SYS.REQ.CLAIMS.DELETE" + + connectEventSubj = "$SYS.ACCOUNT.%s.CONNECT" + disconnectEventSubj = "$SYS.ACCOUNT.%s.DISCONNECT" + accDirectReqSubj = "$SYS.REQ.ACCOUNT.%s.%s" + accPingReqSubj = "$SYS.REQ.ACCOUNT.PING.%s" // atm. only used for STATZ and CONNZ import from system account + // kept for backward compatibility when using http resolver + // this overlaps with the names for events but you'd have to have the operator private key in order to succeed. + accUpdateEventSubjOld = "$SYS.ACCOUNT.%s.CLAIMS.UPDATE" + accUpdateEventSubjNew = "$SYS.REQ.ACCOUNT.%s.CLAIMS.UPDATE" + connsRespSubj = "$SYS._INBOX_.%s" + accConnsEventSubjNew = "$SYS.ACCOUNT.%s.SERVER.CONNS" + accConnsEventSubjOld = "$SYS.SERVER.ACCOUNT.%s.CONNS" // kept for backward compatibility + lameDuckEventSubj = "$SYS.SERVER.%s.LAMEDUCK" + shutdownEventSubj = "$SYS.SERVER.%s.SHUTDOWN" + clientKickReqSubj = "$SYS.REQ.SERVER.%s.KICK" + clientLDMReqSubj = "$SYS.REQ.SERVER.%s.LDM" + authErrorEventSubj = "$SYS.SERVER.%s.CLIENT.AUTH.ERR" + authErrorAccountEventSubj = "$SYS.ACCOUNT.CLIENT.AUTH.ERR" + serverStatsSubj = "$SYS.SERVER.%s.STATSZ" + serverDirectReqSubj = "$SYS.REQ.SERVER.%s.%s" + serverPingReqSubj = "$SYS.REQ.SERVER.PING.%s" + serverStatsPingReqSubj = "$SYS.REQ.SERVER.PING" // use $SYS.REQ.SERVER.PING.STATSZ instead + serverReloadReqSubj = "$SYS.REQ.SERVER.%s.RELOAD" // with server ID + leafNodeConnectEventSubj = "$SYS.ACCOUNT.%s.LEAFNODE.CONNECT" // for internal use only + remoteLatencyEventSubj = "$SYS.LATENCY.M2.%s" + inboxRespSubj = "$SYS._INBOX.%s.%s" + + // Used to return information to a user on bound account and user permissions. + userDirectInfoSubj = "$SYS.REQ.USER.INFO" + userDirectReqSubj = "$SYS.REQ.USER.%s.INFO" + + // FIXME(dlc) - Should account scope, even with wc for now, but later on + // we can then shard as needed. + accNumSubsReqSubj = "$SYS.REQ.ACCOUNT.NSUBS" + + // These are for exported debug services. These are local to this server only. + accSubsSubj = "$SYS.DEBUG.SUBSCRIBERS" + + shutdownEventTokens = 4 + serverSubjectIndex = 2 + accUpdateTokensNew = 6 + accUpdateTokensOld = 5 + accUpdateAccIdxOld = 2 + + accReqTokens = 5 + accReqAccIndex = 3 + + ocspPeerRejectEventSubj = "$SYS.SERVER.%s.OCSP.PEER.CONN.REJECT" + ocspPeerChainlinkInvalidEventSubj = "$SYS.SERVER.%s.OCSP.PEER.LINK.INVALID" +) + +// FIXME(dlc) - make configurable. +var eventsHBInterval = 30 * time.Second +var statsHBInterval = 10 * time.Second + +// Default minimum wait time for sending statsz +const defaultStatszRateLimit = 1 * time.Second + +// Variable version so we can set in tests. +var statszRateLimit = defaultStatszRateLimit + +type sysMsgHandler func(sub *subscription, client *client, acc *Account, subject, reply string, hdr, msg []byte) + +// Used if we have to queue things internally to avoid the route/gw path. +type inSysMsg struct { + sub *subscription + c *client + acc *Account + subj string + rply string + hdr []byte + msg []byte + cb sysMsgHandler +} + +// Used to send and receive messages from inside the server. +type internal struct { + account *Account + client *client + seq uint64 + sid int + servers map[string]*serverUpdate + sweeper *time.Timer + stmr *time.Timer + replies map[string]msgHandler + sendq *ipQueue[*pubMsg] + recvq *ipQueue[*inSysMsg] + recvqp *ipQueue[*inSysMsg] // For STATSZ/Pings + resetCh chan struct{} + wg sync.WaitGroup + sq *sendq + orphMax time.Duration + chkOrph time.Duration + statsz time.Duration + cstatsz time.Duration + shash string + inboxPre string + remoteStatsSub *subscription + lastStatsz time.Time +} + +// ServerStatsMsg is sent periodically with stats updates. +type ServerStatsMsg struct { + Server ServerInfo `json:"server"` + Stats ServerStats `json:"statsz"` +} + +// ConnectEventMsg is sent when a new connection is made that is part of an account. +type ConnectEventMsg struct { + TypedEvent + Server ServerInfo `json:"server"` + Client ClientInfo `json:"client"` +} + +// ConnectEventMsgType is the schema type for ConnectEventMsg +const ConnectEventMsgType = "io.nats.server.advisory.v1.client_connect" + +// DisconnectEventMsg is sent when a new connection previously defined from a +// ConnectEventMsg is closed. +type DisconnectEventMsg struct { + TypedEvent + Server ServerInfo `json:"server"` + Client ClientInfo `json:"client"` + Sent DataStats `json:"sent"` + Received DataStats `json:"received"` + Reason string `json:"reason"` +} + +// DisconnectEventMsgType is the schema type for DisconnectEventMsg +const DisconnectEventMsgType = "io.nats.server.advisory.v1.client_disconnect" + +// OCSPPeerRejectEventMsg is sent when a peer TLS handshake is ultimately rejected due to OCSP invalidation. +// A "peer" can be an inbound client connection or a leaf connection to a remote server. Peer in event payload +// is always the peer's (TLS) leaf cert, which may or may be the invalid cert (See also OCSPPeerChainlinkInvalidEventMsg) +type OCSPPeerRejectEventMsg struct { + TypedEvent + Kind string `json:"kind"` + Peer certidp.CertInfo `json:"peer"` + Server ServerInfo `json:"server"` + Reason string `json:"reason"` +} + +// OCSPPeerRejectEventMsgType is the schema type for OCSPPeerRejectEventMsg +const OCSPPeerRejectEventMsgType = "io.nats.server.advisory.v1.ocsp_peer_reject" + +// OCSPPeerChainlinkInvalidEventMsg is sent when a certificate (link) in a valid TLS chain is found to be OCSP invalid +// during a peer TLS handshake. A "peer" can be an inbound client connection or a leaf connection to a remote server. +// Peer and Link may be the same if the invalid cert was the peer's leaf cert +type OCSPPeerChainlinkInvalidEventMsg struct { + TypedEvent + Link certidp.CertInfo `json:"link"` + Peer certidp.CertInfo `json:"peer"` + Server ServerInfo `json:"server"` + Reason string `json:"reason"` +} + +// OCSPPeerChainlinkInvalidEventMsgType is the schema type for OCSPPeerChainlinkInvalidEventMsg +const OCSPPeerChainlinkInvalidEventMsgType = "io.nats.server.advisory.v1.ocsp_peer_link_invalid" + +// AccountNumConns is an event that will be sent from a server that is tracking +// a given account when the number of connections changes. It will also HB +// updates in the absence of any changes. +type AccountNumConns struct { + TypedEvent + Server ServerInfo `json:"server"` + AccountStat +} + +// AccountStat contains the data common between AccountNumConns and AccountStatz +type AccountStat struct { + Account string `json:"acc"` + Name string `json:"name"` + Conns int `json:"conns"` + LeafNodes int `json:"leafnodes"` + TotalConns int `json:"total_conns"` + NumSubs uint32 `json:"num_subscriptions"` + Sent DataStats `json:"sent"` + Received DataStats `json:"received"` + SlowConsumers int64 `json:"slow_consumers"` +} + +const AccountNumConnsMsgType = "io.nats.server.advisory.v1.account_connections" + +// accNumConnsReq is sent when we are starting to track an account for the first +// time. We will request others send info to us about their local state. +type accNumConnsReq struct { + Server ServerInfo `json:"server"` + Account string `json:"acc"` +} + +// ServerID is basic static info for a server. +type ServerID struct { + Name string `json:"name"` + Host string `json:"host"` + ID string `json:"id"` +} + +// Type for our server capabilities. +type ServerCapability uint64 + +// ServerInfo identifies remote servers. +type ServerInfo struct { + Name string `json:"name"` + Host string `json:"host"` + ID string `json:"id"` + Cluster string `json:"cluster,omitempty"` + Domain string `json:"domain,omitempty"` + Version string `json:"ver"` + Tags []string `json:"tags,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + FeatureFlags map[string]bool `json:"feature_flags,omitempty"` + // Whether JetStream is enabled (deprecated in favor of the `ServerCapability`). + JetStream bool `json:"jetstream"` + // Generic capability flags + Flags ServerCapability `json:"flags"` + // Sequence and Time from the remote server for this message. + Seq uint64 `json:"seq"` + Time time.Time `json:"time"` +} + +const ( + JetStreamEnabled ServerCapability = 1 << iota // Server had JetStream enabled. + BinaryStreamSnapshot // New stream snapshot capability. + AccountNRG // Move NRG traffic out of system account. +) + +// Set JetStream capability. +func (si *ServerInfo) SetJetStreamEnabled() { + si.Flags |= JetStreamEnabled + // Still set old version. + si.JetStream = true +} + +// JetStreamEnabled indicates whether or not we have JetStream enabled. +func (si *ServerInfo) JetStreamEnabled() bool { + // Take into account old version. + return si.Flags&JetStreamEnabled != 0 || si.JetStream +} + +// Set binary stream snapshot capability. +func (si *ServerInfo) SetBinaryStreamSnapshot() { + si.Flags |= BinaryStreamSnapshot +} + +// JetStreamEnabled indicates whether or not we have binary stream snapshot capbilities. +func (si *ServerInfo) BinaryStreamSnapshot() bool { + return si.Flags&BinaryStreamSnapshot != 0 +} + +// Set account NRG capability. +func (si *ServerInfo) SetAccountNRG() { + si.Flags |= AccountNRG +} + +// AccountNRG indicates whether or not we support moving the NRG traffic out of the +// system account and into the asset account. +func (si *ServerInfo) AccountNRG() bool { + return si.Flags&AccountNRG != 0 +} + +// ClientInfo is detailed information about the client forming a connection. +type ClientInfo struct { + Start *time.Time `json:"start,omitempty"` + Host string `json:"host,omitempty"` + ID uint64 `json:"id,omitempty"` + Account string `json:"acc,omitempty"` + Service string `json:"svc,omitempty"` + User string `json:"user,omitempty"` + Name string `json:"name,omitempty"` + Lang string `json:"lang,omitempty"` + Version string `json:"ver,omitempty"` + RTT time.Duration `json:"rtt,omitempty"` + Server string `json:"server,omitempty"` + Cluster string `json:"cluster,omitempty"` + Alternates []string `json:"alts,omitempty"` + Stop *time.Time `json:"stop,omitempty"` + Jwt string `json:"jwt,omitempty"` + IssuerKey string `json:"issuer_key,omitempty"` + NameTag string `json:"name_tag,omitempty"` + Tags jwt.TagList `json:"tags,omitempty"` + Kind string `json:"kind,omitempty"` + ClientType string `json:"client_type,omitempty"` + MQTTClient string `json:"client_id,omitempty"` // This is the MQTT client ID + Nonce string `json:"nonce,omitempty"` + Reply string `json:"reply,omitempty"` // Original reply subject after a service import (only when needed). +} + +// forAssignmentSnap returns the minimum amount of ClientInfo we need for assignment snapshots. +func (ci *ClientInfo) forAssignmentSnap() *ClientInfo { + return &ClientInfo{ + Account: ci.Account, + Service: ci.Service, + Cluster: ci.Cluster, + } +} + +// forProposal returns the minimum amount of ClientInfo we need for assignment proposals. +func (ci *ClientInfo) forProposal() *ClientInfo { + if ci == nil { + return nil + } + cci := *ci + cci.Jwt = _EMPTY_ + cci.IssuerKey = _EMPTY_ + return &cci +} + +// forAdvisory returns the minimum amount of ClientInfo we need for JS advisory events. +func (ci *ClientInfo) forAdvisory() *ClientInfo { + if ci == nil { + return nil + } + cci := *ci + cci.Jwt = _EMPTY_ + cci.Alternates = nil + return &cci +} + +// ServerStats hold various statistics that we will periodically send out. +type ServerStats struct { + Start time.Time `json:"start"` + Mem int64 `json:"mem"` + Cores int `json:"cores"` + CPU float64 `json:"cpu"` + Connections int `json:"connections"` + TotalConnections uint64 `json:"total_connections"` + ActiveAccounts int `json:"active_accounts"` + NumSubs uint32 `json:"subscriptions"` + Sent DataStats `json:"sent"` + SentToClients DataStats `json:"sent_to_clients"` + Received DataStats `json:"received"` + ReceivedFromClients DataStats `json:"received_from_clients"` + SlowConsumers int64 `json:"slow_consumers"` + SlowConsumersStats *SlowConsumersStats `json:"slow_consumer_stats,omitempty"` + StaleConnections int64 `json:"stale_connections,omitempty"` + StaleConnectionStats *StaleConnectionStats `json:"stale_connection_stats,omitempty"` + StalledClients int64 `json:"stalled_clients,omitempty"` + Routes []*RouteStat `json:"routes,omitempty"` + Gateways []*GatewayStat `json:"gateways,omitempty"` + ActiveServers int `json:"active_servers,omitempty"` + JetStream *JetStreamVarz `json:"jetstream,omitempty"` + MemLimit int64 `json:"gomemlimit,omitempty"` + MaxProcs int `json:"gomaxprocs,omitempty"` +} + +// RouteStat holds route statistics. +type RouteStat struct { + ID uint64 `json:"rid"` + Name string `json:"name,omitempty"` + Sent DataStats `json:"sent"` + Received DataStats `json:"received"` + Pending int `json:"pending"` +} + +// GatewayStat holds gateway statistics. +type GatewayStat struct { + ID uint64 `json:"gwid"` + Name string `json:"name"` + Sent DataStats `json:"sent"` + Received DataStats `json:"received"` + NumInbound int `json:"inbound_connections"` +} + +type MsgBytes struct { + Msgs int64 `json:"msgs"` + Bytes int64 `json:"bytes"` +} + +// DataStats reports how may msg and bytes. Applicable for both sent and received. +type DataStats struct { + MsgBytes + Gateways *MsgBytes `json:"gateways,omitempty"` + Routes *MsgBytes `json:"routes,omitempty"` + Leafs *MsgBytes `json:"leafs,omitempty"` +} + +// Used for internally queueing up messages that the server wants to send. +type pubMsg struct { + c *client + sub string + rply string + si *ServerInfo + hdr []byte + msg any + oct compressionType + echo bool + last bool +} + +var pubMsgPool sync.Pool + +func newPubMsg(c *client, sub, rply string, si *ServerInfo, hdr []byte, + msg any, oct compressionType, echo, last bool) *pubMsg { + + var m *pubMsg + pm := pubMsgPool.Get() + if pm != nil { + m = pm.(*pubMsg) + } else { + m = &pubMsg{} + } + // When getting something from a pool it is critical that all fields are + // initialized. Doing this way guarantees that if someone adds a field to + // the structure, the compiler will fail the build if this line is not updated. + (*m) = pubMsg{c, sub, rply, si, hdr, msg, oct, echo, last} + return m +} + +func (pm *pubMsg) returnToPool() { + if pm == nil { + return + } + pm.c, pm.sub, pm.rply, pm.si, pm.hdr, pm.msg = nil, _EMPTY_, _EMPTY_, nil, nil, nil + pubMsgPool.Put(pm) +} + +// Used to track server updates. +type serverUpdate struct { + seq uint64 + ltime time.Time +} + +// TypedEvent is a event or advisory sent by the server that has nats type hints +// typically used for events that might be consumed by 3rd party event systems +type TypedEvent struct { + Type string `json:"type"` + ID string `json:"id"` + Time time.Time `json:"timestamp"` +} + +// internalReceiveLoop will be responsible for dispatching all messages that +// a server receives and needs to internally process, e.g. internal subs. +func (s *Server) internalReceiveLoop(recvq *ipQueue[*inSysMsg]) { + for s.eventsRunning() { + select { + case <-recvq.ch: + msgs := recvq.pop() + for _, m := range msgs { + if m.cb != nil { + m.cb(m.sub, m.c, m.acc, m.subj, m.rply, m.hdr, m.msg) + } + } + recvq.recycle(&msgs) + case <-s.quitCh: + return + } + } +} + +// internalSendLoop will be responsible for serializing all messages that +// a server wants to send. +func (s *Server) internalSendLoop(wg *sync.WaitGroup) { + defer wg.Done() + +RESET: + s.mu.RLock() + if s.sys == nil || s.sys.sendq == nil { + s.mu.RUnlock() + return + } + sysc := s.sys.client + resetCh := s.sys.resetCh + sendq := s.sys.sendq + id := s.info.ID + host := s.info.Host + servername := s.info.Name + domain := s.info.Domain + seqp := &s.sys.seq + js := s.info.JetStream + cluster := s.info.Cluster + if s.gateway.enabled { + cluster = s.getGatewayName() + } + s.mu.RUnlock() + + // Grab tags and metadata. + opts := s.getOpts() + tags, metadata, featureFlags := opts.Tags, opts.Metadata, opts.getMergedFeatureFlags() + + for s.eventsRunning() { + select { + case <-sendq.ch: + msgs := sendq.pop() + for _, pm := range msgs { + if si := pm.si; si != nil { + si.Name = servername + si.Domain = domain + si.Host = host + si.Cluster = cluster + si.ID = id + si.Seq = atomic.AddUint64(seqp, 1) + si.Version = VERSION + si.Time = time.Now().UTC() + si.Tags = tags + si.Metadata = metadata + si.FeatureFlags = featureFlags + si.Flags = 0 + if js { + // New capability based flags. + si.SetJetStreamEnabled() + si.SetBinaryStreamSnapshot() + if s.accountNRGAllowed.Load() { + si.SetAccountNRG() + } + } + } + var b []byte + if pm.msg != nil { + switch v := pm.msg.(type) { + case string: + b = []byte(v) + case []byte: + b = v + default: + b, _ = json.Marshal(pm.msg) + } + } + // Setup our client. If the user wants to use a non-system account use our internal + // account scoped here so that we are not changing out accounts for the system client. + var c *client + if pm.c != nil { + c = pm.c + } else { + c = sysc + } + + // Grab client lock. + c.mu.Lock() + + // Prep internal structures needed to send message. + c.pa.subject, c.pa.reply = []byte(pm.sub), []byte(pm.rply) + c.pa.size, c.pa.szb = len(b), []byte(strconv.FormatInt(int64(len(b)), 10)) + c.pa.hdr, c.pa.hdb = -1, nil + trace := c.trace + + // Now check for optional compression. + var contentHeader string + var bb bytes.Buffer + + if len(b) > 0 { + switch pm.oct { + case gzipCompression: + zw := gzip.NewWriter(&bb) + zw.Write(b) + zw.Close() + b = bb.Bytes() + contentHeader = "gzip" + case snappyCompression: + sw := s2.NewWriter(&bb, s2.WriterSnappyCompat()) + sw.Write(b) + sw.Close() + b = bb.Bytes() + contentHeader = "snappy" + case unsupportedCompression: + contentHeader = "identity" + } + } + // Optional Echo + replaceEcho := c.echo != pm.echo + if replaceEcho { + c.echo = !c.echo + } + c.mu.Unlock() + + // Add in NL + b = append(b, _CRLF_...) + + // Optional raw header addition. + if pm.hdr != nil { + b = append(pm.hdr[:len(pm.hdr):len(pm.hdr)], b...) + nhdr := len(pm.hdr) + nsize := len(b) - LEN_CR_LF + // MQTT producers don't have CRLF, so add it back. + if c.isMqtt() { + nsize += LEN_CR_LF + } + // Update pubArgs + // If others will use this later we need to save and restore original. + c.pa.hdr = nhdr + c.pa.size = nsize + c.pa.hdb = []byte(strconv.Itoa(nhdr)) + c.pa.szb = []byte(strconv.Itoa(nsize)) + } + + // Check if we should set content-encoding + if contentHeader != _EMPTY_ { + b = c.setHeader(contentEncodingHeader, contentHeader, b) + } + + // Tracing + if trace { + c.traceInOp(fmt.Sprintf("PUB %s %s %d", c.pa.subject, c.pa.reply, c.pa.size), nil) + c.traceMsg(b) + } + + // Process like a normal inbound msg. + c.processInboundClientMsg(b) + + // Put echo back if needed. + if replaceEcho { + c.mu.Lock() + c.echo = !c.echo + c.mu.Unlock() + } + + // See if we are doing graceful shutdown. + if !pm.last { + c.flushClients(0) // Never spend time in place. + } else { + // For the Shutdown event, we need to send in place otherwise + // there is a chance that the process will exit before the + // writeLoop has a chance to send it. + c.flushClients(time.Second) + sendq.recycle(&msgs) + return + } + pm.returnToPool() + } + sendq.recycle(&msgs) + case <-resetCh: + goto RESET + case <-s.quitCh: + return + } + } +} + +// Will send a shutdown message for lame-duck. Unlike sendShutdownEvent, this will +// not close off the send queue or reply handler, as we may still have a workload +// that needs migrating off. +// Lock should be held. +func (s *Server) sendLDMShutdownEventLocked() { + if s.sys == nil || s.sys.sendq == nil { + return + } + subj := fmt.Sprintf(lameDuckEventSubj, s.info.ID) + si := &ServerInfo{} + s.sys.sendq.push(newPubMsg(nil, subj, _EMPTY_, si, nil, si, noCompression, false, true)) +} + +// Will send a shutdown message. +func (s *Server) sendShutdownEvent() { + s.mu.Lock() + if s.sys == nil || s.sys.sendq == nil { + s.mu.Unlock() + return + } + subj := fmt.Sprintf(shutdownEventSubj, s.info.ID) + sendq := s.sys.sendq + // Stop any more messages from queueing up. + s.sys.sendq = nil + // Unhook all msgHandlers. Normal client cleanup will deal with subs, etc. + s.sys.replies = nil + // Send to the internal queue and mark as last. + si := &ServerInfo{} + sendq.push(newPubMsg(nil, subj, _EMPTY_, si, nil, si, noCompression, false, true)) + s.mu.Unlock() +} + +// Used to send an internal message to an arbitrary account. +func (s *Server) sendInternalAccountMsg(a *Account, subject string, msg any) error { + return s.sendInternalAccountMsgWithReply(a, subject, _EMPTY_, nil, msg, false) +} + +// Used to send an internal message with an optional reply to an arbitrary account. +func (s *Server) sendInternalAccountMsgWithReply(a *Account, subject, reply string, hdr []byte, msg any, echo bool) error { + s.mu.RLock() + if s.sys == nil || s.sys.sendq == nil { + s.mu.RUnlock() + if s.isShuttingDown() { + // Skip in case this was called at the end phase during shut down + // to avoid too many entries in the logs. + return nil + } + return ErrNoSysAccount + } + c := s.sys.client + // Replace our client with the account's internal client. + if a != nil { + a.mu.Lock() + c = a.internalClient() + a.mu.Unlock() + } + s.sys.sendq.push(newPubMsg(c, subject, reply, nil, hdr, msg, noCompression, echo, false)) + s.mu.RUnlock() + return nil +} + +// Send system style message to an account scope. +func (s *Server) sendInternalAccountSysMsg(a *Account, subj string, si *ServerInfo, msg any, ct compressionType) { + s.mu.RLock() + if s.sys == nil || s.sys.sendq == nil || a == nil { + s.mu.RUnlock() + return + } + sendq := s.sys.sendq + s.mu.RUnlock() + + a.mu.Lock() + c := a.internalClient() + a.mu.Unlock() + + sendq.push(newPubMsg(c, subj, _EMPTY_, si, nil, msg, ct, false, false)) +} + +// This will queue up a message to be sent. +// Lock should not be held. +func (s *Server) sendInternalMsgLocked(subj, rply string, si *ServerInfo, msg any) { + s.mu.RLock() + s.sendInternalMsg(subj, rply, si, msg) + s.mu.RUnlock() +} + +// This will queue up a message to be sent. +// Assumes lock is held on entry. +func (s *Server) sendInternalMsg(subj, rply string, si *ServerInfo, msg any) { + if s.sys == nil || s.sys.sendq == nil { + return + } + s.sys.sendq.push(newPubMsg(nil, subj, rply, si, nil, msg, noCompression, false, false)) +} + +// Will send an api response. +func (s *Server) sendInternalResponse(subj string, response *ServerAPIResponse) { + s.mu.RLock() + if s.sys == nil || s.sys.sendq == nil { + s.mu.RUnlock() + return + } + s.sys.sendq.push(newPubMsg(nil, subj, _EMPTY_, response.Server, nil, response, response.compress, false, false)) + s.mu.RUnlock() +} + +// Used to send internal messages from other system clients to avoid no echo issues. +func (c *client) sendInternalMsg(subj, rply string, si *ServerInfo, msg any) { + if c == nil { + return + } + s := c.srv + if s == nil { + return + } + s.mu.RLock() + if s.sys == nil || s.sys.sendq == nil { + s.mu.RUnlock() + return + } + s.sys.sendq.push(newPubMsg(c, subj, rply, si, nil, msg, noCompression, false, false)) + s.mu.RUnlock() +} + +// Locked version of checking if events system running. Also checks server. +func (s *Server) eventsRunning() bool { + if s == nil { + return false + } + s.mu.RLock() + er := s.isRunning() && s.eventsEnabled() + s.mu.RUnlock() + return er +} + +// EventsEnabled will report if the server has internal events enabled via +// a defined system account. +func (s *Server) EventsEnabled() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.eventsEnabled() +} + +// eventsEnabled will report if events are enabled. +// Lock should be held. +func (s *Server) eventsEnabled() bool { + return s.sys != nil && s.sys.client != nil && s.sys.account != nil +} + +// TrackedRemoteServers returns how many remote servers we are tracking +// from a system events perspective. +func (s *Server) TrackedRemoteServers() int { + s.mu.RLock() + defer s.mu.RUnlock() + if !s.isRunning() || !s.eventsEnabled() { + return -1 + } + return len(s.sys.servers) +} + +// Check for orphan servers who may have gone away without notification. +// This should be wrapChk() to setup common locking. +func (s *Server) checkRemoteServers() { + now := time.Now() + for sid, su := range s.sys.servers { + if now.Sub(su.ltime) > s.sys.orphMax { + s.Debugf("Detected orphan remote server: %q", sid) + // Simulate it going away. + s.processRemoteServerShutdown(sid) + } + } + if s.sys.sweeper != nil { + s.sys.sweeper.Reset(s.sys.chkOrph) + } +} + +// Grab RSS and PCPU +// Server lock will be held but released. +func (s *Server) updateServerUsage(v *ServerStats) { + var vss int64 + pse.ProcUsage(&v.CPU, &v.Mem, &vss) + v.Cores = runtime.NumCPU() + v.MaxProcs = runtime.GOMAXPROCS(-1) + if mm := debug.SetMemoryLimit(-1); mm < math.MaxInt64 { + v.MemLimit = mm + } +} + +// Generate a route stat for our statz update. +func routeStat(r *client) *RouteStat { + if r == nil { + return nil + } + r.mu.Lock() + // Note: *client.out[Msgs|Bytes] are not set using atomics, + // unlike in[Msgs|Bytes]. + rs := &RouteStat{ + ID: r.cid, + Sent: DataStats{ + MsgBytes: MsgBytes{ + Msgs: r.outMsgs, + Bytes: r.outBytes, + }, + }, + Received: DataStats{ + MsgBytes: MsgBytes{ + Msgs: atomic.LoadInt64(&r.inMsgs), + Bytes: atomic.LoadInt64(&r.inBytes), + }, + }, + Pending: int(r.out.pb), + } + if r.route != nil { + rs.Name = r.route.remoteName + } + r.mu.Unlock() + return rs +} + +// Actual send method for statz updates. +// Lock should be held. +func (s *Server) sendStatsz(subj string) { + var m ServerStatsMsg + s.updateServerUsage(&m.Stats) + + if s.limitStatsz(subj) { + return + } + + s.mu.RLock() + defer s.mu.RUnlock() + + // Check that we have a system account, etc. + if s.sys == nil || s.sys.account == nil { + return + } + + shouldCheckInterest := func() bool { + opts := s.getOpts() + if opts.Cluster.Port != 0 || opts.Gateway.Port != 0 || opts.LeafNode.Port != 0 { + return false + } + // If we are here we have no clustering or gateways and are not a leafnode hub. + // Check for leafnode remotes that connect the system account. + if len(opts.LeafNode.Remotes) > 0 { + sysAcc := s.sys.account.GetName() + for _, r := range opts.LeafNode.Remotes { + if r.LocalAccount == sysAcc { + return false + } + } + } + return true + } + + // if we are running standalone, check for interest. + if shouldCheckInterest() { + // Check if we even have interest in this subject. + sacc := s.sys.account + rr := sacc.sl.Match(subj) + totalSubs := len(rr.psubs) + len(rr.qsubs) + if totalSubs == 0 { + return + } else if totalSubs == 1 && len(rr.psubs) == 1 { + // For the broadcast subject we listen to that ourselves with no echo for remote updates. + // If we are the only ones listening do not send either. + if rr.psubs[0] == s.sys.remoteStatsSub { + return + } + } + } + + m.Stats.Start = s.start + m.Stats.Connections = len(s.clients) + m.Stats.TotalConnections = s.totalClients + m.Stats.ActiveAccounts = int(atomic.LoadInt32(&s.activeAccounts)) + m.Stats.Received.Msgs = atomic.LoadInt64(&s.inMsgs) + m.Stats.Received.Bytes = atomic.LoadInt64(&s.inBytes) + m.Stats.ReceivedFromClients.Msgs = atomic.LoadInt64(&s.inClientMsgs) + m.Stats.ReceivedFromClients.Bytes = atomic.LoadInt64(&s.inClientBytes) + m.Stats.Sent.Msgs = atomic.LoadInt64(&s.outMsgs) + m.Stats.Sent.Bytes = atomic.LoadInt64(&s.outBytes) + m.Stats.SentToClients.Msgs = atomic.LoadInt64(&s.outClientMsgs) + m.Stats.SentToClients.Bytes = atomic.LoadInt64(&s.outClientBytes) + m.Stats.SlowConsumers = atomic.LoadInt64(&s.slowConsumers) + // Evaluate the slow consumer stats, but set it only if one of the value is not 0. + scs := &SlowConsumersStats{ + Clients: s.NumSlowConsumersClients(), + Routes: s.NumSlowConsumersRoutes(), + Gateways: s.NumSlowConsumersGateways(), + Leafs: s.NumSlowConsumersLeafs(), + } + if scs.Clients != 0 || scs.Routes != 0 || scs.Gateways != 0 || scs.Leafs != 0 { + m.Stats.SlowConsumersStats = scs + } + m.Stats.StaleConnections = atomic.LoadInt64(&s.staleConnections) + m.Stats.StalledClients = atomic.LoadInt64(&s.stalls) + stcs := &StaleConnectionStats{ + Clients: s.NumStaleConnectionsClients(), + Routes: s.NumStaleConnectionsRoutes(), + Gateways: s.NumStaleConnectionsGateways(), + Leafs: s.NumStaleConnectionsLeafs(), + } + if stcs.Clients != 0 || stcs.Routes != 0 || stcs.Gateways != 0 || stcs.Leafs != 0 { + m.Stats.StaleConnectionStats = stcs + } + m.Stats.NumSubs = s.numSubscriptions() + // Routes + s.forEachRoute(func(r *client) { + m.Stats.Routes = append(m.Stats.Routes, routeStat(r)) + }) + // Gateways + if s.gateway.enabled { + gw := s.gateway + gw.RLock() + for name, c := range gw.out { + gs := &GatewayStat{Name: name} + c.mu.Lock() + gs.ID = c.cid + // Note that *client.out[Msgs|Bytes] are not set using atomic, + // unlike the in[Msgs|bytes]. + gs.Sent = DataStats{ + MsgBytes: MsgBytes{ + Msgs: c.outMsgs, + Bytes: c.outBytes, + }, + } + c.mu.Unlock() + // Gather matching inbound connections + gs.Received = DataStats{} + for _, c := range gw.in { + c.mu.Lock() + if c.gw.name == name { + gs.Received.Msgs += atomic.LoadInt64(&c.inMsgs) + gs.Received.Bytes += atomic.LoadInt64(&c.inBytes) + gs.NumInbound++ + } + c.mu.Unlock() + } + m.Stats.Gateways = append(m.Stats.Gateways, gs) + } + gw.RUnlock() + } + // Active Servers + m.Stats.ActiveServers = len(s.sys.servers) + 1 + + // JetStream + if js := s.js.Load(); js != nil { + jStat := &JetStreamVarz{} + s.mu.RUnlock() + js.mu.RLock() + c := js.config + c.StoreDir = _EMPTY_ + jStat.Config = &c + js.mu.RUnlock() + jStat.Stats = js.usageStats() + // Update our own usage since we do not echo so we will not hear ourselves. + ourNode := getHash(s.serverName()) + if v, ok := s.nodeToInfo.Load(ourNode); ok && v != nil { + ni := v.(nodeInfo) + ni.stats = jStat.Stats + ni.cfg = jStat.Config + s.optsMu.RLock() + ni.tags = copyStrings(s.opts.Tags) + s.optsMu.RUnlock() + s.nodeToInfo.Store(ourNode, ni) + } + // Metagroup info. + if mg := js.getMetaGroup(); mg != nil { + if mg.Leader() { + if ci := s.raftNodeToClusterInfo(mg); ci != nil { + jStat.Meta = &MetaClusterInfo{ + Name: ci.Name, + Leader: ci.Leader, + Replicas: ci.Replicas, + Size: mg.ClusterSize(), + QuorumNeeded: mg.QuorumNeeded(), + Rescue: mg.InRescue(), + } + if ci.Leader != _EMPTY_ { + jStat.Meta.Peer = getHash(ci.Leader) + } + } + } else { + // non leader only include a shortened version without peers + leader := s.serverNameForNode(mg.GroupLeader()) + jStat.Meta = &MetaClusterInfo{ + Name: mg.Group(), + Leader: leader, + Size: mg.ClusterSize(), + QuorumNeeded: mg.QuorumNeeded(), + Rescue: mg.InRescue(), + } + if leader != _EMPTY_ { + jStat.Meta.Peer = getHash(leader) + } + } + if jStat.Meta != nil { + if ipq := s.jsAPIRoutedReqs; ipq != nil { + jStat.Meta.PendingRequests = ipq.len() + } + if ipq := s.jsAPIRoutedInfoReqs; ipq != nil { + jStat.Meta.PendingInfos = ipq.len() + } + jStat.Meta.Pending = jStat.Meta.PendingRequests + jStat.Meta.PendingInfos + jStat.Meta.Snapshot = s.metaClusterSnapshotStats(js, mg) + } + } + jStat.Limits = &s.getOpts().JetStreamLimits + m.Stats.JetStream = jStat + s.mu.RLock() + } + // Send message. + s.sendInternalMsg(subj, _EMPTY_, &m.Server, &m) +} + +// Limit updates to the heartbeat interval, max one second by default. +func (s *Server) limitStatsz(subj string) bool { + s.mu.Lock() + defer s.mu.Unlock() + + if s.sys == nil { + return true + } + + // Only limit the normal broadcast subject. + if subj != fmt.Sprintf(serverStatsSubj, s.ID()) { + return false + } + + interval := statszRateLimit + if s.sys.cstatsz < interval { + interval = s.sys.cstatsz + } + if time.Since(s.sys.lastStatsz) < interval { + // Reschedule heartbeat for the next interval. + if s.sys.stmr != nil { + s.sys.stmr.Reset(time.Until(s.sys.lastStatsz.Add(interval))) + } + return true + } + s.sys.lastStatsz = time.Now() + return false +} + +// Send out our statz update. +// This should be wrapChk() to setup common locking. +func (s *Server) heartbeatStatsz() { + if s.sys.stmr != nil { + // Increase after startup to our max. + if s.sys.cstatsz < s.sys.statsz { + s.sys.cstatsz *= 2 + if s.sys.cstatsz > s.sys.statsz { + s.sys.cstatsz = s.sys.statsz + } + } + s.sys.stmr.Reset(s.sys.cstatsz) + } + // Do in separate Go routine. + go s.sendStatszUpdate() +} + +// Reset statsz rate limit for the next broadcast. +// This should be wrapChk() to setup common locking. +func (s *Server) resetLastStatsz() { + s.sys.lastStatsz = time.Time{} +} + +func (s *Server) sendStatszUpdate() { + s.sendStatsz(fmt.Sprintf(serverStatsSubj, s.ID())) +} + +// This should be wrapChk() to setup common locking. +func (s *Server) startStatszTimer() { + // We will start by sending out more of these and trail off to the statsz being the max. + s.sys.cstatsz = 250 * time.Millisecond + // Send out the first one quickly, we will slowly back off. + s.sys.stmr = time.AfterFunc(s.sys.cstatsz, s.wrapChk(s.heartbeatStatsz)) +} + +// Start a ticker that will fire periodically and check for orphaned servers. +// This should be wrapChk() to setup common locking. +func (s *Server) startRemoteServerSweepTimer() { + s.sys.sweeper = time.AfterFunc(s.sys.chkOrph, s.wrapChk(s.checkRemoteServers)) +} + +// Length of our system hash used for server targeted messages. +const sysHashLen = 8 + +// Computes a hash of 8 characters for the name. +func getHash(name string) string { + return getHashSize(name, sysHashLen) +} + +// Computes a hash for the given `name`. The result will be `size` characters long. +func getHashSize(name string, size int) string { + sha := sha256.New() + sha.Write([]byte(name)) + b := sha.Sum(nil) + for i := 0; i < size; i++ { + b[i] = digits[int(b[i]%base)] + } + return string(b[:size]) +} + +// Returns the node name for this server which is a hash of the server name. +func (s *Server) Node() string { + s.mu.RLock() + defer s.mu.RUnlock() + if s.sys != nil { + return s.sys.shash + } + return _EMPTY_ +} + +// This will setup our system wide tracking subs. +// For now we will setup one wildcard subscription to +// monitor all accounts for changes in number of connections. +// We can make this on a per account tracking basis if needed. +// Tradeoff is subscription and interest graph events vs connect and +// disconnect events, etc. +func (s *Server) initEventTracking() { + // Capture sys in case we are shutdown while setting up. + s.mu.RLock() + sys := s.sys + s.mu.RUnlock() + + if sys == nil || sys.client == nil || sys.account == nil { + return + } + // Create a system hash which we use for other servers to target us specifically. + sys.shash = getHash(s.info.Name) + + // This will be for all inbox responses. + subject := fmt.Sprintf(inboxRespSubj, sys.shash, "*") + if _, err := s.sysSubscribe(subject, s.inboxReply); err != nil { + s.Errorf("Error setting up internal tracking: %v", err) + return + } + sys.inboxPre = subject + // This is for remote updates for connection accounting. + subject = fmt.Sprintf(accConnsEventSubjOld, "*") + if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteConnsUpdate)); err != nil { + s.Errorf("Error setting up internal tracking for %s: %v", subject, err) + return + } + // This will be for responses for account info that we send out. + subject = fmt.Sprintf(connsRespSubj, s.info.ID) + if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteConnsUpdate)); err != nil { + s.Errorf("Error setting up internal tracking: %v", err) + return + } + // Listen for broad requests to respond with number of subscriptions for a given subject. + if _, err := s.sysSubscribe(accNumSubsReqSubj, s.noInlineCallback(s.nsubsRequest)); err != nil { + s.Errorf("Error setting up internal tracking: %v", err) + return + } + // Listen for statsz from others. + subject = fmt.Sprintf(serverStatsSubj, "*") + if sub, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteServerUpdate)); err != nil { + s.Errorf("Error setting up internal tracking: %v", err) + return + } else { + // Keep track of this one. + sys.remoteStatsSub = sub + } + + // Listen for all server shutdowns. + subject = fmt.Sprintf(shutdownEventSubj, "*") + if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteServerShutdown)); err != nil { + s.Errorf("Error setting up internal tracking: %v", err) + return + } + // Listen for servers entering lame-duck mode. + // NOTE: This currently is handled in the same way as a server shutdown, but has + // a different subject in case we need to handle differently in future. + subject = fmt.Sprintf(lameDuckEventSubj, "*") + if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteServerShutdown)); err != nil { + s.Errorf("Error setting up internal tracking: %v", err) + return + } + // Listen for account claims updates. + subscribeToUpdate := true + if s.accResolver != nil { + subscribeToUpdate = !s.accResolver.IsTrackingUpdate() + } + if subscribeToUpdate { + for _, sub := range []string{accUpdateEventSubjOld, accUpdateEventSubjNew} { + if _, err := s.sysSubscribe(fmt.Sprintf(sub, "*"), s.noInlineCallback(s.accountClaimUpdate)); err != nil { + s.Errorf("Error setting up internal tracking: %v", err) + return + } + } + } + // Listen for ping messages that will be sent to all servers for statsz. + // This subscription is kept for backwards compatibility. Got replaced by ...PING.STATZ from below + if _, err := s.sysSubscribe(serverStatsPingReqSubj, s.noInlineCallbackStatsz(s.statszReq)); err != nil { + s.Errorf("Error setting up internal tracking: %v", err) + return + } + monSrvc := map[string]sysMsgHandler{ + "IDZ": s.idzReq, + "STATSZ": s.statszReq, + "VARZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &VarzEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Varz(&optz.VarzOptions) }) + }, + "SUBSZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &SubszEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Subsz(&optz.SubszOptions) }) + }, + "CONNZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &ConnzEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Connz(&optz.ConnzOptions) }) + }, + "ROUTEZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &RoutezEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Routez(&optz.RoutezOptions) }) + }, + "GATEWAYZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &GatewayzEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Gatewayz(&optz.GatewayzOptions) }) + }, + "LEAFZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &LeafzEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Leafz(&optz.LeafzOptions) }) + }, + "ACCOUNTZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &AccountzEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Accountz(&optz.AccountzOptions) }) + }, + "JSZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &JszEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Jsz(&optz.JSzOptions) }) + }, + "HEALTHZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &HealthzEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.healthz(&optz.HealthzOptions), nil }) + }, + "PROFILEZ": nil, // Special case, see below + "EXPVARZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &ExpvarzEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.expvarz(optz), nil }) + }, + "IPQUEUESZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &IpqueueszEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Ipqueuesz(&optz.IpqueueszOptions), nil }) + }, + "RAFTZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &RaftzEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Raftz(&optz.RaftzOptions), nil }) + }, + } + profilez := func(_ *subscription, c *client, _ *Account, _, rply string, rmsg []byte) { + hdr, msg := c.msgParts(rmsg) + // Need to copy since we are passing those to the go routine below. + hdr, msg = copyBytes(hdr), copyBytes(msg) + // Execute in its own go routine because CPU profiling, for instance, + // could take several seconds to complete. + go func() { + optz := &ProfilezEventOptions{} + s.zReq(c, rply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { + return s.profilez(&optz.ProfilezOptions), nil + }) + }() + } + for name, req := range monSrvc { + var h msgHandler + switch name { + case "PROFILEZ": + h = profilez + case "STATSZ": + h = s.noInlineCallbackStatsz(req) + default: + h = s.noInlineCallback(req) + } + subject = fmt.Sprintf(serverDirectReqSubj, s.info.ID, name) + if _, err := s.sysSubscribe(subject, h); err != nil { + s.Errorf("Error setting up internal tracking: %v", err) + return + } + subject = fmt.Sprintf(serverPingReqSubj, name) + if _, err := s.sysSubscribe(subject, h); err != nil { + s.Errorf("Error setting up internal tracking: %v", err) + return + } + } + extractAccount := func(subject string) (string, error) { + if tk := strings.Split(subject, tsep); len(tk) != accReqTokens { + return _EMPTY_, fmt.Errorf("subject %q is malformed", subject) + } else { + return tk[accReqAccIndex], nil + } + } + monAccSrvc := map[string]sysMsgHandler{ + "SUBSZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &SubszEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { + if acc, err := extractAccount(subject); err != nil { + return nil, err + } else { + optz.SubszOptions.Subscriptions = true + optz.SubszOptions.Account = acc + return s.Subsz(&optz.SubszOptions) + } + }) + }, + "CONNZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &ConnzEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { + if acc, err := extractAccount(subject); err != nil { + return nil, err + } else { + optz.ConnzOptions.Account = acc + return s.Connz(&optz.ConnzOptions) + } + }) + }, + "LEAFZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &LeafzEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { + if acc, err := extractAccount(subject); err != nil { + return nil, err + } else { + optz.LeafzOptions.Account = acc + return s.Leafz(&optz.LeafzOptions) + } + }) + }, + "JSZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &JszEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { + if acc, err := extractAccount(subject); err != nil { + return nil, err + } else { + optz.Account = acc + return s.JszAccount(&optz.JSzOptions) + } + }) + }, + "INFO": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &AccInfoEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { + if acc, err := extractAccount(subject); err != nil { + return nil, err + } else { + return s.accountInfo(acc) + } + }) + }, + // STATZ is essentially a duplicate of CONNS with an envelope identical to the others. + // For historical reasons CONNS is the odd one out. + // STATZ is also less heavy weight than INFO + "STATZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &AccountStatzEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { + if acc, err := extractAccount(subject); err != nil { + return nil, err + } else if acc == "PING" { // Filter PING subject. Happens for server as well. But wildcards are not used + return nil, errSkipZreq + } else { + optz.Accounts = []string{acc} + if stz, err := s.AccountStatz(&optz.AccountStatzOptions); err != nil { + return nil, err + } else if len(stz.Accounts) == 0 && !optz.IncludeUnused { + return nil, errSkipZreq + } else { + return stz, nil + } + } + }) + }, + "CONNS": s.connsRequest, + } + for name, req := range monAccSrvc { + if _, err := s.sysSubscribe(fmt.Sprintf(accDirectReqSubj, "*", name), s.noInlineCallback(req)); err != nil { + s.Errorf("Error setting up internal tracking: %v", err) + return + } + } + + // User info. Do not propagate interest so that we know the local server to the connection + // is the only one that will answer the requests. + if _, err := s.sysSubscribeInternal(fmt.Sprintf(userDirectReqSubj, "*"), s.userInfoReq); err != nil { + s.Errorf("Error setting up internal tracking: %v", err) + return + } + + // For now only the STATZ subject has an account specific ping equivalent. + if _, err := s.sysSubscribe(fmt.Sprintf(accPingReqSubj, "STATZ"), + s.noInlineCallback(func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + optz := &AccountStatzEventOptions{} + s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { + if stz, err := s.AccountStatz(&optz.AccountStatzOptions); err != nil { + return nil, err + } else if len(stz.Accounts) == 0 && !optz.IncludeUnused { + return nil, errSkipZreq + } else { + return stz, nil + } + }) + })); err != nil { + s.Errorf("Error setting up internal tracking: %v", err) + return + } + + // Listen for updates when leaf nodes connect for a given account. This will + // force any gateway connections to move to `modeInterestOnly` + subject = fmt.Sprintf(leafNodeConnectEventSubj, "*") + if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.leafNodeConnected)); err != nil { + s.Errorf("Error setting up internal tracking: %v", err) + return + } + // For tracking remote latency measurements. + subject = fmt.Sprintf(remoteLatencyEventSubj, sys.shash) + if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteLatencyUpdate)); err != nil { + s.Errorf("Error setting up internal latency tracking: %v", err) + return + } + // This is for simple debugging of number of subscribers that exist in the system. + if _, err := s.sysSubscribeInternal(accSubsSubj, s.noInlineCallback(s.debugSubscribers)); err != nil { + s.Errorf("Error setting up internal debug service for subscribers: %v", err) + return + } + + // Listen for requests to reload the server configuration. + subject = fmt.Sprintf(serverReloadReqSubj, s.info.ID) + if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.reloadConfig)); err != nil { + s.Errorf("Error setting up server reload handler: %v", err) + return + } + + // Client connection kick + subject = fmt.Sprintf(clientKickReqSubj, s.info.ID) + if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.kickClient)); err != nil { + s.Errorf("Error setting up client kick service: %v", err) + return + } + // Client connection LDM + subject = fmt.Sprintf(clientLDMReqSubj, s.info.ID) + if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.ldmClient)); err != nil { + s.Errorf("Error setting up client LDM service: %v", err) + return + } +} + +// UserInfo returns basic information to a user about bound account and user permissions. +// For account information they will need to ping that separately, and this allows security +// controls on each subsystem if desired, e.g. account info, jetstream account info, etc. +type UserInfo struct { + UserID string `json:"user"` + Account string `json:"account"` + AccountName string `json:"account_name,omitempty"` + UserName string `json:"user_name,omitempty"` + Permissions *Permissions `json:"permissions,omitempty"` + Expires time.Duration `json:"expires,omitempty"` +} + +// Process a user info request. +func (s *Server) userInfoReq(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) { + if !s.EventsEnabled() || reply == _EMPTY_ { + return + } + + response := &ServerAPIResponse{Server: &ServerInfo{}} + + ci, _, _, _, err := s.getRequestInfo(c, msg) + if err != nil { + response.Error = &ApiError{Code: http.StatusBadRequest} + s.sendInternalResponse(reply, response) + return + } + + // Look up the requester's account directly from ci.Account rather than + // using the acc returned by getRequestInfo, which may resolve to the + // service account (ci.Service) when the request arrives via a chained + // service import. + var accountName string + if ci.Account != _EMPTY_ { + if reqAcc, _ := s.LookupAccount(ci.Account); reqAcc != nil { + accountName = reqAcc.getNameTag() + } + } + + response.Data = &UserInfo{ + UserID: ci.User, + Account: ci.Account, + AccountName: accountName, + UserName: ci.NameTag, + Permissions: c.publicPermissions(), + Expires: c.claimExpiration(), + } + s.sendInternalResponse(reply, response) +} + +// register existing accounts with any system exports. +func (s *Server) registerSystemImportsForExisting() { + var accounts []*Account + + s.mu.RLock() + if s.sys == nil { + s.mu.RUnlock() + return + } + sacc := s.sys.account + s.accounts.Range(func(k, v any) bool { + a := v.(*Account) + if a != sacc { + accounts = append(accounts, a) + } + return true + }) + s.mu.RUnlock() + + for _, a := range accounts { + s.registerSystemImports(a) + } +} + +// add all exports a system account will need +func (s *Server) addSystemAccountExports(sacc *Account) { + if !s.EventsEnabled() { + return + } + accConnzSubj := fmt.Sprintf(accDirectReqSubj, "*", "CONNZ") + // prioritize not automatically added exports + if !sacc.hasServiceExportMatching(accConnzSubj) { + // pick export type that clamps importing account id into subject + if err := sacc.addServiceExportWithResponseAndAccountPos(accConnzSubj, Streamed, nil, 4); err != nil { + //if err := sacc.AddServiceExportWithResponse(accConnzSubj, Streamed, nil); err != nil { + s.Errorf("Error adding system service export for %q: %v", accConnzSubj, err) + } + } + // prioritize not automatically added exports + accStatzSubj := fmt.Sprintf(accDirectReqSubj, "*", "STATZ") + if !sacc.hasServiceExportMatching(accStatzSubj) { + // pick export type that clamps importing account id into subject + if err := sacc.addServiceExportWithResponseAndAccountPos(accStatzSubj, Streamed, nil, 4); err != nil { + s.Errorf("Error adding system service export for %q: %v", accStatzSubj, err) + } + } + // FIXME(dlc) - Old experiment, Remove? + if !sacc.hasServiceExportMatching(accSubsSubj) { + if err := sacc.AddServiceExport(accSubsSubj, nil); err != nil { + s.Errorf("Error adding system service export for %q: %v", accSubsSubj, err) + } + } + + // User info export. + userInfoSubj := fmt.Sprintf(userDirectReqSubj, "*") + if !sacc.hasServiceExportMatching(userInfoSubj) { + if err := sacc.AddServiceExport(userInfoSubj, nil); err != nil { + s.Errorf("Error adding system service export for %q: %v", userInfoSubj, err) + } + mappedSubj := fmt.Sprintf(userDirectReqSubj, sacc.GetName()) + if err := sacc.AddServiceImport(sacc, userDirectInfoSubj, mappedSubj); err != nil { + s.Errorf("Error setting up system service import %s: %v", mappedSubj, err) + } + // Make sure to share details. + sacc.setServiceImportSharing(sacc, mappedSubj, false, true) + } + + // Register any accounts that existed prior. + s.registerSystemImportsForExisting() + + // in case of a mixed mode setup, enable js exports anyway + if s.JetStreamEnabled() || !s.standAloneMode() { + s.checkJetStreamExports() + } +} + +// accountClaimUpdate will receive claim updates for accounts. +func (s *Server) accountClaimUpdate(sub *subscription, c *client, _ *Account, subject, resp string, hdr, msg []byte) { + if !s.EventsEnabled() { + return + } + var pubKey string + toks := strings.Split(subject, tsep) + if len(toks) == accUpdateTokensNew { + pubKey = toks[accReqAccIndex] + } else if len(toks) == accUpdateTokensOld { + pubKey = toks[accUpdateAccIdxOld] + } else { + s.Debugf("Received account claims update on bad subject %q", subject) + return + } + if len(msg) == 0 { + err := errors.New("request body is empty") + respondToUpdate(s, resp, pubKey, "jwt update error", err) + } else if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil { + respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err) + } else if claim.Subject != pubKey { + err := errors.New("subject does not match jwt content") + respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err) + } else if v, ok := s.accounts.Load(pubKey); !ok { + respondToUpdate(s, resp, pubKey, "jwt update skipped", nil) + } else if err := s.updateAccountWithClaimJWT(v.(*Account), string(msg)); err != nil { + respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err) + } else { + respondToUpdate(s, resp, pubKey, "jwt updated", nil) + } +} + +// processRemoteServerShutdown will update any affected accounts. +// Will update the remote count for clients. +// Lock assume held. +func (s *Server) processRemoteServerShutdown(sid string) { + s.accounts.Range(func(k, v any) bool { + v.(*Account).removeRemoteServer(sid) + return true + }) + // Update any state in nodeInfo. + s.nodeToInfo.Range(func(k, v any) bool { + ni := v.(nodeInfo) + if ni.id == sid { + ni.offline = true + s.nodeToInfo.Store(k, ni) + return false + } + return true + }) + delete(s.sys.servers, sid) +} + +func (s *Server) sameDomain(domain string) bool { + return domain == _EMPTY_ || s.info.Domain == _EMPTY_ || domain == s.info.Domain +} + +// remoteServerShutdown is called when we get an event from another server shutting down. +func (s *Server) remoteServerShutdown(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + s.mu.Lock() + defer s.mu.Unlock() + if !s.eventsEnabled() { + return + } + toks := strings.Split(subject, tsep) + if len(toks) < shutdownEventTokens { + s.Debugf("Received remote server shutdown on bad subject %q", subject) + return + } + + if len(msg) == 0 { + s.Errorf("Remote server sent invalid (empty) shutdown message to %q", subject) + return + } + + // We have an optional serverInfo here, remove from nodeToX lookups. + var si ServerInfo + if err := json.Unmarshal(msg, &si); err != nil { + s.Debugf("Received bad server info for remote server shutdown") + return + } + + // JetStream node updates if applicable. + node := getHash(si.Name) + if v, ok := s.nodeToInfo.Load(node); ok && v != nil { + ni := v.(nodeInfo) + ni.offline = true + s.nodeToInfo.Store(node, ni) + } + + sid := toks[serverSubjectIndex] + if su := s.sys.servers[sid]; su != nil { + s.processRemoteServerShutdown(sid) + } +} + +// remoteServerUpdate listens for statsz updates from other servers. +func (s *Server) remoteServerUpdate(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + var ssm ServerStatsMsg + if len(msg) == 0 { + s.Debugf("Received empty server info for remote server update") + return + } else if err := json.Unmarshal(msg, &ssm); err != nil { + s.Debugf("Received bad server info for remote server update") + return + } + si := ssm.Server + + // Should do normal updates before bailing if wrong domain. + s.mu.Lock() + if s.isRunning() && s.eventsEnabled() && ssm.Server.ID != s.info.ID { + s.updateRemoteServer(&si) + } + s.mu.Unlock() + + // JetStream node updates. + if !s.sameDomain(si.Domain) { + return + } + + var cfg *JetStreamConfig + var stats *JetStreamStats + + if ssm.Stats.JetStream != nil { + cfg = ssm.Stats.JetStream.Config + stats = ssm.Stats.JetStream.Stats + } + + node := getHash(si.Name) + accountNRG := si.AccountNRG() + newInfo := nodeInfo{ + name: si.Name, + version: si.Version, + cluster: si.Cluster, + domain: si.Domain, + id: si.ID, + tags: si.Tags, + cfg: cfg, + stats: stats, + offline: false, + js: si.JetStreamEnabled(), + binarySnapshots: si.BinaryStreamSnapshot(), + accountNRG: accountNRG, + } + oldInfo, _ := s.nodeToInfo.Swap(node, newInfo) + if oldInfo == nil || accountNRG != oldInfo.(nodeInfo).accountNRG { + // One of the servers we received statsz from changed its mind about + // whether or not it supports in-account NRG, so update the groups + // with this information. + s.updateNRGAccountStatus() + } + + // Process a peer's statsz when it has sufficient information. + if newInfo.selectable() { + if js := s.getJetStream(); js != nil { + js.processPeerStatsz(node) + } + } +} + +// updateRemoteServer is called when we have an update from a remote server. +// This allows us to track remote servers, respond to shutdown messages properly, +// make sure that messages are ordered, and allow us to prune dead servers. +// Lock should be held upon entry. +func (s *Server) updateRemoteServer(si *ServerInfo) { + su := s.sys.servers[si.ID] + if su == nil { + s.sys.servers[si.ID] = &serverUpdate{si.Seq, time.Now()} + s.processNewServer(si) + } else { + // Should always be going up. + if si.Seq <= su.seq { + s.Errorf("Received out of order remote server update from: %q", si.ID) + return + } + su.seq = si.Seq + su.ltime = time.Now() + } +} + +// processNewServer will hold any logic we want to use when we discover a new server. +// Lock should be held upon entry. +func (s *Server) processNewServer(si *ServerInfo) { + // Right now we only check if we have leafnode servers and if so send another + // connect update to make sure they switch this account to interest only mode. + s.ensureGWsInterestOnlyForLeafNodes() + + // Add to our nodeToName + if s.sameDomain(si.Domain) { + node := getHash(si.Name) + // Only update if non-existent + if _, ok := s.nodeToInfo.Load(node); !ok { + s.nodeToInfo.Store(node, nodeInfo{ + name: si.Name, + version: si.Version, + cluster: si.Cluster, + domain: si.Domain, + id: si.ID, + tags: si.Tags, + cfg: nil, + stats: nil, + offline: false, + js: si.JetStreamEnabled(), + binarySnapshots: si.BinaryStreamSnapshot(), + accountNRG: si.AccountNRG(), + }) + } + } + go s.updateNRGAccountStatus() + // Announce ourselves.. + // Do this in a separate Go routine. + go s.sendStatszUpdate() +} + +// Works out whether all nodes support moving the NRG traffic into +// the account and moves it appropriately. +// Server lock MUST NOT be held on entry. +func (s *Server) updateNRGAccountStatus() { + s.rnMu.RLock() + raftNodes := make([]RaftNode, 0, len(s.raftNodes)) + for _, n := range s.raftNodes { + raftNodes = append(raftNodes, n) + } + s.rnMu.RUnlock() + for _, n := range raftNodes { + // In the event that the node is happy that all nodes that + // it cares about haven't changed, this will be a no-op. + if err := n.RecreateInternalSubs(); err != nil { + n.Stop() + } + } +} + +// If GW is enabled on this server and there are any leaf node connections, +// this function will send a LeafNode connect system event to the super cluster +// to ensure that the GWs are in interest-only mode for this account. +// Lock should be held upon entry. +// TODO(dlc) - this will cause this account to be loaded on all servers. Need a better +// way with GW2. +func (s *Server) ensureGWsInterestOnlyForLeafNodes() { + if !s.gateway.enabled || len(s.leafs) == 0 { + return + } + sent := make(map[*Account]bool, len(s.leafs)) + for _, c := range s.leafs { + if !sent[c.acc] { + s.sendLeafNodeConnectMsg(c.acc.Name) + sent[c.acc] = true + } + } +} + +// shutdownEventing will clean up all eventing state. +func (s *Server) shutdownEventing() { + if !s.eventsRunning() { + return + } + + s.mu.Lock() + if s.sys == nil || s.sys.resetCh == nil { + s.mu.Unlock() + return + } + clearTimer(&s.sys.sweeper) + clearTimer(&s.sys.stmr) + // We intentionally do not close resetCh. A sender that captured the channel + // may still attempt to send after the internal send loop has exited. + s.sys.resetCh = nil + wg := &s.sys.wg + s.mu.Unlock() + + // We will queue up a shutdown event and wait for the + // internal send loop to exit. + s.sendShutdownEvent() + wg.Wait() + + s.mu.Lock() + defer s.mu.Unlock() + + // Whip through all accounts. + s.accounts.Range(func(k, v any) bool { + v.(*Account).clearEventing() + return true + }) + // Turn everything off here. + s.sys = nil +} + +// Request for our local connection count. +func (s *Server) connsRequest(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + if !s.eventsRunning() { + return + } + tk := strings.Split(subject, tsep) + if len(tk) != accReqTokens { + s.sys.client.Errorf("Bad subject account connections request message") + return + } + a := tk[accReqAccIndex] + m := accNumConnsReq{Account: a} + if len(msg) > 0 { + if err := json.Unmarshal(msg, &m); err != nil { + s.sys.client.Errorf("Error unmarshalling account connections request message: %v", err) + return + } + } + if m.Account != a { + s.sys.client.Errorf("Error unmarshalled account does not match subject") + return + } + // Here we really only want to lookup the account if its local. We do not want to fetch this + // account if we have no interest in it. + var acc *Account + if v, ok := s.accounts.Load(m.Account); ok { + acc = v.(*Account) + } + if acc == nil { + return + } + // We know this is a local connection. + if nlc := acc.NumLocalConnections(); nlc > 0 { + s.mu.Lock() + s.sendAccConnsUpdate(acc, reply) + s.mu.Unlock() + } +} + +// leafNodeConnected is an event we will receive when a leaf node for a given account connects. +func (s *Server) leafNodeConnected(sub *subscription, _ *client, _ *Account, subject, reply string, hdr, msg []byte) { + m := accNumConnsReq{} + if err := json.Unmarshal(msg, &m); err != nil { + s.sys.client.Errorf("Error unmarshalling account connections request message: %v", err) + return + } + + s.mu.RLock() + na := m.Account == _EMPTY_ || !s.eventsEnabled() || !s.gateway.enabled + s.mu.RUnlock() + + if na { + return + } + + if acc, _ := s.lookupAccount(m.Account); acc != nil { + s.switchAccountToInterestMode(acc.Name) + } +} + +// Common filter options for system requests STATSZ VARZ SUBSZ CONNZ ROUTEZ GATEWAYZ LEAFZ +type EventFilterOptions struct { + Name string `json:"server_name,omitempty"` // filter by server name + Cluster string `json:"cluster,omitempty"` // filter by cluster name + Host string `json:"host,omitempty"` // filter by host name + ExactMatch bool `json:"exact_match,omitempty"` // if the above filters should use exact matching or only "contains" + Tags []string `json:"tags,omitempty"` // filter by tags (must match all tags) + Domain string `json:"domain,omitempty"` // filter by JS domain +} + +// StatszEventOptions are options passed to Statsz +type StatszEventOptions struct { + // No actual options yet + EventFilterOptions +} + +// Options for account Info +type AccInfoEventOptions struct { + // No actual options yet + EventFilterOptions +} + +// In the context of system events, ConnzEventOptions are options passed to Connz +type ConnzEventOptions struct { + ConnzOptions + EventFilterOptions +} + +// In the context of system events, RoutezEventOptions are options passed to Routez +type RoutezEventOptions struct { + RoutezOptions + EventFilterOptions +} + +// In the context of system events, SubzEventOptions are options passed to Subz +type SubszEventOptions struct { + SubszOptions + EventFilterOptions +} + +// In the context of system events, VarzEventOptions are options passed to Varz +type VarzEventOptions struct { + VarzOptions + EventFilterOptions +} + +// In the context of system events, GatewayzEventOptions are options passed to Gatewayz +type GatewayzEventOptions struct { + GatewayzOptions + EventFilterOptions +} + +// In the context of system events, LeafzEventOptions are options passed to Leafz +type LeafzEventOptions struct { + LeafzOptions + EventFilterOptions +} + +// In the context of system events, AccountzEventOptions are options passed to Accountz +type AccountzEventOptions struct { + AccountzOptions + EventFilterOptions +} + +// In the context of system events, AccountzEventOptions are options passed to Accountz +type AccountStatzEventOptions struct { + AccountStatzOptions + EventFilterOptions +} + +// In the context of system events, JszEventOptions are options passed to Jsz +type JszEventOptions struct { + JSzOptions + EventFilterOptions +} + +// In the context of system events, HealthzEventOptions are options passed to Healthz +type HealthzEventOptions struct { + HealthzOptions + EventFilterOptions +} + +// In the context of system events, ProfilezEventOptions are options passed to Profilez +type ProfilezEventOptions struct { + ProfilezOptions + EventFilterOptions +} + +// In the context of system events, ExpvarzEventOptions are options passed to Expvarz +type ExpvarzEventOptions struct { + EventFilterOptions +} + +// In the context of system events, IpqueueszEventOptions are options passed to Ipqueuesz +type IpqueueszEventOptions struct { + EventFilterOptions + IpqueueszOptions +} + +// In the context of system events, RaftzEventOptions are options passed to Raftz +type RaftzEventOptions struct { + EventFilterOptions + RaftzOptions +} + +// returns true if the request does NOT apply to this server and can be ignored. +// DO NOT hold the server lock when calling this. +func (s *Server) filterRequest(fOpts *EventFilterOptions) bool { + if fOpts == nil { + return false + } + if fOpts.ExactMatch { + if (fOpts.Name != _EMPTY_ && fOpts.Name != s.info.Name) || + (fOpts.Host != _EMPTY_ && fOpts.Host != s.info.Host) || + (fOpts.Cluster != _EMPTY_ && fOpts.Cluster != s.ClusterName()) { + return true + } + } else if (fOpts.Name != _EMPTY_ && !strings.Contains(s.info.Name, fOpts.Name)) || + (fOpts.Host != _EMPTY_ && !strings.Contains(s.info.Host, fOpts.Host)) || + (fOpts.Cluster != _EMPTY_ && !strings.Contains(s.ClusterName(), fOpts.Cluster)) { + return true + } + if len(fOpts.Tags) > 0 { + opts := s.getOpts() + for _, t := range fOpts.Tags { + if !opts.Tags.Contains(t) { + return true + } + } + } + if fOpts.Domain != _EMPTY_ && s.getOpts().JetStreamDomain != fOpts.Domain { + return true + } + return false +} + +// Encoding support (compression) +type compressionType int8 + +const ( + noCompression = compressionType(iota) + gzipCompression + snappyCompression + unsupportedCompression +) + +// ServerAPIResponse is the response type for the server API like varz, connz etc. +type ServerAPIResponse struct { + Server *ServerInfo `json:"server"` + Data any `json:"data,omitempty"` + Error *ApiError `json:"error,omitempty"` + + // Private to indicate compression if any. + compress compressionType +} + +// Specialized response types for unmarshalling. These structures are not +// used in the server code and only there for users of the Z endpoints to +// unmarshal the data without having to create these structs in their code + +// ServerAPIConnzResponse is the response type connz +type ServerAPIConnzResponse struct { + Server *ServerInfo `json:"server"` + Data *Connz `json:"data,omitempty"` + Error *ApiError `json:"error,omitempty"` +} + +// ServerAPIRoutezResponse is the response type for routez +type ServerAPIRoutezResponse struct { + Server *ServerInfo `json:"server"` + Data *Routez `json:"data,omitempty"` + Error *ApiError `json:"error,omitempty"` +} + +// ServerAPIGatewayzResponse is the response type for gatewayz +type ServerAPIGatewayzResponse struct { + Server *ServerInfo `json:"server"` + Data *Gatewayz `json:"data,omitempty"` + Error *ApiError `json:"error,omitempty"` +} + +// ServerAPIJszResponse is the response type for jsz +type ServerAPIJszResponse struct { + Server *ServerInfo `json:"server"` + Data *JSInfo `json:"data,omitempty"` + Error *ApiError `json:"error,omitempty"` +} + +// ServerAPIHealthzResponse is the response type for healthz +type ServerAPIHealthzResponse struct { + Server *ServerInfo `json:"server"` + Data *HealthStatus `json:"data,omitempty"` + Error *ApiError `json:"error,omitempty"` +} + +// ServerAPIVarzResponse is the response type for varz +type ServerAPIVarzResponse struct { + Server *ServerInfo `json:"server"` + Data *Varz `json:"data,omitempty"` + Error *ApiError `json:"error,omitempty"` +} + +// ServerAPISubszResponse is the response type for subsz +type ServerAPISubszResponse struct { + Server *ServerInfo `json:"server"` + Data *Subsz `json:"data,omitempty"` + Error *ApiError `json:"error,omitempty"` +} + +// ServerAPILeafzResponse is the response type for leafz +type ServerAPILeafzResponse struct { + Server *ServerInfo `json:"server"` + Data *Leafz `json:"data,omitempty"` + Error *ApiError `json:"error,omitempty"` +} + +// ServerAPIAccountzResponse is the response type for accountz +type ServerAPIAccountzResponse struct { + Server *ServerInfo `json:"server"` + Data *Accountz `json:"data,omitempty"` + Error *ApiError `json:"error,omitempty"` +} + +// ServerAPIExpvarzResponse is the response type for expvarz +type ServerAPIExpvarzResponse struct { + Server *ServerInfo `json:"server"` + Data *ExpvarzStatus `json:"data,omitempty"` + Error *ApiError `json:"error,omitempty"` +} + +// ServerAPIpqueueszResponse is the response type for ipqueuesz +type ServerAPIpqueueszResponse struct { + Server *ServerInfo `json:"server"` + Data *IpqueueszStatus `json:"data,omitempty"` + Error *ApiError `json:"error,omitempty"` +} + +// ServerAPIRaftzResponse is the response type for raftz +type ServerAPIRaftzResponse struct { + Server *ServerInfo `json:"server"` + Data *RaftzStatus `json:"data,omitempty"` + Error *ApiError `json:"error,omitempty"` +} + +// statszReq is a request for us to respond with current statsz. +func (s *Server) statszReq(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + if !s.EventsEnabled() { + return + } + + // No reply is a signal that we should use our normal broadcast subject. + if reply == _EMPTY_ { + reply = fmt.Sprintf(serverStatsSubj, s.info.ID) + s.wrapChk(s.resetLastStatsz)() + } + + opts := StatszEventOptions{} + if len(msg) != 0 { + if err := json.Unmarshal(msg, &opts); err != nil { + response := &ServerAPIResponse{ + Server: &ServerInfo{}, + Error: &ApiError{Code: http.StatusBadRequest, Description: err.Error()}, + } + s.sendInternalMsgLocked(reply, _EMPTY_, response.Server, response) + return + } else if ignore := s.filterRequest(&opts.EventFilterOptions); ignore { + return + } + } + s.sendStatsz(reply) +} + +// idzReq is for a request for basic static server info. +// Try to not hold the write lock or dynamically create data. +func (s *Server) idzReq(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + s.mu.RLock() + defer s.mu.RUnlock() + id := &ServerID{ + Name: s.info.Name, + Host: s.info.Host, + ID: s.info.ID, + } + s.sendInternalMsg(reply, _EMPTY_, nil, &id) +} + +var errSkipZreq = errors.New("filtered response") + +const ( + acceptEncodingHeader = "Accept-Encoding" + contentEncodingHeader = "Content-Encoding" +) + +// This is not as formal as it could be. We see if anything has s2 or snappy first, then gzip. +func getAcceptEncoding(hdr []byte) compressionType { + ae := strings.ToLower(string(getHeader(acceptEncodingHeader, hdr))) + if ae == _EMPTY_ { + return noCompression + } + if strings.Contains(ae, "snappy") || strings.Contains(ae, "s2") { + return snappyCompression + } + if strings.Contains(ae, "gzip") { + return gzipCompression + } + return unsupportedCompression +} + +func (s *Server) zReq(_ *client, reply string, hdr, msg []byte, fOpts *EventFilterOptions, optz any, respf func() (any, error)) { + if !s.EventsEnabled() || reply == _EMPTY_ { + return + } + response := &ServerAPIResponse{Server: &ServerInfo{}} + var err error + status := 0 + if len(msg) != 0 { + if err = json.Unmarshal(msg, optz); err != nil { + status = http.StatusBadRequest // status is only included on error, so record how far execution got + } else if s.filterRequest(fOpts) { + return + } + } + if err == nil { + response.Data, err = respf() + if errors.Is(err, errSkipZreq) { + return + } else if err != nil { + status = http.StatusInternalServerError + } + } + if err != nil { + response.Error = &ApiError{Code: status, Description: err.Error()} + } else if len(hdr) > 0 { + response.compress = getAcceptEncoding(hdr) + } + s.sendInternalResponse(reply, response) +} + +// remoteConnsUpdate gets called when we receive a remote update from another server. +func (s *Server) remoteConnsUpdate(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + if !s.eventsRunning() { + return + } + var m AccountNumConns + if len(msg) == 0 { + s.sys.client.Errorf("No message body provided") + return + } else if err := json.Unmarshal(msg, &m); err != nil { + s.sys.client.Errorf("Error unmarshalling account connection event message: %v", err) + return + } + + // See if we have the account registered, if not drop it. + // Make sure this does not force us to load this account here. + var acc *Account + if v, ok := s.accounts.Load(m.Account); ok { + acc = v.(*Account) + } + // Silently ignore these if we do not have local interest in the account. + if acc == nil { + return + } + + s.mu.Lock() + + // check again here if we have been shutdown. + if !s.isRunning() || !s.eventsEnabled() { + s.mu.Unlock() + return + } + // Double check that this is not us, should never happen, so error if it does. + if m.Server.ID == s.info.ID { + s.sys.client.Errorf("Processing our own account connection event message: ignored") + s.mu.Unlock() + return + } + // If we are here we have interest in tracking this account. Update our accounting. + clients := acc.updateRemoteServer(&m) + s.updateRemoteServer(&m.Server) + s.mu.Unlock() + // Need to close clients outside of server lock + for _, c := range clients { + c.maxAccountConnExceeded() + } +} + +// This will import any system level exports. +func (s *Server) registerSystemImports(a *Account) { + if a == nil || !s.EventsEnabled() { + return + } + sacc := s.SystemAccount() + if sacc == nil || sacc == a { + return + } + dstAccName := sacc.Name + // FIXME(dlc) - make a shared list between sys exports etc. + + importSrvc := func(subj, mappedSubj string) { + if !a.serviceImportExists(dstAccName, subj) { + if err := a.addServiceImportWithClaim(sacc, subj, mappedSubj, nil, true); err != nil { + s.Errorf("Error setting up system service import %s -> %s for account: %v", + subj, mappedSubj, err) + } + } + } + // Add in this to the account in 2 places. + // "$SYS.REQ.SERVER.PING.CONNZ" and "$SYS.REQ.ACCOUNT.PING.CONNZ" + mappedConnzSubj := fmt.Sprintf(accDirectReqSubj, a.Name, "CONNZ") + importSrvc(fmt.Sprintf(accPingReqSubj, "CONNZ"), mappedConnzSubj) + importSrvc(fmt.Sprintf(serverPingReqSubj, "CONNZ"), mappedConnzSubj) + importSrvc(fmt.Sprintf(accPingReqSubj, "STATZ"), fmt.Sprintf(accDirectReqSubj, a.Name, "STATZ")) + + // This is for user's looking up their own info. + mappedSubject := fmt.Sprintf(userDirectReqSubj, a.Name) + importSrvc(userDirectInfoSubj, mappedSubject) + // Make sure to share details. + a.setServiceImportSharing(sacc, mappedSubject, false, true) +} + +// Setup tracking for this account. This allows us to track global account activity. +// Lock should be held on entry. +func (s *Server) enableAccountTracking(a *Account) { + if a == nil || !s.eventsEnabled() { + return + } + + // TODO(ik): Generate payload although message may not be sent. + // May need to ensure we do so only if there is a known interest. + // This can get complicated with gateways. + + subj := fmt.Sprintf(accDirectReqSubj, a.Name, "CONNS") + reply := fmt.Sprintf(connsRespSubj, s.info.ID) + m := accNumConnsReq{Account: a.Name} + s.sendInternalMsg(subj, reply, &m.Server, &m) +} + +// Event on leaf node connect. +// Lock should NOT be held on entry. +func (s *Server) sendLeafNodeConnect(a *Account) { + s.mu.Lock() + // If we are not in operator mode, or do not have any gateways defined, this should also be a no-op. + if a == nil || !s.eventsEnabled() || !s.gateway.enabled { + s.mu.Unlock() + return + } + s.sendLeafNodeConnectMsg(a.Name) + s.mu.Unlock() + + s.switchAccountToInterestMode(a.Name) +} + +// Send the leafnode connect message. +// Lock should be held. +func (s *Server) sendLeafNodeConnectMsg(accName string) { + subj := fmt.Sprintf(leafNodeConnectEventSubj, accName) + m := accNumConnsReq{Account: accName} + s.sendInternalMsg(subj, _EMPTY_, &m.Server, &m) +} + +// sendAccConnsUpdate is called to send out our information on the +// account's local connections. +// Lock should be held on entry. +func (s *Server) sendAccConnsUpdate(a *Account, subj ...string) { + if !s.eventsEnabled() || a == nil { + return + } + sendQ := s.sys.sendq + if sendQ == nil { + return + } + // Build event with account name and number of local clients and leafnodes. + eid := s.nextEventID() + a.mu.Lock() + stat := a.statz() + m := AccountNumConns{ + TypedEvent: TypedEvent{ + Type: AccountNumConnsMsgType, + ID: eid, + Time: time.Now().UTC(), + }, + AccountStat: *stat, + } + // Set timer to fire again unless we are at zero. + if m.TotalConns == 0 { + clearTimer(&a.ctmr) + } else { + // Check to see if we have an HB running and update. + if a.ctmr == nil { + a.ctmr = time.AfterFunc(eventsHBInterval, func() { s.accConnsUpdate(a) }) + } else { + a.ctmr.Reset(eventsHBInterval) + } + } + for _, sub := range subj { + msg := newPubMsg(nil, sub, _EMPTY_, &m.Server, nil, &m, noCompression, false, false) + sendQ.push(msg) + } + a.mu.Unlock() +} + +// Lock should be held on entry. +func (a *Account) statz() *AccountStat { + localConns := a.numLocalConnections() + leafConns := a.numLocalLeafNodes() + + a.stats.Lock() + received := DataStats{ + MsgBytes: MsgBytes{ + Msgs: a.stats.inMsgs, + Bytes: a.stats.inBytes, + }, + Gateways: &MsgBytes{ + Msgs: a.stats.gw.inMsgs, + Bytes: a.stats.gw.inBytes, + }, + Routes: &MsgBytes{ + Msgs: a.stats.rt.inMsgs, + Bytes: a.stats.rt.inBytes, + }, + Leafs: &MsgBytes{ + Msgs: a.stats.ln.inMsgs, + Bytes: a.stats.ln.inBytes, + }, + } + sent := DataStats{ + MsgBytes: MsgBytes{ + Msgs: a.stats.outMsgs, + Bytes: a.stats.outBytes, + }, + Gateways: &MsgBytes{ + Msgs: a.stats.gw.outMsgs, + Bytes: a.stats.gw.outBytes, + }, + Routes: &MsgBytes{ + Msgs: a.stats.rt.outMsgs, + Bytes: a.stats.rt.outBytes, + }, + Leafs: &MsgBytes{ + Msgs: a.stats.ln.outMsgs, + Bytes: a.stats.ln.outBytes, + }, + } + slowConsumers := a.stats.slowConsumers + a.stats.Unlock() + + return &AccountStat{ + Account: a.Name, + Name: a.getNameTagLocked(), + Conns: localConns, + LeafNodes: leafConns, + TotalConns: localConns + leafConns, + NumSubs: a.sl.Count(), + Received: received, + Sent: sent, + SlowConsumers: slowConsumers, + } +} + +// accConnsUpdate is called whenever there is a change to the account's +// number of active connections, or during a heartbeat. +// We will not send for $G. +func (s *Server) accConnsUpdate(a *Account) { + s.mu.Lock() + defer s.mu.Unlock() + if !s.eventsEnabled() || a == nil || a == s.gacc { + return + } + s.sendAccConnsUpdate(a, fmt.Sprintf(accConnsEventSubjOld, a.Name), fmt.Sprintf(accConnsEventSubjNew, a.Name)) +} + +func (s *Server) nextEventID() string { + s.eventIdsMu.Lock() + id := s.eventIds.Next() + s.eventIdsMu.Unlock() + return id +} + +// accountConnectEvent will send an account client connect event if there is interest. +// This is a billing event. +func (s *Server) accountConnectEvent(c *client) { + s.mu.RLock() + eventsEnabled := s.eventsEnabled() + s.mu.RUnlock() + if !eventsEnabled { + return + } + eid := s.nextEventID() + + c.mu.Lock() + if c.acc == nil { + c.mu.Unlock() + return + } + + m := ConnectEventMsg{ + TypedEvent: TypedEvent{ + Type: ConnectEventMsgType, + ID: eid, + Time: time.Now().UTC(), + }, + Client: ClientInfo{ + Start: &c.start, + Host: c.host, + ID: c.cid, + Account: accForClient(c), + User: c.getRawAuthUser(), + Name: c.opts.Name, + Lang: c.opts.Lang, + Version: c.opts.Version, + Jwt: c.opts.JWT, + IssuerKey: issuerForClient(c), + Tags: c.tags, + NameTag: c.acc.getNameTag(), + Kind: c.kindString(), + ClientType: c.clientTypeString(), + MQTTClient: c.getMQTTClientID(), + }, + } + subj := fmt.Sprintf(connectEventSubj, c.acc.Name) + c.mu.Unlock() + + s.sendInternalMsgLocked(subj, _EMPTY_, &m.Server, &m) +} + +// accountDisconnectEvent will send an account client disconnect event if there is interest. +// This is a billing event. +func (s *Server) accountDisconnectEvent(c *client, now time.Time, reason string) { + s.mu.RLock() + eventsEnabled := s.eventsEnabled() + s.mu.RUnlock() + if !eventsEnabled { + return + } + eid := s.nextEventID() + + c.mu.Lock() + + if c.acc == nil { + c.mu.Unlock() + return + } + m := DisconnectEventMsg{ + TypedEvent: TypedEvent{ + Type: DisconnectEventMsgType, + ID: eid, + Time: now, + }, + Client: ClientInfo{ + Start: &c.start, + Stop: &now, + Host: c.host, + ID: c.cid, + Account: accForClient(c), + User: c.getRawAuthUser(), + Name: c.opts.Name, + Lang: c.opts.Lang, + Version: c.opts.Version, + RTT: c.getRTT(), + Jwt: c.opts.JWT, + IssuerKey: issuerForClient(c), + Tags: c.tags, + NameTag: c.acc.getNameTag(), + Kind: c.kindString(), + ClientType: c.clientTypeString(), + MQTTClient: c.getMQTTClientID(), + }, + Sent: DataStats{ + MsgBytes: MsgBytes{ + Msgs: atomic.LoadInt64(&c.inMsgs), + Bytes: atomic.LoadInt64(&c.inBytes), + }, + }, + Received: DataStats{ + MsgBytes: MsgBytes{ + Msgs: c.outMsgs, + Bytes: c.outBytes, + }, + }, + Reason: reason, + } + accName := c.acc.Name + c.mu.Unlock() + + subj := fmt.Sprintf(disconnectEventSubj, accName) + s.sendInternalMsgLocked(subj, _EMPTY_, &m.Server, &m) +} + +// This is the system level event sent to the system account for operators. +func (s *Server) sendAuthErrorEvent(c *client, reason string) { + s.mu.RLock() + eventsEnabled := s.eventsEnabled() + s.mu.RUnlock() + if !eventsEnabled { + return + } + eid := s.nextEventID() + + now := time.Now().UTC() + c.mu.Lock() + m := DisconnectEventMsg{ + TypedEvent: TypedEvent{ + Type: DisconnectEventMsgType, + ID: eid, + Time: now, + }, + Client: ClientInfo{ + Start: &c.start, + Stop: &now, + Host: c.host, + ID: c.cid, + Account: accForClient(c), + User: c.getRawAuthUser(), + Name: c.opts.Name, + Lang: c.opts.Lang, + Version: c.opts.Version, + RTT: c.getRTT(), + Jwt: c.opts.JWT, + IssuerKey: issuerForClient(c), + Tags: c.tags, + NameTag: c.acc.getNameTag(), + Kind: c.kindString(), + ClientType: c.clientTypeString(), + MQTTClient: c.getMQTTClientID(), + }, + Sent: DataStats{ + MsgBytes: MsgBytes{ + Msgs: c.inMsgs, + Bytes: c.inBytes, + }, + }, + Received: DataStats{ + MsgBytes: MsgBytes{ + Msgs: c.outMsgs, + Bytes: c.outBytes, + }, + }, + Reason: reason, + } + c.mu.Unlock() + + s.mu.Lock() + subj := fmt.Sprintf(authErrorEventSubj, s.info.ID) + s.sendInternalMsg(subj, _EMPTY_, &m.Server, &m) + s.mu.Unlock() +} + +// This is the account level event sent to the origin account for account owners. +func (s *Server) sendAccountAuthErrorEvent(c *client, acc *Account, reason string) { + if acc == nil { + return + } + s.mu.RLock() + eventsEnabled := s.eventsEnabled() + s.mu.RUnlock() + if !eventsEnabled { + return + } + eid := s.nextEventID() + + now := time.Now().UTC() + c.mu.Lock() + m := DisconnectEventMsg{ + TypedEvent: TypedEvent{ + Type: DisconnectEventMsgType, + ID: eid, + Time: now, + }, + Client: ClientInfo{ + Start: &c.start, + Stop: &now, + Host: c.host, + ID: c.cid, + Account: acc.Name, + User: c.getRawAuthUser(), + Name: c.opts.Name, + Lang: c.opts.Lang, + Version: c.opts.Version, + RTT: c.getRTT(), + Jwt: c.opts.JWT, + IssuerKey: issuerForClient(c), + Tags: c.tags, + NameTag: c.acc.getNameTag(), + Kind: c.kindString(), + ClientType: c.clientTypeString(), + MQTTClient: c.getMQTTClientID(), + }, + Sent: DataStats{ + MsgBytes: MsgBytes{ + Msgs: c.inMsgs, + Bytes: c.inBytes, + }, + }, + Received: DataStats{ + MsgBytes: MsgBytes{ + Msgs: c.outMsgs, + Bytes: c.outBytes, + }, + }, + Reason: reason, + } + c.mu.Unlock() + + s.sendInternalAccountSysMsg(acc, authErrorAccountEventSubj, &m.Server, &m, noCompression) +} + +// Internal message callback. +// If the msg is needed past the callback it is required to be copied. +// rmsg contains header and the message. use client.msgParts(rmsg) to split them apart +type msgHandler func(sub *subscription, client *client, acc *Account, subject, reply string, rmsg []byte) + +const ( + recvQMuxed = 1 + recvQStatsz = 2 +) + +// Create a wrapped callback handler for the subscription that will move it to an +// internal recvQ for processing not inline with routes etc. +func (s *Server) noInlineCallback(cb sysMsgHandler) msgHandler { + return s.noInlineCallbackRecvQSelect(cb, recvQMuxed) +} + +// Create a wrapped callback handler for the subscription that will move it to an +// internal recvQ for Statsz/Pings for processing not inline with routes etc. +func (s *Server) noInlineCallbackStatsz(cb sysMsgHandler) msgHandler { + return s.noInlineCallbackRecvQSelect(cb, recvQStatsz) +} + +// Create a wrapped callback handler for the subscription that will move it to an +// internal IPQueue for processing not inline with routes etc. +func (s *Server) noInlineCallbackRecvQSelect(cb sysMsgHandler, recvQSelect int) msgHandler { + s.mu.RLock() + if !s.eventsEnabled() { + s.mu.RUnlock() + return nil + } + // Capture here for direct reference to avoid any unnecessary blocking inline with routes, gateways etc. + var recvq *ipQueue[*inSysMsg] + switch recvQSelect { + case recvQStatsz: + recvq = s.sys.recvqp + default: + recvq = s.sys.recvq + } + s.mu.RUnlock() + + return func(sub *subscription, c *client, acc *Account, subj, rply string, rmsg []byte) { + // Need to copy and split here. + hdr, msg := c.msgParts(rmsg) + recvq.push(&inSysMsg{sub, c, acc, subj, rply, copyBytes(hdr), copyBytes(msg), cb}) + } +} + +// Create an internal subscription. sysSubscribeQ for queue groups. +func (s *Server) sysSubscribe(subject string, cb msgHandler) (*subscription, error) { + return s.systemSubscribe(subject, _EMPTY_, false, nil, cb) +} + +// Create an internal subscription with queue +func (s *Server) sysSubscribeQ(subject, queue string, cb msgHandler) (*subscription, error) { + return s.systemSubscribe(subject, queue, false, nil, cb) +} + +// Create an internal subscription but do not forward interest. +func (s *Server) sysSubscribeInternal(subject string, cb msgHandler) (*subscription, error) { + return s.systemSubscribe(subject, _EMPTY_, true, nil, cb) +} + +func (s *Server) systemSubscribe(subject, queue string, internalOnly bool, c *client, cb msgHandler) (*subscription, error) { + s.mu.Lock() + if !s.eventsEnabled() { + s.mu.Unlock() + return nil, ErrNoSysAccount + } + if cb == nil { + s.mu.Unlock() + return nil, fmt.Errorf("undefined message handler") + } + if c == nil { + c = s.sys.client + } + trace := c.trace + s.sys.sid++ + sid := strconv.Itoa(s.sys.sid) + s.mu.Unlock() + + // Now create the subscription + if trace { + c.traceInOp("SUB", []byte(subject+" "+queue+" "+sid)) + } + + var q []byte + if queue != _EMPTY_ { + q = []byte(queue) + } + + // Now create the subscription + return c.processSub([]byte(subject), q, []byte(sid), cb, internalOnly) +} + +func (s *Server) sysUnsubscribe(sub *subscription) { + if sub == nil { + return + } + s.mu.RLock() + if !s.eventsEnabled() { + s.mu.RUnlock() + return + } + c := sub.client + s.mu.RUnlock() + + if c != nil { + c.processUnsub(sub.sid) + } +} + +// This will generate the tracking subject for remote latency from the response subject. +func remoteLatencySubjectForResponse(subject []byte) string { + if !isTrackedReply(subject) { + return "" + } + toks := bytes.Split(subject, []byte(tsep)) + // FIXME(dlc) - Sprintf may become a performance concern at some point. + return fmt.Sprintf(remoteLatencyEventSubj, toks[len(toks)-2]) +} + +// remoteLatencyUpdate is used to track remote latency measurements for tracking on exported services. +func (s *Server) remoteLatencyUpdate(sub *subscription, _ *client, _ *Account, subject, _ string, hdr, msg []byte) { + if !s.eventsRunning() { + return + } + var rl remoteLatency + if err := json.Unmarshal(msg, &rl); err != nil { + s.Errorf("Error unmarshalling remote latency measurement: %v", err) + return + } + // Now we need to look up the responseServiceImport associated with this measurement. + acc, err := s.LookupAccount(rl.Account) + if err != nil { + s.Warnf("Could not lookup account %q for latency measurement", rl.Account) + return + } + // Now get the request id / reply. We need to see if we have a GW prefix and if so strip that off. + reply := rl.ReqId + if gwPrefix, old := isGWRoutedSubjectAndIsOldPrefix([]byte(reply)); gwPrefix { + reply = string(getSubjectFromGWRoutedReply([]byte(reply), old)) + } + acc.mu.RLock() + si := acc.exports.responses[reply] + if si == nil || si.latency == nil { + acc.mu.RUnlock() + return + } + lsub := si.latency.subject + acc.mu.RUnlock() + + si.acc.mu.Lock() + m1 := si.m1 + m2 := rl.M2 + + // So we have not processed the response tracking measurement yet. + if m1 == nil { + // Store our value there for them to pick up. + si.m1 = &m2 + } + si.acc.mu.Unlock() + + if m1 == nil { + return + } + + // Calculate the correct latencies given M1 and M2. + m1.merge(&m2) + + // Clear the requesting client since we send the result here. + acc.mu.Lock() + si.rc = nil + acc.mu.Unlock() + + // Send the metrics + s.sendInternalAccountMsg(acc, lsub, m1) +} + +// This is used for all inbox replies so that we do not send supercluster wide interest +// updates for every request. Same trick used in modern NATS clients. +func (s *Server) inboxReply(sub *subscription, c *client, acc *Account, subject, reply string, msg []byte) { + s.mu.RLock() + if !s.eventsEnabled() || s.sys.replies == nil { + s.mu.RUnlock() + return + } + cb, ok := s.sys.replies[subject] + s.mu.RUnlock() + + if ok && cb != nil { + cb(sub, c, acc, subject, reply, msg) + } +} + +// Copied from go client. +// We could use serviceReply here instead to save some code. +// I prefer these semantics for the moment, when tracing you know what this is. +const ( + InboxPrefix = "$SYS._INBOX." + inboxPrefixLen = len(InboxPrefix) + respInboxPrefixLen = inboxPrefixLen + sysHashLen + 1 + replySuffixLen = 8 // Gives us 62^8 +) + +// Creates an internal inbox used for replies that will be processed by the global wc handler. +func (s *Server) newRespInbox() string { + var b [respInboxPrefixLen + replySuffixLen]byte + pres := b[:respInboxPrefixLen] + copy(pres, s.sys.inboxPre) + rn := rand.Int64() + for i, l := respInboxPrefixLen, rn; i < len(b); i++ { + b[i] = digits[l%base] + l /= base + } + return string(b[:]) +} + +// accNumSubsReq is sent when we need to gather remote info on subs. +type accNumSubsReq struct { + Account string `json:"acc"` + Subject string `json:"subject"` + Queue []byte `json:"queue,omitempty"` +} + +// helper function to total information from results to count subs. +func totalSubs(rr *SublistResult, qg []byte) (nsubs int32) { + if rr == nil { + return + } + checkSub := func(sub *subscription) { + // TODO(dlc) - This could be smarter. + if qg != nil && !bytes.Equal(qg, sub.queue) { + return + } + if sub.client.kind == CLIENT || sub.client.isHubLeafNode() { + nsubs++ + } + } + if qg == nil { + for _, sub := range rr.psubs { + checkSub(sub) + } + } + for _, qsub := range rr.qsubs { + for _, sub := range qsub { + checkSub(sub) + } + } + return +} + +// Allows users of large systems to debug active subscribers for a given subject. +// Payload should be the subject of interest. +func (s *Server) debugSubscribers(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + // Even though this is an internal only subscription, meaning interest was not forwarded, we could + // get one here from a GW in optimistic mode. Ignore for now. + // FIXME(dlc) - Should we send no interest here back to the GW? + if c.kind != CLIENT { + return + } + + var ci ClientInfo + if len(hdr) > 0 { + if err := json.Unmarshal(getHeader(ClientInfoHdr, hdr), &ci); err != nil { + return + } + } + + var acc *Account + if ci.Service != _EMPTY_ { + acc, _ = s.LookupAccount(ci.Service) + } else if ci.Account != _EMPTY_ { + acc, _ = s.LookupAccount(ci.Account) + } else { + // Direct $SYS access. + acc = c.acc + if acc == nil { + acc = s.SystemAccount() + } + } + if acc == nil { + return + } + + // We could have a single subject or we could have a subject and a wildcard separated by whitespace. + args := strings.Split(strings.TrimSpace(string(msg)), " ") + if len(args) == 0 { + s.sendInternalAccountMsg(acc, reply, 0) + return + } + + tsubj := args[0] + var qgroup []byte + if len(args) > 1 { + qgroup = []byte(args[1]) + } + + var nsubs int32 + + if subjectIsLiteral(tsubj) { + // We will look up subscribers locally first then determine if we need to solicit other servers. + rr := acc.sl.Match(tsubj) + nsubs = totalSubs(rr, qgroup) + } else { + // We have a wildcard, so this is a bit slower path. + var _subs [32]*subscription + subs := _subs[:0] + acc.sl.All(&subs) + for _, sub := range subs { + if subjectIsSubsetMatch(string(sub.subject), tsubj) { + if qgroup != nil && !bytes.Equal(qgroup, sub.queue) { + continue + } + if sub.client.kind == CLIENT || sub.client.isHubLeafNode() { + nsubs++ + } + } + } + } + + // We should have an idea of how many responses to expect from remote servers. + var expected = acc.expectedRemoteResponses() + + // If we are only local, go ahead and return. + if expected == 0 { + s.sendInternalAccountMsg(nil, reply, nsubs) + return + } + + // We need to solicit from others. + // To track status. + responses := int32(0) + done := make(chan (bool)) + + s.mu.Lock() + // Create direct reply inbox that we multiplex under the WC replies. + replySubj := s.newRespInbox() + // Store our handler. + s.sys.replies[replySubj] = func(sub *subscription, _ *client, _ *Account, subject, _ string, msg []byte) { + if n, err := strconv.ParseInt(string(msg), 10, 32); err == nil { + atomic.AddInt32(&nsubs, int32(n)) + } + if atomic.AddInt32(&responses, 1) >= expected { + select { + case done <- true: + default: + } + } + } + // Send the request to the other servers. + request := &accNumSubsReq{ + Account: acc.Name, + Subject: tsubj, + Queue: qgroup, + } + s.sendInternalMsg(accNumSubsReqSubj, replySubj, nil, request) + s.mu.Unlock() + + // FIXME(dlc) - We should rate limit here instead of blind Go routine. + go func() { + select { + case <-done: + case <-time.After(500 * time.Millisecond): + } + // Cleanup the WC entry. + var sendResponse bool + s.mu.Lock() + if s.sys != nil && s.sys.replies != nil { + delete(s.sys.replies, replySubj) + sendResponse = true + } + s.mu.Unlock() + if sendResponse { + // Send the response. + s.sendInternalAccountMsg(nil, reply, atomic.LoadInt32(&nsubs)) + } + }() +} + +// Request for our local subscription count. This will come from a remote origin server +// that received the initial request. +func (s *Server) nsubsRequest(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + if !s.eventsRunning() { + return + } + m := accNumSubsReq{} + if len(msg) == 0 { + s.sys.client.Errorf("request requires a body") + return + } else if err := json.Unmarshal(msg, &m); err != nil { + s.sys.client.Errorf("Error unmarshalling account nsubs request message: %v", err) + return + } + // Grab account. + acc, _ := s.lookupAccount(m.Account) + if acc == nil || acc.numLocalAndLeafConnections() == 0 { + return + } + // We will look up subscribers locally first then determine if we need to solicit other servers. + var nsubs int32 + if subjectIsLiteral(m.Subject) { + rr := acc.sl.Match(m.Subject) + nsubs = totalSubs(rr, m.Queue) + } else { + // We have a wildcard, so this is a bit slower path. + var _subs [32]*subscription + subs := _subs[:0] + acc.sl.All(&subs) + for _, sub := range subs { + if (sub.client.kind == CLIENT || sub.client.isHubLeafNode()) && subjectIsSubsetMatch(string(sub.subject), m.Subject) { + if m.Queue != nil && !bytes.Equal(m.Queue, sub.queue) { + continue + } + nsubs++ + } + } + } + s.sendInternalMsgLocked(reply, _EMPTY_, nil, nsubs) +} + +func (s *Server) reloadConfig(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + if !s.eventsRunning() { + return + } + + optz := &EventFilterOptions{} + s.zReq(c, reply, hdr, msg, optz, optz, func() (any, error) { + // Reload the server config, as requested. + return nil, s.Reload() + }) +} + +type KickClientReq struct { + CID uint64 `json:"cid"` +} + +type LDMClientReq struct { + CID uint64 `json:"cid"` +} + +func (s *Server) kickClient(_ *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + if !s.eventsRunning() { + return + } + + var req KickClientReq + if err := json.Unmarshal(msg, &req); err != nil { + s.sys.client.Errorf("Error unmarshalling kick client request: %v", err) + return + } + + optz := &EventFilterOptions{} + s.zReq(c, reply, hdr, msg, optz, optz, func() (any, error) { + return nil, s.DisconnectClientByID(req.CID) + }) + +} + +func (s *Server) ldmClient(_ *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { + if !s.eventsRunning() { + return + } + + var req LDMClientReq + if err := json.Unmarshal(msg, &req); err != nil { + s.sys.client.Errorf("Error unmarshalling kick client request: %v", err) + return + } + + optz := &EventFilterOptions{} + s.zReq(c, reply, hdr, msg, optz, optz, func() (any, error) { + return nil, s.LDMClientByID(req.CID) + }) +} + +// Helper to grab account name for a client. +func accForClient(c *client) string { + if c.acc != nil { + return c.acc.Name + } + return "N/A" +} + +// Helper to grab issuer for a client. +func issuerForClient(c *client) (issuerKey string) { + if c == nil || c.user == nil { + return + } + issuerKey = c.user.SigningKey + if issuerKey == _EMPTY_ && c.user.Account != nil { + issuerKey = c.user.Account.Name + } + return +} + +// Helper to clear timers. +func clearTimer(tp **time.Timer) { + if t := *tp; t != nil { + t.Stop() + *tp = nil + } +} + +// Helper function to wrap functions with common test +// to lock server and return if events not enabled. +func (s *Server) wrapChk(f func()) func() { + return func() { + s.mu.Lock() + if !s.eventsEnabled() { + s.mu.Unlock() + return + } + f() + s.mu.Unlock() + } +} + +// sendOCSPPeerRejectEvent sends a system level event to system account when a peer connection is +// rejected due to OCSP invalid status of its trust chain(s). +func (s *Server) sendOCSPPeerRejectEvent(kind string, peer *x509.Certificate, reason string) { + s.mu.RLock() + eventsEnabled := s.eventsEnabled() + serverID := s.info.ID + s.mu.RUnlock() + if !eventsEnabled { + return + } + if peer == nil { + s.Errorf(certidp.ErrPeerEmptyNoEvent) + return + } + eid := s.nextEventID() + now := time.Now().UTC() + m := OCSPPeerRejectEventMsg{ + TypedEvent: TypedEvent{ + Type: OCSPPeerRejectEventMsgType, + ID: eid, + Time: now, + }, + Kind: kind, + Peer: certidp.CertInfo{ + Subject: certidp.GetSubjectDNForm(peer), + Issuer: certidp.GetIssuerDNForm(peer), + Fingerprint: certidp.GenerateFingerprint(peer), + Raw: peer.Raw, + }, + Reason: reason, + } + subj := fmt.Sprintf(ocspPeerRejectEventSubj, serverID) + s.sendInternalMsgLocked(subj, _EMPTY_, &m.Server, &m) +} + +// sendOCSPPeerChainlinkInvalidEvent sends a system level event to system account when a link in a peer's trust chain +// is OCSP invalid. +func (s *Server) sendOCSPPeerChainlinkInvalidEvent(peer *x509.Certificate, link *x509.Certificate, reason string) { + s.mu.RLock() + eventsEnabled := s.eventsEnabled() + serverID := s.info.ID + s.mu.RUnlock() + if !eventsEnabled { + return + } + if peer == nil || link == nil { + s.Errorf(certidp.ErrPeerEmptyNoEvent) + return + } + eid := s.nextEventID() + now := time.Now().UTC() + m := OCSPPeerChainlinkInvalidEventMsg{ + TypedEvent: TypedEvent{ + Type: OCSPPeerChainlinkInvalidEventMsgType, + ID: eid, + Time: now, + }, + Link: certidp.CertInfo{ + Subject: certidp.GetSubjectDNForm(link), + Issuer: certidp.GetIssuerDNForm(link), + Fingerprint: certidp.GenerateFingerprint(link), + Raw: link.Raw, + }, + Peer: certidp.CertInfo{ + Subject: certidp.GetSubjectDNForm(peer), + Issuer: certidp.GetIssuerDNForm(peer), + Fingerprint: certidp.GenerateFingerprint(peer), + Raw: peer.Raw, + }, + Reason: reason, + } + subj := fmt.Sprintf(ocspPeerChainlinkInvalidEventSubj, serverID) + s.sendInternalMsgLocked(subj, _EMPTY_, &m.Server, &m) +} diff --git a/server/events_test.go b/server/events_test.go new file mode 100644 index 0000000..90ba385 --- /dev/null +++ b/server/events_test.go @@ -0,0 +1,4293 @@ +// Copyright 2018-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "math/rand/v2" + "net/http" + "net/http/httptest" + "os" + "reflect" + "runtime" + "runtime/debug" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/jwt/v2" + "github.com/nats-io/nats.go" + "github.com/nats-io/nkeys" + "github.com/nats-io/nuid" +) + +func createAccount(s *Server) (*Account, nkeys.KeyPair) { + okp, _ := nkeys.FromSeed(oSeed) + akp, _ := nkeys.CreateAccount() + pub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(pub) + jwt, _ := nac.Encode(okp) + addAccountToMemResolver(s, pub, jwt) + acc, err := s.LookupAccount(pub) + if err != nil { + panic(err) + } + return acc, akp +} + +func createUserCredsEx(t *testing.T, nuc *jwt.UserClaims, akp nkeys.KeyPair) nats.Option { + t.Helper() + kp, _ := nkeys.CreateUser() + nuc.Subject, _ = kp.PublicKey() + ujwt, err := nuc.Encode(akp) + if err != nil { + t.Fatalf("Error generating user JWT: %v", err) + } + userCB := func() (string, error) { + return ujwt, nil + } + sigCB := func(nonce []byte) ([]byte, error) { + sig, _ := kp.Sign(nonce) + return sig, nil + } + return nats.UserJWT(userCB, sigCB) +} + +func createUserCreds(t *testing.T, _ *Server, akp nkeys.KeyPair) nats.Option { + return createUserCredsEx(t, jwt.NewUserClaims("test"), akp) +} + +func runTrustedServer(t *testing.T) (*Server, *Options) { + t.Helper() + opts := DefaultOptions() + kp, _ := nkeys.FromSeed(oSeed) + pub, _ := kp.PublicKey() + opts.TrustedKeys = []string{pub} + opts.AccountResolver = &MemAccResolver{} + s := RunServer(opts) + return s, opts +} + +func runTrustedCluster(t *testing.T) (*Server, *Options, *Server, *Options, nkeys.KeyPair) { + t.Helper() + + kp, _ := nkeys.FromSeed(oSeed) + pub, _ := kp.PublicKey() + + mr := &MemAccResolver{} + + // Now create a system account. + // NOTE: This can NOT be shared directly between servers. + // Set via server options. + okp, _ := nkeys.FromSeed(oSeed) + akp, _ := nkeys.CreateAccount() + apub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(apub) + jwt, _ := nac.Encode(okp) + + mr.Store(apub, jwt) + + optsA := DefaultOptions() + optsA.Cluster.Name = "TEST_CLUSTER_22" + optsA.Cluster.Host = "127.0.0.1" + optsA.TrustedKeys = []string{pub} + optsA.AccountResolver = mr + optsA.SystemAccount = apub + optsA.ServerName = "A_SRV" + // Add in dummy gateway + optsA.Gateway.Name = "TEST_CLUSTER_22" + optsA.Gateway.Host = "127.0.0.1" + optsA.Gateway.Port = -1 + optsA.gatewaysSolicitDelay = 30 * time.Second + + sa := RunServer(optsA) + + optsB := nextServerOpts(optsA) + optsB.ServerName = "B_SRV" + optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", optsA.Cluster.Host, optsA.Cluster.Port)) + sb := RunServer(optsB) + + checkClusterFormed(t, sa, sb) + + return sa, optsA, sb, optsB, akp +} + +func runTrustedGateways(t *testing.T) (*Server, *Options, *Server, *Options, nkeys.KeyPair) { + t.Helper() + + kp, _ := nkeys.FromSeed(oSeed) + pub, _ := kp.PublicKey() + + mr := &MemAccResolver{} + + // Now create a system account. + // NOTE: This can NOT be shared directly between servers. + // Set via server options. + okp, _ := nkeys.FromSeed(oSeed) + akp, _ := nkeys.CreateAccount() + apub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(apub) + jwt, _ := nac.Encode(okp) + + mr.Store(apub, jwt) + + optsA := testDefaultOptionsForGateway("A") + optsA.Cluster.Name = "A" + optsA.Cluster.Host = "127.0.0.1" + optsA.TrustedKeys = []string{pub} + optsA.AccountResolver = mr + optsA.SystemAccount = apub + + sa := RunServer(optsA) + + optsB := testGatewayOptionsFromToWithServers(t, "B", "A", sa) + optsB.Cluster.Name = "B" + optsB.TrustedKeys = []string{pub} + optsB.AccountResolver = mr + optsB.SystemAccount = apub + + sb := RunServer(optsB) + + waitForInboundGateways(t, sa, 1, time.Second) + waitForOutboundGateways(t, sa, 1, time.Second) + waitForInboundGateways(t, sb, 1, time.Second) + waitForOutboundGateways(t, sb, 1, time.Second) + + return sa, optsA, sb, optsB, akp +} + +func TestSystemAccount(t *testing.T) { + s, _ := runTrustedServer(t) + defer s.Shutdown() + + acc, _ := createAccount(s) + s.setSystemAccount(acc) + + s.mu.Lock() + defer s.mu.Unlock() + + if s.sys == nil || s.sys.account == nil { + t.Fatalf("Expected sys.account to be non-nil") + } + if s.sys.client == nil { + t.Fatalf("Expected sys.client to be non-nil") + } + + s.sys.client.mu.Lock() + defer s.sys.client.mu.Unlock() + if s.sys.client.echo { + t.Fatalf("Internal clients should always have echo false") + } +} + +func TestSystemAccountNewConnection(t *testing.T) { + s, opts := runTrustedServer(t) + defer s.Shutdown() + + acc, akp := createAccount(s) + s.setSystemAccount(acc) + + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + ncs, err := nats.Connect(url, createUserCreds(t, s, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncs.Close() + + // We may not be able to hear ourselves (if the event is processed + // before we create the sub), so we need to create a second client to + // trigger the connect/disconnect events. + acc2, akp2 := createAccount(s) + + // Be explicit to only receive the event for acc2. + sub, _ := ncs.SubscribeSync(fmt.Sprintf("$SYS.ACCOUNT.%s.>", acc2.Name)) + defer sub.Unsubscribe() + ncs.Flush() + + nc, err := nats.Connect(url, createUserCreds(t, s, akp2), nats.Name("TEST EVENTS")) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + msg, err := sub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Error receiving msg: %v", err) + } + connsMsg, err := sub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Error receiving msg: %v", err) + } + if strings.HasPrefix(msg.Subject, fmt.Sprintf("$SYS.ACCOUNT.%s.SERVER.CONNS", acc2.Name)) { + msg, connsMsg = connsMsg, msg + } + if !strings.HasPrefix(connsMsg.Subject, fmt.Sprintf("$SYS.ACCOUNT.%s.SERVER.CONNS", acc2.Name)) { + t.Fatalf("Expected subject to start with %q, got %q", "$SYS.ACCOUNT..CONNECT", msg.Subject) + } + conns := AccountNumConns{} + if err := json.Unmarshal(connsMsg.Data, &conns); err != nil { + t.Fatalf("Error unmarshalling conns event message: %v", err) + } else if conns.Account != acc2.Name { + t.Fatalf("Wrong account in conns message: %v", conns) + } else if conns.Conns != 1 || conns.TotalConns != 1 || conns.LeafNodes != 0 { + t.Fatalf("Wrong counts in conns message: %v", conns) + } + if !strings.HasPrefix(msg.Subject, fmt.Sprintf("$SYS.ACCOUNT.%s.CONNECT", acc2.Name)) { + t.Fatalf("Expected subject to start with %q, got %q", "$SYS.ACCOUNT..CONNECT", msg.Subject) + } + tokens := strings.Split(msg.Subject, ".") + if len(tokens) < 4 { + t.Fatalf("Expected 4 tokens, got %d", len(tokens)) + } + account := tokens[2] + if account != acc2.Name { + t.Fatalf("Expected %q for account, got %q", acc2.Name, account) + } + + cem := ConnectEventMsg{} + if err := json.Unmarshal(msg.Data, &cem); err != nil { + t.Fatalf("Error unmarshalling connect event message: %v", err) + } + if cem.Type != ConnectEventMsgType { + t.Fatalf("Incorrect schema in connect event: %s", cem.Type) + } + if cem.Time.IsZero() { + t.Fatalf("Event time is not set") + } + if len(cem.ID) != 22 { + t.Fatalf("Event ID is incorrectly set to len %d", len(cem.ID)) + } + if cem.Server.ID != s.ID() { + t.Fatalf("Expected server to be %q, got %q", s.ID(), cem.Server.ID) + } + if cem.Server.Seq == 0 { + t.Fatalf("Expected sequence to be non-zero") + } + if cem.Client.Name != "TEST EVENTS" { + t.Fatalf("Expected client name to be %q, got %q", "TEST EVENTS", cem.Client.Name) + } + if cem.Client.Lang != "go" { + t.Fatalf("Expected client lang to be \"go\", got %q", cem.Client.Lang) + } + + // Now close the other client. Should fire a disconnect event. + // First send and receive some messages. + sub2, _ := nc.SubscribeSync("foo") + defer sub2.Unsubscribe() + sub3, _ := nc.SubscribeSync("*") + defer sub3.Unsubscribe() + + for i := 0; i < 10; i++ { + nc.Publish("foo", []byte("HELLO WORLD")) + } + nc.Flush() + nc.Close() + + msg, err = sub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Error receiving msg: %v", err) + } + connsMsg, err = sub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Error receiving msg: %v", err) + } + if strings.HasPrefix(msg.Subject, fmt.Sprintf("$SYS.ACCOUNT.%s.SERVER.CONNS", acc2.Name)) { + msg, connsMsg = connsMsg, msg + } + if !strings.HasPrefix(connsMsg.Subject, fmt.Sprintf("$SYS.ACCOUNT.%s.SERVER.CONNS", acc2.Name)) { + t.Fatalf("Expected subject to start with %q, got %q", "$SYS.ACCOUNT..CONNECT", msg.Subject) + } else if !strings.Contains(string(connsMsg.Data), `"total_conns":0`) { + t.Fatalf("Expected event to reflect created connection, got: %s", string(connsMsg.Data)) + } + conns = AccountNumConns{} + if err := json.Unmarshal(connsMsg.Data, &conns); err != nil { + t.Fatalf("Error unmarshalling conns event message: %v", err) + } else if conns.Account != acc2.Name { + t.Fatalf("Wrong account in conns message: %v", conns) + } else if conns.Conns != 0 || conns.TotalConns != 0 || conns.LeafNodes != 0 { + t.Fatalf("Wrong counts in conns message: %v", conns) + } + if !strings.HasPrefix(msg.Subject, fmt.Sprintf("$SYS.ACCOUNT.%s.DISCONNECT", acc2.Name)) { + t.Fatalf("Expected subject to start with %q, got %q", "$SYS.ACCOUNT..DISCONNECT", msg.Subject) + } + tokens = strings.Split(msg.Subject, ".") + if len(tokens) < 4 { + t.Fatalf("Expected 4 tokens, got %d", len(tokens)) + } + account = tokens[2] + if account != acc2.Name { + t.Fatalf("Expected %q for account, got %q", acc2.Name, account) + } + + dem := DisconnectEventMsg{} + if err := json.Unmarshal(msg.Data, &dem); err != nil { + t.Fatalf("Error unmarshalling disconnect event message: %v", err) + } + if dem.Type != DisconnectEventMsgType { + t.Fatalf("Incorrect schema in connect event: %s", cem.Type) + } + if dem.Time.IsZero() { + t.Fatalf("Event time is not set") + } + if len(dem.ID) != 22 { + t.Fatalf("Event ID is incorrectly set to len %d", len(cem.ID)) + } + if dem.Server.ID != s.ID() { + t.Fatalf("Expected server to be %q, got %q", s.ID(), dem.Server.ID) + } + if dem.Server.Seq == 0 { + t.Fatalf("Expected sequence to be non-zero") + } + if dem.Server.Seq <= cem.Server.Seq { + t.Fatalf("Expected sequence to be increasing") + } + + if cem.Client.Name != "TEST EVENTS" { + t.Fatalf("Expected client name to be %q, got %q", "TEST EVENTS", dem.Client.Name) + } + if dem.Client.Lang != "go" { + t.Fatalf("Expected client lang to be \"go\", got %q", dem.Client.Lang) + } + + if dem.Sent.Msgs != 10 { + t.Fatalf("Expected 10 msgs sent, got %d", dem.Sent.Msgs) + } + if dem.Sent.Bytes != 110 { + t.Fatalf("Expected 110 bytes sent, got %d", dem.Sent.Bytes) + } + if dem.Received.Msgs != 20 { + t.Fatalf("Expected 20 msgs received, got %d", dem.Sent.Msgs) + } + if dem.Received.Bytes != 220 { + t.Fatalf("Expected 220 bytes sent, got %d", dem.Sent.Bytes) + } +} + +func runTrustedLeafServer(t *testing.T) (*Server, *Options) { + t.Helper() + opts := DefaultOptions() + kp, _ := nkeys.FromSeed(oSeed) + pub, _ := kp.PublicKey() + opts.TrustedKeys = []string{pub} + opts.AccountResolver = &MemAccResolver{} + opts.LeafNode.Port = -1 + s := RunServer(opts) + return s, opts +} + +func genCredsFile(t *testing.T, jwt string, seed []byte) string { + creds := ` + -----BEGIN NATS USER JWT----- + %s + ------END NATS USER JWT------ + + ************************* IMPORTANT ************************* + NKEY Seed printed below can be used to sign and prove identity. + NKEYs are sensitive and should be treated as secrets. + + -----BEGIN USER NKEY SEED----- + %s + ------END USER NKEY SEED------ + + ************************************************************* + ` + return createConfFile(t, []byte(strings.Replace(fmt.Sprintf(creds, jwt, seed), "\t\t", "", -1))) +} + +func runSolicitWithCredentials(t *testing.T, opts *Options, creds string) (*Server, *Options, string) { + content := ` + port: -1 + leafnodes { + remotes = [ + { + url: nats-leaf://127.0.0.1:%d + credentials: '%s' + } + ] + } + ` + config := fmt.Sprintf(content, opts.LeafNode.Port, creds) + conf := createConfFile(t, []byte(config)) + s, opts := RunServerWithConfig(conf) + return s, opts, conf +} + +// Helper function to check that a leaf node has connected to our server. +func checkLeafNodeConnected(t testing.TB, s *Server) { + t.Helper() + checkLeafNodeConnectedCount(t, s, 1) +} + +// Helper function to check that a leaf node has connected to n server. +func checkLeafNodeConnectedCount(t testing.TB, s *Server, lnCons int) { + t.Helper() + checkFor(t, 5*time.Second, 15*time.Millisecond, func() error { + if nln := s.NumLeafNodes(); nln != lnCons { + return fmt.Errorf("Expected %d connected leafnode(s) for server %v, got %d", + lnCons, s, nln) + } + return nil + }) +} + +func TestSystemAccountingWithLeafNodes(t *testing.T) { + s, opts := runTrustedLeafServer(t) + defer s.Shutdown() + + acc, akp := createAccount(s) + s.setSystemAccount(acc) + + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + ncs, err := nats.Connect(url, createUserCreds(t, s, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncs.Close() + + acc2, akp2 := createAccount(s) + + // Be explicit to only receive the event for acc2 account. + sub, _ := ncs.SubscribeSync(fmt.Sprintf("$SYS.ACCOUNT.%s.DISCONNECT", acc2.Name)) + defer sub.Unsubscribe() + ncs.Flush() + + kp, _ := nkeys.CreateUser() + pub, _ := kp.PublicKey() + nuc := jwt.NewUserClaims(pub) + ujwt, err := nuc.Encode(akp2) + if err != nil { + t.Fatalf("Error generating user JWT: %v", err) + } + seed, _ := kp.Seed() + mycreds := genCredsFile(t, ujwt, seed) + + // Create a server that solicits a leafnode connection. + sl, slopts, _ := runSolicitWithCredentials(t, opts, mycreds) + defer sl.Shutdown() + + checkLeafNodeConnected(t, s) + + // Compute the expected number of subs on "sl" based on number + // of existing subs before creating the sub on "s". + expected := int(sl.NumSubscriptions() + 1) + + nc, err := nats.Connect(url, createUserCreds(t, s, akp2), nats.Name("TEST LEAFNODE EVENTS")) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + fooSub := natsSubSync(t, nc, "foo") + natsFlush(t, nc) + + checkExpectedSubs(t, expected, sl) + + surl := fmt.Sprintf("nats://%s:%d", slopts.Host, slopts.Port) + nc2, err := nats.Connect(surl, nats.Name("TEST LEAFNODE EVENTS")) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + // Compute the expected number of subs on "s" based on number + // of existing subs before creating the sub on "sl". + expected = int(s.NumSubscriptions() + 1) + + m := []byte("HELLO WORLD") + + // Now generate some traffic + starSub := natsSubSync(t, nc2, "*") + for i := 0; i < 10; i++ { + nc2.Publish("foo", m) + nc2.Publish("bar", m) + } + natsFlush(t, nc2) + + checkExpectedSubs(t, expected, s) + + // Now send some from the cluster side too. + for i := 0; i < 10; i++ { + nc.Publish("foo", m) + nc.Publish("bar", m) + } + nc.Flush() + + // Make sure all messages are received + for i := 0; i < 20; i++ { + if _, err := fooSub.NextMsg(time.Second); err != nil { + t.Fatalf("Did not get message: %v", err) + } + } + for i := 0; i < 40; i++ { + if _, err := starSub.NextMsg(time.Second); err != nil { + t.Fatalf("Did not get message: %v", err) + } + } + + // Now shutdown the leafnode server since this is where the event tracking should + // happen. Right now we do not track local clients to the leafnode server that + // solicited to the cluster, but we should track usage once the leafnode connection stops. + sl.Shutdown() + + // Make sure we get disconnect event and that tracking is correct. + msg, err := sub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Error receiving msg: %v", err) + } + + dem := DisconnectEventMsg{} + if err := json.Unmarshal(msg.Data, &dem); err != nil { + t.Fatalf("Error unmarshalling disconnect event message: %v", err) + } + if dem.Sent.Msgs != 10 { + t.Fatalf("Expected 10 msgs sent, got %d", dem.Sent.Msgs) + } + if dem.Sent.Bytes != 110 { + t.Fatalf("Expected 110 bytes sent, got %d", dem.Sent.Bytes) + } + if dem.Received.Msgs != 20 { + t.Fatalf("Expected 20 msgs received, got %d", dem.Received.Msgs) + } + if dem.Received.Bytes != 220 { + t.Fatalf("Expected 220 bytes sent, got %d", dem.Received.Bytes) + } +} + +func TestSystemAccountDisconnectBadLogin(t *testing.T) { + s, opts := runTrustedServer(t) + defer s.Shutdown() + + acc, akp := createAccount(s) + s.setSystemAccount(acc) + + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + ncs, err := nats.Connect(url, createUserCreds(t, s, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncs.Close() + + // Listen for auth error events though. + asub, _ := ncs.SubscribeSync("$SYS.SERVER.*.CLIENT.AUTH.ERR") + defer asub.Unsubscribe() + + ncs.Flush() + + nats.Connect(url, nats.Name("TEST BAD LOGIN")) + + m, err := asub.NextMsg(100 * time.Millisecond) + if err != nil { + t.Fatalf("Should have heard an auth error event") + } + dem := DisconnectEventMsg{} + if err := json.Unmarshal(m.Data, &dem); err != nil { + t.Fatalf("Error unmarshalling disconnect event message: %v", err) + } + if dem.Reason != "Authentication Failure" { + t.Fatalf("Expected auth error, got %q", dem.Reason) + } +} + +func TestSysSubscribeRace(t *testing.T) { + s, opts := runTrustedServer(t) + defer s.Shutdown() + + acc, akp := createAccount(s) + s.setSystemAccount(acc) + + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + + nc, err := nats.Connect(url, createUserCreds(t, s, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + done := make(chan struct{}) + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + for { + nc.Publish("foo", []byte("hello")) + select { + case <-done: + return + default: + } + } + }() + + time.Sleep(10 * time.Millisecond) + + received := make(chan struct{}) + // Create message callback handler. + cb := func(sub *subscription, producer *client, _ *Account, subject, reply string, msg []byte) { + select { + case received <- struct{}{}: + default: + } + } + // Now create an internal subscription + sub, err := s.sysSubscribe("foo", cb) + if sub == nil || err != nil { + t.Fatalf("Expected to subscribe, got %v", err) + } + select { + case <-received: + close(done) + case <-time.After(time.Second): + t.Fatalf("Did not receive the message") + } + wg.Wait() +} + +func TestSystemAccountInternalSubscriptions(t *testing.T) { + s, opts := runTrustedServer(t) + defer s.Shutdown() + + sub, err := s.sysSubscribe("foo", nil) + if sub != nil || err != ErrNoSysAccount { + t.Fatalf("Expected to get proper error, got %v", err) + } + + acc, akp := createAccount(s) + s.setSystemAccount(acc) + + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + + nc, err := nats.Connect(url, createUserCreds(t, s, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + sub, err = s.sysSubscribe("foo", nil) + if sub != nil || err == nil { + t.Fatalf("Expected to get error for no handler, got %v", err) + } + + received := make(chan *nats.Msg) + // Create message callback handler. + cb := func(sub *subscription, _ *client, _ *Account, subject, reply string, msg []byte) { + copy := append([]byte(nil), msg...) + received <- &nats.Msg{Subject: subject, Reply: reply, Data: copy} + } + + // Now create an internal subscription + sub, err = s.sysSubscribe("foo", cb) + if sub == nil || err != nil { + t.Fatalf("Expected to subscribe, got %v", err) + } + // Now send out a message from our normal client. + nc.Publish("foo", []byte("HELLO WORLD")) + + var msg *nats.Msg + + select { + case msg = <-received: + if msg.Subject != "foo" { + t.Fatalf("Expected \"foo\" as subject, got %q", msg.Subject) + } + if msg.Reply != "" { + t.Fatalf("Expected no reply, got %q", msg.Reply) + } + if !bytes.Equal(msg.Data, []byte("HELLO WORLD")) { + t.Fatalf("Got the wrong msg payload: %q", msg.Data) + } + break + case <-time.After(time.Second): + t.Fatalf("Did not receive the message") + } + s.sysUnsubscribe(sub) + + // Now send out a message from our normal client. + // We should not see this one. + nc.Publish("foo", []byte("You There?")) + + select { + case <-received: + t.Fatalf("Received a message when we should not have") + case <-time.After(100 * time.Millisecond): + break + } + + // Now make sure we do not hear ourselves. We optimize this for internally + // generated messages. + s.mu.Lock() + s.sendInternalMsg("foo", "", nil, msg.Data) + s.mu.Unlock() + + select { + case <-received: + t.Fatalf("Received a message when we should not have") + case <-time.After(100 * time.Millisecond): + break + } +} + +func TestSystemAccountConnectionUpdatesStopAfterNoLocal(t *testing.T) { + sa, _, sb, optsB, _ := runTrustedCluster(t) + defer sa.Shutdown() + defer sb.Shutdown() + + // Normal Account + okp, _ := nkeys.FromSeed(oSeed) + akp, _ := nkeys.CreateAccount() + pub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(pub) + nac.Limits.Conn = 4 // Limit to 4 connections. + jwt, _ := nac.Encode(okp) + + addAccountToMemResolver(sa, pub, jwt) + + // Listen for updates to the new account connection activity. + received := make(chan *nats.Msg, 10) + cb := func(sub *subscription, _ *client, _ *Account, subject, reply string, msg []byte) { + copy := append([]byte(nil), msg...) + received <- &nats.Msg{Subject: subject, Reply: reply, Data: copy} + } + subj := fmt.Sprintf(accConnsEventSubjNew, pub) + sub, err := sa.sysSubscribe(subj, cb) + if sub == nil || err != nil { + t.Fatalf("Expected to subscribe, got %v", err) + } + defer sa.sysUnsubscribe(sub) + + // Create a few users on the new account. + clients := []*nats.Conn{} + + url := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) + for i := 0; i < 4; i++ { + nc, err := nats.Connect(url, createUserCreds(t, sb, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + clients = append(clients, nc) + } + + // Wait for all 4 notifications. + checkFor(t, time.Second, 50*time.Millisecond, func() error { + if len(received) == 4 { + return nil + } + return fmt.Errorf("Not enough messages, %d vs 4", len(received)) + }) + + // Now lookup the account doing the events on sb. + acc, _ := sb.LookupAccount(pub) + // Make sure we have the timer running. + acc.mu.RLock() + ctmr := acc.ctmr + acc.mu.RUnlock() + if ctmr == nil { + t.Fatalf("Expected event timer for acc conns to be running") + } + + // Now close all of the connections. + for _, nc := range clients { + nc.Close() + } + + // Wait for the 4 new notifications, 8 total (4 for connect, 4 for disconnect) + checkFor(t, time.Second, 50*time.Millisecond, func() error { + if len(received) == 8 { + return nil + } + return fmt.Errorf("Not enough messages, %d vs 4", len(received)) + }) + // Drain the messages. + for i := 0; i < 7; i++ { + <-received + } + // Check last one. + msg := <-received + m := AccountNumConns{} + if err := json.Unmarshal(msg.Data, &m); err != nil { + t.Fatalf("Error unmarshalling account connections request message: %v", err) + } + if m.Conns != 0 { + t.Fatalf("Expected Conns to be 0, got %d", m.Conns) + } + + // Should not receive any more messages.. + select { + case <-received: + t.Fatalf("Did not expect a message here") + case <-time.After(50 * time.Millisecond): + break + } + + // Make sure we have the timer is NOT running. + acc.mu.RLock() + ctmr = acc.ctmr + acc.mu.RUnlock() + if ctmr != nil { + t.Fatalf("Expected event timer for acc conns to NOT be running after reaching zero local clients") + } +} + +func TestSystemAccountConnectionLimits(t *testing.T) { + sa, optsA, sb, optsB, _ := runTrustedCluster(t) + defer sa.Shutdown() + defer sb.Shutdown() + + // We want to test that we are limited to a certain number of active connections + // across multiple servers. + + // Let's create a user account. + okp, _ := nkeys.FromSeed(oSeed) + akp, _ := nkeys.CreateAccount() + pub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(pub) + nac.Limits.Conn = 4 // Limit to 4 connections. + jwt, _ := nac.Encode(okp) + + addAccountToMemResolver(sa, pub, jwt) + + urlA := fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port) + urlB := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) + + // Create a user on each server. Break on first failure. + for { + nca1, err := nats.Connect(urlA, createUserCreds(t, sa, akp)) + if err != nil { + break + } + defer nca1.Close() + ncb1, err := nats.Connect(urlB, createUserCreds(t, sb, akp)) + if err != nil { + break + } + defer ncb1.Close() + } + + checkFor(t, 5*time.Second, 50*time.Millisecond, func() error { + total := sa.NumClients() + sb.NumClients() + if total > int(nac.Limits.Conn) { + return fmt.Errorf("Expected only %d connections, was allowed to connect %d", nac.Limits.Conn, total) + } + return nil + }) +} + +func TestBadAccountUpdate(t *testing.T) { + sa, _ := runTrustedServer(t) + defer sa.Shutdown() + akp1, _ := nkeys.CreateAccount() + pub, _ := akp1.PublicKey() + nac := jwt.NewAccountClaims(pub) + ajwt1, err := nac.Encode(oKp) + require_NoError(t, err) + addAccountToMemResolver(sa, pub, ajwt1) + akp2, _ := nkeys.CreateAccount() + pub2, _ := akp2.PublicKey() + nac.Subject = pub2 // maliciously use a different subject but pretend to remain pub + ajwt2, err := nac.Encode(oKp) + require_NoError(t, err) + acc, err := sa.fetchAccount(pub) + require_NoError(t, err) + if err := sa.updateAccountWithClaimJWT(acc, ajwt2); err != ErrAccountValidation { + t.Fatalf("expected %v but got %v", ErrAccountValidation, err) + } +} + +// Make sure connection limits apply to the system account itself. +func TestSystemAccountSystemConnectionLimitsHonored(t *testing.T) { + sa, optsA, sb, optsB, sakp := runTrustedCluster(t) + defer sa.Shutdown() + defer sb.Shutdown() + + okp, _ := nkeys.FromSeed(oSeed) + // Update system account to have 10 connections + pub, _ := sakp.PublicKey() + nac := jwt.NewAccountClaims(pub) + nac.Limits.Conn = 10 + ajwt, _ := nac.Encode(okp) + + addAccountToMemResolver(sa, pub, ajwt) + addAccountToMemResolver(sb, pub, ajwt) + + // Update the accounts on each server with new claims to force update. + sysAccA := sa.SystemAccount() + sa.updateAccountWithClaimJWT(sysAccA, ajwt) + sysAccB := sb.SystemAccount() + sb.updateAccountWithClaimJWT(sysAccB, ajwt) + + // Check system here first, with no external it should be zero. + sacc := sa.SystemAccount() + if nlc := sacc.NumLocalConnections(); nlc != 0 { + t.Fatalf("Expected no local connections, got %d", nlc) + } + + urlA := fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port) + urlB := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) + + // Create a user on each server. Break on first failure. + tc := 0 + for { + nca1, err := nats.Connect(urlA, createUserCreds(t, sa, sakp)) + if err != nil { + break + } + defer nca1.Close() + tc++ + + ncb1, err := nats.Connect(urlB, createUserCreds(t, sb, sakp)) + if err != nil { + break + } + defer ncb1.Close() + tc++ + + // The account's connection count is exchanged between servers + // so that the local count on each server reflects the total count. + // Pause a bit to give a chance to each server to process the update. + time.Sleep(15 * time.Millisecond) + } + if tc != 10 { + t.Fatalf("Expected to get 10 external connections, got %d", tc) + } + + checkFor(t, 1*time.Second, 50*time.Millisecond, func() error { + total := sa.NumClients() + sb.NumClients() + if total > int(nac.Limits.Conn) { + return fmt.Errorf("Expected only %d connections, was allowed to connect %d", nac.Limits.Conn, total) + } + return nil + }) +} + +// Test that the remote accounting works when a server is started some time later. +func TestSystemAccountConnectionLimitsServersStaggered(t *testing.T) { + sa, optsA, sb, optsB, _ := runTrustedCluster(t) + defer sa.Shutdown() + sb.Shutdown() + + // Let's create a user account. + okp, _ := nkeys.FromSeed(oSeed) + akp, _ := nkeys.CreateAccount() + pub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(pub) + nac.Limits.Conn = 4 // Limit to 4 connections. + jwt, _ := nac.Encode(okp) + + addAccountToMemResolver(sa, pub, jwt) + + urlA := fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port) + // Create max connections on sa. + for i := 0; i < int(nac.Limits.Conn); i++ { + nc, err := nats.Connect(urlA, createUserCreds(t, sa, akp)) + if err != nil { + t.Fatalf("Unexpected error on #%d try: %v", i+1, err) + } + defer nc.Close() + } + + // Restart server B. + optsB.AccountResolver = sa.AccountResolver() + optsB.SystemAccount = sa.SystemAccount().Name + sb = RunServer(optsB) + defer sb.Shutdown() + checkClusterFormed(t, sa, sb) + + // Trigger a load of the user account on the new server + // NOTE: If we do not load the user, the user can be the first + // to request this account, hence the connection will succeed. + checkFor(t, time.Second, 15*time.Millisecond, func() error { + if acc, err := sb.LookupAccount(pub); acc == nil || err != nil { + return fmt.Errorf("LookupAccount did not return account or failed, err=%v", err) + } + return nil + }) + + // Expect this to fail. + urlB := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) + if _, err := nats.Connect(urlB, createUserCreds(t, sb, akp)); err == nil { + t.Fatalf("Expected connection to fail due to max limit") + } +} + +// Test that the remote accounting works when a server is shutdown. +func TestSystemAccountConnectionLimitsServerShutdownGraceful(t *testing.T) { + sa, optsA, sb, optsB, _ := runTrustedCluster(t) + defer sa.Shutdown() + defer sb.Shutdown() + + // Let's create a user account. + okp, _ := nkeys.FromSeed(oSeed) + akp, _ := nkeys.CreateAccount() + pub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(pub) + nac.Limits.Conn = 10 // Limit to 10 connections. + jwt, _ := nac.Encode(okp) + + addAccountToMemResolver(sa, pub, jwt) + addAccountToMemResolver(sb, pub, jwt) + + urlA := fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port) + urlB := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) + + for i := 0; i < 5; i++ { + nc, err := nats.Connect(urlA, nats.NoReconnect(), createUserCreds(t, sa, akp)) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer nc.Close() + nc, err = nats.Connect(urlB, nats.NoReconnect(), createUserCreds(t, sb, akp)) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer nc.Close() + } + + // We are at capacity so both of these should fail. + if _, err := nats.Connect(urlA, createUserCreds(t, sa, akp)); err == nil { + t.Fatalf("Expected connection to fail due to max limit") + } + if _, err := nats.Connect(urlB, createUserCreds(t, sb, akp)); err == nil { + t.Fatalf("Expected connection to fail due to max limit") + } + + // Now shutdown Server B. + sb.Shutdown() + + // Now we should be able to create more on A now. + for i := 0; i < 5; i++ { + nc, err := nats.Connect(urlA, createUserCreds(t, sa, akp)) + if err != nil { + t.Fatalf("Expected to connect on %d, got %v", i, err) + } + defer nc.Close() + } +} + +// Test that the remote accounting works when a server goes away. +func TestSystemAccountConnectionLimitsServerShutdownForced(t *testing.T) { + sa, optsA, sb, optsB, _ := runTrustedCluster(t) + defer sa.Shutdown() + + // Let's create a user account. + okp, _ := nkeys.FromSeed(oSeed) + akp, _ := nkeys.CreateAccount() + pub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(pub) + nac.Limits.Conn = 20 // Limit to 20 connections. + jwt, _ := nac.Encode(okp) + + addAccountToMemResolver(sa, pub, jwt) + addAccountToMemResolver(sb, pub, jwt) + + urlA := fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port) + urlB := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) + + for i := 0; i < 10; i++ { + c, err := nats.Connect(urlA, nats.NoReconnect(), createUserCreds(t, sa, akp)) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer c.Close() + c, err = nats.Connect(urlB, nats.NoReconnect(), createUserCreds(t, sb, akp)) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer c.Close() + } + + // Now shutdown Server B. Do so such that no communications go out. + sb.mu.Lock() + sb.sys = nil + sb.mu.Unlock() + sb.Shutdown() + + if _, err := nats.Connect(urlA, createUserCreds(t, sa, akp)); err == nil { + t.Fatalf("Expected connection to fail due to max limit") + } + + // Let's speed up the checking process. + sa.mu.Lock() + sa.sys.chkOrph = 10 * time.Millisecond + sa.sys.orphMax = 30 * time.Millisecond + sa.sys.sweeper.Reset(sa.sys.chkOrph) + sa.mu.Unlock() + + // We should eventually be able to connect. + checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { + if c, err := nats.Connect(urlA, createUserCreds(t, sa, akp)); err != nil { + return err + } else { + c.Close() + } + return nil + }) +} + +func TestSystemAccountFromConfig(t *testing.T) { + kp, _ := nkeys.FromSeed(oSeed) + opub, _ := kp.PublicKey() + akp, _ := nkeys.CreateAccount() + apub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(apub) + ajwt, err := nac.Encode(kp) + if err != nil { + t.Fatalf("Error generating account JWT: %v", err) + } + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(ajwt)) + })) + defer ts.Close() + + confTemplate := ` + listen: -1 + trusted: %s + system_account: %s + resolver: URL("%s/jwt/v1/accounts/") + ` + + conf := createConfFile(t, []byte(fmt.Sprintf(confTemplate, opub, apub, ts.URL))) + + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + if acc := s.SystemAccount(); acc == nil || acc.Name != apub { + t.Fatalf("System Account not properly set") + } +} + +func TestAccountClaimsUpdates(t *testing.T) { + test := func(subj string) { + s, opts := runTrustedServer(t) + defer s.Shutdown() + + sacc, sakp := createAccount(s) + s.setSystemAccount(sacc) + + // Let's create a normal account with limits we can update. + okp, _ := nkeys.FromSeed(oSeed) + akp, _ := nkeys.CreateAccount() + pub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(pub) + nac.Limits.Conn = 4 + ajwt, _ := nac.Encode(okp) + + addAccountToMemResolver(s, pub, ajwt) + + acc, _ := s.LookupAccount(pub) + if acc.MaxActiveConnections() != 4 { + t.Fatalf("Expected to see a limit of 4 connections") + } + + // Simulate a systems publisher so we can do an account claims update. + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(url, createUserCreds(t, s, sakp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // Update the account + nac = jwt.NewAccountClaims(pub) + nac.Limits.Conn = 8 + issAt := time.Now().Add(-30 * time.Second).Unix() + nac.IssuedAt = issAt + expires := time.Now().Add(2 * time.Second).Unix() + nac.Expires = expires + ajwt, _ = nac.Encode(okp) + + // Publish to the system update subject. + claimUpdateSubj := fmt.Sprintf(subj, pub) + nc.Publish(claimUpdateSubj, []byte(ajwt)) + nc.Flush() + time.Sleep(200 * time.Millisecond) + + acc, _ = s.LookupAccount(pub) + if acc.MaxActiveConnections() != 8 { + t.Fatalf("Account was not updated") + } + } + t.Run("new", func(t *testing.T) { + test(accUpdateEventSubjNew) + }) + t.Run("old", func(t *testing.T) { + test(accUpdateEventSubjOld) + }) +} + +func TestAccountReqMonitoring(t *testing.T) { + s, opts := runTrustedServer(t) + defer s.Shutdown() + sacc, sakp := createAccount(s) + s.setSystemAccount(sacc) + s.EnableJetStream(&JetStreamConfig{StoreDir: t.TempDir()}) + unusedAcc, _ := createAccount(s) + acc, akp := createAccount(s) + acc.EnableJetStream(nil, nil) + subsz := fmt.Sprintf(accDirectReqSubj, acc.Name, "SUBSZ") + connz := fmt.Sprintf(accDirectReqSubj, acc.Name, "CONNZ") + jsz := fmt.Sprintf(accDirectReqSubj, acc.Name, "JSZ") + + pStatz := fmt.Sprintf(accPingReqSubj, "STATZ") + statz := func(name string) string { return fmt.Sprintf(accDirectReqSubj, name, "STATZ") } + // Create system account connection to query + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + ncSys, err := nats.Connect(url, createUserCreds(t, s, sakp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncSys.Close() + // Create a connection that we can query + nc, err := nats.Connect(url, createUserCreds(t, s, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + // query SUBSZ for account + resp, err := ncSys.Request(subsz, nil, time.Second) + require_NoError(t, err) + require_Contains(t, string(resp.Data), `"num_subscriptions":5,`) + // create a subscription + sub, err := nc.Subscribe("foo", func(msg *nats.Msg) {}) + require_NoError(t, err) + defer sub.Unsubscribe() + + require_NoError(t, nc.Flush()) + // query SUBSZ for account + resp, err = ncSys.Request(subsz, nil, time.Second) + require_NoError(t, err) + require_Contains(t, string(resp.Data), `"num_subscriptions":6,`, `"subject":"foo"`) + // query connections for account + resp, err = ncSys.Request(connz, nil, time.Second) + require_NoError(t, err) + require_Contains(t, string(resp.Data), `"num_connections":1,`, `"total":1,`) + // query connections for js account + resp, err = ncSys.Request(jsz, nil, time.Second) + require_NoError(t, err) + require_Contains(t, string(resp.Data), `"memory":0,`, `"storage":0,`) + // query statz/conns for account + resp, err = ncSys.Request(statz(acc.Name), nil, time.Second) + require_NoError(t, err) + respContentAcc := []string{`"conns":1,`, `"total_conns":1`, `"slow_consumers":0`, `"sent":{"msgs":0,"bytes":0`, + `"received":{"msgs":0,"bytes":0`, `"num_subscriptions":`, fmt.Sprintf(`"acc":"%s"`, acc.Name)} + require_Contains(t, string(resp.Data), respContentAcc...) + + rIb := ncSys.NewRespInbox() + rSub, err := ncSys.SubscribeSync(rIb) + require_NoError(t, err) + require_NoError(t, ncSys.PublishRequest(pStatz, rIb, nil)) + minRespContentForBothAcc := []string{`"conns":1,`, `"total_conns":1`, `"slow_consumers":0`, `"acc":"`, `"num_subscriptions":`} + resp, err = rSub.NextMsg(time.Second) + require_NoError(t, err) + require_Contains(t, string(resp.Data), minRespContentForBothAcc...) + // expect one entry per account + require_Contains(t, string(resp.Data), fmt.Sprintf(`"acc":"%s"`, acc.Name), fmt.Sprintf(`"acc":"%s"`, sacc.Name)) + + // Test ping with filter by account name + require_NoError(t, ncSys.PublishRequest(pStatz, rIb, []byte(fmt.Sprintf(`{"accounts":["%s"]}`, sacc.Name)))) + m, err := rSub.NextMsg(time.Second) + require_NoError(t, err) + require_Contains(t, string(m.Data), minRespContentForBothAcc...) + + require_NoError(t, ncSys.PublishRequest(pStatz, rIb, []byte(fmt.Sprintf(`{"accounts":["%s"]}`, acc.Name)))) + m, err = rSub.NextMsg(time.Second) + require_NoError(t, err) + require_Contains(t, string(m.Data), respContentAcc...) + + // Test include unused for statz and ping of statz + unusedContent := []string{`"conns":0,`, `"total_conns":0`, `"slow_consumers":0`, + fmt.Sprintf(`"acc":"%s"`, unusedAcc.Name)} + + resp, err = ncSys.Request(statz(unusedAcc.Name), + []byte(fmt.Sprintf(`{"accounts":["%s"], "include_unused":true}`, unusedAcc.Name)), + time.Second) + require_NoError(t, err) + require_Contains(t, string(resp.Data), unusedContent...) + + require_NoError(t, ncSys.PublishRequest(pStatz, rIb, + []byte(fmt.Sprintf(`{"accounts":["%s"], "include_unused":true}`, unusedAcc.Name)))) + resp, err = rSub.NextMsg(time.Second) + require_NoError(t, err) + require_Contains(t, string(resp.Data), unusedContent...) + + require_NoError(t, ncSys.PublishRequest(pStatz, rIb, []byte(fmt.Sprintf(`{"accounts":["%s"]}`, unusedAcc.Name)))) + _, err = rSub.NextMsg(200 * time.Millisecond) + require_Error(t, err) + + // Test ping from within account, send extra message to check counters. + require_NoError(t, nc.Publish("foo", nil)) + ib := nc.NewRespInbox() + rSub, err = nc.SubscribeSync(ib) + require_NoError(t, err) + require_NoError(t, nc.PublishRequest(pStatz, ib, nil)) + require_NoError(t, nc.Flush()) + resp, err = rSub.NextMsg(time.Second) + require_NoError(t, err) + + // Since we now have processed our own message, sent msgs will be at least 1. + payload := string(resp.Data) + respContentAcc = []string{`"conns":1,`, `"total_conns":1`, `"slow_consumers":0`, `"sent":{"msgs":1,"bytes":0`, fmt.Sprintf(`"acc":"%s"`, acc.Name)} + require_Contains(t, payload, respContentAcc...) + + // Depending on timing, statz message could be accounted too. + receivedOK := strings.Contains(payload, `"received":{"msgs":1,"bytes":0`) || strings.Contains(payload, `"received":{"msgs":2,"bytes":0`) + require_True(t, receivedOK) + _, err = rSub.NextMsg(200 * time.Millisecond) + require_Error(t, err) +} + +func TestAccountReqInfo(t *testing.T) { + s, opts := runTrustedServer(t) + defer s.Shutdown() + sacc, sakp := createAccount(s) + s.setSystemAccount(sacc) + // Let's create an account with service export. + akp, _ := nkeys.CreateAccount() + pub1, _ := akp.PublicKey() + nac1 := jwt.NewAccountClaims(pub1) + nac1.Exports.Add(&jwt.Export{Subject: "req.*", Type: jwt.Service}) + ajwt1, _ := nac1.Encode(oKp) + addAccountToMemResolver(s, pub1, ajwt1) + s.LookupAccount(pub1) + info1 := fmt.Sprintf(accDirectReqSubj, pub1, "INFO") + // Now add an account with service imports. + akp2, _ := nkeys.CreateAccount() + pub2, _ := akp2.PublicKey() + nac2 := jwt.NewAccountClaims(pub2) + nac2.Imports.Add(&jwt.Import{Account: pub1, Subject: "req.1", Type: jwt.Service}) + ajwt2, _ := nac2.Encode(oKp) + addAccountToMemResolver(s, pub2, ajwt2) + s.LookupAccount(pub2) + info2 := fmt.Sprintf(accDirectReqSubj, pub2, "INFO") + // Create system account connection to query + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + ncSys, err := nats.Connect(url, createUserCreds(t, s, sakp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncSys.Close() + checkCommon := func(info *AccountInfo, srv *ServerInfo, pub, jwt string) { + if info.Complete != true { + t.Fatalf("Unexpected value: %v", info.Complete) + } else if info.Expired != false { + t.Fatalf("Unexpected value: %v", info.Expired) + } else if info.JetStream != false { + t.Fatalf("Unexpected value: %v", info.JetStream) + } else if info.ClientCnt != 0 { + t.Fatalf("Unexpected value: %v", info.ClientCnt) + } else if info.AccountName != pub { + t.Fatalf("Unexpected value: %v", info.AccountName) + } else if info.LeafCnt != 0 { + t.Fatalf("Unexpected value: %v", info.LeafCnt) + } else if info.Jwt != jwt { + t.Fatalf("Unexpected value: %v", info.Jwt) + } else if srv.Cluster != "abc" { + t.Fatalf("Unexpected value: %v", srv.Cluster) + } else if srv.Name != s.Name() { + t.Fatalf("Unexpected value: %v", srv.Name) + } else if srv.Host != opts.Host { + t.Fatalf("Unexpected value: %v", srv.Host) + } else if srv.Seq < 1 { + t.Fatalf("Unexpected value: %v", srv.Seq) + } + } + info := AccountInfo{} + srv := ServerInfo{} + msg := struct { + Data *AccountInfo `json:"data"` + Srv *ServerInfo `json:"server"` + }{ + &info, + &srv, + } + if resp, err := ncSys.Request(info1, nil, time.Second); err != nil { + t.Fatalf("Error on request: %v", err) + } else if err := json.Unmarshal(resp.Data, &msg); err != nil { + t.Fatalf("Unmarshalling failed: %v", err) + } else if len(info.Exports) != 1 { + t.Fatalf("Unexpected value: %v", info.Exports) + } else if len(info.Imports) != 4 { + t.Fatalf("Unexpected value: %+v", info.Imports) + } else if info.Exports[0].Subject != "req.*" { + t.Fatalf("Unexpected value: %v", info.Exports) + } else if info.Exports[0].Type != jwt.Service { + t.Fatalf("Unexpected value: %v", info.Exports) + } else if info.Exports[0].ResponseType != jwt.ResponseTypeSingleton { + t.Fatalf("Unexpected value: %v", info.Exports) + } else if info.SubCnt != 4 { + t.Fatalf("Unexpected value: %v", info.SubCnt) + } else { + checkCommon(&info, &srv, pub1, ajwt1) + } + info = AccountInfo{} + srv = ServerInfo{} + if resp, err := ncSys.Request(info2, nil, time.Second); err != nil { + t.Fatalf("Error on request: %v", err) + } else if err := json.Unmarshal(resp.Data, &msg); err != nil { + t.Fatalf("Unmarshalling failed: %v", err) + } else if len(info.Exports) != 0 { + t.Fatalf("Unexpected value: %v", info.Exports) + } else if len(info.Imports) != 5 { + t.Fatalf("Unexpected value: %+v", info.Imports) + } + // Here we need to find our import + var si *ExtImport + for _, im := range info.Imports { + if im.Subject == "req.1" { + si = &im + break + } + } + if si == nil { + t.Fatalf("Could not find our import") + } + if si.Type != jwt.Service { + t.Fatalf("Unexpected value: %+v", si) + } else if si.Account != pub1 { + t.Fatalf("Unexpected value: %+v", si) + } else if info.SubCnt != 5 { + t.Fatalf("Unexpected value: %+v", si) + } else { + checkCommon(&info, &srv, pub2, ajwt2) + } +} + +func TestAccountClaimsUpdatesWithServiceImports(t *testing.T) { + s, opts := runTrustedServer(t) + defer s.Shutdown() + + sacc, sakp := createAccount(s) + s.setSystemAccount(sacc) + + okp, _ := nkeys.FromSeed(oSeed) + + // Let's create an account with service export. + akp, _ := nkeys.CreateAccount() + pub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(pub) + nac.Exports.Add(&jwt.Export{Subject: "req.*", Type: jwt.Service}) + ajwt, _ := nac.Encode(okp) + addAccountToMemResolver(s, pub, ajwt) + s.LookupAccount(pub) + + // Now add an account with multiple service imports. + akp2, _ := nkeys.CreateAccount() + pub2, _ := akp2.PublicKey() + nac2 := jwt.NewAccountClaims(pub2) + nac2.Imports.Add(&jwt.Import{Account: pub, Subject: "req.1", Type: jwt.Service}) + ajwt2, _ := nac2.Encode(okp) + + addAccountToMemResolver(s, pub2, ajwt2) + s.LookupAccount(pub2) + + startSubs := s.NumSubscriptions() + + // Simulate a systems publisher so we can do an account claims update. + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(url, createUserCreds(t, s, sakp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // Update the account several times + for i := 1; i <= 10; i++ { + nac2 = jwt.NewAccountClaims(pub2) + nac2.Limits.Conn = int64(i) + nac2.Imports.Add(&jwt.Import{Account: pub, Subject: "req.1", Type: jwt.Service}) + ajwt2, _ = nac2.Encode(okp) + + // Publish to the system update subject. + claimUpdateSubj := fmt.Sprintf(accUpdateEventSubjNew, pub2) + nc.Publish(claimUpdateSubj, []byte(ajwt2)) + } + nc.Flush() + time.Sleep(50 * time.Millisecond) + + if startSubs < s.NumSubscriptions() { + t.Fatalf("Subscriptions leaked: %d vs %d", startSubs, s.NumSubscriptions()) + } +} + +func TestAccountConnsLimitExceededAfterUpdate(t *testing.T) { + s, opts := runTrustedServer(t) + defer s.Shutdown() + + sacc, _ := createAccount(s) + s.setSystemAccount(sacc) + + // Let's create a normal account with limits we can update. + okp, _ := nkeys.FromSeed(oSeed) + akp, _ := nkeys.CreateAccount() + pub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(pub) + nac.Limits.Conn = 10 + ajwt, _ := nac.Encode(okp) + + addAccountToMemResolver(s, pub, ajwt) + acc, _ := s.LookupAccount(pub) + + // Now create the max connections. + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + for { + nc, err := nats.Connect(url, createUserCreds(t, s, akp)) + if err != nil { + break + } + defer nc.Close() + } + + // We should have max here. + checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { + if total := s.NumClients(); total != acc.MaxActiveConnections() { + return fmt.Errorf("Expected %d connections, got %d", acc.MaxActiveConnections(), total) + } + return nil + }) + + // Now change limits to make current connections over the limit. + nac = jwt.NewAccountClaims(pub) + nac.Limits.Conn = 2 + ajwt, _ = nac.Encode(okp) + + s.updateAccountWithClaimJWT(acc, ajwt) + if acc.MaxActiveConnections() != 2 { + t.Fatalf("Expected max connections to be set to 2, got %d", acc.MaxActiveConnections()) + } + // We should have closed the excess connections. + checkClientsCount(t, s, acc.MaxActiveConnections()) +} + +func TestAccountConnsLimitExceededAfterUpdateDisconnectNewOnly(t *testing.T) { + s, opts := runTrustedServer(t) + defer s.Shutdown() + + sacc, _ := createAccount(s) + s.setSystemAccount(sacc) + + // Let's create a normal account with limits we can update. + okp, _ := nkeys.FromSeed(oSeed) + akp, _ := nkeys.CreateAccount() + pub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(pub) + nac.Limits.Conn = 10 + ajwt, _ := nac.Encode(okp) + + addAccountToMemResolver(s, pub, ajwt) + acc, _ := s.LookupAccount(pub) + + // Now create the max connections. + // We create half then we will wait and then create the rest. + // Will test that we disconnect the newest ones. + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + for i := 0; i < 5; i++ { + nc, err := nats.Connect(url, nats.Name("OLD"), nats.NoReconnect(), createUserCreds(t, s, akp)) + require_NoError(t, err) + defer nc.Close() + } + time.Sleep(500 * time.Millisecond) + for i := 0; i < 5; i++ { + nc, err := nats.Connect(url, nats.Name("NEW"), nats.NoReconnect(), createUserCreds(t, s, akp)) + require_NoError(t, err) + defer nc.Close() + } + + // We should have max here. + checkClientsCount(t, s, acc.MaxActiveConnections()) + + // Now change limits to make current connections over the limit. + nac = jwt.NewAccountClaims(pub) + nac.Limits.Conn = 5 + ajwt, _ = nac.Encode(okp) + + s.updateAccountWithClaimJWT(acc, ajwt) + if acc.MaxActiveConnections() != 5 { + t.Fatalf("Expected max connections to be set to 2, got %d", acc.MaxActiveConnections()) + } + // We should have closed the excess connections. + checkClientsCount(t, s, acc.MaxActiveConnections()) + + connz, err := s.Connz(nil) + require_NoError(t, err) + + // There should only be OLD connections. + for _, c := range connz.Conns { + require_Equal(t, c.Name, "OLD") + } +} + +func TestSystemAccountWithBadRemoteLatencyUpdate(t *testing.T) { + s, _ := runTrustedServer(t) + defer s.Shutdown() + + acc, _ := createAccount(s) + s.setSystemAccount(acc) + + rl := remoteLatency{ + Account: "NONSENSE", + ReqId: "_INBOX.22", + } + b, _ := json.Marshal(&rl) + s.remoteLatencyUpdate(nil, nil, nil, "foo", _EMPTY_, nil, b) +} + +func TestSystemAccountWithGateways(t *testing.T) { + sa, oa, sb, ob, akp := runTrustedGateways(t) + defer sa.Shutdown() + defer sb.Shutdown() + + // Create a client on A that will subscribe on $SYS.ACCOUNT.> + urla := fmt.Sprintf("nats://%s:%d", oa.Host, oa.Port) + nca := natsConnect(t, urla, createUserCreds(t, sa, akp), nats.Name("SYS")) + defer nca.Close() + nca.Flush() + + sub, _ := nca.SubscribeSync("$SYS.ACCOUNT.>") + defer sub.Unsubscribe() + nca.Flush() + + // If this tests fails with wrong number after 10 seconds we may have + // added a new initial subscription for the eventing system. + checkExpectedSubs(t, 62, sa) + + // Create a client on B and see if we receive the event + urlb := fmt.Sprintf("nats://%s:%d", ob.Host, ob.Port) + ncb := natsConnect(t, urlb, createUserCreds(t, sb, akp), nats.Name("TEST EVENTS")) + defer ncb.Close() + + // space for .CONNECT and .CONNS from SYS and $G as well as one extra message + msgs := [4]*nats.Msg{} + var err error + msgs[0], err = sub.NextMsg(time.Second) + require_NoError(t, err) + msgs[1], err = sub.NextMsg(time.Second) + require_NoError(t, err) + // TODO: There is a race currently that can cause the server to process the + // system event *after* the subscription on "A" has been registered, and so + // the "nca" client would receive its own CONNECT message. + msgs[2], _ = sub.NextMsg(250 * time.Millisecond) + + findMsgs := func(sub string) []*nats.Msg { + rMsgs := []*nats.Msg{} + for _, m := range msgs { + if m == nil { + continue + } + if m.Subject == sub { + rMsgs = append(rMsgs, m) + } + } + return rMsgs + } + + msg := findMsgs(fmt.Sprintf("$SYS.ACCOUNT.%s.CONNECT", sa.SystemAccount().Name)) + var bMsg *nats.Msg + if len(msg) < 1 { + t.Fatal("Expected at least one message") + } + bMsg = msg[len(msg)-1] + + require_Contains(t, string(bMsg.Data), sb.ID()) + require_Contains(t, string(bMsg.Data), `"cluster":"B"`) + require_Contains(t, string(bMsg.Data), `"name":"TEST EVENTS"`) + + connsMsgA := findMsgs(fmt.Sprintf("$SYS.ACCOUNT.%s.SERVER.CONNS", sa.SystemAccount().Name)) + if len(connsMsgA) != 1 { + t.Fatal("Expected a message") + } +} + +func TestSystemAccountNoAuthUser(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: "127.0.0.1:-1" + accounts { + $SYS { + users [{user: "admin", password: "pwd"}] + } + } + `)) + defer os.Remove(conf) + s, o := RunServerWithConfig(conf) + defer s.Shutdown() + + for _, test := range []struct { + name string + usrInfo string + ok bool + account string + }{ + {"valid user/pwd", "admin:pwd@", true, "$SYS"}, + {"invalid pwd", "admin:wrong@", false, _EMPTY_}, + {"some token", "sometoken@", false, _EMPTY_}, + {"user used without pwd", "admin@", false, _EMPTY_}, // will be treated as a token + {"user with empty password", "admin:@", false, _EMPTY_}, + {"no user means global account", _EMPTY_, true, globalAccountName}, + } { + t.Run(test.name, func(t *testing.T) { + url := fmt.Sprintf("nats://%s127.0.0.1:%d", test.usrInfo, o.Port) + nc, err := nats.Connect(url) + if err != nil { + if test.ok { + t.Fatalf("Unexpected error: %v", err) + } + return + } else if !test.ok { + nc.Close() + t.Fatalf("Should have failed, did not") + } + var accName string + s.mu.Lock() + for _, c := range s.clients { + c.mu.Lock() + if c.acc != nil { + accName = c.acc.Name + } + c.mu.Unlock() + break + } + s.mu.Unlock() + nc.Close() + checkClientsCount(t, s, 0) + if accName != test.account { + t.Fatalf("The account should have been %q, got %q", test.account, accName) + } + }) + } +} + +func TestServerAccountConns(t *testing.T) { + // speed up hb + orgHBInterval := eventsHBInterval + eventsHBInterval = time.Millisecond * 100 + defer func() { eventsHBInterval = orgHBInterval }() + conf := createConfFile(t, []byte(` + host: 127.0.0.1 + port: -1 + system_account: SYS + accounts: { + SYS: {users: [{user: s, password: s}]} + ACC: {users: [{user: a, password: a}]} + }`)) + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + nc := natsConnect(t, s.ClientURL(), nats.UserInfo("a", "a")) + defer nc.Close() + + subOut, err := nc.SubscribeSync("foo") + require_NoError(t, err) + hw := "HELLO WORLD" + nc.Publish("foo", []byte(hw)) + nc.Publish("bar", []byte(hw)) // will only count towards received + nc.Flush() + m, err := subOut.NextMsg(time.Second) + require_NoError(t, err) + require_Equal(t, string(m.Data), hw) + + ncs := natsConnect(t, s.ClientURL(), nats.UserInfo("s", "s")) + defer ncs.Close() + subs, err := ncs.SubscribeSync("$SYS.ACCOUNT.ACC.SERVER.CONNS") + require_NoError(t, err) + + m, err = subs.NextMsg(time.Second) + require_NoError(t, err) + accConns := &AccountNumConns{} + err = json.Unmarshal(m.Data, accConns) + require_NoError(t, err) + + require_True(t, accConns.Received.Msgs == 2) + require_True(t, accConns.Received.Bytes == 2*int64(len(hw))) + require_True(t, accConns.Sent.Msgs == 1) + require_True(t, accConns.Sent.Bytes == int64(len(hw))) +} + +func TestServerEventsStatsZ(t *testing.T) { + serverStatsReqSubj := "$SYS.REQ.SERVER.%s.STATSZ" + preStart := time.Now().UTC() + // Add little bit of delay to make sure that time check + // between pre-start and actual start does not fail. + time.Sleep(5 * time.Millisecond) + sa, optsA, sb, _, akp := runTrustedCluster(t) + defer sa.Shutdown() + defer sb.Shutdown() + // Same between actual start and post start. + time.Sleep(5 * time.Millisecond) + postStart := time.Now().UTC() + + url := fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port) + ncs, err := nats.Connect(url, createUserCreds(t, sa, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncs.Close() + + subj := fmt.Sprintf(serverStatsSubj, sa.ID()) + sub, _ := ncs.SubscribeSync(subj) + defer sub.Unsubscribe() + ncs.Publish("foo", []byte("HELLO WORLD")) + ncs.Flush() + + // Let's speed up the checking process. + sa.mu.Lock() + sa.sys.statsz = 10 * time.Millisecond + sa.sys.stmr.Reset(sa.sys.statsz) + sa.mu.Unlock() + + _, err = sub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Error receiving msg: %v", err) + } + // Get it the second time so we can check some stats + msg, err := sub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Error receiving msg: %v", err) + } + m := ServerStatsMsg{} + if err := json.Unmarshal(msg.Data, &m); err != nil { + t.Fatalf("Error unmarshalling the statz json: %v", err) + } + if m.Server.ID != sa.ID() { + t.Fatalf("Did not match IDs") + } + if m.Server.Cluster != "TEST_CLUSTER_22" { + t.Fatalf("Did not match cluster name") + } + if m.Server.Version != VERSION { + t.Fatalf("Did not match server version") + } + if !m.Stats.Start.After(preStart) && m.Stats.Start.Before(postStart) { + t.Fatalf("Got a wrong start time for the server %v", m.Stats.Start) + } + if m.Stats.Connections != 1 { + t.Fatalf("Did not match connections of 1, got %d", m.Stats.Connections) + } + if m.Stats.ActiveAccounts != 1 { + t.Fatalf("Did not match active accounts of 1, got %d", m.Stats.ActiveAccounts) + } + if m.Stats.Sent.Msgs < 1 { + t.Fatalf("Did not match sent msgs of >=1, got %d", m.Stats.Sent.Msgs) + } + if m.Stats.Received.Msgs < 1 { + t.Fatalf("Did not match received msgs of >=1, got %d", m.Stats.Received.Msgs) + } + if m.Stats.ReceivedFromClients.Msgs < 1 { + t.Fatalf("Did not match received from client msgs of >=1, got %d", m.Stats.ReceivedFromClients.Msgs) + } + if m.Stats.ReceivedFromClients.Bytes < 1 { + t.Fatalf("Did not match received from client bytes of >=1, got %d", m.Stats.ReceivedFromClients.Bytes) + } + if m.Stats.SentToClients.Msgs < 1 { + t.Fatalf("Did not match sent to client msgs of >= 1, got %d", m.Stats.SentToClients.Msgs) + } + if m.Stats.SentToClients.Bytes < 1 { + t.Fatalf("Did not match sent to client bytes of >= 1, got %d", m.Stats.SentToClients.Bytes) + } + + // Default pool size + 1 for system account + expectedRoutes := DEFAULT_ROUTE_POOL_SIZE + 1 + if lr := len(m.Stats.Routes); lr != expectedRoutes { + t.Fatalf("Expected %d routes, but got %d", expectedRoutes, lr) + } + + // Now let's prompt this server to send us the statsz + subj = fmt.Sprintf(serverStatsReqSubj, sa.ID()) + msg, err = ncs.Request(subj, nil, time.Second) + if err != nil { + t.Fatalf("Error trying to request statsz: %v", err) + } + m2 := ServerStatsMsg{} + if err := json.Unmarshal(msg.Data, &m2); err != nil { + t.Fatalf("Error unmarshalling the statz json: %v", err) + } + if m2.Server.ID != sa.ID() { + t.Fatalf("Did not match IDs") + } + if m2.Stats.Connections != 1 { + t.Fatalf("Did not match connections of 1, got %d", m2.Stats.Connections) + } + if m2.Stats.ActiveAccounts != 1 { + t.Fatalf("Did not match active accounts of 1, got %d", m2.Stats.ActiveAccounts) + } + if m2.Stats.Sent.Msgs < 3 { + t.Fatalf("Did not match sent msgs of >= 3, got %d", m2.Stats.Sent.Msgs) + } + if m2.Stats.Received.Msgs < 1 { + t.Fatalf("Did not match received msgs of >= 1, got %d", m2.Stats.Received.Msgs) + } + if lr := len(m2.Stats.Routes); lr != expectedRoutes { + t.Fatalf("Expected %d routes, but got %d", expectedRoutes, lr) + } + + msg, err = ncs.Request(subj, nil, time.Second) + if err != nil { + t.Fatalf("Error trying to request statsz: %v", err) + } + m3 := ServerStatsMsg{} + if err := json.Unmarshal(msg.Data, &m3); err != nil { + t.Fatalf("Error unmarshalling the statz json: %v", err) + } + if m3.Server.ID != sa.ID() { + t.Fatalf("Did not match IDs") + } + if m3.Stats.Connections != 1 { + t.Fatalf("Did not match connections of 1, got %d", m3.Stats.Connections) + } + if m3.Stats.ActiveAccounts != 1 { + t.Fatalf("Did not match active accounts of 1, got %d", m3.Stats.ActiveAccounts) + } + if m3.Stats.Sent.Msgs < 4 { + t.Fatalf("Did not match sent msgs of >= 4, got %d", m3.Stats.Sent.Msgs) + } + if m3.Stats.Received.Msgs < 2 { + t.Fatalf("Did not match received msgs of >= 2, got %d", m3.Stats.Received.Msgs) + } + if lr := len(m3.Stats.Routes); lr != expectedRoutes { + t.Fatalf("Expected %d routes, but got %d", expectedRoutes, lr) + } + for _, sr := range m3.Stats.Routes { + if sr.Name != "B_SRV" { + t.Fatalf("Expected server A's route to B to have Name set to %q, got %q", "B", sr.Name) + } + } + // Increment stalls to confirm they are reported. + atomic.AddInt64(&sb.stalls, 3) + + // Now query B and check that route's name is "A" + subj = fmt.Sprintf(serverStatsReqSubj, sb.ID()) + ncs.SubscribeSync(subj) + msg, err = ncs.Request(subj, nil, time.Second) + if err != nil { + t.Fatalf("Error trying to request statsz: %v", err) + } + m = ServerStatsMsg{} + if err := json.Unmarshal(msg.Data, &m); err != nil { + t.Fatalf("Error unmarshalling the statz json: %v", err) + } + if lr := len(m.Stats.Routes); lr != expectedRoutes { + t.Fatalf("Expected %d routes, but got %d", expectedRoutes, lr) + } + for _, sr := range m.Stats.Routes { + if sr.Name != "A_SRV" { + t.Fatalf("Expected server B's route to A to have Name set to %q, got %q", "A_SRV", sr.Name) + } + } + require_Equal(t, m.Stats.StalledClients, 3) +} + +func TestServerEventsStatsZForcedBroadcastNotRateLimited(t *testing.T) { + // Tests will set this generally faster, but here we want the rate limiter + // to actually be active. + original := statszRateLimit + statszRateLimit = time.Hour + defer func() { statszRateLimit = original }() + + s, opts := runTrustedServer(t) + defer s.Shutdown() + + acc, akp := createAccount(s) + require_NoError(t, s.setSystemAccount(acc)) + + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(url, createUserCreds(t, s, akp)) + require_NoError(t, err) + defer nc.Close() + + subj := fmt.Sprintf(serverStatsSubj, s.ID()) + + // Arrange so that the rate limiter would suppress any broadcast: + // stop the periodic heartbeat timer so it cannot produce a broadcast, + // and set lastStatsz to "now" so we are well within the interval. + s.mu.Lock() + if s.sys.stmr != nil { + s.sys.stmr.Stop() + } + s.sys.cstatsz = time.Hour + s.sys.statsz = time.Hour + s.sys.lastStatsz = time.Now() + s.mu.Unlock() + + // Let any inflight startup statsz goroutines finish before we subscribe, + // so they cannot land in our subscription as a false positive. + time.Sleep(300 * time.Millisecond) + + // From here on the only thing that can publish to this subject is an + // un-suppressed broadcast. + sub, err := nc.SubscribeSync(subj) + require_NoError(t, err) + defer sub.Unsubscribe() + require_NoError(t, nc.Flush()) + + // Re-arm the suppression window right before the request. + s.mu.Lock() + s.sys.lastStatsz = time.Now() + s.mu.Unlock() + + // Issue a forced (no-reply) STATSZ request. This is the path a new meta + // leader uses to force an immediate broadcast. It must reset lastStatsz + // and broadcast immediately even though we are within the rate-limit + // interval. + s.statszReq(nil, nil, nil, subj, _EMPTY_, nil, nil) + + // Make sure it is a statsz from our server. + msg, err := sub.NextMsg(2 * time.Second) + require_NoError(t, err) + var m ServerStatsMsg + require_NoError(t, json.Unmarshal(msg.Data, &m)) + require_Equal(t, m.Server.ID, s.ID()) + require_Equal(t, m.Stats.Connections, 1) +} + +func TestServerEventsHealthZSingleServer(t *testing.T) { + type healthzResp struct { + Healthz HealthStatus `json:"data"` + Server ServerInfo `json:"server"` + } + cfg := fmt.Sprintf(`listen: 127.0.0.1:-1 + + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + no_auth_user: one + + accounts { + ONE { users = [ { user: "one", pass: "p" } ]; jetstream: enabled } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + }`, t.TempDir()) + + serverHealthzReqSubj := "$SYS.REQ.SERVER.%s.HEALTHZ" + s, _ := RunServerWithConfig(createConfFile(t, []byte(cfg))) + defer s.Shutdown() + + ncs, err := nats.Connect(s.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) + if err != nil { + t.Fatalf("Error connecting to cluster: %v", err) + } + + defer ncs.Close() + ncAcc, err := nats.Connect(s.ClientURL()) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncAcc.Close() + js, err := ncAcc.JetStream() + if err != nil { + t.Fatalf("Error creating JetStream context: %v", err) + } + _, err = js.AddStream(&nats.StreamConfig{ + Name: "test", + Subjects: []string{"foo"}, + }) + if err != nil { + t.Fatalf("Error creating stream: %v", err) + } + _, err = js.AddConsumer("test", &nats.ConsumerConfig{ + Name: "cons", + }) + if err != nil { + t.Fatalf("Error creating consumer: %v", err) + } + + subj := fmt.Sprintf(serverHealthzReqSubj, s.ID()) + + tests := []struct { + name string + req *HealthzEventOptions + expected HealthStatus + }{ + { + name: "no parameters", + expected: HealthStatus{Status: "ok", StatusCode: 200}, + }, + { + name: "with js enabled only", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + JSEnabledOnly: true, + }, + }, + expected: HealthStatus{Status: "ok", StatusCode: 200}, + }, + { + name: "with server only", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + JSServerOnly: true, + }, + }, + expected: HealthStatus{Status: "ok", StatusCode: 200}, + }, + { + name: "with js meta only", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + JSMetaOnly: true, + }, + }, + expected: HealthStatus{Status: "ok", StatusCode: 200}, + }, + { + name: "with account name", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + }, + }, + expected: HealthStatus{Status: "ok", StatusCode: 200}, + }, + { + name: "with account name and stream", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + Stream: "test", + }, + }, + expected: HealthStatus{Status: "ok", StatusCode: 200}, + }, + { + name: "with account name, stream and consumer", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + Stream: "test", + Consumer: "cons", + }, + }, + expected: HealthStatus{Status: "ok", StatusCode: 200}, + }, + { + name: "with stream only", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Stream: "test", + }, + }, + expected: HealthStatus{ + Status: "error", + StatusCode: 400, + Error: `"account" must not be empty when checking stream health`, + }, + }, + { + name: "with stream only, detailed", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Details: true, + Stream: "test", + }, + }, + expected: HealthStatus{ + Status: "error", + StatusCode: 400, + Errors: []HealthzError{ + { + Type: HealthzErrorBadRequest, + Error: `"account" must not be empty when checking stream health`, + }, + }, + }, + }, + { + name: "with account and consumer", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + Consumer: "cons", + }, + }, + expected: HealthStatus{ + Status: "error", + StatusCode: 400, + Error: `"stream" must not be empty when checking consumer health`, + }, + }, + { + name: "with account and consumer, detailed", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + Consumer: "cons", + Details: true, + }, + }, + expected: HealthStatus{ + Status: "error", + StatusCode: 400, + Errors: []HealthzError{ + { + Type: HealthzErrorBadRequest, + Error: `"stream" must not be empty when checking consumer health`, + }, + }, + }, + }, + { + name: "account not found", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "abc", + }, + }, + expected: HealthStatus{ + Status: "unavailable", + StatusCode: 404, + Error: `JetStream account "abc" not found`, + }, + }, + { + name: "account not found, detailed", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "abc", + Details: true, + }, + }, + expected: HealthStatus{ + Status: "error", + StatusCode: 404, + Errors: []HealthzError{ + { + Type: HealthzErrorAccount, + Account: "abc", + Error: `JetStream account "abc" not found`, + }, + }, + }, + }, + { + name: "stream not found", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + Stream: "abc", + }, + }, + expected: HealthStatus{ + Status: "unavailable", + StatusCode: 404, + Error: `JetStream stream "abc" not found on account "ONE"`, + }, + }, + { + name: "stream not found, detailed", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + Stream: "abc", + Details: true, + }, + }, + expected: HealthStatus{ + Status: "error", + StatusCode: 404, + Errors: []HealthzError{ + { + Type: HealthzErrorStream, + Account: "ONE", + Stream: "abc", + Error: `JetStream stream "abc" not found on account "ONE"`, + }, + }, + }, + }, + { + name: "consumer not found", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + Stream: "test", + Consumer: "abc", + }, + }, + expected: HealthStatus{ + Status: "unavailable", + StatusCode: 404, + Error: `JetStream consumer "abc" not found for stream "test" on account "ONE"`, + }, + }, + { + name: "consumer not found, detailed", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + Stream: "test", + Consumer: "abc", + Details: true, + }, + }, + expected: HealthStatus{ + Status: "error", + StatusCode: 404, + Errors: []HealthzError{ + { + Type: HealthzErrorConsumer, + Account: "ONE", + Stream: "test", + Consumer: "abc", + Error: `JetStream consumer "abc" not found for stream "test" on account "ONE"`, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var body []byte + var err error + if test.req != nil { + body, err = json.Marshal(test.req) + if err != nil { + t.Fatalf("Error marshaling request body: %v", err) + } + } + msg, err := ncs.Request(subj, body, 1*time.Second) + if err != nil { + t.Fatalf("Error trying to request healthz: %v", err) + } + var health healthzResp + if err := json.Unmarshal(msg.Data, &health); err != nil { + t.Fatalf("Error unmarshalling the statz json: %v", err) + } + if !reflect.DeepEqual(health.Healthz, test.expected) { + t.Errorf("Invalid healthz status; want: %+v; got: %+v", test.expected, health.Healthz) + } + }) + } +} + +func TestServerEventsHealthZClustered(t *testing.T) { + type healthzResp struct { + Healthz HealthStatus `json:"data"` + Server ServerInfo `json:"server"` + } + serverHealthzReqSubj := "$SYS.REQ.SERVER.%s.HEALTHZ" + c := createJetStreamClusterWithTemplate(t, jsClusterAccountsTempl, "JSC", 3) + defer c.shutdown() + + ncs, err := nats.Connect(c.randomServer().ClientURL(), nats.UserInfo("admin", "s3cr3t!")) + if err != nil { + t.Fatalf("Error connecting to cluster: %v", err) + } + + defer ncs.Close() + ncAcc, err := nats.Connect(c.randomServer().ClientURL()) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncAcc.Close() + js, err := ncAcc.JetStream() + if err != nil { + t.Fatalf("Error creating JetStream context: %v", err) + } + _, err = js.AddStream(&nats.StreamConfig{ + Name: "test", + Subjects: []string{"foo"}, + Replicas: 3, + }) + if err != nil { + t.Fatalf("Error creating stream: %v", err) + } + _, err = js.AddConsumer("test", &nats.ConsumerConfig{ + Name: "cons", + Replicas: 3, + }) + if err != nil { + t.Fatalf("Error creating consumer: %v", err) + } + + c.waitOnStreamLeader("ONE", "test") + c.waitOnConsumerLeader("ONE", "test", "cons") + c.waitOnAllCurrent() + + subj := fmt.Sprintf(serverHealthzReqSubj, c.servers[0].ID()) + pingSubj := fmt.Sprintf(serverHealthzReqSubj, "PING") + + tests := []struct { + name string + req *HealthzEventOptions + expected HealthStatus + expectedError string + }{ + { + name: "no parameters", + expected: HealthStatus{Status: "ok", StatusCode: 200}, + }, + { + name: "with js enabled only", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + JSEnabledOnly: true, + }, + }, + expected: HealthStatus{Status: "ok", StatusCode: 200}, + }, + { + name: "with server only", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + JSServerOnly: true, + }, + }, + expected: HealthStatus{Status: "ok", StatusCode: 200}, + }, + { + name: "with account name", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + }, + }, + expected: HealthStatus{Status: "ok", StatusCode: 200}, + }, + { + name: "with account name and stream", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + Stream: "test", + }, + }, + expected: HealthStatus{Status: "ok", StatusCode: 200}, + }, + { + name: "with account name, stream and consumer", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + Stream: "test", + Consumer: "cons", + }, + }, + expected: HealthStatus{Status: "ok", StatusCode: 200}, + }, + { + name: "with stream only", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Stream: "test", + }, + }, + expected: HealthStatus{ + Status: "error", + StatusCode: 400, + Error: `"account" must not be empty when checking stream health`, + }, + expectedError: "Bad request:", + }, + { + name: "with stream only, detailed", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Details: true, + Stream: "test", + }, + }, + expected: HealthStatus{ + Status: "error", + StatusCode: 400, + Errors: []HealthzError{ + { + Type: HealthzErrorBadRequest, + Error: `"account" must not be empty when checking stream health`, + }, + }, + }, + }, + { + name: "account not found", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "abc", + }, + }, + expected: HealthStatus{ + Status: "unavailable", + StatusCode: 404, + Error: `JetStream account "abc" not found`, + }, + expectedError: `account "abc" not found`, + }, + { + name: "account not found, detailed", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "abc", + Details: true, + }, + }, + expected: HealthStatus{ + Status: "error", + StatusCode: 404, + Errors: []HealthzError{ + { + Type: HealthzErrorAccount, + Account: "abc", + Error: `JetStream account "abc" not found`, + }, + }, + }, + }, + { + name: "stream not found", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + Stream: "abc", + }, + }, + expected: HealthStatus{ + Status: "unavailable", + StatusCode: 404, + Error: `JetStream stream "abc" not found on account "ONE"`, + }, + expectedError: `stream "abc" not found`, + }, + { + name: "stream not found, detailed", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + Stream: "abc", + Details: true, + }, + }, + expected: HealthStatus{ + Status: "error", + StatusCode: 404, + Errors: []HealthzError{ + { + Type: HealthzErrorStream, + Account: "ONE", + Stream: "abc", + Error: `JetStream stream "abc" not found on account "ONE"`, + }, + }, + }, + }, + { + name: "consumer not found", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + Stream: "test", + Consumer: "abc", + }, + }, + expected: HealthStatus{ + Status: "unavailable", + StatusCode: 404, + Error: `JetStream consumer "abc" not found for stream "test" on account "ONE"`, + }, + expectedError: `consumer "abc" not found for stream "test"`, + }, + { + name: "consumer not found, detailed", + req: &HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + Stream: "test", + Consumer: "abc", + Details: true, + }, + }, + expected: HealthStatus{ + Status: "error", + StatusCode: 404, + Errors: []HealthzError{ + { + Type: HealthzErrorConsumer, + Account: "ONE", + Stream: "test", + Consumer: "abc", + Error: `JetStream consumer "abc" not found for stream "test" on account "ONE"`, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var body []byte + var err error + if test.req != nil { + body, err = json.Marshal(test.req) + if err != nil { + t.Fatalf("Error marshaling request body: %v", err) + } + } + msg, err := ncs.Request(subj, body, 1*time.Second) + if err != nil { + t.Fatalf("Error trying to request healthz: %v", err) + } + var health healthzResp + if err := json.Unmarshal(msg.Data, &health); err != nil { + t.Fatalf("Error unmarshalling the statz json: %v", err) + } + if !reflect.DeepEqual(health.Healthz, test.expected) { + t.Errorf("Invalid healthz status; want: %+v; got: %+v", test.expected, health.Healthz) + } + + reply := ncs.NewRespInbox() + sub, err := ncs.SubscribeSync(reply) + if err != nil { + t.Fatalf("Error creating subscription: %v", err) + } + defer sub.Unsubscribe() + + // now PING all servers + if err := ncs.PublishRequest(pingSubj, reply, body); err != nil { + t.Fatalf("Publish error: %v", err) + } + for i := 0; i < 3; i++ { + msg, err := sub.NextMsg(1 * time.Second) + if err != nil { + t.Fatalf("Error fetching healthz PING response: %v", err) + } + var health healthzResp + if err := json.Unmarshal(msg.Data, &health); err != nil { + t.Fatalf("Error unmarshalling the statz json: %v", err) + } + if !reflect.DeepEqual(health.Healthz, test.expected) { + t.Errorf("Invalid healthz status; want: %+v; got: %+v", test.expected, health.Healthz) + } + } + if _, err := sub.NextMsg(50 * time.Millisecond); !errors.Is(err, nats.ErrTimeout) { + t.Fatalf("Expected timeout error; got: %v", err) + } + }) + } +} + +func TestServerEventsHealthZClustered_NoReplicas(t *testing.T) { + type healthzResp struct { + Healthz HealthStatus `json:"data"` + Server ServerInfo `json:"server"` + } + serverHealthzReqSubj := "$SYS.REQ.SERVER.%s.HEALTHZ" + c := createJetStreamClusterWithTemplate(t, jsClusterAccountsTempl, "JSC", 3) + defer c.shutdown() + + ncs, err := nats.Connect(c.randomServer().ClientURL(), nats.UserInfo("admin", "s3cr3t!")) + if err != nil { + t.Fatalf("Error connecting to cluster: %v", err) + } + + defer ncs.Close() + ncAcc, err := nats.Connect(c.randomServer().ClientURL()) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncAcc.Close() + js, err := ncAcc.JetStream() + if err != nil { + t.Fatalf("Error creating JetStream context: %v", err) + } + + pingSubj := fmt.Sprintf(serverHealthzReqSubj, "PING") + + t.Run("non-replicated stream", func(t *testing.T) { + _, err = js.AddStream(&nats.StreamConfig{ + Name: "test", + Subjects: []string{"foo"}, + }) + if err != nil { + t.Fatalf("Error creating stream: %v", err) + } + _, err = js.AddConsumer("test", &nats.ConsumerConfig{ + Name: "cons", + }) + if err != nil { + t.Fatalf("Error creating consumer: %v", err) + } + body, err := json.Marshal(HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + Stream: "test", + }, + }) + if err != nil { + t.Fatalf("Error marshaling request body: %v", err) + } + + reply := ncs.NewRespInbox() + sub, err := ncs.SubscribeSync(reply) + if err != nil { + t.Fatalf("Error creating subscription: %v", err) + } + defer sub.Unsubscribe() + + // now PING all servers + if err := ncs.PublishRequest(pingSubj, reply, body); err != nil { + t.Fatalf("Publish error: %v", err) + } + var healthy int + for i := 0; i < 3; i++ { + msg, err := sub.NextMsg(1 * time.Second) + if err != nil { + t.Fatalf("Error fetching healthz PING response: %v", err) + } + var health healthzResp + if err := json.Unmarshal(msg.Data, &health); err != nil { + t.Fatalf("Error unmarshalling the statz json: %v", err) + } + if health.Healthz.Status == "ok" { + healthy++ + continue + } + if !strings.Contains(health.Healthz.Error, `stream "test" not found`) { + t.Errorf("Expected error to contain: %q, got: %s", `stream "test" not found`, health.Healthz.Error) + } + } + if healthy != 1 { + t.Fatalf("Expected 1 healthy server; got: %d", healthy) + } + if _, err := sub.NextMsg(50 * time.Millisecond); !errors.Is(err, nats.ErrTimeout) { + t.Fatalf("Expected timeout error; got: %v", err) + } + }) + + t.Run("non-replicated consumer", func(t *testing.T) { + _, err = js.AddStream(&nats.StreamConfig{ + Name: "test-repl", + Subjects: []string{"bar"}, + Replicas: 3, + }) + if err != nil { + t.Fatalf("Error creating stream: %v", err) + } + _, err = js.AddConsumer("test-repl", &nats.ConsumerConfig{ + Name: "cons-single", + }) + if err != nil { + t.Fatalf("Error creating consumer: %v", err) + } + body, err := json.Marshal(HealthzEventOptions{ + HealthzOptions: HealthzOptions{ + Account: "ONE", + Stream: "test-repl", + Consumer: "cons-single", + }, + }) + if err != nil { + t.Fatalf("Error marshaling request body: %v", err) + } + + reply := ncs.NewRespInbox() + sub, err := ncs.SubscribeSync(reply) + if err != nil { + t.Fatalf("Error creating subscription: %v", err) + } + defer sub.Unsubscribe() + + // now PING all servers + if err := ncs.PublishRequest(pingSubj, reply, body); err != nil { + t.Fatalf("Publish error: %v", err) + } + var healthy int + for i := 0; i < 3; i++ { + msg, err := sub.NextMsg(1 * time.Second) + if err != nil { + t.Fatalf("Error fetching healthz PING response: %v", err) + } + var health healthzResp + if err := json.Unmarshal(msg.Data, &health); err != nil { + t.Fatalf("Error unmarshalling the statz json: %v", err) + } + if health.Healthz.Status == "ok" { + healthy++ + continue + } + if !strings.Contains(health.Healthz.Error, `consumer "cons-single" not found`) { + t.Errorf("Expected error to contain: %q, got: %s", `consumer "cons-single" not found`, health.Healthz.Error) + } + } + if healthy != 1 { + t.Fatalf("Expected 1 healthy server; got: %d", healthy) + } + if _, err := sub.NextMsg(50 * time.Millisecond); !errors.Is(err, nats.ErrTimeout) { + t.Fatalf("Expected timeout error; got: %v", err) + } + }) + +} + +func TestServerEventsHealthZJetStreamNotEnabled(t *testing.T) { + type healthzResp struct { + Healthz HealthStatus `json:"data"` + Server ServerInfo `json:"server"` + } + cfg := `listen: 127.0.0.1:-1 + + accounts { + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + }` + + serverHealthzReqSubj := "$SYS.REQ.SERVER.%s.HEALTHZ" + s, _ := RunServerWithConfig(createConfFile(t, []byte(cfg))) + defer s.Shutdown() + + ncs, err := nats.Connect(s.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) + if err != nil { + t.Fatalf("Error connecting to cluster: %v", err) + } + + defer ncs.Close() + + subj := fmt.Sprintf(serverHealthzReqSubj, s.ID()) + + msg, err := ncs.Request(subj, nil, 1*time.Second) + if err != nil { + t.Fatalf("Error trying to request healthz: %v", err) + } + var health healthzResp + if err := json.Unmarshal(msg.Data, &health); err != nil { + t.Fatalf("Error unmarshalling the statz json: %v", err) + } + if health.Healthz.Status != "ok" { + t.Errorf("Invalid healthz status; want: %q; got: %q", "ok", health.Healthz.Status) + } + if health.Healthz.Error != "" { + t.Errorf("HealthZ error: %s", health.Healthz.Error) + } +} + +func TestServerEventsPingStatsZ(t *testing.T) { + sa, _, sb, optsB, akp := runTrustedCluster(t) + defer sa.Shutdown() + defer sb.Shutdown() + url := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) + nc, err := nats.Connect(url, createUserCreds(t, sb, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + test := func(req []byte) { + reply := nc.NewRespInbox() + sub, _ := nc.SubscribeSync(reply) + nc.PublishRequest(serverStatsPingReqSubj, reply, req) + // Make sure its a statsz + m := ServerStatsMsg{} + // Receive both manually. + msg, err := sub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Error receiving msg: %v", err) + } + if err := json.Unmarshal(msg.Data, &m); err != nil { + t.Fatalf("Error unmarshalling the statz json: %v", err) + } + msg, err = sub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Error receiving msg: %v", err) + } + if err := json.Unmarshal(msg.Data, &m); err != nil { + t.Fatalf("Error unmarshalling the statz json: %v", err) + } + } + strRequestTbl := []string{ + `{"cluster":"TEST"}`, + `{"cluster":"CLUSTER"}`, + `{"server_name":"SRV"}`, + `{"server_name":"_"}`, + fmt.Sprintf(`{"host":"%s"}`, optsB.Host), + fmt.Sprintf(`{"host":"%s", "cluster":"CLUSTER", "name":"SRV"}`, optsB.Host), + } + for i, opt := range strRequestTbl { + t.Run(fmt.Sprintf("%s-%d", t.Name(), i), func(t *testing.T) { + test([]byte(opt)) + }) + } + requestTbl := []StatszEventOptions{ + {EventFilterOptions: EventFilterOptions{Cluster: "TEST"}}, + {EventFilterOptions: EventFilterOptions{Cluster: "CLUSTER"}}, + {EventFilterOptions: EventFilterOptions{Name: "SRV"}}, + {EventFilterOptions: EventFilterOptions{Name: "_"}}, + {EventFilterOptions: EventFilterOptions{Host: optsB.Host}}, + {EventFilterOptions: EventFilterOptions{Host: optsB.Host, Cluster: "CLUSTER", Name: "SRV"}}, + } + for i, opt := range requestTbl { + t.Run(fmt.Sprintf("%s-%d", t.Name(), i), func(t *testing.T) { + msg, _ := json.MarshalIndent(&opt, "", " ") + test(msg) + }) + } +} + +func TestServerEventsPingStatsZDedicatedRecvQ(t *testing.T) { + sa, _, sb, optsB, akp := runTrustedCluster(t) + defer sa.Shutdown() + defer sb.Shutdown() + url := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) + nc, err := nats.Connect(url, createUserCreds(t, sb, akp)) + require_NoError(t, err) + defer nc.Close() + // We need to wait a little bit for the $SYS.SERVER.ACCOUNT.%s.CONNS + // event to be pushed in the mux'ed queue. + time.Sleep(300 * time.Millisecond) + + testReq := func(t *testing.T, subj string, expectTwo bool) { + for _, s := range []*Server{sa, sb} { + s.mu.RLock() + recvq := s.sys.recvq + s.mu.RUnlock() + recvq.Lock() + defer recvq.Unlock() + } + reply := nc.NewRespInbox() + sub := natsSubSync(t, nc, reply) + nc.PublishRequest(subj, reply, nil) + msg := natsNexMsg(t, sub, time.Second) + if len(msg.Data) == 0 { + t.Fatal("Unexpected empty response") + } + // Make sure its a statsz + m := ServerStatsMsg{} + err := json.Unmarshal(msg.Data, &m) + require_NoError(t, err) + require_False(t, m.Stats.Start.IsZero()) + if expectTwo { + msg = natsNexMsg(t, sub, time.Second) + err = json.Unmarshal(msg.Data, &m) + require_NoError(t, err) + require_False(t, m.Stats.Start.IsZero()) + } + } + const statsz = "STATSZ" + for _, test := range []struct { + name string + f func() string + expectTwo bool + }{ + {"server stats ping request subject", func() string { return serverStatsPingReqSubj }, true}, + {"server ping request subject", func() string { return fmt.Sprintf(serverPingReqSubj, statsz) }, true}, + {"server a direct request subject", func() string { return fmt.Sprintf(serverDirectReqSubj, sa.ID(), statsz) }, false}, + {"server b direct request subject", func() string { return fmt.Sprintf(serverDirectReqSubj, sb.ID(), statsz) }, false}, + } { + t.Run(test.name, func(t *testing.T) { + testReq(t, test.f(), test.expectTwo) + }) + } +} + +func TestServerEventsPingStatsZFilter(t *testing.T) { + sa, _, sb, optsB, akp := runTrustedCluster(t) + defer sa.Shutdown() + defer sb.Shutdown() + + url := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) + nc, err := nats.Connect(url, createUserCreds(t, sb, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + requestTbl := []string{ + `{"cluster":"DOESNOTEXIST"}`, + `{"host":"DOESNOTEXIST"}`, + `{"server_name":"DOESNOTEXIST"}`, + } + for i, msg := range requestTbl { + t.Run(fmt.Sprintf("%s-%d", t.Name(), i), func(t *testing.T) { + // Receive both manually. + if _, err := nc.Request(serverStatsPingReqSubj, []byte(msg), time.Second/4); err != nats.ErrTimeout { + t.Fatalf("Error, expected timeout: %v", err) + } + }) + } + requestObjTbl := []EventFilterOptions{ + {Cluster: "DOESNOTEXIST"}, + {Host: "DOESNOTEXIST"}, + {Name: "DOESNOTEXIST"}, + } + for i, opt := range requestObjTbl { + t.Run(fmt.Sprintf("%s-%d", t.Name(), i), func(t *testing.T) { + msg, _ := json.MarshalIndent(&opt, "", " ") + // Receive both manually. + if _, err := nc.Request(serverStatsPingReqSubj, []byte(msg), time.Second/4); err != nats.ErrTimeout { + t.Fatalf("Error, expected timeout: %v", err) + } + }) + } +} + +func TestServerEventsPingStatsZFailFilter(t *testing.T) { + sa, _, sb, optsB, akp := runTrustedCluster(t) + defer sa.Shutdown() + defer sb.Shutdown() + + url := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) + nc, err := nats.Connect(url, createUserCreds(t, sb, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // Receive both manually. + if msg, err := nc.Request(serverStatsPingReqSubj, []byte(`{MALFORMEDJSON`), time.Second/4); err != nil { + t.Fatalf("Error: %v", err) + } else { + resp := make(map[string]map[string]any) + if err := json.Unmarshal(msg.Data, &resp); err != nil { + t.Fatalf("Error unmarshalling the response json: %v", err) + } + if resp["error"]["code"].(float64) != http.StatusBadRequest { + t.Fatal("bad error code") + } + } +} + +func TestServerEventsPingMonitorz(t *testing.T) { + sa, _, sb, optsB, akp := runTrustedCluster(t) + defer sa.Shutdown() + defer sb.Shutdown() + sysAcc, _ := akp.PublicKey() + url := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) + nc, err := nats.Connect(url, createUserCreds(t, sb, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + nc.Flush() + + tests := []struct { + endpoint string + opt any + resp any + respField []string + }{ + {"VARZ", nil, &Varz{}, + []string{"now", "cpu", "system_account"}}, + {"SUBSZ", nil, &Subsz{}, + []string{"num_subscriptions", "num_cache"}}, + {"CONNZ", nil, &Connz{}, + []string{"now", "connections"}}, + {"ROUTEZ", nil, &Routez{}, + []string{"now", "routes"}}, + {"GATEWAYZ", nil, &Gatewayz{}, + []string{"now", "outbound_gateways", "inbound_gateways"}}, + {"LEAFZ", nil, &Leafz{}, + []string{"now", "leafs"}}, + + {"SUBSZ", &SubszOptions{}, &Subsz{}, + []string{"num_subscriptions", "num_cache"}}, + {"CONNZ", &ConnzOptions{}, &Connz{}, + []string{"now", "connections"}}, + {"ROUTEZ", &RoutezOptions{}, &Routez{}, + []string{"now", "routes"}}, + {"GATEWAYZ", &GatewayzOptions{}, &Gatewayz{}, + []string{"now", "outbound_gateways", "inbound_gateways"}}, + {"LEAFZ", &LeafzOptions{}, &Leafz{}, + []string{"now", "leafs"}}, + {"ACCOUNTZ", &AccountzOptions{}, &Accountz{}, + []string{"now", "accounts"}}, + + {"SUBSZ", &SubszOptions{Limit: 5}, &Subsz{}, + []string{"num_subscriptions", "num_cache"}}, + {"CONNZ", &ConnzOptions{Limit: 5}, &Connz{}, + []string{"now", "connections"}}, + {"ROUTEZ", &RoutezOptions{SubscriptionsDetail: true}, &Routez{}, + []string{"now", "routes"}}, + {"GATEWAYZ", &GatewayzOptions{Accounts: true}, &Gatewayz{}, + []string{"now", "outbound_gateways", "inbound_gateways"}}, + {"LEAFZ", &LeafzOptions{Subscriptions: true}, &Leafz{}, + []string{"now", "leafs"}}, + {"ACCOUNTZ", &AccountzOptions{Account: sysAcc}, &Accountz{}, + []string{"now", "account_detail"}}, + {"LEAFZ", &LeafzOptions{Account: sysAcc}, &Leafz{}, + []string{"now", "leafs"}}, + + {"ROUTEZ", json.RawMessage(`{"cluster":""}`), &Routez{}, + []string{"now", "routes"}}, + {"ROUTEZ", json.RawMessage(`{"name":""}`), &Routez{}, + []string{"now", "routes"}}, + {"ROUTEZ", json.RawMessage(`{"cluster":"TEST_CLUSTER_22"}`), &Routez{}, + []string{"now", "routes"}}, + {"ROUTEZ", json.RawMessage(`{"cluster":"CLUSTER"}`), &Routez{}, + []string{"now", "routes"}}, + {"ROUTEZ", json.RawMessage(`{"cluster":"TEST_CLUSTER_22", "subscriptions":true}`), &Routez{}, + []string{"now", "routes"}}, + + {"JSZ", nil, &JSzOptions{}, []string{"now", "disabled"}}, + + {"HEALTHZ", nil, &JSzOptions{}, []string{"status"}}, + {"HEALTHZ", &HealthzOptions{JSEnabledOnly: true}, &JSzOptions{}, []string{"status"}}, + {"HEALTHZ", &HealthzOptions{JSServerOnly: true}, &JSzOptions{}, []string{"status"}}, + {"HEALTHZ", &HealthzOptions{JSMetaOnly: true}, &JSzOptions{}, []string{"status"}}, + {"EXPVARZ", nil, &ExpvarzStatus{}, []string{"memstats", "cmdline"}}, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("%s-%d", test.endpoint, i), func(t *testing.T) { + var opt []byte + if test.opt != nil { + opt, err = json.Marshal(test.opt) + if err != nil { + t.Fatalf("Error marshaling opts: %v", err) + } + } + reply := nc.NewRespInbox() + replySubj, _ := nc.SubscribeSync(reply) + + // set a header to make sure request parsing knows to ignore them + nc.PublishMsg(&nats.Msg{ + Subject: fmt.Sprintf("%s.%s", serverStatsPingReqSubj, test.endpoint), + Reply: reply, + Header: nats.Header{"header": []string{"for header sake"}}, + Data: opt, + }) + + // Receive both manually. + msg, err := replySubj.NextMsg(time.Second) + if err != nil { + t.Fatalf("Error receiving msg: %v", err) + } + response1 := make(map[string]map[string]any) + + if err := json.Unmarshal(msg.Data, &response1); err != nil { + t.Fatalf("Error unmarshalling response1 json: %v", err) + } + + serverName := "" + if response1["server"]["name"] == "A_SRV" { + serverName = "B_SRV" + } else if response1["server"]["name"] == "B_SRV" { + serverName = "A_SRV" + } else { + t.Fatalf("Error finding server in %s", string(msg.Data)) + } + if resp, ok := response1["data"]; !ok { + t.Fatalf("Error finding: %s in %s", + strings.ToLower(test.endpoint), string(msg.Data)) + } else { + for _, respField := range test.respField { + if _, ok := resp[respField]; !ok { + t.Fatalf("Error finding: %s in %s", respField, resp) + } + } + } + + msg, err = replySubj.NextMsg(time.Second) + if err != nil { + t.Fatalf("Error receiving msg: %v", err) + } + response2 := make(map[string]map[string]any) + if err := json.Unmarshal(msg.Data, &response2); err != nil { + t.Fatalf("Error unmarshalling the response2 json: %v", err) + } + if response2["server"]["name"] != serverName { + t.Fatalf("Error finding server %s in %s", serverName, string(msg.Data)) + } + if resp, ok := response2["data"]; !ok { + t.Fatalf("Error finding: %s in %s", + strings.ToLower(test.endpoint), string(msg.Data)) + } else { + for _, respField := range test.respField { + if val, ok := resp[respField]; !ok { + t.Fatalf("Error finding: %s in %s", respField, resp) + } else if val == nil { + t.Fatalf("Nil value found: %s in %s", respField, resp) + } + } + } + }) + } +} + +func TestGatewayNameClientInfo(t *testing.T) { + sa, _, sb, _, _ := runTrustedCluster(t) + defer sa.Shutdown() + defer sb.Shutdown() + + c, _, l := newClientForServer(sa) + defer c.close() + + var info Info + err := json.Unmarshal([]byte(l[5:]), &info) + if err != nil { + t.Fatalf("Could not parse INFO json: %v\n", err) + } + if info.Cluster != "TEST_CLUSTER_22" { + t.Fatalf("Expected a cluster name of 'TEST_CLUSTER_22', got %q", info.Cluster) + } +} + +type slowAccResolver struct { + sync.Mutex + AccountResolver + acc string +} + +func (sr *slowAccResolver) Fetch(name string) (string, error) { + sr.Lock() + delay := sr.acc == name + sr.Unlock() + if delay { + time.Sleep(200 * time.Millisecond) + } + return sr.AccountResolver.Fetch(name) +} + +func TestConnectionUpdatesTimerProperlySet(t *testing.T) { + origEventsHBInterval := eventsHBInterval + eventsHBInterval = 50 * time.Millisecond + defer func() { eventsHBInterval = origEventsHBInterval }() + + sa, _, sb, optsB, _ := runTrustedCluster(t) + defer sa.Shutdown() + defer sb.Shutdown() + + // Normal Account + okp, _ := nkeys.FromSeed(oSeed) + akp, _ := nkeys.CreateAccount() + pub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(pub) + nac.Limits.Conn = 10 // set any limit... + jwt, _ := nac.Encode(okp) + + addAccountToMemResolver(sa, pub, jwt) + + // Listen for HB updates... + count := int32(0) + cb := func(sub *subscription, _ *client, _ *Account, subject, reply string, msg []byte) { + atomic.AddInt32(&count, 1) + } + subj := fmt.Sprintf(accConnsEventSubjNew, pub) + sub, err := sa.sysSubscribe(subj, cb) + if sub == nil || err != nil { + t.Fatalf("Expected to subscribe, got %v", err) + } + defer sa.sysUnsubscribe(sub) + + url := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) + nc := natsConnect(t, url, createUserCreds(t, sb, akp)) + defer nc.Close() + + time.Sleep(500 * time.Millisecond) + // After waiting 500ms with HB interval of 50ms, we should get + // about 10 updates, no much more + if n := atomic.LoadInt32(&count); n > 15 { + t.Fatalf("Expected about 10 updates, got %v", n) + } + + // Now lookup the account doing the events on sb. + acc, _ := sb.LookupAccount(pub) + // Make sure we have the timer running. + acc.mu.RLock() + ctmr := acc.ctmr + acc.mu.RUnlock() + if ctmr == nil { + t.Fatalf("Expected event timer for acc conns to be running") + } + + nc.Close() + + checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { + // Make sure we have the timer is NOT running. + acc.mu.RLock() + ctmr = acc.ctmr + acc.mu.RUnlock() + if ctmr != nil { + return fmt.Errorf("Expected event timer for acc conns to NOT be running after reaching zero local clients") + } + return nil + }) +} + +func TestServerEventsReceivedByQSubs(t *testing.T) { + s, opts := runTrustedServer(t) + defer s.Shutdown() + + acc, akp := createAccount(s) + s.setSystemAccount(acc) + + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + ncs, err := nats.Connect(url, createUserCreds(t, s, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncs.Close() + + // Listen for auth error events. + qsub, _ := ncs.QueueSubscribeSync("$SYS.SERVER.*.CLIENT.AUTH.ERR", "queue") + defer qsub.Unsubscribe() + + ncs.Flush() + + nats.Connect(url, nats.Name("TEST BAD LOGIN")) + + m, err := qsub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Should have heard an auth error event") + } + dem := DisconnectEventMsg{} + if err := json.Unmarshal(m.Data, &dem); err != nil { + t.Fatalf("Error unmarshalling disconnect event message: %v", err) + } + if dem.Reason != "Authentication Failure" { + t.Fatalf("Expected auth error, got %q", dem.Reason) + } +} + +func TestServerEventsFilteredByTag(t *testing.T) { + confA := createConfFile(t, []byte(` + listen: -1 + server_name: srv-A + server_tags: ["foo", "bar"] + cluster { + name: clust + listen: -1 + no_advertise: true + } + system_account: SYS + accounts: { + SYS: { + users: [ + {user: b, password: b} + ] + } + } + no_auth_user: b + `)) + sA, _ := RunServerWithConfig(confA) + defer sA.Shutdown() + confB := createConfFile(t, []byte(fmt.Sprintf(` + listen: -1 + server_name: srv-B + server_tags: ["bar", "baz"] + cluster { + name: clust + listen: -1 + no_advertise: true + routes [ + nats-route://127.0.0.1:%d + ] + } + system_account: SYS + accounts: { + SYS: { + users: [ + {user: b, password: b} + ] + } + } + no_auth_user: b + `, sA.opts.Cluster.Port))) + sB, _ := RunServerWithConfig(confB) + defer sB.Shutdown() + checkClusterFormed(t, sA, sB) + nc := natsConnect(t, sA.ClientURL()) + defer nc.Close() + + ib := nats.NewInbox() + req := func(tags ...string) { + t.Helper() + r, err := json.Marshal(VarzEventOptions{EventFilterOptions: EventFilterOptions{Tags: tags}}) + require_NoError(t, err) + err = nc.PublishRequest(fmt.Sprintf(serverPingReqSubj, "VARZ"), ib, r) + require_NoError(t, err) + } + + msgs := make(chan *nats.Msg, 10) + defer close(msgs) + _, err := nc.ChanSubscribe(ib, msgs) + require_NoError(t, err) + req("none") + select { + case <-msgs: + t.Fatalf("no message expected") + case <-time.After(200 * time.Millisecond): + } + req("foo") + m := <-msgs + require_Contains(t, string(m.Data), "srv-A", "foo", "bar") + req("foo", "bar") + m = <-msgs + require_Contains(t, string(m.Data), "srv-A", "foo", "bar") + req("baz") + m = <-msgs + require_Contains(t, string(m.Data), "srv-B", "bar", "baz") + req("bar") + m1 := <-msgs + m2 := <-msgs + require_Contains(t, string(m1.Data)+string(m2.Data), "srv-A", "srv-B", "foo", "bar", "baz") + require_Len(t, len(msgs), 0) +} + +func TestServerUnstableEventFilterMatch(t *testing.T) { + confA := createConfFile(t, []byte(` + listen: -1 + server_name: srv1 + server_tags: ["foo", "bar"] + cluster { + name: clust + listen: -1 + no_advertise: true + } + system_account: SYS + accounts: { + SYS: { + users: [ + {user: b, password: b} + ] + } + } + no_auth_user: b + `)) + sA, _ := RunServerWithConfig(confA) + defer sA.Shutdown() + confB := createConfFile(t, []byte(fmt.Sprintf(` + listen: -1 + server_name: srv10 + server_tags: ["bar", "baz"] + cluster { + name: clust + listen: -1 + no_advertise: true + routes [ + nats-route://127.0.0.1:%d + ] + } + system_account: SYS + accounts: { + SYS: { + users: [ + {user: b, password: b} + ] + } + } + no_auth_user: b + `, sA.opts.Cluster.Port))) + sB, _ := RunServerWithConfig(confB) + defer sB.Shutdown() + checkClusterFormed(t, sA, sB) + + tester := func(t *testing.T, nc *nats.Conn, name string, count int) { + t.Helper() + + r, err := json.Marshal(VarzEventOptions{EventFilterOptions: EventFilterOptions{Name: name, ExactMatch: true}}) + require_NoError(t, err) + + for i := 0; i < count; i++ { + res, err := nc.Request(fmt.Sprintf(serverPingReqSubj, "VARZ"), r, time.Second) + require_NoError(t, err) + + var vz ServerAPIVarzResponse + err = json.Unmarshal(res.Data, &vz) + require_NoError(t, err) + + if vz.Server.Name != name { + t.Fatalf("Expected server name to be %q, got %q", name, vz.Server.Name) + } + } + } + + // Connects to srv1 so it's most likely to respond first while we are asking srv10 to respond. + nc := natsConnect(t, sA.ClientURL()) + defer nc.Close() + tester(t, nc, "srv10", 10) + + // Connects to srv10 so it's most likely to respond first while we are asking srv1 to respond. + nc.Close() + nc = natsConnect(t, sB.ClientURL()) + defer nc.Close() + tester(t, nc, "srv1", 10) +} + +// https://github.com/nats-io/nats-server/issues/3177 +func TestServerEventsAndDQSubscribers(t *testing.T) { + c := createJetStreamClusterWithTemplate(t, jsClusterAccountsTempl, "DDQ", 3) + defer c.shutdown() + + nc, err := nats.Connect(c.randomServer().ClientURL(), nats.UserInfo("admin", "s3cr3t!")) + require_NoError(t, err) + defer nc.Close() + + sub, err := nc.QueueSubscribeSync("$SYS.ACCOUNT.*.DISCONNECT", "qq") + require_NoError(t, err) + nc.Flush() + + // Create and disconnect 10 random connections. + for i := 0; i < 10; i++ { + nc, err := nats.Connect(c.randomServer().ClientURL()) + require_NoError(t, err) + nc.Close() + } + + checkSubsPending(t, sub, 10) +} + +func TestServerEventsStatszSingleServer(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: "127.0.0.1:-1" + accounts { $SYS { users [{user: "admin", password: "p1d"}]} } + `)) + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + // Grab internal system client. + s.mu.RLock() + sysc := s.sys.client + wait := s.sys.cstatsz + 25*time.Millisecond + s.mu.RUnlock() + + // Wait for when first statsz would have gone out.. + time.Sleep(wait) + + sysc.mu.Lock() + outMsgs := sysc.stats.outMsgs + sysc.mu.Unlock() + + require_True(t, outMsgs == 0) + + // Connect as a system user and make sure if there is + // subscription interest that we will receive updates. + nc, _ := jsClientConnect(t, s, nats.UserInfo("admin", "p1d")) + defer nc.Close() + + sub, err := nc.SubscribeSync(fmt.Sprintf(serverStatsSubj, "*")) + require_NoError(t, err) + + checkSubsPending(t, sub, 1) +} + +func TestServerEventsReload(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: "127.0.0.1:-1" + accounts: { + $SYS { users [{user: "admin", password: "p1d"}]} + test { users [{user: "foo", password: "bar"}]} + } + ping_interval: "100ms" + `)) + opts := LoadConfig(conf) + s := RunServer(opts) + defer s.Shutdown() + subject := fmt.Sprintf(serverReloadReqSubj, s.info.ID) + + // Connect as a test user and make sure the reload endpoint is not + // accessible. + ncTest, _ := jsClientConnect(t, s, nats.UserInfo("foo", "bar")) + defer ncTest.Close() + testReply := ncTest.NewRespInbox() + sub, err := ncTest.SubscribeSync(testReply) + require_NoError(t, err) + err = ncTest.PublishRequest(subject, testReply, nil) + require_NoError(t, err) + _, err = sub.NextMsg(time.Second) + require_Error(t, err) + + require_True(t, s.getOpts().PingInterval == 100*time.Millisecond) + + // Connect as a system user. + nc, _ := jsClientConnect(t, s, nats.UserInfo("admin", "p1d")) + defer nc.Close() + + // rewrite the config file with a different ping interval + err = os.WriteFile(conf, []byte(` + listen: "127.0.0.1:-1" + accounts: { + $SYS { users [{user: "admin", password: "p1d"}]} + test { users [{user: "foo", password: "bar"}]} + } + ping_interval: "200ms" + `), 0666) + require_NoError(t, err) + + msg, err := nc.Request(subject, nil, time.Second) + require_NoError(t, err) + + var apiResp = ServerAPIResponse{} + err = json.Unmarshal(msg.Data, &apiResp) + require_NoError(t, err) + + require_True(t, apiResp.Data == nil) + require_True(t, apiResp.Error == nil) + + // See that the ping interval has changed. + require_True(t, s.getOpts().PingInterval == 200*time.Millisecond) + + // rewrite the config file with a different ping interval + err = os.WriteFile(conf, []byte(`garbage and nonsense`), 0666) + require_NoError(t, err) + + // Request the server to reload and wait for the response. + msg, err = nc.Request(subject, nil, time.Second) + require_NoError(t, err) + + apiResp = ServerAPIResponse{} + err = json.Unmarshal(msg.Data, &apiResp) + require_NoError(t, err) + + require_True(t, apiResp.Data == nil) + require_Error(t, apiResp.Error, fmt.Errorf("Parse error on line 1: 'Expected a top-level value to end with a new line, comment or EOF, but got 'n' instead.'")) + + // See that the ping interval has not changed. + require_True(t, s.getOpts().PingInterval == 200*time.Millisecond) +} + +func TestServerEventsLDMKick(t *testing.T) { + ldmed := make(chan bool, 1) + disconnected := make(chan bool, 1) + + s, opts := runTrustedServer(t) + defer s.Shutdown() + + acc, akp := createAccount(s) + s.setSystemAccount(acc) + + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + ncs, err := nats.Connect(url, createUserCreds(t, s, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncs.Close() + + _, akp2 := createAccount(s) + + nc, err := nats.Connect(url, createUserCreds(t, s, akp2), nats.Name("TEST EVENTS LDM+KICK"), nats.LameDuckModeHandler(func(_ *nats.Conn) { ldmed <- true })) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + nc.SetDisconnectErrHandler(func(_ *nats.Conn, err error) { disconnected <- true }) + + cid, err := nc.GetClientID() + if err != nil { + t.Fatalf("Error on getting the CID: %v", err) + } + + reqldm := LDMClientReq{CID: cid} + reqldmpayload, _ := json.Marshal(reqldm) + reqkick := KickClientReq{CID: cid} + reqkickpayload, _ := json.Marshal(reqkick) + + // Test for data races when getting the client by ID + uc := createUserCreds(t, s, akp2) + totalClients := 100 + someClients := make([]*nats.Conn, 0, totalClients) + for i := 0; i < totalClients; i++ { + nc, err := nats.Connect(s.ClientURL(), uc) + require_NoError(t, err) + defer nc.Close() + someClients = append(someClients, nc) + } + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < totalClients; i++ { + someClients[i].Close() + } + }() + defer wg.Wait() + + _, err = ncs.Request(fmt.Sprintf("$SYS.REQ.SERVER.%s.LDM", s.ID()), reqldmpayload, time.Second) + require_NoError(t, err) + + select { + case <-ldmed: + case <-time.After(time.Second): + t.Fatalf("timeout waiting for the connection to receive the LDM signal") + } + + _, err = ncs.Request(fmt.Sprintf("$SYS.REQ.SERVER.%s.KICK", s.ID()), reqkickpayload, time.Second) + require_NoError(t, err) + + select { + case <-disconnected: + case <-time.After(time.Second): + t.Fatalf("timeout waiting for the client to get disconnected") + } +} + +func Benchmark_GetHash(b *testing.B) { + b.StopTimer() + // Get 100 random names + names := make([]string, 0, 100) + for i := 0; i < 100; i++ { + names = append(names, nuid.Next()) + } + hashes := make([]string, 0, 100) + for j := 0; j < 100; j++ { + sha := sha256.New() + sha.Write([]byte(names[j])) + b := sha.Sum(nil) + for i := 0; i < 8; i++ { + b[i] = digits[int(b[i]%base)] + } + hashes = append(hashes, string(b[:8])) + } + wg := sync.WaitGroup{} + wg.Add(8) + errCh := make(chan error, 8) + b.StartTimer() + for i := 0; i < 8; i++ { + go func() { + defer wg.Done() + for i := 0; i < b.N; i++ { + idx := rand.IntN(100) + if h := getHash(names[idx]); h != hashes[idx] { + errCh <- fmt.Errorf("Hash for name %q was %q, but should be %q", names[idx], h, hashes[idx]) + return + } + } + }() + } + wg.Wait() + select { + case err := <-errCh: + b.Fatal(err.Error()) + default: + } +} + +func TestClusterSetupMsgs(t *testing.T) { + // Tests will set this general faster, but here we want default for production. + original := statszRateLimit + statszRateLimit = defaultStatszRateLimit + defer func() { statszRateLimit = original }() + + numServers := 10 + c := createClusterEx(t, false, 0, false, "cluster", numServers) + defer shutdownCluster(c) + + checkFor(t, 3*time.Second, 500*time.Millisecond, func() error { + var totalOut int + for _, server := range c.servers { + totalOut += int(atomic.LoadInt64(&server.outMsgs)) + } + totalExpected := numServers * numServers + if totalOut >= totalExpected { + return fmt.Errorf("Total outMsgs is %d, expected < %d\n", totalOut, totalExpected) + } + return nil + }) +} + +func TestServerEventsProfileZNotBlockingRecvQ(t *testing.T) { + sa, _, sb, optsB, akp := runTrustedCluster(t) + defer sa.Shutdown() + defer sb.Shutdown() + // For this test, we will run a single server because the profiling + // would fail for the second server since it would detect that + // one profiling is already running (2 servers, but same process). + sa.Shutdown() + url := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) + nc, err := nats.Connect(url, createUserCreds(t, sb, akp)) + require_NoError(t, err) + defer nc.Close() + // We need to wait a little bit for the $SYS.SERVER.ACCOUNT.%s.CONNS + // event to be pushed in the mux'ed queue. + time.Sleep(300 * time.Millisecond) + + po := ProfilezOptions{Name: "cpu", Duration: 1 * time.Second} + req, err := json.Marshal(po) + require_NoError(t, err) + + testReq := func(t *testing.T, subj string) { + // Block the recvQ by locking it for the duration of this test. + sb.mu.RLock() + recvq := sb.sys.recvq + sb.mu.RUnlock() + recvq.Lock() + defer recvq.Unlock() + + // Send the profilez request on the given subject. + reply := nc.NewRespInbox() + sub := natsSubSync(t, nc, reply) + nc.PublishRequest(subj, reply, req) + msg := natsNexMsg(t, sub, 10*time.Second) + if len(msg.Data) == 0 { + t.Fatal("Unexpected empty response") + } + // Make sure its a ServerAPIResponse + resp := ServerAPIResponse{Data: &ProfilezStatus{}} + err := json.Unmarshal(msg.Data, &resp) + require_NoError(t, err) + // Check profile status to make sure that we got something. + ps := resp.Data.(*ProfilezStatus) + if ps.Error != _EMPTY_ { + t.Fatalf("%s", ps.Error) + } + require_True(t, len(ps.Profile) > 0) + } + const profilez = "PROFILEZ" + for _, test := range []struct { + name string + f func() string + }{ + {"server profilez request subject", func() string { return fmt.Sprintf(serverPingReqSubj, profilez) }}, + {"server direct request subject", func() string { return fmt.Sprintf(serverDirectReqSubj, sb.ID(), profilez) }}, + } { + t.Run(test.name, func(t *testing.T) { + testReq(t, test.f()) + }) + } +} + +func TestServerEventsStatsZJetStreamApiLevel(t *testing.T) { + s, opts := runTrustedServer(t) + defer s.Shutdown() + + acc, akp := createAccount(s) + s.setSystemAccount(acc) + s.EnableJetStream(&JetStreamConfig{StoreDir: t.TempDir()}) + + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + ncs, err := nats.Connect(url, createUserCreds(t, s, akp)) + require_NoError(t, err) + defer ncs.Close() + + msg, err := ncs.Request("$SYS.REQ.SERVER.PING.STATSZ", nil, time.Second) + require_NoError(t, err) + + var stats ServerStatsMsg + err = json.Unmarshal(msg.Data, &stats) + require_NoError(t, err) + + require_Equal(t, stats.Stats.JetStream.Stats.API.Level, JSApiLevel) +} + +func TestServerEventsPingStatsSlowConsumersStats(t *testing.T) { + s, _ := runTrustedServer(t) + defer s.Shutdown() + + acc, akp := createAccount(s) + s.setSystemAccount(acc) + ncs, err := nats.Connect(s.ClientURL(), createUserCreds(t, s, akp)) + require_NoError(t, err) + defer ncs.Close() + + const statsz = "STATSZ" + for _, test := range []struct { + name string + f func() string + expectTwo bool + }{ + {"server stats ping request subject", func() string { return serverStatsPingReqSubj }, true}, + {"server ping request subject", func() string { return fmt.Sprintf(serverPingReqSubj, statsz) }, true}, + {"server direct request subject", func() string { return fmt.Sprintf(serverDirectReqSubj, s.ID(), statsz) }, false}, + } { + t.Run(test.name, func(t *testing.T) { + // Clear all slow consumers values + s.scStats.clients.Store(0) + s.scStats.routes.Store(0) + s.scStats.gateways.Store(0) + s.scStats.leafs.Store(0) + + msg, err := ncs.Request(test.f(), nil, time.Second) + require_NoError(t, err) + + var ssm ServerStatsMsg + err = json.Unmarshal(msg.Data, &ssm) + require_NoError(t, err) + + // No slow consumer stats, so should be nil + require_True(t, ssm.Stats.SlowConsumersStats == nil) + + // Now set some values + s.scStats.clients.Store(1) + s.scStats.routes.Store(2) + s.scStats.gateways.Store(3) + s.scStats.leafs.Store(4) + + msg, err = ncs.Request(test.f(), nil, time.Second) + require_NoError(t, err) + + ssm = ServerStatsMsg{} + err = json.Unmarshal(msg.Data, &ssm) + require_NoError(t, err) + + require_NotNil(t, ssm.Stats.SlowConsumersStats) + scs := ssm.Stats.SlowConsumersStats + require_Equal(t, scs.Clients, 1) + require_Equal(t, scs.Routes, 2) + require_Equal(t, scs.Gateways, 3) + require_Equal(t, scs.Leafs, 4) + }) + } +} + +func TestServerEventsPingStatsStaleConnectionStats(t *testing.T) { + templ := ` + listen: "127.0.0.1:-1" + system_account = sys + accounts { + a { + users = [{ user: a, pass: a }] + } + b { + users = [{ user: b, pass: b }] + } + sys { + users = [{ user: sys, pass: sys }] + } + } + ` + conf := createConfFile(t, []byte(templ)) + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + // 3 different ways to get statz: + const statsz = "STATSZ" + + // $SYS.REQ.SERVER.NCBT6MBA7Q7ZF4R4WVXTSZTCZDPPXX2ALLN3XM75VCBCNMPIKQFPFLKV.STATSZ + dirReqSubject := func(s *Server) string { + return fmt.Sprintf(serverDirectReqSubj, s.ID(), statsz) + } + // $SYS.REQ.SERVER.PING.STATSZ + pingReqSubject := func(s *Server) string { + return fmt.Sprintf(serverPingReqSubj, statsz) + } + // $SYS.REQ.SERVER.PING + statsPingSubject := func(s *Server) string { + return serverStatsPingReqSubj + } + subjects := []string{dirReqSubject(s), pingReqSubject(s), statsPingSubject(s)} + + ncs, err := nats.Connect(s.ClientURL(), nats.UserInfo("sys", "sys")) + require_NoError(t, err) + + for _, subject := range subjects { + msg, err := ncs.Request(subject, nil, time.Second) + require_NoError(t, err) + + var ssm ServerStatsMsg + err = json.Unmarshal(msg.Data, &ssm) + require_NoError(t, err) + + // No stale connection stats. + require_True(t, ssm.Stats.StaleConnectionStats == nil) + require_Equal(t, s.NumStaleConnections(), int64(0)) + require_Equal(t, s.NumStaleConnectionsClients(), uint64(0)) + require_Equal(t, s.NumStaleConnectionsRoutes(), uint64(0)) + require_Equal(t, s.NumStaleConnectionsGateways(), uint64(0)) + require_Equal(t, s.NumStaleConnectionsLeafs(), uint64(0)) + } + + // Set some values and confirm. + s.staleStats.clients.Store(1) + s.staleStats.routes.Store(2) + s.staleStats.gateways.Store(3) + s.staleStats.leafs.Store(4) + atomic.StoreInt64(&s.staleConnections, 10) + require_Equal(t, s.NumStaleConnections(), int64(10)) + require_Equal(t, s.NumStaleConnectionsClients(), uint64(1)) + require_Equal(t, s.NumStaleConnectionsRoutes(), uint64(2)) + require_Equal(t, s.NumStaleConnectionsGateways(), uint64(3)) + require_Equal(t, s.NumStaleConnectionsLeafs(), uint64(4)) + + for _, subject := range subjects { + msg, err := ncs.Request(subject, nil, time.Second) + require_NoError(t, err) + + ssm := ServerStatsMsg{} + err = json.Unmarshal(msg.Data, &ssm) + require_NoError(t, err) + + require_NotNil(t, ssm.Stats.StaleConnectionStats) + stcs := ssm.Stats.StaleConnectionStats + require_Equal(t, stcs.Clients, 1) + require_Equal(t, stcs.Routes, 2) + require_Equal(t, stcs.Gateways, 3) + require_Equal(t, stcs.Leafs, 4) + require_Equal(t, ssm.Stats.StaleConnections, int64(10)) + } +} + +func TestServerEventsStatszMaxProcsMemLimit(t *testing.T) { + // We want to prove that our set values are reflected in STATSZ, + // so we can't use constants that might match the system that + // the test is run on. + omp, omm := runtime.GOMAXPROCS(-1), debug.SetMemoryLimit(-1) + mp, mm := runtime.GOMAXPROCS(omp*2)*2, debug.SetMemoryLimit(omm/2)/2 + + // When we're done, put everything back. + defer runtime.GOMAXPROCS(omp) + defer debug.SetMemoryLimit(omm) + + s, opts := runTrustedServer(t) + defer s.Shutdown() + + acc, akp := createAccount(s) + s.setSystemAccount(acc) + + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + ncs, err := nats.Connect(url, createUserCreds(t, s, akp)) + require_NoError(t, err) + defer ncs.Close() + + msg, err := ncs.Request("$SYS.REQ.SERVER.PING.STATSZ", nil, time.Second) + require_NoError(t, err) + + var stats ServerStatsMsg + require_NoError(t, json.Unmarshal(msg.Data, &stats)) + require_Equal(t, stats.Stats.MaxProcs, mp) + require_Equal(t, stats.Stats.MemLimit, mm) +} + +func TestSubszPagination(t *testing.T) { + type subszResp struct { + Subsz Subsz `json:"data"` + Server ServerInfo `json:"server"` + } + s, opts := runTrustedServer(t) + defer s.Shutdown() + + sysAcc, sysAkp := createAccount(s) + s.setSystemAccount(sysAcc) + + acc, akp := createAccount(s) + + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + ncSys, err := nats.Connect(url, createUserCreds(t, s, sysAkp)) + require_NoError(t, err) + defer ncSys.Close() + + nc, err := nats.Connect(url, createUserCreds(t, s, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // Create 100 subscriptions. + for i := range 100 { + nc.Subscribe(fmt.Sprintf("foo.%d", i), func(_ *nats.Msg) {}) + } + + time.Sleep(100 * time.Millisecond) + + reqSubject := fmt.Sprintf(accDirectReqSubj, acc.Name, "SUBSZ") + + // Request the first page. + subszReq := SubszOptions{Subscriptions: true, Limit: 10} + req, _ := json.Marshal(subszReq) + msg, err := ncSys.Request(reqSubject, req, time.Second) + require_NoError(t, err) + + var subsz subszResp + require_NoError(t, json.Unmarshal(msg.Data, &subsz)) + require_Equal(t, len(subsz.Subsz.Subs), 10) + + // we cannot check for equality since we have to account for the monitoring subscriptions + if subsz.Subsz.Total < 100 || subsz.Subsz.Total > 110 { + t.Fatalf("Expected total subscriptions to be more than 100 and less than 110, got %d", subsz.Subsz.Total) + } + + // Now test with a sub filter + + // create 10 subs on "bar.*" + for range 10 { + nc.Subscribe("bar.*", func(_ *nats.Msg) {}) + } + + time.Sleep(100 * time.Millisecond) + + subszReq = SubszOptions{Subscriptions: true, Limit: 5, Test: "bar.A"} + req, _ = json.Marshal(subszReq) + msg, err = ncSys.Request(reqSubject, req, time.Second) + require_NoError(t, err) + + var subszFiltered subszResp + require_NoError(t, json.Unmarshal(msg.Data, &subszFiltered)) + require_Equal(t, len(subszFiltered.Subsz.Subs), 5) + require_Equal(t, subszFiltered.Subsz.Total, 10) +} + +func TestServerEventsConnectDisconnectForGlobalAcc(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: "127.0.0.1:-1" + accounts { + $SYS { + users [{user: "admin", password: "pwd"}] + } + } + `)) + defer os.Remove(conf) + + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + ncs, err := nats.Connect(url, nats.UserInfo("admin", "pwd")) + require_NoError(t, err) + defer ncs.Close() + + s1, err := ncs.SubscribeSync(fmt.Sprintf(connectEventSubj, globalAccountName)) + require_NoError(t, err) + s2, err := ncs.SubscribeSync(fmt.Sprintf(disconnectEventSubj, globalAccountName)) + require_NoError(t, err) + + // Flush to make sure subscriptions are established + require_NoError(t, ncs.Flush()) + + // Connect to global account + ncg, err := nats.Connect(url, nats.UserInfo("", "")) + require_NoError(t, err) + + // System account should get a connect event + msg, err := s1.NextMsg(5 * time.Second) + require_NoError(t, err) + require_Equal(t, msg.Subject, fmt.Sprintf(connectEventSubj, globalAccountName)) + + // Disconnect from global account + ncg.Close() + + // System account should get a disconnect event + msg, err = s2.NextMsg(5 * time.Second) + require_NoError(t, err) + require_Equal(t, msg.Subject, fmt.Sprintf(disconnectEventSubj, globalAccountName)) +} + +func TestSendInternalAccountMsgWithReplyDoesNotMutateHeaderSlice(t *testing.T) { + s := RunServer(DefaultOptions()) + defer s.Shutdown() + + nc, err := nats.Connect(s.ClientURL()) + require_NoError(t, err) + defer nc.Close() + + sub, err := nc.SubscribeSync("foo") + require_NoError(t, err) + require_NoError(t, nc.Flush()) + + // Build a header slice with significant spare capacity, like callers + // constructing headers with append/genHeader naturally do. + hdr := make([]byte, 0, 256) + hdr = append(hdr, "NATS/1.0\r\nFoo: bar\r\n\r\n"...) + + // Snapshot the entire backing array so we can detect any mutation, + // including writes past len(hdr) into spare capacity. + snapshot := make([]byte, cap(hdr)) + copy(snapshot, hdr[:cap(hdr)]) + + body := []byte("REPRO_LEAK_BODY") + err = s.sendInternalAccountMsgWithReply(s.globalAccount(), "foo", _EMPTY_, hdr, body, false) + require_NoError(t, err) + + msg, err := sub.NextMsg(2 * time.Second) + require_NoError(t, err) + require_Equal(t, string(msg.Data), "REPRO_LEAK_BODY") + require_Equal(t, msg.Header.Get("Foo"), "bar") + + // The caller's hdr slice (and its full backing array) must be untouched. + got := hdr[:cap(hdr)] + if !bytes.Equal(got, snapshot) { + t.Fatalf("hdr backing array was mutated by send:\n want: %q\n got: %q", snapshot, got) + } +} + +func TestEventsResetChSendNoPanicAfterShutdownEventing(t *testing.T) { + s := RunServer(DefaultOptions()) + defer s.Shutdown() + + // Simulate a sender that grabbed the channel reference before it was nil-ed, + // exactly as the production senders do under s.mu. + s.mu.Lock() + if s.sys == nil || s.sys.resetCh == nil { + s.mu.Unlock() + t.Fatal("expected system eventing to be running") + } + rc := s.sys.resetCh + s.mu.Unlock() + + // Drive eventing shutdown to completion. With the old code this closed rc. + s.shutdownEventing() + + // Stand in for the (now exited) internal send loop so that, on a healthy + // open channel, the send can complete. + go func() { <-rc }() + + // Perform the send using the same select-against-quitCh pattern as the + // production senders. On the buggy code rc is closed and this panics. + done := make(chan any, 1) + go func() { + defer func() { done <- recover() }() + select { + case rc <- struct{}{}: + case <-s.quitCh: + } + }() + + select { + case r := <-done: + if r != nil { + t.Fatalf("send on resetCh panicked during shutdown: %v", r) + } + case <-time.After(5 * time.Second): + t.Fatal("send on resetCh blocked unexpectedly") + } +} + +func TestEventsRemoteLatencyUpdateNilLatency(t *testing.T) { + s, _ := runTrustedServer(t) + defer s.Shutdown() + + // Enable internal events by setting a system account. + sacc, _ := createAccount(s) + require_NoError(t, s.setSystemAccount(sacc)) + + // Target account that will own the tracked response serviceImport. + acc, _ := createAccount(s) + + // Construct the offending state directly: a response serviceImport with a + // .T-suffixed tracked reply, tracking enabled, but a nil latency. + reply := "_R_.abcdef.T" + si := &serviceImport{ + acc: acc, + from: reply, + to: reply, + response: true, + tracking: true, // tracking on, but... + latency: nil, // ...no latency object -> nil deref at si.latency.subject + } + acc.mu.Lock() + if acc.exports.responses == nil { + acc.exports.responses = make(map[string]*serviceImport) + } + acc.exports.responses[reply] = si + acc.mu.Unlock() + + // Craft a remote latency measurement that resolves to this serviceImport. + rl := remoteLatency{ + Account: acc.Name, + ReqId: reply, + } + msg, err := json.Marshal(&rl) + require_NoError(t, err) + + // On unfixed code this panics with a nil pointer dereference on + // si.latency.subject. With the fix it returns cleanly. + var recovered any + func() { + defer func() { + if r := recover(); r != nil { + recovered = r + // The panic occurred while holding acc.mu.RLock() (taken just + // before the offending deref and released just after). Release + // it so the deferred s.Shutdown does not deadlock. + acc.mu.RUnlock() + } + }() + s.remoteLatencyUpdate(nil, nil, nil, "$SYS.LATENCY.M2."+reply, "", nil, msg) + }() + + if recovered != nil { + t.Fatalf("remoteLatencyUpdate panicked on nil si.latency: %v", recovered) + } +} diff --git a/server/feature_flags.go b/server/feature_flags.go new file mode 100644 index 0000000..6121438 --- /dev/null +++ b/server/feature_flags.go @@ -0,0 +1,157 @@ +// Copyright 2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "maps" + "slices" + "strings" +) + +const ( + FeatureFlagJsAckFormatV2 = "js_ack_fc_v2" + FeatureFlagJsAPIReplyFormatV2 = "js_api_reply_v2" + FeatureFlagJsRaftDeleteRange = "js_raft_delete_range" + FeatureFlagJsSnapshotSources = "js_snapshot_sources" +) + +var featureFlags = map[string]bool{ + // Use v2 format for `$JS.ACK.>` and `$JS.FC.>`. + // - Introduced: 2.14.0, both v1 and v2 supported, only using v1. + // - Enabled: TBD, both supported, v2 becomes the default. + // + // - v1: $JS.ACK....... + // - v2: $JS.ACK......... + // See also: https://github.com/nats-io/nats-architecture-and-design/blob/main/adr/ADR-15.md#jsack + FeatureFlagJsAckFormatV2: false, + + // Use a scoped v2 format for replies to internally-issued JetStream API + // requests. This permits a leafnode to authorize only the replies for a + // specific remote stream/consumer instead of every $JSC.R reply in an + // account. + // + // - v1: $JSC.R. + // - v2: $JSC.R..... + // + // The reply subject is opaque to the remote API server, so this is safe to + // enable independently on the server issuing the request. + FeatureFlagJsAPIReplyFormatV2: false, + + // Propose delete range gaps as a single `deleteRangeOp` Raft append entry + // instead of one entry per deleted sequence. Dramatically reduces Raft cost + // on mirrors whose origin has a large number of interior deletes. + // - Introduced: 2.14.0, apply-side always supports receiving `deleteRangeOp`. + // - Enabled: 2.15.0, when upgrading all servers should be 2.14.0+ + // - To be removed in 2.16.0 + // + // WARNING: Only enable once every peer in the cluster is on a version that + // supports receiving `deleteRangeOp`. Older peers panic on apply of an + // unknown stream entry operation. + FeatureFlagJsRaftDeleteRange: true, + + // Include the stream's sourcing state in the replicated stream snapshot. + // Unlike other per-message derived state, it outlives the messages it was + // collected from, so a replica that caught up after those messages were + // removed can't derive it locally and would resume sourcing from an earlier + // position once it becomes leader. + // - Introduced: 2.15.0, both encoding versions are accepted, only emitting v1. + // - Enabled: TBD, once all supported versions accept v2. + // + // WARNING: Only enable once every peer in the cluster is on a version that + // accepts the v2 encoding. Older peers reject the snapshot outright. + FeatureFlagJsSnapshotSources: false, +} + +// getFeatureFlag is used to retrieve either the default or overwritten value for a feature flag. +// The user's value takes precedence over the system's default. However, if the flag doesn't exist, it's disabled. +// The *Options returned by Server.getOpts() is treated as immutable, mutations go through setOpts, +// so no lock is required on the map read here. +func (o *Options) getFeatureFlag(k string) bool { + defaultValue, ok := featureFlags[k] + if !ok { + return false // Not supported. + } + if userValue, ok := o.FeatureFlags[k]; ok { + return userValue + } + return defaultValue +} + +// getMergedFeatureFlags returns a merged map of feature flags, with the user's values taking precedence. +func (o *Options) getMergedFeatureFlags() map[string]bool { + merged := make(map[string]bool) + for k, v := range featureFlags { + merged[k] = v + } + for k, v := range o.FeatureFlags { + if _, ok := featureFlags[k]; !ok { + continue + } + merged[k] = v + } + return merged +} + +// printFeatureFlags logs the currently used feature flags on server startup. +func (s *Server) printFeatureFlags(o *Options) { + if len(o.FeatureFlags) == 0 { + return + } + keys := slices.Sorted(maps.Keys(o.FeatureFlags)) + + var ( + configured strings.Builder + unsupported strings.Builder + ) + + for _, k := range keys { + // Unsupported + defaultValue, ok := featureFlags[k] + if !ok { + if unsupported.Len() > 0 { + unsupported.WriteString(", ") + } + unsupported.WriteString(k) + continue + } + + v := o.FeatureFlags[k] + if configured.Len() > 0 { + configured.WriteString(", ") + } + configured.WriteString(k) + configured.WriteString(" (") + if defaultValue { + if v { + configured.WriteString("enabled") + } else { + configured.WriteString("opt-out") + } + } else if v { + configured.WriteString("opt-in") + } else { + configured.WriteString("disabled") + } + configured.WriteString(")") + } + if configured.Len() == 0 { + configured.WriteString("none") + } + + s.Noticef(" Feature flags:") + s.Noticef(" Configured: %s", configured.String()) + if unsupported.Len() > 0 { + s.Noticef(" Unsupported: %s", unsupported.String()) + } +} diff --git a/server/feature_flags_test.go b/server/feature_flags_test.go new file mode 100644 index 0000000..b5de8d7 --- /dev/null +++ b/server/feature_flags_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "reflect" + "testing" +) + +func TestGetFeatureFlag(t *testing.T) { + featureFlags["test_flag"] = true + t.Cleanup(func() { delete(featureFlags, "test_flag") }) + + // Unset feature flags gets defaults. + o := &Options{} + require_Equal(t, o.getFeatureFlag(""), false) + require_Equal(t, o.getFeatureFlag("test_flag"), true) + + // User disables the feature flag, it takes precedence. + o.FeatureFlags = map[string]bool{"test_flag": false} + require_Equal(t, o.getFeatureFlag("test_flag"), false) + + // User enables the feature flag, but we don't support it anymore. + // It's allowed for the user to provide stale flags, but we shouldn't return they're enabled. + delete(featureFlags, "test_flag") + o.FeatureFlags = map[string]bool{"test_flag": true} + require_Equal(t, o.getFeatureFlag("test_flag"), false) +} + +func TestFeatureFlagsConfigParseNonBool(t *testing.T) { + conf := createConfFile(t, []byte(` + feature_flags { + opt_in_flag: "true" + } + `)) + + _, err := ProcessConfigFile(conf) + require_Error(t, err) + require_Contains(t, err.Error(), `error parsing feature flag "opt_in_flag": expected bool, got string`) +} + +func TestGetMergedFeatureFlags(t *testing.T) { + previous := featureFlags + featureFlags = make(map[string]bool) + featureFlags["test_flag"] = true + featureFlags["test_flag_default"] = true + t.Cleanup(func() { featureFlags = previous }) + + o := &Options{} + o.FeatureFlags = map[string]bool{"test_flag": false, "unknown_flag": false} + + // The merged flags should only contain known flags, with the user's values taking precedence. + merged := o.getMergedFeatureFlags() + expected := map[string]bool{"test_flag": false, "test_flag_default": true} + if !reflect.DeepEqual(expected, merged) { + t.Fatalf("Feature flags are incorrect.\nexpected: %+v\ngot: %+v", expected, merged) + } +} diff --git a/server/gateway.go b/server/gateway.go new file mode 100644 index 0000000..4d49440 --- /dev/null +++ b/server/gateway.go @@ -0,0 +1,3426 @@ +// Copyright 2018-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "cmp" + "crypto/sha256" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "math/rand/v2" + "net" + "net/url" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +const ( + defaultSolicitGatewaysDelay = time.Second + defaultGatewayConnectDelay = time.Second + defaultGatewayConnectMaxDelay = 30 * time.Second + defaultGatewayReconnectDelay = time.Second + defaultGatewayRecentSubExpiration = 2 * time.Second + defaultGatewayMaxRUnsubBeforeSwitch = 1000 + + oldGWReplyPrefix = "$GR." + oldGWReplyPrefixLen = len(oldGWReplyPrefix) + oldGWReplyStart = oldGWReplyPrefixLen + 5 // len of prefix above + len of hash (4) + "." + + // The new prefix is "_GR_..." where is 6 characters + // hash of origin cluster name and is 6 characters hash of origin server pub key. + gwReplyPrefix = "_GR_." + gwReplyPrefixLen = len(gwReplyPrefix) + gwHashLen = 6 + gwClusterOffset = gwReplyPrefixLen + gwServerOffset = gwClusterOffset + gwHashLen + 1 + gwSubjectOffset = gwServerOffset + gwHashLen + 1 + + // Gateway connections send PINGs regardless of traffic. The interval is + // either Options.PingInterval or this value, whichever is the smallest. + gwMaxPingInterval = 15 * time.Second +) + +var ( + gatewayConnectDelay = defaultGatewayConnectDelay + gatewayConnectMaxDelay = defaultGatewayConnectMaxDelay + gatewayReconnectDelay = defaultGatewayReconnectDelay + gatewayMaxRUnsubBeforeSwitch = defaultGatewayMaxRUnsubBeforeSwitch + gatewaySolicitDelay = int64(defaultSolicitGatewaysDelay) + gatewayMaxPingInterval = gwMaxPingInterval +) + +// Warning when user configures gateway TLS insecure +const gatewayTLSInsecureWarning = "TLS certificate chain and hostname of solicited gateways will not be verified. DO NOT USE IN PRODUCTION!" + +// SetGatewaysSolicitDelay sets the initial delay before gateways +// connections are initiated. +// Used by tests. +func SetGatewaysSolicitDelay(delay time.Duration) { + atomic.StoreInt64(&gatewaySolicitDelay, int64(delay)) +} + +// ResetGatewaysSolicitDelay resets the initial delay before gateways +// connections are initiated to its default values. +// Used by tests. +func ResetGatewaysSolicitDelay() { + atomic.StoreInt64(&gatewaySolicitDelay, int64(defaultSolicitGatewaysDelay)) +} + +const ( + gatewayCmdGossip byte = 1 + gatewayCmdAllSubsStart byte = 2 + gatewayCmdAllSubsComplete byte = 3 +) + +// GatewayInterestMode represents an account interest mode for a gateway connection +type GatewayInterestMode byte + +// GatewayInterestMode values +const ( + // optimistic is the default mode where a cluster will send + // to a gateway unless it is been told that there is no interest + // (this is for plain subscribers only). + Optimistic GatewayInterestMode = iota + // transitioning is when a gateway has to send too many + // no interest on subjects to the remote and decides that it is + // now time to move to modeInterestOnly (this is on a per account + // basis). + Transitioning + // interestOnly means that a cluster sends all it subscriptions + // interest to the gateway, which in return does not send a message + // unless it knows that there is explicit interest. + InterestOnly +) + +func (im GatewayInterestMode) String() string { + switch im { + case Optimistic: + return "Optimistic" + case InterestOnly: + return "Interest-Only" + case Transitioning: + return "Transitioning" + default: + return "Unknown" + } +} + +var gwDoNotForceInterestOnlyMode bool + +// GatewayDoNotForceInterestOnlyMode is used ONLY in tests. +// DO NOT USE in normal code or if you embed the NATS Server. +func GatewayDoNotForceInterestOnlyMode(doNotForce bool) { + gwDoNotForceInterestOnlyMode = doNotForce +} + +type srvGateway struct { + totalQSubs int64 //total number of queue subs in all remote gateways (used with atomic operations) + sync.RWMutex + enabled bool // Immutable, true if both a name and port are configured + name string // Name of the Gateway on this server + out map[string]*client // outbound gateways + outo []*client // outbound gateways maintained in an order suitable for sending msgs (currently based on RTT) + in map[uint64]*client // inbound gateways + remotes map[string]*gatewayCfg // Config of remote gateways + URLs refCountedUrlSet // Set of all Gateway URLs in the cluster + URL string // This server gateway URL (after possible random port is resolved) + info *Info // Gateway Info protocol + infoJSON []byte // Marshal'ed Info protocol + runknown bool // Rejects unknown (not configured) gateway connections + replyPfx []byte // Will be "$GNR.<1:reserved>.<8:cluster hash>.<8:server hash>." + + // For backward compatibility + oldReplyPfx []byte + oldHash []byte + + // We maintain the interest of subjects and queues per account. + // For a given account, entries in the map could be something like this: + // foo.bar {n: 3} // 3 subs on foo.bar + // foo.> {n: 6} // 6 subs on foo.> + // foo bar {n: 1, q: true} // 1 qsub on foo, queue bar + // foo baz {n: 3, q: true} // 3 qsubs on foo, queue baz + pasi struct { + // Protect map since accessed from different go-routine and avoid + // possible race resulting in RS+ being sent before RS- resulting + // in incorrect interest suppression. + // Will use while sending QSubs (on GW connection accept) and when + // switching to the send-all-subs mode. + sync.Mutex + m map[string]map[string]*sitally + } + + // This is to track recent subscriptions for a given account + rsubs sync.Map + + resolver netResolver // Used to resolve host name before calling net.Dial() + sqbsz int // Max buffer size to send queue subs protocol. Used for testing. + recSubExp time.Duration // For how long do we check if there is a subscription match for a message with reply + + // These are used for routing of mapped replies. + sIDHash []byte // Server ID hash (6 bytes) + routesIDByHash sync.Map // Route's server ID is hashed (6 bytes) and stored in this map. + + // If a server has its own configuration in the "Gateways" remotes configuration + // we will keep track of the URLs that are defined in the config so they can + // be reported in monitoring. + ownCfgURLs []string +} + +// Subject interest tally. Also indicates if the key in the map is a +// queue or not. +type sitally struct { + n int32 // number of subscriptions directly matching + q bool // indicate that this is a queue +} + +type gatewayCfg struct { + sync.RWMutex + *RemoteGatewayOpts + hash []byte + oldHash []byte + urls map[string]*url.URL + connAttempts int + tlsName string + implicit bool + varzUpdateURLs bool // Tells monitoring code to update URLs when varz is inspected. +} + +// Struct for client's gateway related fields +type gateway struct { + name string + cfg *gatewayCfg + connectURL *url.URL // Needed when sending CONNECT after receiving INFO from remote + outsim *sync.Map // Per-account subject interest (or no-interest) (outbound conn) + insim map[string]*insie // Per-account subject no-interest sent or modeInterestOnly mode (inbound conn) + + // This is an outbound GW connection + outbound bool + // Set/check in readLoop without lock. This is to know that an inbound has sent the CONNECT protocol first + connected bool + // Set to true if outbound is to a server that only knows about $GR, not $GNR + useOldPrefix bool + // If true, it indicates that the inbound side will switch any account to + // interest-only mode "immediately", so the outbound should disregard + // the optimistic mode when checking for interest. + interestOnlyMode bool + // Name of the remote server + remoteName string +} + +// Outbound subject interest entry. +type outsie struct { + sync.RWMutex + // Indicate that all subs should be stored. This is + // set to true when receiving the command from the + // remote that we are about to receive all its subs. + mode GatewayInterestMode + // If not nil, used for no-interest for plain subs. + // If a subject is present in this map, it means that + // the remote is not interested in that subject. + // When we have received the command that says that + // the remote has sent all its subs, this is set to nil. + ni map[string]struct{} + // Contains queue subscriptions when in optimistic mode, + // and all subs when pk is > 0. + sl *Sublist + // Number of queue subs + qsubs int +} + +// Inbound subject interest entry. +// If `ni` is not nil, it stores the subjects for which an +// RS- was sent to the remote gateway. When a subscription +// is created, this is used to know if we need to send +// an RS+ to clear the no-interest in the remote. +// When an account is switched to modeInterestOnly (we send +// all subs of an account to the remote), then `ni` is nil and +// when all subs have been sent, mode is set to modeInterestOnly +type insie struct { + ni map[string]struct{} // Record if RS- was sent for given subject + mode GatewayInterestMode +} + +type gwReplyMap struct { + ms string + exp int64 +} + +type gwReplyMapping struct { + // Indicate if we should check the map or not. Since checking the map is done + // when processing inbound messages and requires the lock we want to + // check only when needed. This is set/get using atomic, so needs to + // be memory aligned. + check int32 + // To keep track of gateway replies mapping + mapping map[string]*gwReplyMap +} + +// Returns the corresponding gw routed subject, and `true` to indicate that a +// mapping was found. If no entry is found, the passed subject is returned +// as-is and `false` is returned to indicate that no mapping was found. +// Caller is responsible to ensure the locking. +func (g *gwReplyMapping) get(subject []byte) ([]byte, bool) { + rm, ok := g.mapping[string(subject)] + if !ok { + return subject, false + } + subj := []byte(rm.ms) + return subj, true +} + +// clone returns a deep copy of the RemoteGatewayOpts object +func (r *RemoteGatewayOpts) clone() *RemoteGatewayOpts { + if r == nil { + return nil + } + clone := &RemoteGatewayOpts{ + Name: r.Name, + URLs: deepCopyURLs(r.URLs), + } + if r.TLSConfig != nil { + clone.TLSConfig = r.TLSConfig.Clone() + clone.TLSTimeout = r.TLSTimeout + } + return clone +} + +// Ensure that gateway is properly configured. +func validateGatewayOptions(o *Options) error { + if o.Gateway.Name == _EMPTY_ && o.Gateway.Port == 0 { + return nil + } + if o.Gateway.Name == _EMPTY_ { + return errors.New("gateway has no name") + } + if strings.Contains(o.Gateway.Name, " ") { + return ErrGatewayNameHasSpaces + } + if o.Gateway.Port == 0 { + return fmt.Errorf("gateway %q has no port specified (select -1 for random port)", o.Gateway.Name) + } + for i, g := range o.Gateway.Gateways { + if g.Name == _EMPTY_ { + return fmt.Errorf("gateway in the list %d has no name", i) + } + if len(g.URLs) == 0 { + return fmt.Errorf("gateway %q has no URL", g.Name) + } + } + if err := validatePinnedCerts(o.Gateway.TLSPinnedCerts); err != nil { + return fmt.Errorf("gateway %q: %v", o.Gateway.Name, err) + } + return nil +} + +// Computes a hash of 6 characters for the name. +// This will be used for routing of replies. +func getGWHash(name string) []byte { + return []byte(getHashSize(name, gwHashLen)) +} + +func getOldHash(name string) []byte { + sha := sha256.New() + sha.Write([]byte(name)) + fullHash := []byte(fmt.Sprintf("%x", sha.Sum(nil))) + return fullHash[:4] +} + +// Initialize the s.gateway structure. We do this even if the server +// does not have a gateway configured. In some part of the code, the +// server will check the number of outbound gateways, etc.. and so +// we don't have to check if s.gateway is nil or not. +func (s *Server) newGateway(opts *Options) error { + gateway := &srvGateway{ + name: opts.Gateway.Name, + out: make(map[string]*client), + outo: make([]*client, 0, 4), + in: make(map[uint64]*client), + remotes: make(map[string]*gatewayCfg), + URLs: make(refCountedUrlSet), + resolver: opts.Gateway.resolver, + runknown: opts.Gateway.RejectUnknown, + oldHash: getOldHash(opts.Gateway.Name), + } + gateway.Lock() + defer gateway.Unlock() + + gateway.sIDHash = getGWHash(s.info.ID) + clusterHash := getGWHash(opts.Gateway.Name) + prefix := make([]byte, 0, gwSubjectOffset) + prefix = append(prefix, gwReplyPrefix...) + prefix = append(prefix, clusterHash...) + prefix = append(prefix, '.') + prefix = append(prefix, gateway.sIDHash...) + prefix = append(prefix, '.') + gateway.replyPfx = prefix + + prefix = make([]byte, 0, oldGWReplyStart) + prefix = append(prefix, oldGWReplyPrefix...) + prefix = append(prefix, gateway.oldHash...) + prefix = append(prefix, '.') + gateway.oldReplyPfx = prefix + + gateway.pasi.m = make(map[string]map[string]*sitally) + + if gateway.resolver == nil { + gateway.resolver = netResolver(net.DefaultResolver) + } + + // Create remote gateways + for _, rgo := range opts.Gateway.Gateways { + // Ignore if there is a remote gateway with our name. + if rgo.Name == gateway.name { + gateway.ownCfgURLs = getURLsAsString(rgo.URLs) + continue + } + cfg := &gatewayCfg{ + RemoteGatewayOpts: rgo.clone(), + hash: getGWHash(rgo.Name), + oldHash: getOldHash(rgo.Name), + urls: make(map[string]*url.URL, len(rgo.URLs)), + } + if opts.Gateway.TLSConfig != nil && cfg.TLSConfig == nil { + cfg.TLSConfig = opts.Gateway.TLSConfig.Clone() + } + if cfg.TLSTimeout == 0 { + cfg.TLSTimeout = opts.Gateway.TLSTimeout + } + for _, u := range rgo.URLs { + // For TLS, look for a hostname that we can use for TLSConfig.ServerName + cfg.saveTLSHostname(u) + cfg.urls[u.Host] = u + } + gateway.remotes[cfg.Name] = cfg + } + + gateway.sqbsz = opts.Gateway.sendQSubsBufSize + if gateway.sqbsz == 0 { + gateway.sqbsz = maxBufSize + } + gateway.recSubExp = defaultGatewayRecentSubExpiration + + gateway.enabled = opts.Gateway.Name != "" && opts.Gateway.Port != 0 + s.gateway = gateway + return nil +} + +// Update remote gateways TLS configurations after a config reload. +func (g *srvGateway) updateRemotesTLSConfig(opts *Options) { + g.Lock() + defer g.Unlock() + // Instead of going over opts.Gateway.Gateways, which would include only + // explicit remotes, we are going to go through g.remotes. + for name, cfg := range g.remotes { + if name == g.name { + continue + } + var ro *RemoteGatewayOpts + // We now need to go back and find the RemoteGatewayOpts but only if + // this remote is explicit (otherwise it won't be found). + if !cfg.isImplicit() { + for _, r := range opts.Gateway.Gateways { + if r.Name == name { + ro = r + break + } + } + } + cfg.Lock() + // If we have an `ro` (that means an explicitly defined remote gateway) + // and it has an explicit TLS config, use that one, otherwise (no explicit + // TLS config in the remote, or implicit remote), use the TLS config from + // the main block. + if ro != nil && ro.TLSConfig != nil { + cfg.TLSConfig = ro.TLSConfig.Clone() + } else if opts.Gateway.TLSConfig != nil { + cfg.TLSConfig = opts.Gateway.TLSConfig.Clone() + } + // Ensure that OCSP callbacks are always setup after a reload if needed. + mustStaple := opts.OCSPConfig != nil && opts.OCSPConfig.Mode == OCSPModeAlways + if mustStaple && opts.Gateway.TLSConfig != nil { + clientCB := opts.Gateway.TLSConfig.GetClientCertificate + verifyCB := opts.Gateway.TLSConfig.VerifyConnection + if mustStaple && cfg.TLSConfig != nil { + if clientCB != nil && cfg.TLSConfig.GetClientCertificate == nil { + cfg.TLSConfig.GetClientCertificate = clientCB + } + if verifyCB != nil && cfg.TLSConfig.VerifyConnection == nil { + cfg.TLSConfig.VerifyConnection = verifyCB + } + } + } + cfg.Unlock() + } +} + +// Returns if this server rejects connections from gateways that are not +// explicitly configured. +func (g *srvGateway) rejectUnknown() bool { + g.RLock() + reject := g.runknown + g.RUnlock() + return reject +} + +// Starts the gateways accept loop and solicit explicit gateways +// after an initial delay. This delay is meant to give a chance to +// the cluster to form and this server gathers gateway URLs for this +// cluster in order to send that as part of the connect/info process. +func (s *Server) startGateways() { + s.startGatewayAcceptLoop() + + // Delay start of creation of gateways to give a chance + // to the local cluster to form. + s.startGoRoutine(func() { + defer s.grWG.Done() + + dur := s.getOpts().gatewaysSolicitDelay + if dur == 0 { + dur = time.Duration(atomic.LoadInt64(&gatewaySolicitDelay)) + } + + select { + case <-time.After(dur): + s.solicitGateways() + case <-s.quitCh: + return + } + }) +} + +// This starts the gateway accept loop in a go routine, unless it +// is detected that the server has already been shutdown. +func (s *Server) startGatewayAcceptLoop() { + if s.isShuttingDown() { + return + } + + // Snapshot server options. + opts := s.getOpts() + + port := opts.Gateway.Port + if port == -1 { + port = 0 + } + + s.mu.Lock() + hp := net.JoinHostPort(opts.Gateway.Host, strconv.Itoa(port)) + l, e := natsListen("tcp", hp) + s.gatewayListenerErr = e + if e != nil { + s.mu.Unlock() + s.Fatalf("Error listening on gateway port: %d - %v", opts.Gateway.Port, e) + return + } + s.Noticef("Gateway name is %s", s.getGatewayName()) + s.Noticef("Listening for gateways connections on %s", + net.JoinHostPort(opts.Gateway.Host, strconv.Itoa(l.Addr().(*net.TCPAddr).Port))) + + tlsReq := opts.Gateway.TLSConfig != nil + authRequired := opts.Gateway.Username != "" + info := &Info{ + ID: s.info.ID, + Name: opts.ServerName, + Version: s.info.Version, + AuthRequired: authRequired, + TLSRequired: tlsReq, + TLSVerify: tlsReq, + MaxPayload: s.info.MaxPayload, + Gateway: opts.Gateway.Name, + GatewayNRP: true, + Headers: s.supportsHeaders(), + Proto: s.getServerProto(), + } + // Unless in some tests we want to keep the old behavior, we are now + // (since v2.9.0) indicate that this server will switch all accounts + // to InterestOnly mode when accepting an inbound or when a new + // account is fetched. + if !gwDoNotForceInterestOnlyMode { + info.GatewayIOM = true + } + + // If we have selected a random port... + if port == 0 { + // Write resolved port back to options. + opts.Gateway.Port = l.Addr().(*net.TCPAddr).Port + } + // Possibly override Host/Port based on Gateway.Advertise + if err := s.setGatewayInfoHostPort(info, opts); err != nil { + s.Fatalf("Error setting gateway INFO with Gateway.Advertise value of %s, err=%v", opts.Gateway.Advertise, err) + l.Close() + s.mu.Unlock() + return + } + // Setup state that can enable shutdown + s.gatewayListener = l + + // Warn if insecure is configured in the main Gateway configuration + // or any of the RemoteGateway's. This means that we need to check + // remotes even if TLS would not be configured for the accept. + warn := tlsReq && opts.Gateway.TLSConfig.InsecureSkipVerify + if !warn { + for _, g := range opts.Gateway.Gateways { + if g.TLSConfig != nil && g.TLSConfig.InsecureSkipVerify { + warn = true + break + } + } + } + if warn { + s.Warnf(gatewayTLSInsecureWarning) + } + go s.acceptConnections(l, "Gateway", func(conn net.Conn) { s.createGateway(nil, nil, conn) }, nil) + s.mu.Unlock() +} + +// Similar to setInfoHostPortAndGenerateJSON, but for gatewayInfo. +func (s *Server) setGatewayInfoHostPort(info *Info, o *Options) error { + gw := s.gateway + gw.Lock() + defer gw.Unlock() + gw.URLs.removeUrl(gw.URL) + if o.Gateway.Advertise != "" { + advHost, advPort, err := parseHostPort(o.Gateway.Advertise, o.Gateway.Port) + if err != nil { + return err + } + info.Host = advHost + info.Port = advPort + } else { + info.Host = o.Gateway.Host + info.Port = o.Gateway.Port + // If the host is "0.0.0.0" or "::" we need to resolve to a public IP. + // This will return at most 1 IP. + hostIsIPAny, ips, err := s.getNonLocalIPsIfHostIsIPAny(info.Host, false) + if err != nil { + return err + } + if hostIsIPAny { + if len(ips) == 0 { + // TODO(ik): Should we fail here (prevent starting)? If not, we + // are going to "advertise" the 0.0.0.0: url, which means + // that remote are going to try to connect to 0.0.0.0:, + // which means a connect to loopback address, which is going + // to fail with either TLS error, conn refused if the remote + // is using different gateway port than this one, or error + // saying that it tried to connect to itself. + s.Errorf("Could not find any non-local IP for gateway %q with listen specification %q", + gw.name, info.Host) + } else { + // Take the first from the list... + info.Host = ips[0] + } + } + } + gw.URL = net.JoinHostPort(info.Host, strconv.Itoa(info.Port)) + if o.Gateway.Advertise != "" { + s.Noticef("Advertise address for gateway %q is set to %s", gw.name, gw.URL) + } else { + s.Noticef("Address for gateway %q is %s", gw.name, gw.URL) + } + gw.URLs[gw.URL]++ + gw.info = info + info.GatewayURL = gw.URL + // (re)generate the gatewayInfoJSON byte array + gw.generateInfoJSON() + return nil +} + +// Generates the Gateway INFO protocol. +// The gateway lock is held on entry +func (g *srvGateway) generateInfoJSON() { + // We could be here when processing a route INFO that has a gateway URL, + // but this server is not configured for gateways, so simply ignore here. + // The configuration mismatch is reported somewhere else. + if !g.enabled || g.info == nil { + return + } + g.info.GatewayURLs = g.URLs.getAsStringSlice() + b, err := json.Marshal(g.info) + if err != nil { + panic(err) + } + g.infoJSON = []byte(fmt.Sprintf(InfoProto, b)) +} + +// Goes through the list of registered gateways and try to connect to those. +// The list (remotes) is initially containing the explicit remote gateways, +// but the list is augmented with any implicit (discovered) gateway. Therefore, +// this function only solicit explicit ones. +func (s *Server) solicitGateways() { + gw := s.gateway + gw.RLock() + defer gw.RUnlock() + for _, cfg := range gw.remotes { + // Since we delay the creation of gateways, it is + // possible that server starts to receive inbound from + // other clusters and in turn create outbounds. So here + // we create only the ones that are configured. + if !cfg.isImplicit() { + cfg := cfg // Create new instance for the goroutine. + s.startGoRoutine(func() { + s.solicitGateway(cfg, true) + s.grWG.Done() + }) + } + } +} + +// Reconnect to the gateway after a little wait period. For explicit +// gateways, we also wait for the default reconnect time. +func (s *Server) reconnectGateway(cfg *gatewayCfg) { + defer s.grWG.Done() + + delay := time.Duration(rand.IntN(100)) * time.Millisecond + if !cfg.isImplicit() { + delay += gatewayReconnectDelay + } + select { + case <-time.After(delay): + case <-s.quitCh: + return + } + s.solicitGateway(cfg, false) +} + +// This function will loop trying to connect to any URL attached +// to the given Gateway. It will return once a connection has been created. +func (s *Server) solicitGateway(cfg *gatewayCfg, firstConnect bool) { + var ( + opts = s.getOpts() + isImplicit = cfg.isImplicit() + attemptDelay = gatewayConnectDelay + attempts int + typeStr string + ) + if isImplicit { + typeStr = "implicit" + } else { + typeStr = "explicit" + } + + const connFmt = "Connecting to %s gateway %q (%s) at %s (attempt %v)" + const connErrFmt = "Error connecting to %s gateway %q (%s) at %s (attempt %v): %v" + + for s.isRunning() { + urls := cfg.getURLs() + if len(urls) == 0 { + break + } + attempts++ + report := s.shouldReportConnectErr(firstConnect, attempts) + // Iteration is random + for _, u := range urls { + address, err := s.getRandomIP(s.gateway.resolver, u.Host, nil) + if err != nil { + s.Errorf("Error getting IP for %s gateway %q (%s): %v", typeStr, cfg.Name, u.Host, err) + continue + } + if report { + s.Noticef(connFmt, typeStr, cfg.Name, u.Host, address, attempts) + } else { + s.Debugf(connFmt, typeStr, cfg.Name, u.Host, address, attempts) + } + conn, err := natsDialTimeout("tcp", address, DEFAULT_ROUTE_DIAL) + if err == nil { + // We could connect, create the gateway connection and return. + s.createGateway(cfg, u, conn) + return + } + if report { + s.Errorf(connErrFmt, typeStr, cfg.Name, u.Host, address, attempts, err) + } else { + s.Debugf(connErrFmt, typeStr, cfg.Name, u.Host, address, attempts, err) + } + // Break this loop if server is being shutdown... + if !s.isRunning() { + break + } + } + if isImplicit { + if opts.Gateway.ConnectRetries == 0 || attempts > opts.Gateway.ConnectRetries { + s.gateway.Lock() + // We could have just accepted an inbound for this remote gateway. + // So if there is an inbound, let's try again to connect. + if s.gateway.hasInbound(cfg.Name) { + s.gateway.Unlock() + continue + } + delete(s.gateway.remotes, cfg.Name) + s.gateway.Unlock() + return + } + } + select { + case <-s.quitCh: + return + case <-time.After(attemptDelay): + if opts.Gateway.ConnectBackoff { + // Use exponential backoff for connection attempts. + attemptDelay *= 2 + if attemptDelay > gatewayConnectMaxDelay { + attemptDelay = gatewayConnectMaxDelay + } + } + continue + } + } +} + +// Returns true if there is an inbound for the given `name`. +// Lock held on entry. +func (g *srvGateway) hasInbound(name string) bool { + for _, ig := range g.in { + ig.mu.Lock() + igname := ig.gw.name + ig.mu.Unlock() + if igname == name { + return true + } + } + return false +} + +// Called when a gateway connection is either accepted or solicited. +// If accepted, the gateway is marked as inbound. +// If solicited, the gateway is marked as outbound. +func (s *Server) createGateway(cfg *gatewayCfg, url *url.URL, conn net.Conn) { + // Snapshot server options. + opts := s.getOpts() + + now := time.Now() + c := &client{srv: s, nc: conn, start: now, last: now, kind: GATEWAY} + + // Are we creating the gateway based on the configuration + solicit := cfg != nil + var tlsRequired bool + + s.gateway.RLock() + infoJSON := s.gateway.infoJSON + s.gateway.RUnlock() + + // Perform some initialization under the client lock + c.mu.Lock() + c.initClient() + c.gw = &gateway{} + if solicit { + // This is an outbound gateway connection + cfg.RLock() + tlsRequired = cfg.TLSConfig != nil + cfgName := cfg.Name + cfg.RUnlock() + c.gw.outbound = true + c.gw.name = cfgName + c.gw.cfg = cfg + cfg.bumpConnAttempts() + // Since we are delaying the connect until after receiving + // the remote's INFO protocol, save the URL we need to connect to. + c.gw.connectURL = url + + c.Noticef("Creating outbound gateway connection to %q", cfgName) + } else { + c.flags.set(expectConnect) + // Inbound gateway connection + c.Noticef("Processing inbound gateway connection") + // Check if TLS is required for inbound GW connections. + tlsRequired = opts.Gateway.TLSConfig != nil + // We expect a CONNECT from the accepted connection. + c.setAuthTimer(secondsToDuration(opts.Gateway.AuthTimeout)) + } + + // Check for TLS + if tlsRequired { + var tlsConfig *tls.Config + var tlsName string + var timeout float64 + + if solicit { + var ( + mustStaple = opts.OCSPConfig != nil && opts.OCSPConfig.Mode == OCSPModeAlways + clientCB func(*tls.CertificateRequestInfo) (*tls.Certificate, error) + verifyCB func(tls.ConnectionState) error + ) + // Snapshot callbacks for OCSP outside an ongoing reload which might be happening. + if mustStaple { + s.reloadMu.RLock() + s.optsMu.RLock() + clientCB = s.opts.Gateway.TLSConfig.GetClientCertificate + verifyCB = s.opts.Gateway.TLSConfig.VerifyConnection + s.optsMu.RUnlock() + s.reloadMu.RUnlock() + } + + cfg.RLock() + tlsName = cfg.tlsName + tlsConfig = cfg.TLSConfig.Clone() + timeout = cfg.TLSTimeout + + // Ensure that OCSP callbacks are always setup on gateway reconnect when OCSP policy is set to always. + if mustStaple { + if clientCB != nil && tlsConfig.GetClientCertificate == nil { + tlsConfig.GetClientCertificate = clientCB + } + if verifyCB != nil && tlsConfig.VerifyConnection == nil { + tlsConfig.VerifyConnection = verifyCB + } + } + cfg.RUnlock() + } else { + tlsConfig = opts.Gateway.TLSConfig + timeout = opts.Gateway.TLSTimeout + } + + // Perform (either server or client side) TLS handshake. + if resetTLSName, err := c.doTLSHandshake("gateway", solicit, url, tlsConfig, tlsName, timeout, opts.Gateway.TLSPinnedCerts); err != nil { + if resetTLSName { + cfg.Lock() + cfg.tlsName = _EMPTY_ + cfg.Unlock() + } + c.mu.Unlock() + return + } + } + + // Do final client initialization + c.in.pacache = make(map[string]*perAccountCache) + if solicit { + // This is an outbound gateway connection + c.gw.outsim = &sync.Map{} + } else { + // Inbound gateway connection + c.gw.insim = make(map[string]*insie) + } + + // Register in temp map for now until gateway properly registered + // in out or in gateways. + if !s.addToTempClients(c.cid, c) { + c.mu.Unlock() + c.closeConnection(ServerShutdown) + return + } + + // Only send if we accept a connection. Will send CONNECT+INFO as an + // outbound only after processing peer's INFO protocol. + if !solicit { + c.enqueueProto(infoJSON) + } + + // Spin up the read loop. + s.startGoRoutine(func() { c.readLoop(nil) }) + + // Spin up the write loop. + s.startGoRoutine(func() { c.writeLoop() }) + + if tlsRequired { + c.Debugf("TLS handshake complete") + cs := c.nc.(*tls.Conn).ConnectionState() + c.Debugf("TLS version %s, cipher suite %s", tlsVersion(cs.Version), tls.CipherSuiteName(cs.CipherSuite)) + } + + // For outbound, we can't set the normal ping timer yet since the other + // side would fail with a parse error should it receive anything but the + // CONNECT protocol as the first protocol. We still want to make sure + // that the connection is not stale until the first INFO from the remote + // is received. + if solicit { + c.watchForStaleConnection(adjustPingInterval(GATEWAY, opts.PingInterval), opts.MaxPingsOut) + } + + c.mu.Unlock() + + // Announce ourselves again to new connections. + if solicit && s.EventsEnabled() { + s.sendStatszUpdate() + } +} + +// Builds and sends the CONNECT protocol for a gateway. +// Client lock held on entry. +func (c *client) sendGatewayConnect(opts *Options) { + c.gw.cfg.RLock() + tlsRequired := c.gw.cfg.TLSConfig != nil + c.gw.cfg.RUnlock() + url := c.gw.connectURL + c.gw.connectURL = nil + var user, pass string + if userInfo := url.User; userInfo != nil { + user = userInfo.Username() + pass, _ = userInfo.Password() + } else if opts != nil { + user = opts.Gateway.Username + pass = opts.Gateway.Password + } + cinfo := connectInfo{ + Verbose: false, + Pedantic: false, + User: user, + Pass: pass, + TLS: tlsRequired, + Name: c.srv.info.ID, + Gateway: c.srv.gateway.name, + } + b, err := json.Marshal(cinfo) + if err != nil { + panic(err) + } + c.enqueueProto([]byte(fmt.Sprintf(ConProto, b))) +} + +// Process the CONNECT protocol from a gateway connection. +// Returns an error to the connection if the CONNECT is not from a gateway +// (for instance a client or route connecting to the gateway port), or +// if the destination does not match the gateway name of this server. +// +// +func (c *client) processGatewayConnect(arg []byte) error { + connect := &connectInfo{} + if err := json.Unmarshal(arg, connect); err != nil { + return err + } + + // Coming from a client or a route, reject + if connect.Gateway == "" { + c.sendErrAndErr(ErrClientOrRouteConnectedToGatewayPort.Error()) + c.closeConnection(WrongPort) + return ErrClientOrRouteConnectedToGatewayPort + } + + c.mu.Lock() + s := c.srv + c.mu.Unlock() + + // If we reject unknown gateways, make sure we have it configured, + // otherwise return an error. + if s.gateway.rejectUnknown() && s.getRemoteGateway(connect.Gateway) == nil { + c.Errorf("Rejecting connection from gateway %q", connect.Gateway) + c.sendErr(fmt.Sprintf("Connection to gateway %q rejected", s.getGatewayName())) + c.closeConnection(WrongGateway) + return ErrWrongGateway + } + + c.mu.Lock() + c.gw.connected = true + // Set the Ping timer after sending connect and info. + c.setFirstPingTimer() + c.mu.Unlock() + + return nil +} + +// Process the INFO protocol from a gateway connection. +// +// If the gateway connection is an outbound (this server initiated the connection), +// this function checks that the incoming INFO contains the Gateway field. If empty, +// it means that this is a response from an older server or that this server connected +// to the wrong port. +// The outbound gateway may also receive a gossip INFO protocol from the remote gateway, +// indicating other gateways that the remote knows about. This server will try to connect +// to those gateways (if not explicitly configured or already implicitly connected). +// In both cases (explicit or implicit), the local cluster is notified about the existence +// of this new gateway. This allows servers in the cluster to ensure that they have an +// outbound connection to this gateway. +// +// For an inbound gateway, the gateway is simply registered and the info protocol +// is saved to be used after processing the CONNECT. +// +// +func (c *client) processGatewayInfo(info *Info) { + var ( + gwName string + cfg *gatewayCfg + ) + c.mu.Lock() + s := c.srv + cid := c.cid + + // Check if this is the first INFO. (this call sets the flag if not already set). + isFirstINFO := c.flags.setIfNotSet(infoReceived) + + isOutbound := c.gw.outbound + if isOutbound { + gwName = c.gw.name + cfg = c.gw.cfg + } else if isFirstINFO { + c.gw.name = info.Gateway + } + if isFirstINFO { + c.opts.Name = info.ID + // Get the protocol version from the INFO protocol. This will be checked + // to see if this connection supports message tracing for instance. + c.opts.Protocol = info.Proto + c.gw.remoteName = info.Name + } + c.mu.Unlock() + + // For an outbound connection... + if isOutbound { + // Check content of INFO for fields indicating that it comes from a gateway. + // If we incorrectly connect to the wrong port (client or route), we won't + // have the Gateway field set. + if info.Gateway == "" { + c.sendErrAndErr(fmt.Sprintf("Attempt to connect to gateway %q using wrong port", gwName)) + c.closeConnection(WrongPort) + return + } + // Check that the gateway name we got is what we expect + if info.Gateway != gwName { + // Unless this is the very first INFO, it may be ok if this is + // a gossip request to connect to other gateways. + if !isFirstINFO && info.GatewayCmd == gatewayCmdGossip { + // If we are configured to reject unknown, do not attempt to + // connect to one that we don't have configured. + if s.gateway.rejectUnknown() && s.getRemoteGateway(info.Gateway) == nil { + return + } + s.processImplicitGateway(info) + return + } + // Otherwise, this is a failure... + // We are reporting this error in the log... + c.Errorf("Failing connection to gateway %q, remote gateway name is %q", + gwName, info.Gateway) + // ...and sending this back to the remote so that the error + // makes more sense in the remote server's log. + c.sendErr(fmt.Sprintf("Connection from %q rejected, wanted to connect to %q, this is %q", + s.getGatewayName(), gwName, info.Gateway)) + c.closeConnection(WrongGateway) + return + } + + // Check for duplicate server name with servers in our cluster + if s.isDuplicateServerName(info.Name) { + c.Errorf("Remote server has a duplicate name: %q", info.Name) + c.closeConnection(DuplicateServerName) + return + } + + // Possibly add URLs that we get from the INFO protocol. + if len(info.GatewayURLs) > 0 { + cfg.updateURLs(info.GatewayURLs) + } + + // If this is the first INFO, send our connect + if isFirstINFO { + s.gateway.RLock() + infoJSON := s.gateway.infoJSON + s.gateway.RUnlock() + + supportsHeaders := s.supportsHeaders() + opts := s.getOpts() + + // Note, if we want to support NKeys, then we would get the nonce + // from this INFO protocol and can sign it in the CONNECT we are + // going to send now. + c.mu.Lock() + c.gw.interestOnlyMode = info.GatewayIOM + c.sendGatewayConnect(opts) + c.Debugf("Gateway connect protocol sent to %q", gwName) + // Send INFO too + c.enqueueProto(infoJSON) + c.gw.useOldPrefix = !info.GatewayNRP + c.headers = supportsHeaders && info.Headers + c.mu.Unlock() + + // Register as an outbound gateway.. if we had a protocol to ack our connect, + // then we should do that when process that ack. + if s.registerOutboundGatewayConnection(gwName, c) { + c.Noticef("Outbound gateway connection to %q (%s) registered", gwName, info.ID) + // Now that the outbound gateway is registered, we can remove from temp map. + s.removeFromTempClients(cid) + // Set the Ping timer after sending connect and info. + c.mu.Lock() + c.setFirstPingTimer() + c.mu.Unlock() + } else { + // There was a bug that would cause a connection to possibly + // be called twice resulting in reconnection of twice the + // same outbound connection. The issue is fixed, but adding + // defensive code above that if we did not register this connection + // because we already have an outbound for this name, then + // close this connection (and make sure it does not try to reconnect) + c.setNoReconnect() + c.closeConnection(WrongGateway) + return + } + } else if info.GatewayCmd > 0 { + switch info.GatewayCmd { + case gatewayCmdAllSubsStart: + c.gatewayAllSubsReceiveStart(info) + return + case gatewayCmdAllSubsComplete: + c.gatewayAllSubsReceiveComplete(info) + return + default: + s.Warnf("Received unknown command %v from gateway %q", info.GatewayCmd, gwName) + return + } + } + + // Flood local cluster with information about this gateway. + // Servers in this cluster will ensure that they have (or otherwise create) + // an outbound connection to this gateway. + s.forwardNewGatewayToLocalCluster(info) + + } else if isFirstINFO { + // This is the first INFO of an inbound connection... + + // Check for duplicate server name with servers in our cluster + if s.isDuplicateServerName(info.Name) { + c.Errorf("Remote server has a duplicate name: %q", info.Name) + c.closeConnection(DuplicateServerName) + return + } + + s.registerInboundGatewayConnection(cid, c) + c.Noticef("Inbound gateway connection from %q (%s) registered", info.Gateway, info.ID) + + // Now that it is registered, we can remove from temp map. + s.removeFromTempClients(cid) + + // Send our QSubs. + s.sendQueueSubsToGateway(c) + + // Initiate outbound connection. This function will behave correctly if + // we have already one. + s.processImplicitGateway(info) + + // Send back to the server that initiated this gateway connection the + // list of all remote gateways known on this server. + s.gossipGatewaysToInboundGateway(info.Gateway, c) + + // Now make sure if we have any knowledge of connected leafnodes that we resend the + // connect events to switch those accounts into interest only mode. + s.mu.Lock() + s.ensureGWsInterestOnlyForLeafNodes() + s.mu.Unlock() + js := s.js.Load() + + // If running in some tests, maintain the original behavior. + if gwDoNotForceInterestOnlyMode && js != nil { + // Switch JetStream accounts to interest-only mode. + var accounts []string + js.mu.Lock() + if len(js.accounts) > 0 { + accounts = make([]string, 0, len(js.accounts)) + for accName := range js.accounts { + accounts = append(accounts, accName) + } + } + js.mu.Unlock() + for _, accName := range accounts { + if acc, err := s.LookupAccount(accName); err == nil && acc != nil { + if acc.JetStreamEnabled() { + s.switchAccountToInterestMode(acc.GetName()) + } + } + } + } else if !gwDoNotForceInterestOnlyMode { + // Starting 2.9.0, we are phasing out the optimistic mode, so change + // all accounts to interest-only mode, unless instructed not to do so + // in some tests. + s.accounts.Range(func(_, v any) bool { + acc := v.(*Account) + s.switchAccountToInterestMode(acc.GetName()) + return true + }) + } + } +} + +// Sends to the given inbound gateway connection a gossip INFO protocol +// for each gateway known by this server. This allows for a "full mesh" +// of gateways. +func (s *Server) gossipGatewaysToInboundGateway(gwName string, c *client) { + gw := s.gateway + gw.RLock() + defer gw.RUnlock() + for gwCfgName, cfg := range gw.remotes { + // Skip the gateway that we just created + if gwCfgName == gwName { + continue + } + info := Info{ + ID: s.info.ID, + GatewayCmd: gatewayCmdGossip, + } + urls := cfg.getURLsAsStrings() + if len(urls) > 0 { + info.Gateway = gwCfgName + info.GatewayURLs = urls + b, _ := json.Marshal(&info) + c.mu.Lock() + c.enqueueProto([]byte(fmt.Sprintf(InfoProto, b))) + c.mu.Unlock() + } + } +} + +// Sends the INFO protocol of a gateway to all routes known by this server. +func (s *Server) forwardNewGatewayToLocalCluster(oinfo *Info) { + // Need to protect s.routes here, so use server's lock + s.mu.Lock() + defer s.mu.Unlock() + + // We don't really need the ID to be set, but, we need to make sure + // that it is not set to the server ID so that if we were to connect + // to an older server that does not expect a "gateway" INFO, it + // would think that it needs to create an implicit route (since info.ID + // would not match the route's remoteID), but will fail to do so because + // the sent protocol will not have host/port defined. + info := &Info{ + ID: "GW" + s.info.ID, + Name: s.getOpts().ServerName, + Gateway: oinfo.Gateway, + GatewayURLs: oinfo.GatewayURLs, + GatewayCmd: gatewayCmdGossip, + } + b, _ := json.Marshal(info) + infoJSON := []byte(fmt.Sprintf(InfoProto, b)) + + s.forEachRemote(func(r *client) { + r.mu.Lock() + r.enqueueProto(infoJSON) + r.mu.Unlock() + }) +} + +// Sends queue subscriptions interest to remote gateway. +// This is sent from the inbound side, that is, the side that receives +// messages from the remote's outbound connection. This side is +// the one sending the subscription interest. +func (s *Server) sendQueueSubsToGateway(c *client) { + s.sendSubsToGateway(c, _EMPTY_) +} + +// Sends all subscriptions for the given account to the remove gateway +// This is sent from the inbound side, that is, the side that receives +// messages from the remote's outbound connection. This side is +// the one sending the subscription interest. +func (s *Server) sendAccountSubsToGateway(c *client, accName string) { + s.sendSubsToGateway(c, accName) +} + +func gwBuildSubProto(buf *bytes.Buffer, accName string, acc map[string]*sitally, doQueues bool) { + for saq, si := range acc { + if doQueues && si.q || !doQueues && !si.q { + buf.Write(rSubBytes) + buf.WriteString(accName) + buf.WriteByte(' ') + // For queue subs (si.q is true), saq will be + // subject + ' ' + queue, for plain subs, this is + // just the subject. + buf.WriteString(saq) + if doQueues { + buf.WriteString(" 1") + } + buf.WriteString(CR_LF) + } + } +} + +// Sends subscriptions to remote gateway. +func (s *Server) sendSubsToGateway(c *client, accountName string) { + var ( + bufa = [32 * 1024]byte{} + bbuf = bytes.NewBuffer(bufa[:0]) + ) + + gw := s.gateway + + // This needs to run under this lock for the whole duration + gw.pasi.Lock() + defer gw.pasi.Unlock() + + // If account is specified... + if accountName != _EMPTY_ { + // Simply send all plain subs (no queues) for this specific account + gwBuildSubProto(bbuf, accountName, gw.pasi.m[accountName], false) + // Instruct to send all subs (RS+/-) for this account from now on. + c.mu.Lock() + e := c.gw.insim[accountName] + if e == nil { + e = &insie{} + c.gw.insim[accountName] = e + } + e.mode = InterestOnly + c.mu.Unlock() + } else { + // Send queues for all accounts + for accName, acc := range gw.pasi.m { + gwBuildSubProto(bbuf, accName, acc, true) + } + } + + buf := bbuf.Bytes() + + // Nothing to send. + if len(buf) == 0 { + return + } + if len(buf) > cap(bufa) { + s.Debugf("Sending subscriptions to %q, buffer size: %v", c.gw.name, len(buf)) + } + // Send + c.mu.Lock() + c.enqueueProto(buf) + c.Debugf("Sent queue subscriptions to gateway") + c.mu.Unlock() +} + +// This is invoked when getting an INFO protocol for gateway on the ROUTER port. +// This function will then execute appropriate function based on the command +// contained in the protocol. +// +func (s *Server) processGatewayInfoFromRoute(info *Info, routeSrvID string) { + switch info.GatewayCmd { + case gatewayCmdGossip: + s.processImplicitGateway(info) + default: + s.Errorf("Unknown command %d from server %v", info.GatewayCmd, routeSrvID) + } +} + +// Sends INFO protocols to the given route connection for each known Gateway. +// These will be processed by the route and delegated to the gateway code to +// invoke processImplicitGateway. +func (s *Server) sendGatewayConfigsToRoute(route *client) { + gw := s.gateway + gw.RLock() + // Send only to gateways for which we have actual outbound connection to. + if len(gw.out) == 0 { + gw.RUnlock() + return + } + // Collect gateway configs for which we have an outbound connection. + gwCfgsa := [16]*gatewayCfg{} + gwCfgs := gwCfgsa[:0] + for _, c := range gw.out { + c.mu.Lock() + if c.gw.cfg != nil { + gwCfgs = append(gwCfgs, c.gw.cfg) + } + c.mu.Unlock() + } + gw.RUnlock() + if len(gwCfgs) == 0 { + return + } + + // Check forwardNewGatewayToLocalCluster() as to why we set ID this way. + info := Info{ + ID: "GW" + s.info.ID, + GatewayCmd: gatewayCmdGossip, + } + for _, cfg := range gwCfgs { + urls := cfg.getURLsAsStrings() + if len(urls) > 0 { + info.Gateway = cfg.Name + info.GatewayURLs = urls + b, _ := json.Marshal(&info) + route.mu.Lock() + route.enqueueProto([]byte(fmt.Sprintf(InfoProto, b))) + route.mu.Unlock() + } + } +} + +// Initiates a gateway connection using the info contained in the INFO protocol. +// If a gateway with the same name is already registered (either because explicitly +// configured, or already implicitly connected), this function will augmment the +// remote URLs with URLs present in the info protocol and return. +// Otherwise, this function will register this remote (to prevent multiple connections +// to the same remote) and call solicitGateway (which will run in a different go-routine). +func (s *Server) processImplicitGateway(info *Info) { + s.gateway.Lock() + defer s.gateway.Unlock() + // Name of the gateway to connect to is the Info.Gateway field. + gwName := info.Gateway + // If this is our name, bail. + if gwName == s.gateway.name { + return + } + // Check if we already have this config, and if so, we are done + cfg := s.gateway.remotes[gwName] + if cfg != nil { + // However, possibly augment the list of URLs with the given + // info.GatewayURLs content. + cfg.Lock() + cfg.addURLs(info.GatewayURLs) + cfg.Unlock() + return + } + opts := s.getOpts() + cfg = &gatewayCfg{ + RemoteGatewayOpts: &RemoteGatewayOpts{Name: gwName}, + hash: getGWHash(gwName), + oldHash: getOldHash(gwName), + urls: make(map[string]*url.URL, len(info.GatewayURLs)), + implicit: true, + } + if opts.Gateway.TLSConfig != nil { + cfg.TLSConfig = opts.Gateway.TLSConfig.Clone() + cfg.TLSTimeout = opts.Gateway.TLSTimeout + } + + // Since we know we don't have URLs (no config, so just based on what we + // get from INFO), directly call addURLs(). We don't need locking since + // we just created that structure and no one else has access to it yet. + cfg.addURLs(info.GatewayURLs) + // If there is no URL, we can't proceed. + if len(cfg.urls) == 0 { + return + } + s.gateway.remotes[gwName] = cfg + s.startGoRoutine(func() { + s.solicitGateway(cfg, true) + s.grWG.Done() + }) +} + +// NumOutboundGateways is public here mostly for testing. +func (s *Server) NumOutboundGateways() int { + return s.numOutboundGateways() +} + +// Returns the number of outbound gateway connections +func (s *Server) numOutboundGateways() int { + s.gateway.RLock() + n := len(s.gateway.out) + s.gateway.RUnlock() + return n +} + +// Returns the number of inbound gateway connections +func (s *Server) numInboundGateways() int { + s.gateway.RLock() + n := len(s.gateway.in) + s.gateway.RUnlock() + return n +} + +// Returns the remoteGateway (if any) that has the given `name` +func (s *Server) getRemoteGateway(name string) *gatewayCfg { + s.gateway.RLock() + cfg := s.gateway.remotes[name] + s.gateway.RUnlock() + return cfg +} + +// Used in tests +func (g *gatewayCfg) bumpConnAttempts() { + g.Lock() + g.connAttempts++ + g.Unlock() +} + +// Used in tests +func (g *gatewayCfg) getConnAttempts() int { + g.Lock() + ca := g.connAttempts + g.Unlock() + return ca +} + +// Used in tests +func (g *gatewayCfg) resetConnAttempts() { + g.Lock() + g.connAttempts = 0 + g.Unlock() +} + +// Returns if this remote gateway is implicit or not. +func (g *gatewayCfg) isImplicit() bool { + g.RLock() + ii := g.implicit + g.RUnlock() + return ii +} + +// getURLs returns an array of URLs in random order suitable for +// an iteration to try to connect. +func (g *gatewayCfg) getURLs() []*url.URL { + g.RLock() + a := make([]*url.URL, 0, len(g.urls)) + for _, u := range g.urls { + a = append(a, u) + } + g.RUnlock() + // Map iteration is random, but not that good with small maps. + rand.Shuffle(len(a), func(i, j int) { + a[i], a[j] = a[j], a[i] + }) + return a +} + +// Similar to getURLs but returns the urls as an array of strings. +func (g *gatewayCfg) getURLsAsStrings() []string { + g.RLock() + a := make([]string, 0, len(g.urls)) + for _, u := range g.urls { + a = append(a, u.Host) + } + g.RUnlock() + return a +} + +// updateURLs creates the urls map with the content of the config's URLs array +// and the given array that we get from the INFO protocol. +func (g *gatewayCfg) updateURLs(infoURLs []string) { + g.Lock() + // Clear the map... + g.urls = make(map[string]*url.URL, len(g.URLs)+len(infoURLs)) + // Add the urls from the config URLs array. + for _, u := range g.URLs { + g.urls[u.Host] = u + } + // Then add the ones from the infoURLs array we got. + g.addURLs(infoURLs) + // The call above will set varzUpdateURLs only when finding ULRs in infoURLs + // that are not present in the config. That does not cover the case where + // previously "discovered" URLs are now gone. We could check "before" size + // of g.urls and if bigger than current size, set the boolean to true. + // Not worth it... simply set this to true to allow a refresh of gateway + // URLs in varz. + g.varzUpdateURLs = true + g.Unlock() +} + +// Saves the hostname of the given URL (if not already done). +// This may be used as the ServerName of the TLSConfig when initiating a +// TLS connection. +// Write lock held on entry. +func (g *gatewayCfg) saveTLSHostname(u *url.URL) { + if g.TLSConfig != nil && g.tlsName == "" && net.ParseIP(u.Hostname()) == nil { + g.tlsName = u.Hostname() + } +} + +// add URLs from the given array to the urls map only if not already present. +// remoteGateway write lock is assumed to be held on entry. +// Write lock is held on entry. +func (g *gatewayCfg) addURLs(infoURLs []string) { + var scheme string + if g.TLSConfig != nil { + scheme = "tls" + } else { + scheme = "nats" + } + for _, iu := range infoURLs { + if _, present := g.urls[iu]; !present { + // Urls in Info.GatewayURLs come without scheme. Add it to parse + // the url (otherwise it fails). + if u, err := url.Parse(fmt.Sprintf("%s://%s", scheme, iu)); err == nil { + // Also, if a tlsName has not been set yet and we are dealing + // with a hostname and not a bare IP, save the hostname. + g.saveTLSHostname(u) + // Use u.Host for the key. + g.urls[u.Host] = u + // Signal that we have updated the list. Used by monitoring code. + g.varzUpdateURLs = true + } + } + } +} + +// Adds this URL to the set of Gateway URLs. +// Returns true if the URL has been added, false otherwise. +// Server lock held on entry +func (s *Server) addGatewayURL(urlStr string) bool { + s.gateway.Lock() + added := s.gateway.URLs.addUrl(urlStr) + if added { + s.gateway.generateInfoJSON() + } + s.gateway.Unlock() + return added +} + +// Removes this URL from the set of gateway URLs. +// Returns true if the URL has been removed, false otherwise. +// Server lock held on entry +func (s *Server) removeGatewayURL(urlStr string) bool { + if s.isShuttingDown() { + return false + } + s.gateway.Lock() + removed := s.gateway.URLs.removeUrl(urlStr) + if removed { + s.gateway.generateInfoJSON() + } + s.gateway.Unlock() + return removed +} + +// Sends a Gateway's INFO to all inbound GW connections. +// Server lock is held on entry +func (s *Server) sendAsyncGatewayInfo() { + s.gateway.RLock() + for _, ig := range s.gateway.in { + ig.mu.Lock() + ig.enqueueProto(s.gateway.infoJSON) + ig.mu.Unlock() + } + s.gateway.RUnlock() +} + +// This returns the URL of the Gateway listen spec, or empty string +// if the server has no gateway configured. +func (s *Server) getGatewayURL() string { + s.gateway.RLock() + url := s.gateway.URL + s.gateway.RUnlock() + return url +} + +// Returns this server gateway name. +// Same than calling s.gateway.getName() +func (s *Server) getGatewayName() string { + // This is immutable + return s.gateway.name +} + +// All gateway connections (outbound and inbound) are put in the given map. +func (s *Server) getAllGatewayConnections(conns map[uint64]*client) { + gw := s.gateway + gw.RLock() + for _, c := range gw.out { + c.mu.Lock() + cid := c.cid + c.mu.Unlock() + conns[cid] = c + } + for cid, c := range gw.in { + conns[cid] = c + } + gw.RUnlock() +} + +// Register the given gateway connection (*client) in the inbound gateways +// map. The key is the connection ID (like for clients and routes). +func (s *Server) registerInboundGatewayConnection(cid uint64, gwc *client) { + s.gateway.Lock() + s.gateway.in[cid] = gwc + s.gateway.Unlock() +} + +// Register the given gateway connection (*client) in the outbound gateways +// map with the given name as the key. +func (s *Server) registerOutboundGatewayConnection(name string, gwc *client) bool { + s.gateway.Lock() + if _, exist := s.gateway.out[name]; exist { + s.gateway.Unlock() + return false + } + s.gateway.out[name] = gwc + s.gateway.outo = append(s.gateway.outo, gwc) + s.gateway.orderOutboundConnectionsLocked() + s.gateway.Unlock() + return true +} + +// Returns the outbound gateway connection (*client) with the given name, +// or nil if not found +func (s *Server) getOutboundGatewayConnection(name string) *client { + s.gateway.RLock() + gwc := s.gateway.out[name] + s.gateway.RUnlock() + return gwc +} + +// Returns all outbound gateway connections in the provided array. +// The order of the gateways is suited for the sending of a message. +// Current ordering is based on individual gateway's RTT value. +func (s *Server) getOutboundGatewayConnections(a *[]*client) { + s.gateway.RLock() + for i := 0; i < len(s.gateway.outo); i++ { + *a = append(*a, s.gateway.outo[i]) + } + s.gateway.RUnlock() +} + +// Orders the array of outbound connections. +// Current ordering is by lowest RTT. +// Gateway write lock is held on entry +func (g *srvGateway) orderOutboundConnectionsLocked() { + // Order the gateways by lowest RTT + slices.SortFunc(g.outo, func(i, j *client) int { return cmp.Compare(i.getRTTValue(), j.getRTTValue()) }) +} + +// Orders the array of outbound connections. +// Current ordering is by lowest RTT. +func (g *srvGateway) orderOutboundConnections() { + g.Lock() + g.orderOutboundConnectionsLocked() + g.Unlock() +} + +// Returns all inbound gateway connections in the provided array +func (s *Server) getInboundGatewayConnections(a *[]*client) { + s.gateway.RLock() + for _, gwc := range s.gateway.in { + *a = append(*a, gwc) + } + s.gateway.RUnlock() +} + +// This is invoked when a gateway connection is closed and the server +// is removing this connection from its state. +func (s *Server) removeRemoteGatewayConnection(c *client) { + c.mu.Lock() + cid := c.cid + isOutbound := c.gw.outbound + gwName := c.gw.name + if isOutbound && c.gw.outsim != nil { + // We do this to allow the GC to release this connection. + // Since the map is used by the rest of the code without client lock, + // we can't simply set it to nil, instead, just make sure we empty it. + c.gw.outsim.Range(func(k, _ any) bool { + c.gw.outsim.Delete(k) + return true + }) + } + c.mu.Unlock() + + gw := s.gateway + gw.Lock() + if isOutbound { + delete(gw.out, gwName) + louto := len(gw.outo) + reorder := false + for i := 0; i < len(gw.outo); i++ { + if gw.outo[i] == c { + // If last, simply remove and no need to reorder + if i != louto-1 { + gw.outo[i] = gw.outo[louto-1] + reorder = true + } + gw.outo = gw.outo[:louto-1] + } + } + if reorder { + gw.orderOutboundConnectionsLocked() + } + } else { + delete(gw.in, cid) + } + gw.Unlock() + s.removeFromTempClients(cid) + + if isOutbound { + // Update number of totalQSubs for this gateway + qSubsRemoved := int64(0) + c.mu.Lock() + for _, sub := range c.subs { + if sub.queue != nil { + qSubsRemoved++ + } + } + c.subs = nil + c.mu.Unlock() + // Update total count of qsubs in remote gateways. + atomic.AddInt64(&c.srv.gateway.totalQSubs, -qSubsRemoved) + + } else { + var subsa [1024]*subscription + var subs = subsa[:0] + + // For inbound GW connection, if we have subs, those are + // local subs on "_R_." subjects. + c.mu.Lock() + for _, sub := range c.subs { + subs = append(subs, sub) + } + c.subs = nil + c.mu.Unlock() + for _, sub := range subs { + c.removeReplySub(sub) + } + } +} + +// GatewayAddr returns the net.Addr object for the gateway listener. +func (s *Server) GatewayAddr() *net.TCPAddr { + s.mu.Lock() + defer s.mu.Unlock() + if s.gatewayListener == nil { + return nil + } + return s.gatewayListener.Addr().(*net.TCPAddr) +} + +// A- protocol received from the remote after sending messages +// on an account that it has no interest in. Mark this account +// with a "no interest" marker to prevent further messages send. +// +func (c *client) processGatewayAccountUnsub(accName string) { + // Just to indicate activity around "subscriptions" events. + c.in.subs++ + // This account may have an entry because of queue subs. + // If that's the case, we can reset the no-interest map, + // but not set the entry to nil. + setToNil := true + if ei, ok := c.gw.outsim.Load(accName); ei != nil { + e := ei.(*outsie) + e.Lock() + // Reset the no-interest map if we have queue subs + // and don't set the entry to nil. + if e.qsubs > 0 { + e.ni = make(map[string]struct{}) + setToNil = false + } + e.Unlock() + } else if ok { + // Already set to nil, so skip + setToNil = false + } + if setToNil { + c.gw.outsim.Store(accName, nil) + } +} + +// A+ protocol received from remote gateway if it had previously +// sent an A-. Clear the "no interest" marker for this account. +// +func (c *client) processGatewayAccountSub(accName string) error { + // Just to indicate activity around "subscriptions" events. + c.in.subs++ + // If this account has an entry because of queue subs, we + // can't delete the entry. + remove := true + if ei, ok := c.gw.outsim.Load(accName); ei != nil { + e := ei.(*outsie) + e.Lock() + if e.qsubs > 0 { + remove = false + } + e.Unlock() + } else if !ok { + // There is no entry, so skip + remove = false + } + if remove { + c.gw.outsim.Delete(accName) + } + return nil +} + +// RS- protocol received from the remote after sending messages +// on a subject that it has no interest in (but knows about the +// account). Mark this subject with a "no interest" marker to +// prevent further messages being sent. +// If in modeInterestOnly or for a queue sub, remove from +// the sublist if present. +// +func (c *client) processGatewayRUnsub(arg []byte) error { + _, accName, subject, queue, err := c.parseUnsubProto(arg, true, false) + if err != nil { + return fmt.Errorf("processGatewaySubjectUnsub %s", err.Error()) + } + + var ( + e *outsie + useSl bool + newe bool + callUpdate bool + srv *Server + sub *subscription + ) + + // Possibly execute this on exit after all locks have been released. + // If callUpdate is true, srv and sub will be not nil. + defer func() { + if callUpdate { + srv.updateInterestForAccountOnGateway(accName, sub, -1) + } + }() + + c.mu.Lock() + if c.gw.outsim == nil { + c.Errorf("Received RS- from gateway on inbound connection") + c.mu.Unlock() + c.closeConnection(ProtocolViolation) + return nil + } + defer c.mu.Unlock() + // If closed, c.subs map will be nil, so bail out. + if c.isClosed() { + return nil + } + + ei, _ := c.gw.outsim.Load(accName) + if ei != nil { + e = ei.(*outsie) + e.Lock() + defer e.Unlock() + // If there is an entry, for plain sub we need + // to know if we should store the sub + useSl = queue != nil || e.mode != Optimistic + } else if queue != nil { + // should not even happen... + c.Debugf("Received RS- without prior RS+ for subject %q, queue %q", subject, queue) + return nil + } else { + // Plain sub, assume optimistic sends, create entry. + e = &outsie{ni: make(map[string]struct{}), sl: NewSublistForServer(c.srv)} + newe = true + } + // This is when a sub or queue sub is supposed to be in + // the sublist. Look for it and remove. + if useSl { + var ok bool + key := arg + // m[string()] does not cause mem allocation + sub, ok = c.subs[string(key)] + // if RS- for a sub that we don't have, just ignore. + if !ok { + return nil + } + if e.sl.Remove(sub) == nil { + delete(c.subs, bytesToString(key)) + if queue != nil { + e.qsubs-- + atomic.AddInt64(&c.srv.gateway.totalQSubs, -1) + } + // If last, we can remove the whole entry only + // when in optimistic mode and there is no element + // in the `ni` map. + if e.sl.Count() == 0 && e.mode == Optimistic && len(e.ni) == 0 { + c.gw.outsim.Delete(accName) + } + } + // We are going to call updateInterestForAccountOnGateway on exit. + srv = c.srv + callUpdate = true + } else { + e.ni[string(subject)] = struct{}{} + if newe { + c.gw.outsim.Store(accName, e) + } + } + return nil +} + +// For plain subs, RS+ protocol received from remote gateway if it +// had previously sent a RS-. Clear the "no interest" marker for +// this subject (under this account). +// For queue subs, or if in modeInterestOnly, register interest +// from remote gateway. +// +func (c *client) processGatewayRSub(arg []byte) error { + // Indicate activity. + c.in.subs++ + + var ( + queue []byte + qw int32 + ) + + args := splitArg(arg) + switch len(args) { + case 2: + case 4: + queue = args[2] + qw = int32(parseSize(args[3])) + default: + return fmt.Errorf("processGatewaySubjectSub Parse Error: '%s'", arg) + } + accName := args[0] + subject := args[1] + + var ( + e *outsie + useSl bool + newe bool + callUpdate bool + srv *Server + sub *subscription + ) + + // Possibly execute this on exit after all locks have been released. + // If callUpdate is true, srv and sub will be not nil. + defer func() { + if callUpdate { + srv.updateInterestForAccountOnGateway(string(accName), sub, 1) + } + }() + + c.mu.Lock() + if c.gw.outsim == nil { + c.Errorf("Received RS+ from gateway on inbound connection") + c.mu.Unlock() + c.closeConnection(ProtocolViolation) + return nil + } + defer c.mu.Unlock() + // If closed, c.subs map will be nil, so bail out. + if c.isClosed() { + return nil + } + + ei, _ := c.gw.outsim.Load(bytesToString(accName)) + // We should always have an existing entry for plain subs because + // in optimistic mode we would have received RS- first, and + // in full knowledge, we are receiving RS+ for an account after + // getting many RS- from the remote.. + if ei != nil { + e = ei.(*outsie) + e.Lock() + defer e.Unlock() + useSl = queue != nil || e.mode != Optimistic + } else if queue == nil { + return nil + } else { + e = &outsie{ni: make(map[string]struct{}), sl: NewSublistForServer(c.srv)} + newe = true + useSl = true + } + if useSl { + var key []byte + // We store remote subs by account/subject[/queue]. + // For queue, remove the trailing weight + if queue != nil { + key = arg[:len(arg)-len(args[3])-1] + } else { + key = arg + } + // If RS+ for a sub that we already have, ignore. + // (m[string()] does not allocate memory) + if _, ok := c.subs[string(key)]; ok { + return nil + } + // new subscription. copy subject (and queue) to + // not reference the underlying possibly big buffer. + var csubject []byte + var cqueue []byte + if queue != nil { + // make single allocation and use different slices + // to point to subject and queue name. + cbuf := make([]byte, len(subject)+1+len(queue)) + copy(cbuf, key[len(accName)+1:]) + csubject = cbuf[:len(subject)] + cqueue = cbuf[len(subject)+1:] + } else { + csubject = make([]byte, len(subject)) + copy(csubject, subject) + } + sub = &subscription{client: c, subject: csubject, queue: cqueue, qw: qw} + // If no error inserting in sublist... + if e.sl.Insert(sub) == nil { + c.subs[string(key)] = sub + if queue != nil { + e.qsubs++ + atomic.AddInt64(&c.srv.gateway.totalQSubs, 1) + } + if newe { + c.gw.outsim.Store(string(accName), e) + } + } + // We are going to call updateInterestForAccountOnGateway on exit. + srv = c.srv + callUpdate = true + } else { + subj := bytesToString(subject) + // If this is an RS+ for a wc subject, then + // remove from the no interest map all subjects + // that are a subset of this wc subject. + if subjectHasWildcard(subj) { + for k := range e.ni { + if subjectIsSubsetMatch(k, subj) { + delete(e.ni, k) + } + } + } else { + delete(e.ni, subj) + } + } + return nil +} + +// Returns true if this gateway has possible interest in the +// given account/subject (which means, it does not have a registered +// no-interest on the account and/or subject) and the sublist result +// for queue subscriptions. +// +func (c *client) gatewayInterest(acc string, subj []byte) (bool, *SublistResult) { + ei, accountInMap := c.gw.outsim.Load(acc) + // If there is an entry for this account and ei is nil, + // it means that the remote is not interested at all in + // this account and we could not possibly have queue subs. + if accountInMap && ei == nil { + return false, nil + } + // Assume interest if account not in map, unless we support + // only interest-only mode. + psi := !accountInMap && !c.gw.interestOnlyMode + var r *SublistResult + if accountInMap { + // If in map, check for subs interest with sublist. + e := ei.(*outsie) + e.RLock() + // Unless each side has agreed on interest-only mode, + // we may be in transition to modeInterestOnly + // but until e.ni is nil, use it to know if we + // should suppress interest or not. + if !c.gw.interestOnlyMode && e.ni != nil { + if _, inMap := e.ni[string(subj)]; !inMap { + psi = true + } + } + // If we are in modeInterestOnly (e.ni will be nil) + // or if we have queue subs, we also need to check sl.Match. + if e.ni == nil || e.qsubs > 0 { + r = e.sl.MatchBytes(subj) + if len(r.psubs) > 0 { + psi = true + } + } + e.RUnlock() + // Since callers may just check if the sublist result is nil or not, + // make sure that if what is returned by sl.Match() is the emptyResult, then + // we return nil to the caller. + if r == emptyResult { + r = nil + } + } + return psi, r +} + +// switchAccountToInterestMode will switch an account over to interestMode. +// Lock should NOT be held. +func (s *Server) switchAccountToInterestMode(accName string) { + gwsa := [16]*client{} + gws := gwsa[:0] + s.getInboundGatewayConnections(&gws) + + for _, gin := range gws { + var e *insie + var ok bool + + gin.mu.Lock() + if e, ok = gin.gw.insim[accName]; !ok || e == nil { + e = &insie{} + gin.gw.insim[accName] = e + } + // Do it only if we are in Optimistic mode + if e.mode == Optimistic { + gin.gatewaySwitchAccountToSendAllSubs(e, accName) + } + gin.mu.Unlock() + } +} + +// This is invoked when registering (or unregistering) the first +// (or last) subscription on a given account/subject. For each +// GWs inbound connections, we will check if we need to send an RS+ or A+ +// protocol. +func (s *Server) maybeSendSubOrUnsubToGateways(accName string, sub *subscription, added bool) { + if sub.queue != nil { + return + } + gwsa := [16]*client{} + gws := gwsa[:0] + s.getInboundGatewayConnections(&gws) + if len(gws) == 0 { + return + } + var ( + rsProtoa [512]byte + rsProto []byte + accProtoa [256]byte + accProto []byte + proto []byte + subject = bytesToString(sub.subject) + hasWC = subjectHasWildcard(subject) + ) + for _, c := range gws { + proto = nil + c.mu.Lock() + e, inMap := c.gw.insim[accName] + // If there is a inbound subject interest entry... + if e != nil { + sendProto := false + // In optimistic mode, we care only about possibly sending RS+ (or A+) + // so if e.ni is not nil we do things only when adding a new subscription. + if e.ni != nil && added { + // For wildcard subjects, we will remove from our no-interest + // map, all subjects that are a subset of this wc subject, but we + // still send the wc subject and let the remote do its own cleanup. + if hasWC { + for enis := range e.ni { + if subjectIsSubsetMatch(enis, subject) { + delete(e.ni, enis) + sendProto = true + } + } + } else if _, noInterest := e.ni[subject]; noInterest { + delete(e.ni, subject) + sendProto = true + } + } else if e.mode == InterestOnly { + // We are in the mode where we always send RS+/- protocols. + sendProto = true + } + if sendProto { + if rsProto == nil { + // Construct the RS+/- only once + proto = rsProtoa[:0] + if added { + proto = append(proto, rSubBytes...) + } else { + proto = append(proto, rUnsubBytes...) + } + proto = append(proto, accName...) + proto = append(proto, ' ') + proto = append(proto, sub.subject...) + proto = append(proto, CR_LF...) + rsProto = proto + } else { + // Point to the already constructed RS+/- + proto = rsProto + } + } + } else if added && inMap { + // Here, we have a `nil` entry for this account in + // the map, which means that we have previously sent + // an A-. We have a new subscription, so we need to + // send an A+ and delete the entry from the map so + // that we do this only once. + delete(c.gw.insim, accName) + if accProto == nil { + // Construct the A+ only once + proto = accProtoa[:0] + proto = append(proto, aSubBytes...) + proto = append(proto, accName...) + proto = append(proto, CR_LF...) + accProto = proto + } else { + // Point to the already constructed A+ + proto = accProto + } + } + if proto != nil { + c.enqueueProto(proto) + if c.trace { + c.traceOutOp("", proto[:len(proto)-LEN_CR_LF]) + } + } + c.mu.Unlock() + } +} + +// This is invoked when the first (or last) queue subscription on a +// given subject/group is registered (or unregistered). Sent to all +// inbound gateways. +func (s *Server) sendQueueSubOrUnsubToGateways(accName string, qsub *subscription, added bool) { + if qsub.queue == nil { + return + } + + gwsa := [16]*client{} + gws := gwsa[:0] + s.getInboundGatewayConnections(&gws) + if len(gws) == 0 { + return + } + + var protoa [512]byte + var proto []byte + for _, c := range gws { + if proto == nil { + proto = protoa[:0] + if added { + proto = append(proto, rSubBytes...) + } else { + proto = append(proto, rUnsubBytes...) + } + proto = append(proto, accName...) + proto = append(proto, ' ') + proto = append(proto, qsub.subject...) + proto = append(proto, ' ') + proto = append(proto, qsub.queue...) + if added { + // For now, just use 1 for the weight + proto = append(proto, ' ', '1') + } + proto = append(proto, CR_LF...) + } + c.mu.Lock() + // If we add a queue sub, and we had previously sent an A-, + // we don't need to send an A+ here, but we need to clear + // the fact that we did sent the A- so that we don't send + // an A+ when we will get the first non-queue sub registered. + if added { + if ei, ok := c.gw.insim[accName]; ok && ei == nil { + delete(c.gw.insim, accName) + } + } + c.enqueueProto(proto) + if c.trace { + c.traceOutOp("", proto[:len(proto)-LEN_CR_LF]) + } + c.mu.Unlock() + } +} + +// This is invoked when a subscription (plain or queue) is +// added/removed locally or in our cluster. We use ref counting +// to know when to update the inbound gateways. +// +func (s *Server) gatewayUpdateSubInterest(accName string, sub *subscription, change int32) { + if sub.si { + return + } + + var ( + keya [1024]byte + key = keya[:0] + entry *sitally + isNew bool + ) + + s.gateway.pasi.Lock() + defer s.gateway.pasi.Unlock() + + accMap := s.gateway.pasi.m + + // First see if we have the account + st := accMap[accName] + if st == nil { + // Ignore remove of something we don't have + if change < 0 { + return + } + st = make(map[string]*sitally) + accMap[accName] = st + isNew = true + } + // Lookup: build the key as subject[+' '+queue] + key = append(key, sub.subject...) + if sub.queue != nil { + key = append(key, ' ') + key = append(key, sub.queue...) + } + if !isNew { + entry = st[string(key)] + } + first := false + last := false + if entry == nil { + // Ignore remove of something we don't have + if change < 0 { + return + } + entry = &sitally{n: change, q: sub.queue != nil} + st[string(key)] = entry + first = true + } else { + entry.n += change + if entry.n <= 0 { + delete(st, bytesToString(key)) + last = true + if len(st) == 0 { + delete(accMap, accName) + } + } + } + if sub.client != nil { + rsubs := &s.gateway.rsubs + acc := sub.client.acc + sli, _ := rsubs.Load(acc) + if change > 0 { + var sl *Sublist + if sli == nil { + sl = NewSublistNoCache() + rsubs.Store(acc, sl) + } else { + sl = sli.(*Sublist) + } + sl.Insert(sub) + time.AfterFunc(s.gateway.recSubExp, func() { + sl.Remove(sub) + }) + } else if sli != nil { + sl := sli.(*Sublist) + sl.Remove(sub) + if sl.Count() == 0 { + rsubs.Delete(acc) + } + } + } + if first || last { + if entry.q { + s.sendQueueSubOrUnsubToGateways(accName, sub, first) + } else { + s.maybeSendSubOrUnsubToGateways(accName, sub, first) + } + } +} + +// Returns true if the given subject is a GW routed reply subject, +// that is, starts with $GNR and is long enough to contain cluster/server hash +// and subject. +func isGWRoutedReply(subj []byte) bool { + return len(subj) > gwSubjectOffset && bytesToString(subj[:gwReplyPrefixLen]) == gwReplyPrefix +} + +// Same than isGWRoutedReply but accepts the old prefix $GR and returns +// a boolean indicating if this is the old prefix +func isGWRoutedSubjectAndIsOldPrefix(subj []byte) (bool, bool) { + if isGWRoutedReply(subj) { + return true, false + } + if len(subj) > oldGWReplyStart && bytesToString(subj[:oldGWReplyPrefixLen]) == oldGWReplyPrefix { + return true, true + } + return false, false +} + +// Returns true if subject starts with "$GNR.". This is to check that +// clients can't publish on this subject. +func hasGWRoutedReplyPrefix(subj []byte) bool { + return len(subj) > gwReplyPrefixLen && bytesToString(subj[:gwReplyPrefixLen]) == gwReplyPrefix +} + +// Evaluates if the given reply should be mapped or not. +func (g *srvGateway) shouldMapReplyForGatewaySend(acc *Account, reply []byte) bool { + // If for this account there is a recent matching subscription interest + // then we will map. + sli, _ := g.rsubs.Load(acc) + if sli == nil { + return false + } + sl := sli.(*Sublist) + if sl.Count() > 0 { + if sl.HasInterest(string(reply)) { + return true + } + } + + return false +} + +var subPool = &sync.Pool{ + New: func() any { + return &subscription{} + }, +} + +// May send a message to all outbound gateways. It is possible +// that the message is not sent to a given gateway if for instance +// it is known that this gateway has no interest in the account or +// subject, etc.. +// When invoked from a LEAF connection, `checkLeafQF` should be passed as `true` +// so that we skip any queue subscription interest that is not part of the +// `c.pa.queues` filter (similar to what we do in `processMsgResults`). However, +// when processing service imports, then this boolean should be passes as `false`, +// regardless if it is a LEAF connection or not. +// +func (c *client) sendMsgToGateways(acc *Account, msg, subject, reply []byte, qgroups [][]byte, checkLeafQF bool) bool { + // We had some times when we were sending across a GW with no subject, and the other side would break + // due to parser error. These need to be fixed upstream but also double check here. + if len(subject) == 0 { + return false + } + gwsa := [16]*client{} + gws := gwsa[:0] + // This is in fast path, so avoid calling functions when possible. + // Get the outbound connections in place instead of calling + // getOutboundGatewayConnections(). + srv := c.srv + gw := srv.gateway + gw.RLock() + for i := 0; i < len(gw.outo); i++ { + gws = append(gws, gw.outo[i]) + } + thisClusterReplyPrefix := gw.replyPfx + thisClusterOldReplyPrefix := gw.oldReplyPfx + gw.RUnlock() + if len(gws) == 0 { + return false + } + + // Copy off original pa in case it changes. + pa := c.pa + + mt, _ := c.isMsgTraceEnabled() + + var ( + queuesa = [512]byte{} + queues = queuesa[:0] + accName = acc.Name + mreplya [256]byte + mreply []byte + dstHash []byte + checkReply = len(reply) > 0 + didDeliver bool + prodIsMQTT = c.isMqtt() + dlvMsgs int64 + dlvExtraSz int64 + ) + + // Get a subscription from the pool + sub := subPool.Get().(*subscription) + + // Check if the subject is on the reply prefix, if so, we + // need to send that message directly to the origin cluster. + directSend, old := isGWRoutedSubjectAndIsOldPrefix(subject) + if directSend { + if old { + dstHash = subject[oldGWReplyPrefixLen : oldGWReplyStart-1] + } else { + dstHash = subject[gwClusterOffset : gwClusterOffset+gwHashLen] + } + } + for i := 0; i < len(gws); i++ { + gwc := gws[i] + if directSend { + gwc.mu.Lock() + var ok bool + if gwc.gw.cfg != nil { + if old { + ok = bytes.Equal(dstHash, gwc.gw.cfg.oldHash) + } else { + ok = bytes.Equal(dstHash, gwc.gw.cfg.hash) + } + } + gwc.mu.Unlock() + if !ok { + continue + } + } else { + // Plain sub interest and queue sub results for this account/subject + psi, qr := gwc.gatewayInterest(accName, subject) + if !psi && qr == nil { + continue + } + queues = queuesa[:0] + if qr != nil { + for i := 0; i < len(qr.qsubs); i++ { + qsubs := qr.qsubs[i] + if len(qsubs) > 0 { + queue := qsubs[0].queue + if checkLeafQF { + // Skip any queue that is not in the leaf's queue filter. + skip := true + for _, qn := range c.pa.queues { + if bytes.Equal(queue, qn) { + skip = false + break + } + } + if skip { + continue + } + // Now we still need to check that it was not delivered + // locally by checking the given `qgroups`. + } + add := true + for _, qn := range qgroups { + if bytes.Equal(queue, qn) { + add = false + break + } + } + if add { + qgroups = append(qgroups, queue) + queues = append(queues, queue...) + queues = append(queues, ' ') + } + } + } + } + if !psi && len(queues) == 0 { + continue + } + } + if checkReply { + // Check/map only once + checkReply = false + // Assume we will use original + mreply = reply + // Decide if we should map. + if gw.shouldMapReplyForGatewaySend(acc, reply) { + mreply = mreplya[:0] + gwc.mu.Lock() + useOldPrefix := gwc.gw.useOldPrefix + gwc.mu.Unlock() + if useOldPrefix { + mreply = append(mreply, thisClusterOldReplyPrefix...) + } else { + mreply = append(mreply, thisClusterReplyPrefix...) + } + mreply = append(mreply, reply...) + } + } + + // Assume original message + dmsg := msg + if mt != nil { + // If trace is enabled, we need to set the hop header per gateway. + dmsg = mt.setHopHeader(c, dmsg) + } + + // Setup the message header. + // Make sure we are an 'R' proto by default + c.msgb[0] = 'R' + mh := c.msgb[:msgHeadProtoLen] + mh = append(mh, accName...) + mh = append(mh, ' ') + mh = append(mh, subject...) + mh = append(mh, ' ') + if len(queues) > 0 { + if len(reply) > 0 { + mh = append(mh, "+ "...) // Signal that there is a reply. + mh = append(mh, mreply...) + mh = append(mh, ' ') + } else { + mh = append(mh, "| "...) // Only queues + } + mh = append(mh, queues...) + } else if len(reply) > 0 { + mh = append(mh, mreply...) + mh = append(mh, ' ') + } + // Headers + hasHeader := c.pa.hdr > 0 + canReceiveHeader := gwc.headers + + if hasHeader { + if canReceiveHeader { + mh[0] = 'H' + mh = append(mh, c.pa.hdb...) + mh = append(mh, ' ') + mh = append(mh, c.pa.szb...) + } else { + // If we are here we need to truncate the payload size + nsz := strconv.Itoa(c.pa.size - c.pa.hdr) + mh = append(mh, nsz...) + } + } else { + mh = append(mh, c.pa.szb...) + } + + mh = append(mh, CR_LF...) + + // We reuse the subscription object that we pass to deliverMsg. + // So set/reset important fields. + sub.nm, sub.max = 0, 0 + sub.client = gwc + sub.subject = subject + if c.deliverMsg(prodIsMQTT, sub, acc, subject, mreply, mh, dmsg, false) { + // We don't count internal deliveries so count only if sub.icb is nil + if sub.icb == nil { + dlvMsgs++ + dlvExtraSz += int64(len(dmsg) - len(msg)) + } + didDeliver = true + } + + // If we set the header reset the origin pub args. + if mt != nil { + c.pa = pa + } + } + if dlvMsgs > 0 { + totalBytes := dlvMsgs*int64(len(msg)) + dlvExtraSz + // For non MQTT producers, remove the CR_LF * number of messages + if !prodIsMQTT { + totalBytes -= dlvMsgs * int64(LEN_CR_LF) + } + if acc != nil { + acc.stats.Lock() + acc.stats.outMsgs += dlvMsgs + acc.stats.outBytes += totalBytes + acc.stats.gw.outMsgs += dlvMsgs + acc.stats.gw.outBytes += totalBytes + acc.stats.Unlock() + } + atomic.AddInt64(&srv.outMsgs, dlvMsgs) + atomic.AddInt64(&srv.outBytes, totalBytes) + } + // Done with subscription, put back to pool. We don't need + // to reset content since we explicitly set when using it. + // However, make sure to not hold a reference to a connection. + sub.client = nil + subPool.Put(sub) + return didDeliver +} + +// Possibly sends an A- to the remote gateway `c`. +// Invoked when processing an inbound message and the account is not found. +// A check under a lock that protects processing of SUBs and UNSUBs is +// done to make sure that we don't send the A- if a subscription has just +// been created at the same time, which would otherwise results in the +// remote never sending messages on this account until a new subscription +// is created. +func (s *Server) gatewayHandleAccountNoInterest(c *client, accName []byte) { + // Check and possibly send the A- under this lock. + s.gateway.pasi.Lock() + defer s.gateway.pasi.Unlock() + + si, inMap := s.gateway.pasi.m[string(accName)] + if inMap && si != nil && len(si) > 0 { + return + } + c.sendAccountUnsubToGateway(accName) +} + +// Helper that sends an A- to this remote gateway if not already done. +// This function should not be invoked directly but instead be invoked +// by functions holding the gateway.pasi's Lock. +func (c *client) sendAccountUnsubToGateway(accName []byte) { + // Check if we have sent the A- or not. + c.mu.Lock() + e, sent := c.gw.insim[string(accName)] + if e != nil || !sent { + // Add a nil value to indicate that we have sent an A- + // so that we know to send A+ when needed. + c.gw.insim[string(accName)] = nil + var protoa [256]byte + proto := protoa[:0] + proto = append(proto, aUnsubBytes...) + proto = append(proto, accName...) + proto = append(proto, CR_LF...) + c.enqueueProto(proto) + if c.trace { + c.traceOutOp("", proto[:len(proto)-LEN_CR_LF]) + } + } + c.mu.Unlock() +} + +// Possibly sends an A- for this account or RS- for this subject. +// Invoked when processing an inbound message and the account is found +// but there is no interest on this subject. +// A test is done under a lock that protects processing of SUBs and UNSUBs +// and if there is no subscription at this time, we send an A-. If there +// is at least a subscription, but no interest on this subject, we send +// an RS- for this subject (if not already done). +func (s *Server) gatewayHandleSubjectNoInterest(c *client, acc *Account, accName, subject []byte) { + s.gateway.pasi.Lock() + defer s.gateway.pasi.Unlock() + + // If there is no subscription for this account, we would normally + // send an A-, however, if this account has the internal subscription + // for service reply, send a specific RS- for the subject instead. + // Need to grab the lock here since sublist can change during reload. + acc.mu.RLock() + hasSubs := acc.sl.Count() > 0 || acc.siReply != nil + acc.mu.RUnlock() + + // If there is at least a subscription, possibly send RS- + if hasSubs { + sendProto := false + c.mu.Lock() + // Send an RS- protocol if not already done and only if + // not in the modeInterestOnly. + e := c.gw.insim[string(accName)] + if e == nil { + e = &insie{ni: make(map[string]struct{})} + e.ni[string(subject)] = struct{}{} + c.gw.insim[string(accName)] = e + sendProto = true + } else if e.ni != nil { + // If we are not in modeInterestOnly, check if we + // have already sent an RS- + if _, alreadySent := e.ni[string(subject)]; !alreadySent { + // TODO(ik): pick some threshold as to when + // we need to switch mode + if len(e.ni) >= gatewayMaxRUnsubBeforeSwitch { + // If too many RS-, switch to all-subs-mode. + c.gatewaySwitchAccountToSendAllSubs(e, string(accName)) + } else { + e.ni[string(subject)] = struct{}{} + sendProto = true + } + } + } + if sendProto { + var ( + protoa = [512]byte{} + proto = protoa[:0] + ) + proto = append(proto, rUnsubBytes...) + proto = append(proto, accName...) + proto = append(proto, ' ') + proto = append(proto, subject...) + proto = append(proto, CR_LF...) + c.enqueueProto(proto) + if c.trace { + c.traceOutOp("", proto[:len(proto)-LEN_CR_LF]) + } + } + c.mu.Unlock() + } else { + // There is not a single subscription, send an A- (if not already done). + c.sendAccountUnsubToGateway([]byte(acc.Name)) + } +} + +// Returns the cluster hash from the gateway reply prefix +func (g *srvGateway) getClusterHash() []byte { + g.RLock() + clusterHash := g.replyPfx[gwClusterOffset : gwClusterOffset+gwHashLen] + g.RUnlock() + return clusterHash +} + +// Store this route in map with the key being the remote server's name hash +// and the remote server's ID hash used by gateway replies mapping routing. +func (s *Server) storeRouteByHash(srvIDHash string, c *client) { + if !s.gateway.enabled { + return + } + s.gateway.routesIDByHash.Store(srvIDHash, c) +} + +// Remove the route with the given keys from the map. +func (s *Server) removeRouteByHash(srvIDHash string) { + if !s.gateway.enabled { + return + } + s.gateway.routesIDByHash.Delete(srvIDHash) +} + +// Returns the route with given hash or nil if not found. +// This is for gateways only. +func (s *Server) getRouteByHash(hash, accName []byte) (*client, bool) { + id := bytesToString(hash) + var perAccount bool + if v, ok := s.accRouteByHash.Load(bytesToString(accName)); ok { + if v == nil { + id += bytesToString(accName) + perAccount = true + } else { + id += strconv.Itoa(v.(int)) + } + } + if v, ok := s.gateway.routesIDByHash.Load(id); ok { + return v.(*client), perAccount + } else if !perAccount { + // Check if we have a "no pool" connection at index 0. + if v, ok := s.gateway.routesIDByHash.Load(bytesToString(hash) + "0"); ok { + if r := v.(*client); r != nil { + r.mu.Lock() + noPool := r.route.noPool + r.mu.Unlock() + if noPool { + return r, false + } + } + } + } + return nil, perAccount +} + +// Returns the subject from the routed reply +func getSubjectFromGWRoutedReply(reply []byte, isOldPrefix bool) []byte { + if isOldPrefix { + return reply[oldGWReplyStart:] + } + return reply[gwSubjectOffset:] +} + +// Returns the subject embedded in the given routed +// reply subject and whether the prefix was stripped. +// If the subject is not routed, returns it unchanged. +func getGWRoutedSubjectOrSelf(subject []byte) ([]byte, bool) { + if isGWPrefix, oldPrefix := isGWRoutedSubjectAndIsOldPrefix(subject); isGWPrefix { + return getSubjectFromGWRoutedReply(subject, oldPrefix), true + } + return subject, false +} + +// This should be invoked only from processInboundGatewayMsg() or +// processInboundRoutedMsg() and is checking if the subject +// (c.pa.subject) has the _GR_ prefix. If so, this is processed +// as a GW reply and `true` is returned to indicate to the caller +// that it should stop processing. +// If gateway is not enabled on this server or if the subject +// does not start with _GR_, `false` is returned and caller should +// process message as usual. +func (c *client) handleGatewayReply(msg []byte) (processed bool) { + // Do not handle GW prefixed messages if this server does not have + // gateway enabled or if the subject does not start with the previx. + if !c.srv.gateway.enabled { + return false + } + isGWPrefix, oldPrefix := isGWRoutedSubjectAndIsOldPrefix(c.pa.subject) + if !isGWPrefix { + return false + } + // Save original subject (in case we have to forward) + orgSubject := c.pa.subject + + var clusterHash []byte + var srvHash []byte + var subject []byte + + if oldPrefix { + clusterHash = c.pa.subject[oldGWReplyPrefixLen : oldGWReplyStart-1] + // Check if this reply is intended for our cluster. + if !bytes.Equal(clusterHash, c.srv.gateway.oldHash) { + // We could report, for now, just drop. + return true + } + subject = c.pa.subject[oldGWReplyStart:] + } else { + clusterHash = c.pa.subject[gwClusterOffset : gwClusterOffset+gwHashLen] + // Check if this reply is intended for our cluster. + if !bytes.Equal(clusterHash, c.srv.gateway.getClusterHash()) { + // We could report, for now, just drop. + return true + } + srvHash = c.pa.subject[gwServerOffset : gwServerOffset+gwHashLen] + subject = c.pa.subject[gwSubjectOffset:] + } + + var route *client + var perAccount bool + + // If the origin is not this server, get the route this should be sent to. + if c.kind == GATEWAY && srvHash != nil && !bytes.Equal(srvHash, c.srv.gateway.sIDHash) { + route, perAccount = c.srv.getRouteByHash(srvHash, c.pa.account) + // This will be possibly nil, and in this case we will try to process + // the interest from this server. + } + + // Adjust the subject + c.pa.subject = subject + + // Use a stack buffer to rewrite c.pa.cache since we only need it for + // getAccAndResultFromCache() + var _pacache [256]byte + pacache := _pacache[:0] + // For routes that are dedicated to an account, do not put the account + // name in the pacache. + if c.kind == GATEWAY || (c.kind == ROUTER && c.route != nil && len(c.route.accName) == 0) { + pacache = append(pacache, c.pa.account...) + pacache = append(pacache, ' ') + } + pacache = append(pacache, c.pa.subject...) + c.pa.pacache = pacache + + acc, r := c.getAccAndResultFromCache() + if acc == nil { + typeConn := "routed" + if c.kind == GATEWAY { + typeConn = "gateway" + } + c.Debugf("Unknown account %q for %s message on subject: %q", c.pa.account, typeConn, c.pa.subject) + if c.kind == GATEWAY { + c.srv.gatewayHandleAccountNoInterest(c, c.pa.account) + } + return true + } + + // If route is nil, we will process the incoming message locally. + if route == nil { + // Check if this is a service reply subject (_R_) + isServiceReply := isServiceReply(c.pa.subject) + + var queues [][]byte + if len(r.psubs)+len(r.qsubs) > 0 { + flags := pmrCollectQueueNames | pmrIgnoreEmptyQueueFilter + // If this message came from a ROUTE, allow to pick queue subs + // only if the message was directly sent by the "gateway" server + // in our cluster that received it. + if c.kind == ROUTER { + flags |= pmrAllowSendFromRouteToRoute + } + _, queues = c.processMsgResults(acc, r, msg, nil, c.pa.subject, c.pa.reply, flags) + } + // Since this was a reply that made it to the origin cluster, + // we now need to send the message with the real subject to + // gateways in case they have interest on that reply subject. + if !isServiceReply { + c.sendMsgToGateways(acc, msg, c.pa.subject, c.pa.reply, queues, false) + } + } else if c.kind == GATEWAY { + // Only if we are a gateway connection should we try to route + // to the server where the request originated. + var bufa [256]byte + var buf = bufa[:0] + buf = append(buf, msgHeadProto...) + if !perAccount { + buf = append(buf, acc.Name...) + buf = append(buf, ' ') + } + buf = append(buf, orgSubject...) + buf = append(buf, ' ') + if len(c.pa.reply) > 0 { + buf = append(buf, c.pa.reply...) + buf = append(buf, ' ') + } + szb := c.pa.szb + if c.pa.hdr >= 0 { + if route.headers { + buf[0] = 'H' + buf = append(buf, c.pa.hdb...) + buf = append(buf, ' ') + } else { + szb = []byte(strconv.Itoa(c.pa.size - c.pa.hdr)) + msg = msg[c.pa.hdr:] + } + } + buf = append(buf, szb...) + mhEnd := len(buf) + buf = append(buf, _CRLF_...) + buf = append(buf, msg...) + + route.mu.Lock() + route.enqueueProto(buf) + if route.trace { + route.traceOutOp("", buf[:mhEnd]) + } + route.mu.Unlock() + } + return true +} + +// Process a message coming from a remote gateway. Send to any sub/qsub +// in our cluster that is matching. When receiving a message for an +// account or subject for which there is no interest in this cluster +// an A-/RS- protocol may be send back. +// +func (c *client) processInboundGatewayMsg(msg []byte) { + // Update statistics + c.in.msgs++ + // The msg includes the CR_LF, so pull back out for accounting. + size := len(msg) - LEN_CR_LF + c.in.bytes += int32(size) + + if c.opts.Verbose { + c.sendOK() + } + + // Mostly under testing scenarios. + if c.srv == nil { + return + } + + // If the subject (c.pa.subject) has the gateway prefix, this function will + // handle it. + if c.handleGatewayReply(msg) { + // We are done here. + return + } + + acc, r := c.getAccAndResultFromCache() + if acc == nil { + c.Debugf("Unknown account %q for gateway message on subject: %q", c.pa.account, c.pa.subject) + c.srv.gatewayHandleAccountNoInterest(c, c.pa.account) + return + } + + acc.stats.Lock() + acc.stats.inMsgs++ + acc.stats.inBytes += int64(size) + acc.stats.gw.inMsgs++ + acc.stats.gw.inBytes += int64(size) + acc.stats.Unlock() + + // Check if this is a service reply subject (_R_) + noInterest := len(r.psubs) == 0 + checkNoInterest := true + if acc.NumServiceImports() > 0 { + if isServiceReply(c.pa.subject) { + checkNoInterest = false + } else { + // We need to eliminate the subject interest from the service imports here to + // make sure we send the proper no interest if the service import is the only interest. + noInterest = true + for _, sub := range r.psubs { + // sub.si indicates that this is a subscription for service import, and is immutable. + // So sub.si is false, then this is a subscription for something else, so there is + // actually proper interest. + if !sub.si { + noInterest = false + break + } + } + } + } + if checkNoInterest && noInterest { + // If there is no interest on plain subs, possibly send an RS-, + // even if there is qsubs interest. + c.srv.gatewayHandleSubjectNoInterest(c, acc, c.pa.account, c.pa.subject) + + // If there is also no queue filter, then no point in continuing + // (even if r.qsubs i > 0). + if len(c.pa.queues) == 0 { + return + } + } + c.processMsgResults(acc, r, msg, nil, c.pa.subject, c.pa.reply, pmrNoFlag) +} + +// Indicates that the remote which we are sending messages to +// has decided to send us all its subs interest so that we +// stop doing optimistic sends. +// +func (c *client) gatewayAllSubsReceiveStart(info *Info) { + account := getAccountFromGatewayCommand(c, info, "start") + if account == "" { + return + } + + c.Debugf("Gateway %q: switching account %q to %s mode", + info.Gateway, account, InterestOnly) + + // Since the remote would send us this start command + // only after sending us too many RS- for this account, + // we should always have an entry here. + // TODO(ik): Should we close connection with protocol violation + // error if that happens? + ei, _ := c.gw.outsim.Load(account) + if ei != nil { + e := ei.(*outsie) + e.Lock() + e.mode = Transitioning + e.Unlock() + } else { + e := &outsie{sl: NewSublistForServer(c.srv)} + e.mode = Transitioning + c.mu.Lock() + c.gw.outsim.Store(account, e) + c.mu.Unlock() + } +} + +// Indicates that the remote has finished sending all its +// subscriptions and we should now not send unless we know +// there is explicit interest. +// +func (c *client) gatewayAllSubsReceiveComplete(info *Info) { + account := getAccountFromGatewayCommand(c, info, "complete") + if account == _EMPTY_ { + return + } + // Done receiving all subs from remote. Set the `ni` + // map to nil so that gatewayInterest() no longer + // uses it. + ei, _ := c.gw.outsim.Load(account) + if ei != nil { + e := ei.(*outsie) + // Needs locking here since `ni` is checked by + // many go-routines calling gatewayInterest() + e.Lock() + e.ni = nil + e.mode = InterestOnly + e.Unlock() + + c.Debugf("Gateway %q: switching account %q to %s mode complete", + info.Gateway, account, InterestOnly) + } +} + +// small helper to get the account name from the INFO command. +func getAccountFromGatewayCommand(c *client, info *Info, cmd string) string { + if info.GatewayCmdPayload == nil { + c.sendErrAndErr(fmt.Sprintf("Account absent from receive-all-subscriptions-%s command", cmd)) + c.closeConnection(ProtocolViolation) + return _EMPTY_ + } + return string(info.GatewayCmdPayload) +} + +// Switch to send-all-subs mode for the given gateway and account. +// This is invoked when processing an inbound message and we +// reach a point where we had to send a lot of RS- for this +// account. We will send an INFO protocol to indicate that we +// start sending all our subs (for this account), followed by +// all subs (RS+) and finally an INFO to indicate the end of it. +// The remote will then send messages only if it finds explicit +// interest in the sublist created based on all RS+ that we just +// sent. +// The client's lock is held on entry. +// +func (c *client) gatewaySwitchAccountToSendAllSubs(e *insie, accName string) { + // Set this map to nil so that the no-interest is no longer checked. + e.ni = nil + // Switch mode to transitioning to prevent switchAccountToInterestMode + // to possibly call this function multiple times. + e.mode = Transitioning + s := c.srv + + remoteGWName := c.gw.name + c.Debugf("Gateway %q: switching account %q to %s mode", + remoteGWName, accName, InterestOnly) + + // Function that will create an INFO protocol + // and set proper command. + sendCmd := func(cmd byte, useLock bool) { + // Use bare server info and simply set the + // gateway name and command + info := Info{ + Gateway: s.gateway.name, + GatewayCmd: cmd, + GatewayCmdPayload: stringToBytes(accName), + } + + b, _ := json.Marshal(&info) + infoJSON := []byte(fmt.Sprintf(InfoProto, b)) + if useLock { + c.mu.Lock() + } + c.enqueueProto(infoJSON) + if useLock { + c.mu.Unlock() + } + } + // Send the start command. When remote receives this, + // it may continue to send optimistic messages, but + // it will start to register RS+/RS- in sublist instead + // of noInterest map. + sendCmd(gatewayCmdAllSubsStart, false) + + // Execute this in separate go-routine as to not block + // the readLoop (which may cause the otherside to close + // the connection due to slow consumer) + s.startGoRoutine(func() { + defer s.grWG.Done() + + s.sendAccountSubsToGateway(c, accName) + // Send the complete command. When the remote receives + // this, it will not send a message unless it has a + // matching sub from us. + sendCmd(gatewayCmdAllSubsComplete, true) + + c.Debugf("Gateway %q: switching account %q to %s mode complete", + remoteGWName, accName, InterestOnly) + }) +} + +// Keeps track of the routed reply to be used when/if application sends back a +// message on the reply without the prefix. +// If `client` is not nil, it will be stored in the client gwReplyMapping structure, +// and client lock is held on entry. +// If `client` is nil, the mapping is stored in the client's account's gwReplyMapping +// structure. Account lock will be explicitly acquired. +// This is a server receiver because we use a timer interval that is avail in +// Server.gateway object. +func (s *Server) trackGWReply(c *client, acc *Account, reply, routedReply []byte) { + var l sync.Locker + var g *gwReplyMapping + if acc != nil { + acc.mu.Lock() + defer acc.mu.Unlock() + g = &acc.gwReplyMapping + l = &acc.mu + } else { + g = &c.gwReplyMapping + l = &c.mu + } + ttl := s.gateway.recSubExp + wasEmpty := len(g.mapping) == 0 + if g.mapping == nil { + g.mapping = make(map[string]*gwReplyMap) + } + // The reason we pass both `reply` and `routedReply`, is that in some cases, + // `routedReply` may have a deliver subject appended, something look like: + // "_GR_.xxx.yyy.$JS.ACK.$MQTT_msgs.someid.1.1.1.1620086713306484000.0@$MQTT.msgs.foo" + // but `reply` has already been cleaned up (delivery subject removed from tail): + // "$JS.ACK.$MQTT_msgs.someid.1.1.1.1620086713306484000.0" + // So we will use that knowledge so we don't have to make any cleaning here. + routedReply = routedReply[:gwSubjectOffset+len(reply)] + // We need to make a copy so that we don't reference the underlying + // read buffer. + ms := string(routedReply) + grm := &gwReplyMap{ms: ms, exp: time.Now().Add(ttl).UnixNano()} + // If we are here with the same key but different mapped replies + // (say $GNR._.A.srv1.bar and then $GNR._.B.srv2.bar), we need to + // store it otherwise we would take the risk of the reply not + // making it back. + g.mapping[ms[gwSubjectOffset:]] = grm + if wasEmpty { + atomic.StoreInt32(&g.check, 1) + s.gwrm.m.Store(g, l) + if atomic.CompareAndSwapInt32(&s.gwrm.w, 0, 1) { + select { + case s.gwrm.ch <- ttl: + default: + } + } + } +} + +// Starts a long lived go routine that is responsible to +// remove GW reply mapping that have expired. +func (s *Server) startGWReplyMapExpiration() { + s.mu.Lock() + s.gwrm.ch = make(chan time.Duration, 1) + s.mu.Unlock() + s.startGoRoutine(func() { + defer s.grWG.Done() + + t := time.NewTimer(time.Hour) + var ttl time.Duration + for { + select { + case <-t.C: + if ttl == 0 { + t.Reset(time.Hour) + continue + } + now := time.Now().UnixNano() + mapEmpty := true + s.gwrm.m.Range(func(k, v any) bool { + g := k.(*gwReplyMapping) + l := v.(sync.Locker) + l.Lock() + for k, grm := range g.mapping { + if grm.exp <= now { + delete(g.mapping, k) + if len(g.mapping) == 0 { + atomic.StoreInt32(&g.check, 0) + s.gwrm.m.Delete(g) + } + } + } + l.Unlock() + mapEmpty = false + return true + }) + if mapEmpty && atomic.CompareAndSwapInt32(&s.gwrm.w, 1, 0) { + ttl = 0 + t.Reset(time.Hour) + } else { + t.Reset(ttl) + } + case cttl := <-s.gwrm.ch: + ttl = cttl + if !t.Stop() { + select { + case <-t.C: + default: + } + } + t.Reset(ttl) + case <-s.quitCh: + return + } + } + }) +} diff --git a/server/gsl/gsl.go b/server/gsl/gsl.go new file mode 100644 index 0000000..f92fec6 --- /dev/null +++ b/server/gsl/gsl.go @@ -0,0 +1,566 @@ +// Copyright 2025-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gsl + +import ( + "errors" + "strings" + "sync" +) + +// Sublist is a routing mechanism to handle subject distribution and +// provides a facility to match subjects from published messages to +// interested subscribers. Subscribers can have wildcard subjects to +// match multiple published subjects. + +// Common byte variables for wildcards and token separator. +const ( + pwc = '*' + pwcs = "*" + fwc = '>' + fwcs = ">" + tsep = "." + btsep = '.' + _EMPTY_ = "" +) + +// Sublist related errors +var ( + ErrInvalidSubject = errors.New("gsl: invalid subject") + ErrNotFound = errors.New("gsl: no matches found") + ErrNilChan = errors.New("gsl: nil channel") + ErrAlreadyRegistered = errors.New("gsl: notification already registered") +) + +// SimpleSublist is an alias type for GenericSublist that takes +// empty values, useful for tracking interest only without any +// unnecessary allocations. +type SimpleSublist = GenericSublist[struct{}] + +// NewSimpleSublist will create a simple sublist. +func NewSimpleSublist() *SimpleSublist { + return &GenericSublist[struct{}]{root: newLevel[struct{}]()} +} + +// A GenericSublist stores and efficiently retrieves subscriptions. +type GenericSublist[T comparable] struct { + sync.RWMutex + root *level[T] + count uint32 +} + +// A node contains subscriptions and a pointer to the next level. +type node[T comparable] struct { + next *level[T] + subs map[T]string // value -> subject +} + +// A level represents a group of nodes and special pointers to +// wildcard nodes. +type level[T comparable] struct { + nodes map[string]*node[T] + pwc, fwc *node[T] +} + +// Create a new default node. +func newNode[T comparable]() *node[T] { + return &node[T]{subs: make(map[T]string)} +} + +// Create a new default level. +func newLevel[T comparable]() *level[T] { + return &level[T]{nodes: make(map[string]*node[T])} +} + +// NewSublist will create a default sublist with caching enabled per the flag. +func NewSublist[T comparable]() *GenericSublist[T] { + return &GenericSublist[T]{root: newLevel[T]()} +} + +// Insert adds a subscription into the sublist +func (s *GenericSublist[T]) Insert(subject string, value T) error { + s.Lock() + + var sfwc bool + var n *node[T] + l := s.root + + for t := range strings.SplitSeq(subject, tsep) { + lt := len(t) + if lt == 0 || sfwc { + s.Unlock() + return ErrInvalidSubject + } + + if lt > 1 { + n = l.nodes[t] + } else { + switch t[0] { + case pwc: + n = l.pwc + case fwc: + n = l.fwc + sfwc = true + default: + n = l.nodes[t] + } + } + if n == nil { + n = newNode[T]() + if lt > 1 { + l.nodes[t] = n + } else { + switch t[0] { + case pwc: + l.pwc = n + case fwc: + l.fwc = n + default: + l.nodes[t] = n + } + } + } + if n.next == nil { + n.next = newLevel[T]() + } + l = n.next + } + + n.subs[value] = subject + + s.count++ + s.Unlock() + + return nil +} + +// Match will match all entries to the literal subject. +// It will return a set of results for both normal and queue subscribers. +func (s *GenericSublist[T]) Match(subject string, cb func(T)) { + s.match(subject, cb, true) +} + +// MatchBytes will match all entries to the literal subject. +// It will return a set of results for both normal and queue subscribers. +func (s *GenericSublist[T]) MatchBytes(subject []byte, cb func(T)) { + s.match(string(subject), cb, true) +} + +// HasInterest will return whether or not there is any interest in the subject. +// In cases where more detail is not required, this may be faster than Match. +func (s *GenericSublist[T]) HasInterest(subject string) bool { + return s.hasInterest(subject, true, nil) +} + +// NumInterest will return the number of subs interested in the subject. +// In cases where more detail is not required, this may be faster than Match. +func (s *GenericSublist[T]) NumInterest(subject string) (np int) { + s.hasInterest(subject, true, &np) + return +} + +// MatchesFullWildcard returns true if there is top-level ">" interest. +func (s *GenericSublist[T]) MatchesFullWildcard() bool { + if s == nil { + return false + } + s.RLock() + defer s.RUnlock() + return s.root.fwc != nil +} + +// MatchesSingleFilter returns the filter when the sublist contains exactly one unique subject. +func (s *GenericSublist[T]) MatchesSingleFilter() (string, bool) { + if s == nil { + return _EMPTY_, false + } + s.RLock() + defer s.RUnlock() + return singleFilter(s.root, _EMPTY_) +} + +func singleFilter[T comparable](l *level[T], filter string) (string, bool) { + if l == nil { + return filter, filter != _EMPTY_ + } + if len(l.nodes) > 1 { + return _EMPTY_, false + } + var next *node[T] + branches := 0 + if l.pwc != nil { + next = l.pwc + branches++ + } + if l.fwc != nil { + next = l.fwc + branches++ + } + for _, n := range l.nodes { + next = n + branches++ + } + if branches != 1 { + return _EMPTY_, false + } + for _, subj := range next.subs { + filter = subj + break + } + if next.next == nil { + return filter, filter != _EMPTY_ + } + if filter != _EMPTY_ { + if next.next.numNodes() > 0 { + return _EMPTY_, false + } + return filter, true + } + return singleFilter(next.next, filter) +} + +func (s *GenericSublist[T]) match(subject string, cb func(T), doLock bool) { + tsa := [32]string{} + tokens := tsa[:0] + start := 0 + for i := 0; i < len(subject); i++ { + if subject[i] == btsep { + if i-start == 0 { + return + } + tokens = append(tokens, subject[start:i]) + start = i + 1 + } + } + if start >= len(subject) { + return + } + tokens = append(tokens, subject[start:]) + + if doLock { + s.RLock() + defer s.RUnlock() + } + matchLevel(s.root, tokens, cb) +} + +func (s *GenericSublist[T]) hasInterest(subject string, doLock bool, np *int) bool { + tsa := [32]string{} + tokens := tsa[:0] + start := 0 + for i := 0; i < len(subject); i++ { + if subject[i] == btsep { + if i-start == 0 { + return false + } + tokens = append(tokens, subject[start:i]) + start = i + 1 + } + } + if start >= len(subject) { + return false + } + tokens = append(tokens, subject[start:]) + + if doLock { + s.RLock() + defer s.RUnlock() + } + return matchLevelForAny(s.root, tokens, np) +} + +func matchLevelForAny[T comparable](l *level[T], toks []string, np *int) bool { + var pwc, n *node[T] + for i, t := range toks { + if l == nil { + return false + } + if l.fwc != nil { + if np != nil { + *np += len(l.fwc.subs) + } + return true + } + if pwc = l.pwc; pwc != nil { + if match := matchLevelForAny(pwc.next, toks[i+1:], np); match { + return true + } + } + n = l.nodes[t] + if n != nil { + l = n.next + } else { + l = nil + } + } + if n != nil { + if np != nil { + *np += len(n.subs) + } + if len(n.subs) > 0 { + return true + } + } + if pwc != nil { + if np != nil { + *np += len(pwc.subs) + } + return len(pwc.subs) > 0 + } + return false +} + +// callbacksForResults will make the necessary callbacks for each +// result in this node. +func callbacksForResults[T comparable](n *node[T], cb func(T)) { + for sub := range n.subs { + cb(sub) + } +} + +// matchLevel is used to recursively descend into the trie. +func matchLevel[T comparable](l *level[T], toks []string, cb func(T)) { + var pwc, n *node[T] + for i, t := range toks { + if l == nil { + return + } + if l.fwc != nil { + callbacksForResults(l.fwc, cb) + } + if pwc = l.pwc; pwc != nil { + matchLevel(pwc.next, toks[i+1:], cb) + } + n = l.nodes[t] + if n != nil { + l = n.next + } else { + l = nil + } + } + if n != nil { + callbacksForResults(n, cb) + } + if pwc != nil { + callbacksForResults(pwc, cb) + } +} + +// lnt is used to track descent into levels for a removal for pruning. +type lnt[T comparable] struct { + l *level[T] + n *node[T] + t string +} + +// Raw low level remove, can do batches with lock held outside. +func (s *GenericSublist[T]) remove(subject string, value T, shouldLock bool) error { + if shouldLock { + s.Lock() + defer s.Unlock() + } + + var sfwc bool + var n *node[T] + l := s.root + + // Track levels for pruning + var lnts [32]lnt[T] + levels := lnts[:0] + + for t := range strings.SplitSeq(subject, tsep) { + lt := len(t) + if lt == 0 || sfwc { + return ErrInvalidSubject + } + if l == nil { + return ErrNotFound + } + if lt > 1 { + n = l.nodes[t] + } else { + switch t[0] { + case pwc: + n = l.pwc + case fwc: + n = l.fwc + sfwc = true + default: + n = l.nodes[t] + } + } + if n != nil { + levels = append(levels, lnt[T]{l, n, t}) + l = n.next + } else { + l = nil + } + } + + if !s.removeFromNode(n, value) { + return ErrNotFound + } + + s.count-- + + for i := len(levels) - 1; i >= 0; i-- { + l, n, t := levels[i].l, levels[i].n, levels[i].t + if n.isEmpty() { + l.pruneNode(n, t) + } + } + + return nil +} + +// Remove will remove a subscription. +func (s *GenericSublist[T]) Remove(subject string, value T) error { + return s.remove(subject, value, true) +} + +// HasInterestStartingIn is a helper for subject tree intersection. +func (s *GenericSublist[T]) HasInterestStartingIn(subj string) bool { + s.RLock() + defer s.RUnlock() + var _tokens [64]string + tokens := tokenizeSubjectIntoSlice(_tokens[:0], subj) + return hasInterestStartingIn(s.root, tokens) +} + +func hasInterestStartingIn[T comparable](l *level[T], tokens []string) bool { + if l == nil { + return false + } + if len(tokens) == 0 { + return true + } + token := tokens[0] + if l.fwc != nil { + return true + } + found := false + if pwc := l.pwc; pwc != nil { + found = found || hasInterestStartingIn(pwc.next, tokens[1:]) + } + if n := l.nodes[token]; n != nil { + found = found || hasInterestStartingIn(n.next, tokens[1:]) + } + return found +} + +// pruneNode is used to prune an empty node from the tree. +func (l *level[T]) pruneNode(n *node[T], t string) { + if n == nil { + return + } + if n == l.fwc { + l.fwc = nil + } else if n == l.pwc { + l.pwc = nil + } else { + delete(l.nodes, t) + } +} + +// isEmpty will test if the node has any entries. Used +// in pruning. +func (n *node[T]) isEmpty() bool { + return len(n.subs) == 0 && (n.next == nil || n.next.numNodes() == 0) +} + +// Return the number of nodes for the given level. +func (l *level[T]) numNodes() int { + if l == nil { + return 0 + } + num := len(l.nodes) + if l.pwc != nil { + num++ + } + if l.fwc != nil { + num++ + } + return num +} + +// Remove the sub for the given node. +func (s *GenericSublist[T]) removeFromNode(n *node[T], value T) (found bool) { + if n == nil { + return false + } + if _, found = n.subs[value]; found { + delete(n.subs, value) + } + return found +} + +// Count returns the number of subscriptions. +func (s *GenericSublist[T]) Count() uint32 { + s.RLock() + defer s.RUnlock() + return s.count +} + +// numLevels will return the maximum number of levels +// contained in the Sublist tree. +func (s *GenericSublist[T]) numLevels() int { + return visitLevel(s.root, 0) +} + +// visitLevel is used to descend the Sublist tree structure +// recursively. +func visitLevel[T comparable](l *level[T], depth int) int { + if l == nil || l.numNodes() == 0 { + return depth + } + + depth++ + maxDepth := depth + + for _, n := range l.nodes { + if n == nil { + continue + } + newDepth := visitLevel(n.next, depth) + if newDepth > maxDepth { + maxDepth = newDepth + } + } + if l.pwc != nil { + pwcDepth := visitLevel(l.pwc.next, depth) + if pwcDepth > maxDepth { + maxDepth = pwcDepth + } + } + if l.fwc != nil { + fwcDepth := visitLevel(l.fwc.next, depth) + if fwcDepth > maxDepth { + maxDepth = fwcDepth + } + } + return maxDepth +} + +// use similar to append. meaning, the updated slice will be returned +func tokenizeSubjectIntoSlice(tts []string, subject string) []string { + start := 0 + for i := 0; i < len(subject); i++ { + if subject[i] == btsep { + tts = append(tts, subject[start:i]) + start = i + 1 + } + } + tts = append(tts, subject[start:]) + return tts +} diff --git a/server/gsl/gsl_test.go b/server/gsl/gsl_test.go new file mode 100644 index 0000000..0eb84a1 --- /dev/null +++ b/server/gsl/gsl_test.go @@ -0,0 +1,489 @@ +// Copyright 2016-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gsl + +import ( + "strings" + "testing" + + "github.com/nats-io/nats-server/v2/internal/antithesis" +) + +func TestGenericSublistInit(t *testing.T) { + s := NewSublist[struct{}]() + require_Equal(t, s.count, 0) + require_Equal(t, s.Count(), s.count) +} + +func TestGenericSublistInsertCount(t *testing.T) { + s := NewSublist[struct{}]() + require_NoError(t, s.Insert("foo", struct{}{})) + require_NoError(t, s.Insert("bar", struct{}{})) + require_NoError(t, s.Insert("foo.bar", struct{}{})) + require_Equal(t, s.Count(), 3) +} + +func TestGenericSublistSimple(t *testing.T) { + s := NewSublist[struct{}]() + require_NoError(t, s.Insert("foo", struct{}{})) + require_Matches(t, s, "foo", 1) +} + +func TestGenericSublistSimpleMultiTokens(t *testing.T) { + s := NewSublist[struct{}]() + require_NoError(t, s.Insert("foo.bar.baz", struct{}{})) + require_Matches(t, s, "foo.bar.baz", 1) +} + +func TestGenericSublistPartialWildcard(t *testing.T) { + s := NewSublist[struct{}]() + require_NoError(t, s.Insert("a.b.c", struct{}{})) + require_NoError(t, s.Insert("a.*.c", struct{}{})) + require_Matches(t, s, "a.b.c", 2) +} + +func TestGenericSublistPartialWildcardAtEnd(t *testing.T) { + s := NewSublist[struct{}]() + require_NoError(t, s.Insert("a.b.c", struct{}{})) + require_NoError(t, s.Insert("a.b.*", struct{}{})) + require_Matches(t, s, "a.b.c", 2) +} + +func TestGenericSublistFullWildcard(t *testing.T) { + s := NewSublist[struct{}]() + require_NoError(t, s.Insert("a.b.c", struct{}{})) + require_NoError(t, s.Insert("a.>", struct{}{})) + require_Matches(t, s, "a.b.c", 2) + require_Matches(t, s, "a.>", 1) +} + +func TestGenericSublistRemove(t *testing.T) { + s := NewSublist[struct{}]() + + require_NoError(t, s.Insert("a.b.c.d", struct{}{})) + require_Equal(t, s.Count(), 1) + require_Matches(t, s, "a.b.c.d", 1) + + require_NoError(t, s.Remove("a.b.c.d", struct{}{})) + require_Equal(t, s.Count(), 0) + require_Matches(t, s, "a.b.c.d", 0) +} + +func TestGenericSublistRemoveWildcard(t *testing.T) { + s := NewSublist[int]() + + require_NoError(t, s.Insert("a.b.c.d", 11)) + require_NoError(t, s.Insert("a.b.*.d", 22)) + require_NoError(t, s.Insert("a.b.>", 33)) + require_Equal(t, s.Count(), 3) + require_Matches(t, s, "a.b.c.d", 3) + + require_NoError(t, s.Remove("a.b.*.d", 22)) + require_Equal(t, s.Count(), 2) + require_Matches(t, s, "a.b.c.d", 2) + + require_NoError(t, s.Remove("a.b.>", 33)) + require_Equal(t, s.Count(), 1) + require_Matches(t, s, "a.b.c.d", 1) + + require_NoError(t, s.Remove("a.b.c.d", 11)) + require_Equal(t, s.Count(), 0) + require_Matches(t, s, "a.b.c.d", 0) +} + +func TestGenericSublistRemoveCleanup(t *testing.T) { + s := NewSublist[struct{}]() + require_Equal(t, s.numLevels(), 0) + require_NoError(t, s.Insert("a.b.c.d.e.f", struct{}{})) + require_Equal(t, s.numLevels(), 6) + require_NoError(t, s.Remove("a.b.c.d.e.f", struct{}{})) + require_Equal(t, s.numLevels(), 0) +} + +func TestGenericSublistRemoveCleanupWildcards(t *testing.T) { + s := NewSublist[struct{}]() + require_Equal(t, s.numLevels(), 0) + require_NoError(t, s.Insert("a.b.*.d.e.>", struct{}{})) + require_Equal(t, s.numLevels(), 6) + require_NoError(t, s.Remove("a.b.*.d.e.>", struct{}{})) + require_Equal(t, s.numLevels(), 0) +} + +func TestGenericSublistInvalidSubjectsInsert(t *testing.T) { + s := NewSublist[struct{}]() + // Insert, or subscriptions, can have wildcards, but not empty tokens, + // and can not have a FWC that is not the terminal token. + require_Error(t, s.Insert(".foo", struct{}{}), ErrInvalidSubject) + require_Error(t, s.Insert("foo.", struct{}{}), ErrInvalidSubject) + require_Error(t, s.Insert("foo..bar", struct{}{}), ErrInvalidSubject) + require_Error(t, s.Insert("foo.bar..baz", struct{}{}), ErrInvalidSubject) + require_Error(t, s.Insert("foo.>.baz", struct{}{}), ErrInvalidSubject) +} + +func TestGenericSublistBadSubjectOnRemove(t *testing.T) { + s := NewSublist[struct{}]() + require_Error(t, s.Insert("a.b..d", struct{}{}), ErrInvalidSubject) + require_Error(t, s.Remove("a.b..d", struct{}{}), ErrInvalidSubject) + require_Error(t, s.Remove("a.>.b", struct{}{}), ErrInvalidSubject) +} + +func TestGenericSublistTwoTokenPubMatchSingleTokenSub(t *testing.T) { + s := NewSublist[struct{}]() + require_NoError(t, s.Insert("foo", struct{}{})) + require_Matches(t, s, "foo", 1) + require_Matches(t, s, "foo.bar", 0) +} + +func TestGenericSublistInsertWithWildcardsAsLiterals(t *testing.T) { + s := NewSublist[int]() + for i, subject := range []string{"foo.*-", "foo.>-"} { + require_NoError(t, s.Insert(subject, i)) + require_Matches(t, s, "foo.bar", 0) + require_Matches(t, s, subject, 1) + } +} + +func TestGenericSublistRemoveWithWildcardsAsLiterals(t *testing.T) { + s := NewSublist[int]() + for i, subject := range []string{"foo.*-", "foo.>-"} { + require_NoError(t, s.Insert(subject, i)) + require_Matches(t, s, "foo.bar", 0) + require_Matches(t, s, subject, 1) + require_Error(t, s.Remove("foo.bar", i), ErrNotFound) + require_Equal(t, s.Count(), 1) + require_NoError(t, s.Remove(subject, i)) + require_Equal(t, s.Count(), 0) + } +} + +func TestGenericSublistMatchWithEmptyTokens(t *testing.T) { + s := NewSublist[struct{}]() + require_NoError(t, s.Insert(">", struct{}{})) + for _, subject := range []string{".foo", "..foo", "foo..", "foo.", "foo..bar", "foo...bar"} { + t.Run(subject, func(t *testing.T) { + require_Matches(t, s, subject, 0) + }) + } +} + +func TestGenericSublistHasInterest(t *testing.T) { + s := NewSublist[int]() + require_NoError(t, s.Insert("foo", 11)) + + // Expect to find that "foo" matches but "bar" doesn't. + // At this point nothing should be in the cache. + require_True(t, s.HasInterest("foo")) + require_False(t, s.HasInterest("bar")) + + // Call Match on a subject we know there is no match. + require_Matches(t, s, "bar", 0) + require_False(t, s.HasInterest("bar")) + + // Remove fooSub and check interest again + require_NoError(t, s.Remove("foo", 11)) + require_False(t, s.HasInterest("foo")) + + // Try with some wildcards + require_NoError(t, s.Insert("foo.*", 22)) + require_False(t, s.HasInterest("foo")) + require_True(t, s.HasInterest("foo.bar")) + require_False(t, s.HasInterest("foo.bar.baz")) + + // Remove sub, there should be no interest + require_NoError(t, s.Remove("foo.*", 22)) + require_False(t, s.HasInterest("foo")) + require_False(t, s.HasInterest("foo.bar")) + require_False(t, s.HasInterest("foo.bar.baz")) + + require_NoError(t, s.Insert("foo.>", 33)) + require_False(t, s.HasInterest("foo")) + require_True(t, s.HasInterest("foo.bar")) + require_True(t, s.HasInterest("foo.bar.baz")) + + require_NoError(t, s.Remove("foo.>", 33)) + require_False(t, s.HasInterest("foo")) + require_False(t, s.HasInterest("foo.bar")) + require_False(t, s.HasInterest("foo.bar.baz")) + + require_NoError(t, s.Insert("*.>", 44)) + require_False(t, s.HasInterest("foo")) + require_True(t, s.HasInterest("foo.bar")) + require_True(t, s.HasInterest("foo.baz")) + require_NoError(t, s.Remove("*.>", 44)) + + require_NoError(t, s.Insert("*.bar", 55)) + require_False(t, s.HasInterest("foo")) + require_True(t, s.HasInterest("foo.bar")) + require_False(t, s.HasInterest("foo.baz")) + require_NoError(t, s.Remove("*.bar", 55)) + + require_NoError(t, s.Insert("*", 66)) + require_True(t, s.HasInterest("foo")) + require_False(t, s.HasInterest("foo.bar")) + require_NoError(t, s.Remove("*", 66)) +} + +func TestGenericSublistHasInterestOverlapping(t *testing.T) { + s := NewSublist[int]() + require_NoError(t, s.Insert("stream.A.child", 11)) + require_NoError(t, s.Insert("stream.*", 11)) + require_True(t, s.HasInterest("stream.A.child")) + require_True(t, s.HasInterest("stream.A")) +} + +func TestGenericSublistMatchesFullWildcard(t *testing.T) { + s := NewSublist[int]() + require_NoError(t, s.Insert("stream.A.child", 11)) + require_NoError(t, s.Insert("stream.*", 11)) + require_True(t, s.HasInterest("stream.A.child")) + require_True(t, s.HasInterest("stream.A")) + require_False(t, s.MatchesFullWildcard()) + + require_NoError(t, s.Insert(">", 11)) + require_True(t, s.MatchesFullWildcard()) +} + +func TestGenericSublistSingleFilter(t *testing.T) { + var sn *GenericSublist[int] + filter, ok := sn.MatchesSingleFilter() + require_False(t, ok) + require_Equal(t, filter, _EMPTY_) + + s := NewSublist[int]() + filter, ok = s.MatchesSingleFilter() + require_False(t, ok) + require_Equal(t, filter, _EMPTY_) + + require_NoError(t, s.Insert("stream.A.child", 11)) + filter, ok = s.MatchesSingleFilter() + require_True(t, ok) + require_Equal(t, filter, "stream.A.child") + + require_NoError(t, s.Insert("stream.A.child", 22)) + filter, ok = s.MatchesSingleFilter() + require_True(t, ok) + require_Equal(t, filter, "stream.A.child") + + require_NoError(t, s.Insert("stream.A.other", 22)) + filter, ok = s.MatchesSingleFilter() + require_False(t, ok) + require_Equal(t, filter, _EMPTY_) + + require_NoError(t, s.Insert("stream.*", 33)) + filter, ok = s.MatchesSingleFilter() + require_False(t, ok) + require_Equal(t, filter, _EMPTY_) + + // Test that an ancestor with descendants correctly returns false. + s2 := NewSublist[int]() + require_NoError(t, s2.Insert("foo", 1)) + require_NoError(t, s2.Insert("foo.bar", 2)) + require_NoError(t, s2.Insert("foo.baz", 3)) + filter, ok = s2.MatchesSingleFilter() + require_False(t, ok) + require_Equal(t, filter, _EMPTY_) + + // Test that an ancestor with only descendants correctly returns false. + s3 := NewSublist[int]() + require_NoError(t, s3.Insert("foo.bar", 2)) + require_NoError(t, s3.Insert("foo.baz", 3)) + filter, ok = s3.MatchesSingleFilter() + require_False(t, ok) + require_Equal(t, filter, _EMPTY_) +} + +// TestGenericSublistHasInterestStartingInRace tests that HasInterestStartingIn +// is safe to call concurrently with modifications to the sublist. +func TestGenericSublistHasInterestStartingInRace(t *testing.T) { + s := NewSublist[int]() + + // Pre-populate with some patterns + for i := 0; i < 10; i++ { + s.Insert("foo.bar.baz", i) + s.Insert("foo.*.baz", i+10) + s.Insert("foo.>", i+20) + } + + done := make(chan struct{}) + const iterations = 1000 + + // Goroutine 1: repeatedly call HasInterestStartingIn + go func() { + for i := 0; i < iterations; i++ { + s.HasInterestStartingIn("foo") + s.HasInterestStartingIn("foo.bar") + s.HasInterestStartingIn("foo.bar.baz") + s.HasInterestStartingIn("other.subject") + } + done <- struct{}{} + }() + + // Goroutine 2: repeatedly modify the sublist + go func() { + for i := 0; i < iterations; i++ { + val := 1000 + i + s.Insert("test.subject."+string(rune('a'+i%26)), val) + s.Insert("foo.*.test", val) + s.Remove("test.subject."+string(rune('a'+i%26)), val) + s.Remove("foo.*.test", val) + } + done <- struct{}{} + }() + + // Goroutine 3: also call HasInterest (which does lock) + go func() { + for i := 0; i < iterations; i++ { + s.HasInterest("foo.bar.baz") + s.HasInterest("foo.something.baz") + } + done <- struct{}{} + }() + + // Wait for all goroutines + <-done + <-done + <-done +} + +func TestGenericSublistNumInterest(t *testing.T) { + s := NewSublist[int]() + require_NoError(t, s.Insert("foo", 11)) + + require_NumInterest := func(t *testing.T, subj string, wnp int) { + t.Helper() + require_Matches(t, s, subj, wnp) + require_Equal(t, s.NumInterest(subj), wnp) + } + + // Expect to find that "foo" matches but "bar" doesn't. + // At this point nothing should be in the cache. + require_NumInterest(t, "foo", 1) + require_NumInterest(t, "bar", 0) + + // Remove fooSub and check interest again + require_NoError(t, s.Remove("foo", 11)) + require_NumInterest(t, "foo", 0) + + // Try with some wildcards + require_NoError(t, s.Insert("foo.*", 22)) + require_NumInterest(t, "foo", 0) + require_NumInterest(t, "foo.bar", 1) + require_NumInterest(t, "foo.bar.baz", 0) + + // Remove sub, there should be no interest + require_NoError(t, s.Remove("foo.*", 22)) + require_NumInterest(t, "foo", 0) + require_NumInterest(t, "foo.bar", 0) + require_NumInterest(t, "foo.bar.baz", 0) + + require_NoError(t, s.Insert("foo.>", 33)) + require_NumInterest(t, "foo", 0) + require_NumInterest(t, "foo.bar", 1) + require_NumInterest(t, "foo.bar.baz", 1) + + require_NoError(t, s.Remove("foo.>", 33)) + require_NumInterest(t, "foo", 0) + require_NumInterest(t, "foo.bar", 0) + require_NumInterest(t, "foo.bar.baz", 0) + + require_NoError(t, s.Insert("*.>", 44)) + require_NumInterest(t, "foo", 0) + require_NumInterest(t, "foo.bar", 1) + require_NumInterest(t, "foo.bar.baz", 1) + require_NoError(t, s.Remove("*.>", 44)) + + require_NoError(t, s.Insert("*.bar", 55)) + require_NumInterest(t, "foo", 0) + require_NumInterest(t, "foo.bar", 1) + require_NumInterest(t, "foo.bar.baz", 0) + require_NoError(t, s.Remove("*.bar", 55)) + + require_NoError(t, s.Insert("*", 66)) + require_NumInterest(t, "foo", 1) + require_NumInterest(t, "foo.bar", 0) + require_NoError(t, s.Remove("*", 66)) +} + +// --- TEST HELPERS --- + +func require_Matches[T comparable](t *testing.T, s *GenericSublist[T], sub string, c int) { + t.Helper() + matches := 0 + s.Match(sub, func(_ T) { + matches++ + }) + require_Equal(t, matches, c) +} + +func require_True(t testing.TB, b bool) { + t.Helper() + if !b { + antithesis.AssertUnreachable(t, "Failed require_True check", nil) + t.Fatalf("require true, but got false") + } +} + +func require_False(t testing.TB, b bool) { + t.Helper() + if b { + antithesis.AssertUnreachable(t, "Failed require_False check", nil) + t.Fatalf("require false, but got true") + } +} + +func require_NoError(t testing.TB, err error) { + t.Helper() + if err != nil { + antithesis.AssertUnreachable(t, "Failed require_NoError check", map[string]any{ + "error": err.Error(), + }) + t.Fatalf("require no error, but got: %v", err) + } +} + +func require_Error(t testing.TB, err error, expected ...error) { + t.Helper() + if err == nil { + antithesis.AssertUnreachable(t, "Failed require_Error check (nil error)", nil) + t.Fatalf("require error, but got none") + } + if len(expected) == 0 { + return + } + // Try to strip nats prefix from Go library if present. + const natsErrPre = "nats: " + eStr := err.Error() + if strings.HasPrefix(eStr, natsErrPre) { + eStr = strings.Replace(eStr, natsErrPre, _EMPTY_, 1) + } + + for _, e := range expected { + if err == e || strings.Contains(eStr, e.Error()) || strings.Contains(e.Error(), eStr) { + return + } + } + + antithesis.AssertUnreachable(t, "Failed require_Error check (unexpected error)", map[string]any{ + "error": err.Error(), + }) + t.Fatalf("Expected one of %v, got '%v'", expected, err) +} + +func require_Equal[T comparable](t testing.TB, a, b T) { + t.Helper() + if a != b { + antithesis.AssertUnreachable(t, "Failed require_Equal check", nil) + t.Fatalf("require %T equal, but got: %v != %v", a, a, b) + } +} diff --git a/server/ipqueue.go b/server/ipqueue.go new file mode 100644 index 0000000..5df1335 --- /dev/null +++ b/server/ipqueue.go @@ -0,0 +1,334 @@ +// Copyright 2021-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "errors" + "iter" + "sync" + "sync/atomic" +) + +const ipQueueDefaultMaxRecycleSize = 4 * 1024 + +// This is a generic intra-process queue. +type ipQueue[T any] struct { + inprogress int64 + sync.Mutex + ch chan struct{} + elts []T + pos int + pool *sync.Pool + sz uint64 // Calculated size (only if calc != nil) + name string + m *sync.Map + ipQueueOpts[T] +} + +type ipQueueOpts[T any] struct { + mrs int // Max recycle size + calc func(e T) uint64 // Calc function for tracking size + msz uint64 // Limit by total calculated size + mlen int // Limit by number of entries +} + +type ipQueueOpt[T any] func(*ipQueueOpts[T]) + +// This option allows to set the maximum recycle size when attempting +// to put back a slice to the pool. +func ipqMaxRecycleSize[T any](max int) ipQueueOpt[T] { + return func(o *ipQueueOpts[T]) { + o.mrs = max + } +} + +// This option enables total queue size counting by passing in a function +// that evaluates the size of each entry as it is pushed/popped. This option +// enables the size() function. +func ipqSizeCalculation[T any](calc func(e T) uint64) ipQueueOpt[T] { + return func(o *ipQueueOpts[T]) { + o.calc = calc + } +} + +// This option allows setting the maximum queue size. Once the limit is +// reached, then push() will stop returning true and no more entries will +// be stored until some more are popped. The ipQueue_SizeCalculation must +// be provided for this to work. +func ipqLimitBySize[T any](max uint64) ipQueueOpt[T] { + return func(o *ipQueueOpts[T]) { + o.msz = max + } +} + +// This option allows setting the maximum queue length. Once the limit is +// reached, then push() will stop returning true and no more entries will +// be stored until some more are popped. +func ipqLimitByLen[T any](max int) ipQueueOpt[T] { + return func(o *ipQueueOpts[T]) { + o.mlen = max + } +} + +var errIPQLenLimitReached = errors.New("IPQ len limit reached") +var errIPQSizeLimitReached = errors.New("IPQ size limit reached") + +func newIPQueue[T any](s *Server, name string, opts ...ipQueueOpt[T]) *ipQueue[T] { + q := &ipQueue[T]{ + ch: make(chan struct{}, 1), + pool: &sync.Pool{ + New: func() any { + // Reason we use pointer to slice instead of slice is explained + // here: https://staticcheck.io/docs/checks#SA6002 + res := make([]T, 0, 32) + return &res + }, + }, + name: name, + m: &s.ipQueues, + ipQueueOpts: ipQueueOpts[T]{ + mrs: ipQueueDefaultMaxRecycleSize, + }, + } + for _, o := range opts { + o(&q.ipQueueOpts) + } + s.ipQueues.Store(name, q) + return q +} + +// Add the element `e` to the queue, notifying the queue channel's `ch` if the +// entry is the first to be added, and returns the length of the queue after +// this element is added. +func (q *ipQueue[T]) push(e T) (int, error) { + q.Lock() + l := len(q.elts) - q.pos + if q.mlen > 0 && l == q.mlen { + q.Unlock() + return l, errIPQLenLimitReached + } + if q.calc != nil { + sz := q.calc(e) + if q.msz > 0 && q.sz+sz > q.msz { + q.Unlock() + return l, errIPQSizeLimitReached + } + q.sz += sz + } + if q.elts == nil { + // What comes out of the pool is already of size 0, so no need for [:0]. + q.elts = *(q.pool.Get().(*[]T)) + } + q.elts = append(q.elts, e) + q.Unlock() + if l == 0 { + select { + case q.ch <- struct{}{}: + default: + } + } + return l + 1, nil +} + +// Add all elements yielded by seq to the queue while holding the queue +// lock, preventing other producers from pushing interleaving elements. +// On success, it returns the queue length after adding all yielded elements. +// If a queue limit is reached, no yielded elements are retained, and it +// returns the unchanged queue length and the first limit error. +func (q *ipQueue[T]) pushMany(seq iter.Seq[T]) (int, error) { + q.Lock() + l, added, start := len(q.elts)-q.pos, 0, len(q.elts) + initialSize := q.sz + revert := func() { + clear(q.elts[start:]) + q.elts = q.elts[:start] + q.sz, added = initialSize, 0 + } + defer func() { + q.Unlock() + if l == 0 && added > 0 { + select { + case q.ch <- struct{}{}: + default: + } + } + }() + + for e := range seq { + if q.mlen > 0 && l+added == q.mlen { + revert() + return l, errIPQLenLimitReached + } + if q.calc != nil { + sz := q.calc(e) + if q.msz > 0 && q.sz+sz > q.msz { + revert() + return l, errIPQSizeLimitReached + } + q.sz += sz + } + if q.elts == nil { + // What comes out of the pool is already of size 0, so no need for [:0]. + q.elts = *(q.pool.Get().(*[]T)) + } + q.elts = append(q.elts, e) + added++ + } + return l + added, nil +} + +// Returns the whole list of elements currently present in the queue, +// emptying the queue. This should be called after receiving a notification +// from the queue's `ch` notification channel that indicates that there +// is something in the queue. +// However, in cases where `drain()` may be called from another go +// routine, it is possible that a routine is notified that there is +// something, but by the time it calls `pop()`, the drain() would have +// emptied the queue. So the caller should never assume that pop() will +// return a slice of 1 or more, it could return `nil`. +func (q *ipQueue[T]) pop() []T { + if q == nil { + return nil + } + q.Lock() + if len(q.elts)-q.pos == 0 { + q.Unlock() + return nil + } + var elts []T + if q.pos == 0 { + elts = q.elts + } else { + elts = q.elts[q.pos:] + } + q.elts, q.pos, q.sz = nil, 0, 0 + atomic.AddInt64(&q.inprogress, int64(len(elts))) + q.Unlock() + return elts +} + +// Returns the first element from the queue, if any. See comment above +// regarding calling after being notified that there is something and +// the use of drain(). In short, the caller should always check the +// boolean return value to ensure that the value is genuine and not a +// default empty value. +func (q *ipQueue[T]) popOne() (T, bool) { + q.Lock() + l := len(q.elts) - q.pos + if l == 0 { + q.Unlock() + var empty T + return empty, false + } + e := q.elts[q.pos] + if l--; l > 0 { + q.pos++ + if q.calc != nil { + q.sz -= q.calc(e) + } + // We need to re-signal + select { + case q.ch <- struct{}{}: + default: + } + } else { + // We have just emptied the queue, so we can reuse unless it is too big. + if cap(q.elts) <= q.mrs { + q.elts = q.elts[:0] + } else { + q.elts = nil + } + q.pos, q.sz = 0, 0 + } + q.Unlock() + return e, true +} + +// After a pop(), the slice can be recycled for the next push() when +// a first element is added to the queue. +// This will also decrement the "in progress" count with the length +// of the slice. +// WARNING: The caller MUST never reuse `elts`. +func (q *ipQueue[T]) recycle(elts *[]T) { + // If invoked with a nil list, nothing to do. + if elts == nil || *elts == nil { + return + } + // Update the in progress count. + if len(*elts) > 0 { + atomic.AddInt64(&q.inprogress, int64(-(len(*elts)))) + } + // We also don't want to recycle huge slices, so check against the max. + // q.mrs is normally immutable but can be changed, in a safe way, in some tests. + if cap(*elts) > q.mrs { + return + } + (*elts) = (*elts)[:0] + q.pool.Put(elts) +} + +// Returns the current length of the queue. +func (q *ipQueue[T]) len() int { + q.Lock() + defer q.Unlock() + return len(q.elts) - q.pos +} + +// Returns the calculated size of the queue (if ipQueue_SizeCalculation has been +// passed in), otherwise returns zero. +func (q *ipQueue[T]) size() uint64 { + q.Lock() + defer q.Unlock() + return q.sz +} + +// Empty the queue and consumes the notification signal if present. +// Returns the number of items that were drained from the queue. +// Note that this could cause a reader go routine that has been +// notified that there is something in the queue (reading from queue's `ch`) +// may then get nothing if `drain()` is invoked before the `pop()` or `popOne()`. +func (q *ipQueue[T]) drain() int { + if q == nil { + return 0 + } + q.Lock() + olen := len(q.elts) - q.pos + q.elts, q.pos, q.sz = nil, 0, 0 + // Consume the signal if it was present to reduce the chance of a reader + // routine to be think that there is something in the queue... + select { + case <-q.ch: + default: + } + q.Unlock() + return olen +} + +// Since the length of the queue goes to 0 after a pop(), it is good to +// have an insight on how many elements are yet to be processed after a pop(). +// For that reason, the queue maintains a count of elements returned through +// the pop() API. When the caller will call q.recycle(), this count will +// be reduced by the size of the slice returned by pop(). +func (q *ipQueue[T]) inProgress() int64 { + return atomic.LoadInt64(&q.inprogress) +} + +// Remove this queue from the server's map of ipQueues. +// All ipQueue operations (such as push/pop/etc..) are still possible. +func (q *ipQueue[T]) unregister() { + if q == nil { + return + } + q.m.Delete(q.name) +} diff --git a/server/ipqueue_test.go b/server/ipqueue_test.go new file mode 100644 index 0000000..85bb34c --- /dev/null +++ b/server/ipqueue_test.go @@ -0,0 +1,735 @@ +// Copyright 2021-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestIPQueueBasic(t *testing.T) { + s := &Server{} + q := newIPQueue[int](s, "test") + // Check that pool has been created + if q.pool == nil { + t.Fatal("Expected pool to have been created") + } + // Check for the default mrs + if q.mrs != ipQueueDefaultMaxRecycleSize { + t.Fatalf("Expected default max recycle size to be %v, got %v", + ipQueueDefaultMaxRecycleSize, q.mrs) + } + select { + case <-q.ch: + t.Fatalf("Should not have been notified") + default: + // OK! + } + if l := q.len(); l != 0 { + t.Fatalf("Expected len to be 0, got %v", l) + } + + // Try to change the max recycle size + q2 := newIPQueue[int](s, "test2", ipqMaxRecycleSize[int](10)) + if q2.mrs != 10 { + t.Fatalf("Expected max recycle size to be 10, got %v", q2.mrs) + } + + // Check that those 2 queues are registered + var gotFirst bool + var gotSecond bool + s.ipQueues.Range(func(k, v any) bool { + switch k.(string) { + case "test": + gotFirst = true + case "test2": + gotSecond = true + default: + t.Fatalf("Unknown queue: %q", k.(string)) + } + return true + }) + if !gotFirst { + t.Fatalf("Did not find queue %q", "test") + } + if !gotSecond { + t.Fatalf("Did not find queue %q", "test2") + } + // Unregister them + q.unregister() + q2.unregister() + // They should have been removed from the map + s.ipQueues.Range(func(k, v any) bool { + t.Fatalf("Got queue %q", k.(string)) + return false + }) + // But verify that we can still push/pop + q.push(1) + elts := q.pop() + if len(elts) != 1 { + t.Fatalf("Should have gotten 1 element, got %v", len(elts)) + } + q2.push(2) + if e, ok := q2.popOne(); !ok || e != 2 { + t.Fatalf("popOne failed: %+v", e) + } +} + +func TestIPQueuePush(t *testing.T) { + s := &Server{} + q := newIPQueue[int](s, "test") + q.push(1) + if l := q.len(); l != 1 { + t.Fatalf("Expected len to be 1, got %v", l) + } + select { + case <-q.ch: + // OK + default: + t.Fatalf("Should have been notified of addition") + } + // Push a new element, we should not be notified. + q.push(2) + if l := q.len(); l != 2 { + t.Fatalf("Expected len to be 2, got %v", l) + } + select { + case <-q.ch: + t.Fatalf("Should not have been notified of addition") + default: + // OK + } +} + +func TestIPQueuePushMany(t *testing.T) { + s := &Server{} + q := newIPQueue[int](s, "test") + + l, err := q.pushMany(func(yield func(int) bool) { + for i := 1; i <= 3; i++ { + if !yield(i) { + return + } + } + }) + require_NoError(t, err) + require_Equal(t, l, 3) + select { + case <-q.ch: + default: + t.Fatal("Should have been notified of additions") + } + values := q.pop() + require_Len(t, len(values), 3) + for i, value := range values { + require_Equal(t, value, i+1) + } + q.recycle(&values) + + l, err = q.pushMany(func(func(int) bool) {}) + require_NoError(t, err) + require_Equal(t, l, 0) + select { + case <-q.ch: + t.Fatal("Should not have been notified of additions") + default: + } + + limited := newIPQueue[int](s, "limited", ipqLimitByLen[int](2)) + l, err = limited.pushMany(func(yield func(int) bool) { + for i := 1; i <= 3; i++ { + if !yield(i) { + return + } + } + }) + require_Error(t, err, errIPQLenLimitReached) + require_Equal(t, l, 0) + require_Equal(t, limited.len(), 0) + select { + case <-limited.ch: + t.Fatal("Should not have been notified of rejected additions") + default: + } +} + +func TestIPQueuePop(t *testing.T) { + s := &Server{} + q := newIPQueue[int](s, "test") + q.push(1) + <-q.ch + elts := q.pop() + if l := len(elts); l != 1 { + t.Fatalf("Expected 1 elt, got %v", l) + } + if l := q.len(); l != 0 { + t.Fatalf("Expected len to be 0, got %v", l) + } + // The channel notification should be empty + select { + case <-q.ch: + t.Fatalf("Should not have been notified of addition") + default: + // OK + } + // Since pop() brings the number of pending to 0, we keep track of the + // number of "in progress" elements. Check that the value is 1 here. + if n := q.inProgress(); n != 1 { + t.Fatalf("Expected count to be 1, got %v", n) + } + // Recycling will bring it down to 0. + q.recycle(&elts) + if n := q.inProgress(); n != 0 { + t.Fatalf("Expected count to be 0, got %v", n) + } + // If we call pop() now, we should get nil. + if elts = q.pop(); elts != nil { + t.Fatalf("Expected nil, got %v", elts) + } + // The in progress count should still be 0 + if n := q.inProgress(); n != 0 { + t.Fatalf("Expected count to be 0, got %v", n) + } + // Calling pop() on an unitialized queue will return nil + var q2 *ipQueue[int] + elts = q2.pop() + require_True(t, elts == nil) +} + +func TestIPQueuePopOne(t *testing.T) { + s := &Server{} + q := newIPQueue[int](s, "test") + q.push(1) + <-q.ch + e, ok := q.popOne() + if !ok { + t.Fatal("Got nil") + } + if i := e; i != 1 { + t.Fatalf("Expected 1, got %v", i) + } + if l := q.len(); l != 0 { + t.Fatalf("Expected len to be 0, got %v", l) + } + // That does not affect the number of notProcessed + if n := q.inProgress(); n != 0 { + t.Fatalf("Expected count to be 0, got %v", n) + } + select { + case <-q.ch: + t.Fatalf("Should not have been notified of addition") + default: + // OK + } + q.push(2) + q.push(3) + e, ok = q.popOne() + if !ok { + t.Fatal("Got nil") + } + if i := e; i != 2 { + t.Fatalf("Expected 2, got %v", i) + } + if l := q.len(); l != 1 { + t.Fatalf("Expected len to be 1, got %v", l) + } + select { + case <-q.ch: + // OK + default: + t.Fatalf("Should have been notified that there is more") + } + e, ok = q.popOne() + if !ok { + t.Fatal("Got nil") + } + if i := e; i != 3 { + t.Fatalf("Expected 3, got %v", i) + } + if l := q.len(); l != 0 { + t.Fatalf("Expected len to be 0, got %v", l) + } + select { + case <-q.ch: + t.Fatalf("Should not have been notified that there is more") + default: + // OK + } + // Calling it again now that we know there is nothing, we + // should get nil. + if e, ok = q.popOne(); ok { + t.Fatalf("Expected nil, got %v", e) + } + + q = newIPQueue[int](s, "test2") + q.push(1) + q.push(2) + // Capture current capacity + q.Lock() + c := cap(q.elts) + q.Unlock() + e, ok = q.popOne() + if !ok || e != 1 { + t.Fatalf("Invalid value: %v", e) + } + if l := q.len(); l != 1 { + t.Fatalf("Expected len to be 1, got %v", l) + } + values := q.pop() + if len(values) != 1 || values[0] != 2 { + t.Fatalf("Unexpected values: %v", values) + } + if cap(values) != c-1 { + t.Fatalf("Unexpected capacity: %v vs %v", cap(values), c-1) + } + if l := q.len(); l != 0 { + t.Fatalf("Expected len to be 0, got %v", l) + } + // Just make sure that this is ok... + q.recycle(&values) +} + +func TestIPQueueMultiProducers(t *testing.T) { + s := &Server{} + q := newIPQueue[int](s, "test") + + wg := sync.WaitGroup{} + wg.Add(3) + send := func(start, end int) { + defer wg.Done() + + for i := start; i <= end; i++ { + q.push(i) + } + } + go send(1, 100) + go send(101, 200) + go send(201, 300) + + tm := time.NewTimer(2 * time.Second) + m := make(map[int]struct{}) + for done := false; !done; { + select { + case <-q.ch: + values := q.pop() + for _, v := range values { + m[v] = struct{}{} + } + q.recycle(&values) + if n := q.inProgress(); n != 0 { + t.Fatalf("Expected count to be 0, got %v", n) + } + done = len(m) == 300 + case <-tm.C: + t.Fatalf("Did not receive all elements: %v", m) + } + } + wg.Wait() +} + +func TestIPQueueRecycle(t *testing.T) { + s := &Server{} + q := newIPQueue[int](s, "test") + total := 1000 + for iter := 0; iter < 5; iter++ { + var sz int + for i := 0; i < total; i++ { + sz, _ = q.push(i) + } + if sz != total { + t.Fatalf("Expected size to be %v, got %v", total, sz) + } + values := q.pop() + preRecycleCap := cap(values) + q.recycle(&values) + sz, _ = q.push(1001) + if sz != 1 { + t.Fatalf("Expected size to be %v, got %v", 1, sz) + } + values = q.pop() + if l := len(values); l != 1 { + t.Fatalf("Len should be 1, got %v", l) + } + if c := cap(values); c == preRecycleCap { + break + } else if iter == 4 { + // We can't fail the test since there is no guarantee that the slice + // is still present in the pool when we do a Get(), but let's log that + // recycling did not occur even after all iterations.. + t.Logf("Seem like the previous slice was not recycled, old cap=%v new cap=%v", + preRecycleCap, c) + } + } + + q = newIPQueue[int](s, "test2", ipqMaxRecycleSize[int](10)) + for i := 0; i < 100; i++ { + q.push(i) + } + values := q.pop() + preRecycleCap := cap(values) + q.recycle(&values) + q.push(1001) + values = q.pop() + if l := len(values); l != 1 { + t.Fatalf("Len should be 1, got %v", l) + } + // This time, we should not have recycled it, so the new cap should be whatever + // the default is, but less than the preRecycleCap. + if c := cap(values); c >= preRecycleCap { + t.Fatalf("The slice should not have been put back in the pool, got cap of %v", c) + } + q.recycle(&values) + + // Check that we don't crash when recycling a nil or empty slice + values = q.pop() + q.recycle(&values) + q.recycle(nil) +} + +func TestIPQueueDrain(t *testing.T) { + s := &Server{} + q := newIPQueue[int](s, "test") + for iter, recycled := 0, false; iter < 5 && !recycled; iter++ { + for i := 0; i < 100; i++ { + q.push(i + 1) + } + q.drain() + // Try to get something from the pool right away + s := q.pool.Get() + recycled := s != nil + if !recycled { + // We can't fail the test, since we have no guarantee it will be recycled + // especially when running with `-race` flag... + if iter == 4 { + t.Log("nothing was recycled") + } + } + // Check that we have consumed the signal... + select { + case <-q.ch: + t.Fatal("Signal should have been consumed by drain") + default: + // OK! + } + // Check len + if l := q.len(); l != 0 { + t.Fatalf("Expected len to be 0, got %v", l) + } + if recycled { + break + } + } +} + +func TestIPQueueSizeCalculation(t *testing.T) { + type testType = [16]byte + var testValue testType + + calc := ipqSizeCalculation[testType](func(e testType) uint64 { + return uint64(len(e)) + }) + s := &Server{} + q := newIPQueue[testType](s, "test", calc) + + for i := 0; i < 10; i++ { + q.push(testValue) + require_Equal(t, q.len(), i+1) + require_Equal(t, q.size(), uint64(i+1)*uint64(len(testValue))) + } + + for i := 10; i > 5; i-- { + q.popOne() + require_Equal(t, q.len(), i-1) + require_Equal(t, q.size(), uint64(i-1)*uint64(len(testValue))) + } + + q.pop() + require_Equal(t, q.len(), 0) + require_Equal(t, q.size(), 0) +} + +func TestIPQueueSizeCalculationWithLimits(t *testing.T) { + type testType = [16]byte + var testValue testType + + calc := ipqSizeCalculation[testType](func(e testType) uint64 { + return uint64(len(e)) + }) + s := &Server{} + + t.Run("LimitByLen", func(t *testing.T) { + q := newIPQueue[testType](s, "test", calc, ipqLimitByLen[testType](5)) + for i := 0; i < 10; i++ { + n, err := q.push(testValue) + if i >= 5 { + require_Error(t, err, errIPQLenLimitReached) + } else { + require_NoError(t, err) + } + require_LessThan(t, n, 6) + } + }) + + t.Run("LimitBySize", func(t *testing.T) { + q := newIPQueue[testType](s, "test", calc, ipqLimitBySize[testType](16*5)) + for i := 0; i < 10; i++ { + n, err := q.push(testValue) + if i >= 5 { + require_Error(t, err, errIPQSizeLimitReached) + } else { + require_NoError(t, err) + } + require_LessThan(t, n, 6) + } + }) +} + +func Benchmark_IPQueueSizeCalculation(b *testing.B) { + type testType = [16]byte + var testValue testType + + s := &Server{} + + runPopOne := func(b *testing.B, q *ipQueue[testType]) { + b.SetBytes(16) + for i := 0; i < b.N; i++ { + q.push(testValue) + } + for i := b.N; i > 0; i-- { + q.popOne() + } + } + runPopAll := func(b *testing.B, q *ipQueue[testType]) { + b.SetBytes(16) + for i := 0; i < b.N; i++ { + q.push(testValue) + } + elts := q.pop() + for _, v := range elts { + _ = v + } + q.recycle(&elts) + } + for _, test := range []struct { + name string + run func(b *testing.B, q *ipQueue[testType]) + }{ + {"pop one", runPopOne}, + {"pop all", runPopAll}, + } { + b.Run(test.name, func(b *testing.B) { + // Measures without calculation function overheads. + b.Run("WithoutCalc", func(b *testing.B) { + test.run(b, newIPQueue[testType](s, "test")) + }) + + // Measures the raw overhead of having a calculation function. + b.Run("WithEmptyCalc", func(b *testing.B) { + calc := ipqSizeCalculation[testType](func(e testType) uint64 { + return 0 + }) + test.run(b, newIPQueue[testType](s, "test", calc)) + }) + + // Measures the overhead of having a calculation function that + // actually measures something useful. + b.Run("WithLenCalc", func(b *testing.B) { + calc := ipqSizeCalculation[testType](func(e testType) uint64 { + return uint64(len(e)) + }) + test.run(b, newIPQueue[testType](s, "test", calc)) + }) + }) + } +} + +func Benchmark_IPQueue1Prod1ConsPopOne(b *testing.B) { + benchIPQueue(b, 1, 1, true, false) +} + +func Benchmark_IPQueue5Prod1ConsPopOne(b *testing.B) { + benchIPQueue(b, 5, 1, true, false) +} + +func Benchmark_IPQueue1Prod5ConsPopOne(b *testing.B) { + benchIPQueue(b, 1, 5, true, false) +} + +func Benchmark_IPQueue5Prod5ConsPopOne(b *testing.B) { + benchIPQueue(b, 5, 5, true, false) +} + +func Benchmark_IPQueue1Prod1ConsPopAll(b *testing.B) { + benchIPQueue(b, 1, 1, false, false) +} + +func Benchmark_IPQueue5Prod1ConsPopAll(b *testing.B) { + benchIPQueue(b, 5, 1, false, false) +} + +func Benchmark_IPQueue1Prod5ConsPopAll(b *testing.B) { + benchIPQueue(b, 1, 5, false, false) +} + +func Benchmark_IPQueue5Prod5ConsPopAll(b *testing.B) { + benchIPQueue(b, 5, 5, false, false) +} + +func Benchmark_IPQueueWithLim1Prod1ConsPopOne(b *testing.B) { + benchIPQueue(b, 1, 1, true, true) +} + +func Benchmark_IPQueueWithLim5Prod1ConsPopOne(b *testing.B) { + benchIPQueue(b, 5, 1, true, true) +} + +func Benchmark_IPQueueWithLim1Prod5ConsPopOne(b *testing.B) { + benchIPQueue(b, 1, 5, true, true) +} + +func Benchmark_IPQueueWithLim5Prod5ConsPopOne(b *testing.B) { + benchIPQueue(b, 5, 5, true, true) +} + +func Benchmark_IPQueueWithLim1Prod1ConsPopAll(b *testing.B) { + benchIPQueue(b, 1, 1, false, true) +} + +func Benchmark_IPQueueWithLim5Prod1ConsPopAll(b *testing.B) { + benchIPQueue(b, 5, 1, false, true) +} + +func Benchmark_IPQueueWithLim1Prod5ConsPopAll(b *testing.B) { + benchIPQueue(b, 1, 5, false, true) +} + +func Benchmark_IPQueueWithLim5Prod5ConsPopAll(b *testing.B) { + benchIPQueue(b, 5, 5, false, true) +} + +func benchIPQueue(b *testing.B, numProd, numCons int, popOne, withLimits bool) { + var count atomic.Int64 + var sum atomic.Int64 + + perProd := (b.N / numProd) + 1 + total := int64(perProd * numProd) + + var opts []ipQueueOpt[int] + if withLimits { + // Add len/size options but make sure they are big enough that we don't fail push() calls. + // What we want is to activate code that is checking for limits. + opts = append(opts, ipqLimitByLen[int](int(total+100))) + opts = append(opts, ipqLimitBySize[int](uint64((total+100)*5))) + opts = append(opts, ipqSizeCalculation[int](func(e int) uint64 { return uint64(5) })) + } + queue := newIPQueue[int](&Server{}, "bench", opts...) + + cdone := make(chan struct{}, numCons) + for i := 0; i < numCons; i++ { + go func() { + for range queue.ch { + if popOne { + v, ok := queue.popOne() + if !ok { + continue + } + sum.Add(int64(v)) + if count.Add(1) >= total { + cdone <- struct{}{} + return + } + } else { + elts := queue.pop() + for _, v := range elts { + sum.Add(int64(v)) + if count.Add(1) >= total { + queue.recycle(&elts) + cdone <- struct{}{} + return + } + } + queue.recycle(&elts) + } + } + }() + } + + pwg := sync.WaitGroup{} + pwg.Add(numProd) + var expected atomic.Int64 + for i := 0; i < numProd; i++ { + go func() { + defer pwg.Done() + for i := 0; i < perProd; i++ { + expected.Add(int64(i + 1)) + queue.push(i + 1) + } + }() + } + + pwg.Wait() + <-cdone + b.StopTimer() + + for i := 1; i < numCons; i++ { + queue.push(0) + <-cdone + } + if sum.Load() != expected.Load() { + b.Fatalf("Invalid sum, expected %v, got %v", expected.Load(), sum.Load()) + } +} + +func Benchmark_IPQueuePushAndPopOne(b *testing.B) { + s := &Server{} + q := newIPQueue[int](s, "test") + for i := 0; i < b.N; i++ { + q.push(i + 1) + } + require_Equal[int](b, q.len(), b.N) + for i := 0; i < b.N; i++ { + v, ok := q.popOne() + require_True(b, ok) + require_Equal[int](b, v, i+1) + } + require_Equal[int](b, q.len(), 0) + require_Equal(b, q.inProgress(), 0) +} + +func Benchmark_IPQueuePushAndPopAll(b *testing.B) { + s := &Server{} + q := newIPQueue[int](s, "test") + expected := 0 + for i := 0; i < b.N; i++ { + q.push(i + 1) + expected += i + 1 + } + require_Equal(b, q.len(), b.N) + <-q.ch + + elts := q.pop() + require_Equal(b, len(elts), b.N) + require_Equal(b, q.len(), 0) + require_Equal(b, q.inProgress(), int64(b.N)) + sum := 0 + for _, v := range elts { + sum += v + } + q.recycle(&elts) + require_Equal(b, expected, sum) + require_Equal(b, q.len(), 0) + require_Equal(b, q.inProgress(), 0) +} diff --git a/server/jetstream.go b/server/jetstream.go new file mode 100644 index 0000000..1a6915a --- /dev/null +++ b/server/jetstream.go @@ -0,0 +1,3017 @@ +// Copyright 2019-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "runtime/debug" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/minio/highwayhash" + "github.com/nats-io/nats-server/v2/server/gsl" + "github.com/nats-io/nats-server/v2/server/sysmem" + "github.com/nats-io/nats-server/v2/server/tpm" + "github.com/nats-io/nkeys" + "github.com/nats-io/nuid" +) + +// JetStreamConfig determines this server's configuration. +// MaxMemory and MaxStore are in bytes. +type JetStreamConfig struct { + MaxMemory int64 `json:"max_memory"` // MaxMemory is the maximum size of memory type streams + MaxStore int64 `json:"max_storage"` // MaxStore is the maximum size of file store type streams + StoreDir string `json:"store_dir,omitempty"` // StoreDir is where storage files are stored + SyncInterval time.Duration `json:"sync_interval,omitempty"` // SyncInterval is how frequently we sync to disk in the background by calling fsync + SyncAlways bool `json:"sync_always,omitempty"` // SyncAlways indicates flushes are done after every write + Domain string `json:"domain,omitempty"` // Domain is the JetStream domain + CompressOK bool `json:"compress_ok,omitempty"` // CompressOK indicates if compression is supported + UniqueTag string `json:"unique_tag,omitempty"` // UniqueTag is the unique tag assigned to this instance + Strict bool `json:"strict,omitempty"` // Strict indicates if strict JSON parsing is performed + + // maxStorePending is set when MaxStore was derived from the available disk + // space rather than configured, and has not been adjusted yet for the space + // that recovered streams already occupy. See finalizeDynamicMaxStore. + maxStorePending bool +} + +// Statistics about JetStream for this server. +type JetStreamStats struct { + Memory uint64 `json:"memory"` + Store uint64 `json:"storage"` + ReservedMemory uint64 `json:"reserved_memory"` + ReservedStore uint64 `json:"reserved_storage"` + Accounts int `json:"accounts"` + HAAssets int `json:"ha_assets"` + API JetStreamAPIStats `json:"api"` +} + +type JetStreamAccountLimits struct { + MaxMemory int64 `json:"max_memory"` + MaxStore int64 `json:"max_storage"` + MaxStreams int `json:"max_streams"` + MaxConsumers int `json:"max_consumers"` + MaxAckPending int `json:"max_ack_pending"` + MemoryMaxStreamBytes int64 `json:"memory_max_stream_bytes"` + StoreMaxStreamBytes int64 `json:"storage_max_stream_bytes"` + MaxBytesRequired bool `json:"max_bytes_required"` +} + +type JetStreamTier struct { + Memory uint64 `json:"memory"` + Store uint64 `json:"storage"` + ReservedMemory uint64 `json:"reserved_memory"` + ReservedStore uint64 `json:"reserved_storage"` + Streams int `json:"streams"` + Consumers int `json:"consumers"` + Limits JetStreamAccountLimits `json:"limits"` +} + +// JetStreamAccountStats returns current statistics about the account's JetStream usage. +type JetStreamAccountStats struct { + JetStreamTier // in case tiers are used, reflects totals with limits not set + Domain string `json:"domain,omitempty"` + API JetStreamAPIStats `json:"api"` + Tiers map[string]JetStreamTier `json:"tiers,omitempty"` // indexed by tier name +} + +// JetStreamAPIStats holds stats about the API usage for this server +type JetStreamAPIStats struct { + Level int `json:"level"` // Level is the active API level this server implements + Total uint64 `json:"total"` // Total is the total API requests received since start + Errors uint64 `json:"errors"` // Errors is the total API requests that resulted in error responses + Inflight uint64 `json:"inflight,omitempty"` // Inflight are the number of API requests currently being served +} + +// This is for internal accounting for JetStream for this server. +type jetStream struct { + // These are here first because of atomics on 32bit systems. + apiInflight int64 + apiTotal int64 + apiErrors int64 + memMax int64 + memReserved int64 // Requires JS lock to be held. + memUsed int64 + storeMax int64 + storeReserved int64 // Requires JS lock to be held. + storeUsed int64 + queueLimit int64 + infoQueueLimit int64 + clustered int32 + mu sync.RWMutex + srv *Server + config JetStreamConfig + cluster *jetStreamCluster + accounts map[string]*jsAccount + apiSubs *Sublist + infoSubs *gsl.SimpleSublist // Subjects for info-specific queue. + started time.Time + + // System level request to purge a stream move + accountPurge *subscription + + // Debounced reconcile of assignments for peers that became selectable. + // Has its own lock so signaling is cheap and never contends on the JS lock. + prMu sync.Mutex + prPeers map[string]struct{} + // Meta peers that were just added but that we can't place on yet, because + // we have no STATSZ for them. Their first STATSZ moves them into prPeers. + prNewPeers map[string]struct{} + prRunning bool + + // Some bools regarding general state. + metaRecovering bool + standAlone bool + oos bool + shuttingDown bool + + // Atomic versions + disabled atomic.Bool +} + +type remoteUsage struct { + tiers map[string]*jsaUsage // indexed by tier name + api uint64 + err uint64 +} + +type jsaStorage struct { + total jsaUsage + local jsaUsage +} + +// This represents a jetstream enabled account. +// Worth noting that we include the jetstream pointer, this is because +// in general we want to be very efficient when receiving messages on +// an internal sub for a stream, so we will direct link to the stream +// and walk backwards as needed vs multiple hash lookups and locks, etc. +type jsAccount struct { + mu sync.RWMutex + js *jetStream + account *Account + storeDir string + inflight sync.Map + streams map[string]*stream + + // From server + sendq *ipQueue[*pubMsg] + + // For limiting only running one checkAndSync at a time. + sync atomic.Bool + + // Usage/limits related fields that will be protected by usageMu + usageMu sync.RWMutex + limits map[string]JetStreamAccountLimits // indexed by tierName + usage map[string]*jsaStorage // indexed by tierName + rusage map[string]*remoteUsage // indexed by node id + apiTotal uint64 + apiErrors uint64 + usageApi uint64 + usageErr uint64 + updatesPub string + updatesSub *subscription + lupdate time.Time + utimer *time.Timer +} + +// Track general usage for this account. +type jsaUsage struct { + mem int64 + store int64 +} + +// EnableJetStream will enable JetStream support on this server with the given configuration. +// A nil configuration will dynamically choose the limits and temporary file storage directory. +func (s *Server) EnableJetStream(config *JetStreamConfig) error { + if s.JetStreamEnabled() { + return fmt.Errorf("jetstream already enabled") + } + + s.Noticef("Starting JetStream") + start := time.Now() + defer func() { + s.Noticef("Took %s to start JetStream", time.Since(start)) + }() + + if config == nil || config.MaxMemory <= 0 || config.MaxStore <= 0 { + var storeDir, domain, uniqueTag string + var maxStore, maxMem int64 + if config != nil { + storeDir, domain, uniqueTag = config.StoreDir, config.Domain, config.UniqueTag + maxStore, maxMem = config.MaxStore, config.MaxMemory + } + config = s.dynJetStreamConfig(storeDir, maxStore, maxMem) + if domain != _EMPTY_ { + config.Domain = domain + } + if uniqueTag != _EMPTY_ { + config.UniqueTag = uniqueTag + } + s.Debugf("JetStream creating dynamic configuration - %s memory, %s disk", friendlyBytes(config.MaxMemory), friendlyBytes(config.MaxStore)) + } else if config.StoreDir != _EMPTY_ { + config.StoreDir = filepath.Join(config.StoreDir, JetStreamStoreDir) + } + + cfg := *config + if cfg.StoreDir == _EMPTY_ { + cfg.StoreDir = filepath.Join(os.TempDir(), JetStreamStoreDir) + s.Warnf("Temporary storage directory used, data could be lost on system reboot") + } + + // We will consistently place the 'jetstream' directory under the storedir that was handed to us. Prior to 2.2.3 though + // we could have a directory on disk without the 'jetstream' directory. This will check and fix if needed. + if err := s.checkStoreDir(&cfg); err != nil { + return err + } + + return s.enableJetStream(cfg) +} + +// Function signature to generate a key encryption key. +type keyGen func(context []byte) ([]byte, error) + +// Return a key generation function or nil if encryption not enabled. +func (s *Server) jsKeyGen(jsKey, info string) keyGen { + if ek := jsKey; ek != _EMPTY_ { + return func(context []byte) ([]byte, error) { + h := hmac.New(sha256.New, []byte(ek)) + if _, err := h.Write([]byte(info)); err != nil { + return nil, err + } + if _, err := h.Write(context); err != nil { + return nil, err + } + return h.Sum(nil), nil + } + } + return nil +} + +// Decode the encrypted metafile. +func (s *Server) decryptMeta(sc StoreCipher, ekey, buf []byte, acc, context string) ([]byte, bool, error) { + if len(ekey) < minMetaKeySize { + return nil, false, errBadKeySize + } + var osc StoreCipher + switch sc { + case AES: + osc = ChaCha + case ChaCha: + osc = AES + } + type prfWithCipher struct { + keyGen + StoreCipher + } + var prfs []prfWithCipher + if prf := s.jsKeyGen(s.getOpts().JetStreamKey, acc); prf == nil { + return nil, false, errNoEncryption + } else { + // First of all, try our current encryption keys with both + // store cipher algorithms. + prfs = append(prfs, prfWithCipher{prf, sc}) + prfs = append(prfs, prfWithCipher{prf, osc}) + } + if prf := s.jsKeyGen(s.getOpts().JetStreamOldKey, acc); prf != nil { + // Then, if we have an old encryption key, try with also with + // both store cipher algorithms. + prfs = append(prfs, prfWithCipher{prf, sc}) + prfs = append(prfs, prfWithCipher{prf, osc}) + } + + for i, prf := range prfs { + rb, err := prf.keyGen([]byte(context)) + if err != nil { + continue + } + kek, err := genEncryptionKey(prf.StoreCipher, rb) + if err != nil { + continue + } + ns := kek.NonceSize() + seed, err := kek.Open(nil, ekey[:ns], ekey[ns:], nil) + if err != nil { + continue + } + aek, err := genEncryptionKey(prf.StoreCipher, seed) + if err != nil { + continue + } + if aek.NonceSize() != kek.NonceSize() { + continue + } + plain, err := aek.Open(nil, buf[:ns], buf[ns:], nil) + if err != nil { + continue + } + return plain, i > 0, nil + } + return nil, false, fmt.Errorf("unable to recover keys") +} + +// Check to make sure directory has the jetstream directory. +// We will have it properly configured here now regardless, so need to look inside. +func (s *Server) checkStoreDir(cfg *JetStreamConfig) error { + fis, _ := os.ReadDir(cfg.StoreDir) + // If we have nothing underneath us, could be just starting new, but if we see this we can check. + if len(fis) != 0 { + return nil + } + // Let's check the directory above. If it has us 'jetstream' but also other stuff that we can + // identify as accounts then we can fix. + fis, _ = os.ReadDir(filepath.Dir(cfg.StoreDir)) + // If just one that is us 'jetstream' and all is ok. + if len(fis) == 1 { + return nil + } + + haveJetstreamDir := false + for _, fi := range fis { + if fi.Name() == JetStreamStoreDir { + haveJetstreamDir = true + break + } + } + + for _, fi := range fis { + // Skip the 'jetstream' directory. + if fi.Name() == JetStreamStoreDir { + continue + } + // Let's see if this is an account. + if accName := fi.Name(); accName != _EMPTY_ { + _, ok := s.accounts.Load(accName) + if !ok && s.AccountResolver() != nil && nkeys.IsValidPublicAccountKey(accName) { + // Account is not local but matches the NKEY account public key, + // this is enough indication to move this directory, no need to + // fetch the account. + ok = true + } + // If this seems to be an account go ahead and move the directory. This will include all assets + // like streams and consumers. + if ok { + if !haveJetstreamDir { + err := os.Mkdir(filepath.Join(filepath.Dir(cfg.StoreDir), JetStreamStoreDir), defaultDirPerms) + if err != nil { + return err + } + haveJetstreamDir = true + } + old := filepath.Join(filepath.Dir(cfg.StoreDir), fi.Name()) + new := filepath.Join(cfg.StoreDir, fi.Name()) + s.Noticef("JetStream relocated account %q to %q", old, new) + if err := os.Rename(old, new); err != nil { + return err + } + } + } + } + + return nil +} + +// This function sets/updates the jetstream encryption key and cipher based +// on options. If the TPM options have been specified, a key is generated +// and sealed by the TPM. +func (s *Server) initJetStreamEncryption() (err error) { + opts := s.getOpts() + + // The TPM settings and other encryption settings are mutually exclusive. + if opts.JetStreamKey != _EMPTY_ && opts.JetStreamTpm.KeysFile != _EMPTY_ { + return fmt.Errorf("JetStream encryption key may not be used with TPM options") + } + // if we are using the standard method to set the encryption key just return and carry on. + if opts.JetStreamKey != _EMPTY_ { + return nil + } + // if the tpm options are not used then no encryption has been configured and return. + if opts.JetStreamTpm.KeysFile == _EMPTY_ { + return nil + } + + if opts.JetStreamTpm.Pcr == 0 { + // Default PCR to use in the TPM. Values can be 0-23, and most platforms + // reserve values 0-12 for the OS, boot locker, disc encryption, etc. + // 16 used for debugging. In sticking to NATS tradition, we'll use 22 + // as the default with the option being configurable. + opts.JetStreamTpm.Pcr = 22 + } + + // Using the TPM to generate or get the encryption key and update the encryption options. + opts.JetStreamKey, err = tpm.LoadJetStreamEncryptionKeyFromTPM(opts.JetStreamTpm.SrkPassword, + opts.JetStreamTpm.KeysFile, opts.JetStreamTpm.KeyPassword, opts.JetStreamTpm.Pcr) + + return err +} + +// enableJetStream will start up the JetStream subsystem. +func (s *Server) enableJetStream(cfg JetStreamConfig) error { + js := &jetStream{srv: s, config: cfg, accounts: make(map[string]*jsAccount), apiSubs: NewSublistNoCache(), infoSubs: gsl.NewSimpleSublist()} + s.gcbMu.Lock() + if s.gcbOutMax = s.getOpts().JetStreamMaxCatchup; s.gcbOutMax == 0 { + s.gcbOutMax = defaultMaxTotalCatchupOutBytes + } + s.gcbMu.Unlock() + + atomic.StoreInt64(&js.memMax, cfg.MaxMemory) + atomic.StoreInt64(&js.storeMax, cfg.MaxStore) + + // TODO: Not currently reloadable. + atomic.StoreInt64(&js.queueLimit, s.getOpts().JetStreamRequestQueueLimit) + atomic.StoreInt64(&js.infoQueueLimit, s.getOpts().JetStreamInfoQueueLimit) + + s.js.Store(js) + + // FIXME(dlc) - Allow memory only operation? + if stat, err := os.Stat(cfg.StoreDir); os.IsNotExist(err) { + if err := os.MkdirAll(cfg.StoreDir, defaultDirPerms); err != nil { + return fmt.Errorf("could not create storage directory - %v", err) + } + } else { + // Make sure its a directory and that we can write to it. + if stat == nil || !stat.IsDir() { + return fmt.Errorf("storage directory is not a directory") + } + tmpfile, err := os.CreateTemp(cfg.StoreDir, "_test_") + if err != nil { + return fmt.Errorf("storage directory is not writable") + } + tmpfile.Close() + os.Remove(tmpfile.Name()) + } + + if err := s.initJetStreamEncryption(); err != nil { + return err + } + + // JetStream is an internal service so we need to make sure we have a system account. + // This system account will export the JetStream service endpoints. + if s.SystemAccount() == nil { + s.SetDefaultSystemAccount() + } + + opts := s.getOpts() + if !opts.DisableJetStreamBanner { + s.Noticef(" _ ___ _____ ___ _____ ___ ___ _ __ __") + s.Noticef(" _ | | __|_ _/ __|_ _| _ \\ __| /_\\ | \\/ |") + s.Noticef("| || | _| | | \\__ \\ | | | / _| / _ \\| |\\/| |") + s.Noticef(" \\__/|___| |_| |___/ |_| |_|_\\___/_/ \\_\\_| |_|") + s.Noticef("") + s.Noticef(" https://docs.nats.io/jetstream") + s.Noticef("") + } + s.Noticef("---------------- JETSTREAM ----------------") + + if cfg.Strict { + s.Noticef(" Strict: %t", cfg.Strict) + } + + s.Noticef(" Max Memory: %s", friendlyBytes(cfg.MaxMemory)) + s.Noticef(" Max Storage: %s", friendlyBytes(cfg.MaxStore)) + s.Noticef(" Store Directory: \"%s\"", cfg.StoreDir) + if cfg.Domain != _EMPTY_ { + s.Noticef(" Domain: %s", cfg.Domain) + } + + if ek := opts.JetStreamKey; ek != _EMPTY_ { + s.Noticef(" Encryption: %s", opts.JetStreamCipher) + } + if opts.JetStreamTpm.KeysFile != _EMPTY_ { + s.Noticef(" TPM File: %q, Pcr: %d", opts.JetStreamTpm.KeysFile, + opts.JetStreamTpm.Pcr) + } + s.Noticef(" API Level: %d", JSApiLevel) + s.Noticef("-------------------------------------------") + + // Setup our internal subscriptions. + if err := s.setJetStreamExportSubs(); err != nil { + return fmt.Errorf("setting up internal jetstream subscriptions failed: %v", err) + } + + // Setup our internal system exports. + s.Debugf(" Exports:") + s.Debugf(" %s", jsAllAPI) + s.setupJetStreamExports() + + standAlone, canExtend := s.standAloneMode(), s.canExtendOtherDomain() + if standAlone && canExtend && s.getOpts().JetStreamExtHint != jsWillExtend { + canExtend = false + s.Noticef("Standalone server started in clustered mode do not support extending domains") + s.Noticef(`Manually disable standalone mode by setting the JetStream Option "extension_hint: %s"`, jsWillExtend) + } + + // Indicate if we will be standalone for checking resource reservations, etc. + js.setJetStreamStandAlone(standAlone && !canExtend) + + // Enable accounts and restore state before starting clustering. + if err := s.enableJetStreamAccounts(); err != nil { + return err + } + + // All file based streams have been recovered, so we now know how much of + // the disk we already occupy and can settle on a dynamic limit. + js.finalizeDynamicMaxStore() + + // If we are in clustered mode go ahead and start the meta controller. + if !standAlone || canExtend { + if err := s.enableJetStreamClustering(); err != nil { + return err + } + // Set our atomic bool to clustered. + s.jsClustered.Store(true) + } + + // Mark when we are up and running. + js.setStarted() + + return nil +} + +// finalizeDynamicMaxStore settles a dynamic max store limit once all file based +// streams have been recovered. diskAvailable only reports free space, so add back +// what we occupy ourselves to keep the limit stable across restarts. +func (js *jetStream) finalizeDynamicMaxStore() { + js.mu.Lock() + if !js.config.maxStorePending { + js.mu.Unlock() + return + } + // From here on the limit is real and can be enforced, see sufficientResources. + js.config.maxStorePending = false + recovered := atomic.LoadInt64(&js.storeUsed) + if recovered <= 0 { + js.mu.Unlock() + return + } + // diskAvailable is already scaled down to 75% of what is free, so scale the + // recovered bytes the same way before adding them back. + maxStore := addSaturate(js.config.MaxStore, recovered/4*3) + js.config.MaxStore = maxStore + atomic.StoreInt64(&js.storeMax, maxStore) + js.mu.Unlock() +} + +const jsNoExtend = "no_extend" +const jsWillExtend = "will_extend" + +// This will check if we have a solicited leafnode that shares the system account +// and extension is not manually disabled +func (s *Server) canExtendOtherDomain() bool { + opts := s.getOpts() + sysAcc := s.SystemAccount().GetName() + for _, r := range opts.LeafNode.Remotes { + if r.LocalAccount == sysAcc { + for _, denySub := range r.DenyImports { + if subjectIsSubsetMatch(denySub, raftAllSubj) { + return false + } + } + return true + } + } + return false +} + +func (s *Server) updateJetStreamInfoStatus(enabled bool) { + s.mu.Lock() + s.info.JetStream = enabled + s.mu.Unlock() +} + +// restartJetStream will try to re-enable JetStream during a reload if it had been disabled during runtime. +func (s *Server) restartJetStream() error { + opts := s.getOpts() + cfg := JetStreamConfig{ + StoreDir: opts.StoreDir, + SyncInterval: opts.SyncInterval, + SyncAlways: opts.SyncAlways, + MaxMemory: opts.JetStreamMaxMemory, + MaxStore: opts.JetStreamMaxStore, + Domain: opts.JetStreamDomain, + Strict: !opts.NoJetStreamStrict, + } + s.Noticef("Restarting JetStream") + err := s.EnableJetStream(&cfg) + if err != nil { + s.Warnf("Can't start JetStream: %v", err) + return s.ShutdownJetStream() + } + s.updateJetStreamInfoStatus(true) + return nil +} + +// checkJetStreamExports will check if we have the JS exports setup +// on the system account, and if not go ahead and set them up. +func (s *Server) checkJetStreamExports() { + if sacc := s.SystemAccount(); sacc != nil { + sacc.mu.RLock() + se := sacc.getServiceExport(jsAllAPI) + sacc.mu.RUnlock() + if se == nil { + s.setupJetStreamExports() + } + } +} + +func (s *Server) setupJetStreamExports() { + // Setup our internal system export. + sacc := s.SystemAccount() + if err := sacc.AddServiceExport(jsAllAPI, nil); err != nil { + s.Warnf("Error setting up jetstream service exports: %v", err) + } + // Map the domain prefixed API too, so an isolated JetStream is addressable by + // domain like an account is. Unprefixed always means this server's own JetStream. + if domain := s.getOpts().JetStreamDomain; domain != _EMPTY_ { + src, dest := fmt.Sprintf(jsDomainAPI, domain), jsAllAPI + if err := sacc.AddMapping(src, dest); err != nil { + s.Errorf("Error adding JetStream domain mapping to system account: %v", err) + } else { + s.Debugf("Adding JetStream Domain Mapping %q -> %s to system account", src, dest) + } + } +} + +func (s *Server) jetStreamOOSPending() (wasPending bool) { + if js := s.getJetStream(); js != nil { + js.mu.Lock() + wasPending = js.oos + js.oos = true + js.mu.Unlock() + } + return wasPending +} + +func (s *Server) setJetStreamDisabled() { + if js := s.getJetStream(); js != nil { + js.disabled.Store(true) + } +} + +func (s *Server) handleOutOfSpace(mset *stream) { + if s.JetStreamEnabled() && !s.jetStreamOOSPending() { + var stream string + if mset != nil { + stream = mset.name() + s.Errorf("JetStream out of %s resources, will be DISABLED", mset.Store().Type()) + } else { + s.Errorf("JetStream out of resources, will be DISABLED") + } + + go s.ShutdownJetStream() + + adv := &JSServerOutOfSpaceAdvisory{ + TypedEvent: TypedEvent{ + Type: JSServerOutOfStorageAdvisoryType, + ID: nuid.Next(), + Time: time.Now().UTC(), + }, + Server: s.Name(), + ServerID: s.ID(), + Stream: stream, + Cluster: s.cachedClusterName(), + Domain: s.getOpts().JetStreamDomain, + } + s.publishAdvisory(nil, JSAdvisoryServerOutOfStorage, adv) + } +} + +// DisableJetStream will turn off JetStream and signals in clustered mode +// to have the metacontroller remove us from the peer list. Persistent +// meta-raft state on disk is removed. For transient runtime errors where +// the server should rejoin its existing meta group on restart, use +// ShutdownJetStream instead. +func (s *Server) DisableJetStream() error { + return s.disableJetStream(true) +} + +// ShutdownJetStream is like DisableJetStream but preserves persistent +// meta-raft state on disk so the server can rejoin the existing meta +// group on restart. Use for transient runtime errors that the operator +// is expected to fix before restarting. +func (s *Server) ShutdownJetStream() error { + return s.disableJetStream(false) +} + +func (s *Server) disableJetStream(deleteState bool) error { + if !s.JetStreamEnabled() { + return nil + } + + s.setJetStreamDisabled() + + if s.JetStreamIsClustered() { + isLeader := s.JetStreamIsLeader() + js, cc := s.getJetStreamCluster() + if js == nil { + s.shutdownJetStream() + return nil + } + js.mu.RLock() + meta := cc.meta + js.mu.RUnlock() + + if meta != nil { + if isLeader { + s.Warnf("JetStream initiating meta leader transfer") + meta.StepDown() + select { + case <-s.quitCh: + return nil + case <-time.After(2 * time.Second): + } + if !s.JetStreamIsCurrent() { + s.Warnf("JetStream timeout waiting for meta leader transfer") + } + } + if deleteState { + meta.Delete() + } else { + meta.Stop() + meta.WaitForStop() + } + } + } + + // Update our info status. + s.updateJetStreamInfoStatus(false) + + // Normal shutdown. + s.shutdownJetStream() + + // Shut down the RAFT groups. + s.shutdownRaftNodes() + + return nil +} + +func (s *Server) enableJetStreamAccounts() error { + // Reuse the same task workers across all accounts, so that we don't explode + // with a large number of goroutines on multi-account systems. + tq := parallelTaskQueue(min(64, s.diskIOSemaphore().cap())) + defer close(tq) + + // If we have no configured accounts setup then setup imports on global account. + if s.globalAccountOnly() { + gacc := s.GlobalAccount() + gacc.mu.Lock() + if len(gacc.jsLimits) == 0 { + gacc.jsLimits = defaultJSAccountTiers + } + gacc.mu.Unlock() + if err := s.configJetStream(gacc, tq); err != nil { + return err + } + } else if err := s.configAllJetStreamAccounts(tq); err != nil { + return fmt.Errorf("Error enabling jetstream on configured accounts: %v", err) + } + return nil +} + +// enableAllJetStreamServiceImportsAndMappings turns on all service imports and mappings for jetstream for this account. +func (a *Account) enableAllJetStreamServiceImportsAndMappings() error { + a.mu.RLock() + s := a.srv + a.mu.RUnlock() + + if s == nil { + return fmt.Errorf("jetstream account not registered") + } + + var dstAccName string + if sacc := s.SystemAccount(); sacc != nil { + dstAccName = sacc.Name + } + + if !a.serviceImportExists(dstAccName, jsAllAPI) { + // Capture si so we can turn on implicit sharing with JetStream layer. + // Make sure to set "to" otherwise will incur performance slow down. + si, err := a.addServiceImport(s.SystemAccount(), jsAllAPI, jsAllAPI, nil) + if err != nil { + return fmt.Errorf("Error setting up jetstream service imports for account: %v", err) + } + a.mu.Lock() + si.share = true + a.mu.Unlock() + } + + // Check if we have a Domain specified. + // If so add in a subject mapping that will allow local connected clients to reach us here as well. + if opts := s.getOpts(); opts.JetStreamDomain != _EMPTY_ { + mappings := generateJSMappingTable(opts.JetStreamDomain) + a.mu.RLock() + for _, m := range a.mappings { + delete(mappings, m.src) + } + a.mu.RUnlock() + for src, dest := range mappings { + if err := a.AddMapping(src, dest); err != nil { + s.Errorf("Error adding JetStream domain mapping: %v", err) + } + } + } + + return nil +} + +// enableJetStreamInfoServiceImportOnly will enable the single service import responder. +// Should we do them all regardless? +func (a *Account) enableJetStreamInfoServiceImportOnly() error { + // Check if this import would be overshadowed. This can happen when accounts + // are importing from another account for JS access. + if a.serviceImportShadowed(JSApiAccountInfo) { + return nil + } + + return a.enableAllJetStreamServiceImportsAndMappings() +} + +func (s *Server) configJetStream(acc *Account, tq chan<- func()) error { + if acc == nil { + return nil + } + acc.mu.RLock() + jsLimits := acc.jsLimits + acc.mu.RUnlock() + if jsLimits != nil { + // Check if already enabled. This can be during a reload. + if acc.JetStreamEnabled() { + if err := acc.enableAllJetStreamServiceImportsAndMappings(); err != nil { + return err + } + if err := acc.UpdateJetStreamLimits(jsLimits); err != nil { + return err + } + } else { + if err := acc.EnableJetStream(jsLimits, tq); err != nil { + return err + } + if s.gateway.enabled { + s.switchAccountToInterestMode(acc.GetName()) + } + } + } else if acc != s.SystemAccount() { + if acc.JetStreamEnabled() { + acc.DisableJetStream() + } + // We will setup basic service imports to respond to + // requests if JS is enabled for this account. + if err := acc.enableJetStreamInfoServiceImportOnly(); err != nil { + return err + } + } + return nil +} + +// configAllJetStreamAccounts walk all configured accounts and turn on jetstream if requested. +func (s *Server) configAllJetStreamAccounts(tq chan<- func()) error { + // Check to see if system account has been enabled. We could arrive here via reload and + // a non-default system account. + s.checkJetStreamExports() + + // Bail if server not enabled. If it was enabled and a reload turns it off + // that will be handled elsewhere. + js := s.getJetStream() + if js == nil { + return nil + } + + // Snapshot into our own list. Might not be needed. + s.mu.RLock() + if s.sys != nil { + // clustered stream removal will perform this cleanup as well + // this is mainly for initial cleanup + saccName := s.sys.account.Name + accStoreDirs, _ := os.ReadDir(js.config.StoreDir) + for _, acc := range accStoreDirs { + if accName := acc.Name(); accName != saccName { + // no op if not empty + accDir := filepath.Join(js.config.StoreDir, accName) + os.Remove(filepath.Join(accDir, streamsDir)) + os.Remove(accDir) + } + } + } + + var jsAccounts []*Account + s.accounts.Range(func(k, v any) bool { + jsAccounts = append(jsAccounts, v.(*Account)) + return true + }) + accounts := &s.accounts + s.mu.RUnlock() + + // Process any jetstream enabled accounts here. These will be accounts we are + // already aware of at startup etc. + for _, acc := range jsAccounts { + if err := s.configJetStream(acc, tq); err != nil { + return err + } + } + + // Now walk all the storage we have and resolve any accounts that we did not process already. + // This is important in resolver/operator models. + fis, _ := os.ReadDir(js.config.StoreDir) + for _, fi := range fis { + if accName := fi.Name(); accName != _EMPTY_ { + // Only load up ones not already loaded since they are processed above. + if _, ok := accounts.Load(accName); !ok { + if acc, err := s.lookupAccount(accName); err != nil && acc != nil { + if err := s.configJetStream(acc, tq); err != nil { + return err + } + } + } + } + } + + return nil +} + +// Mark our started time. +func (js *jetStream) setStarted() { + js.mu.Lock() + defer js.mu.Unlock() + js.started = time.Now() +} + +func (js *jetStream) isEnabled() bool { + if js == nil { + return false + } + return !js.disabled.Load() +} + +// Mark that we will be in standlone mode. +func (js *jetStream) setJetStreamStandAlone(isStandAlone bool) { + if js == nil { + return + } + js.mu.Lock() + defer js.mu.Unlock() + if js.standAlone = isStandAlone; js.standAlone { + // Update our server atomic. + js.srv.isMetaLeader.Store(true) + js.accountPurge, _ = js.srv.systemSubscribe(JSApiAccountPurge, _EMPTY_, false, nil, js.srv.jsLeaderAccountPurgeRequest) + } else if js.accountPurge != nil { + js.srv.sysUnsubscribe(js.accountPurge) + } +} + +// JetStreamEnabled reports if jetstream is enabled for this server. +func (s *Server) JetStreamEnabled() bool { + return s.getJetStream().isEnabled() +} + +// JetStreamEnabledForDomain will report if any servers have JetStream enabled within this domain. +func (s *Server) JetStreamEnabledForDomain() bool { + if s.JetStreamEnabled() { + return true + } + + var jsFound bool + // If we are here we do not have JetStream enabled for ourselves, but we need to check all connected servers. + // TODO(dlc) - Could optimize and memoize this. + s.nodeToInfo.Range(func(k, v any) bool { + // This should not be dependent on online status, so only check js. + if v.(nodeInfo).js { + jsFound = true + return false + } + return true + }) + + return jsFound +} + +// Will signal that all pull requests for consumers on this server are now invalid. +func (s *Server) signalPullConsumers() { + js := s.getJetStream() + if js == nil { + return + } + + js.mu.RLock() + defer js.mu.RUnlock() + + // In case we have stale pending requests. + const hdr = "NATS/1.0 409 Server Shutdown\r\n" + JSPullRequestPendingMsgs + ": %d\r\n" + JSPullRequestPendingBytes + ": %d\r\n\r\n" + var didSend bool + + for _, jsa := range js.accounts { + jsa.mu.RLock() + for _, stream := range jsa.streams { + stream.mu.RLock() + for _, o := range stream.consumers { + o.mu.RLock() + // Only signal on R1. + if o.cfg.Replicas <= 1 { + for reply, wr := range o.pendingRequests() { + shdr := fmt.Sprintf(hdr, wr.n, wr.b) + o.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, []byte(shdr), nil, nil, 0)) + didSend = true + } + } + o.mu.RUnlock() + } + stream.mu.RUnlock() + } + jsa.mu.RUnlock() + } + // Give time for migration information to make it out of our server. + if didSend { + time.Sleep(50 * time.Millisecond) + } +} + +// Helper for determining if we are shutting down. +func (js *jetStream) isShuttingDown() bool { + js.mu.RLock() + defer js.mu.RUnlock() + return js.shuttingDown +} + +// Shutdown jetstream for this server. +func (s *Server) shutdownJetStream() { + js := s.getJetStream() + if js == nil { + return + } + + s.Noticef("Initiating JetStream Shutdown...") + defer s.Noticef("JetStream Shutdown") + + // If we have folks blocked on sync requests, unblock. + // Send 1 is enough, but use select in case they were all present. + select { + case s.syncOutSem <- struct{}{}: + default: + } + + var _a [512]*Account + accounts := _a[:0] + + js.mu.Lock() + // Collect accounts. + for _, jsa := range js.accounts { + if a := jsa.acc(); a != nil { + accounts = append(accounts, a) + } + } + accPurgeSub := js.accountPurge + js.accountPurge = nil + // Signal we are shutting down. + js.shuttingDown = true + js.mu.Unlock() + + if accPurgeSub != nil { + s.sysUnsubscribe(accPurgeSub) + } + + for _, a := range accounts { + a.removeJetStream() + } + + s.js.Store(nil) + + js.mu.Lock() + js.accounts = nil + + var qch chan struct{} + var stopped chan struct{} + if cc := js.cluster; cc != nil { + if cc.qch != nil { + qch, stopped = cc.qch, cc.stopped + cc.qch, cc.stopped = nil, nil + } + js.stopUpdatesSub() + if cc.metaRescue != nil { + s.sysUnsubscribe(cc.metaRescue) + cc.metaRescue = nil + } + if cc.c != nil { + cc.c.closeConnection(ClientClosed) + cc.c = nil + } + cc.meta = nil + // Set our atomic bool to false. + s.jsClustered.Store(false) + } + js.mu.Unlock() + + // If we were clustered signal the monitor cluster go routine. + // We will wait for a bit for it to close. + // Do this without the lock. + if qch != nil { + close(qch) // Must be close() to signal *all* listeners + select { + case <-stopped: + case <-time.After(10 * time.Second): + s.Warnf("Did not receive signal for successful shutdown of cluster routine") + } + } +} + +// JetStreamConfig will return the current config. Useful if the system +// created a dynamic configuration. A copy is returned. +func (s *Server) JetStreamConfig() *JetStreamConfig { + var c *JetStreamConfig + if js := s.getJetStream(); js != nil { + js.mu.RLock() + copy := js.config + c = &(copy) + js.mu.RUnlock() + } + return c +} + +// StoreDir returns the current JetStream directory. +func (s *Server) StoreDir() string { + js := s.getJetStream() + if js == nil { + return _EMPTY_ + } + return js.config.StoreDir +} + +// JetStreamNumAccounts returns the number of enabled accounts this server is tracking. +func (s *Server) JetStreamNumAccounts() int { + js := s.getJetStream() + if js == nil { + return 0 + } + js.mu.Lock() + defer js.mu.Unlock() + return len(js.accounts) +} + +// JetStreamReservedResources returns the reserved resources if JetStream is enabled. +func (s *Server) JetStreamReservedResources() (int64, int64, error) { + js := s.getJetStream() + if js == nil { + return -1, -1, NewJSNotEnabledForAccountError() + } + js.mu.RLock() + defer js.mu.RUnlock() + return js.memReserved, js.storeReserved, nil +} + +func (s *Server) getJetStream() *jetStream { + return s.js.Load() +} + +func (a *Account) assignJetStreamLimits(limits map[string]JetStreamAccountLimits) { + a.mu.Lock() + a.jsLimits = limits + a.mu.Unlock() +} + +// EnableJetStream will enable JetStream on this account with the defined limits. +// This is a helper for JetStreamEnableAccount. +func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits, tq chan<- func()) error { + a.mu.RLock() + s := a.srv + a.mu.RUnlock() + + if s == nil { + return fmt.Errorf("jetstream account not registered") + } + + if s.SystemAccount() == a { + return fmt.Errorf("jetstream can not be enabled on the system account") + } + + s.mu.RLock() + if s.sys == nil { + s.mu.RUnlock() + return ErrServerNotRunning + } + sendq := s.sys.sendq + s.mu.RUnlock() + + // No limits means we dynamically set up limits. + // We also place limits here so we know that the account is configured for JetStream. + if len(limits) == 0 { + limits = defaultJSAccountTiers + } + + a.assignJetStreamLimits(limits) + + js := s.getJetStream() + if js == nil { + return NewJSNotEnabledError() + } + + js.mu.Lock() + // Accounts get reset to nil on shutdown, since we re-acquire the locks here, we need to check again. + if js.accounts == nil { + js.mu.Unlock() + return NewJSNotEnabledError() + } + + if jsa, ok := js.accounts[a.Name]; ok { + a.mu.Lock() + a.js = jsa + a.mu.Unlock() + js.mu.Unlock() + return a.enableAllJetStreamServiceImportsAndMappings() + } + + // Check the limits against existing reservations. + if err := js.sufficientResources(limits); err != nil { + js.mu.Unlock() + return err + } + + sysNode := s.Node() + + jsa := &jsAccount{js: js, account: a, limits: limits, streams: make(map[string]*stream), sendq: sendq, usage: make(map[string]*jsaStorage)} + jsa.storeDir = filepath.Join(js.config.StoreDir, a.Name) + + // A single server does not need to do the account updates at this point. + if js.cluster != nil || !s.standAloneMode() { + jsa.usageMu.Lock() + jsa.utimer = time.AfterFunc(usageTick, jsa.sendClusterUsageUpdateTimer) + // Cluster mode updates to resource usage. System internal prevents echos. + jsa.updatesPub = fmt.Sprintf(jsaUpdatesPubT, a.Name, sysNode) + jsa.updatesSub, _ = s.sysSubscribe(fmt.Sprintf(jsaUpdatesSubT, a.Name), jsa.remoteUpdateUsage) + jsa.usageMu.Unlock() + } + + js.accounts[a.Name] = jsa + // Stamp inside account as well. Needs to be done under js's lock. + a.mu.Lock() + a.js = jsa + a.mu.Unlock() + js.mu.Unlock() + + // Create the proper imports here. + if err := a.enableAllJetStreamServiceImportsAndMappings(); err != nil { + return err + } + + s.Debugf("Enabled JetStream for account %q", a.Name) + if l, ok := limits[_EMPTY_]; ok { + s.Debugf(" Max Memory: %s", friendlyBytes(l.MaxMemory)) + s.Debugf(" Max Storage: %s", friendlyBytes(l.MaxStore)) + } else { + for t, l := range limits { + s.Debugf(" Tier: %s", t) + s.Debugf(" Max Memory: %s", friendlyBytes(l.MaxMemory)) + s.Debugf(" Max Storage: %s", friendlyBytes(l.MaxStore)) + } + } + + // Clean up any old snapshots that were orphaned while staging. + os.RemoveAll(filepath.Join(js.config.StoreDir, snapStagingDir)) + + sdir := filepath.Join(jsa.storeDir, streamsDir) + if _, err := os.Stat(sdir); os.IsNotExist(err) { + if err := os.MkdirAll(sdir, defaultDirPerms); err != nil { + return fmt.Errorf("could not create storage streams directory - %v", err) + } + // Just need to make sure we can write to the directory. + // Remove the directory will create later if needed. + os.RemoveAll(sdir) + // when empty remove parent directory, which may have been created as well + os.Remove(jsa.storeDir) + } else { + // Restore any state here. + s.Debugf("Recovering JetStream state for account %q", a.Name) + } + + // Remember if we should be encrypted and what cipher we think we should use. + encrypted := s.getOpts().JetStreamKey != _EMPTY_ + sc := s.getOpts().JetStreamCipher + + doConsumers := func(mset *stream, odir string) { + ofis, _ := os.ReadDir(odir) + if len(ofis) > 0 { + s.Noticef(" Recovering %d consumers for stream - '%s > %s'", len(ofis), mset.accName(), mset.name()) + } + for _, ofi := range ofis { + metafile := filepath.Join(odir, ofi.Name(), JetStreamMetaFile) + metasum := filepath.Join(odir, ofi.Name(), JetStreamMetaFileSum) + if _, err := os.Stat(metafile); os.IsNotExist(err) { + s.Warnf(" Missing consumer metafile %q", metafile) + continue + } + buf, err := os.ReadFile(metafile) + if err != nil { + s.Warnf(" Error reading consumer metafile %q: %v", metafile, err) + continue + } + if _, err := os.Stat(metasum); os.IsNotExist(err) { + s.Warnf(" Missing consumer checksum for %q", metasum) + continue + } + + // Check if we are encrypted. + if key, err := os.ReadFile(filepath.Join(odir, ofi.Name(), JetStreamMetaFileKey)); err == nil { + s.Debugf(" Consumer metafile is encrypted, reading encrypted keyfile") + // Decode the buffer before proceeding. + ctxName := mset.name() + tsep + ofi.Name() + nbuf, _, err := s.decryptMeta(sc, key, buf, a.Name, ctxName) + if err != nil { + s.Warnf(" Error decrypting our consumer metafile: %v", err) + continue + } + buf = nbuf + } + + var cfg FileConsumerInfo + decoder := json.NewDecoder(bytes.NewReader(buf)) + decoder.DisallowUnknownFields() + strictErr := decoder.Decode(&cfg) + if strictErr != nil { + cfg = FileConsumerInfo{} + if err := json.Unmarshal(buf, &cfg); err != nil { + s.Warnf(" Error unmarshalling consumer metafile %q: %v", metafile, err) + continue + } + } + if supported := supportsRequiredApiLevel(cfg.Metadata); !supported || strictErr != nil { + var offlineReason string + if !supported { + apiLevel := getRequiredApiLevel(cfg.Metadata) + if strictErr != nil { + offlineReason = fmt.Sprintf("unsupported - config error: %s", strings.TrimPrefix(strictErr.Error(), "json: ")) + } else { + offlineReason = fmt.Sprintf("unsupported - required API level: %s, current API level: %d", apiLevel, JSApiLevel) + } + s.Warnf(" Detected unsupported consumer '%s > %s > %s': %s", a.Name, mset.name(), cfg.Name, offlineReason) + } else { + offlineReason = fmt.Sprintf("decoding error: %v", strictErr) + s.Warnf(" Error unmarshalling consumer metafile %q: %v", metafile, strictErr) + } + singleServerMode := !s.JetStreamIsClustered() && s.standAloneMode() + if singleServerMode { + if !mset.closed.Load() { + s.Warnf(" Stopping unsupported stream '%s > %s'", a.Name, mset.name()) + mset.mu.Lock() + mset.offlineReason = fmt.Sprintf("stopped - unsupported consumer %q", cfg.Name) + mset.mu.Unlock() + mset.stop(false, false) + } + + // Fake a consumer, so we can respond to API requests as single-server. + o := &consumer{ + mset: mset, + js: s.getJetStream(), + acc: a, + srv: s, + cfg: cfg.ConsumerConfig, + direct: cfg.Direct, + sourcing: cfg.Sourcing, + active: false, + stream: mset.name(), + name: cfg.Name, + dseq: 1, + sseq: 1, + created: time.Now().UTC(), + closed: true, + offlineReason: offlineReason, + } + if !cfg.Created.IsZero() { + o.created = cfg.Created + } + + mset.mu.Lock() + mset.setConsumer(o) + mset.mu.Unlock() + } + continue + } + + isEphemeral := !isDurableConsumer(&cfg.ConsumerConfig) + if isEphemeral { + // This is an ephemeral consumer and this could fail on restart until + // the consumer can reconnect. We will create it as a durable and switch it. + cfg.ConsumerConfig.Durable = ofi.Name() + } + obs, err := mset.addConsumerWithAssignment(&cfg.ConsumerConfig, _EMPTY_, nil, true, ActionCreateOrUpdate, false) + if err != nil { + s.Warnf(" Error adding consumer '%s > %s > %s': %v", a.Name, mset.name(), cfg.Name, err) + continue + } + if isEphemeral { + obs.switchToEphemeral() + } + if !cfg.Created.IsZero() { + obs.setCreatedTime(cfg.Created) + } + } + } + + // Now recover the streams. + fis, _ := os.ReadDir(sdir) + doStream := func(fi os.DirEntry) error { + plaintext := true + mdir := filepath.Join(sdir, fi.Name()) + // Check for partially deleted streams. They are marked with "." prefix. + if strings.HasPrefix(fi.Name(), tsep) { + go os.RemoveAll(mdir) + return nil + } + key := sha256.Sum256([]byte(fi.Name())) + hh, err := highwayhash.NewDigest64(key[:]) + if err != nil { + return err + } + metafile := filepath.Join(mdir, JetStreamMetaFile) + metasum := filepath.Join(mdir, JetStreamMetaFileSum) + if _, err := os.Stat(metafile); os.IsNotExist(err) { + s.Warnf(" Missing stream metafile for %q", metafile) + return nil + } + buf, err := os.ReadFile(metafile) + if err != nil { + s.Warnf(" Error reading metafile %q: %v", metafile, err) + return nil + } + if _, err := os.Stat(metasum); os.IsNotExist(err) { + s.Warnf(" Missing stream checksum file %q", metasum) + return nil + } + sum, err := os.ReadFile(metasum) + if err != nil { + s.Warnf(" Error reading Stream metafile checksum %q: %v", metasum, err) + return nil + } + hh.Write(buf) + var hb [highwayhash.Size64]byte + checksum := hex.EncodeToString(hh.Sum(hb[:0])) + if checksum != string(sum) { + s.Warnf(" Stream metafile %q: checksums do not match %q vs %q", metafile, sum, checksum) + return nil + } + + // Track if we are converting ciphers. + var convertingCiphers bool + + // Check if we are encrypted. + keyFile := filepath.Join(mdir, JetStreamMetaFileKey) + keyBuf, err := os.ReadFile(keyFile) + if err == nil { + s.Debugf(" Stream metafile is encrypted, reading encrypted keyfile") + if len(keyBuf) < minMetaKeySize { + s.Warnf(" Bad stream encryption key length of %d", len(keyBuf)) + return nil + } + // Decode the buffer before proceeding. + var nbuf []byte + nbuf, convertingCiphers, err = s.decryptMeta(sc, keyBuf, buf, a.Name, fi.Name()) + if err != nil { + s.Warnf(" Error decrypting our stream metafile: %v", err) + return nil + } + buf = nbuf + plaintext = false + } + + var cfg FileStreamInfo + decoder := json.NewDecoder(bytes.NewReader(buf)) + decoder.DisallowUnknownFields() + strictErr := decoder.Decode(&cfg) + if strictErr != nil { + cfg = FileStreamInfo{} + if err := json.Unmarshal(buf, &cfg); err != nil { + s.Warnf(" Error unmarshalling stream metafile %q: %v", metafile, err) + return nil + } + } + if supported := supportsRequiredApiLevel(cfg.Metadata); !supported || strictErr != nil { + var offlineReason string + if !supported { + apiLevel := getRequiredApiLevel(cfg.Metadata) + if strictErr != nil { + offlineReason = fmt.Sprintf("unsupported - config error: %s", strings.TrimPrefix(strictErr.Error(), "json: ")) + } else { + offlineReason = fmt.Sprintf("unsupported - required API level: %s, current API level: %d", apiLevel, JSApiLevel) + } + s.Warnf(" Detected unsupported stream '%s > %s': %s", a.Name, cfg.StreamConfig.Name, offlineReason) + } else { + offlineReason = fmt.Sprintf("decoding error: %v", strictErr) + s.Warnf(" Error unmarshalling stream metafile %q: %v", metafile, strictErr) + } + singleServerMode := !s.JetStreamIsClustered() && s.standAloneMode() + if singleServerMode { + // Fake a stream, so we can respond to API requests as single-server. + mset := &stream{ + acc: a, + jsa: jsa, + cfg: cfg.StreamConfig, + js: js, + srv: s, + stype: cfg.Storage, + consumers: make(map[string]*consumer), + active: false, + created: time.Now().UTC(), + offlineReason: offlineReason, + } + if !cfg.Created.IsZero() { + mset.created = cfg.Created + } + mset.closed.Store(true) + + jsa.mu.Lock() + jsa.streams[cfg.Name] = mset + jsa.mu.Unlock() + + // Now do the consumers. + odir := filepath.Join(sdir, fi.Name(), consumerDir) + doConsumers(mset, odir) + } + return nil + } + + // We had a bug that set a default de dupe window on mirror, despite that being not a valid config + fixCfgMirrorWithDedupWindow(&cfg.StreamConfig) + + // We had a bug that could allow subjects in that had prefix or suffix spaces. We check for that here + // and will patch them on the fly for now. We will warn about them. + var hadSubjErr bool + for i, subj := range cfg.StreamConfig.Subjects { + if !IsValidSubject(subj) { + s.Warnf(" Detected bad subject %q while adding stream %q, will attempt to repair", subj, cfg.Name) + if nsubj := strings.TrimSpace(subj); IsValidSubject(nsubj) { + s.Warnf(" Bad subject %q repaired to %q", subj, nsubj) + cfg.StreamConfig.Subjects[i] = nsubj + } else { + s.Warnf(" Error recreating stream %q: %v", cfg.Name, "invalid subject") + hadSubjErr = true + break + } + } + } + if hadSubjErr { + return nil + } + + // The other possible bug is assigning subjects to mirrors, so check for that and patch as well. + if cfg.StreamConfig.Mirror != nil && len(cfg.StreamConfig.Subjects) > 0 { + s.Warnf(" Detected subjects on a mirrored stream %q, will remove", cfg.Name) + cfg.StreamConfig.Subjects = nil + } + + s.Noticef(" Starting restore for stream '%s > %s'", a.Name, cfg.StreamConfig.Name) + rt := time.Now() + + // Log if we are converting from plaintext to encrypted. + if encrypted { + if plaintext { + s.Noticef(" Encrypting stream '%s > %s'", a.Name, cfg.StreamConfig.Name) + } else if convertingCiphers { + s.Noticef(" Converting to %s for stream '%s > %s'", sc, a.Name, cfg.StreamConfig.Name) + // Remove the key file to have system regenerate with the new cipher. + os.Remove(keyFile) + } + } + + // Add in the stream. + mset, err := a.recoverStream(&cfg.StreamConfig) + if err != nil { + s.Warnf(" Error recreating stream %q: %v", cfg.Name, err) + // If we removed a keyfile from above make sure to put it back. + if convertingCiphers { + err := os.WriteFile(keyFile, keyBuf, defaultFilePerms) + if err != nil { + s.Warnf(" Error replacing meta keyfile for stream %q: %v", cfg.Name, err) + } + } + return nil + } + if !cfg.Created.IsZero() { + mset.setCreatedTime(cfg.Created) + } + + // Might need to recover from a partial batch write, but only if a single replica stream. + if cfg.AllowAtomicPublish && cfg.Replicas == 1 { + var ( + ok bool + smv StoreMsg + batchId string + batchSeq uint64 + commit bool + commitEob bool + batchStoreDir string + store StreamStore + state StreamState + ) + // Check if the last message was part of a batch. + sm, err := mset.store.LoadLastMsg(fwcs, &smv) + if err != nil || sm == nil { + goto SKIP + } + batchId = getBatchId(sm.hdr) + batchSeq, ok = getBatchSequence(sm.hdr) + commit = len(sliceHeader(JSBatchCommit, sm.hdr)) != 0 + if batchId == _EMPTY_ || !ok || commit { + goto SKIP + } + // We've observed a partial batch write. Write the remainder of the batch. + batchSeq++ + _, batchStoreDir = getBatchStoreDir(jsa.storeDir, cfg.Name, batchId) + if _, err = os.Stat(batchStoreDir); err != nil { + s.Errorf(" Failed restoring partial batch write for stream '%s > %s' at sequence %d: %v", + mset.accName(), mset.name(), batchSeq, err) + goto SKIP + } + store, err = newBatchStore(mset, batchId, cfg.Replicas, cfg.Storage, jsa.storeDir, cfg.Name) + if err != nil { + s.Errorf(" Failed restoring partial batch write for stream '%s > %s' at sequence %d: %v", + mset.accName(), mset.name(), batchSeq, err) + goto SKIP + } + store.FastState(&state) + sm, err = store.LoadMsg(state.LastSeq, &smv) + if err != nil || sm == nil { + s.Errorf(" Failed restoring partial batch write for stream '%s > %s' at sequence %d: last msg not found %d", + mset.accName(), mset.name(), batchSeq, state.LastSeq) + goto SKIP + } + commitEob = bytes.Equal(sliceHeader(JSBatchCommit, sm.hdr), []byte("eob")) + // If the commit ends with an "End Of Batch" message, we don't store this. + if commitEob { + state.LastSeq-- + } + s.Noticef(" Restoring partial batch write for stream '%s > %s' (seq %d to %d)", + mset.accName(), mset.name(), batchSeq, state.LastSeq) + // Loop through items that weren't persisted yet. + for seq := batchSeq; seq <= state.LastSeq; seq++ { + sm, err = store.LoadMsg(seq, &smv) + if err != nil || sm == nil { + s.Errorf(" Failed restoring partial batch write for stream '%s > %s' at sequence %d: %v", + mset.accName(), mset.name(), seq, err) + break + } + hdr := sm.hdr + // If committed by EOB, the last message must get the normal commit header. + if commitEob && seq == state.LastSeq { + hdr = genHeader(hdr, JSBatchCommit, "1") + } + mset.processJetStreamMsg(sm.subj, _EMPTY_, hdr, sm.msg, 0, 0, nil, false, true) + } + store.Delete(true) + SKIP: + os.RemoveAll(filepath.Join(sdir, fi.Name(), batchesDir)) + } + + state := mset.state() + s.Noticef(" Restored %s messages for stream '%s > %s' in %v", + comma(int64(state.Msgs)), mset.accName(), mset.name(), time.Since(rt).Round(time.Millisecond)) + + // Now do the consumers. + odir := filepath.Join(sdir, fi.Name(), consumerDir) + doConsumers(mset, odir) + + // Collect to check for dangling messages. + // TODO(dlc) - Can be removed eventually. + if cfg.StreamConfig.Retention == InterestPolicy { + mset.checkForOrphanMsgs() + mset.checkConsumerReplication() + } + + return nil + } + + if tq != nil { + // If a parallelTaskQueue was provided then use that for concurrency. + var wg sync.WaitGroup + wg.Add(len(fis)) + for _, fi := range fis { + tq <- func() { + doStream(fi) + wg.Done() + } + } + wg.Wait() + } else { + // No parallelTaskQueue provided, do inline as before. + for _, fi := range fis { + doStream(fi) + } + } + + // Make sure to cleanup any old remaining snapshots. + os.RemoveAll(filepath.Join(jsa.storeDir, snapsDir)) + + s.Debugf("JetStream state for account %q recovered", a.Name) + + return nil +} + +// Return whether we require MaxBytes to be set and if > 0 an upper limit for stream size exists +// Both limits are independent of each other. +func (a *Account) maxBytesLimits(cfg *StreamConfig) (bool, int64) { + a.mu.RLock() + jsa := a.js + a.mu.RUnlock() + if jsa == nil { + return false, 0 + } + jsa.usageMu.RLock() + var replicas int + if cfg != nil { + replicas = cfg.Replicas + } + selectedLimits, _, ok := jsa.selectLimits(replicas) + jsa.usageMu.RUnlock() + if !ok { + return false, 0 + } + maxStreamBytes := int64(0) + if cfg.Storage == MemoryStorage { + maxStreamBytes = selectedLimits.MemoryMaxStreamBytes + } else { + maxStreamBytes = selectedLimits.StoreMaxStreamBytes + } + return selectedLimits.MaxBytesRequired, maxStreamBytes +} + +// NumStreams will return how many streams we have. +func (a *Account) numStreams() int { + a.mu.RLock() + jsa := a.js + a.mu.RUnlock() + if jsa == nil { + return 0 + } + jsa.mu.Lock() + n := len(jsa.streams) + jsa.mu.Unlock() + return n +} + +// Streams will return all known streams. +func (a *Account) streams() []*stream { + return a.filteredStreams(_EMPTY_) +} + +func (a *Account) filteredStreams(filter string) []*stream { + a.mu.RLock() + jsa := a.js + a.mu.RUnlock() + + if jsa == nil { + return nil + } + + jsa.mu.RLock() + defer jsa.mu.RUnlock() + + var msets []*stream + for _, mset := range jsa.streams { + if filter != _EMPTY_ { + mset.cfgMu.RLock() + for _, subj := range mset.cfg.Subjects { + if SubjectsCollide(filter, subj) { + msets = append(msets, mset) + break + } + } + mset.cfgMu.RUnlock() + } else { + msets = append(msets, mset) + } + } + + return msets +} + +// lookupStream will lookup a stream by name. +func (a *Account) lookupStream(name string) (*stream, error) { + a.mu.RLock() + jsa := a.js + a.mu.RUnlock() + + if jsa == nil { + return nil, NewJSNotEnabledForAccountError() + } + jsa.mu.RLock() + defer jsa.mu.RUnlock() + + mset, ok := jsa.streams[name] + if !ok { + return nil, NewJSStreamNotFoundError() + } + return mset, nil +} + +// UpdateJetStreamLimits will update the account limits for a JetStream enabled account. +func (a *Account) UpdateJetStreamLimits(limits map[string]JetStreamAccountLimits) error { + a.mu.RLock() + s, jsa := a.srv, a.js + a.mu.RUnlock() + + if s == nil { + return fmt.Errorf("jetstream account not registered") + } + js := s.getJetStream() + if js == nil { + return NewJSNotEnabledError() + } + if jsa == nil { + return NewJSNotEnabledForAccountError() + } + + if len(limits) == 0 { + limits = defaultJSAccountTiers + } + + // Calculate the delta between what we have and what we want. + jsa.usageMu.RLock() + dl := diffCheckedLimits(jsa.limits, limits) + jsa.usageMu.RUnlock() + + js.mu.Lock() + // Check the limits against existing reservations. + if err := js.sufficientResources(dl); err != nil { + js.mu.Unlock() + return err + } + js.mu.Unlock() + + // Update + jsa.usageMu.Lock() + jsa.limits = limits + jsa.usageMu.Unlock() + + return nil +} + +func diffCheckedLimits(a, b map[string]JetStreamAccountLimits) map[string]JetStreamAccountLimits { + diff := map[string]JetStreamAccountLimits{} + for t, la := range a { + // in a, not in b will return 0 + lb := b[t] + diff[t] = JetStreamAccountLimits{ + MaxMemory: lb.MaxMemory - la.MaxMemory, + MaxStore: lb.MaxStore - la.MaxStore, + } + } + for t, lb := range b { + if la, ok := a[t]; !ok { + // only in b not in a. (in a and b already covered) + diff[t] = JetStreamAccountLimits{ + MaxMemory: lb.MaxMemory - la.MaxMemory, + MaxStore: lb.MaxStore - la.MaxStore, + } + } + } + return diff +} + +// Return reserved bytes for memory and file store streams for this account on this server. +// Lock should be held. +func (jsa *jsAccount) reservedStorage(tier string) (mem, store uint64) { + for _, mset := range jsa.streams { + mset.cfgMu.RLock() + storage, replicas, maxBytes := mset.cfg.Storage, mset.cfg.Replicas, mset.cfg.MaxBytes + mset.cfgMu.RUnlock() + if (tier == _EMPTY_ || tier == tierName(replicas)) && maxBytes > 0 { + switch storage { + case FileStorage: + store += uint64(maxBytes) + case MemoryStorage: + mem += uint64(maxBytes) + } + } + } + return mem, store +} + +// Return reserved bytes for memory and file store streams for this account in clustered mode. +// js lock should be held. +func reservedStorage(sas map[string]*streamAssignment, tier string) (mem, store uint64) { + for _, sa := range sas { + cfg := sa.Config + if (tier == _EMPTY_ || tier == tierName(cfg.Replicas)) && cfg.MaxBytes > 0 { + switch cfg.Storage { + case FileStorage: + store += uint64(cfg.MaxBytes) + case MemoryStorage: + mem += uint64(cfg.MaxBytes) + } + } + } + return mem, store +} + +// JetStreamUsage reports on JetStream usage and limits for an account. +func (a *Account) JetStreamUsage() JetStreamAccountStats { + a.mu.RLock() + jsa, aname := a.js, a.Name + accJsLimits := a.jsLimits + a.mu.RUnlock() + + var stats JetStreamAccountStats + if jsa != nil { + js := jsa.js + js.mu.RLock() + cc := js.cluster + singleServer := cc == nil + jsa.mu.RLock() + jsa.usageMu.RLock() + stats.Memory, stats.Store = jsa.storageTotals() + stats.Domain = js.config.Domain + stats.API = JetStreamAPIStats{ + Level: JSApiLevel, + Total: jsa.apiTotal, + Errors: jsa.apiErrors, + } + if singleServer { + stats.ReservedMemory, stats.ReservedStore = jsa.reservedStorage(_EMPTY_) + } else { + stats.ReservedMemory, stats.ReservedStore = reservedStorage(cc.streams[aname], _EMPTY_) + } + l, defaultTier := jsa.limits[_EMPTY_] + if defaultTier { + stats.Limits = l + } else { + skipped := 0 + stats.Tiers = make(map[string]JetStreamTier) + for t, total := range jsa.usage { + if _, ok := jsa.limits[t]; !ok && (*total) == (jsaStorage{}) { + // skip tiers not present that don't contain a count + // In case this shows an empty stream, that tier will be added when iterating over streams + skipped++ + } else { + tier := JetStreamTier{ + Memory: uint64(total.total.mem), + Store: uint64(total.total.store), + Limits: jsa.limits[t], + } + if singleServer { + tier.ReservedMemory, tier.ReservedStore = jsa.reservedStorage(t) + } else { + tier.ReservedMemory, tier.ReservedStore = reservedStorage(cc.streams[aname], t) + } + stats.Tiers[t] = tier + } + } + if len(accJsLimits) != len(jsa.usage)-skipped { + // insert unused limits + for t, lim := range accJsLimits { + if _, ok := stats.Tiers[t]; !ok { + tier := JetStreamTier{Limits: lim} + if singleServer { + tier.ReservedMemory, tier.ReservedStore = jsa.reservedStorage(t) + } else { + tier.ReservedMemory, tier.ReservedStore = reservedStorage(cc.streams[aname], t) + } + stats.Tiers[t] = tier + } + } + } + } + jsa.usageMu.RUnlock() + + // Clustered + if cc := js.cluster; cc != nil { + sas := cc.streams[aname] + if defaultTier { + stats.Streams = len(sas) + stats.ReservedMemory, stats.ReservedStore = reservedStorage(sas, _EMPTY_) + } + for _, sa := range sas { + if defaultTier { + stats.Consumers += len(sa.consumers) + } else { + stats.Streams++ + streamTier := tierName(sa.Config.Replicas) + su, ok := stats.Tiers[streamTier] + if !ok { + su = JetStreamTier{} + } + su.Streams++ + stats.Tiers[streamTier] = su + + // Now consumers, check each since could be different tiers. + for _, ca := range sa.consumers { + stats.Consumers++ + consumerTier := tierName(ca.Config.replicas(sa.Config)) + cu, ok := stats.Tiers[consumerTier] + if !ok { + cu = JetStreamTier{} + } + cu.Consumers++ + stats.Tiers[consumerTier] = cu + } + } + } + } else { + if defaultTier { + stats.Streams = len(jsa.streams) + } + for _, mset := range jsa.streams { + consCount := mset.numConsumers() + stats.Consumers += consCount + if !defaultTier { + u, ok := stats.Tiers[mset.tier] + if !ok { + u = JetStreamTier{} + } + u.Streams++ + stats.Streams++ + u.Consumers += consCount + stats.Tiers[mset.tier] = u + } + } + } + jsa.mu.RUnlock() + js.mu.RUnlock() + } + return stats +} + +// DisableJetStream will disable JetStream for this account. +func (a *Account) DisableJetStream() error { + return a.removeJetStream() +} + +// removeJetStream is called when JetStream has been disabled for this account. +func (a *Account) removeJetStream() error { + a.mu.Lock() + s := a.srv + a.js = nil + a.mu.Unlock() + + if s == nil { + return fmt.Errorf("jetstream account not registered") + } + + js := s.getJetStream() + if js == nil { + return NewJSNotEnabledForAccountError() + } + + return js.disableJetStream(js.lookupAccount(a)) +} + +// Disable JetStream for the account. +func (js *jetStream) disableJetStream(jsa *jsAccount) error { + if jsa == nil || jsa.account == nil { + return NewJSNotEnabledForAccountError() + } + + js.mu.Lock() + delete(js.accounts, jsa.account.Name) + js.mu.Unlock() + + jsa.delete() + + return nil +} + +// jetStreamConfigured reports whether the account has JetStream configured, regardless of this +// servers JetStream status. +func (a *Account) jetStreamConfigured() bool { + if a == nil { + return false + } + a.mu.RLock() + defer a.mu.RUnlock() + return len(a.jsLimits) > 0 +} + +// JetStreamEnabled is a helper to determine if jetstream is enabled for an account. +func (a *Account) JetStreamEnabled() bool { + if a == nil { + return false + } + a.mu.RLock() + enabled := a.js != nil + a.mu.RUnlock() + return enabled +} + +func (jsa *jsAccount) removeRemoteUsage(rnode string) { + jsa.usageMu.Lock() + defer jsa.usageMu.Unlock() + + rUsage := jsa.rusage[rnode] + if rUsage == nil { + return + } + for tierName, usage := range rUsage.tiers { + if total := jsa.usage[tierName]; total != nil { + total.total.mem -= usage.mem + total.total.store -= usage.store + } + } + jsa.apiTotal -= rUsage.api + jsa.apiErrors -= rUsage.err + delete(jsa.rusage, rnode) +} + +func (jsa *jsAccount) remoteUpdateUsage(sub *subscription, c *client, _ *Account, subject, _ string, msg []byte) { + // jsa.js.srv is immutable and guaranteed to no be nil, so no lock needed. + s := jsa.js.srv + + if len(msg) < minUsageUpdateLen { + s.Warnf("Ignoring remote usage update with size too short") + return + } + var rnode string + if li := strings.LastIndexByte(subject, btsep); li > 0 && li < len(subject) { + rnode = subject[li+1:] + } + if rnode == _EMPTY_ { + s.Warnf("Received remote usage update with no remote node") + return + } + + // Capture the meta group before the usage lock so we do not invert lock ordering. + meta := jsa.js.getMetaGroup() + jsa.usageMu.Lock() + defer jsa.usageMu.Unlock() + + rUsage, ok := jsa.rusage[rnode] + if !ok { + // Once a server has been removed from the meta group, a usage update that + // was already in flight must not recreate its remote usage entry. Only do + // the membership check for new entries; steady-state updates avoid it. + if meta != nil { + var current bool + for _, peer := range meta.VotingPeerNames() { + if peer == rnode { + current = true + break + } + } + if !current { + return + } + } + if jsa.rusage == nil { + jsa.rusage = make(map[string]*remoteUsage) + } + rUsage = &remoteUsage{tiers: make(map[string]*jsaUsage)} + jsa.rusage[rnode] = rUsage + } + updateTotal := func(tierName string, memUsed, storeUsed int64) { + total, ok := jsa.usage[tierName] + if !ok { + total = &jsaStorage{} + jsa.usage[tierName] = total + } + // Update the usage for this remote. + if usage := rUsage.tiers[tierName]; usage != nil { + // Decrement our old values. + total.total.mem -= usage.mem + total.total.store -= usage.store + usage.mem, usage.store = memUsed, storeUsed + } else { + rUsage.tiers[tierName] = &jsaUsage{memUsed, storeUsed} + } + total.total.mem += memUsed + total.total.store += storeUsed + } + + var le = binary.LittleEndian + apiTotal, apiErrors := le.Uint64(msg[16:]), le.Uint64(msg[24:]) + memUsed, storeUsed := int64(le.Uint64(msg[0:])), int64(le.Uint64(msg[8:])) + + // We later extended the data structure to support multiple tiers + var excessRecordCnt uint32 + var tierName string + + if len(msg) >= usageMultiTiersLen { + excessRecordCnt = le.Uint32(msg[minUsageUpdateLen:]) + length := le.Uint64(msg[minUsageUpdateLen+4:]) + // Need to protect past this point in case this is wrong. + if length > uint64(len(msg))-usageMultiTiersLen { + s.Warnf("Received corrupt remote usage update") + return + } + tierName = string(msg[usageMultiTiersLen : usageMultiTiersLen+length]) + msg = msg[usageMultiTiersLen+length:] + } + updateTotal(tierName, memUsed, storeUsed) + for ; excessRecordCnt > 0 && len(msg) >= usageRecordLen; excessRecordCnt-- { + memUsed, storeUsed := int64(le.Uint64(msg[0:])), int64(le.Uint64(msg[8:])) + length := le.Uint64(msg[16:]) + if length > uint64(len(msg))-usageRecordLen { + s.Warnf("Received corrupt remote usage update on excess record") + return + } + tierName = string(msg[usageRecordLen : usageRecordLen+length]) + msg = msg[usageRecordLen+length:] + updateTotal(tierName, memUsed, storeUsed) + } + jsa.apiTotal -= rUsage.api + jsa.apiErrors -= rUsage.err + rUsage.api = apiTotal + rUsage.err = apiErrors + jsa.apiTotal += apiTotal + jsa.apiErrors += apiErrors +} + +// When we detect a skew of some sort this will verify the usage reporting is correct. +// No locks should be held. +func (jsa *jsAccount) checkAndSyncUsage(tierName string, storeType StorageType) { + // This will run in a separate go routine, so check that we are only running once. + if !jsa.sync.CompareAndSwap(false, true) { + return + } + defer jsa.sync.Store(false) + + // Hold the account read lock and the usage lock while we calculate. + // We scope by tier and storage type, but if R3 File has 200 streams etc. could + // show a pause. I did test with > 100 non-active streams and was 80-200ns or so. + // Should be rare this gets called as well. + jsa.mu.RLock() + defer jsa.mu.RUnlock() + js := jsa.js + if js == nil { + return + } + s := js.srv + + // We need to collect the stream stores before we acquire the usage lock since in storeUpdates the + // stream lock could be held if deletion are inline with storing a new message, e.g. via limits. + var stores []StreamStore + for _, mset := range jsa.streams { + mset.mu.RLock() + if mset.tier == tierName && mset.stype == storeType && mset.store != nil { + stores = append(stores, mset.store) + } + mset.mu.RUnlock() + } + + // Now range and qualify, hold usage lock to prevent updates. + jsa.usageMu.Lock() + defer jsa.usageMu.Unlock() + + usage, ok := jsa.usage[tierName] + if !ok { + return + } + + // Collect current total for all stream stores that matched. + var total int64 + var state StreamState + for _, store := range stores { + store.FastState(&state) + total += int64(state.Bytes) + } + + var needClusterUpdate bool + // If we do not match on our calculations compute delta and adjust. + if storeType == MemoryStorage { + if total != usage.local.mem { + s.Warnf("MemStore usage drift of %v vs %v detected for account %q", + friendlyBytes(total), friendlyBytes(usage.local.mem), jsa.account.GetName()) + delta := total - usage.local.mem + usage.local.mem += delta + usage.total.mem += delta + atomic.AddInt64(&js.memUsed, delta) + needClusterUpdate = true + } + } else { + if total != usage.local.store { + s.Warnf("FileStore usage drift of %v vs %v detected for account %q", + friendlyBytes(total), friendlyBytes(usage.local.store), jsa.account.GetName()) + delta := total - usage.local.store + usage.local.store += delta + usage.total.store += delta + atomic.AddInt64(&js.storeUsed, delta) + needClusterUpdate = true + } + } + + // Publish our local updates if in clustered mode. + if needClusterUpdate && js.isClusteredNoLock() { + jsa.sendClusterUsageUpdate() + } +} + +// Updates accounting on in use memory and storage. This is called from locally +// by the lower storage layers. +func (jsa *jsAccount) updateUsage(tierName string, storeType StorageType, delta int64) { + // jsa.js is immutable and cannot be nil, so ok w/o lock. + js := jsa.js + // updateUsage() may be invoked under the mset's lock, so we can't get + // the js' lock to check if clustered. So use this function that make + // use of an atomic to do the check without having data race reports. + isClustered := js.isClusteredNoLock() + + var needsCheck bool + jsa.usageMu.Lock() + s, ok := jsa.usage[tierName] + if !ok { + s = &jsaStorage{} + jsa.usage[tierName] = s + } + if storeType == MemoryStorage { + s.local.mem += delta + s.total.mem += delta + atomic.AddInt64(&js.memUsed, delta) + needsCheck = s.local.mem < 0 + } else { + s.local.store += delta + s.total.store += delta + atomic.AddInt64(&js.storeUsed, delta) + needsCheck = s.local.store < 0 + } + // Publish our local updates if in clustered mode. + if isClustered { + jsa.sendClusterUsageUpdate() + } + jsa.usageMu.Unlock() + + if needsCheck { + // We could be holding the stream lock from up in the stack, and this + // will want the jsa lock, which would violate locking order. + // So do this in a Go routine. The function will check if it is already running. + go jsa.checkAndSyncUsage(tierName, storeType) + } +} + +var usageTick = 1500 * time.Millisecond + +func (jsa *jsAccount) sendClusterUsageUpdateTimer() { + jsa.usageMu.Lock() + defer jsa.usageMu.Unlock() + jsa.sendClusterUsageUpdate() + if jsa.utimer != nil { + jsa.utimer.Reset(usageTick) + } +} + +// For usage fields. +const ( + minUsageUpdateLen = 32 + stackUsageUpdate = 72 + usageRecordLen = 24 + usageMultiTiersLen = 44 + apiStatsAndNumTiers = 20 + minUsageUpdateWindow = 250 * time.Millisecond +) + +// Send updates to our account usage for this server. +// jsa.usageMu lock should be held. +func (jsa *jsAccount) sendClusterUsageUpdate() { + // These values are absolute so we can limit send rates. + now := time.Now() + if now.Sub(jsa.lupdate) < minUsageUpdateWindow { + return + } + jsa.lupdate = now + + lenUsage := len(jsa.usage) + if lenUsage == 0 { + return + } + // every base record contains mem/store/len(tier) as well as the tier name + l := usageRecordLen * lenUsage + for tier := range jsa.usage { + l += len(tier) + } + // first record contains api/usage errors as well as count for extra base records + l += apiStatsAndNumTiers + + var raw [stackUsageUpdate]byte + var b []byte + if l > stackUsageUpdate { + b = make([]byte, l) + } else { + b = raw[:l] + } + + var i int + var le = binary.LittleEndian + for tier, usage := range jsa.usage { + le.PutUint64(b[i+0:], uint64(usage.local.mem)) + le.PutUint64(b[i+8:], uint64(usage.local.store)) + if i == 0 { + le.PutUint64(b[16:], jsa.usageApi) + le.PutUint64(b[24:], jsa.usageErr) + le.PutUint32(b[32:], uint32(len(jsa.usage)-1)) + le.PutUint64(b[36:], uint64(len(tier))) + copy(b[usageMultiTiersLen:], tier) + i = usageMultiTiersLen + len(tier) + } else { + le.PutUint64(b[i+16:], uint64(len(tier))) + copy(b[i+usageRecordLen:], tier) + i += usageRecordLen + len(tier) + } + } + jsa.sendq.push(newPubMsg(nil, jsa.updatesPub, _EMPTY_, nil, nil, b, noCompression, false, false)) +} + +func (js *jetStream) wouldExceedLimits(storeType StorageType, sz int) bool { + var ( + total *int64 + max *int64 + ) + if storeType == MemoryStorage { + total, max = &js.memUsed, &js.memMax + } else { + total, max = &js.storeUsed, &js.storeMax + } + return (atomic.LoadInt64(total) + int64(sz)) > atomic.LoadInt64(max) +} + +func (js *jetStream) limitsExceeded(storeType StorageType) bool { + return js.wouldExceedLimits(storeType, 0) +} + +func tierName(replicas int) string { + // TODO (mh) this is where we could select based off a placement tag as well "qos:tier" + if replicas == 0 { + replicas = 1 + } + return fmt.Sprintf("R%d", replicas) +} + +func isSameTier(replicasA, replicasB int) bool { + a := max(1, replicasA) + b := max(1, replicasB) + // TODO (mh) this is where we could select based off a placement tag as well "qos:tier" + return a == b +} + +func (jsa *jsAccount) jetStreamAndClustered() (*jetStream, bool) { + jsa.mu.RLock() + js := jsa.js + jsa.mu.RUnlock() + return js, js.isClustered() +} + +// jsa.usageMu read lock should be held. +func (jsa *jsAccount) selectLimits(replicas int) (JetStreamAccountLimits, string, bool) { + if selectedLimits, ok := jsa.limits[_EMPTY_]; ok { + return selectedLimits, _EMPTY_, true + } + tier := tierName(replicas) + if selectedLimits, ok := jsa.limits[tier]; ok { + return selectedLimits, tier, true + } + return JetStreamAccountLimits{}, _EMPTY_, false +} + +// Lock should be held. +func (jsa *jsAccount) countStreams(tier string, cfg *StreamConfig) (streams int) { + for _, mset := range jsa.streams { + mset.cfgMu.RLock() + name, replicas := mset.cfg.Name, mset.cfg.Replicas + mset.cfgMu.RUnlock() + // Don't count the stream toward the limit if it already exists. + if (tier == _EMPTY_ || isSameTier(replicas, cfg.Replicas)) && name != cfg.Name { + streams++ + } + } + return streams +} + +// jsa.usageMu read lock (at least) should be held. +func (jsa *jsAccount) storageTotals() (uint64, uint64) { + mem := uint64(0) + store := uint64(0) + for _, sa := range jsa.usage { + mem += uint64(sa.total.mem) + store += uint64(sa.total.store) + } + return mem, store +} + +func (jsa *jsAccount) limitsExceeded(storeType StorageType, tierName string, replicas int) (bool, *ApiError) { + return jsa.wouldExceedLimits(storeType, tierName, replicas, _EMPTY_, nil, nil) +} + +func (jsa *jsAccount) wouldExceedLimits(storeType StorageType, tierName string, replicas int, subj string, hdr, msg []byte) (bool, *ApiError) { + jsa.usageMu.RLock() + defer jsa.usageMu.RUnlock() + + selectedLimits, ok := jsa.limits[tierName] + if !ok { + return true, NewJSNoLimitsError() + } + inUse := jsa.usage[tierName] + if inUse == nil { + // Imply totals of 0 + return false, nil + } + r := int64(replicas) + // Make sure replicas is correct. + if r < 1 { + r = 1 + } + // This is for limits. If we have no tier, consider all to be flat, vs tiers like R3 where we want to scale limit by replication. + lr := r + if tierName == _EMPTY_ { + lr = 1 + } + + // Since tiers are flat we need to scale limit up by replicas when checking. + if storeType == MemoryStorage { + totalMem := inUse.total.mem + (int64(memStoreMsgSize(subj, hdr, msg)) * r) + if selectedLimits.MaxMemory >= 0 && totalMem > selectedLimits.MaxMemory*lr { + return true, nil + } + } else { + totalStore := inUse.total.store + (int64(fileStoreMsgSize(subj, hdr, msg)) * r) + if selectedLimits.MaxStore >= 0 && totalStore > selectedLimits.MaxStore*lr { + return true, nil + } + } + + return false, nil +} + +// Check account limits. +// Read Lock should be held +func (js *jetStream) checkAccountLimits(selected *JetStreamAccountLimits, tier string, config *StreamConfig, currentRes int64) error { + return js.checkLimits(selected, tier, config, false, currentRes, 0) +} + +// Check account and server limits. +// Read Lock should be held +func (js *jetStream) checkAllLimits(selected *JetStreamAccountLimits, tier string, config *StreamConfig, currentRes, maxBytesOffset int64) error { + return js.checkLimits(selected, tier, config, true, currentRes, maxBytesOffset) +} + +// Check if a new proposed msg set while exceed our account limits. +// Lock should be held. +func (js *jetStream) checkLimits(selected *JetStreamAccountLimits, tier string, config *StreamConfig, checkServer bool, currentRes, maxBytesOffset int64) error { + // Check MaxConsumers + if config.MaxConsumers > 0 && selected.MaxConsumers > 0 && config.MaxConsumers > selected.MaxConsumers { + return NewJSMaximumConsumersLimitError() + } + // stream limit is checked separately on stream create only! + // Check storage, memory or disk. + return js.checkBytesLimits(selected, tier, config.MaxBytes, config.Replicas, config.Storage, checkServer, currentRes, maxBytesOffset) +} + +// accountReservation returns how many bytes count against the account limit +// for a stream with the given replica count. Un-tiered limits are flat, so R>1 +// is counted as Replicas*bytes; tiered limits already bake in replication. +func accountReservation(tier string, replicas int, bytes int64) int64 { + if bytes <= 0 { + return 0 + } + if tier == _EMPTY_ && replicas > 1 { + return mulSaturate(int64(replicas), bytes) + } + return bytes +} + +// Check if additional bytes will exceed our account limits and optionally the server itself. +// Read Lock should be held. +func (js *jetStream) checkBytesLimits(selectedLimits *JetStreamAccountLimits, tier string, addBytes int64, replicas int, storage StorageType, checkServer bool, currentRes, maxBytesOffset int64) error { + if addBytes < 0 { + addBytes = 1 + } + // The per-server footprint is a single replica's worth of bytes; the + // account footprint additionally accounts for replication in un-tiered setups. + serverBytes := addSaturate(addBytes, maxBytesOffset) + accountBytes := accountReservation(tier, replicas, serverBytes) + + switch storage { + case MemoryStorage: + // Account limits defined. + if selectedLimits.MaxMemory >= 0 && (currentRes > selectedLimits.MaxMemory || accountBytes > selectedLimits.MaxMemory-currentRes) { + return NewJSMemoryResourcesExceededError() + } + // Check if this server can handle request. + if checkServer && (js.memReserved > js.config.MaxMemory || serverBytes > js.config.MaxMemory-js.memReserved) { + return NewJSMemoryResourcesExceededError() + } + case FileStorage: + // Account limits defined. + if selectedLimits.MaxStore >= 0 && (currentRes > selectedLimits.MaxStore || accountBytes > selectedLimits.MaxStore-currentRes) { + return NewJSStorageResourcesExceededError() + } + // Check if this server can handle request. + if checkServer && (js.storeReserved > js.config.MaxStore || serverBytes > js.config.MaxStore-js.storeReserved) { + return NewJSStorageResourcesExceededError() + } + } + + return nil +} + +func (jsa *jsAccount) acc() *Account { + return jsa.account +} + +// Delete the JetStream resources. +func (jsa *jsAccount) delete() { + var streams []*stream + + jsa.mu.Lock() + // The update timer and subs need to be protected by usageMu lock + jsa.usageMu.Lock() + if jsa.utimer != nil { + jsa.utimer.Stop() + jsa.utimer = nil + } + if jsa.updatesSub != nil && jsa.js.srv != nil { + s := jsa.js.srv + s.sysUnsubscribe(jsa.updatesSub) + jsa.updatesSub = nil + } + jsa.usageMu.Unlock() + + for _, ms := range jsa.streams { + streams = append(streams, ms) + } + jsa.mu.Unlock() + + for _, mset := range streams { + mset.stop(false, false) + } +} + +// Lookup the jetstream account for a given account. +func (js *jetStream) lookupAccount(a *Account) *jsAccount { + if a == nil { + return nil + } + js.mu.RLock() + jsa := js.accounts[a.Name] + js.mu.RUnlock() + return jsa +} + +// Report on JetStream stats and usage for this server. +func (js *jetStream) usageStats() *JetStreamStats { + var stats JetStreamStats + js.mu.RLock() + stats.Accounts = len(js.accounts) + stats.ReservedMemory = uint64(js.memReserved) + stats.ReservedStore = uint64(js.storeReserved) + s := js.srv + js.mu.RUnlock() + stats.API.Level = JSApiLevel + stats.API.Total = uint64(atomic.LoadInt64(&js.apiTotal)) + stats.API.Errors = uint64(atomic.LoadInt64(&js.apiErrors)) + stats.API.Inflight = uint64(atomic.LoadInt64(&js.apiInflight)) + // Make sure we do not report negative. + used := atomic.LoadInt64(&js.memUsed) + if used < 0 { + used = 0 + } + stats.Memory = uint64(used) + used = atomic.LoadInt64(&js.storeUsed) + if used < 0 { + used = 0 + } + stats.Store = uint64(used) + stats.HAAssets = s.numRaftNodes() + return &stats +} + +// Check to see if we have enough system resources for this account. +// Lock should be held. +func (js *jetStream) sufficientResources(limits map[string]JetStreamAccountLimits) error { + // If we are clustered we do not really know how many resources will be ultimately available. + // This needs to be handled out of band. + // If we are a single server, we can make decisions here. + if limits == nil || !js.standAlone { + return nil + } + + totalMaxBytes := func(limits map[string]JetStreamAccountLimits) (int64, int64) { + totalMaxMemory := int64(0) + totalMaxStore := int64(0) + for _, l := range limits { + if l.MaxMemory > 0 { + totalMaxMemory += l.MaxMemory + } + if l.MaxStore > 0 { + totalMaxStore += l.MaxStore + } + } + return totalMaxMemory, totalMaxStore + } + + totalMaxMemory, totalMaxStore := totalMaxBytes(limits) + + // Reserved is now specific to the MaxBytes for streams. + if js.memReserved+totalMaxMemory > js.config.MaxMemory { + return NewJSMemoryResourcesExceededError() + } + // A dynamic limit is still provisional until recovery has run. + recovering := js.config.maxStorePending + if !recovering && js.storeReserved+totalMaxStore > js.config.MaxStore { + return NewJSStorageResourcesExceededError() + } + + // Since we know if we are here we are single server mode, check the account reservations. + var storeReserved, memReserved int64 + for _, jsa := range js.accounts { + if jsa.account.IsExpired() { + continue + } + jsa.usageMu.RLock() + maxMemory, maxStore := totalMaxBytes(jsa.limits) + jsa.usageMu.RUnlock() + memReserved += maxMemory + storeReserved += maxStore + } + + if memReserved+totalMaxMemory > js.config.MaxMemory { + return NewJSMemoryResourcesExceededError() + } + if !recovering && storeReserved+totalMaxStore > js.config.MaxStore { + return NewJSStorageResourcesExceededError() + } + + return nil +} + +// This will reserve the stream resources requested. +// This will spin off off of MaxBytes. +func (js *jetStream) reserveStreamResources(cfg *StreamConfig) { + if cfg == nil || cfg.MaxBytes <= 0 { + return + } + + js.mu.Lock() + switch cfg.Storage { + case MemoryStorage: + js.memReserved += cfg.MaxBytes + case FileStorage: + js.storeReserved += cfg.MaxBytes + } + s, clustered := js.srv, !js.standAlone + js.mu.Unlock() + // If clustered send an update to the system immediately. + if clustered { + s.sendStatszUpdate() + } +} + +// Release reserved resources held by a stream. +func (js *jetStream) releaseStreamResources(cfg *StreamConfig) { + if cfg == nil || cfg.MaxBytes <= 0 { + return + } + + js.mu.Lock() + switch cfg.Storage { + case MemoryStorage: + js.memReserved -= cfg.MaxBytes + case FileStorage: + js.storeReserved -= cfg.MaxBytes + } + s, clustered := js.srv, !js.standAlone + js.mu.Unlock() + // If clustered send an update to the system immediately. + if clustered { + s.sendStatszUpdate() + } +} + +const ( + // JetStreamStoreDir is the prefix we use. + JetStreamStoreDir = "jetstream" + // JetStreamMaxStoreDefault is the default disk storage limit. 1TB + JetStreamMaxStoreDefault = 1024 * 1024 * 1024 * 1024 + // JetStreamMaxMemDefault is only used when we can't determine system memory. 256MB + JetStreamMaxMemDefault = 1024 * 1024 * 256 + // snapshot staging for restores. + snapStagingDir = ".snap-staging" +) + +// Dynamically create a config with a tmp based directory (repeatable) and 75% of system memory. +func (s *Server) dynJetStreamConfig(storeDir string, maxStore, maxMem int64) *JetStreamConfig { + jsc := &JetStreamConfig{} + if storeDir != _EMPTY_ { + jsc.StoreDir = filepath.Join(storeDir, JetStreamStoreDir) + } else { + // Create one in tmp directory, but make it consistent for restarts. + jsc.StoreDir = filepath.Join(os.TempDir(), "nats", JetStreamStoreDir) + s.Warnf("Temporary storage directory used, data could be lost on system reboot") + } + + opts := s.getOpts() + + // Strict mode. + jsc.Strict = !opts.NoJetStreamStrict + + // Sync options. + jsc.SyncInterval = opts.SyncInterval + jsc.SyncAlways = opts.SyncAlways + + if maxStore > 0 || (opts.maxStoreSet && maxStore == 0) { + jsc.MaxStore = maxStore + } else { + jsc.MaxStore = diskAvailable(jsc.StoreDir) + jsc.maxStorePending = true + } + + if maxMem > 0 || (opts.maxMemSet && maxMem == 0) { + jsc.MaxMemory = maxMem + } else { + // Estimate to 75% of total memory if we can determine system memory. + if sysMem := sysmem.Memory(); sysMem > 0 { + // Check if we have been limited with GOMEMLIMIT and if lower use that value. + if gml := debug.SetMemoryLimit(-1); gml != math.MaxInt64 && gml < sysMem { + s.Debugf("JetStream detected GOMEMLIMIT of %v", friendlyBytes(gml)) + sysMem = gml + } + jsc.MaxMemory = sysMem / 4 * 3 + } else { + jsc.MaxMemory = JetStreamMaxMemDefault + } + } + + return jsc +} + +// Helper function. +func (a *Account) checkForJetStream() (*Server, *jsAccount, error) { + a.mu.RLock() + s := a.srv + jsa := a.js + a.mu.RUnlock() + + if s == nil || jsa == nil { + return nil, nil, NewJSNotEnabledForAccountError() + } + + return s, jsa, nil +} + +type Number interface { + int | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64 | float32 | float64 +} + +// friendlyBytes returns a string with the given bytes int64 +// represented as a size, such as 1KB, 10MB, etc... +func friendlyBytes[T Number](bytes T) string { + fbytes := float64(bytes) + base := 1024 + pre := []string{"K", "M", "G", "T", "P", "E"} + if fbytes < float64(base) { + return fmt.Sprintf("%v B", fbytes) + } + exp := int(math.Log(fbytes) / math.Log(float64(base))) + index := exp - 1 + return fmt.Sprintf("%.2f %sB", fbytes/math.Pow(float64(base), float64(exp)), pre[index]) +} + +func isValidName(name string) bool { + if name == _EMPTY_ { + return false + } + return !strings.ContainsAny(name, " \t\r\n\f.*>") +} + +func isValidAssetName(name string) bool { + if name == _EMPTY_ { + return false + } + return !strings.ContainsAny(name, " \t\r\n\f.*>\\/") +} + +// To throttle the out of resources errors. +func (s *Server) resourcesExceededError(storeType StorageType) { + var didAlert bool + + s.rerrMu.Lock() + if now := time.Now(); now.Sub(s.rerrLast) > 10*time.Second { + s.Errorf("JetStream %s resource limits exceeded for server", strings.ToLower(storeType.String())) + s.rerrLast = now + didAlert = true + } + s.rerrMu.Unlock() + + // If we are meta leader we should relinquish that here. + if didAlert { + if js := s.getJetStream(); js != nil { + js.mu.RLock() + if cc := js.cluster; cc != nil && cc.meta != nil { + cc.meta.StepDown() + } + js.mu.RUnlock() + } + } +} + +// For validating options. +func validateJetStreamOptions(o *Options) error { + // in non operator mode, the account names need to be configured + if len(o.JsAccDefaultDomain) > 0 { + if len(o.TrustedOperators) == 0 { + for a, domain := range o.JsAccDefaultDomain { + found := false + if isReservedAccount(a) { + found = true + } else { + for _, acc := range o.Accounts { + if a == acc.GetName() { + if len(acc.jsLimits) > 0 && domain != _EMPTY_ { + return fmt.Errorf("default_js_domain contains account name %q with enabled JetStream", a) + } + found = true + break + } + } + } + if !found { + return fmt.Errorf("in non operator mode, `default_js_domain` references non existing account %q", a) + } + } + } else { + for a := range o.JsAccDefaultDomain { + if !nkeys.IsValidPublicAccountKey(a) { + return fmt.Errorf("default_js_domain contains account name %q, which is not a valid public account nkey", a) + } + } + } + for a, d := range o.JsAccDefaultDomain { + sacc := DEFAULT_SYSTEM_ACCOUNT + if o.SystemAccount != _EMPTY_ { + sacc = o.SystemAccount + } + if a == sacc { + return fmt.Errorf("system account %q can not be in default_js_domain", a) + } + if d == _EMPTY_ { + continue + } + if sub := fmt.Sprintf(jsDomainAPI, d); !IsValidSubject(sub) { + return fmt.Errorf("default_js_domain contains account %q with invalid domain name %q", a, d) + } + } + } + if o.JetStreamDomain != _EMPTY_ { + if subj := fmt.Sprintf(jsDomainAPI, o.JetStreamDomain); !IsValidSubject(subj) { + return fmt.Errorf("invalid domain name: derived %q is not a valid subject", subj) + } + + if !isValidName(o.JetStreamDomain) { + return fmt.Errorf("invalid domain name: may not contain ., * or >") + } + } + // If not clustered no checks needed past here. + if !o.JetStream || o.Cluster.Port == 0 { + return nil + } + if o.ServerName == _EMPTY_ { + return fmt.Errorf("jetstream cluster requires `server_name` to be set") + } + if o.Cluster.Name == _EMPTY_ { + return fmt.Errorf("jetstream cluster requires `cluster.name` to be set") + } + + h := strings.ToLower(o.JetStreamExtHint) + switch h { + case jsWillExtend, jsNoExtend, _EMPTY_: + o.JetStreamExtHint = h + default: + return fmt.Errorf("expected 'no_extend' for string value, got '%s'", h) + } + + if o.JetStreamMaxCatchup < 0 { + return fmt.Errorf("jetstream max catchup cannot be negative") + } + return nil +} + +// We had a bug that set a default de dupe window on mirror, despite that being not a valid config +func fixCfgMirrorWithDedupWindow(cfg *StreamConfig) { + if cfg == nil || cfg.Mirror == nil { + return + } + if cfg.Duplicates != 0 { + cfg.Duplicates = 0 + } +} + +func (s *Server) handleWritePermissionError() { + //TODO Check if we should add s.jetStreamOOSPending in condition + if s.JetStreamEnabled() { + s.Errorf("File system permission denied while writing, disabling JetStream") + + go s.ShutdownJetStream() + + //TODO Send respective advisory if needed, same as in handleOutOfSpace + } +} diff --git a/server/jetstream_batching.go b/server/jetstream_batching.go new file mode 100644 index 0000000..0aa0219 --- /dev/null +++ b/server/jetstream_batching.go @@ -0,0 +1,1102 @@ +// Copyright 2025-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "encoding/json" + "errors" + "fmt" + "maps" + "math" + "math/big" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +var ( + // Tracks the total inflight batches, across all streams and accounts that enable batching. + globalInflightAtomicBatches atomic.Int64 + globalInflightFastBatches atomic.Int64 +) + +type batching struct { + mu sync.Mutex + atomic map[string]*atomicBatch + fast map[string]*fastBatch +} + +type atomicBatch struct { + timer *time.Timer // Inactivity timer for the batch. + lseq uint64 // The highest sequence for this batch. + store StreamStore // Where the batch is staged before committing. +} + +type fastBatch struct { + timer *time.Timer // Inactivity timer for the batch. + lseq uint64 // The highest sequence for this batch. + sseq uint64 // Last persisted stream sequence. + pseq uint64 // Last persisted batch sequence (is always lower or equal to lseq). + fseq uint64 // Sequence of when we last sent a flow message (is always lower or equal to pseq). + pending uint32 // Number of pending messages in the batch waiting to be persisted. + ackMessages uint16 // Ack will be sent every N messages. + maxAckMessages uint16 // Maximum ackMessages value the client allows. + reply string // The last reply subject seen when persisting a message. + gapOk bool // Whether a gap is okay, if not, the batch would be rejected. + commit bool // If the batch is committed. +} + +// newAtomicBatch creates an atomic batch publish object. +// Lock should be held. +func (batches *batching) newAtomicBatch(mset *stream, batchId string, replicas int, storage StorageType, storeDir, streamName string) (*atomicBatch, error) { + store, err := newBatchStore(mset, batchId, replicas, storage, storeDir, streamName) + if err != nil { + return nil, err + } + b := &atomicBatch{store: store} + b.setupCleanupTimer(mset, batchId, batches) + return b, nil +} + +// setupCleanupTimer sets up a timer to clean up the batch after a timeout. +func (b *atomicBatch) setupCleanupTimer(mset *stream, batchId string, batches *batching) { + // Create a timer to clean up after timeout. + timeout := getCleanupTimeout(mset) + b.timer = time.AfterFunc(timeout, func() { + b.cleanup(batchId, batches) + mset.sendStreamBatchAbandonedAdvisory(batchId, BatchTimeout) + }) +} + +// resetCleanupTimer resets the cleanup timer, allowing to extend the lifetime of the batch. +// Returns whether the timer was reset without it having expired before. +func (b *atomicBatch) resetCleanupTimer(mset *stream) bool { + timeout := getCleanupTimeout(mset) + return b.timer.Reset(timeout) +} + +// cleanup deletes underlying resources associated with the batch and unregisters it from the stream's batches. +func (b *atomicBatch) cleanup(batchId string, batches *batching) { + batches.mu.Lock() + defer batches.mu.Unlock() + b.cleanupLocked(batchId, batches) +} + +// Lock should be held. +func (b *atomicBatch) cleanupLocked(batchId string, batches *batching) { + if b.timer == nil { + return + } + globalInflightAtomicBatches.Add(-1) + b.timer.Stop() + b.store.Delete(true) + delete(batches.atomic, batchId) + // Reset so that another invocation doesn't double-account. + b.timer = nil +} + +// Lock should be held. +func (b *atomicBatch) stopLocked() { + if b.timer == nil { + return + } + globalInflightAtomicBatches.Add(-1) + b.timer.Stop() + b.store.Stop() + // Reset so that another invocation doesn't double-account. + b.timer = nil +} + +func getBatchStoreDir(storeDir, streamName, batchId string) (string, string) { + bname := getHash(batchId) + return bname, filepath.Join(storeDir, streamsDir, streamName, batchesDir, bname) +} + +func newBatchStore(mset *stream, batchId string, replicas int, storage StorageType, storeDir, streamName string) (StreamStore, error) { + if replicas == 1 && storage == FileStorage { + bname, storeDir := getBatchStoreDir(storeDir, streamName, batchId) + s := mset.srv + fcfg := FileStoreConfig{ + AsyncFlush: true, + SyncOnFlush: true, + BlockSize: defaultLargeBlockSize, + StoreDir: storeDir, + srv: s, + } + prf := s.jsKeyGen(s.getOpts().JetStreamKey, mset.acc.Name) + if prf != nil { + // We are encrypted here, fill in correct cipher selection. + fcfg.Cipher = s.getOpts().JetStreamCipher + } + oldprf := s.jsKeyGen(s.getOpts().JetStreamOldKey, mset.acc.Name) + cfg := StreamConfig{Name: bname, Storage: FileStorage} + return newFileStoreWithCreated(fcfg, cfg, time.Time{}, prf, oldprf) + } + return newMemStore(&StreamConfig{Name: _EMPTY_, Storage: MemoryStorage}) +} + +// readyForCommit indicates the batch is ready to be committed. +// If the timer has already cleaned up the batch, we can't commit. +// Otherwise, we ensure the timer does not clean up the batch in the meantime. +// Lock should be held. +func (b *atomicBatch) readyForCommit() *BatchAbandonReason { + if !b.timer.Stop() { + return &BatchTimeout + } + if b.store.FlushAllPending() != nil { + return &BatchIncomplete + } + return nil +} + +// newFastBatch creates a fast batch publish object and registers it in batches.fast. +// Lock should be held. +func (batches *batching) newFastBatch(mset *stream, batchId string, gapOk bool, maxAckMessages uint16) *fastBatch { + b := &fastBatch{gapOk: gapOk, maxAckMessages: maxAckMessages} + if batches.fast == nil { + batches.fast = make(map[string]*fastBatch, 1) + } + batches.fast[batchId] = b + batches.fastBatchInit(b) + b.setupCleanupTimer(mset, batchId, batches) + return b +} + +// fastBatchInit (re)initializes the ackMessages field for a fast batch. +// The batch must already be registered in batches.fast. +// Lock should be held. +func (batches *batching) fastBatchInit(b *fastBatch) { + // If it's the only batch, just allow what the client wants, otherwise we'll + // need to coordinate and slowly ramp up this publisher. + // TODO(mvv): fast ingest's initial flow value improvements? + ackMessages := min(500, b.maxAckMessages) + if len(batches.fast) > 1 { + ackMessages = 1 + } + b.ackMessages = ackMessages +} + +// fastBatchReset resets the fast batch to an empty state and sends a flow control message. +// Lock should be held. +func (batches *batching) fastBatchReset(mset *stream, batchId string, b *fastBatch) { + // If the timer already stopped before we could commit, we clean it up. + if b.timer == nil || (!b.commit && !b.timer.Stop()) { + b.cleanupLocked(batchId, batches) + return + } + // Otherwise, reset the state. + batches.fastBatchInit(b) + b.timer.Reset(getCleanupTimeout(mset)) + b.commit = false + b.pending = 0 + b.fseq, b.lseq = b.pseq, b.pseq + b.sendFlowControl(b.fseq, mset, b.reply) +} + +// fastBatchRegisterSequences registers the highest stored batch and stream sequence and returns +// whether a PubAck should be sent if the batch has been committed. +// If this is called on a follower, it only registers the highest stream and persisted batch sequences. +// Lock should be held. +func (batches *batching) fastBatchRegisterSequences(mset *stream, reply string, streamSeq uint64, isLeader bool, batch *FastBatch) bool { + b, ok := batches.fast[batch.id] + if !ok || !isLeader { + // If this batch has committed, we can clean it up. + if batch.commit { + if b != nil { + b.cleanupLocked(batch.id, batches) + } + return false + } + // Otherwise, even as a follower, we record the latest state of this batch. + if b == nil || !b.resetCleanupTimer(mset) { + if b != nil { + // The timer couldn't be reset, this means the timer already runs and is likely + // waiting to acquire the lock. We reset the timer here so it doesn't clean up + // this batch that we're about to overwrite. + b.timer = nil + } else { + // If this is a new batch for us, even though we're a follower, we still need + // to account toward the global inflight limit. + globalInflightFastBatches.Add(1) + } + // We'll need a copy as we'll use it as a key and later for cleanup. + batchId := copyString(batch.id) + b = batches.newFastBatch(mset, batchId, batch.gapOk, batch.flow) + } + b.sseq = streamSeq + b.pseq, b.lseq = batch.seq, batch.seq + b.reply = reply + return false + } + b.reply = reply + if b.pending > 0 { + b.pending-- + } + b.sseq = streamSeq + // Store last persisted batch sequence. + // If we have no remaining pending writes, we might have had duplicate messages + // and need to send additional flow control messages. + var skipped bool + if b.pending == 0 { + skipped = true + b.pseq = b.lseq + } else { + b.pseq = batch.seq + } + // If the PubAck needs to be sent now as a result of a commit. + if b.lseq == b.pseq && b.commit { + b.cleanupLocked(batch.id, batches) + // If we skipped ahead due to duplicate messages, send the PubAck with the highest sequence. + if skipped { + var buf [256]byte + pubAck := append(buf[:0], mset.pubAck...) + response := append(pubAck, strconv.FormatUint(b.sseq, 10)...) + response = append(response, fmt.Sprintf(",\"batch\":%q,\"count\":%d}", batch.id, b.lseq)...) + if len(reply) > 0 { + mset.outq.sendMsg(reply, response) + } + return false + } + return true + } + b.checkFlowControl(mset, reply, batches) + return false +} + +// checkFlowControl checks whether a flow control message should be sent. +// If so, it updates the flow values to speed up or slow down the publisher if needed. +// Returns whether a flow control message was sent. +// Lock should be held. +func (b *fastBatch) checkFlowControl(mset *stream, reply string, batches *batching) bool { + am := uint64(b.ackMessages) + if b.pseq < b.fseq+am { + return false + } + // Instead of sending multiple flow control messages, skip ahead to only send the last. + steps := (b.pseq - b.fseq) / am + b.fseq += steps * am + + // TODO(mvv): fast ingest's dynamic flow value improvements? + // This is currently just a simple value to have a working version. Should take average + // message sizes into account and compare how much this client is contributing to the + // ingest IPQ total size and messages and have publishers share based on that. + maxAckMessages := uint16(500 / len(batches.fast)) + if maxAckMessages < 1 { + maxAckMessages = 1 + } + // Limit to the client's allowed maximum. + if maxAckMessages > b.maxAckMessages { + maxAckMessages = b.maxAckMessages + } + + if b.ackMessages < maxAckMessages { + // Ramp up. + b.ackMessages *= 2 + if b.ackMessages > maxAckMessages { + b.ackMessages = maxAckMessages + } + } else if b.ackMessages > maxAckMessages { + // Slow down. + b.ackMessages /= 2 + if b.ackMessages <= maxAckMessages { + b.ackMessages = maxAckMessages + } + } + + // Finally, send the flow control message. + b.sendFlowControl(b.fseq, mset, reply) + return true +} + +// sendFlowControl sends a fast batch flow control message for the current highest sequence. +// Lock should be held. +func (b *fastBatch) sendFlowControl(batchSeq uint64, mset *stream, reply string) { + if len(reply) == 0 { + return + } + response, _ := BatchFlowAck{Sequence: batchSeq, Messages: b.ackMessages}.MarshalJSON() + mset.outq.sendMsg(reply, response) +} + +// fastBatchCommit ends the batch and commits the data up to that point. If all messages +// have already been persisted, a PubAck is sent immediately. Otherwise, it will be sent +// after the last message has been persisted. +// Lock should be held. +func (batches *batching) fastBatchCommit(b *fastBatch, batchId string, mset *stream, reply string) bool { + // Either we commit now, or we clean up later, so stop the timer. + if b.timer == nil || (!b.commit && !b.timer.Stop()) { + // Shouldn't be possible for the timer to already be stopped if we haven't committed yet, + // since we pre-check being able to reset the timer. But guard against it anyhow. + return true + } + // Mark that this batch commits. + b.commit = true + // If the whole batch has been persisted, we can respond with the PubAck now. + if b.lseq == b.pseq { + b.cleanupLocked(batchId, batches) + var buf [256]byte + pubAck := append(buf[:0], mset.pubAck...) + response := append(pubAck, strconv.FormatUint(b.sseq, 10)...) + response = append(response, fmt.Sprintf(",\"batch\":%q,\"count\":%d}", batchId, b.lseq)...) + if len(reply) > 0 { + mset.outq.sendMsg(reply, response) + } + return true + } + // Otherwise, we need to wait and the PubAck will be sent when the last message is persisted. + return false +} + +// setupCleanupTimer sets up a timer to clean up the batch after a timeout. +func (b *fastBatch) setupCleanupTimer(mset *stream, batchId string, batches *batching) { + // Create a timer to clean up after timeout. + timeout := getCleanupTimeout(mset) + b.timer = time.AfterFunc(timeout, func() { + b.cleanup(batchId, batches) + }) +} + +// resetCleanupTimer resets the cleanup timer, allowing to extend the lifetime of the batch. +// Returns whether the timer was reset without it having expired before. +func (b *fastBatch) resetCleanupTimer(mset *stream) bool { + if b.commit { + return true + } + if b.timer == nil { + return false + } + timeout := getCleanupTimeout(mset) + return b.timer.Reset(timeout) +} + +// cleanup deletes underlying resources associated with the batch and unregisters it from the stream's batches. +func (b *fastBatch) cleanup(batchId string, batches *batching) { + batches.mu.Lock() + defer batches.mu.Unlock() + b.cleanupLocked(batchId, batches) +} + +// Lock should be held. +func (b *fastBatch) cleanupLocked(batchId string, batches *batching) { + // If the timer is nil, it means this batch has been replaced with a new one. + // This can happen on a follower depending on timing. + if b.timer == nil { + return + } + globalInflightFastBatches.Add(-1) + b.timer.Stop() + delete(batches.fast, batchId) + // Reset so that another invocation doesn't double-account. + b.timer = nil +} + +// getCleanupTimeout returns the timeout for the batch, taking into account the server's limits. +func getCleanupTimeout(mset *stream) time.Duration { + timeout := streamMaxBatchTimeout + if maxBatchTimeout := mset.srv.getOpts().JetStreamLimits.MaxBatchTimeout; maxBatchTimeout > 0 { + timeout = maxBatchTimeout + } + return timeout +} + +// batchStagedDiff stages all changes for consistency checks until commit. +type batchStagedDiff struct { + msgIds map[string]struct{} + counter map[string]*msgCounterRunningTotal + inflight map[string]*inflightSubjectRunningTotal + inflightTransform map[uint64]string + expectedPerSubject map[string]*batchExpectedPerSubject +} + +type batchExpectedPerSubject struct { + sseq uint64 // Stream sequence. + clseq uint64 // Clustered proposal sequence. +} + +// copyCounterSources returns a deep copy of the given counter sources. +func copyCounterSources(src CounterSources) CounterSources { + if src == nil { + return nil + } + dst := make(CounterSources, len(src)) + for stream, subjects := range src { + dst[stream] = maps.Clone(subjects) + } + return dst +} + +func (diff *batchStagedDiff) commit(mset *stream) { + if len(diff.msgIds) > 0 { + ts := time.Now().UnixNano() + mset.ddMu.Lock() + for msgId := range diff.msgIds { + // We stage with zero, and will update in processJetStreamMsg once we know the sequence. + mset.storeMsgIdLocked(&ddentry{msgId, 0, ts}) + } + mset.ddMu.Unlock() + } + + // Store running totals for counters, we could have multiple counter increments proposed, but not applied yet. + if len(diff.counter) > 0 { + if mset.clusteredCounterTotal == nil { + mset.clusteredCounterTotal = make(map[string]*msgCounterRunningTotal, len(diff.counter)) + } + for k, c := range diff.counter { + mset.clusteredCounterTotal[k] = c + } + } + + // Track inflight. + if len(diff.inflight) > 0 { + if mset.inflight == nil { + mset.inflight = make(map[string]*inflightSubjectRunningTotal, len(diff.inflight)) + } + for subj, i := range diff.inflight { + if c, ok := mset.inflight[subj]; ok { + c.bytes += i.bytes + c.ops += i.ops + c.schedule = i.schedule + } else { + mset.inflight[subj] = i + } + } + } + + // Track inflight subject transforms. + if len(diff.inflightTransform) > 0 { + if mset.inflightTransform == nil { + mset.inflightTransform = make(map[uint64]string, len(diff.inflightTransform)) + } + for clseq, subj := range diff.inflightTransform { + mset.inflightTransform[clseq] = subj + } + } + + // Track sequence and subject. + if len(diff.expectedPerSubject) > 0 { + if mset.expectedPerSubjectSequence == nil { + mset.expectedPerSubjectSequence = make(map[uint64]string, len(diff.expectedPerSubject)) + } + if mset.expectedPerSubjectInProcess == nil { + mset.expectedPerSubjectInProcess = make(map[string]struct{}, len(diff.expectedPerSubject)) + } + for subj, e := range diff.expectedPerSubject { + mset.expectedPerSubjectSequence[e.clseq] = subj + mset.expectedPerSubjectInProcess[subj] = struct{}{} + } + } +} + +type batchApply struct { + id string // ID of the current batch. + count uint64 // Number of entries in the batch, for consistency checks. + entries []*CommittedEntry // Previous entries that are part of this batch. + entryStart int // The index into an entry indicating the first message of the batch. + maxApplied uint64 // Applied value before the entry containing the first message of the batch. +} + +// clearBatchState clears in-memory apply-batch-related state. +func (batch *batchApply) clearBatchState() { + batch.id = _EMPTY_ + batch.count = 0 + batch.entries = nil + batch.entryStart = 0 + batch.maxApplied = 0 +} + +// rejectBatchStateLocked rejects the batch and clears in-memory apply-batch-related state. +// Corrects mset.clfs to take the failed batch into account. +func (batch *batchApply) rejectBatchState(mset *stream) { + mset.clMu.Lock() + mset.clfs += batch.count + mset.clMu.Unlock() + // We're rejecting the batch, so all entries need to be returned to the pool. + for _, bce := range batch.entries { + bce.ReturnToPool() + } + batch.clearBatchState() +} + +// checkMsgHeadersPreClusteredProposal checks the message for expected/consistency headers. +// mset.mu lock must NOT be held or used. +// mset.clMu lock must be held. +func checkMsgHeadersPreClusteredProposal( + diff *batchStagedDiff, mset *stream, subject, rsubject string, hdr []byte, msg []byte, sourced bool, name string, + jsa *jsAccount, allowRollup, denyPurge, allowTTL, allowMsgCounter, allowMsgSchedules bool, + discard DiscardPolicy, discardNewPer bool, maxMsgSize int, maxMsgs int64, maxMsgsPer int64, maxBytes int64, +) ([]byte, []byte, uint64, *ApiError, error) { + var incr *big.Int + var hasSchedule bool + + // Do this before staging any proposal state. All clustered publish paths, + // including atomic and fast batches, use this helper. + if mset.store.Type() == FileStorage && isFileStoreMsgTooLarge(fileStoreMsgSize(subject, hdr, msg)) { + return hdr, msg, 0, NewJSStreamStoreFailedError(ErrMsgTooLarge), ErrMsgTooLarge + } + + // Some header checks must be checked pre proposal. + if len(hdr) > 0 { + // Since we encode header len as u16 make sure we do not exceed. + // Again this works if it goes through but better to be pre-emptive. + if len(hdr) > math.MaxUint16 { + err := fmt.Errorf("JetStream header size exceeds limits for '%s > %s'", jsa.acc().Name, name) + return hdr, msg, 0, NewJSStreamHeaderExceedsMaximumError(), err + } + // Counter increments. + // Only supported on counter streams, and payload must be empty (if not coming from a source). + var ok bool + if incr, ok = getMessageIncr(hdr); !ok { + apiErr := NewJSMessageIncrInvalidError() + return hdr, msg, 0, apiErr, apiErr + } else if incr != nil && !sourced { + // Only do checks if the message isn't sourced. Otherwise, we need to store verbatim. + if !allowMsgCounter { + apiErr := NewJSMessageIncrDisabledError() + return hdr, msg, 0, apiErr, apiErr + } else if len(msg) > 0 { + apiErr := NewJSMessageIncrPayloadError() + return hdr, msg, 0, apiErr, apiErr + } else { + // Check for incompatible headers. + var doErr bool + if getRollup(hdr) != _EMPTY_ || + getExpectedStream(hdr) != _EMPTY_ || + getExpectedLastMsgId(hdr) != _EMPTY_ || + getExpectedLastSeqPerSubjectForSubject(hdr) != _EMPTY_ { + doErr = true + } else if _, ok = getExpectedLastSeq(hdr); ok { + doErr = true + } else if _, ok = getExpectedLastSeqPerSubject(hdr); ok { + doErr = true + } + + if doErr { + apiErr := NewJSMessageIncrInvalidError() + return hdr, msg, 0, apiErr, apiErr + } + } + } + // Expected stream name can also be pre-checked. + if sname := getExpectedStream(hdr); sname != _EMPTY_ && sname != name { + return hdr, msg, 0, NewJSStreamNotMatchError(), errStreamMismatch + } + // TTL'd messages are rejected entirely if TTLs are not enabled on the stream, or if the TTL is invalid. + if ttl, err := getMessageTTL(hdr); !sourced && (ttl != 0 || err != nil) { + if !allowTTL { + return hdr, msg, 0, NewJSMessageTTLDisabledError(), errMsgTTLDisabled + } else if err != nil { + return hdr, msg, 0, NewJSMessageTTLInvalidError(), err + } + } + // Check for MsgIds here at the cluster level to avoid excessive CLFS accounting. + // Will help during restarts. + if msgId := getMsgId(hdr); msgId != _EMPTY_ { + // Dedupe if staged. + if _, ok = diff.msgIds[msgId]; ok { + return hdr, msg, 0, NewJSAtomicPublishContainsDuplicateMessageError(), errMsgIdDuplicate + } + mset.ddMu.Lock() + if dde := mset.checkMsgId(msgId); dde != nil { + seq := dde.seq + mset.ddMu.Unlock() + // Should not return an invalid sequence, in that case error. + if seq > 0 { + return hdr, msg, seq, NewJSAtomicPublishContainsDuplicateMessageError(), errMsgIdDuplicate + } else { + return hdr, msg, 0, NewJSStreamDuplicateMessageConflictError(), errMsgIdDuplicate + } + } + if diff.msgIds == nil { + diff.msgIds = map[string]struct{}{msgId: {}} + } else { + diff.msgIds[msgId] = struct{}{} + } + mset.ddMu.Unlock() + } + + // Non-sourced messages aren't allowed to have the stream source header. + if !sourced && len(sliceHeader(JSStreamSource, hdr)) > 0 { + apiErr := NewJSMessageSourceHdrNotAllowedError() + return hdr, msg, 0, apiErr, apiErr + } + } + + // Apply increment for counter. + // But only if it's allowed for this stream. This can happen when we store verbatim for a sourced stream. + if incr == nil && allowMsgCounter { + apiErr := NewJSMessageIncrMissingError() + return hdr, msg, 0, apiErr, apiErr + } + if incr != nil && allowMsgCounter { + var initial big.Int + var sources CounterSources + + // If we've got a running total, update that, since we have inflight proposals updating the same counter. + var ok bool + var counter *msgCounterRunningTotal + if counter, ok = diff.counter[subject]; ok { + initial = *counter.total + sources = counter.sources + } else if counter, ok = mset.clusteredCounterTotal[subject]; ok { + // Make an explicit copy to separate the staged data from what's committed. + initial.Set(counter.total) + sources = copyCounterSources(counter.sources) + // Don't need to initialize all values, they'll be overwritten later. + counter = &msgCounterRunningTotal{ops: counter.ops} + } else { + // Load last message, and store as inflight running total. + var smv StoreMsg + sm, err := mset.store.LoadLastMsg(subject, &smv) + if err == nil && sm != nil { + var val CounterValue + // Return an error if the counter is broken somehow. + if json.Unmarshal(sm.msg, &val) != nil { + apiErr := NewJSMessageCounterBrokenError() + return hdr, msg, 0, apiErr, apiErr + } + if ncs := sliceHeader(JSMessageCounterSources, sm.hdr); len(ncs) > 0 { + if err := json.Unmarshal(ncs, &sources); err != nil { + apiErr := NewJSMessageCounterBrokenError() + return hdr, msg, 0, apiErr, apiErr + } + } + initial.SetString(val.Value, 10) + } + } + srchdr := sliceHeader(JSStreamSource, hdr) + if len(srchdr) > 0 { + // This is a sourced message, so we can't apply Nats-Incr but + // instead should just update the source count header. + fields := strings.Split(string(srchdr), " ") + origStream := fields[0] + origSubj := subject + if len(fields) >= 5 { + origSubj = fields[4] + } + var val CounterValue + if json.Unmarshal(msg, &val) != nil { + apiErr := NewJSMessageCounterBrokenError() + return hdr, msg, 0, apiErr, apiErr + } + var sourced big.Int + sourced.SetString(val.Value, 10) + if sources == nil { + sources = map[string]map[string]string{} + } + if sources[origStream] == nil { + sources[origStream] = map[string]string{} + } + prevVal := sources[origStream][origSubj] + sources[origStream][origSubj] = sourced.String() + // We will also replace the Nats-Incr header with the diff + // between our last value from this source and this one, so + // that the arithmetic is always correct. + var previous big.Int + previous.SetString(prevVal, 10) + incr.Sub(&sourced, &previous) + hdr = setHeader(JSMessageIncr, incr.String(), hdr) + } + // Now make the change. + initial.Add(&initial, incr) + // Generate the new payload. + var _msg [128]byte + msg = fmt.Appendf(_msg[:0], "{%q:%q}", "val", initial.String()) + // Write the updated source count headers. + if len(sources) > 0 { + nhdr, err := json.Marshal(sources) + if err != nil { + return hdr, msg, 0, NewJSMessageCounterBrokenError(), err + } + hdr = setHeader(JSMessageCounterSources, string(nhdr), hdr) + } + + // Check to see if we are over the max msg size. + maxSize := int64(mset.srv.getOpts().MaxPayload) + if maxMsgSize >= 0 && int64(maxMsgSize) < maxSize { + maxSize = int64(maxMsgSize) + } + hdrLen, msgLen := int64(len(hdr)), int64(len(msg)) + // Subtract to prevent against overflows. + if hdrLen > maxSize || msgLen > maxSize-hdrLen { + return hdr, msg, 0, NewJSStreamMessageExceedsMaximumError(), ErrMaxPayload + } + + // Keep the in-memory counters up-to-date. + if counter == nil { + counter = &msgCounterRunningTotal{} + } + counter.total = &initial + counter.sources = sources + counter.ops++ + if diff.counter == nil { + diff.counter = map[string]*msgCounterRunningTotal{subject: counter} + } else { + diff.counter[subject] = counter + } + } + + if len(hdr) > 0 { + // Expected last sequence. + if seq, exists := getExpectedLastSeq(hdr); exists && seq != mset.clseq-mset.clfs { + mlseq := mset.clseq - mset.clfs + err := fmt.Errorf("last sequence mismatch: %d vs %d", seq, mlseq) + return hdr, msg, 0, NewJSStreamWrongLastSequenceError(mlseq), err + } else if exists && len(diff.inflight) > 0 { + // Only the first message in a batch can contain an expected last sequence. + err := fmt.Errorf("last sequence mismatch") + return hdr, msg, 0, NewJSStreamWrongLastSequenceConstantError(), err + } + + // Expected last sequence per subject. + if seq, exists := getExpectedLastSeqPerSubject(hdr); exists { + // Allow override of the subject used for the check. + seqSubj := subject + if optSubj := getExpectedLastSeqPerSubjectForSubject(hdr); optSubj != _EMPTY_ { + seqSubj = copyString(optSubj) + } + + // The subject is already written to in this batch, we can't allow + // expected checks since they would be incorrect. + if _, ok := diff.inflight[seqSubj]; ok { + err := errors.New("last sequence by subject mismatch") + return hdr, msg, 0, NewJSStreamWrongLastSequenceConstantError(), err + } + + // If the subject is already in process, block as otherwise we could have + // multiple messages inflight with the same subject. + if _, found := mset.expectedPerSubjectInProcess[seqSubj]; found { + err := errors.New("last sequence by subject mismatch") + return hdr, msg, 0, NewJSStreamWrongLastSequenceConstantError(), err + } + + // If the subject is already in process but without expected headers, block as we would have + // multiple messages inflight with the same subject. + if _, ok := mset.inflight[seqSubj]; ok { + err := errors.New("last sequence by subject mismatch") + return hdr, msg, 0, NewJSStreamWrongLastSequenceConstantError(), err + } + + // If we've already done an expected-check on this subject, use the cached result. + if e, ok := diff.expectedPerSubject[seqSubj]; ok { + if e.sseq != seq { + err := fmt.Errorf("last sequence by subject mismatch: %d vs %d", seq, e.sseq) + return hdr, msg, 0, NewJSStreamWrongLastSequenceError(e.sseq), err + } + e.clseq = mset.clseq + } else { + var smv StoreMsg + var fseq uint64 + sm, err := mset.store.LoadLastMsg(seqSubj, &smv) + if sm != nil { + fseq = sm.seq + } + if err == ErrStoreMsgNotFound && seq == 0 { + fseq, err = 0, nil + } + if err != nil || fseq != seq { + err = fmt.Errorf("last sequence by subject mismatch: %d vs %d", seq, fseq) + return hdr, msg, 0, NewJSStreamWrongLastSequenceError(fseq), err + } + + e = &batchExpectedPerSubject{sseq: fseq, clseq: mset.clseq} + if diff.expectedPerSubject == nil { + diff.expectedPerSubject = map[string]*batchExpectedPerSubject{seqSubj: e} + } else { + diff.expectedPerSubject[seqSubj] = e + } + } + } else if getExpectedLastSeqPerSubjectForSubject(hdr) != _EMPTY_ { + apiErr := NewJSStreamExpectedLastSeqPerSubjectInvalidError() + return hdr, msg, 0, apiErr, apiErr + } + + // Message scheduling. + if sourced { + // noop, sourced messages were already validated by the origin stream. + } else if schedule, apiErr := getMessageSchedule(hdr); apiErr != nil { + if !allowMsgSchedules { + apiErr = NewJSMessageSchedulesDisabledError() + } + return hdr, msg, 0, apiErr, apiErr + } else if !schedule.IsZero() { + hasSchedule = true + if !allowMsgSchedules { + apiErr := NewJSMessageSchedulesDisabledError() + return hdr, msg, 0, apiErr, apiErr + } else if scheduleTtl, ok := getMessageScheduleTTL(hdr); !ok { + apiErr := NewJSMessageSchedulesTTLInvalidError() + return hdr, msg, 0, apiErr, apiErr + } else if scheduleRollup := getMessageScheduleRollup(hdr); scheduleRollup != _EMPTY_ && scheduleRollup != JSMsgRollupSubject { + apiErr := NewJSMessageSchedulesRollupInvalidError() + return hdr, msg, 0, apiErr, apiErr + } else if scheduleTtl != _EMPTY_ && !allowTTL { + return hdr, msg, 0, NewJSMessageTTLDisabledError(), errMsgTTLDisabled + } else if scheduleTarget := getMessageScheduleTarget(hdr); scheduleTarget == _EMPTY_ || + !IsValidPublishSubject(scheduleTarget) || scheduleTarget == subject { + apiErr := NewJSMessageSchedulesTargetInvalidError() + return hdr, msg, 0, apiErr, apiErr + } else if scheduleSource := getMessageScheduleSource(hdr); scheduleSource != _EMPTY_ && + (scheduleSource == scheduleTarget || scheduleSource == subject || !IsValidPublishSubject(scheduleSource)) { + apiErr := NewJSMessageSchedulesSourceInvalidError() + return hdr, msg, 0, apiErr, apiErr + } else { + mset.cfgMu.RLock() + match := slices.ContainsFunc(mset.cfg.Subjects, func(subj string) bool { + return SubjectsCollide(subj, scheduleTarget) + }) + if !match { + mset.cfgMu.RUnlock() + apiErr := NewJSMessageSchedulesTargetInvalidError() + return hdr, msg, 0, apiErr, apiErr + } + if scheduleSource != _EMPTY_ { + match = slices.ContainsFunc(mset.cfg.Subjects, func(subj string) bool { + return SubjectsCollide(subj, scheduleSource) + }) + if !match { + mset.cfgMu.RUnlock() + apiErr := NewJSMessageSchedulesSourceInvalidError() + return hdr, msg, 0, apiErr, apiErr + } + } + mset.cfgMu.RUnlock() + + // Add a rollup sub header if it doesn't already exist. + // Otherwise, it must exist already as a rollup on the subject. + if rollup := getRollup(hdr); rollup == _EMPTY_ { + hdr = genHeader(hdr, JSMsgRollup, JSMsgRollupSubject) + } else if rollup != JSMsgRollupSubject { + apiErr := NewJSMessageSchedulesRollupInvalidError() + return hdr, msg, 0, apiErr, apiErr + } + } + } + if scheduleNext := sliceHeader(JSScheduleNext, hdr); len(scheduleNext) > 0 && !sourced { + // Clients may only use Nats-Schedule-Next to purge a schedule. + if bytesToString(scheduleNext) != JSScheduleNextPurge { + apiErr := NewJSMessageSchedulesSchedulerInvalidError() + return hdr, msg, 0, apiErr, apiErr + } + // Nats-Scheduler must accompany the purge and: + // - it must NOT be empty. + // - it must NOT match the publish subject. + if scheduler := sliceHeader(JSScheduler, hdr); len(scheduler) == 0 || + bytesToString(scheduler) == subject || !IsValidPublishSubject(bytesToString(scheduler)) { + apiErr := NewJSMessageSchedulesSchedulerInvalidError() + return hdr, msg, 0, apiErr, apiErr + } else if !allowMsgSchedules { + apiErr := NewJSMessageSchedulesDisabledError() + return hdr, msg, 0, apiErr, apiErr + } else { + // Check that the to-be-purged subject is a schedule message. + // We still allow this message through if there exists no message for this subject, + // to remain backward-compatible. An "expected at sequence" check can still be + // performed to make this stricter. + schedSubj := bytesToString(scheduler) + var invalid bool + if i, ok := diff.inflight[schedSubj]; ok { + invalid = !i.schedule + } else if i, ok = mset.inflight[schedSubj]; ok { + invalid = !i.schedule + } else { + var smv StoreMsg + sm, _ := mset.store.LoadLastMsg(schedSubj, &smv) + invalid = sm != nil && len(sliceHeader(JSSchedulePattern, sm.hdr)) == 0 + } + if invalid { + apiErr := NewJSMessageSchedulesSchedulerInvalidError() + return hdr, msg, 0, apiErr, apiErr + } + } + } else if !sourced && len(sliceHeader(JSScheduler, hdr)) > 0 { + // Clients may only use Nats-Scheduler alongside Nats-Schedule-Next. + apiErr := NewJSMessageSchedulesSchedulerInvalidError() + return hdr, msg, 0, apiErr, apiErr + } + + // Check for any rollups. + if rollup := getRollup(hdr); rollup != _EMPTY_ { + if (!allowRollup || denyPurge) && !sourced { + err := errors.New("rollup not permitted") + return hdr, msg, 0, NewJSStreamRollupFailedError(err), err + } + switch rollup { + case JSMsgRollupSubject: + // Rolling up the subject is only allowed if the first occurrence of this subject in the batch. + if _, ok := diff.inflight[subject]; ok { + err := errors.New("batch rollup sub invalid") + return hdr, msg, 0, NewJSStreamRollupFailedError(err), err + } + case JSMsgRollupAll: + // Rolling up the whole stream is only allowed if this is the first message of the batch. + if len(diff.inflight) > 0 { + err := errors.New("batch rollup all invalid") + return hdr, msg, 0, NewJSStreamRollupFailedError(err), err + } + default: + err := fmt.Errorf("rollup value invalid: %q", rollup) + return hdr, msg, 0, NewJSStreamRollupFailedError(err), err + } + } + } + + // Track inflight. + // Store the subject to ensure other messages in this batch using + // an expected check or rollup on the same subject fail. + if diff.inflight == nil { + diff.inflight = make(map[string]*inflightSubjectRunningTotal, 1) + } + var sz uint64 + if mset.store.Type() == FileStorage { + sz = fileStoreMsgSizeRaw(len(subject), len(hdr), len(msg)) + } else { + sz = memStoreMsgSizeRaw(len(subject), len(hdr), len(msg)) + } + var ( + i *inflightSubjectRunningTotal + ok bool + err error + ) + if i, ok = diff.inflight[subject]; ok { + i.bytes += sz + i.ops++ + i.schedule = hasSchedule + } else { + i = &inflightSubjectRunningTotal{bytes: sz, ops: 1, schedule: hasSchedule} + diff.inflight[subject] = i + } + + // Subject transform. + if subject != rsubject { + // The 'subject' is a transformed subject used for consistency checks. + // But since we propose the original (raw) subject to our peers, we need + // to store the transformed subject separately for when we apply. + // TODO(mvv): since subject transforms are handled by each replica individually, this has a + // potential for desync given out-of-order stream subject transform updates. + if diff.inflightTransform == nil { + diff.inflightTransform = make(map[uint64]string, 1) + } + diff.inflightTransform[mset.clseq] = subject + } + + // Check if we have discard new with max msgs or bytes. + // We need to deny here otherwise we'd need to bump CLFS, and it could succeed on some + // peers and not others depending on consumer ack state (if interest policy). + // So we deny here, if we allow that means we know it would succeed on every peer. + if discard == DiscardNew { + if maxMsgs > 0 || maxBytes > 0 { + // Track usual max msgs/bytes thresholds for DiscardNew. + var state StreamState + mset.store.FastState(&state) + + totalMsgs := state.Msgs + totalBytes := state.Bytes + for _, i = range mset.inflight { + totalMsgs += i.ops + totalBytes += i.bytes + } + for _, i = range diff.inflight { + totalMsgs += i.ops + totalBytes += i.bytes + } + + if maxMsgs > 0 && totalMsgs > uint64(maxMsgs) { + err = ErrMaxMsgs + } else if maxBytes > 0 && totalBytes > uint64(maxBytes) { + err = ErrMaxBytes + } + if err != nil { + return hdr, msg, 0, NewJSStreamStoreFailedError(err, Unless(err)), err + } + } + + // Similarly, check DiscardNew per-subject threshold to not need to bump CLFS. + // Allow rollup messages through since they will purge after storing. + if discardNewPer && maxMsgsPer > 0 && len(sliceHeader(JSMsgRollup, hdr)) == 0 { + // Get the current total for this subject. + totalMsgsForSubject := mset.store.SubjectsTotals(subject)[subject] + // Add inflight count in this batch and for this stream. + totalMsgsForSubject += i.ops + if i, ok = mset.inflight[subject]; ok { + totalMsgsForSubject += i.ops + } + if totalMsgsForSubject > uint64(maxMsgsPer) { + err = ErrMaxMsgsPerSubject + return hdr, msg, 0, NewJSStreamStoreFailedError(err, Unless(err)), err + } + } + } + + return hdr, msg, 0, nil, nil +} + +// recalculateClusteredSeq initializes or updates mset.clseq, for example after a leader change. +// This is reused for normal clustered publishing into a stream, and for atomic and fast batch publishing. +// mset.clMu lock must be held. +func recalculateClusteredSeq(mset *stream, needStreamLock bool) (lseq uint64) { + // Need to unlock and re-acquire the locks in the proper order. + // Locking order is stream -> clMu + if needStreamLock { + mset.clMu.Unlock() + mset.mu.RLock() + mset.clMu.Lock() + } + // Re-capture + lseq = mset.lseq + mset.clseq = lseq + mset.clfs + // Keep hold of the mset.clMu, but unlock the others. + if needStreamLock { + mset.mu.RUnlock() + } + return lseq +} + +// commitSingleMsg commits and proposes a single message to the node. +// This is reused both for normal publishing into a stream, and for fast batch publishing. +// mset.clMu lock must be held. +func commitSingleMsg( + diff *batchStagedDiff, mset *stream, subject string, reply string, hdr []byte, msg []byte, name string, + jsa *jsAccount, mt *msgTrace, node RaftNode, term uint64, replicas int, lseq uint64, +) error { + // Do proposal. + esm := encodeStreamMsgAllowCompress(subject, reply, hdr, msg, mset.clseq, time.Now().UnixNano(), false) + if err := node.Propose(term, esm); err != nil { + return err + } + + var mtKey uint64 + if mt != nil { + mtKey = mset.clseq + if mset.mt == nil { + mset.mt = make(map[uint64]*msgTrace) + } + mset.mt[mtKey] = mt + } + + diff.commit(mset) + mset.clseq++ + mset.trackReplicationTraffic(node, len(esm), replicas) + + // Check to see if we are being overrun. + // TODO(dlc) - Make this a limit where we drop messages to protect ourselves, but allow to be configured. + if mset.clseq-(lseq+mset.clfs) > streamLagWarnThreshold { + lerr := fmt.Errorf("JetStream stream '%s > %s' has high message lag", jsa.acc().Name, name) + mset.srv.RateLimitWarnf("%s", lerr.Error()) + } + return nil +} diff --git a/server/jetstream_benchmark_test.go b/server/jetstream_benchmark_test.go new file mode 100644 index 0000000..49a0d26 --- /dev/null +++ b/server/jetstream_benchmark_test.go @@ -0,0 +1,2494 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !skip_js_tests && !skip_js_cluster_tests && !skip_js_cluster_tests_2 + +package server + +import ( + "encoding/json" + "fmt" + "math" + "math/bits" + "math/rand/v2" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +func BenchmarkJetStreamConsume(b *testing.B) { + const ( + verbose = false + streamName = "S" + subject = "s" + seed = 42 + publishTimeout = 30 * time.Second + // Publishing is setup for this benchmark. Keep batches small enough to + // avoid overwhelming replicated streams on slower machines. + publishBatchSize = 1000 + ) + + runSyncPushConsumer := func(b *testing.B, js nats.JetStreamContext, streamName string) (int, int, int) { + const nextMsgTimeout = 3 * time.Second + + subOpts := []nats.SubOpt{ + nats.BindStream(streamName), + } + sub, err := js.SubscribeSync(_EMPTY_, subOpts...) + if err != nil { + b.Fatalf("Failed to subscribe: %v", err) + } + defer sub.Unsubscribe() + + bitset := NewBitset(uint64(b.N)) + uniqueConsumed, duplicates, errors := 0, 0, 0 + + b.ResetTimer() + + for uniqueConsumed < b.N { + msg, err := sub.NextMsg(nextMsgTimeout) + if err != nil { + b.Fatalf("No more messages (received: %d/%d)", uniqueConsumed, b.N) + } + + metadata, mdErr := msg.Metadata() + if mdErr != nil { + errors++ + continue + } + + ackErr := msg.Ack() + if ackErr != nil { + errors++ + continue + } + + seq := metadata.Sequence.Stream + + index := seq - 1 + if bitset.get(index) { + duplicates++ + continue + } + + uniqueConsumed++ + bitset.set(index, true) + + if verbose && uniqueConsumed%1000 == 0 { + b.Logf("Consumed: %d/%d", bitset.count(), b.N) + } + } + + b.StopTimer() + + return uniqueConsumed, duplicates, errors + } + + runAsyncPushConsumer := func(b *testing.B, js nats.JetStreamContext, streamName string, ordered, durable bool) (int, int, int) { + const timeout = 3 * time.Minute + bitset := NewBitset(uint64(b.N)) + doneCh := make(chan bool, 1) + uniqueConsumed, duplicates, errors := 0, 0, 0 + + handleMsg := func(msg *nats.Msg) { + metadata, mdErr := msg.Metadata() + if mdErr != nil { + // fmt.Printf("Metadata error: %v\n", mdErr) + errors++ + return + } + + // Ordered defaults to AckNone policy, don't try to ACK + if !ordered { + ackErr := msg.Ack() + if ackErr != nil { + // fmt.Printf("Ack error: %v\n", ackErr) + errors++ + return + } + } + + seq := metadata.Sequence.Stream + + index := seq - 1 + if bitset.get(index) { + duplicates++ + return + } + + uniqueConsumed++ + bitset.set(index, true) + + if uniqueConsumed == b.N { + msg.Sub.Unsubscribe() + doneCh <- true + } + if verbose && uniqueConsumed%1000 == 0 { + b.Logf("Consumed %d/%d", uniqueConsumed, b.N) + } + } + + subOpts := []nats.SubOpt{ + nats.BindStream(streamName), + } + + if ordered { + subOpts = append(subOpts, nats.OrderedConsumer()) + } + + if durable { + subOpts = append(subOpts, nats.Durable("c")) + } + + sub, err := js.Subscribe(_EMPTY_, handleMsg, subOpts...) + if err != nil { + b.Fatalf("Failed to subscribe: %v", err) + } + defer sub.Unsubscribe() + + b.ResetTimer() + + select { + case <-doneCh: + b.StopTimer() + case <-time.After(timeout): + b.Fatalf("Timeout, %d/%d received, %d errors", uniqueConsumed, b.N, errors) + } + + return uniqueConsumed, duplicates, errors + } + + runPullConsumer := func(b *testing.B, js nats.JetStreamContext, streamName string, durable bool) (int, int, int) { + const fetchMaxWait = nats.MaxWait(3 * time.Second) + const fetchMaxMessages = 1000 + + bitset := NewBitset(uint64(b.N)) + uniqueConsumed, duplicates, errors := 0, 0, 0 + + subOpts := []nats.SubOpt{ + nats.BindStream(streamName), + } + + consumerName := _EMPTY_ // Default ephemeral + if durable { + consumerName = "c" // Durable + } + + sub, err := js.PullSubscribe("", consumerName, subOpts...) + if err != nil { + b.Fatalf("Failed to subscribe: %v", err) + } + defer sub.Unsubscribe() + + b.ResetTimer() + + fetchLoop: + for { + msgs, err := sub.Fetch(fetchMaxMessages, fetchMaxWait) + if err != nil { + b.Fatalf("Failed to fetch: %v", err) + } + + processMsgsLoop: + for _, msg := range msgs { + metadata, mdErr := msg.Metadata() + if mdErr != nil { + errors++ + continue processMsgsLoop + } + + ackErr := msg.Ack() + if ackErr != nil { + errors++ + continue processMsgsLoop + } + + seq := metadata.Sequence.Stream + + index := seq - 1 + if bitset.get(index) { + duplicates++ + continue processMsgsLoop + } + + uniqueConsumed++ + bitset.set(index, true) + + if uniqueConsumed == b.N { + msg.Sub.Unsubscribe() + break fetchLoop + } + + if verbose && uniqueConsumed%1000 == 0 { + b.Logf("Consumed %d/%d", uniqueConsumed, b.N) + } + } + } + + b.StopTimer() + + return uniqueConsumed, duplicates, errors + } + + type ConsumerType string + const ( + PushSync ConsumerType = "PUSH[Sync,Ephemeral]" + PushAsync ConsumerType = "PUSH[Async,Ephemeral]" + PushAsyncOrdered ConsumerType = "PUSH[Async,Ordered]" + PushAsyncDurable ConsumerType = "PUSH[Async,Durable]" + PullDurable ConsumerType = "PULL[Durable]" + PullEphemeral ConsumerType = "PULL[Ephemeral]" + ) + + benchmarksCases := []struct { + clusterSize int + replicas int + messageSize int + minMessages int + }{ + {1, 1, 10, 100_000}, // Single node, 10B messages, ~1MiB minimum + {1, 1, 1024, 1_000}, // Single node, 1KB messages, ~1MiB minimum + {3, 3, 10, 100_000}, // Cluster, R3, 10B messages, ~1MiB minimum + {3, 3, 1024, 1_000}, // Cluster, R3, 1KB messages, ~1MiB minimum + } + + // Each of the cases above is run with each of the consumer types + consumerTypes := []ConsumerType{ + PushSync, + PushAsync, + PushAsyncOrdered, + PushAsyncDurable, + PullDurable, + PullEphemeral, + } + + for _, bc := range benchmarksCases { + + name := fmt.Sprintf( + "N=%d,R=%d,MsgSz=%db", + bc.clusterSize, + bc.replicas, + bc.messageSize, + ) + + b.Run( + name, + func(b *testing.B) { + for _, ct := range consumerTypes { + name := fmt.Sprintf( + "%v", + ct, + ) + b.Run( + name, + func(b *testing.B) { + // Skip short runs, benchmark gets re-executed with a larger N + if b.N < bc.minMessages { + b.ResetTimer() + return + } + + if verbose { + b.Logf("Running %s with %d messages", name, b.N) + } + + if verbose { + b.Logf("Setting up %d nodes", bc.clusterSize) + } + + cl, _, shutdown, nc, js := startJSClusterAndConnect(b, bc.clusterSize) + defer shutdown() + defer func() { nc.Close() }() + + if verbose { + b.Logf("Creating stream with R=%d", bc.replicas) + } + streamConfig := &nats.StreamConfig{ + Name: streamName, + Subjects: []string{subject}, + Replicas: bc.replicas, + } + if _, err := js.AddStream(streamConfig); err != nil { + b.Fatalf("Error creating stream: %v", err) + } + + // If replicated resource, connect to stream leader for lower variability + if bc.replicas > 1 { + connectURL := cl.streamLeader("$G", streamName).ClientURL() + nc.Close() + nc, _ = jsClientConnectURL(b, connectURL) + } + + js, err := nc.JetStream(nats.PublishAsyncMaxPending(publishBatchSize)) + if err != nil { + b.Fatalf("Failed to create JetStream context: %v", err) + } + + message := make([]byte, bc.messageSize) + rand.NewChaCha8([32]byte{seed}).Read(message) + + // Publish b.N messages to the stream (in batches) + for i := 1; i <= b.N; i++ { + fastRandomMutation(message, 10) + _, err := js.PublishAsync(subject, message) + if err != nil { + b.Fatalf("Failed to publish: %s", err) + } + // Drain each batch before publishing more messages. + if i%publishBatchSize == 0 || i == b.N { + select { + case <-js.PublishAsyncComplete(): + if verbose { + b.Logf("Published %d/%d messages", i, b.N) + } + case <-time.After(publishTimeout): + b.Fatalf("Publish timed out with %d acknowledgements pending after publishing %d/%d messages", js.PublishAsyncPending(), i, b.N) + } + } + } + + // Set size of each operation, for throughput calculation + b.SetBytes(int64(bc.messageSize)) + + // Discard time spent during setup + // Consumer may reset again further in + b.ResetTimer() + + var consumed, duplicates, errors int + + const ( + ordered = true + unordered = false + durable = true + ephemeral = false + ) + + switch ct { + case PushSync: + consumed, duplicates, errors = runSyncPushConsumer(b, js, streamName) + case PushAsync: + consumed, duplicates, errors = runAsyncPushConsumer(b, js, streamName, unordered, ephemeral) + case PushAsyncOrdered: + consumed, duplicates, errors = runAsyncPushConsumer(b, js, streamName, ordered, ephemeral) + case PushAsyncDurable: + consumed, duplicates, errors = runAsyncPushConsumer(b, js, streamName, unordered, durable) + case PullDurable: + consumed, duplicates, errors = runPullConsumer(b, js, streamName, durable) + case PullEphemeral: + consumed, duplicates, errors = runPullConsumer(b, js, streamName, ephemeral) + default: + b.Fatalf("Unknown consumer type: %v", ct) + } + + // Benchmark ends here, (consumer may have stopped earlier) + b.StopTimer() + + if consumed != b.N { + b.Fatalf("Something doesn't add up: %d != %d", consumed, b.N) + } + + b.ReportMetric(float64(duplicates)*100/float64(b.N), "%dupe") + b.ReportMetric(float64(errors)*100/float64(b.N), "%error") + }, + ) + } + }, + ) + } +} + +// BenchmarkJetStreamConsumeFilteredContiguous verifies the fix in +// https://github.com/nats-io/nats-server/pull/7015 and should +// capture future regressions. +func BenchmarkJetStreamConsumeFilteredContiguous(b *testing.B) { + clusterSizeCases := []struct { + clusterSize int // Single node or cluster + replicas int // Stream replicas + storage nats.StorageType // Stream storage + filters int // How many subject filters? + }{ + {1, 1, nats.MemoryStorage, 1}, + {1, 1, nats.MemoryStorage, 2}, + {3, 3, nats.MemoryStorage, 1}, + {3, 3, nats.MemoryStorage, 2}, + {1, 1, nats.FileStorage, 1}, + {1, 1, nats.FileStorage, 2}, + {3, 3, nats.FileStorage, 1}, + {3, 3, nats.FileStorage, 2}, + } + + for _, cs := range clusterSizeCases { + name := fmt.Sprintf( + "N=%d,R=%d,storage=%s", + cs.clusterSize, + cs.replicas, + cs.storage.String(), + ) + if cs.filters != 2 { // historical default is 2 + name = name + ",SF" + } + b.Run(name, func(b *testing.B) { + _, _, shutdown, nc, js := startJSClusterAndConnect(b, cs.clusterSize) + defer shutdown() + defer nc.Close() + + msgs := b.N + payload := make([]byte, 1024) + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "test", + Subjects: []string{"foo"}, + Retention: nats.LimitsPolicy, + Storage: cs.storage, + Replicas: cs.replicas, + }) + require_NoError(b, err) + + for range msgs { + _, err = js.Publish("foo", payload) + require_NoError(b, err) + } + + // Subject filters deliberately vary from the stream, ensures that we hit + // the right paths in the filestore, rather than detecting 1:1 overlap. + ocfg := &nats.ConsumerConfig{ + Name: "test_consumer", + DeliverPolicy: nats.DeliverAllPolicy, + AckPolicy: nats.AckNonePolicy, + Replicas: cs.replicas, + MemoryStorage: true, + } + switch cs.filters { + case 1: + ocfg.FilterSubject = "foo" + case 2: + ocfg.FilterSubjects = []string{"foo", "bar"} + } + _, err = js.AddConsumer("test", ocfg) + require_NoError(b, err) + + ps, err := js.PullSubscribe("foo", _EMPTY_, nats.Bind("test", "test_consumer")) + require_NoError(b, err) + + b.SetBytes(int64(len(payload))) + b.ReportAllocs() + b.ResetTimer() + for range msgs { + msgs, err := ps.Fetch(1) + require_NoError(b, err) + require_Len(b, len(msgs), 1) + } + b.StopTimer() + }) + } +} + +func BenchmarkJetStreamConsumeWithFilters(b *testing.B) { + const ( + verbose = false + streamName = "S" + subjectPrefix = "s" + seed = 42 + messageSize = 32 + consumerReplicas = 1 + domainNameLength = 36 // Length of domain portion of subject, must be an even number + publishBatchSize = 1000 + publishTimeout = 10 * time.Second + ) + + clusterSizeCases := []struct { + clusterSize int // Single node or cluster + replicas int // Stream replicas + storage nats.StorageType // Stream storage + }{ + {1, 1, nats.MemoryStorage}, + {3, 3, nats.MemoryStorage}, + {1, 1, nats.FileStorage}, + {3, 3, nats.FileStorage}, + } + + benchmarksCases := []struct { + domains int // Number of distinct domains + subjectsPerDomain int // Number of distinct subjects within each domain + filters int // Number of filters (..>) per consumer + concurrentConsumers int // Number of consumer running + }{ + {100, 10, 5, 12}, + {1000, 10, 25, 12}, + {10_000, 10, 50, 12}, + } + + for _, cs := range clusterSizeCases { + name := fmt.Sprintf( + "N=%d,R=%d,storage=%s", + cs.clusterSize, + cs.replicas, + cs.storage.String(), + ) + b.Run( + name, + func(b *testing.B) { + for _, bc := range benchmarksCases { + + name := fmt.Sprintf( + "D=%d,DS=%d,F=%d,C=%d", + bc.domains, + bc.subjectsPerDomain, + bc.filters, + bc.concurrentConsumers, + ) + + b.Run( + name, + func(b *testing.B) { + cl, s, shutdown, nc, js := startJSClusterAndConnect(b, cs.clusterSize) + defer shutdown() + defer nc.Close() + + if verbose { + b.Logf("Creating stream with R=%d", cs.replicas) + } + streamConfig := &nats.StreamConfig{ + Name: streamName, + Subjects: []string{subjectPrefix + ".>"}, + Storage: cs.storage, + Retention: nats.LimitsPolicy, + MaxAge: time.Hour, + Duplicates: 10 * time.Second, + Discard: nats.DiscardOld, + NoAck: false, + MaxMsgs: -1, + MaxBytes: -1, + MaxConsumers: -1, + Replicas: 1, + MaxMsgsPerSubject: 1, + } + if _, err := js.AddStream(streamConfig); err != nil { + b.Fatalf("Error creating stream: %v", err) + } + + // If replicated resource, connect to stream leader for lower variability + connectURL := s.ClientURL() + if cs.replicas > 1 { + connectURL = cl.streamLeader("$G", streamName).ClientURL() + nc.Close() + _, js = jsClientConnectURL(b, connectURL) + } + + rng := rand.NewChaCha8([32]byte{seed}) + message := make([]byte, messageSize) + domain := make([]byte, domainNameLength/2) + + domains := make([]string, 0, bc.domains*bc.subjectsPerDomain) + + // Publish one message per subject for each domain + published := 0 + totalMessages := bc.domains * bc.subjectsPerDomain + for d := 1; d <= bc.domains; d++ { + rng.Read(domain) + for s := 1; s <= bc.subjectsPerDomain; s++ { + rng.Read(message) + domainString := fmt.Sprintf("%X", domain) + domains = append(domains, domainString) + subject := fmt.Sprintf("%s.%s.%d", subjectPrefix, domainString, s) + _, err := js.PublishAsync(subject, message) + if err != nil { + b.Fatalf("failed to publish: %s", err) + } + published += 1 + + // Wait for all pending to be published before trying to publish the next batch + if published%publishBatchSize == 0 || published == totalMessages { + select { + case <-js.PublishAsyncComplete(): + if verbose { + b.Logf("Published %d/%d messages", published, totalMessages) + } + case <-time.After(publishTimeout): + b.Fatalf("Publish timed out") + } + } + + } + } + + // Number of messages that each new consumer expects to consume + messagesPerIteration := bc.filters * bc.subjectsPerDomain + + // Each call to 'subscribe_consume_unsubscribe' is one benchmark operation. + // i.e. subscribe_consume_unsubscribe will be called a total of b.N times (split among C threads) + // Each operation consists of: + // - Create filter + // - Create consumer / Subscribe + // - Consume expected number of messages + // - Unsubscribe + subscribeConsumeUnsubscribe := func(js nats.JetStreamContext, rng *rand.Rand) { + // Select F unique domains to create F non-overlapping filters + filterDomains := make(map[string]bool, bc.filters) + filters := make([]string, 0, bc.filters) + for len(filterDomains) < bc.filters { + domain := domains[rng.IntN(len(domains))] + if _, found := filterDomains[domain]; found { + // Collision with existing filter, try again + continue + } + filterDomains[domain] = true + filters = append(filters, fmt.Sprintf("%s.%s.>", subjectPrefix, domain)) + } + + if verbose { + b.Logf("Subscribe with filters: %+v", filters) + } + + // Consumer callback + received := 0 + consumeWg := sync.WaitGroup{} + consumeWg.Add(1) + cb := func(msg *nats.Msg) { + received += 1 + if received == messagesPerIteration { + consumeWg.Done() + if verbose { + b.Logf("Received %d/%d messages", received, messagesPerIteration) + } + } + } + + // Create consumer + subOpts := []nats.SubOpt{ + nats.BindStream(streamName), + nats.OrderedConsumer(), + nats.ConsumerReplicas(consumerReplicas), + nats.ConsumerFilterSubjects(filters...), + nats.ConsumerMemoryStorage(), + } + + var sub *nats.Subscription + + sub, err := js.Subscribe("", cb, subOpts...) + if err != nil { + b.Fatalf("Failed to subscribe: %s", err) + } + + defer func(sub *nats.Subscription) { + err := sub.Unsubscribe() + if err != nil { + b.Logf("Failed to unsubscribe: %s", err) + } + }(sub) + + consumeWg.Wait() + } + + // Wait for all consumer threads and main to be ready + wgReady := sync.WaitGroup{} + wgReady.Add(bc.concurrentConsumers + 1) + // Wait until all consumer threads have completed + wgCompleted := sync.WaitGroup{} + wgCompleted.Add(bc.concurrentConsumers) + // Operations left for consumer threads + opsCount := atomic.Int32{} + opsCount.Store(int32(b.N)) + + // Start a pool of C goroutines, each one with a dedicated connection. + for i := 1; i <= bc.concurrentConsumers; i++ { + go func(consumerId int) { + // Connect + nc, js := jsClientConnectURL(b, connectURL) + defer nc.Close() + + // Signal completion of work + defer wgCompleted.Done() + + rng := rand.New(rand.NewPCG(seed, uint64(consumerId))) + // Ready, wait for everyone else + wgReady.Done() + wgReady.Wait() + + completed := 0 + for opsCount.Add(-1) >= 0 { + subscribeConsumeUnsubscribe(js, rng) + completed += 1 + } + if verbose { + b.Logf("Consumer thread %d completed %d of %d operations", consumerId, completed, b.N) + } + }(i) + } + + // Wait for all consumers to be ready + wgReady.Done() + wgReady.Wait() + + // Start measuring time + b.ResetTimer() + + // Wait for consumers to have chewed through b.N operations + wgCompleted.Wait() + b.StopTimer() + + // Throughput is not very important in this benchmark since each operation includes + // subscribe, unsubscribe and retrieves just a few bytes + // b.SetBytes(int64(messageSize * messagesPerIteration)) + }, + ) + } + }, + ) + } +} + +func BenchmarkJetStreamPublish(b *testing.B) { + const ( + verbose = false + seed = 42 + streamName = "S" + ) + + runSyncPublisher := func(b *testing.B, js nats.JetStreamContext, messageSize int, subjects []string) (int, int) { + published, errors := 0, 0 + message := make([]byte, messageSize) + rand.NewChaCha8([32]byte{seed}).Read(message) + + b.ResetTimer() + + for i := 1; i <= b.N; i++ { + fastRandomMutation(message, 10) + subject := subjects[rand.Uint32N(uint32(len(subjects)))] + _, pubErr := js.Publish(subject, message) + if pubErr != nil { + errors++ + } else { + published++ + } + + if verbose && i%1000 == 0 { + b.Logf("Published %d/%d, %d errors", i, b.N, errors) + } + } + + b.StopTimer() + + return published, errors + } + + runAsyncPublisher := func(b *testing.B, js nats.JetStreamContext, messageSize int, subjects []string, asyncWindow int) (int, int) { + const publishCompleteMaxWait = 30 * time.Second + message := make([]byte, messageSize) + src := rand.NewChaCha8([32]byte{seed}) + src.Read(message) + rng := rand.New(src) + + published, errors := 0, 0 + + b.ResetTimer() + + for published < b.N { + + // Normally publish a full batch (of size `asyncWindow`) + publishBatchSize := asyncWindow + // Unless fewer are left to complete the benchmark + if b.N-published < asyncWindow { + publishBatchSize = b.N - published + } + + pending := make([]nats.PubAckFuture, 0, publishBatchSize) + + for i := 0; i < publishBatchSize; i++ { + fastRandomMutation(message, 10) + subject := subjects[rng.IntN(len(subjects))] + pubAckFuture, err := js.PublishAsync(subject, message) + if err != nil { + errors++ + continue + } + pending = append(pending, pubAckFuture) + } + + // All in this batch published, wait for completed + select { + case <-js.PublishAsyncComplete(): + case <-time.After(publishCompleteMaxWait): + b.Fatalf("Publish timed out") + } + + // Verify one by one if they were published successfully + for _, pubAckFuture := range pending { + select { + case <-pubAckFuture.Ok(): + published++ + case <-pubAckFuture.Err(): + errors++ + default: + b.Fatalf("PubAck is still pending after publish completed") + } + } + + if verbose { + b.Logf("Published %d/%d", published, b.N) + } + } + + b.StopTimer() + + return published, errors + } + + type PublishType string + const ( + Sync PublishType = "Sync" + Async PublishType = "Async" + ) + + benchmarksCases := []struct { + clusterSize int + replicas int + messageSize int + numSubjects int + }{ + {1, 1, 10, 1}, // Single node, 10B messages + {1, 1, 1024, 1}, // Single node, 1KB messages + {3, 3, 10, 1}, // 3-nodes cluster, R=3, 10B messages + {3, 3, 1024, 1}, // 3-nodes cluster, R=3, 1KB messages + {3, 3, 10, 1}, // 3-nodes cluster, R=3, 10B messages (async flush) + {3, 3, 1024, 1}, // 3-nodes cluster, R=3, 1KB messages (async flush) + } + + // All the cases above are run with each of the publisher cases below + publisherCases := []struct { + pType PublishType + asyncWindow int + }{ + {Sync, -1}, + {Async, 1000}, + {Async, 4000}, + {Async, 8000}, + } + + for _, bc := range benchmarksCases { + name := fmt.Sprintf( + "N=%d,R=%d,MsgSz=%db,Subjs=%d", + bc.clusterSize, + bc.replicas, + bc.messageSize, + bc.numSubjects, + ) + b.Run( + name, + func(b *testing.B) { + for _, pc := range publisherCases { + name := fmt.Sprintf("%v", pc.pType) + if pc.pType == Async && pc.asyncWindow > 0 { + name = fmt.Sprintf("%s[W:%d]", name, pc.asyncWindow) + } + + b.Run( + name, + func(b *testing.B) { + subjects := make([]string, bc.numSubjects) + for i := 0; i < bc.numSubjects; i++ { + subjects[i] = fmt.Sprintf("s-%d", i+1) + } + + if verbose { + b.Logf("Running %s with %d ops", name, b.N) + } + + if verbose { + b.Logf("Setting up %d nodes", bc.clusterSize) + } + + cl, _, shutdown, nc, _ := startJSClusterAndConnect(b, bc.clusterSize) + defer shutdown() + defer nc.Close() + + jsOpts := []nats.JSOpt{ + nats.MaxWait(10 * time.Second), + } + + if pc.asyncWindow > 0 && pc.pType == Async { + jsOpts = append(jsOpts, nats.PublishAsyncMaxPending(pc.asyncWindow)) + } + + js, err := nc.JetStream(jsOpts...) + if err != nil { + b.Fatalf("Unexpected error getting JetStream context: %v", err) + } + + if verbose { + b.Logf("Creating stream with R=%d and %d input subjects", bc.replicas, bc.numSubjects) + } + _, err = jsStreamCreate(b, nc, &StreamConfig{ + Name: streamName, + Subjects: subjects, + Replicas: bc.replicas, + Storage: FileStorage, + }) + if err != nil { + b.Fatalf("Error creating stream: %v", err) + } + + // If replicated resource, connect to stream leader for lower variability + if bc.replicas > 1 { + connectURL := cl.streamLeader("$G", streamName).ClientURL() + nc.Close() + nc, err = nats.Connect(connectURL) + if err != nil { + b.Fatalf("Failed to create client connection to stream leader: %v", err) + } + defer nc.Close() + js, err = nc.JetStream(jsOpts...) + if err != nil { + b.Fatalf("Unexpected error getting JetStream context for stream leader: %v", err) + } + } + + if verbose { + b.Logf("Running %v publisher with message size: %dB", pc.pType, bc.messageSize) + } + + b.SetBytes(int64(bc.messageSize)) + + // Benchmark starts here + b.ResetTimer() + + var published, errors int + switch pc.pType { + case Sync: + published, errors = runSyncPublisher(b, js, bc.messageSize, subjects) + case Async: + published, errors = runAsyncPublisher(b, js, bc.messageSize, subjects, pc.asyncWindow) + } + + // Benchmark ends here + b.StopTimer() + + if published+errors != b.N { + b.Fatalf("Something doesn't add up: %d + %d != %d", published, errors, b.N) + } + + b.ReportMetric(float64(errors)*100/float64(b.N), "%error") + }, + ) + } + }, + ) + } +} + +func BenchmarkJetStreamMetaSnapshot(b *testing.B) { + c := createJetStreamClusterExplicit(b, "R3S", 3) + defer c.shutdown() + + setup := func(reqLevel string) *jetStream { + ml := c.leader() + acc, js := ml.globalAccount(), ml.getJetStream() + n := js.getMetaGroup() + + // Create all streams and consumers. + numStreams := 200 + numConsumers := 500 + ci := &ClientInfo{Cluster: "R3S", Account: globalAccountName} + js.mu.Lock() + metadata := map[string]string{JSRequiredLevelMetadataKey: reqLevel} + for i := 0; i < numStreams; i++ { + scfg := &StreamConfig{ + Name: fmt.Sprintf("STREAM-%d", i), + Subjects: []string{fmt.Sprintf("SUBJECT-%d", i)}, + Storage: MemoryStorage, + Metadata: metadata, + } + cfg, _ := ml.checkStreamCfgLocked(scfg, acc, false) + rg, _ := js.createGroupForStream(ci, &cfg) + sa := &streamAssignment{Group: rg, Sync: syncSubjForStream(), Config: &cfg, Client: ci, Created: time.Now().UTC()} + n.Propose(n.Term(), encodeAddStreamAssignment(sa)) + + for j := 0; j < numConsumers; j++ { + ccfg := &ConsumerConfig{ + Durable: fmt.Sprintf("CONSUMER-%d", j), + MemoryStorage: true, + Metadata: metadata, + } + selectedLimits, _, _, _ := acc.selectLimits(ccfg.replicas(&cfg)) + srvLim := &ml.getOpts().JetStreamLimits + setConsumerConfigDefaults(ccfg, &cfg, srvLim, selectedLimits, false) + rg, _ = js.cluster.createGroupForConsumer(ccfg, sa) + ca := &consumerAssignment{Group: rg, Stream: cfg.Name, Name: ccfg.Durable, Config: ccfg, Client: ci, Created: time.Now().UTC()} + n.Propose(n.Term(), encodeAddConsumerAssignment(ca)) + } + } + js.mu.Unlock() + + // Wait for all servers to have created all assets. + checkFor(b, 20*time.Second, 200*time.Millisecond, func() error { + for _, s := range c.servers { + sjs := s.getJetStream() + sjs.mu.RLock() + streams := sjs.cluster.streams[globalAccountName] + if len(streams) != numStreams { + sjs.mu.RUnlock() + return fmt.Errorf("expected %d streams, got %d", numStreams, len(streams)) + } + for _, sa := range streams { + if nc := len(sa.consumers); nc != numConsumers { + sjs.mu.RUnlock() + return fmt.Errorf("expected %d consumers, got %d", numConsumers, nc) + } + } + sjs.mu.RUnlock() + } + return nil + }) + return js + } + + for _, t := range []struct { + title string + reqLevel string + }{ + {title: "Default", reqLevel: "0"}, + {title: "AllUnsupported", reqLevel: strconv.Itoa(math.MaxInt)}, + } { + b.Run(t.title, func(b *testing.B) { + js := setup(t.reqLevel) + b.ResetTimer() + for range b.N { + js.metaSnapshot() + } + b.StopTimer() + }) + } +} + +func BenchmarkJetStreamCounters(b *testing.B) { + const ( + verbose = false + seed = 12345 + streamName = "S" + ) + + // We don't actually create real sourcing streams, we just populate + // the Nats-Counter-Sources header to make it look like we have some, + // so that we can see how the code performs for bringing forward the + // latest headers each time. + generateSources := func(t testing.TB, count int) string { + t.Helper() + sources := CounterSources{} + for i := range count { + streamName := fmt.Sprintf("STREAM_%d", i%10) + subjectName := fmt.Sprintf("subject.%d", i) + if sources[streamName] == nil { + sources[streamName] = map[string]string{} + } + sources[streamName][subjectName] = "12345" + } + j, err := json.Marshal(sources) + require_NoError(t, err) + return string(j) + } + + runSyncPublisher := func(b *testing.B, js nats.JetStreamContext, subjects []string, sources int) (int, int) { + published, errors := 0, 0 + msg := &nats.Msg{ + Header: nats.Header{}, + } + msg.Header.Set(JSMessageIncr, "1") + if sources > 0 { + msg.Header.Set(JSMessageCounterSources, generateSources(b, sources)) + } + b.ResetTimer() + + for i := 1; i <= b.N; i++ { + msg.Subject = subjects[rand.Uint32N(uint32(len(subjects)))] + if _, pubErr := js.PublishMsg(msg); pubErr != nil { + errors++ + } else { + published++ + } + + if verbose && i%1000 == 0 { + b.Logf("Published %d/%d, %d errors", i, b.N, errors) + } + } + + b.StopTimer() + return published, errors + } + + runAsyncPublisher := func(b *testing.B, js nats.JetStreamContext, subjects []string, sources int, asyncWindow int) (int, int) { + const publishCompleteMaxWait = 30 * time.Second + msg := &nats.Msg{ + Header: nats.Header{}, + } + msg.Header.Set(JSMessageIncr, "1") + if sources > 0 { + msg.Header.Set(JSMessageCounterSources, generateSources(b, sources)) + } + published, errors := 0, 0 + b.ResetTimer() + + for published < b.N { + // Normally publish a full batch (of size `asyncWindow`) + publishBatchSize := min(b.N-published, asyncWindow) + pending := make([]nats.PubAckFuture, 0, publishBatchSize) + + for range publishBatchSize { + msg.Subject = subjects[rand.Uint32N(uint32(len(subjects)))] + pubAckFuture, err := js.PublishMsgAsync(msg) + if err != nil { + errors++ + continue + } + pending = append(pending, pubAckFuture) + } + + // All in this batch published, wait for completed + select { + case <-js.PublishAsyncComplete(): + case <-time.After(publishCompleteMaxWait): + b.Fatalf("Publish timed out") + } + + // Verify one by one if they were published successfully + for _, pubAckFuture := range pending { + select { + case <-pubAckFuture.Ok(): + published++ + case <-pubAckFuture.Err(): + errors++ + default: + b.Fatalf("PubAck is still pending after publish completed") + } + } + + if verbose { + b.Logf("Published %d/%d", published, b.N) + } + } + + b.StopTimer() + return published, errors + } + + type PublishType string + const ( + Sync PublishType = "Sync" + Async PublishType = "Async" + ) + + type benchmarksCase struct { + storageType StorageType + clusterSize int + replicas int + numSubjects int + sources int + } + var benchmarksCases []benchmarksCase + for _, storage := range []StorageType{FileStorage, MemoryStorage} { + for _, replicas := range []int{1, 3} { + for _, numSubjects := range []int{1, 1000} { + for _, sources := range []int{0, 10, 25, 250} { + benchmarksCases = append(benchmarksCases, benchmarksCase{ + storageType: storage, + clusterSize: 3, + replicas: replicas, + numSubjects: numSubjects, + sources: sources, + }) + } + } + } + } + + // All the cases above are run with each of the publisher cases below + publisherCases := []struct { + pType PublishType + asyncWindow int + }{ + {Sync, -1}, + {Async, 1000}, + {Async, 4000}, + {Async, 8000}, + } + + for _, bc := range benchmarksCases { + name := fmt.Sprintf( + "S=%s,N=%d,R=%d,Subjs=%d,Srcs=%d", + bc.storageType, + bc.clusterSize, + bc.replicas, + bc.numSubjects, + bc.sources, + ) + + b.Run(name, func(b *testing.B) { + for _, pc := range publisherCases { + name := fmt.Sprintf("%v", pc.pType) + if pc.pType == Async && pc.asyncWindow > 0 { + name = fmt.Sprintf("%s[W:%d]", name, pc.asyncWindow) + } + + b.Run(name, func(b *testing.B) { + subjects := make([]string, bc.numSubjects) + for i := range bc.numSubjects { + subjects[i] = fmt.Sprintf("s-%d", i+1) + } + + if verbose { + b.Logf("Running %s with %d ops", name, b.N) + } + + if verbose { + b.Logf("Setting up %d nodes", bc.clusterSize) + } + + cl, _, shutdown, nc, _ := startJSClusterAndConnect(b, bc.clusterSize) + defer shutdown() + defer nc.Close() + + jsOpts := []nats.JSOpt{ + nats.MaxWait(10 * time.Second), + } + + if pc.asyncWindow > 0 && pc.pType == Async { + jsOpts = append(jsOpts, nats.PublishAsyncMaxPending(pc.asyncWindow)) + } + + js, err := nc.JetStream(jsOpts...) + if err != nil { + b.Fatalf("Unexpected error getting JetStream context: %v", err) + } + + if verbose { + b.Logf("Creating stream with R=%d and %d input subjects", bc.replicas, bc.numSubjects) + } + if _, err := jsStreamCreate(b, nc, &StreamConfig{ + Name: streamName, + Storage: bc.storageType, + Subjects: subjects, + Replicas: bc.replicas, + AllowMsgCounter: true, + }); err != nil { + b.Fatalf("Error creating stream: %v", err) + } + + // If replicated resource, connect to stream leader for lower variability + if bc.replicas > 1 { + connectURL := cl.streamLeader("$G", streamName).ClientURL() + nc.Close() + nc, err = nats.Connect(connectURL) + if err != nil { + b.Fatalf("Failed to create client connection to stream leader: %v", err) + } + defer nc.Close() + js, err = nc.JetStream(jsOpts...) + if err != nil { + b.Fatalf("Unexpected error getting JetStream context for stream leader: %v", err) + } + } + + if verbose { + b.Logf("Running %v publisher", pc.pType) + } + + // Benchmark starts here + b.ResetTimer() + + var published, errors int + switch pc.pType { + case Sync: + published, errors = runSyncPublisher(b, js, subjects, bc.sources) + case Async: + published, errors = runAsyncPublisher(b, js, subjects, bc.sources, pc.asyncWindow) + } + + // Benchmark ends here + b.StopTimer() + + if published+errors != b.N { + b.Fatalf("Something doesn't add up: %d + %d != %d", published, errors, b.N) + } + + b.ReportMetric(float64(errors)*100/float64(b.N), "%error") + }) + } + }) + } +} + +func BenchmarkJetStreamInterestStreamWithLimit(b *testing.B) { + const ( + verbose = true + seed = 42 + publishBatchSize = 100 + messageSize = 256 + numSubjects = 2500 + subjectPrefix = "S" + numPublishers = 4 + randomData = true + warmupMessages = 1 + ) + + if verbose { + b.Logf( + "BatchSize: %d, MsgSize: %d, Subjects: %d, Publishers: %d, Random Message: %v", + publishBatchSize, + messageSize, + numSubjects, + numPublishers, + randomData, + ) + } + + // Benchmark parameters: sub-benchmarks are executed for every combination of the following 3 groups + // Unless a more restrictive filter is specified, e.g.: + // BenchmarkJetStreamInterestStreamWithLimit/.*R=3.*/Storage=Memory/unlimited + + // Parameter: Number of nodes and number of stream replicas + clusterAndReplicasCases := []struct { + clusterSize int + replicas int + }{ + {1, 1}, // Single node, R=1 + {3, 3}, // 3-nodes cluster, R=3 + } + + // Parameter: Stream storage type + storageTypeCases := []nats.StorageType{ + nats.MemoryStorage, + nats.FileStorage, + } + + // Parameter: Stream limit configuration + limitConfigCases := map[string]func(*nats.StreamConfig){ + "unlimited": func(config *nats.StreamConfig) { + }, + "MaxMsg=1000": func(config *nats.StreamConfig) { + config.MaxMsgs = 100 + }, + "MaxMsg=10": func(config *nats.StreamConfig) { + config.MaxMsgs = 10 + }, + "MaxPerSubject=10": func(config *nats.StreamConfig) { + config.MaxMsgsPerSubject = 10 + }, + "MaxAge=1s": func(config *nats.StreamConfig) { + config.MaxAge = 1 * time.Second + }, + "MaxBytes=1MB": func(config *nats.StreamConfig) { + config.MaxBytes = 1024 * 1024 + }, + } + + // Context shared by publishers routines + type PublishersContext = struct { + readyWg sync.WaitGroup + completedWg sync.WaitGroup + messagesLeft int + lock sync.Mutex + errors int + } + + // Helper: Publish synchronously as Goroutine + publish := func(publisherId int, ctx *PublishersContext, js nats.JetStreamContext) { + defer ctx.completedWg.Done() + errors := 0 + messageBuf := make([]byte, messageSize) + rand.NewChaCha8([32]byte{0: seed, 1: byte(publisherId)}).Read(messageBuf) + + // Warm up: publish a few messages + for i := 0; i < warmupMessages; i++ { + subject := fmt.Sprintf("%s.%d", subjectPrefix, rand.Uint32N(numSubjects)) + if randomData { + fastRandomMutation(messageBuf, 10) + } + _, err := js.Publish(subject, messageBuf) + if err != nil { + b.Logf("Warning: failed to publish warmup message: %s", err) + } + } + + // Signal this publisher is ready + ctx.readyWg.Done() + + for { + // Obtain a batch of messages to publish + batchSize := 0 + { + ctx.lock.Lock() + if ctx.messagesLeft >= publishBatchSize { + batchSize = publishBatchSize + } else if ctx.messagesLeft < publishBatchSize { + batchSize = ctx.messagesLeft + } + ctx.messagesLeft -= batchSize + ctx.lock.Unlock() + } + + // Nothing left to publish, terminate + if batchSize == 0 { + ctx.lock.Lock() + ctx.errors += errors + ctx.lock.Unlock() + return + } + + // Publish a batch of messages + for i := 0; i < batchSize; i++ { + subject := fmt.Sprintf("%s.%d", subjectPrefix, rand.Uint32N(numSubjects)) + if randomData { + fastRandomMutation(messageBuf, 10) + } + _, err := js.Publish(subject, messageBuf) + if err != nil { + errors += 1 + } + } + } + } + + // Benchmark matrix: (cluster and replicas) * (storage type) * (stream limit) + for _, benchmarkCase := range clusterAndReplicasCases { + b.Run( + fmt.Sprintf( + "N=%d,R=%d", + benchmarkCase.clusterSize, + benchmarkCase.replicas, + ), + func(b *testing.B) { + for _, storageType := range storageTypeCases { + b.Run( + fmt.Sprintf("Storage=%v", storageType), + func(b *testing.B) { + for limitDescription, limitConfigFunc := range limitConfigCases { + b.Run( + limitDescription, + func(b *testing.B) { + // Print benchmark parameters + if verbose { + b.Logf( + "Stream: %+v, Storage: [%v] Limit: [%s], Ops: %d", + benchmarkCase, + storageType, + limitDescription, + b.N, + ) + } + + // Setup server or cluster + cl, ls, shutdown, nc, js := startJSClusterAndConnect(b, benchmarkCase.clusterSize) + defer shutdown() + defer nc.Close() + + // Common stream configuration + streamConfig := &nats.StreamConfig{ + Name: "S", + Subjects: []string{fmt.Sprintf("%s.>", subjectPrefix)}, + Replicas: benchmarkCase.replicas, + Storage: storageType, + Discard: DiscardOld, + Retention: DiscardOld, + } + // Configure stream limit + limitConfigFunc(streamConfig) + + // Create stream + if _, err := js.AddStream(streamConfig); err != nil { + b.Fatalf("Error creating stream: %v", err) + } + + // Set up publishers shared context + var pubCtx PublishersContext + pubCtx.readyWg.Add(numPublishers) + pubCtx.completedWg.Add(numPublishers) + + // Hold this lock until all publishers are ready + pubCtx.lock.Lock() + pubCtx.messagesLeft = b.N + + connectURL := ls.ClientURL() + // If replicated resource, connect to stream leader for lower variability + if benchmarkCase.replicas > 1 { + connectURL = cl.streamLeader("$G", "S").ClientURL() + } + + // Spawn publishers routines, each with its own connection and JS context + for i := 0; i < numPublishers; i++ { + nc, err := nats.Connect(connectURL) + if err != nil { + b.Fatal(err) + } + defer nc.Close() + js, err := nc.JetStream() + if err != nil { + b.Fatal(err) + } + go publish(i, &pubCtx, js) + } + + // Wait for all publishers to be ready + pubCtx.readyWg.Wait() + + // Set size of each operation, for throughput calculation + b.SetBytes(messageSize) + + // Benchmark starts here + b.ResetTimer() + + // Unblock the publishers + pubCtx.lock.Unlock() + + // Wait for all publishers to complete + pubCtx.completedWg.Wait() + + // Benchmark ends here + b.StopTimer() + + // Sanity check, publishers may have died before completing + if pubCtx.messagesLeft != 0 { + b.Fatalf("Some messages left: %d", pubCtx.messagesLeft) + } + + b.ReportMetric(float64(pubCtx.errors)*100/float64(b.N), "%error") + }, + ) + } + }, + ) + } + }, + ) + } +} + +func BenchmarkJetStreamKV(b *testing.B) { + const ( + verbose = false + kvName = "BUCKET" + keyPrefix = "K_" + seed = 42 + ) + + runKVGet := func(b *testing.B, kv nats.KeyValue, keys []string) int { + rng := rand.New(rand.NewChaCha8([32]byte{seed})) + errors := 0 + + b.ResetTimer() + + for i := 1; i <= b.N; i++ { + key := keys[rng.IntN(len(keys))] + _, err := kv.Get(key) + if err != nil { + errors++ + continue + } + + if verbose && i%1000 == 0 { + b.Logf("Completed %d/%d Get ops", i, b.N) + } + } + + b.StopTimer() + return errors + } + + runKVPut := func(b *testing.B, kv nats.KeyValue, keys []string, valueSize int) int { + value := make([]byte, valueSize) + rand.NewChaCha8([32]byte{seed}).Read(value) + errors := 0 + + b.ResetTimer() + + for i := 1; i <= b.N; i++ { + key := keys[rand.Uint32N(uint32(len(keys)))] + fastRandomMutation(value, 10) + _, err := kv.Put(key, value) + if err != nil { + errors++ + continue + } + + if verbose && i%1000 == 0 { + b.Logf("Completed %d/%d Put ops", i, b.N) + } + } + + b.StopTimer() + return errors + } + + runKVUpdate := func(b *testing.B, kv nats.KeyValue, keys []string, valueSize int) int { + value := make([]byte, valueSize) + rand.NewChaCha8([32]byte{seed}).Read(value) + errors := 0 + b.ResetTimer() + + for i := 1; i <= b.N; i++ { + key := keys[rand.Uint32N(uint32(len(keys)))] + + kve, getErr := kv.Get(key) + if getErr != nil { + errors++ + continue + } + + fastRandomMutation(value, 10) + _, updateErr := kv.Update(key, value, kve.Revision()) + if updateErr != nil { + errors++ + continue + } + + if verbose && i%1000 == 0 { + b.Logf("Completed %d/%d Update ops", i, b.N) + } + } + + b.StopTimer() + return errors + } + + type WorkloadType string + const ( + Get WorkloadType = "GET" + Put WorkloadType = "PUT" + Update WorkloadType = "CAS" + ) + + benchmarksCases := []struct { + clusterSize int + replicas int + numKeys int + valueSize int + }{ + {1, 1, 100, 100}, // 1 node with 100 keys, 100B values + {1, 1, 1000, 100}, // 1 node with 1000 keys, 100B values + {3, 3, 100, 100}, // 3 nodes with 100 keys, 100B values + {3, 3, 1000, 100}, // 3 nodes with 1000 keys, 100B values + {3, 3, 1000, 1024}, // 3 nodes with 1000 keys, 1KB values + } + + workloadCases := []WorkloadType{ + Get, + Put, + Update, + } + + for _, bc := range benchmarksCases { + + bName := fmt.Sprintf( + "N=%d,R=%d,B=1,K=%d,ValSz=%db", + bc.clusterSize, + bc.replicas, + bc.numKeys, + bc.valueSize, + ) + + b.Run( + bName, + func(b *testing.B) { + for _, wc := range workloadCases { + wName := fmt.Sprintf("%v", wc) + b.Run( + wName, + func(b *testing.B) { + if verbose { + b.Logf("Running %s workload %s with %d messages", wName, bName, b.N) + } + + if verbose { + b.Logf("Setting up %d nodes", bc.clusterSize) + } + + // Pre-generate all keys + keys := make([]string, 0, bc.numKeys) + for i := 1; i <= bc.numKeys; i++ { + key := fmt.Sprintf("%s%d", keyPrefix, i) + keys = append(keys, key) + } + + // Setup server or cluster + cl, _, shutdown, nc, js := startJSClusterAndConnect(b, bc.clusterSize) + defer shutdown() + defer nc.Close() + + // Create bucket + if verbose { + b.Logf("Creating KV %s with R=%d", kvName, bc.replicas) + } + kvConfig := &nats.KeyValueConfig{ + Bucket: kvName, + Replicas: bc.replicas, + } + kv, err := js.CreateKeyValue(kvConfig) + if err != nil { + b.Fatalf("Error creating KV: %v", err) + } + + // Initialize all keys + rng := rand.NewChaCha8([32]byte{seed}) + value := make([]byte, bc.valueSize) + for _, key := range keys { + rng.Read(value) + _, err := kv.Create(key, value) + if err != nil { + b.Fatalf("Failed to initialize %s/%s: %v", kvName, key, err) + } + } + + // If replicated resource, connect to stream leader for lower variability + if bc.replicas > 1 { + nc.Close() + connectURL := cl.streamLeader("$G", fmt.Sprintf("KV_%s", kvName)).ClientURL() + nc, js = jsClientConnectURL(b, connectURL) + defer nc.Close() + } + + kv, err = js.KeyValue(kv.Bucket()) + if err != nil { + b.Fatalf("Error binding to KV: %v", err) + } + + // Set size of each operation, for throughput calculation + b.SetBytes(int64(bc.valueSize)) + + // Discard time spent during setup + // May reset again further in + b.ResetTimer() + + var errors int + + switch wc { + case Get: + errors = runKVGet(b, kv, keys) + case Put: + errors = runKVPut(b, kv, keys, bc.valueSize) + case Update: + errors = runKVUpdate(b, kv, keys, bc.valueSize) + default: + b.Fatalf("Unknown workload type: %v", wc) + } + + // Benchmark ends here, (may have stopped earlier) + b.StopTimer() + + b.ReportMetric(float64(errors)*100/float64(b.N), "%error") + }, + ) + } + }, + ) + } +} + +func BenchmarkJetStreamObjStore(b *testing.B) { + const ( + verbose = false + objStoreName = "B" + keyPrefix = "K_" + seed = 42 + initKeys = true + + // read/write ratios + ReadOnly = 1.0 + WriteOnly = 0.0 + ) + + // rwRatio to string + rwRatioToString := func(rwRatio float64) string { + switch rwRatio { + case ReadOnly: + return "readOnly" + case WriteOnly: + return "writeOnly" + default: + return fmt.Sprintf("%0.1f", rwRatio) + } + } + + // benchmark for object store by performing read/write operations with data of random size + RunObjStoreBenchmark := func(b *testing.B, objStore nats.ObjectStore, minObjSz int, maxObjSz int, numKeys int, rwRatio float64) (int, int, int) { + var ( + errors int + reads int + writes int + ) + + dataBuf := make([]byte, maxObjSz) + src := rand.NewChaCha8([32]byte{seed}) + src.Read(dataBuf) + + rng := rand.New(src) + + // Each operation is processing a random amount of bytes within a size range which + // will be either read from or written to an object store bucket. However, here we are + // approximating the size of the processed data with a simple average of the range. + b.SetBytes(int64((minObjSz + maxObjSz) / 2)) + + for i := 1; i <= b.N; i++ { + key := fmt.Sprintf("%s_%d", keyPrefix, rng.IntN(numKeys)) + var err error + + rwOp := rng.Float64() + switch { + case rwOp <= rwRatio: + // Read Op + _, err = objStore.GetBytes(key) + reads++ + case rwOp > rwRatio: + // Write Op + // dataSz is a random value between min-max object size and cannot be less than 1 byte + dataSz := rng.IntN(maxObjSz-minObjSz+1) + minObjSz + data := dataBuf[:dataSz] + fastRandomMutation(data, 10) + _, err = objStore.PutBytes(key, data) + writes++ + } + if err != nil { + errors++ + } + + if verbose && i%1000 == 0 { + b.Logf("Completed: %d reads, %d writes, %d errors. %d/%d total operations have been completed.", reads, writes, errors, i, b.N) + } + } + return errors, reads, writes + } + + // benchmark cases table + benchmarkCases := []struct { + storage nats.StorageType + numKeys int + minObjSz int + maxObjSz int + }{ + {nats.MemoryStorage, 100, 1024, 102400}, // mem storage, 100 objects sized (1KB-100KB) + {nats.MemoryStorage, 100, 102400, 1048576}, // mem storage, 100 objects sized (100KB-1MB) + {nats.MemoryStorage, 1000, 10240, 102400}, // mem storage, 1k objects of various size (10KB - 100KB) + {nats.FileStorage, 100, 1024, 102400}, // file storage, 100 objects sized (1KB-100KB) + {nats.FileStorage, 1000, 10240, 1048576}, // file storage, 1k objects of various size (10KB - 1MB) + {nats.FileStorage, 100, 102400, 1048576}, // file storage, 100 objects sized (100KB-1MB) + {nats.FileStorage, 100, 1048576, 10485760}, // file storage, 100 objects sized (1MB-10MB) + {nats.FileStorage, 10, 10485760, 104857600}, // file storage, 10 objects sized (10MB-100MB) + } + + var ( + clusterSizeCases = []int{1, 3} + rwRatioCases = []float64{ReadOnly, WriteOnly, 0.8} + ) + + // Test with either single node or 3 node cluster + for _, clusterSize := range clusterSizeCases { + replicas := clusterSize + cName := fmt.Sprintf("N=%d,R=%d", clusterSize, replicas) + b.Run( + cName, + func(b *testing.B) { + for _, rwRatio := range rwRatioCases { + rName := fmt.Sprintf("workload=%s", rwRatioToString(rwRatio)) + b.Run( + rName, + func(b *testing.B) { + // Test all tabled benchmark cases + for _, bc := range benchmarkCases { + bName := fmt.Sprintf("K=%d,storage=%s,minObjSz=%db,maxObjSz=%db", bc.numKeys, bc.storage, bc.minObjSz, bc.maxObjSz) + b.Run( + bName, + func(b *testing.B) { + // Test setup + if verbose { + b.Logf("Setting up %d nodes", replicas) + } + + // Setup server or cluster + cl, _, shutdown, nc, js := startJSClusterAndConnect(b, clusterSize) + defer shutdown() + defer nc.Close() + + // Initialize object store + if verbose { + b.Logf("Creating ObjectStore %s with R=%d", objStoreName, replicas) + } + objStoreConfig := &nats.ObjectStoreConfig{ + Bucket: objStoreName, + Replicas: replicas, + Storage: bc.storage, + } + objStore, err := js.CreateObjectStore(objStoreConfig) + if err != nil { + b.Fatalf("Error creating ObjectStore: %v", err) + } + + // If replicated resource, connect to stream leader for lower variability + if clusterSize > 1 { + nc.Close() + connectURL := cl.streamLeader("$G", fmt.Sprintf("OBJ_%s", objStoreName)).ClientURL() + nc, js := jsClientConnectURL(b, connectURL) + defer nc.Close() + objStore, err = js.ObjectStore(objStoreName) + if err != nil { + b.Fatalf("Error binding to ObjectStore: %v", err) + } + } + + // Initialize keys + if initKeys { + rng := rand.NewChaCha8([32]byte{seed}) + for n := 0; n < bc.numKeys; n++ { + key := fmt.Sprintf("%s_%d", keyPrefix, n) + dataSz := rand.New(rng).IntN(bc.maxObjSz-bc.minObjSz+1) + bc.minObjSz + value := make([]byte, dataSz) + rng.Read(value) + _, err := objStore.PutBytes(key, value) + if err != nil { + b.Fatalf("Failed to initialize %s/%s: %v", objStoreName, key, err) + } + } + } + + b.ResetTimer() + + // Run benchmark + errors, reads, writes := RunObjStoreBenchmark(b, objStore, bc.minObjSz, bc.maxObjSz, bc.numKeys, rwRatio) + + // Report metrics + b.ReportMetric(float64(errors)*100/float64(b.N), "%error") + b.ReportMetric(float64(reads), "reads") + b.ReportMetric(float64(writes), "writes") + }, + ) + } + }, + ) + } + }, + ) + } +} + +func BenchmarkJetStreamPublishConcurrent(b *testing.B) { + const ( + subject = "test-subject" + streamName = "test-stream" + ) + + type BenchPublisher struct { + // nats connection for this publisher + conn *nats.Conn + // jetstream context + js nats.JetStreamContext + // message buffer + messageData []byte + // number of publish calls + publishCalls int + // number of publish errors + publishErrors int + } + + messageSizeCases := []int64{ + 10, // 10B + 1024, // 1KiB + 102400, // 100KiB + } + numPubsCases := []int{ + 12, + } + + replicasCases := []struct { + clusterSize int + replicas int + }{ + {1, 1}, + {3, 3}, + {3, 3}, + } + + workload := func(b *testing.B, numPubs int, messageSize int64, clientUrl string) { + // create N publishers + publishers := make([]BenchPublisher, numPubs) + for i := range publishers { + // create publisher connection and jetstream context + ncPub, err := nats.Connect(clientUrl) + if err != nil { + b.Fatal(err) + } + defer ncPub.Close() + jsPub, err := ncPub.JetStream() + if err != nil { + b.Fatal(err) + } + + // initialize publisher + publishers[i] = BenchPublisher{ + conn: ncPub, + js: jsPub, + messageData: make([]byte, messageSize), + publishCalls: 0, + publishErrors: 0, + } + rand.NewChaCha8([32]byte{byte(i)}).Read(publishers[i].messageData) + } + + // waits for all publishers sub-routines and for main thread to be ready + var workloadReadyWg sync.WaitGroup + workloadReadyWg.Add(1 + numPubs) + + // wait group blocks main thread until publish workload is completed, it is decremented after stream receives b.N messages from all publishers + var benchCompleteWg sync.WaitGroup + benchCompleteWg.Add(1) + + // wait group to ensure all publishers have been torn down + var finishedPublishersWg sync.WaitGroup + finishedPublishersWg.Add(numPubs) + + // start go routines for all publishers, wait till all publishers are initialized before starting publish workload + for i := range publishers { + go func(pubId int) { + // signal that this publisher has been torn down + defer finishedPublishersWg.Done() + + // publisher sub-routine is ready + workloadReadyWg.Done() + + // start workload when main thread and all other publishers are ready + workloadReadyWg.Wait() + + // publish until stream receives b.N messages + for { + // random bytes as payload + fastRandomMutation(publishers[pubId].messageData, 10) + // attempt to publish message + pubAck, err := publishers[pubId].js.Publish(subject, publishers[pubId].messageData) + publishers[pubId].publishCalls += 1 + if err != nil { + publishers[pubId].publishErrors += 1 + continue + } + // all messages have been published to stream + if pubAck.Sequence == uint64(b.N) { + benchCompleteWg.Done() + } + // a publisher has already published b.N messages, stop publishing + if pubAck.Sequence >= uint64(b.N) { + return + } + } + }(i) + } + + // set bytes per operation + b.SetBytes(messageSize) + + // main thread is ready + workloadReadyWg.Done() + // start the clock + b.ResetTimer() + + // wait till termination cond reached + benchCompleteWg.Wait() + // stop the clock + b.StopTimer() + + // wait for all publishers to shutdown + finishedPublishersWg.Wait() + + // sum up publish calls and errors + publishCalls := 0 + publishErrors := 0 + for _, pub := range publishers { + publishCalls += pub.publishCalls + publishErrors += pub.publishErrors + } + + // report error rate + errorRate := 100 * float64(publishErrors) / float64(publishCalls) + b.ReportMetric(errorRate, "%error") + } + + // benchmark case matrix + for _, replicasCase := range replicasCases { + title := fmt.Sprintf("N=%d,R=%d", replicasCase.clusterSize, replicasCase.replicas) + b.Run( + title, + func(b *testing.B) { + for _, messageSize := range messageSizeCases { + b.Run( + fmt.Sprintf("msgSz=%db", messageSize), + func(b *testing.B) { + for _, numPubs := range numPubsCases { + b.Run( + fmt.Sprintf("pubs=%d", numPubs), + func(b *testing.B) { + // start jetstream cluster + cl, ls, shutdown, nc, js := startJSClusterAndConnect(b, replicasCase.clusterSize) + defer shutdown() + defer nc.Close() + clientUrl := ls.ClientURL() + + // create stream + _, err := jsStreamCreate(b, nc, &StreamConfig{ + Name: streamName, + Subjects: []string{subject}, + Replicas: replicasCase.replicas, + Storage: FileStorage, + }) + if err != nil { + b.Fatal(err) + } + defer js.DeleteStream(streamName) + + // If replicated resource, connect to stream leader for lower variability + if replicasCase.replicas > 1 { + nc.Close() + clientUrl = cl.streamLeader("$G", streamName).ClientURL() + nc, _ = jsClientConnectURL(b, clientUrl) + defer nc.Close() + } + + // run workload + workload(b, numPubs, messageSize, clientUrl) + }, + ) + } + }, + ) + } + }, + ) + } +} + +func BenchmarkJetStreamParallelStartup(b *testing.B) { + omp := runtime.GOMAXPROCS(-1) + streams, msgs, cardinality := omp, 100_000, 10_000 + + _, s, shutdown, nc, js := startJSClusterAndConnect(b, 1) + defer shutdown() + jsc := *s.JetStreamConfig() + sd := strings.TrimSuffix(jsc.StoreDir, "/jetstream") + + b.Logf("Building %d streams with %d messages, %d subjects...", streams, msgs, cardinality) + start := time.Now() + for i := range streams { + jsStreamCreate(b, nc, &StreamConfig{ + Name: fmt.Sprintf("stream_%d", i), + Subjects: []string{fmt.Sprintf("%d.>", i)}, + Storage: FileStorage, + }) + for n := range msgs { + subj := fmt.Sprintf("%d.%d", i, n%cardinality) + _, err := js.Publish(subj, nil) + require_NoError(b, err) + } + } + b.Logf("Streams built in %s", time.Since(start)) + + bench := func(b *testing.B) { + s.shutdownJetStream() + jsc.StoreDir = sd + require_NoError(b, filepath.Walk(jsc.StoreDir, func(path string, info os.FileInfo, err error) error { + require_NoError(b, err) + if info.Mode().IsRegular() && info.Name() == "index.db" { + return os.Truncate(path, 0) + } + return nil + })) + b.ResetTimer() + s.EnableJetStream(&jsc) + } + + // Try to step down GOMAXPROCS in common CPU core counts. + mp := 1 << (bits.Len(uint(omp)) - 1) + if omp > mp { + b.Run(fmt.Sprintf("GOMAXPROCS=%d", omp), func(b *testing.B) { + bench(b) + }) + } + for ; mp >= 1; mp >>= 1 { + b.Run(fmt.Sprintf("GOMAXPROCS=%d", mp), func(b *testing.B) { + runtime.GOMAXPROCS(mp) + defer runtime.GOMAXPROCS(omp) + bench(b) + }) + } +} + +func BenchmarkJetStreamScanForSources(b *testing.B) { + _, s, shutdown, nc, js := startJSClusterAndConnect(b, 1) + defer shutdown() + + jsStreamCreate(b, nc, &StreamConfig{ + Name: "origin", + Subjects: []string{"foo"}, + Storage: FileStorage, + }) + + jsStreamCreate(b, nc, &StreamConfig{ + Name: "stream", + Subjects: []string{"bar"}, + Storage: FileStorage, + Sources: []*StreamSource{ + { + Name: "origin", + FilterSubject: "foo", + }, + }, + }) + + // Start by publishing some messages to the sourcing stream. + for range 1000 { + _, err := js.Publish("bar", nil) + require_NoError(b, err) + } + + // Then publish a message to the origin stream. + _, err := js.Publish("foo", nil) + require_NoError(b, err) + + // Wait for the sourced message from the origin stream. + checkFor(b, 5*time.Second, 100*time.Millisecond, func() error { + si, err := js.StreamInfo("stream") + if err != nil { + return err + } + if si.State.Msgs != 1001 { + return fmt.Errorf("waiting for sourcing") + } + return nil + }) + + // Now publish a LOT more messages to the sourcing stream + // which previously the reverse scan would have had to scan + // over linearly. + for range 100_000 { + _, err := js.Publish("bar", nil) + require_NoError(b, err) + } + + mset, err := s.globalAccount().lookupStream("stream") + require_NoError(b, err) + + b.Run("StartingSequenceForSources", func(b *testing.B) { + mset.resetSourceInfo() + }) +} + +// Helper function to stand up a JS-enabled single server or cluster +func startJSClusterAndConnect(b *testing.B, clusterSize int) (c *cluster, s *Server, shutdown func(), nc *nats.Conn, js nats.JetStreamContext) { + b.Helper() + var err error + + if clusterSize == 1 { + s = RunBasicJetStreamServer(b) + shutdown = func() { + s.Shutdown() + } + s.optsMu.Lock() + s.opts.SyncInterval = 5 * time.Minute + s.optsMu.Unlock() + } else { + tmpl := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 2GB, max_file_store: 4GB, store_dir: '%s'} + + leaf { + listen: 127.0.0.1:-1 + } + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } + ` + c = createJetStreamClusterWithTemplate(b, tmpl, "BENCH_PUB", clusterSize) + c.waitOnClusterReadyWithNumPeers(clusterSize) + c.waitOnLeader() + s = c.leader() + shutdown = func() { + c.shutdown() + } + for _, s := range c.servers { + s.optsMu.Lock() + s.opts.SyncInterval = 5 * time.Minute + s.optsMu.Unlock() + } + } + + nc, err = nats.Connect(s.ClientURL()) + if err != nil { + b.Fatalf("failed to connect: %s", err) + } + + js, err = nc.JetStream() + if err != nil { + b.Fatalf("failed to init jetstream: %s", err) + } + + return c, s, shutdown, nc, js +} + +// BenchmarkJetStreamDirectGet measures direct get latency, single and multi, +// idle and while a concurrent writer floods the stream, plus the publish path +// on its own to spot read-isolation cost on writes. +func BenchmarkJetStreamDirectGet(b *testing.B) { + const numSubjects = 256 + + setup := func(b *testing.B) (*Server, *nats.Conn) { + b.Helper() + s := RunBasicJetStreamServer(b) + nc, js := jsClientConnect(b, s) + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo.*"}, + Storage: nats.FileStorage, + MaxMsgsPerSubject: 1, + AllowDirect: true, + }) + if err != nil { + b.Fatalf("add stream: %v", err) + } + payload := make([]byte, 64) + for i := 0; i < numSubjects; i++ { + if _, err := js.PublishAsync(fmt.Sprintf("foo.%d", i), payload); err != nil { + b.Fatalf("preload: %v", err) + } + } + select { + case <-js.PublishAsyncComplete(): + case <-time.After(10 * time.Second): + b.Fatalf("preload did not complete") + } + return s, nc + } + + // Hammer the stream with async publishes over the same subject space for + // the duration of the benchmark. The stop func reports publishes acked. + startWriter := func(b *testing.B, s *Server) (stop func() uint64) { + b.Helper() + nc, js := jsClientConnect(b, s, nats.NoEcho()) + done := make(chan struct{}) + finished := make(chan struct{}) + var acked atomic.Uint64 + go func() { + defer close(finished) + payload := make([]byte, 64) + for i := 0; ; i++ { + select { + case <-done: + return + default: + } + pa, err := js.PublishAsync(fmt.Sprintf("foo.%d", i%numSubjects), payload) + if err != nil { + // Pending limit reached, wait for completions. + select { + case <-js.PublishAsyncComplete(): + case <-time.After(time.Second): + } + continue + } + go func() { + select { + case <-pa.Ok(): + acked.Add(1) + case <-pa.Err(): + case <-done: + } + }() + } + }() + return func() uint64 { + close(done) + <-finished + nc.Close() + return acked.Load() + } + } + + single := `{"last_by_subj":"foo.7"}` + multi := `{"multi_last":["foo.7","foo.11","foo.13","foo.17","foo.19","foo.23","foo.29","foo.31"]}` + for _, bc := range []struct { + name string + req string + withWrites bool + }{ + {"Single/Idle", single, false}, + {"Single/UnderWrites", single, true}, + {"Multi/Idle", multi, false}, + {"Multi/UnderWrites", multi, true}, + } { + b.Run(bc.name, func(b *testing.B) { + s, nc := setup(b) + defer s.Shutdown() + defer nc.Close() + + var stop func() uint64 + if bc.withWrites { + stop = startWriter(b, s) + // Let the writer get going. + time.Sleep(100 * time.Millisecond) + } + + getSubj := fmt.Sprintf(JSDirectMsgGetT, "TEST") + req := []byte(bc.req) + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := nc.Request(getSubj, req, 5*time.Second); err != nil { + b.Fatalf("direct get: %v", err) + } + } + b.StopTimer() + if stop != nil { + writes := stop() + b.ReportMetric(float64(writes)/b.Elapsed().Seconds(), "writes/s") + } + }) + } + + b.Run("PublishOnly", func(b *testing.B) { + s, nc := setup(b) + defer s.Shutdown() + defer nc.Close() + + _, js := jsClientConnect(b, s) + payload := make([]byte, 64) + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := js.PublishAsync(fmt.Sprintf("foo.%d", i%numSubjects), payload); err != nil { + i-- + select { + case <-js.PublishAsyncComplete(): + case <-time.After(time.Second): + } + } + } + select { + case <-js.PublishAsyncComplete(): + case <-time.After(30 * time.Second): + b.Fatalf("publishes did not complete") + } + }) +} diff --git a/server/jetstream_cluster_long_test.go b/server/jetstream_cluster_long_test.go new file mode 100644 index 0000000..3750219 --- /dev/null +++ b/server/jetstream_cluster_long_test.go @@ -0,0 +1,1193 @@ +// Copyright 2022-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This test file is skipped by default to avoid accidentally running (e.g. `go test ./server`) +//go:build !skip_js_tests && include_js_long_tests + +package server + +import ( + "context" + crand "crypto/rand" + "errors" + "fmt" + "math/rand/v2" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +// TestLongKVPutWithServerRestarts overwrites values in a replicated KV bucket for a fixed amount of time. +// Also restarts a random server at fixed interval. +// The test fails if updates fail for a continuous interval of time, or if a server fails to restart and catch up. +func TestLongKVPutWithServerRestarts(t *testing.T) { + // RNG Seed + const Seed = 123456 + // Number of keys in bucket + const NumKeys = 1000 + // Size of (random) values + const ValueSize = 1024 + // Test duration + const Duration = 3 * time.Minute + // If no updates successful for this interval, then fail the test + const MaxRetry = 5 * time.Second + // Minimum time between put operations + const UpdatesInterval = 1 * time.Millisecond + // Time between progress reports to console + const ProgressInterval = 10 * time.Second + // Time between server restarts + const ServerRestartInterval = 5 * time.Second + + type Parameters struct { + numServers int + replicas int + storage nats.StorageType + } + + test := func(t *testing.T, p Parameters) { + rng := rand.New(rand.NewPCG(uint64(Seed), 0)) + + // Create cluster + clusterName := fmt.Sprintf("C_%d-%s", p.numServers, p.storage) + cluster := createJetStreamClusterExplicit(t, clusterName, p.numServers) + defer cluster.shutdown() + + // Connect to a random server but client will discover others too. + nc, js := jsClientConnect(t, cluster.randomServer()) + defer nc.Close() + + // Create bucket + kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ + Bucket: "TEST", + Replicas: p.replicas, + Storage: p.storage, + }) + require_NoError(t, err) + + // Initialize list of keys + keys := make([]string, NumKeys) + for i := 0; i < NumKeys; i++ { + keys[i] = fmt.Sprintf("key-%d", i) + } + + // Initialize keys in bucket with an empty value + for _, key := range keys { + _, err := kv.Put(key, []byte{}) + require_NoError(t, err) + } + + // Track statistics + var stats = struct { + start time.Time + lastSuccessfulUpdate time.Time + updateOk uint64 + updateErr uint64 + restarts uint64 + restartsMap map[string]int + }{ + start: time.Now(), + lastSuccessfulUpdate: time.Now(), + restartsMap: make(map[string]int, p.numServers), + } + for _, server := range cluster.servers { + stats.restartsMap[server.Name()] = 0 + } + + // Print statistics + printProgress := func() { + + t.Logf( + "[%s] %d updates %d errors, %d server restarts (%v)", + time.Since(stats.start).Round(time.Second), + stats.updateOk, + stats.updateErr, + stats.restarts, + stats.restartsMap, + ) + } + + // Print update on completion + defer printProgress() + + // Pick a random key and update it with a random value + valueBuffer := make([]byte, ValueSize) + updateRandomKey := func() error { + key := keys[rand.IntN(NumKeys)] + _, err := crand.Read(valueBuffer) + require_NoError(t, err) + _, err = kv.Put(key, valueBuffer) + return err + } + + // Set up timers and tickers + endTestTimer := time.After(Duration) + nextUpdateTicker := time.NewTicker(UpdatesInterval) + progressTicker := time.NewTicker(ProgressInterval) + restartServerTicker := time.NewTicker(ServerRestartInterval) + + runLoop: + for { + select { + + case <-endTestTimer: + break runLoop + + case <-nextUpdateTicker.C: + err := updateRandomKey() + if err == nil { + stats.updateOk++ + stats.lastSuccessfulUpdate = time.Now() + } else { + stats.updateErr++ + if time.Since(stats.lastSuccessfulUpdate) > MaxRetry { + t.Fatalf("Could not successfully update for over %s", MaxRetry) + } + } + + case <-restartServerTicker.C: + randomServer := cluster.servers[rng.IntN(len(cluster.servers))] + randomServer.Shutdown() + randomServer.WaitForShutdown() + restartedServer := cluster.restartServer(randomServer) + cluster.waitOnClusterReady() + cluster.waitOnServerHealthz(restartedServer) + cluster.waitOnAllCurrent() + stats.restarts++ + stats.restartsMap[randomServer.Name()]++ + + case <-progressTicker.C: + printProgress() + } + } + } + + testCases := []Parameters{ + {numServers: 5, replicas: 3, storage: nats.MemoryStorage}, + {numServers: 5, replicas: 3, storage: nats.FileStorage}, + } + + for _, testCase := range testCases { + name := fmt.Sprintf("N:%d,R:%d,%s", testCase.numServers, testCase.replicas, testCase.storage) + t.Run( + name, + func(t *testing.T) { test(t, testCase) }, + ) + } +} + +// This is a RaftChainOfBlocks test that randomly starts and stops nodes to exercise recovery and snapshots. +func TestLongNRGChainOfBlocks(t *testing.T) { + const ( + ClusterSize = 3 + GroupSize = 3 + ConvergenceTimeout = 30 * time.Second + Duration = 10 * time.Minute + PrintStateInterval = 3 * time.Second + ) + + // Create cluster + c := createJetStreamClusterExplicit(t, "Test", ClusterSize) + defer c.shutdown() + + rg := c.createRaftGroup("ChainOfBlocks", GroupSize, newRaftChainStateMachine) + rg.waitOnLeader() + + // Available operations + type TestOperation string + const ( + StopOne TestOperation = "Stop one active node" + StopAll = "Stop all active nodes" + RestartOne = "Restart one stopped node" + RestartAll = "Restart all stopped nodes" + Snapshot = "Snapshot one active node" + Propose = "Propose a value via one active node" + ProposeLeader = "Propose a value via leader" + Pause = "Let things run undisturbed for a while" + Check = "Wait for nodes to converge" + ) + + // Weighted distribution of operations, one is randomly chosen from this vector in each iteration + opsWeighted := []TestOperation{ + StopOne, + StopOne, + StopOne, + StopAll, + StopAll, + RestartOne, + RestartOne, + RestartOne, + RestartAll, + RestartAll, + RestartAll, + Snapshot, + Snapshot, + Propose, + Propose, + Propose, + Propose, + ProposeLeader, + ProposeLeader, + ProposeLeader, + ProposeLeader, + Pause, + Pause, + Pause, + Pause, + Check, + Check, + Check, + Check, + } + + rngSeed := time.Now().UnixNano() + rng := rand.New(rand.NewPCG(uint64(rngSeed), 0)) + t.Logf("Seed: %d", rngSeed) + + // Chose a node from the list (and remove it) + pickRandomNode := func(nodes []stateMachine) ([]stateMachine, stateMachine) { + if len(nodes) == 0 { + // Input list is empty + return nodes, nil + } + // Pick random node + i := rng.IntN(len(nodes)) + node := nodes[i] + // Move last element in its place + nodes[i] = nodes[len(nodes)-1] + // Return slice excluding last element + return nodes[:len(nodes)-1], node + } + + // Create summary status string for all replicas + chainStatusString := func() string { + b := strings.Builder{} + for _, sm := range rg { + csm := sm.(*RCOBStateMachine) + running, blocksCount, blockHash := csm.getCurrentHash() + if running { + b.WriteString( + fmt.Sprintf( + " [%s (%s): %d blocks, hash=%s],", + csm.server().Name(), + csm.node().ID(), + blocksCount, + blockHash, + ), + ) + } else { + b.WriteString( + fmt.Sprintf( + " [%s (%s): STOPPED],", + csm.server().Name(), + csm.node().ID(), + ), + ) + + } + } + return b.String() + } + + // Track the highest number of blocks applied by any of the replicas + highestBlocksCount := uint64(0) + + // Track active and stopped nodes + activeNodes := make([]stateMachine, 0, GroupSize) + stoppedNodes := make([]stateMachine, 0, GroupSize) + + // Initially all nodes are active + activeNodes = append(activeNodes, rg...) + + defer func() { + t.Logf("Final state: %s", chainStatusString()) + }() + + printStateTicker := time.NewTicker(PrintStateInterval) + testTimer := time.NewTimer(Duration) + start := time.Now() + iteration := 0 + stopCooldown := 0 + + for { + + iteration++ + select { + case <-printStateTicker.C: + t.Logf( + "[%s] State: %s", + time.Since(start).Round(time.Second), + chainStatusString(), + ) + case <-testTimer.C: + // Test completed + return + default: + // Continue + } + + // Choose a random operation to perform in this iteration + nextOperation := opsWeighted[rng.IntN(len(opsWeighted))] + + // If we're about to stop one or all nodes, do some sanity checks to ensure we don't + // spam stops which would constantly interrupt the chance of making progress. + if nextOperation == StopOne || nextOperation == StopAll { + if stopCooldown > 0 { + nextOperation = Propose + } else { + stopCooldown = 50 + } + } + if stopCooldown > 0 { + stopCooldown-- + } + + if RCOBOptions.verbose { + t.Logf("Iteration %d: %s", iteration, nextOperation) + } + + switch nextOperation { + + case StopOne: + // Stop an active node (if any are left active) + var n stateMachine + activeNodes, n = pickRandomNode(activeNodes) + if n != nil { + n.stop() + stoppedNodes = append(stoppedNodes, n) + } + + case StopAll: + // Stop all active nodes (if any are active) + for _, node := range activeNodes { + node.stop() + } + stoppedNodes = append(stoppedNodes, activeNodes...) + activeNodes = make([]stateMachine, 0, GroupSize) + + case RestartOne: + // Restart a stopped node (if any are stopped) + var n stateMachine + stoppedNodes, n = pickRandomNode(stoppedNodes) + if n != nil { + n.restart() + activeNodes = append(activeNodes, n) + } + + case RestartAll: + // Restart all stopped nodes (if any) + for _, node := range stoppedNodes { + node.restart() + } + activeNodes = append(activeNodes, stoppedNodes...) + stoppedNodes = make([]stateMachine, 0, GroupSize) + + case Snapshot: + // Choose a random active node and tell it to create a snapshot + if len(activeNodes) > 0 { + n := activeNodes[rng.IntN(len(activeNodes))] + n.(*RCOBStateMachine).createSnapshot() + } + + case Propose: + // Make an active node propose the next block (if any nodes are active) + if len(activeNodes) > 0 { + n := activeNodes[rng.IntN(len(activeNodes))] + n.(*RCOBStateMachine).proposeBlock() + } + + case ProposeLeader: + // Make the leader propose the next block (if a leader is active) + leader := rg.leader() + if leader != nil { + leader.(*RCOBStateMachine).proposeBlock() + } + + case Pause: + // Noop, let things run undisturbed for a little bit + time.Sleep(time.Duration(rng.IntN(250)) * time.Millisecond) + + case Check: + // Restart any stopped node + for _, node := range stoppedNodes { + node.restart() + } + activeNodes = append(activeNodes, stoppedNodes...) + stoppedNodes = make([]stateMachine, 0, GroupSize) + + // Ensure all nodes (eventually) converge + checkFor(t, ConvergenceTimeout, 250*time.Millisecond, + func() error { + referenceNode := rg[0] + // Save block count and hash of first node as reference + _, referenceBlocksCount, referenceHash := referenceNode.(*RCOBStateMachine).getCurrentHash() + + // Compare each node against reference + for _, n := range rg { + sm := n.(*RCOBStateMachine) + running, blocksCount, blockHash := sm.getCurrentHash() + if !running { + return fmt.Errorf( + "node not running: %s (%s)", + sm.server().Name(), + sm.node().ID(), + ) + } + + // Track the highest block delivered by any node + if blocksCount > highestBlocksCount { + if RCOBOptions.verbose { + t.Logf( + "New highest blocks count: %d (%s (%s))", + blocksCount, + sm.s.Name(), + sm.n.ID(), + ) + } + highestBlocksCount = blocksCount + } + + // Each replica must match the reference node + + if blocksCount != referenceBlocksCount { + return fmt.Errorf( + "different number of blocks %d (%s (%s) vs. %d (%s (%s))", + blocksCount, + sm.server().Name(), + sm.node().ID(), + referenceBlocksCount, + referenceNode.server().Name(), + referenceNode.node().ID(), + ) + } else if blockHash != referenceHash { + return fmt.Errorf( + "different hash after %d blocks %s (%s (%s) vs. %s (%s (%s))", + blocksCount, + blockHash, + sm.server().Name(), + sm.node().ID(), + referenceHash, + referenceNode.server().Name(), + referenceNode.node().ID(), + ) + } + } + + // Verify consistency check was against the highest block known + if referenceBlocksCount < highestBlocksCount { + return fmt.Errorf( + "nodes converged below highest known block count: %d: %s", + highestBlocksCount, + chainStatusString(), + ) + } + + // All nodes reached the same state, check passed + return nil + }, + ) + } + } +} + +func TestLongClusterWorkQueueMessagesNotSkipped(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + iterations := 500_000 + stream := "s1" + subjf := "subj.>" + consumers := map[string]string{ + "c1": "subj.c1", + "c2": "subj.c2.*", + "c3": "subj.c3", + } + + sig := make(chan *nats.Msg, 900) + + _, err := js.AddStream(&nats.StreamConfig{ + Name: stream, + Storage: nats.FileStorage, + Subjects: []string{subjf}, + Retention: nats.WorkQueuePolicy, + Duplicates: time.Minute * 2, + Replicas: 3, + MaxAge: time.Hour, + }) + require_NoError(t, err) + + for name, subjf := range consumers { + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + _, err = js.AddConsumer(stream, &nats.ConsumerConfig{ + Name: name, + FilterSubject: subjf, + Replicas: 3, + AckPolicy: nats.AckExplicitPolicy, + DeliverPolicy: nats.DeliverAllPolicy, + }) + require_NoError(t, err) + + ps, err := js.PullSubscribe(subjf, "", nats.Bind(stream, name)) + require_NoError(t, err) + + go func() { + for { + msgs, err := ps.FetchBatch(300) + if errors.Is(err, nats.ErrTimeout) { + continue + } + if errors.Is(err, nats.ErrConnectionClosed) || errors.Is(err, nats.ErrSubscriptionClosed) { + return // ... for when the test finishes + } + require_NoError(t, err) + for msg := range msgs.Messages() { + go func() { + time.Sleep(time.Millisecond * time.Duration(rand.Int32N(100))) + require_NoError(t, msg.Ack()) + sig <- msg + }() + } + } + }() + } + + go func() { + hdrs := nats.Header{} + for i := 1; i <= iterations; i++ { + // Pick a random consumer to hit this time (map iteration order is + // non-deterministic, but break to do it just once). + for _, subj := range consumers { + hdrs.Set("Nats-Msg-Id", fmt.Sprintf("msg-%d", i)) + if strings.HasPrefix(subj, "*") { + subj = strings.Replace(subj, "*", fmt.Sprintf("%d", i), 1) + } + _, err := js.PublishMsg(&nats.Msg{ + Subject: subj, + Header: hdrs, + }) + require_NoError(t, err) + break + } + } + }() + + for i := 1; i <= iterations; i++ { + if i%10000 == 0 { + t.Logf("%d messages out of %d", i, iterations) + } + + select { + case <-sig: + case <-time.After(time.Second * 2): + si, err := js.StreamInfo(stream) + require_NoError(t, err) + t.Logf("Stream info: %+v", si.State) + + for name := range consumers { + ci, err := js.ConsumerInfo(stream, name) + require_NoError(t, err) + t.Logf("Consumer %q info: %+v, %+v", name, ci.AckFloor, ci.Delivered) + } + + t.Fatalf("Didn't receive message %d", i) + } + } +} + +func TestLongClusterJetStreamKeyValueSync(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + for _, s := range c.servers { + s.optsMu.Lock() + s.opts.LameDuckDuration = 15 * time.Second + s.opts.LameDuckGracePeriod = -15 * time.Second + s.optsMu.Unlock() + } + s := c.randomNonLeader() + connect := func(t *testing.T) (*nats.Conn, nats.JetStreamContext) { + return jsClientConnect(t, s) + } + + const accountName = "$G" + const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + createData := func(n int) []byte { + b := make([]byte, n) + for i := range b { + b[i] = letterBytes[rand.IntN(len(letterBytes))] + } + return b + } + getOrCreateKvStore := func(kvname string) (nats.KeyValue, error) { + _, js := connect(t) + kvExists := false + existingKvnames := js.KeyValueStoreNames() + for existingKvname := range existingKvnames { + if existingKvname == kvname { + kvExists = true + break + } + } + if !kvExists { + return js.CreateKeyValue(&nats.KeyValueConfig{ + Bucket: kvname, + Replicas: 3, + Storage: nats.FileStorage, + }) + } else { + return js.KeyValue(kvname) + } + } + abs := func(x int64) int64 { + if x < 0 { + return -x + } + return x + } + var counter int64 + var errorCounter int64 + + checkMsgsEqual := func(t *testing.T, accountName, streamName string) error { + // Gather all the streams replicas and compare contents. + msets := make(map[*Server]*stream) + for _, s := range c.servers { + acc, err := s.LookupAccount(accountName) + if err != nil { + return err + } + mset, err := acc.lookupStream(streamName) + if err != nil { + return err + } + msets[s] = mset + } + + str := getStreamDetails(t, c, accountName, streamName) + if str == nil { + return fmt.Errorf("could not get stream leader state") + } + state := str.State + for seq := state.FirstSeq; seq <= state.LastSeq; seq++ { + var msgId string + var smv StoreMsg + for replica, mset := range msets { + mset.mu.RLock() + sm, err := mset.store.LoadMsg(seq, &smv) + mset.mu.RUnlock() + if err != nil { + if err == ErrStoreMsgNotFound || err == errDeletedMsg { + // Skip these. + } else { + t.Logf("WRN: Error loading message (seq=%d) from stream %q on replica %q: %v", seq, streamName, replica, err) + } + continue + } + if msgId == _EMPTY_ { + msgId = string(sm.hdr) + } else if msgId != string(sm.hdr) { + t.Errorf("MsgIds do not match for seq %d on stream %q: %q vs %q", seq, streamName, msgId, sm.hdr) + } + } + } + return nil + } + + keyUpdater := func(ctx context.Context, cancel context.CancelFunc, kvname string, numKeys int) { + kv, err := getOrCreateKvStore(kvname) + if err != nil { + t.Fatalf("[%s]:%v", kvname, err) + } + for i := 0; i < numKeys; i++ { + key := fmt.Sprintf("key-%d", i) + kv.Create(key, createData(160)) + } + lastData := make(map[string][]byte) + revisions := make(map[string]uint64) + for { + select { + case <-ctx.Done(): + return + default: + } + r := rand.IntN(numKeys) + key := fmt.Sprintf("key-%d", r) + + for i := 0; i < 5; i++ { + _, err := kv.Get(key) + if err != nil { + atomic.AddInt64(&errorCounter, 1) + if err == nats.ErrKeyNotFound { + t.Logf("WRN: Key not found! [%s/%s] - [%s]", kvname, key, err) + cancel() + } + } + } + + k, err := kv.Get(key) + if err != nil { + atomic.AddInt64(&errorCounter, 1) + } else { + if revisions[key] != 0 && abs(int64(k.Revision())-int64(revisions[key])) < 2 { + lastDataVal, ok := lastData[key] + if ok && k.Revision() == revisions[key] && slices.Compare(lastDataVal, k.Value()) != 0 { + t.Logf("data loss [%s/%s][rev:%d] expected:[%v] is:[%v]", kvname, key, revisions[key], string(lastDataVal), string(k.Value())) + } + } + newData := createData(160) + revisions[key], err = kv.Update(key, newData, k.Revision()) + if err != nil && err != nats.ErrTimeout { + atomic.AddInt64(&errorCounter, 1) + } else { + lastData[key] = newData + } + atomic.AddInt64(&counter, 1) + } + } + } + + streamCount := 50 + keysCount := 100 + streamPrefix := "IKV" + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + // The keyUpdaters will run for less time. + kctx, kcancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer kcancel() + + var wg sync.WaitGroup + var streams []string + for i := 0; i < streamCount; i++ { + streamName := fmt.Sprintf("%s-%d", streamPrefix, i) + streams = append(streams, "KV_"+streamName) + + wg.Add(1) + go func(i int) { + defer wg.Done() + keyUpdater(kctx, cancel, streamName, keysCount) + }(i) + } + + debug := false + nc2, _ := jsClientConnect(t, s) + if debug { + go func() { + for range time.NewTicker(5 * time.Second).C { + select { + case <-ctx.Done(): + return + default: + } + for _, str := range streams { + leaderSrv := c.streamLeader(accountName, str) + if leaderSrv == nil { + continue + } + streamLeader := getStreamDetails(t, c, accountName, str) + if streamLeader == nil { + continue + } + t.Logf("|------------------------------------------------------------------------------------------------------------------------|") + lstate := streamLeader.State + t.Logf("| %-10s | %-10s | msgs:%-10d | bytes:%-10d | deleted:%-10d | first:%-10d | last:%-10d |", + str, leaderSrv.String()+"*", lstate.Msgs, lstate.Bytes, lstate.NumDeleted, lstate.FirstSeq, lstate.LastSeq, + ) + for _, srv := range c.servers { + if srv == leaderSrv { + continue + } + acc, err := srv.LookupAccount(accountName) + if err != nil { + continue + } + stream, err := acc.lookupStream(str) + if err != nil { + t.Logf("Error looking up stream %s on %s replica", str, srv) + continue + } + state := stream.state() + + unsynced := lstate.Msgs != state.Msgs || lstate.Bytes != state.Bytes || + lstate.NumDeleted != state.NumDeleted || lstate.FirstSeq != state.FirstSeq || lstate.LastSeq != state.LastSeq + + var result string + if unsynced { + result = "UNSYNCED" + } + t.Logf("| %-10s | %-10s | msgs:%-10d | bytes:%-10d | deleted:%-10d | first:%-10d | last:%-10d | %s", + str, srv, state.Msgs, state.Bytes, state.NumDeleted, state.FirstSeq, state.LastSeq, result, + ) + } + } + t.Logf("|------------------------------------------------------------------------------------------------------------------------| %v", nc2.ConnectedUrl()) + } + }() + } + + checkStreams := func(t *testing.T) { + for _, str := range streams { + checkFor(t, time.Minute, 500*time.Millisecond, func() error { + return checkState(t, c, accountName, str) + }) + checkFor(t, time.Minute, 500*time.Millisecond, func() error { + return checkMsgsEqual(t, accountName, str) + }) + } + } + +Loop: + for range time.NewTicker(30 * time.Second).C { + select { + case <-ctx.Done(): + break Loop + default: + } + rollout := func(t *testing.T) { + for _, s := range c.servers { + // For graceful mode + s.lameDuckMode() + s.WaitForShutdown() + s = c.restartServer(s) + + hctx, hcancel := context.WithTimeout(context.Background(), 15*time.Second) + defer hcancel() + + Healthz: + for range time.NewTicker(2 * time.Second).C { + select { + case <-hctx.Done(): + default: + } + + status := s.healthz(nil) + if status.StatusCode == 200 { + break Healthz + } + } + c.waitOnClusterReady() + checkStreams(t) + } + } + rollout(t) + checkStreams(t) + } + wg.Wait() + checkStreams(t) +} + +func TestLongClusterCLFSOnDuplicates(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + nc2, js2 := jsClientConnect(t, c.randomServer()) + defer nc2.Close() + + streamName := "TESTW" + _, err := js.AddStream(&nats.StreamConfig{ + Name: streamName, + Subjects: []string{"foo"}, + Replicas: 3, + Storage: nats.FileStorage, + MaxAge: 3 * time.Minute, + Duplicates: 2 * time.Minute, + }) + require_NoError(t, err) + + c.waitOnStreamLeader(globalAccountName, streamName) + + var wg sync.WaitGroup + + // The test will be successful if it runs for this long without dup issues. + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + go func() { + tick := time.NewTicker(10 * time.Second) + for { + select { + case <-ctx.Done(): + wg.Done() + return + case <-tick.C: + c.streamLeader(globalAccountName, streamName).JetStreamStepdownStream(globalAccountName, streamName) + } + } + }() + wg.Add(1) + + for i := 0; i < 5; i++ { + go func(i int) { + var err error + sub, err := js2.PullSubscribe("foo", fmt.Sprintf("A:%d", i)) + require_NoError(t, err) + + for { + select { + case <-ctx.Done(): + wg.Done() + return + default: + } + + msgs, err := sub.Fetch(100, nats.MaxWait(200*time.Millisecond)) + if err != nil { + continue + } + for _, msg := range msgs { + msg.Ack() + } + } + }(i) + wg.Add(1) + } + + // Sync producer that only does a couple of duplicates, cancel the test + // if we get too many errors without responses. + errCh := make(chan error, 10) + go func() { + // Try sync publishes normally in this state and see if it times out. + for i := 0; ; i++ { + select { + case <-ctx.Done(): + wg.Done() + return + default: + } + + var succeeded bool + var failures int + for n := 0; n < 10; n++ { + _, err := js.Publish( + "foo", []byte("test"), + nats.MsgId(fmt.Sprintf("sync:checking:%d", i)), + nats.RetryAttempts(30), + nats.AckWait(500*time.Millisecond), + ) + if err != nil { + failures++ + continue + } + succeeded = true + } + if !succeeded { + errCh <- fmt.Errorf("Too many publishes failed with timeout: failures=%d, i=%d", failures, i) + } + } + }() + wg.Add(1) + +Loop: + for n := uint64(0); true; n++ { + select { + case <-ctx.Done(): + break Loop + case e := <-errCh: + t.Error(e) + break Loop + default: + } + // Cause a lot of duplicates very fast until producer stalls. + for i := 0; i < 128; i++ { + msgID := nats.MsgId(fmt.Sprintf("id.%d.%d", n, i)) + js.PublishAsync("foo", []byte("test"), msgID, nats.RetryAttempts(10)) + } + } + cancel() + wg.Wait() +} + +func TestLongClusterJetStreamRestartThenScaleStreamReplicas(t *testing.T) { + t.Skip("This test takes too long, need to make shorter") + + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + s := c.randomNonLeader() + nc, js := jsClientConnect(t, s) + defer nc.Close() + + nc2, producer := jsClientConnect(t, s) + defer nc2.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo"}, + Replicas: 3, + }) + require_NoError(t, err) + c.waitOnStreamLeader(globalAccountName, "TEST") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + end := time.Now().Add(2 * time.Second) + for time.Now().Before(end) { + producer.Publish("foo", []byte(strings.Repeat("A", 128))) + time.Sleep(time.Millisecond) + } + + var wg sync.WaitGroup + for i := 0; i < 5; i++ { + sub, err := js.PullSubscribe("foo", fmt.Sprintf("C-%d", i)) + require_NoError(t, err) + + wg.Add(1) + go func() { + defer wg.Done() + for range time.NewTicker(10 * time.Millisecond).C { + select { + case <-ctx.Done(): + return + default: + } + + msgs, err := sub.Fetch(1) + if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, nats.ErrConnectionClosed) { + t.Logf("Pull Error: %v", err) + } + for _, msg := range msgs { + msg.Ack() + } + } + }() + } + c.lameDuckRestartAll() + c.waitOnStreamLeader(globalAccountName, "TEST") + + // Swap the logger to try to detect the condition after the restart. + loggers := make([]*captureDebugLogger, 3) + for i, srv := range c.servers { + l := &captureDebugLogger{dbgCh: make(chan string, 10)} + loggers[i] = l + srv.SetLogger(l, true, false) + } + condition := `Direct proposal ignored, not leader (state: CLOSED)` + errCh := make(chan error, 10) + + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case dl := <-loggers[0].dbgCh: + if strings.Contains(dl, condition) { + errCh <- errors.New(condition) + } + case dl := <-loggers[1].dbgCh: + if strings.Contains(dl, condition) { + errCh <- errors.New(condition) + } + case dl := <-loggers[2].dbgCh: + if strings.Contains(dl, condition) { + errCh <- errors.New(condition) + } + case <-ctx.Done(): + return + } + } + }() + + // Start publishing again for a while. + end = time.Now().Add(2 * time.Second) + for time.Now().Before(end) { + producer.Publish("foo", []byte(strings.Repeat("A", 128))) + time.Sleep(time.Millisecond) + } + + // Try to do a stream edit back to R=1 after doing all the upgrade. + info, _ := js.StreamInfo("TEST") + sconfig := info.Config + sconfig.Replicas = 1 + _, err = js.UpdateStream(&sconfig) + require_NoError(t, err) + + // Leave running for some time after the update. + time.Sleep(2 * time.Second) + + info, _ = js.StreamInfo("TEST") + sconfig = info.Config + sconfig.Replicas = 3 + _, err = js.UpdateStream(&sconfig) + require_NoError(t, err) + + select { + case e := <-errCh: + t.Fatalf("Bad condition on raft node: %v", e) + case <-time.After(2 * time.Second): + // Done + } + + // Stop goroutines and wait for them to exit. + cancel() + wg.Wait() +} + +func TestLongFileStoreEnforceMsgPerSubjectLimit(t *testing.T) { + td := t.TempDir() + fs, err := newFileStore( + FileStoreConfig{StoreDir: td, BlockSize: 1024}, + StreamConfig{ + Name: "zzz", Subjects: []string{"test.>"}, Storage: FileStorage, + }, + ) + require_NoError(t, err) + defer fs.Stop() + + t.Logf("Starting publishes") + for i := 0; i < 100_000; i++ { + _, _, err := fs.StoreMsg(fmt.Sprintf("test.%06d", i), nil, nil, 0) + require_NoError(t, err) + } + // Now update some of them. Leave a bit of a mess with some big gaps. + for i := 0; i < 5_000_000; i++ { + n := rand.Int32N(100_000) + if n < 5000 { + continue + } + _, _, err := fs.StoreMsg(fmt.Sprintf("test.%06d", n), nil, nil, 0) + require_NoError(t, err) + } + t.Logf("Publish complete") + + require_NoError(t, fs.Stop()) + fs, err = newFileStore( + FileStoreConfig{StoreDir: td, BlockSize: 1024}, + StreamConfig{ + Name: "zzz", Subjects: []string{"test.>"}, Storage: FileStorage, + MaxMsgsPer: 1, + }, + ) + require_NoError(t, err) + defer fs.Stop() + + // Mangle the filestore state and then see how long it takes to enforce + // the per-subject limit. + fs.state.Msgs++ + start := time.Now() + fs.enforceMsgPerSubjectLimit(false) + require_LessThan(t, time.Since(start), time.Minute) + t.Logf("Took %s", time.Since(start)) +} diff --git a/server/jetstream_errors.go b/server/jetstream_errors.go new file mode 100644 index 0000000..5a81aa6 --- /dev/null +++ b/server/jetstream_errors.go @@ -0,0 +1,102 @@ +package server + +import ( + "fmt" +) + +type errOpts struct { + err error +} + +// ErrorOption configures a NATS Error helper +type ErrorOption func(*errOpts) + +// Unless ensures that if err is a ApiErr that err will be returned rather than the one being created via the helper +func Unless(err error) ErrorOption { + return func(opts *errOpts) { + opts.err = err + } +} + +func parseOpts(opts []ErrorOption) *errOpts { + eopts := &errOpts{} + for _, opt := range opts { + opt(eopts) + } + return eopts +} + +type ErrorIdentifier uint16 + +// IsNatsErr determines if an error matches ID, if multiple IDs are given if the error matches any of these the function will be true +func IsNatsErr(err error, ids ...ErrorIdentifier) bool { + if err == nil { + return false + } + + ce, ok := err.(*ApiError) + if !ok || ce == nil { + return false + } + + for _, id := range ids { + ae, ok := ApiErrors[id] + if !ok || ae == nil { + continue + } + + if ce.ErrCode == ae.ErrCode { + return true + } + } + + return false +} + +// ApiError is included in all responses if there was an error. +type ApiError struct { + Code int `json:"code"` + ErrCode uint16 `json:"err_code,omitempty"` + Description string `json:"description,omitempty"` +} + +// ErrorsData is the source data for generated errors as found in errors.json +type ErrorsData struct { + Constant string `json:"constant"` + Code int `json:"code"` + ErrCode uint16 `json:"error_code"` + Description string `json:"description"` + Comment string `json:"comment"` + Help string `json:"help"` + URL string `json:"url"` + Deprecates string `json:"deprecates"` +} + +func (e *ApiError) Error() string { + return fmt.Sprintf("%s (%d)", e.Description, e.ErrCode) +} + +func (e *ApiError) toReplacerArgs(replacements []any) []string { + var ( + ra []string + key string + ) + + for i, replacement := range replacements { + if i%2 == 0 { + key = replacement.(string) + continue + } + + switch v := replacement.(type) { + case string: + ra = append(ra, key, v) + case error: + ra = append(ra, key, v.Error()) + default: + ra = append(ra, key, fmt.Sprintf("%v", v)) + } + } + + return ra +} diff --git a/server/jetstream_errors_test.go b/server/jetstream_errors_test.go new file mode 100644 index 0000000..c7adcee --- /dev/null +++ b/server/jetstream_errors_test.go @@ -0,0 +1,83 @@ +package server + +import ( + "errors" + "testing" +) + +func TestIsNatsErr(t *testing.T) { + if !IsNatsErr(ApiErrors[JSNotEnabledForAccountErr], JSNotEnabledForAccountErr) { + t.Fatalf("Expected error match") + } + + if IsNatsErr(ApiErrors[JSNotEnabledForAccountErr], JSClusterNotActiveErr) { + t.Fatalf("Expected error mismatch") + } + + if IsNatsErr(ApiErrors[JSNotEnabledForAccountErr], JSClusterNotActiveErr, JSClusterNotAvailErr) { + t.Fatalf("Expected error mismatch") + } + + if !IsNatsErr(ApiErrors[JSNotEnabledForAccountErr], JSClusterNotActiveErr, JSNotEnabledForAccountErr) { + t.Fatalf("Expected error match") + } + + if !IsNatsErr(&ApiError{ErrCode: 10039}, 1, JSClusterNotActiveErr, JSNotEnabledForAccountErr) { + t.Fatalf("Expected error match") + } + + if IsNatsErr(&ApiError{ErrCode: 10039}, 1, 2, JSClusterNotActiveErr) { + t.Fatalf("Expected error mismatch") + } + + if IsNatsErr(nil, JSClusterNotActiveErr) { + t.Fatalf("Expected error mismatch") + } + + if IsNatsErr(errors.New("x"), JSClusterNotActiveErr) { + t.Fatalf("Expected error mismatch") + } +} + +func TestApiError_Error(t *testing.T) { + if es := ApiErrors[JSClusterNotActiveErr].Error(); es != "JetStream not in clustered mode (10006)" { + t.Fatalf("Expected 'JetStream not in clustered mode (10006)', got %q", es) + } +} + +func TestApiError_NewWithTags(t *testing.T) { + ne := NewJSRestoreSubscribeFailedError(errors.New("failed error"), "the.subject") + if ne.Description != "JetStream unable to subscribe to restore snapshot the.subject: failed error" { + t.Fatalf("Expected 'JetStream unable to subscribe to restore snapshot the.subject: failed error' got %q", ne.Description) + } + + if ne == ApiErrors[JSRestoreSubscribeFailedErrF] { + t.Fatalf("Expected a new instance") + } +} + +func TestApiError_NewWithUnless(t *testing.T) { + if ne := NewJSStreamRestoreError(errors.New("failed error"), Unless(ApiErrors[JSNotEnabledForAccountErr])); !IsNatsErr(ne, JSNotEnabledForAccountErr) { + t.Fatalf("Expected JSNotEnabledForAccountErr got %s", ne) + } + + if ne := NewJSStreamRestoreError(errors.New("failed error")); !IsNatsErr(ne, JSStreamRestoreErrF) { + t.Fatalf("Expected JSStreamRestoreErrF got %s", ne) + } + + if ne := NewJSStreamRestoreError(errors.New("failed error"), Unless(errors.New("other error"))); !IsNatsErr(ne, JSStreamRestoreErrF) { + t.Fatalf("Expected JSStreamRestoreErrF got %s", ne) + } + + if ne := NewJSPeerRemapError(Unless(ApiErrors[JSNotEnabledForAccountErr])); !IsNatsErr(ne, JSNotEnabledForAccountErr) { + t.Fatalf("Expected JSNotEnabledForAccountErr got %s", ne) + } + + if ne := NewJSPeerRemapError(Unless(nil)); !IsNatsErr(ne, JSPeerRemapErr) { + t.Fatalf("Expected JSPeerRemapErr got %s", ne) + } + + if ne := NewJSPeerRemapError(Unless(errors.New("other error"))); !IsNatsErr(ne, JSPeerRemapErr) { + t.Fatalf("Expected JSPeerRemapErr got %s", ne) + } +} diff --git a/server/jetstream_events.go b/server/jetstream_events.go new file mode 100644 index 0000000..2f6180f --- /dev/null +++ b/server/jetstream_events.go @@ -0,0 +1,383 @@ +// Copyright 2020-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "encoding/json" + "time" +) + +// publishAdvisory sends the given advisory into the account. Returns true if +// it was sent, false if not (i.e. due to lack of interest or a marshal error). +func (s *Server) publishAdvisory(acc *Account, subject string, adv any) bool { + if acc == nil { + acc = s.SystemAccount() + if acc == nil { + return false + } + } + + // If there is no one listening for this advisory then save ourselves the effort + // and don't bother encoding the JSON or sending it. + if sl := acc.sl; (sl != nil && !sl.HasInterest(subject)) && !s.hasGatewayInterest(acc.Name, subject) { + return false + } + + ej, err := json.Marshal(adv) + if err == nil { + err = s.sendInternalAccountMsg(acc, subject, ej) + if err != nil { + s.Warnf("Advisory could not be sent for account %q: %v", acc.Name, err) + } + } else { + s.Warnf("Advisory could not be serialized for account %q: %v", acc.Name, err) + } + return err == nil +} + +// JSAPIAudit is an advisory about administrative actions taken on JetStream +type JSAPIAudit struct { + TypedEvent + Server string `json:"server"` + Client *ClientInfo `json:"client"` + Subject string `json:"subject"` + Request string `json:"request,omitempty"` + Response string `json:"response"` + Domain string `json:"domain,omitempty"` +} + +const JSAPIAuditType = "io.nats.jetstream.advisory.v1.api_audit" + +// ActionAdvisoryType indicates which action against a stream, consumer or template triggered an advisory +type ActionAdvisoryType string + +const ( + CreateEvent ActionAdvisoryType = "create" + DeleteEvent ActionAdvisoryType = "delete" + ModifyEvent ActionAdvisoryType = "modify" +) + +// JSStreamActionAdvisory indicates that a stream was created, edited or deleted +type JSStreamActionAdvisory struct { + TypedEvent + Stream string `json:"stream"` + Action ActionAdvisoryType `json:"action"` + Domain string `json:"domain,omitempty"` +} + +const JSStreamActionAdvisoryType = "io.nats.jetstream.advisory.v1.stream_action" + +// JSConsumerActionAdvisory indicates that a consumer was created or deleted +type JSConsumerActionAdvisory struct { + TypedEvent + Stream string `json:"stream"` + Consumer string `json:"consumer"` + Action ActionAdvisoryType `json:"action"` + Domain string `json:"domain,omitempty"` +} + +const JSConsumerActionAdvisoryType = "io.nats.jetstream.advisory.v1.consumer_action" + +// JSConsumerPauseAdvisory indicates that a consumer was paused or unpaused +type JSConsumerPauseAdvisory struct { + TypedEvent + Stream string `json:"stream"` + Consumer string `json:"consumer"` + Paused bool `json:"paused"` + PauseUntil time.Time `json:"pause_until,omitempty"` + Domain string `json:"domain,omitempty"` +} + +const JSConsumerPauseAdvisoryType = "io.nats.jetstream.advisory.v1.consumer_pause" + +// JSConsumerAckMetric is a metric published when a user acknowledges a message, the +// number of these that will be published is dependent on SampleFrequency +type JSConsumerAckMetric struct { + TypedEvent + Stream string `json:"stream"` + Consumer string `json:"consumer"` + ConsumerSeq uint64 `json:"consumer_seq"` + StreamSeq uint64 `json:"stream_seq"` + Delay int64 `json:"ack_time"` + Deliveries uint64 `json:"deliveries"` + Domain string `json:"domain,omitempty"` +} + +// JSConsumerAckMetricType is the schema type for JSConsumerAckMetricType +const JSConsumerAckMetricType = "io.nats.jetstream.metric.v1.consumer_ack" + +// JSConsumerDeliveryExceededAdvisory is an advisory informing that a message hit +// its MaxDeliver threshold and so might be a candidate for DLQ handling +type JSConsumerDeliveryExceededAdvisory struct { + TypedEvent + Stream string `json:"stream"` + Consumer string `json:"consumer"` + StreamSeq uint64 `json:"stream_seq"` + Deliveries uint64 `json:"deliveries"` + Domain string `json:"domain,omitempty"` +} + +// JSConsumerDeliveryExceededAdvisoryType is the schema type for JSConsumerDeliveryExceededAdvisory +const JSConsumerDeliveryExceededAdvisoryType = "io.nats.jetstream.advisory.v1.max_deliver" + +// JSConsumerDeliveryNakAdvisory is an advisory informing that a message was +// naked by the consumer +type JSConsumerDeliveryNakAdvisory struct { + TypedEvent + Stream string `json:"stream"` + Consumer string `json:"consumer"` + ConsumerSeq uint64 `json:"consumer_seq"` + StreamSeq uint64 `json:"stream_seq"` + Deliveries uint64 `json:"deliveries"` + Domain string `json:"domain,omitempty"` +} + +// JSConsumerDeliveryNakAdvisoryType is the schema type for JSConsumerDeliveryNakAdvisory +const JSConsumerDeliveryNakAdvisoryType = "io.nats.jetstream.advisory.v1.nak" + +// JSConsumerDeliveryTerminatedAdvisory is an advisory informing that a message was +// terminated by the consumer, so might be a candidate for DLQ handling +type JSConsumerDeliveryTerminatedAdvisory struct { + TypedEvent + Stream string `json:"stream"` + Consumer string `json:"consumer"` + ConsumerSeq uint64 `json:"consumer_seq"` + StreamSeq uint64 `json:"stream_seq"` + Deliveries uint64 `json:"deliveries"` + Reason string `json:"reason,omitempty"` + Domain string `json:"domain,omitempty"` +} + +// JSConsumerDeliveryTerminatedAdvisoryType is the schema type for JSConsumerDeliveryTerminatedAdvisory +const JSConsumerDeliveryTerminatedAdvisoryType = "io.nats.jetstream.advisory.v1.terminated" + +// JSSnapshotCreateAdvisory is an advisory sent after a snapshot is successfully started +type JSSnapshotCreateAdvisory struct { + TypedEvent + Stream string `json:"stream"` + State StreamState `json:"state"` + Client *ClientInfo `json:"client"` + Domain string `json:"domain,omitempty"` +} + +// JSSnapshotCreatedAdvisoryType is the schema type for JSSnapshotCreateAdvisory +const JSSnapshotCreatedAdvisoryType = "io.nats.jetstream.advisory.v1.snapshot_create" + +// JSSnapshotCompleteAdvisory is an advisory sent after a snapshot is successfully started +type JSSnapshotCompleteAdvisory struct { + TypedEvent + Stream string `json:"stream"` + Start time.Time `json:"start"` + End time.Time `json:"end"` + Client *ClientInfo `json:"client"` + Domain string `json:"domain,omitempty"` + Error string `json:"error,omitempty"` +} + +// JSSnapshotCompleteAdvisoryType is the schema type for JSSnapshotCreateAdvisory +const JSSnapshotCompleteAdvisoryType = "io.nats.jetstream.advisory.v1.snapshot_complete" + +// JSRestoreCreateAdvisory is an advisory sent after a snapshot is successfully started +type JSRestoreCreateAdvisory struct { + TypedEvent + Stream string `json:"stream"` + Client *ClientInfo `json:"client"` + Domain string `json:"domain,omitempty"` +} + +// JSRestoreCreateAdvisoryType is the schema type for JSSnapshotCreateAdvisory +const JSRestoreCreateAdvisoryType = "io.nats.jetstream.advisory.v1.restore_create" + +// JSRestoreCompleteAdvisory is an advisory sent after a snapshot is successfully started +type JSRestoreCompleteAdvisory struct { + TypedEvent + Stream string `json:"stream"` + Start time.Time `json:"start"` + End time.Time `json:"end"` + Bytes int64 `json:"bytes"` + Client *ClientInfo `json:"client"` + Domain string `json:"domain,omitempty"` +} + +// JSRestoreCompleteAdvisoryType is the schema type for JSSnapshotCreateAdvisory +const JSRestoreCompleteAdvisoryType = "io.nats.jetstream.advisory.v1.restore_complete" + +// Clustering specific. + +// JSClusterLeaderElectedAdvisoryType is sent when the system elects a new meta leader. +const JSDomainLeaderElectedAdvisoryType = "io.nats.jetstream.advisory.v1.domain_leader_elected" + +// JSClusterLeaderElectedAdvisory indicates that a domain has elected a new leader. +type JSDomainLeaderElectedAdvisory struct { + TypedEvent + Leader string `json:"leader"` + Replicas []*PeerInfo `json:"replicas"` + Cluster string `json:"cluster"` + Domain string `json:"domain,omitempty"` +} + +// JSStreamLeaderElectedAdvisoryType is sent when the system elects a new leader for a stream. +const JSStreamLeaderElectedAdvisoryType = "io.nats.jetstream.advisory.v1.stream_leader_elected" + +// JSStreamLeaderElectedAdvisory indicates that a stream has elected a new leader. +type JSStreamLeaderElectedAdvisory struct { + TypedEvent + Account string `json:"account,omitempty"` + Stream string `json:"stream"` + Leader string `json:"leader"` + Replicas []*PeerInfo `json:"replicas"` + Domain string `json:"domain,omitempty"` +} + +// JSStreamQuorumLostAdvisoryType is sent when the system detects a clustered stream and +// its consumers are stalled and unable to make progress. +const JSStreamQuorumLostAdvisoryType = "io.nats.jetstream.advisory.v1.stream_quorum_lost" + +// JSStreamQuorumLostAdvisory indicates that a stream has lost quorum and is stalled. +type JSStreamQuorumLostAdvisory struct { + TypedEvent + Account string `json:"account,omitempty"` + Stream string `json:"stream"` + Replicas []*PeerInfo `json:"replicas"` + Domain string `json:"domain,omitempty"` +} + +// JSStreamBatchAbandonedAdvisoryType is sent when a stream's atomic batch is abandoned. +const JSStreamBatchAbandonedAdvisoryType = "io.nats.jetstream.advisory.v1.stream_batch_abandoned" + +// JSStreamBatchAbandonedAdvisory indicates that a stream's batch was abandoned. +type JSStreamBatchAbandonedAdvisory struct { + TypedEvent + Account string `json:"account,omitempty"` + Stream string `json:"stream"` + Domain string `json:"domain,omitempty"` + BatchId string `json:"batch"` + Reason BatchAbandonReason `json:"reason"` +} + +type BatchAbandonReason string + +var ( + BatchTimeout BatchAbandonReason = "timeout" + BatchLarge BatchAbandonReason = "large" + BatchIncomplete BatchAbandonReason = "incomplete" + BatchRequirementsNotMet BatchAbandonReason = "unsupported" +) + +// JSConsumerLeaderElectedAdvisoryType is sent when the system elects a leader for a consumer. +const JSConsumerLeaderElectedAdvisoryType = "io.nats.jetstream.advisory.v1.consumer_leader_elected" + +// JSConsumerLeaderElectedAdvisory indicates that a consumer has elected a new leader. +type JSConsumerLeaderElectedAdvisory struct { + TypedEvent + Account string `json:"account,omitempty"` + Stream string `json:"stream"` + Consumer string `json:"consumer"` + Leader string `json:"leader"` + Replicas []*PeerInfo `json:"replicas"` + Domain string `json:"domain,omitempty"` +} + +// JSConsumerQuorumLostAdvisoryType is sent when the system detects a clustered consumer and +// is stalled and unable to make progress. +const JSConsumerQuorumLostAdvisoryType = "io.nats.jetstream.advisory.v1.consumer_quorum_lost" + +// JSConsumerQuorumLostAdvisory indicates that a consumer has lost quorum and is stalled. +type JSConsumerQuorumLostAdvisory struct { + TypedEvent + Account string `json:"account,omitempty"` + Stream string `json:"stream"` + Consumer string `json:"consumer"` + Replicas []*PeerInfo `json:"replicas"` + Domain string `json:"domain,omitempty"` +} + +const JSConsumerGroupPinnedAdvisoryType = "io.nats.jetstream.advisory.v1.consumer_group_pinned" + +// JSConsumerGroupPinnedAdvisory that a group switched to a new pinned client +type JSConsumerGroupPinnedAdvisory struct { + TypedEvent + Account string `json:"account,omitempty"` + Stream string `json:"stream"` + Consumer string `json:"consumer"` + Domain string `json:"domain,omitempty"` + Group string `json:"group"` + PinnedClientId string `json:"pinned_id"` +} + +const JSConsumerGroupUnpinnedAdvisoryType = "io.nats.jetstream.advisory.v1.consumer_group_unpinned" + +// JSConsumerGroupUnpinnedAdvisory indicates that a pin was lost +type JSConsumerGroupUnpinnedAdvisory struct { + TypedEvent + Account string `json:"account,omitempty"` + Stream string `json:"stream"` + Consumer string `json:"consumer"` + Domain string `json:"domain,omitempty"` + Group string `json:"group"` + // one of "admin" or "timeout", could be an enum up to the implementor to decide + Reason string `json:"reason"` +} + +// JSServerOutOfStorageAdvisoryType is sent when the server is out of storage space. +const JSServerOutOfStorageAdvisoryType = "io.nats.jetstream.advisory.v1.server_out_of_space" + +// JSServerOutOfSpaceAdvisory indicates that a stream has lost quorum and is stalled. +type JSServerOutOfSpaceAdvisory struct { + TypedEvent + Server string `json:"server"` + ServerID string `json:"server_id"` + Stream string `json:"stream,omitempty"` + Cluster string `json:"cluster"` + Domain string `json:"domain,omitempty"` +} + +// JSServerRemovedAdvisoryType is sent when the server has been removed and JS disabled. +const JSServerRemovedAdvisoryType = "io.nats.jetstream.advisory.v1.server_removed" + +// JSServerRemovedAdvisory indicates that a stream has lost quorum and is stalled. +type JSServerRemovedAdvisory struct { + TypedEvent + Server string `json:"server"` + ServerID string `json:"server_id"` + Cluster string `json:"cluster"` + Domain string `json:"domain,omitempty"` +} + +// JSMetaRescueAdvisoryType is sent when a server unsafely lowers the meta +// group's quorum requirement for disaster recovery. +const JSMetaRescueAdvisoryType = "io.nats.jetstream.advisory.v1.meta_rescue" + +// JSMetaRescueAdvisory indicates that a server has unsafely lowered the meta +// group's quorum requirement for disaster recovery. +type JSMetaRescueAdvisory struct { + TypedEvent + Server string `json:"server"` + ServerID string `json:"server_id"` + PrevQuorum int `json:"prev_quorum"` + NewQuorum int `json:"new_quorum"` + Cluster string `json:"cluster"` + Domain string `json:"domain,omitempty"` +} + +// JSAPILimitReachedAdvisoryType is sent when the JS API request queue limit is reached. +const JSAPILimitReachedAdvisoryType = "io.nats.jetstream.advisory.v1.api_limit_reached" + +// JSAPILimitReachedAdvisory is a advisory published when JetStream hits the queue length limit. +type JSAPILimitReachedAdvisory struct { + TypedEvent + Server string `json:"server"` // Server that created the event, name or ID + Domain string `json:"domain,omitempty"` // Domain the server belongs to + Dropped int64 `json:"dropped"` // How many messages did we drop from the queue +} diff --git a/server/jetstream_helpers_test.go b/server/jetstream_helpers_test.go new file mode 100644 index 0000000..814e584 --- /dev/null +++ b/server/jetstream_helpers_test.go @@ -0,0 +1,2551 @@ +// Copyright 2020-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Do not exlude this file with the !skip_js_tests since those helpers +// are also used by MQTT. + +//lint:file-ignore U1000 Avoid detecting as unused code + +package server + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "math/rand/v2" + "net" + "net/url" + "os" + "path" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "golang.org/x/time/rate" + + "github.com/nats-io/nats-server/v2/internal/antithesis" +) + +// Support functions + +func init() { + // Speed up raft for tests. + hbInterval = 50 * time.Millisecond + minElectionTimeout = 1500 * time.Millisecond + maxElectionTimeout = 3500 * time.Millisecond + lostQuorumInterval = 5 * time.Second + lostQuorumCheck = 4 * hbInterval + + // For statz and jetstream placement speedups as well. + statszRateLimit = 0 +} + +// Used to setup clusters of clusters for tests. +type cluster struct { + servers []*Server + opts []*Options + name string + t testing.TB + nproxies []*netProxy +} + +// Used to setup superclusters for tests. +type supercluster struct { + t *testing.T + clusters []*cluster + nproxies []*netProxy +} + +func (sc *supercluster) shutdown() { + if sc == nil { + return + } + for _, np := range sc.nproxies { + np.stop() + } + for _, c := range sc.clusters { + shutdownCluster(c) + } +} + +func (sc *supercluster) randomServer() *Server { + return sc.randomCluster().randomServer() +} + +func (sc *supercluster) serverByName(sname string) *Server { + for _, c := range sc.clusters { + if s := c.serverByName(sname); s != nil { + return s + } + } + return nil +} + +func (sc *supercluster) waitOnStreamLeader(account, stream string) { + sc.t.Helper() + expires := time.Now().Add(30 * time.Second) + for time.Now().Before(expires) { + for _, c := range sc.clusters { + leader := c.streamLeader(account, stream) + if leader == nil { + time.Sleep(100 * time.Millisecond) + continue + } + // Also wait for the stream leader to finish any scales/moves. + js := leader.getJetStream() + js.mu.RLock() + sa := js.streamAssignment(account, stream) + wait := sa == nil || (sa.Group != nil && sa.Group.Desired != nil) + js.mu.RUnlock() + if wait { + time.Sleep(100 * time.Millisecond) + continue + } + time.Sleep(200 * time.Millisecond) + return + } + time.Sleep(100 * time.Millisecond) + } + antithesis.AssertUnreachable(sc.t, "Timeout in supercluster.waitOnStreamLeader", map[string]any{ + "account": account, + "stream": stream, + }) + sc.t.Fatalf("Expected a stream leader for %q %q, got none", account, stream) +} + +var jsClusterAccountsTempl = ` + listen: 127.0.0.1:-1 + + server_name: %s + jetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'} + + leaf { + listen: 127.0.0.1:-1 + } + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + websocket { + listen: 127.0.0.1:-1 + compression: true + handshake_timeout: "5s" + no_tls: true + } + + no_auth_user: one + + accounts { + ONE { users = [ { user: "one", pass: "p" } ]; jetstream: enabled } + TWO { users = [ { user: "two", pass: "p" } ]; jetstream: enabled } + NOJS { users = [ { user: "nojs", pass: "p" } ] } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } +` + +var jsClusterTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'} + + leaf { + listen: 127.0.0.1:-1 + } + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } +` + +var jsClusterSyncAlwaysTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s', sync_interval: always} + + leaf { + listen: 127.0.0.1:-1 + } + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } +` + +var jsClusterEncryptedTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s', key: "s3cr3t!"} + + leaf { + listen: 127.0.0.1:-1 + } + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } +` + +var jsClusterMaxBytesTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'} + + leaf { + listen: 127.0.0.1:-1 + } + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + no_auth_user: u + + accounts { + $U { + users = [ { user: "u", pass: "p" } ] + jetstream: { + max_mem: 128MB + max_file: 18GB + max_bytes: true // Forces streams to indicate max_bytes. + } + } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } +` + +var jsClusterMaxBytesAccountLimitTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 4GB, store_dir: '%s'} + + leaf { + listen: 127.0.0.1:-1 + } + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + no_auth_user: u + + accounts { + $U { + users = [ { user: "u", pass: "p" } ] + jetstream: { + max_mem: 128MB + max_file: 3GB + max_bytes: true // Forces streams to indicate max_bytes. + } + } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } +` + +var jsSuperClusterTempl = ` + %s + gateway { + name: %s + listen: 127.0.0.1:%d + gateways = [%s + ] + } + + system_account: "$SYS" +` + +var jsClusterLimitsTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'} + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + no_auth_user: u + + accounts { + ONE { + users = [ { user: "u", pass: "s3cr3t!" } ] + jetstream: enabled + } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } +` + +var jsMixedModeGlobalAccountTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'} + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + accounts {$SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } +` + +var jsGWTempl = `%s{name: %s, urls: [%s]}` + +var jsClusterAccountLimitsTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'} + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + no_auth_user: js + + accounts { + $JS { users = [ { user: "js", pass: "p" } ]; jetstream: {max_store: 1MB, max_mem: 0} } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } +` + +func createJetStreamTaggedSuperCluster(t *testing.T) *supercluster { + return createJetStreamTaggedSuperClusterWithGWProxy(t, nil) +} + +func createJetStreamTaggedSuperClusterWithGWProxy(t *testing.T, gwm gwProxyMap) *supercluster { + sc := createJetStreamSuperClusterWithTemplateAndModHook(t, jsClusterTempl, 3, 3, nil, gwm) + sc.waitOnPeerCount(9) + + reset := func(s *Server) { + s.mu.Lock() + rch := s.sys.resetCh + s.mu.Unlock() + if rch != nil { + rch <- struct{}{} + } + s.sendStatszUpdate() + } + + // Make first cluster AWS, US country code. + for i, s := range sc.clusterForName("C1").servers { + s.optsMu.Lock() + s.opts.Tags.Add("cloud:aws") + s.opts.Tags.Add("country:us") + s.opts.Tags.Add(fmt.Sprintf("node:%d", i+1)) + s.optsMu.Unlock() + reset(s) + } + // Make second cluster GCP, UK country code. + for i, s := range sc.clusterForName("C2").servers { + s.optsMu.Lock() + s.opts.Tags.Add("cloud:gcp") + s.opts.Tags.Add("country:uk") + s.opts.Tags.Add(fmt.Sprintf("node:%d", i+1)) + s.optsMu.Unlock() + reset(s) + } + // Make third cluster AZ, JP country code. + for i, s := range sc.clusterForName("C3").servers { + s.optsMu.Lock() + s.opts.Tags.Add("cloud:az") + s.opts.Tags.Add("country:jp") + s.opts.Tags.Add(fmt.Sprintf("node:%d", i+1)) + s.optsMu.Unlock() + reset(s) + } + + sc.waitOnLeader() + ml := sc.leader() + require_True(t, ml != nil) + js := ml.getJetStream() + require_True(t, js != nil) + js.mu.RLock() + defer js.mu.RUnlock() + cc := js.cluster + require_True(t, cc != nil) + + // Walk and make sure all tags are registered. + expires := time.Now().Add(10 * time.Second) + for time.Now().Before(expires) { + allOK := true + for _, p := range cc.meta.Peers() { + si, ok := ml.nodeToInfo.Load(p.ID) + require_True(t, ok) + ni := si.(nodeInfo) + if len(ni.tags) == 0 { + allOK = false + reset(sc.serverByName(ni.name)) + } + } + if allOK { + break + } + } + + return sc +} + +func createJetStreamSuperCluster(t *testing.T, numServersPer, numClusters int) *supercluster { + return createJetStreamSuperClusterWithTemplate(t, jsClusterTempl, numServersPer, numClusters) +} + +func createJetStreamSuperClusterWithTemplate(t *testing.T, tmpl string, numServersPer, numClusters int) *supercluster { + return createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, numServersPer, numClusters, nil, nil) +} + +// For doing proxyies in GWs. +type gwProxy struct { + rtt time.Duration + up int + down int +} + +// For use in normal clusters. +type clusterProxy = gwProxy + +// Maps cluster names to proxy settings. +type gwProxyMap map[string]*gwProxy + +func createJetStreamSuperClusterWithTemplateAndModHook(t *testing.T, tmpl string, numServersPer, numClusters int, modify modifyCb, gwm gwProxyMap) *supercluster { + t.Helper() + if numServersPer < 1 { + t.Fatalf("Number of servers must be >= 1") + } + if numClusters <= 1 { + t.Fatalf("Number of clusters must be > 1") + } + + startClusterPorts := []int{20_022, 22_022, 24_022} + startGatewayPorts := []int{20_122, 22_122, 24_122} + startClusterPort := startClusterPorts[rand.IntN(len(startClusterPorts))] + startGWPort := startGatewayPorts[rand.IntN(len(startGatewayPorts))] + + // Make the GWs form faster for the tests. + SetGatewaysSolicitDelay(10 * time.Millisecond) + defer ResetGatewaysSolicitDelay() + + cp, gp := startClusterPort, startGWPort + var clusters []*cluster + var nproxies []*netProxy + var gws []string + + // Build GWs first, will be same for all servers. + for i, port := 1, gp; i <= numClusters; i++ { + cn := fmt.Sprintf("C%d", i) + var gwp *gwProxy + if len(gwm) > 0 { + gwp = gwm[cn] + } + var urls []string + for n := 0; n < numServersPer; n++ { + routeURL := fmt.Sprintf("nats-route://127.0.0.1:%d", port) + if gwp != nil { + np := createNetProxy(gwp.rtt, gwp.up, gwp.down, routeURL, false) + nproxies = append(nproxies, np) + routeURL = np.routeURL() + } + urls = append(urls, routeURL) + port++ + } + gws = append(gws, fmt.Sprintf(jsGWTempl, "\n\t\t\t", cn, strings.Join(urls, ","))) + } + gwconf := strings.Join(gws, _EMPTY_) + + for i := 1; i <= numClusters; i++ { + cn := fmt.Sprintf("C%d", i) + // Go ahead and build configurations. + c := &cluster{servers: make([]*Server, 0, numServersPer), opts: make([]*Options, 0, numServersPer), name: cn} + + // Build out the routes that will be shared with all configs. + var routes []string + for port := cp; port < cp+numServersPer; port++ { + routes = append(routes, fmt.Sprintf("nats-route://127.0.0.1:%d", port)) + } + routeConfig := strings.Join(routes, ",") + + for si := 0; si < numServersPer; si++ { + storeDir := t.TempDir() + sn := fmt.Sprintf("%s-S%d", cn, si+1) + bconf := fmt.Sprintf(tmpl, sn, storeDir, cn, cp+si, routeConfig) + conf := fmt.Sprintf(jsSuperClusterTempl, bconf, cn, gp, gwconf) + gp++ + if modify != nil { + conf = modify(sn, cn, storeDir, conf) + } + s, o := RunServerWithConfig(createConfFile(t, []byte(conf))) + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + } + checkClusterFormed(t, c.servers...) + clusters = append(clusters, c) + cp += numServersPer + c.t = t + } + + // Start any proxies. + for _, np := range nproxies { + np.start() + } + + // Wait for the supercluster to be formed. + egws := numClusters - 1 + for _, c := range clusters { + for _, s := range c.servers { + waitForOutboundGateways(t, s, egws, 10*time.Second) + } + } + + sc := &supercluster{t, clusters, nproxies} + sc.campaignMetaImmediately() + sc.waitOnLeader() + sc.waitOnAllCurrent() + + // Wait for all the peer nodes to be registered. + checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { + var peers []string + if ml := sc.leader(); ml != nil { + peers = ml.ActivePeers() + if len(peers) == numClusters*numServersPer { + return nil + } + } + return fmt.Errorf("Not correct number of peers, expected %d, got %d", numClusters*numServersPer, len(peers)) + }) + + if sc.leader() == nil { + sc.t.Fatalf("Expected a cluster leader, got none") + } + + return sc +} + +func (sc *supercluster) createLeafNodes(clusterName string, numServers int) *cluster { + sc.t.Helper() + + // Create our leafnode cluster template first. + return sc.createLeafNodesWithDomain(clusterName, numServers, "") +} + +func (sc *supercluster) createLeafNodesWithDomain(clusterName string, numServers int, domain string) *cluster { + sc.t.Helper() + + // Create our leafnode cluster template first. + return sc.randomCluster().createLeafNodes(clusterName, numServers, domain) +} + +func (sc *supercluster) createSingleLeafNode(extend bool) *Server { + sc.t.Helper() + + return sc.randomCluster().createLeafNode(extend) +} + +func (sc *supercluster) leader() *Server { + for _, c := range sc.clusters { + if leader := c.leader(); leader != nil { + return leader + } + } + return nil +} + +func (sc *supercluster) waitOnLeader() { + sc.t.Helper() + expires := time.Now().Add(30 * time.Second) + for time.Now().Before(expires) { + for _, c := range sc.clusters { + if leader := c.leader(); leader != nil { + time.Sleep(250 * time.Millisecond) + return + } + } + time.Sleep(25 * time.Millisecond) + } + antithesis.AssertUnreachable(sc.t, "Timeout in supercluster.waitOnLeader", nil) + sc.t.Fatalf("Expected a cluster leader, got none") +} + +func (sc *supercluster) waitOnAccount(account string) { + sc.t.Helper() + expires := time.Now().Add(40 * time.Second) + for time.Now().Before(expires) { + found := true + for _, c := range sc.clusters { + for _, s := range c.servers { + acc, err := s.fetchAccount(account) + found = found && err == nil && acc != nil + } + } + if found { + return + } + time.Sleep(100 * time.Millisecond) + continue + } + + sc.t.Fatalf("Expected account %q to exist but didn't", account) +} + +func (sc *supercluster) campaignMetaImmediately() { + var servers []*Server + for _, c := range sc.clusters { + servers = append(servers, c.servers...) + } + campaignMetaImmediately(servers) +} + +func (sc *supercluster) waitOnAllCurrent() { + sc.t.Helper() + for _, c := range sc.clusters { + c.waitOnAllCurrent() + } +} + +func (sc *supercluster) clusterForName(name string) *cluster { + for _, c := range sc.clusters { + if c.name == name { + return c + } + } + return nil +} + +func (sc *supercluster) randomCluster() *cluster { + clusters := append(sc.clusters[:0:0], sc.clusters...) + rand.Shuffle(len(clusters), func(i, j int) { clusters[i], clusters[j] = clusters[j], clusters[i] }) + return clusters[0] +} + +func (sc *supercluster) waitOnPeerCount(n int) { + sc.t.Helper() + sc.waitOnLeader() + leader := sc.leader() + expires := time.Now().Add(30 * time.Second) + // Make sure we have all peers, and take into account the meta leader could still change. + for time.Now().Before(expires) { + if leader = sc.leader(); leader == nil { + time.Sleep(50 * time.Millisecond) + continue + } + if len(leader.JetStreamClusterPeers()) == n { + return + } + time.Sleep(100 * time.Millisecond) + } + antithesis.AssertUnreachable(sc.t, "Timeout in supercluster.waitOnPeerCount", nil) + sc.t.Fatalf("Expected a super cluster peer count of %d, got %d", n, len(leader.JetStreamClusterPeers())) +} + +var jsClusterMirrorSourceImportsTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'} + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + no_auth_user: dlc + + accounts { + JS { + jetstream: enabled + users = [ { user: "rip", pass: "pass" } ] + exports [ + { service: "$JS.API.CONSUMER.>" } # To create internal consumers to mirror/source. + { stream: "RI.DELIVER.SYNC.>" } # For the mirror/source consumers sending to IA via delivery subject. + { service: "$JS.FC.>" } + ] + } + IA { + jetstream: enabled + users = [ { user: "dlc", pass: "pass" } ] + imports [ + { service: { account: JS, subject: "$JS.API.CONSUMER.>"}, to: "RI.JS.API.CONSUMER.>" } + { stream: { account: JS, subject: "RI.DELIVER.SYNC.>"} } + { service: {account: JS, subject: "$JS.FC.>" }} + ] + } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } +` + +var jsClusterImportsTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'} + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + no_auth_user: dlc + + accounts { + JS { + jetstream: enabled + users = [ { user: "rip", pass: "pass" } ] + exports [ + { service: "$JS.API.>", response: stream } + { service: "TEST" } # For publishing to the stream. + { service: "$JS.ACK.TEST.*.>" } + { service: "$JS.ACK.*.*.TEST.*.>" } + ] + } + IA { + users = [ { user: "dlc", pass: "pass" } ] + imports [ + { service: { subject: "$JS.API.>", account: JS }} + { service: { subject: "TEST", account: JS }} + { service: { subject: "$JS.ACK.TEST.*.>", account: JS }} + { service: { subject: "$JS.ACK.*.*.TEST.*.>", account: JS }} + ] + } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } +` + +func createMixedModeCluster(t testing.TB, tmpl string, clusterName, snPre string, numJsServers, numNonServers int, doJSConfig bool) *cluster { + t.Helper() + + if clusterName == _EMPTY_ || numJsServers < 0 || numNonServers < 1 { + t.Fatalf("Bad params") + } + + numServers := numJsServers + numNonServers + const startClusterPort = 23232 + + // Build out the routes that will be shared with all configs. + var routes []string + for cp := startClusterPort; cp < startClusterPort+numServers; cp++ { + routes = append(routes, fmt.Sprintf("nats-route://127.0.0.1:%d", cp)) + } + routeConfig := strings.Join(routes, ",") + + // Go ahead and build configurations and start servers. + c := &cluster{servers: make([]*Server, 0, numServers), opts: make([]*Options, 0, numServers), name: clusterName} + + for cp := startClusterPort; cp < startClusterPort+numServers; cp++ { + storeDir := t.TempDir() + + sn := fmt.Sprintf("%sS-%d", snPre, cp-startClusterPort+1) + conf := fmt.Sprintf(tmpl, sn, storeDir, clusterName, cp, routeConfig) + + // Disable JS here. + if cp-startClusterPort >= numJsServers { + // We can disable by commmenting it out, meaning no JS config, or can set the config up and just set disabled. + // e.g. jetstream: {domain: "SPOKE", enabled: false} + if doJSConfig { + conf = strings.Replace(conf, "jetstream: {", "jetstream: { enabled: false, ", 1) + } else { + conf = strings.Replace(conf, "jetstream: ", "# jetstream: ", 1) + } + } + + s, o := RunServerWithConfig(createConfFile(t, []byte(conf))) + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + } + c.t = t + + // Wait til we are formed and have a leader. + c.checkClusterFormed() + if numJsServers > 0 { + c.campaignMetaImmediately() + c.waitOnPeerCount(numJsServers) + } + + return c +} + +// This will create a cluster that is explicitly configured for the routes, etc. +// and also has a defined clustername. All configs for routes and cluster name will be the same. +func createJetStreamClusterExplicit(t testing.TB, clusterName string, numServers int) *cluster { + return createJetStreamClusterWithTemplate(t, jsClusterTempl, clusterName, numServers) +} + +func createJetStreamClusterWithTemplate(t testing.TB, tmpl string, clusterName string, numServers int) *cluster { + return createJetStreamClusterWithTemplateAndModHook(t, tmpl, clusterName, numServers, nil) +} + +func createJetStreamClusterWithTemplateAndModHook(t testing.TB, tmpl string, clusterName string, numServers int, modify modifyCb) *cluster { + startPorts := []int{7_022, 9_022, 11_022, 15_022} + port := startPorts[rand.IntN(len(startPorts))] + return createJetStreamClusterAndModHook(t, tmpl, clusterName, _EMPTY_, numServers, port, true, modify) +} + +func createJetStreamCluster(t testing.TB, tmpl string, clusterName, snPre string, numServers int, portStart int, waitOnReady bool) *cluster { + return createJetStreamClusterAndModHook(t, tmpl, clusterName, snPre, numServers, portStart, waitOnReady, nil) +} + +type modifyCb func(serverName, clusterName, storeDir, conf string) string + +func createJetStreamClusterAndModHook(t testing.TB, tmpl, cName, snPre string, numServers int, portStart int, waitOnReady bool, modify modifyCb) *cluster { + return createJetStreamClusterEx(t, tmpl, cName, snPre, numServers, portStart, waitOnReady, modify, nil) +} + +func createJetStreamClusterWithNetProxy(t testing.TB, cName string, numServers int, cnp *clusterProxy) *cluster { + startPorts := []int{7_122, 9_122, 11_122, 15_122} + port := startPorts[rand.IntN(len(startPorts))] + return createJetStreamClusterEx(t, jsClusterTempl, cName, _EMPTY_, numServers, port, true, nil, cnp) +} + +func createJetStreamClusterEx(t testing.TB, tmpl, cName, snPre string, numServers int, portStart int, wait bool, modify modifyCb, cnp *clusterProxy) *cluster { + t.Helper() + if cName == _EMPTY_ || numServers < 1 { + t.Fatalf("Bad params") + } + + // Flaky test prevention: + // Binding a socket to IP stack port 0 will bind an ephemeral port from an OS-specific range. + // If someone passes in to us a port spec which would cover that range, the test would be flaky. + // Adjust these ports to be the most inclusive across the port runner OSes. + // Linux: /proc/sys/net/ipv4/ip_local_port_range : 32768:60999 + // is useful, and shows there's no safe available range without OS-specific tuning. + // Our tests are usually run on Linux. Folks who care about other OSes: if you can't tune your test-runner OS to match, please + // propose a viable alternative. + const prohibitedPortFirst = 32768 + const prohibitedPortLast = 60999 + if (portStart >= prohibitedPortFirst && portStart <= prohibitedPortLast) || + (portStart+numServers-1 >= prohibitedPortFirst && portStart+numServers-1 <= prohibitedPortLast) { + t.Fatalf("test setup failure: may not specify a cluster port range which falls within %d:%d", prohibitedPortFirst, prohibitedPortLast) + } + + // Build out the routes that will be shared with all configs. + var routes []string + var nproxies []*netProxy + for cp := portStart; cp < portStart+numServers; cp++ { + routeURL := fmt.Sprintf("nats-route://127.0.0.1:%d", cp) + if cnp != nil { + np := createNetProxy(cnp.rtt, cnp.up, cnp.down, routeURL, false) + nproxies = append(nproxies, np) + routeURL = np.routeURL() + } + routes = append(routes, routeURL) + } + routeConfig := strings.Join(routes, ",") + + // Go ahead and build configurations and start servers. + c := &cluster{servers: make([]*Server, 0, numServers), opts: make([]*Options, 0, numServers), name: cName, nproxies: nproxies} + + // Start any proxies. + for _, np := range nproxies { + np.start() + } + + for cp := portStart; cp < portStart+numServers; cp++ { + storeDir := t.TempDir() + sn := fmt.Sprintf("%sS-%d", snPre, cp-portStart+1) + conf := fmt.Sprintf(tmpl, sn, storeDir, cName, cp, routeConfig) + if modify != nil { + conf = modify(sn, cName, storeDir, conf) + } + s, o := RunServerWithConfig(createConfFile(t, []byte(conf))) + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + } + c.t = t + + // Wait til we are formed and have a leader. + c.checkClusterFormed() + if wait { + c.campaignMetaImmediately() + c.waitOnClusterReady() + } + + return c +} + +func (c *cluster) campaignMetaImmediately() { + campaignMetaImmediately(c.servers) +} + +func campaignMetaImmediately(servers []*Server) { + // Only campaign on a single meta group, but check all servers first + // since one could already know of a leader. Pick the campaigning server + // at random so tests don't always end up with the same meta leader. + var nodes []RaftNode + for _, s := range servers { + js := s.getJetStream() + if js == nil { + continue + } + n := js.getMetaGroup() + if n == nil { + continue + } + if n.GroupLeader() != _EMPTY_ { + return + } + nodes = append(nodes, n) + } + if len(nodes) > 0 { + nodes[rand.IntN(len(nodes))].CampaignImmediately() + } +} + +func (c *cluster) addInNewServer() *Server { + c.t.Helper() + sn := fmt.Sprintf("S-%d", len(c.servers)+1) + storeDir := c.t.TempDir() + seedRoute := fmt.Sprintf("nats-route://127.0.0.1:%d", c.opts[0].Cluster.Port) + conf := fmt.Sprintf(jsClusterTempl, sn, storeDir, c.name, -1, seedRoute) + s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + c.checkClusterFormed() + return s +} + +// This is tied to jsClusterAccountsTempl, so changes there to users needs to be reflected here. +func (c *cluster) createSingleLeafNodeNoSystemAccount() *Server { + as := c.randomServer() + lno := as.getOpts().LeafNode + ln1 := fmt.Sprintf("nats://one:p@%s:%d", lno.Host, lno.Port) + ln2 := fmt.Sprintf("nats://two:p@%s:%d", lno.Host, lno.Port) + conf := fmt.Sprintf(jsClusterSingleLeafNodeTempl, c.t.TempDir(), ln1, ln2) + s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + + checkLeafNodeConnectedCount(c.t, as, 2) + + return s +} + +// This is tied to jsClusterAccountsTempl, so changes there to users needs to be reflected here. +func (c *cluster) createSingleLeafNodeNoSystemAccountAndEnablesJetStream() *Server { + return c.createSingleLeafNodeNoSystemAccountAndEnablesJetStreamWithDomain(_EMPTY_, "nojs") +} + +func (c *cluster) createSingleLeafNodeNoSystemAccountAndEnablesJetStreamWithDomain(domain, user string) *Server { + tmpl := jsClusterSingleLeafNodeLikeNGSTempl + if domain != _EMPTY_ { + nsc := fmt.Sprintf("domain: %s, store_dir:", domain) + tmpl = strings.Replace(jsClusterSingleLeafNodeLikeNGSTempl, "store_dir:", nsc, 1) + } + as := c.randomServer() + lno := as.getOpts().LeafNode + ln := fmt.Sprintf("nats://%s:p@%s:%d", user, lno.Host, lno.Port) + conf := fmt.Sprintf(tmpl, c.t.TempDir(), ln) + s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + + checkLeafNodeConnectedCount(c.t, as, 1) + + return s +} + +var jsClusterSingleLeafNodeLikeNGSTempl = ` + listen: 127.0.0.1:-1 + server_name: LNJS + jetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'} + + leaf { remotes [ { urls: [ %s ] } ] } +` + +var jsClusterSingleLeafNodeTempl = ` + listen: 127.0.0.1:-1 + server_name: LNJS + jetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'} + + leaf { remotes [ + { urls: [ %s ], account: "JSY" } + { urls: [ %s ], account: "JSN" } ] + } + + accounts { + JSY { users = [ { user: "y", pass: "p" } ]; jetstream: true } + JSN { users = [ { user: "n", pass: "p" } ] } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } +` + +var jsClusterTemplWithLeafNode = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'} + + {{leaf}} + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } +` + +var jsClusterTemplWithLeafNodeNoJS = ` + listen: 127.0.0.1:-1 + server_name: %s + + # Need to keep below since it fills in the store dir by default so just comment out. + # jetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'} + + {{leaf}} + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } +` + +var jsClusterTemplWithSingleLeafNode = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'} + + {{leaf}} + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } +` + +var jsClusterTemplWithSingleFleetLeafNode = ` + listen: 127.0.0.1:-1 + server_name: %s + cluster: { name: fleet } + jetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'} + + {{leaf}} + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } +` + +var jsClusterTemplWithSingleLeafNodeNoJS = ` + listen: 127.0.0.1:-1 + server_name: %s + + # jetstream: {store_dir: '%s'} + + {{leaf}} + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } +` + +var jsLeafFrag = ` + leaf { + remotes [ + { urls: [ %s ] } + { urls: [ %s ], account: "$SYS" } + ] + } +` +var jsLeafNoSysFrag = ` + leaf { + remotes [ + { urls: [ %s ] } + ] + } +` + +func (c *cluster) createLeafNodes(clusterName string, numServers int, domain string) *cluster { + c.t.Helper() + return c.createLeafNodesWithStartPortAndDomain(clusterName, numServers, 22111, domain) +} + +func (c *cluster) createLeafNodesNoJS(clusterName string, numServers int) *cluster { + c.t.Helper() + return c.createLeafNodesWithTemplateAndStartPort(jsClusterTemplWithLeafNodeNoJS, clusterName, numServers, 21333) +} + +func (c *cluster) createLeafNodesWithStartPortAndDomain(clusterName string, numServers int, portStart int, domain string) *cluster { + c.t.Helper() + if domain == _EMPTY_ { + return c.createLeafNodesWithTemplateAndStartPort(jsClusterTemplWithLeafNode, clusterName, numServers, portStart) + } + tmpl := strings.Replace(jsClusterTemplWithLeafNode, "store_dir:", fmt.Sprintf(`domain: "%s", store_dir:`, domain), 1) + return c.createLeafNodesWithTemplateAndStartPort(tmpl, clusterName, numServers, portStart) +} + +func (c *cluster) createLeafNode(extend bool) *Server { + c.t.Helper() + if extend { + return c.createLeafNodeWithTemplate("LNS", + strings.ReplaceAll(jsClusterTemplWithSingleLeafNode, "store_dir:", " extension_hint: will_extend, store_dir:")) + } else { + return c.createLeafNodeWithTemplate("LNS", jsClusterTemplWithSingleLeafNode) + } +} + +func (c *cluster) createLeafNodeWithTemplate(name, template string) *Server { + c.t.Helper() + tmpl := c.createLeafSolicit(template) + conf := fmt.Sprintf(tmpl, name, c.t.TempDir()) + s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + return s +} + +func (c *cluster) createLeafNodeWithTemplateNoSystem(name, template string) *Server { + return c.createLeafNodeWithTemplateNoSystemWithProto(name, template, "nats") +} + +func (c *cluster) createLeafNodeWithTemplateNoSystemWithProto(name, template, proto string) *Server { + c.t.Helper() + tmpl := c.createLeafSolicitNoSystemWithProto(template, proto) + conf := fmt.Sprintf(tmpl, name, c.t.TempDir()) + s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + return s +} + +// Helper to generate the leaf solicit configs. +func (c *cluster) createLeafSolicit(tmpl string) string { + return c.createLeafSolicitWithProto(tmpl, "nats") +} + +func (c *cluster) createLeafSolicitWithProto(tmpl, proto string) string { + c.t.Helper() + + // Create our leafnode cluster template first. + var lns, lnss []string + for _, s := range c.servers { + if s.ClusterName() != c.name { + continue + } + ln := s.getOpts().LeafNode + lns = append(lns, fmt.Sprintf("%s://%s:%d", proto, ln.Host, ln.Port)) + lnss = append(lnss, fmt.Sprintf("%s://admin:s3cr3t!@%s:%d", proto, ln.Host, ln.Port)) + } + lnc := strings.Join(lns, ", ") + lnsc := strings.Join(lnss, ", ") + lconf := fmt.Sprintf(jsLeafFrag, lnc, lnsc) + return strings.Replace(tmpl, "{{leaf}}", lconf, 1) +} + +func (c *cluster) createLeafSolicitNoSystemWithProto(tmpl, proto string) string { + c.t.Helper() + + // Create our leafnode cluster template first. + var lns []string + for _, s := range c.servers { + if s.ClusterName() != c.name { + continue + } + switch proto { + case "nats", "tls": + ln := s.getOpts().LeafNode + lns = append(lns, fmt.Sprintf("%s://%s:%d", proto, ln.Host, ln.Port)) + case "ws", "wss": + ln := s.getOpts().Websocket + lns = append(lns, fmt.Sprintf("%s://%s:%d", proto, ln.Host, ln.Port)) + } + } + lnc := strings.Join(lns, ", ") + return strings.Replace(tmpl, "{{leaf}}", fmt.Sprintf(jsLeafNoSysFrag, lnc), 1) +} + +func (c *cluster) createLeafNodesWithTemplateMixedMode(template, clusterName string, numJsServers, numNonServers int, doJSConfig bool) *cluster { + c.t.Helper() + + // Create our leafnode cluster template first. + tmpl := c.createLeafSolicit(template) + pre := clusterName + "-" + + lc := createMixedModeCluster(c.t, tmpl, clusterName, pre, numJsServers, numNonServers, doJSConfig) + for _, s := range lc.servers { + checkLeafNodeConnectedCount(c.t, s, 2) + } + lc.waitOnClusterReadyWithNumPeers(numJsServers) + + return lc +} + +func (c *cluster) createLeafNodesWithTemplateAndStartPort(template, clusterName string, numServers int, portStart int) *cluster { + c.t.Helper() + + // Create our leafnode cluster template first. + tmpl := c.createLeafSolicit(template) + pre := clusterName + "-" + lc := createJetStreamCluster(c.t, tmpl, clusterName, pre, numServers, portStart, false) + for _, s := range lc.servers { + checkLeafNodeConnectedCount(c.t, s, 2) + } + return lc +} + +// Helper function to close and disable leafnodes. +func (s *Server) closeAndDisableLeafnodes() { + var leafs []*client + s.mu.Lock() + for _, ln := range s.leafs { + leafs = append(leafs, ln) + } + // Disable leafnodes for now. + s.leafDisableConnect = true + s.mu.Unlock() + + for _, ln := range leafs { + ln.closeConnection(Revocation) + } +} + +// Helper function to re-enable leafnode connections. +func (s *Server) reEnableLeafnodes() { + s.mu.Lock() + // Re-enable leafnodes. + s.leafDisableConnect = false + s.mu.Unlock() +} + +// Helper to set the remote migrate feature. +func (s *Server) setJetStreamMigrateOnRemoteLeaf() { + s.mu.Lock() + for cfg := range s.leafRemoteCfgs { + cfg.JetStreamClusterMigrate = true + } + s.mu.Unlock() +} + +// Helper to set the remote migrate feature. +func (s *Server) setJetStreamMigrateOnRemoteLeafWithDelay(delay time.Duration) { + s.mu.Lock() + for cfg := range s.leafRemoteCfgs { + cfg.JetStreamClusterMigrate = true + cfg.JetStreamClusterMigrateDelay = delay + } + s.mu.Unlock() +} + +// Will add in the mapping for the account to each server. +func (c *cluster) addSubjectMapping(account, src, dest string) { + c.t.Helper() + + for _, s := range c.servers { + if s.ClusterName() != c.name { + continue + } + acc, err := s.LookupAccount(account) + if err != nil { + c.t.Fatalf("Unexpected error on %v: %v", s, err) + } + if err := acc.AddMapping(src, dest); err != nil { + c.t.Fatalf("Error adding mapping: %v", err) + } + } + // Make sure interest propagates. + time.Sleep(200 * time.Millisecond) +} + +// Adjust limits for the given account. +func (c *cluster) updateLimits(account string, newLimits map[string]JetStreamAccountLimits) { + c.t.Helper() + for _, s := range c.servers { + acc, err := s.LookupAccount(account) + if err != nil { + c.t.Fatalf("Unexpected error: %v", err) + } + if err := acc.UpdateJetStreamLimits(newLimits); err != nil { + c.t.Fatalf("Unexpected error: %v", err) + } + } +} + +// Hack for staticcheck +var skip = func(t *testing.T) { + t.SkipNow() +} + +func jsClientConnect(t testing.TB, s *Server, opts ...nats.Option) (*nats.Conn, nats.JetStreamContext) { + t.Helper() + nc, err := nats.Connect(s.ClientURL(), opts...) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + js, err := nc.JetStream(nats.MaxWait(10 * time.Second)) + if err != nil { + t.Fatalf("Unexpected error getting JetStream context: %v", err) + } + return nc, js +} + +func jsClientConnectEx(t testing.TB, s *Server, jsOpts []nats.JSOpt, opts ...nats.Option) (*nats.Conn, nats.JetStreamContext) { + t.Helper() + nc, err := nats.Connect(s.ClientURL(), opts...) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + jo := []nats.JSOpt{nats.MaxWait(10 * time.Second)} + if len(jsOpts) > 0 { + jo = append(jo, jsOpts...) + } + js, err := nc.JetStream(jo...) + if err != nil { + t.Fatalf("Unexpected error getting JetStream context: %v", err) + } + return nc, js +} + +func jsClientConnectNewAPI(t testing.TB, s *Server, opts ...nats.Option) (*nats.Conn, jetstream.JetStream) { + t.Helper() + nc, err := nats.Connect(s.ClientURL(), opts...) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + js, err := jetstream.New(nc, jetstream.WithDefaultTimeout(10*time.Second)) + if err != nil { + t.Fatalf("Unexpected error getting JetStream context: %v", err) + } + return nc, js +} + +func jsClientConnectURL(t testing.TB, url string, opts ...nats.Option) (*nats.Conn, nats.JetStreamContext) { + t.Helper() + + nc, err := nats.Connect(url, opts...) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + js, err := nc.JetStream(nats.MaxWait(10 * time.Second)) + if err != nil { + t.Fatalf("Unexpected error getting JetStream context: %v", err) + } + return nc, js +} + +// jsStreamCreate is for sending a stream create for fields that nats.go does not know about yet. +func jsStreamCreate(t testing.TB, nc *nats.Conn, cfg *StreamConfig) (*StreamConfig, error) { + t.Helper() + + j, err := json.Marshal(cfg) + require_NoError(t, err) + + msg, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), j, time.Second*3) + require_NoError(t, err) + + var resp JSApiStreamUpdateResponse + require_NoError(t, json.Unmarshal(msg.Data, &resp)) + + if resp.Error != nil { + return nil, resp.Error + } + + require_NotNil(t, resp.StreamInfo) + return &resp.Config, nil +} + +// jsStreamUpdate is for sending a stream create for fields that nats.go does not know about yet. +func jsStreamUpdate(t testing.TB, nc *nats.Conn, cfg *StreamConfig) (*StreamConfig, error) { + t.Helper() + + j, err := json.Marshal(cfg) + require_NoError(t, err) + + msg, err := nc.Request(fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), j, time.Second*3) + require_NoError(t, err) + + var resp JSApiStreamUpdateResponse + require_NoError(t, json.Unmarshal(msg.Data, &resp)) + + if resp.Error != nil { + return nil, resp.Error + } + + require_NotNil(t, resp.StreamInfo) + return &resp.Config, nil +} + +// jsConsumerCreate is for sending a consumer create for fields that nats.go does not know about yet. +func jsConsumerCreate(t testing.TB, nc *nats.Conn, stream string, cfg ConsumerConfig, pedantic bool) (*ConsumerConfig, error) { + t.Helper() + + j, err := json.Marshal(CreateConsumerRequest{Stream: stream, Config: cfg, Pedantic: pedantic}) + require_NoError(t, err) + + msg, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, stream, cfg.Durable), j, time.Second*3) + require_NoError(t, err) + + var resp JSApiConsumerCreateResponse + require_NoError(t, json.Unmarshal(msg.Data, &resp)) + + if resp.Error != nil { + return nil, resp.Error + } + + require_NotNil(t, resp.ConsumerInfo) + return resp.Config, nil +} + +func checkSubsPending(t *testing.T, sub *nats.Subscription, numExpected int) { + t.Helper() + checkFor(t, 10*time.Second, 20*time.Millisecond, func() error { + if nmsgs, _, err := sub.Pending(); err != nil || nmsgs != numExpected { + return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, numExpected) + } + return nil + }) +} + +func fetchMsgs(t *testing.T, sub *nats.Subscription, numExpected int, totalWait time.Duration) []*nats.Msg { + t.Helper() + result := make([]*nats.Msg, 0, numExpected) + for start, count, wait := time.Now(), numExpected, totalWait; len(result) != numExpected; { + msgs, err := sub.Fetch(count, nats.MaxWait(wait)) + if err != nil { + antithesis.AssertUnreachable(t, "Fetch error", map[string]any{ + "error": err.Error(), + }) + t.Fatal(err) + } + result = append(result, msgs...) + count -= len(msgs) + if wait = totalWait - time.Since(start); wait < 0 { + break + } + } + if len(result) != numExpected { + antithesis.AssertUnreachable(t, "Unexpected fetch messages count", map[string]any{ + "expected": numExpected, + "actual": len(result), + }) + t.Fatalf("Unexpected msg count, got %d, want %d", len(result), numExpected) + } + return result +} + +func (c *cluster) restartServer(rs *Server) *Server { + c.t.Helper() + index := -1 + var opts *Options + for i, s := range c.servers { + if s == rs { + index = i + break + } + } + if index < 0 { + c.t.Fatalf("Could not find server %v to restart", rs) + } + opts = c.opts[index] + s, o := RunServerWithConfig(opts.ConfigFile) + c.servers[index] = s + c.opts[index] = o + return s +} + +func (c *cluster) checkClusterFormed() { + c.t.Helper() + checkClusterFormed(c.t, c.servers...) +} + +func (c *cluster) waitOnPeerCount(n int) { + c.t.Helper() + c.waitOnLeader() + leader := c.leader() + expires := time.Now().Add(30 * time.Second) + // Make sure we have all peers, and take into account the meta leader could still change. + for time.Now().Before(expires) { + if leader = c.leader(); leader == nil { + time.Sleep(50 * time.Millisecond) + continue + } + if len(leader.JetStreamClusterPeers()) == n { + return + } + time.Sleep(100 * time.Millisecond) + } + antithesis.AssertUnreachable(c.t, "Timeout in cluster.waitOnPeerCount", map[string]any{ + "cluster": c.name, + }) + c.t.Fatalf("Expected a cluster peer count of %d, got %d", n, len(leader.JetStreamClusterPeers())) +} + +func (c *cluster) waitOnConsumerLeader(account, stream, consumer string) { + c.t.Helper() + expires := time.Now().Add(30 * time.Second) + for time.Now().Before(expires) { + leader := c.consumerLeader(account, stream, consumer) + if leader == nil { + time.Sleep(100 * time.Millisecond) + continue + } + // Also wait for the consumer leader to finish any scales/moves. + js := leader.getJetStream() + js.mu.RLock() + ca := js.consumerAssignment(account, stream, consumer) + wait := ca == nil || (ca.Group != nil && ca.Group.Desired != nil) + js.mu.RUnlock() + if wait { + time.Sleep(100 * time.Millisecond) + continue + } + time.Sleep(200 * time.Millisecond) + return + } + antithesis.AssertUnreachable(c.t, "Timeout in cluster.waitOnConsumerLeader", map[string]any{ + "cluster": c.name, + "account": account, + "stream": stream, + "consumer": consumer, + }) + c.t.Fatalf("Expected a consumer leader for %q %q %q, got none", account, stream, consumer) +} + +func (c *cluster) consumerLeader(account, stream, consumer string) *Server { + c.t.Helper() + for _, s := range c.servers { + if s.JetStreamIsConsumerLeader(account, stream, consumer) { + return s + } + } + return nil +} + +func (c *cluster) randomNonConsumerLeader(account, stream, consumer string) *Server { + c.t.Helper() + for _, s := range c.servers { + if !s.JetStreamIsConsumerLeader(account, stream, consumer) { + return s + } + } + return nil +} + +// stepDownStreamLeader steps the stream leader down to the given server if it's not equal. +func (c *cluster) stepDownStreamLeader(nc *nats.Conn, account, stream string, s *Server) { + c.t.Helper() + req, err := json.Marshal(JSApiLeaderStepdownRequest{Placement: &Placement{Preferred: s.Name()}}) + require_NoError(c.t, err) + sl := c.streamLeader(account, stream) + if sl != s { + _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, stream), req, 5*time.Second) + require_NoError(c.t, err) + c.waitOnStreamLeader(account, stream) + sl = c.streamLeader(account, stream) + } + require_Equal(c.t, sl, s) +} + +// stepDownConsumerLeader steps the consumer leader down to the given server if it's not equal. +func (c *cluster) stepDownConsumerLeader(nc *nats.Conn, account, stream, consumer string, s *Server) { + c.t.Helper() + req, err := json.Marshal(JSApiLeaderStepdownRequest{Placement: &Placement{Preferred: s.Name()}}) + require_NoError(c.t, err) + cl := c.consumerLeader(account, stream, consumer) + if cl != s { + _, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, stream, consumer), req, 5*time.Second) + require_NoError(c.t, err) + c.waitOnConsumerLeader(account, stream, consumer) + cl = c.consumerLeader(account, stream, consumer) + } + require_Equal(c.t, cl, s) +} + +func (c *cluster) waitOnStreamLeader(account, stream string) { + c.t.Helper() + expires := time.Now().Add(30 * time.Second) + for time.Now().Before(expires) { + leader := c.streamLeader(account, stream) + if leader == nil { + time.Sleep(100 * time.Millisecond) + continue + } + // Also wait for the stream leader to finish any scales/moves. + js := leader.getJetStream() + js.mu.RLock() + sa := js.streamAssignment(account, stream) + wait := sa == nil || (sa.Group != nil && sa.Group.Desired != nil) + js.mu.RUnlock() + if wait { + time.Sleep(100 * time.Millisecond) + continue + } + time.Sleep(200 * time.Millisecond) + return + } + antithesis.AssertUnreachable(c.t, "Timeout in cluster.waitOnStreamLeader", map[string]any{ + "cluster": c.name, + "account": account, + "stream": stream, + }) + c.t.Fatalf("Expected a stream leader for %q %q, got none", account, stream) +} + +func (c *cluster) randomNonStreamLeader(account, stream string) *Server { + c.t.Helper() + for _, s := range c.servers { + if s.JetStreamIsStreamAssigned(account, stream) && !s.JetStreamIsStreamLeader(account, stream) { + return s + } + } + return nil +} + +func (c *cluster) streamLeader(account, stream string) *Server { + c.t.Helper() + for _, s := range c.servers { + if s.JetStreamIsStreamLeader(account, stream) { + return s + } + } + return nil +} + +func (c *cluster) waitOnStreamCurrent(s *Server, account, stream string) { + c.t.Helper() + expires := time.Now().Add(30 * time.Second) + for time.Now().Before(expires) { + if s.JetStreamIsStreamCurrent(account, stream) { + time.Sleep(100 * time.Millisecond) + return + } + time.Sleep(100 * time.Millisecond) + } + antithesis.AssertUnreachable(c.t, "Timeout in cluster.waitOnStreamCurrent", map[string]any{ + "cluster": c.name, + "account": account, + "stream": stream, + }) + c.t.Fatalf("Expected server %q to eventually be current for stream %q", s, stream) +} + +func (c *cluster) waitOnServerHealthz(s *Server) { + c.t.Helper() + expires := time.Now().Add(30 * time.Second) + for time.Now().Before(expires) { + hs := s.healthz(nil) + if hs.Status == "ok" && hs.Error == _EMPTY_ { + return + } + time.Sleep(100 * time.Millisecond) + } + antithesis.AssertUnreachable(c.t, "Timeout in cluster.waitOnServerHealthz", map[string]any{ + "cluster": c.name, + "server": s.Name(), + }) + c.t.Fatalf("Expected server %q to eventually return healthz 'ok', but got %q", s, s.healthz(nil).Error) +} + +func (c *cluster) waitOnServerCurrent(s *Server) { + c.t.Helper() + expires := time.Now().Add(30 * time.Second) + for time.Now().Before(expires) { + time.Sleep(100 * time.Millisecond) + if !s.JetStreamEnabled() || s.JetStreamIsCurrent() { + return + } + } + antithesis.AssertUnreachable(c.t, "Timeout in cluster.waitOnServerCurrent", map[string]any{ + "cluster": c.name, + "server": s.Name(), + }) + c.t.Fatalf("Expected server %q to eventually be current", s) +} + +func (c *cluster) waitOnAllCurrent() { + c.t.Helper() + // Callers use this to wait for a proposal they just made to be applied, + // so give it a chance to be committed before checking for currency. + time.Sleep(100 * time.Millisecond) + expires := time.Now().Add(30 * time.Second) + for time.Now().Before(expires) { + current := true + for _, s := range c.servers { + if s.JetStreamEnabled() && !s.JetStreamIsCurrent() { + current = false + break + } + } + if current { + return + } + time.Sleep(10 * time.Millisecond) + } + antithesis.AssertUnreachable(c.t, "Timeout in cluster.waitOnAllCurrent", map[string]any{ + "cluster": c.name, + }) + c.t.Fatalf("Expected all servers in cluster %q to eventually be current", c.name) +} + +func (c *cluster) serverByName(sname string) *Server { + for _, s := range c.servers { + if s.Name() == sname { + return s + } + } + return nil +} + +func (c *cluster) randomNonLeader() *Server { + // range should randomize.. but.. + for _, s := range c.servers { + if s.Running() && !s.JetStreamIsLeader() { + return s + } + } + return nil +} + +func (c *cluster) leader() *Server { + for _, s := range c.servers { + if s.JetStreamIsLeader() { + return s + } + } + return nil +} + +func (c *cluster) expectNoLeader() { + c.t.Helper() + expires := time.Now().Add(maxElectionTimeout) + for time.Now().Before(expires) { + if c.leader() == nil { + return + } + time.Sleep(20 * time.Millisecond) + } + antithesis.AssertUnreachable(c.t, "Timeout in cluster.expectNoLeader", map[string]any{ + "cluster": c.name, + }) + c.t.Fatalf("Expected no leader but have one") +} + +func (c *cluster) waitOnLeader() { + c.t.Helper() + expires := time.Now().Add(40 * time.Second) + for time.Now().Before(expires) { + if leader := c.leader(); leader != nil { + time.Sleep(100 * time.Millisecond) + return + } + time.Sleep(10 * time.Millisecond) + } + + antithesis.AssertUnreachable(c.t, "Timeout in cluster.waitOnLeader", map[string]any{ + "cluster": c.name, + }) + c.t.Fatalf("Expected a cluster leader, got none") +} + +func (c *cluster) waitOnAccount(account string) { + c.t.Helper() + expires := time.Now().Add(40 * time.Second) + for time.Now().Before(expires) { + found := true + for _, s := range c.servers { + s.optsMu.RLock() + wantJS := s.opts.JetStream + s.optsMu.RUnlock() + acc, err := s.fetchAccount(account) + if found = found && err == nil && acc != nil; wantJS { + found = found && acc.JetStreamEnabled() + } + } + if found { + return + } + time.Sleep(100 * time.Millisecond) + continue + } + + antithesis.AssertUnreachable(c.t, "Timeout in cluster.waitOnAccount", map[string]any{ + "account": account, + }) + c.t.Fatalf("Expected account %q to exist but didn't", account) +} + +// Helper function to check that a cluster is formed +func (c *cluster) waitOnClusterReady() { + c.t.Helper() + c.waitOnClusterReadyWithNumPeers(len(c.servers)) +} + +func (c *cluster) waitOnClusterReadyWithNumPeers(numPeersExpected int) { + c.t.Helper() + var leader *Server + expires := time.Now().Add(40 * time.Second) + // Make sure we have all peers, and take into account the meta leader could still change. + for time.Now().Before(expires) { + if leader = c.leader(); leader == nil { + time.Sleep(50 * time.Millisecond) + continue + } + if len(leader.JetStreamClusterPeers()) == numPeersExpected { + current := true + for _, s := range c.servers { + if s.Running() && s.JetStreamEnabled() && !s.JetStreamIsCurrent() { + current = false + break + } + } + if current { + return + } + } + time.Sleep(10 * time.Millisecond) + } + + if leader == nil { + c.shutdown() + antithesis.AssertUnreachable(c.t, "Timeout in cluster.waitOnClusterReadyWithNumPeers (1)", map[string]any{ + "cluster": c.name, + }) + c.t.Fatalf("Failed to elect a meta-leader") + } + + peersSeen := len(leader.JetStreamClusterPeers()) + c.shutdown() + + if leader == nil { + antithesis.AssertUnreachable(c.t, "Timeout in cluster.waitOnClusterReadyWithNumPeers (2)", map[string]any{ + "cluster": c.name, + }) + c.t.Fatalf("Expected a cluster leader and fully formed cluster, no leader") + } else { + antithesis.AssertUnreachable(c.t, "Timeout in cluster.waitOnClusterReadyWithNumPeers (3)", map[string]any{ + "cluster": c.name, + "peers_expected": numPeersExpected, + "peers_seen": peersSeen, + }) + c.t.Fatalf("Expected a fully formed cluster, only %d of %d peers seen", peersSeen, numPeersExpected) + } +} + +// Helper function to remove JetStream from a server. +func (c *cluster) removeJetStream(s *Server) { + c.t.Helper() + index := -1 + for i, cs := range c.servers { + if cs == s { + index = i + break + } + } + cf := c.opts[index].ConfigFile + cb, _ := os.ReadFile(cf) + var sb strings.Builder + for _, l := range strings.Split(string(cb), "\n") { + if !strings.HasPrefix(strings.TrimSpace(l), "jetstream") { + sb.WriteString(l + "\n") + } + } + if err := os.WriteFile(cf, []byte(sb.String()), 0644); err != nil { + c.t.Fatalf("Error writing updated config file: %v", err) + } + if err := s.Reload(); err != nil { + c.t.Fatalf("Error on server reload: %v", err) + } + time.Sleep(100 * time.Millisecond) +} + +func (c *cluster) stopAll() { + c.t.Helper() + for _, s := range c.servers { + s.Shutdown() + } + for _, s := range c.servers { + s.WaitForShutdown() + } +} + +func (c *cluster) restartAll() { + c.t.Helper() + for i, s := range c.servers { + if !s.Running() { + opts := c.opts[i] + s, o := RunServerWithConfig(opts.ConfigFile) + c.servers[i] = s + c.opts[i] = o + } + } + c.checkClusterFormed() + c.campaignMetaImmediately() + c.waitOnClusterReady() +} + +func (c *cluster) lameDuckRestartAll() { + c.t.Helper() + for _, s := range c.servers { + s.lameDuckMode() + } + for i, s := range c.servers { + s.WaitForShutdown() + if !s.Running() { + opts := c.opts[i] + s, o := RunServerWithConfig(opts.ConfigFile) + c.servers[i] = s + c.opts[i] = o + } + } + c.checkClusterFormed() + c.campaignMetaImmediately() + c.waitOnClusterReady() +} + +func (c *cluster) restartAllSamePorts() { + c.t.Helper() + for i, s := range c.servers { + if !s.Running() { + opts := c.opts[i] + s := RunServer(opts) + c.servers[i] = s + } + } + c.checkClusterFormed() + c.campaignMetaImmediately() + c.waitOnClusterReady() +} + +func (c *cluster) totalSubs() (total int) { + c.t.Helper() + for _, s := range c.servers { + total += int(s.NumSubscriptions()) + } + return total +} + +func (c *cluster) stableTotalSubs() (total int) { + nsubs := -1 + checkFor(c.t, 2*time.Second, 250*time.Millisecond, func() error { + subs := c.totalSubs() + if subs == nsubs { + return nil + } + nsubs = subs + return fmt.Errorf("Still stabilizing") + }) + return nsubs + +} + +func addStreamPedanticWithError(t *testing.T, nc *nats.Conn, cfg *StreamConfigRequest) (*StreamInfo, *ApiError) { + t.Helper() + req, err := json.Marshal(cfg) + require_NoError(t, err) + rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, 5*time.Second) + require_NoError(t, err) + var resp JSApiStreamCreateResponse + err = json.Unmarshal(rmsg.Data, &resp) + require_NoError(t, err) + if resp.Type != JSApiStreamCreateResponseType { + t.Fatalf("Invalid response type %s expected %s", resp.Type, JSApiStreamCreateResponseType) + } + return resp.StreamInfo, resp.Error +} + +func updateStreamPedanticWithError(t *testing.T, nc *nats.Conn, cfg *StreamConfigRequest) (*StreamInfo, *ApiError) { + t.Helper() + req, err := json.Marshal(cfg) + require_NoError(t, err) + rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), req, time.Second) + require_NoError(t, err) + var resp JSApiStreamCreateResponse + err = json.Unmarshal(rmsg.Data, &resp) + require_NoError(t, err) + if resp.Type != JSApiStreamUpdateResponseType { + t.Fatalf("Invalid response type %s expected %s", resp.Type, JSApiStreamUpdateResponseType) + } + return resp.StreamInfo, resp.Error +} + +func addStream(t *testing.T, nc *nats.Conn, cfg *StreamConfig) *StreamInfo { + t.Helper() + si, err := addStreamWithError(t, nc, cfg) + if err != nil { + t.Fatalf("Unexpected error: %+v", err) + } + return si +} + +func addStreamWithError(t *testing.T, nc *nats.Conn, cfg *StreamConfig) (*StreamInfo, *ApiError) { + t.Helper() + req, err := json.Marshal(cfg) + require_NoError(t, err) + rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, 5*time.Second) + require_NoError(t, err) + var resp JSApiStreamCreateResponse + err = json.Unmarshal(rmsg.Data, &resp) + require_NoError(t, err) + if resp.Type != JSApiStreamCreateResponseType { + t.Fatalf("Invalid response type %s expected %s", resp.Type, JSApiStreamCreateResponseType) + } + return resp.StreamInfo, resp.Error +} + +func updateStream(t *testing.T, nc *nats.Conn, cfg *StreamConfig) *StreamInfo { + t.Helper() + req, err := json.Marshal(cfg) + require_NoError(t, err) + rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), req, time.Second) + require_NoError(t, err) + var resp JSApiStreamCreateResponse + err = json.Unmarshal(rmsg.Data, &resp) + require_NoError(t, err) + if resp.Type != JSApiStreamUpdateResponseType { + t.Fatalf("Invalid response type %s expected %s", resp.Type, JSApiStreamUpdateResponseType) + } + if resp.Error != nil { + t.Fatalf("Unexpected error: %+v", resp.Error) + } + return resp.StreamInfo +} + +// setInActiveDeleteThreshold sets the delete threshold for how long to wait +// before deleting an inactive consumer. +func (o *consumer) setInActiveDeleteThreshold(dthresh time.Duration) error { + o.mu.Lock() + defer o.mu.Unlock() + + deleteWasRunning := o.dtmr != nil + stopAndClearTimer(&o.dtmr) + // Do not add jitter if set via here. + o.dthresh = dthresh + if deleteWasRunning { + o.dtmr = time.AfterFunc(o.dthresh, func() { o.deleteNotActive() }) + } + return nil +} + +// Net Proxy - For introducing RTT and BW constraints. +type netProxy struct { + sync.RWMutex + listener net.Listener + conns []net.Conn + rtt time.Duration + up int + down int + url string + surl string + port int +} + +func newNetProxy(rtt time.Duration, upRate, downRate int, serverURL string) *netProxy { + return createNetProxy(rtt, upRate, downRate, serverURL, true) +} + +func createNetProxy(rtt time.Duration, upRate, downRate int, serverURL string, start bool) *netProxy { + hp := net.JoinHostPort("127.0.0.1", "0") + l, e := net.Listen("tcp", hp) + if e != nil { + panic(fmt.Sprintf("Error listening on port: %s, %q", hp, e)) + } + u, err := url.Parse(serverURL) + if err != nil { + panic(fmt.Sprintf("Could not parse server URL: %v", err)) + } + + var clientURL string + port := l.Addr().(*net.TCPAddr).Port + if u.User != nil { + clientURL = fmt.Sprintf("nats://%v@127.0.0.1:%d", u.User, port) + } else { + clientURL = fmt.Sprintf("nats://127.0.0.1:%d", port) + } + + proxy := &netProxy{ + listener: l, + rtt: rtt, + up: upRate, + down: downRate, + url: clientURL, + surl: serverURL, + port: port, + } + if start { + proxy.start() + } + return proxy +} + +func (np *netProxy) start() { + u, err := url.Parse(np.surl) + if err != nil { + panic(fmt.Sprintf("Could not parse server URL: %v", err)) + } + host := u.Host + + // Check if this is restart. + // We nil out listener on stop() + if np.listener == nil && np.port != 0 { + hp := net.JoinHostPort("127.0.0.1", fmt.Sprintf("%d", np.port)) + np.listener, err = net.Listen("tcp", hp) + if err != nil { + panic(fmt.Sprintf("Error listening on port: %s, %q", hp, err)) + } + } + + go func() { + for { + client, err := np.listener.Accept() + if err != nil { + return + } + server, err := net.DialTimeout("tcp", host, time.Second) + if err != nil { + continue + } + np.conns = append(np.conns, client, server) + go np.loop(np.up, client, server) + go np.loop(np.down, server, client) + } + }() +} + +func (np *netProxy) clientURL() string { + return np.url +} + +func (np *netProxy) routeURL() string { + return strings.Replace(np.url, "nats", "nats-route", 1) +} + +func (np *netProxy) leafURL() string { + return strings.Replace(np.url, "nats", "nats-leaf", 1) +} + +func (np *netProxy) loop(tbw int, r, w net.Conn) { + const rbl = 8192 + var buf [rbl]byte + ctx := context.Background() + + rl := rate.NewLimiter(rate.Limit(tbw), rbl) + + for { + n, err := r.Read(buf[:]) + if err != nil { + w.Close() + return + } + // RTT delays + np.RLock() + delay := np.rtt / 2 + np.RUnlock() + if delay > 0 { + time.Sleep(delay) + } + if err := rl.WaitN(ctx, n); err != nil { + return + } + if _, err = w.Write(buf[:n]); err != nil { + r.Close() + return + } + } +} + +func (np *netProxy) updateRTT(rtt time.Duration) { + np.Lock() + np.rtt = rtt + np.Unlock() +} + +func (np *netProxy) stop() { + if np.listener != nil { + np.listener.Close() + np.listener = nil + for _, c := range np.conns { + c.Close() + } + } +} + +// Bitset, aka bitvector, allows tracking of large number of bits efficiently +type bitset struct { + // Bit map storage + bitmap []uint8 + // Number of bits currently set to 1 + currentCount uint64 + // Number of bits stored + size uint64 +} + +func NewBitset(size uint64) *bitset { + byteSize := (size + 7) / 8 //Round up to the nearest byte + + return &bitset{ + bitmap: make([]uint8, int(byteSize)), + size: size, + currentCount: 0, + } +} + +func (b *bitset) get(index uint64) bool { + if index >= b.size { + panic(fmt.Sprintf("Index %d out of bounds, size %d", index, b.size)) + } + byteIndex := index / 8 + bitIndex := uint(index % 8) + bit := (b.bitmap[byteIndex] & (uint8(1) << bitIndex)) + return bit != 0 +} + +func (b *bitset) set(index uint64, value bool) { + if index >= b.size { + panic(fmt.Sprintf("Index %d out of bounds, size %d", index, b.size)) + } + byteIndex := index / 8 + bitIndex := uint(index % 8) + byteMask := uint8(1) << bitIndex + isSet := (b.bitmap[byteIndex] & (uint8(1) << bitIndex)) != 0 + if value { + b.bitmap[byteIndex] |= byteMask + if !isSet { + b.currentCount += 1 + } + } else { + b.bitmap[byteIndex] &= ^byteMask + if isSet { + b.currentCount -= 1 + } + } +} + +func (b *bitset) count() uint64 { + return b.currentCount +} + +func (b *bitset) String() string { + const block = 8 // 8 bytes, 64 bits per line + sb := strings.Builder{} + + sb.WriteString(fmt.Sprintf("Bits set: %d/%d\n", b.currentCount, b.size)) + for i := 0; i < len(b.bitmap); i++ { + if i%block == 0 { + if i > 0 { + sb.WriteString("\n") + } + sb.WriteString(fmt.Sprintf("[%4d] ", i*8)) + } + for j := uint8(0); j < 8; j++ { + if b.bitmap[i]&(1< 0 { + sb.WriteString("1") + } else { + sb.WriteString("0") + } + } + } + sb.WriteString("\n") + return sb.String() +} + +func copyDir(t *testing.T, dst, src string) error { + t.Helper() + srcFS := os.DirFS(src) + return fs.WalkDir(srcFS, ".", func(p string, d os.DirEntry, err error) error { + if err != nil { + return err + } + // A purge moves msgs to __msgs__ and removes that directory + // asynchronously. It is disposable and may disappear while walking a + // live store, so do not include it in the copy. + if d.IsDir() && d.Name() == purgeDir { + return fs.SkipDir + } + // Atomic writes use .tmp files that are renamed into place when complete. + if strings.HasSuffix(d.Name(), blkTmpSuffix) { + if d.IsDir() { + return fs.SkipDir + } + return nil + } + newPath := path.Join(dst, p) + if d.IsDir() { + return os.MkdirAll(newPath, defaultDirPerms) + } + r, err := srcFS.Open(p) + if err != nil { + return err + } + defer r.Close() + + w, err := os.OpenFile(newPath, os.O_CREATE|os.O_WRONLY, defaultFilePerms) + if err != nil { + return err + } + defer w.Close() + _, err = io.Copy(w, r) + return err + }) +} + +func getStreamDetails(t *testing.T, c *cluster, accountName, streamName string) *StreamDetail { + t.Helper() + srv := c.streamLeader(accountName, streamName) + if srv == nil { + return nil + } + jsz, err := srv.Jsz(&JSzOptions{Accounts: true, Streams: true, Consumer: true}) + require_NoError(t, err) + for _, acc := range jsz.AccountDetails { + if acc.Name == accountName { + for _, stream := range acc.Streams { + if stream.Name == streamName { + return &stream + } + } + } + } + t.Error("Could not find account details") + return nil +} + +func checkState(t *testing.T, c *cluster, accountName, streamName string) error { + t.Helper() + _, err := checkStateAndErr(t, c, accountName, streamName) + return err +} + +func checkStateAndErr(t *testing.T, c *cluster, accountName, streamName string) (StreamState, error) { + t.Helper() + + leaderSrv := c.streamLeader(accountName, streamName) + if leaderSrv == nil { + return StreamState{}, fmt.Errorf("no leader server found for stream %q", streamName) + } + streamLeader := getStreamDetails(t, c, accountName, streamName) + if streamLeader == nil { + return StreamState{}, fmt.Errorf("no leader found for stream %q", streamName) + } + + acc, err := leaderSrv.LookupAccount(accountName) + require_NoError(t, err) + mset, err := acc.lookupStream(streamName) + require_NoError(t, err) + + cfgReplicas := mset.config().Replicas + foundReplicas := 1 // We already know the leader. + var errs []error + for _, srv := range c.servers { + if srv == leaderSrv { + // Skip self + continue + } + if srv.isShuttingDown() { + // Skip if shutdown. + for _, replica := range streamLeader.Cluster.Replicas { + if replica.Name == srv.Name() { + foundReplicas++ + break + } + } + continue + } + acc, err = srv.LookupAccount(accountName) + if err != nil { + continue + } + mset, err = acc.lookupStream(streamName) + if err != nil { + continue + } + + foundReplicas++ + state := mset.state() + + if state.Msgs != streamLeader.State.Msgs { + err := fmt.Errorf("[%s] Leader %v has %d messages, Follower %v has %d messages", + streamName, leaderSrv, streamLeader.State.Msgs, + srv, state.Msgs, + ) + errs = append(errs, err) + } + if state.FirstSeq != streamLeader.State.FirstSeq { + err := fmt.Errorf("[%s] Leader %v FirstSeq is %d, Follower %v is at %d", + streamName, leaderSrv, streamLeader.State.FirstSeq, + srv, state.FirstSeq, + ) + errs = append(errs, err) + } + if state.LastSeq != streamLeader.State.LastSeq { + err := fmt.Errorf("[%s] Leader %v LastSeq is %d, Follower %v is at %d", + streamName, leaderSrv, streamLeader.State.LastSeq, + srv, state.LastSeq, + ) + errs = append(errs, err) + } + if state.NumDeleted != streamLeader.State.NumDeleted { + err := fmt.Errorf("[%s] Leader %v NumDeleted is %d, Follower %v is at %d\nSTATE_A: %+v\nSTATE_B: %+v\n", + streamName, leaderSrv, streamLeader.State.NumDeleted, + srv, state.NumDeleted, streamLeader.State, state, + ) + errs = append(errs, err) + } + } + if cfgReplicas != foundReplicas { + err := fmt.Errorf("[%s] Expected %d Replicas, got %d\n", streamName, cfgReplicas, foundReplicas) + errs = append(errs, err) + } + if len(errs) > 0 { + return StreamState{}, errors.Join(errs...) + } + return streamLeader.State, nil +} + +// Helpers for the meta leader peer reconciliation tests. +func metaStreamPeers(s *Server, account, stream string) []string { + js := s.getJetStream() + js.mu.RLock() + defer js.mu.RUnlock() + sa := js.streamAssignmentOrInflight(account, stream) + if sa == nil { + return nil + } + peers := copyStrings(sa.Group.Peers) + slices.Sort(peers) + return peers +} + +func metaConsumerPeers(s *Server, account, stream, consumer string) []string { + js := s.getJetStream() + js.mu.RLock() + defer js.mu.RUnlock() + ca := js.consumerAssignmentOrInflight(account, stream, consumer) + if ca == nil { + return nil + } + peers := copyStrings(ca.Group.Peers) + slices.Sort(peers) + return peers +} + +func performStreamBackup(t *testing.T, nc *nats.Conn, streamName string) (StreamConfig, StreamState, []byte) { + t.Helper() + endpoint := fmt.Sprintf(JSApiStreamSnapshotT, streamName) + inbox := nc.NewRespInbox() + + sub, err := nc.SubscribeSync(inbox) + require_NoError(t, err) + + resp, err := nc.RequestMsg(&nats.Msg{ + Subject: endpoint, + Data: fmt.Appendf(nil, `{"deliver_subject": %q}`, inbox), + }, 5*time.Second) + require_NoError(t, err) + + var body struct { + StreamState `json:"state"` + StreamConfig `json:"config"` + } + require_NoError(t, json.Unmarshal(resp.Data, &body)) + + buf := bytes.NewBuffer(nil) + for { + msg, err := sub.NextMsg(5 * time.Second) + require_NoError(t, err) + if msg.Reply != _EMPTY_ { + require_NoError(t, msg.Respond(nil)) + } + if len(msg.Data) == 0 { + break + } + _, err = buf.Write(msg.Data) + require_NoError(t, err) + } + return body.StreamConfig, body.StreamState, buf.Bytes() +} + +func performStreamRestore(t *testing.T, nc *nats.Conn, sc StreamConfig, ss StreamState, archive []byte) bool { + t.Helper() + endpoint := fmt.Sprintf(JSApiStreamRestoreT, sc.Name) + + req, err := json.Marshal(struct { + StreamState `json:"state"` + StreamConfig `json:"config"` + }{ + StreamState: ss, + StreamConfig: sc, + }) + require_NoError(t, err) + + resp, err := nc.RequestMsg(&nats.Msg{ + Subject: endpoint, + Data: req, + }, 5*time.Second) + require_NoError(t, err) + + var res struct { + DeliverSubject string `json:"deliver_subject"` + } + require_NoError(t, json.Unmarshal(resp.Data, &res)) + if !IsValidLiteralSubject(res.DeliverSubject) { + return false + } + + reader := bytes.NewReader(archive) + buf := make([]byte, 128*1024) + for { + n, err := reader.Read(buf) + if err == io.EOF { + break + } + require_NoError(t, err) + + _, err = nc.Request(res.DeliverSubject, buf[:n], time.Second) + require_NoError(t, err) + } + + _, err = nc.Request(res.DeliverSubject, nil, time.Second) + require_NoError(t, err) + return true +} + +// publishAsync publishes a message asynchronously, retrying if we're stalled +// on async publishes. +func publishAsync(t testing.TB, js nats.JetStreamContext, subj string, msg []byte) nats.PubAckFuture { + t.Helper() + for { + pubAck, err := js.PublishAsync(subj, msg) + if err == nil { + return pubAck + } + require_Error(t, err, nats.ErrTooManyStalledMsgs) + } +} diff --git a/server/jetstream_jwt_test.go b/server/jetstream_jwt_test.go new file mode 100644 index 0000000..0f05eb7 --- /dev/null +++ b/server/jetstream_jwt_test.go @@ -0,0 +1,2332 @@ +// Copyright 2020-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !skip_js_tests + +package server + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nkeys" + + jwt "github.com/nats-io/jwt/v2" +) + +func TestJetStreamJWTLimits(t *testing.T) { + updateJwt := func(url string, creds string, pubKey string, jwt string) { + t.Helper() + c := natsConnect(t, url, nats.UserCredentials(creds)) + defer c.Close() + if msg, err := c.Request(fmt.Sprintf(accUpdateEventSubjNew, pubKey), []byte(jwt), time.Second); err != nil { + t.Fatal("error not expected in this test", err) + } else { + content := make(map[string]any) + if err := json.Unmarshal(msg.Data, &content); err != nil { + t.Fatalf("%v", err) + } else if _, ok := content["data"]; !ok { + t.Fatalf("did not get an ok response got: %v", content) + } + } + } + require_IdenticalLimits := func(infoLim JetStreamAccountLimits, lim jwt.JetStreamLimits) { + t.Helper() + if int64(infoLim.MaxConsumers) != lim.Consumer || int64(infoLim.MaxStreams) != lim.Streams || + infoLim.MaxMemory != lim.MemoryStorage || infoLim.MaxStore != lim.DiskStorage { + t.Fatalf("limits do not match %v != %v", infoLim, lim) + } + } + expect_JSDisabledForAccount := func(c *nats.Conn) { + t.Helper() + if _, err := c.Request("$JS.API.INFO", nil, time.Second); err != nats.ErrTimeout && err != nats.ErrNoResponders { + t.Fatalf("Unexpected error: %v", err) + } + } + expect_InfoError := func(c *nats.Conn) { + t.Helper() + var info JSApiAccountInfoResponse + if resp, err := c.Request("$JS.API.INFO", nil, time.Second); err != nil { + t.Fatalf("Unexpected error: %v", err) + } else if err = json.Unmarshal(resp.Data, &info); err != nil { + t.Fatalf("response1 %v got error %v", string(resp.Data), err) + } else if info.Error == nil { + t.Fatalf("expected error") + } + } + validate_limits := func(c *nats.Conn, expectedLimits jwt.JetStreamLimits) { + t.Helper() + var info JSApiAccountInfoResponse + if resp, err := c.Request("$JS.API.INFO", nil, time.Second); err != nil { + t.Fatalf("Unexpected error: %v", err) + } else if err = json.Unmarshal(resp.Data, &info); err != nil { + t.Fatalf("response1 %v got error %v", string(resp.Data), err) + } else { + require_IdenticalLimits(info.Limits, expectedLimits) + } + } + // create system account + sysKp, _ := nkeys.CreateAccount() + sysPub, _ := sysKp.PublicKey() + sysUKp, _ := nkeys.CreateUser() + sysUSeed, _ := sysUKp.Seed() + uclaim := newJWTTestUserClaims() + uclaim.Subject, _ = sysUKp.PublicKey() + sysUserJwt, err := uclaim.Encode(sysKp) + require_NoError(t, err) + sysKp.Seed() + sysCreds := genCredsFile(t, sysUserJwt, sysUSeed) + // limits to apply and check + limits1 := jwt.JetStreamLimits{MemoryStorage: 1024 * 1024, DiskStorage: 2048 * 1024, Streams: 1, Consumer: 2, MaxBytesRequired: true} + // has valid limits that would fail when incorrectly applied twice + limits2 := jwt.JetStreamLimits{MemoryStorage: 4096 * 1024, DiskStorage: 8192 * 1024, Streams: 3, Consumer: 4} + // limits exceeding actual configured value of DiskStorage + limitsExceeded := jwt.JetStreamLimits{MemoryStorage: 8192 * 1024, DiskStorage: 16384 * 1024, Streams: 5, Consumer: 6} + // create account using jetstream with both limits + akp, _ := nkeys.CreateAccount() + aPub, _ := akp.PublicKey() + claim := jwt.NewAccountClaims(aPub) + claim.Limits.JetStreamLimits = limits1 + aJwt1, err := claim.Encode(oKp) + require_NoError(t, err) + claim.Limits.JetStreamLimits = limits2 + aJwt2, err := claim.Encode(oKp) + require_NoError(t, err) + claim.Limits.JetStreamLimits = limitsExceeded + aJwtLimitsExceeded, err := claim.Encode(oKp) + require_NoError(t, err) + claim.Limits.JetStreamLimits = jwt.JetStreamLimits{} // disabled + aJwt4, err := claim.Encode(oKp) + require_NoError(t, err) + // account user + uKp, _ := nkeys.CreateUser() + uSeed, _ := uKp.Seed() + uclaim = newJWTTestUserClaims() + uclaim.Subject, _ = uKp.PublicKey() + userJwt, err := uclaim.Encode(akp) + require_NoError(t, err) + userCreds := genCredsFile(t, userJwt, uSeed) + dir := t.TempDir() + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:-1 + jetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: "%s"} + operator: %s + resolver: { + type: full + dir: '%s' + } + system_account: %s + `, dir, ojwt, dir, sysPub))) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + port := opts.Port + updateJwt(s.ClientURL(), sysCreds, aPub, aJwt1) + c := natsConnect(t, s.ClientURL(), nats.UserCredentials(userCreds), nats.ReconnectWait(200*time.Millisecond)) + defer c.Close() + validate_limits(c, limits1) + // keep using the same connection + updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) + validate_limits(c, limits2) + // keep using the same connection but do NOT CHANGE anything. + // This tests if the jwt is applied a second time (would fail) + updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) + validate_limits(c, limits2) + // keep using the same connection. This update EXCEEDS LIMITS + updateJwt(s.ClientURL(), sysCreds, aPub, aJwtLimitsExceeded) + validate_limits(c, limits2) + // disable test after failure + updateJwt(s.ClientURL(), sysCreds, aPub, aJwt4) + expect_InfoError(c) + // re enable, again testing with a value that can't be applied twice + updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) + validate_limits(c, limits2) + // disable test no prior failure + updateJwt(s.ClientURL(), sysCreds, aPub, aJwt4) + expect_InfoError(c) + // Wrong limits form start + updateJwt(s.ClientURL(), sysCreds, aPub, aJwtLimitsExceeded) + expect_JSDisabledForAccount(c) + // enable js but exceed limits. Followed by fix via restart + updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) + validate_limits(c, limits2) + updateJwt(s.ClientURL(), sysCreds, aPub, aJwtLimitsExceeded) + validate_limits(c, limits2) + s.Shutdown() + conf = createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:%d + jetstream: {max_mem_store: 20Mb, max_file_store: 20Mb, store_dir: "%s"} + operator: %s + resolver: { + type: full + dir: '%s' + } + system_account: %s + `, port, dir, ojwt, dir, sysPub))) + s, _ = RunServerWithConfig(conf) + defer s.Shutdown() + c.Flush() // force client to discover the disconnect + checkClientsCount(t, s, 1) + validate_limits(c, limitsExceeded) + s.Shutdown() + // disable jetstream test + conf = createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:%d + operator: %s + resolver: { + type: full + dir: '%s' + } + system_account: %s + `, port, ojwt, dir, sysPub))) + s, _ = RunServerWithConfig(conf) + defer s.Shutdown() + c.Flush() // force client to discover the disconnect + checkClientsCount(t, s, 1) + expect_JSDisabledForAccount(c) + // test that it stays disabled + updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) + expect_JSDisabledForAccount(c) + c.Close() +} + +func TestJetStreamJWTDisallowBearer(t *testing.T) { + sysKp, syspub := createKey(t) + sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) + sysCreds := newUser(t, sysKp) + + accKp, err := nkeys.CreateAccount() + require_NoError(t, err) + accIdPub, err := accKp.PublicKey() + require_NoError(t, err) + aClaim := jwt.NewAccountClaims(accIdPub) + accJwt1, err := aClaim.Encode(oKp) + require_NoError(t, err) + aClaim.Limits.DisallowBearer = true + accJwt2, err := aClaim.Encode(oKp) + require_NoError(t, err) + + uc := jwt.NewUserClaims("dummy") + uc.BearerToken = true + uOpt1 := createUserCredsEx(t, uc, accKp) + uc.BearerToken = false + uOpt2 := createUserCredsEx(t, uc, accKp) + + dir := t.TempDir() + cf := createConfFile(t, []byte(fmt.Sprintf(` + port: -1 + operator = %s + system_account: %s + resolver: { + type: full + dir: '%s/jwt' + } + resolver_preload = { + %s : "%s" + } + `, ojwt, syspub, dir, syspub, sysJwt))) + s, _ := RunServerWithConfig(cf) + defer s.Shutdown() + + updateJwt(t, s.ClientURL(), sysCreds, accJwt1, 1) + disconnectErrCh := make(chan error, 10) + defer close(disconnectErrCh) + nc1, err := nats.Connect(s.ClientURL(), uOpt1, + nats.NoReconnect(), + nats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) { + disconnectErrCh <- err + })) + require_NoError(t, err) + defer nc1.Close() + + // update jwt and observe bearer token get disconnected + updateJwt(t, s.ClientURL(), sysCreds, accJwt2, 1) + select { + case err := <-disconnectErrCh: + require_Contains(t, err.Error(), "authorization violation") + case <-time.After(time.Second): + t.Fatalf("expected error on disconnect") + } + + // assure bearer token is not allowed to connect + _, err = nats.Connect(s.ClientURL(), uOpt1) + require_Error(t, err) + + // assure non bearer token can connect + nc2, err := nats.Connect(s.ClientURL(), uOpt2) + require_NoError(t, err) + defer nc2.Close() +} + +func TestJetStreamJWTMove(t *testing.T) { + sysKp, syspub := createKey(t) + sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) + sysCreds := newUser(t, sysKp) + + accKp, aExpPub := createKey(t) + + test := func(t *testing.T, replicas int, accClaim *jwt.AccountClaims) { + accClaim.Name = "acc" + accJwt := encodeClaim(t, accClaim, aExpPub) + accCreds := newUser(t, accKp) + + tmlp := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + leaf { + listen: 127.0.0.1:-1 + } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + ` + sc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmlp, 5, 2, + func(serverName, clustername, storeDir, conf string) string { + switch sname := serverName[strings.Index(serverName, "-")+1:]; sname { + case "S1", "S2": + conf = strings.ReplaceAll(conf, "jetstream", "#jetstream") + } + return conf + fmt.Sprintf(` + server_tags: [cloud:%s-tag] + operator: %s + system_account: %s + resolver: { + type: full + dir: '%s/jwt' + } + resolver_preload = { + %s : %s + } + `, clustername, ojwt, syspub, storeDir, syspub, sysJwt) + }, nil) + defer sc.shutdown() + + s := sc.serverByName("C1-S1") + require_False(t, s.JetStreamEnabled()) + updateJwt(t, s.ClientURL(), sysCreds, accJwt, 10) + + sc.waitOnAccount(aExpPub) + + s = sc.serverByName("C2-S1") + require_False(t, s.JetStreamEnabled()) + + nc := natsConnect(t, s.ClientURL(), nats.UserCredentials(accCreds)) + defer nc.Close() + + js, err := nc.JetStream() + require_NoError(t, err) + + ci, err := js.AddStream(&nats.StreamConfig{Name: "MOVE-ME", Replicas: replicas, + Placement: &nats.Placement{Tags: []string{"cloud:C1-tag"}}}) + require_NoError(t, err) + require_Equal(t, ci.Cluster.Name, "C1") + + _, err = js.AddConsumer("MOVE-ME", &nats.ConsumerConfig{Durable: "dur", AckPolicy: nats.AckExplicitPolicy}) + require_NoError(t, err) + _, err = js.Publish("MOVE-ME", []byte("hello world")) + require_NoError(t, err) + + // Perform actual move + _, err = js.UpdateStream(&nats.StreamConfig{Name: "MOVE-ME", Replicas: replicas, + Placement: &nats.Placement{Tags: []string{"cloud:C2-tag"}}}) + require_NoError(t, err) + + sc.clusterForName("C2").waitOnStreamLeader(aExpPub, "MOVE-ME") + + checkFor(t, 30*time.Second, 250*time.Millisecond, func() error { + if si, err := js.StreamInfo("MOVE-ME"); err != nil { + return fmt.Errorf("stream: %v", err) + } else if si.Cluster.Name != "C2" { + return fmt.Errorf("Wrong cluster: %q", si.Cluster.Name) + } else if !strings.HasPrefix(si.Cluster.Leader, "C2-") { + return fmt.Errorf("Wrong leader: %q", si.Cluster.Leader) + } else if len(si.Cluster.Replicas) != replicas-1 { + return fmt.Errorf("Expected %d replicas, got %d", replicas-1, len(si.Cluster.Replicas)) + } else if si.State.Msgs != 1 { + return fmt.Errorf("expected one message") + } + // Now make sure consumer has leader etc.. + if ci, err := js.ConsumerInfo("MOVE-ME", "dur"); err != nil { + return fmt.Errorf("stream: %v", err) + } else if ci.Cluster.Name != "C2" { + return fmt.Errorf("Wrong cluster: %q", ci.Cluster.Name) + } else if ci.Cluster.Leader == _EMPTY_ { + return fmt.Errorf("No leader yet") + } + return nil + }) + + sub, err := js.PullSubscribe("", "dur", nats.BindStream("MOVE-ME")) + require_NoError(t, err) + m, err := sub.Fetch(1) + require_NoError(t, err) + require_NoError(t, m[0].AckSync()) + } + + t.Run("tiered", func(t *testing.T) { + accClaim := jwt.NewAccountClaims(aExpPub) + accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ + DiskStorage: 1100, Consumer: 1, Streams: 1} + accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ + DiskStorage: 3300, Consumer: 1, Streams: 1} + + t.Run("R3", func(t *testing.T) { + test(t, 3, accClaim) + }) + t.Run("R1", func(t *testing.T) { + test(t, 1, accClaim) + }) + }) + + t.Run("non-tiered", func(t *testing.T) { + accClaim := jwt.NewAccountClaims(aExpPub) + accClaim.Limits.JetStreamLimits = jwt.JetStreamLimits{ + DiskStorage: 4400, Consumer: 2, Streams: 2} + + t.Run("R3", func(t *testing.T) { + test(t, 3, accClaim) + }) + t.Run("R1", func(t *testing.T) { + test(t, 1, accClaim) + }) + }) +} + +func TestJetStreamJWTClusteredTiers(t *testing.T) { + sysKp, syspub := createKey(t) + sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) + newUser(t, sysKp) + + accKp, aExpPub := createKey(t) + accClaim := jwt.NewAccountClaims(aExpPub) + accClaim.Name = "acc" + accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ + DiskStorage: 1100, Consumer: 2, Streams: 2} + accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ + DiskStorage: 1100, Consumer: 1, Streams: 1} + accJwt := encodeClaim(t, accClaim, aExpPub) + accCreds := newUser(t, accKp) + tmlp := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + leaf { + listen: 127.0.0.1:-1 + } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + ` + fmt.Sprintf(` + operator: %s + system_account: %s + resolver = MEMORY + resolver_preload = { + %s : %s + %s : %s + } + `, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt) + + c := createJetStreamClusterWithTemplate(t, tmlp, "cluster", 3) + defer c.shutdown() + + nc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds)) + defer nc.Close() + + js, err := nc.JetStream() + require_NoError(t, err) + + // Prevent 'nats: JetStream not enabled for account' when creating the first stream. + c.waitOnAccount(aExpPub) + + // Test absent tiers + _, err = js.AddStream(&nats.StreamConfig{Name: "testR2", Replicas: 2, Subjects: []string{"testR2"}}) + require_Error(t, err) + require_Equal(t, err.Error(), "nats: no JetStream default or applicable tiered limit present") + _, err = js.AddStream(&nats.StreamConfig{Name: "testR5", Replicas: 5, Subjects: []string{"testR5"}}) + require_Error(t, err) + require_Equal(t, err.Error(), "nats: no JetStream default or applicable tiered limit present") + + // Test tiers up to stream limits + _, err = js.AddStream(&nats.StreamConfig{Name: "testR1-1", Replicas: 1, Subjects: []string{"testR1-1"}}) + require_NoError(t, err) + _, err = js.AddStream(&nats.StreamConfig{Name: "testR3-1", Replicas: 3, Subjects: []string{"testR3-1"}}) + require_NoError(t, err) + _, err = js.AddStream(&nats.StreamConfig{Name: "testR1-2", Replicas: 1, Subjects: []string{"testR1-2"}}) + require_NoError(t, err) + + // Test exceeding tiered stream limit + _, err = js.AddStream(&nats.StreamConfig{Name: "testR1-3", Replicas: 1, Subjects: []string{"testR1-3"}}) + require_Error(t, err) + require_Equal(t, err.Error(), "nats: maximum number of streams reached") + _, err = js.AddStream(&nats.StreamConfig{Name: "testR3-3", Replicas: 3, Subjects: []string{"testR3-3"}}) + require_Error(t, err) + require_Equal(t, err.Error(), "nats: maximum number of streams reached") + + // Test tiers up to consumer limits + _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur1", AckPolicy: nats.AckExplicitPolicy}) + require_NoError(t, err) + _, err = js.AddConsumer("testR3-1", &nats.ConsumerConfig{Durable: "dur2", AckPolicy: nats.AckExplicitPolicy}) + require_NoError(t, err) + _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur3", AckPolicy: nats.AckExplicitPolicy}) + require_NoError(t, err) + + // test exceeding tiered consumer limits + _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur4", AckPolicy: nats.AckExplicitPolicy}) + require_Error(t, err) + require_Equal(t, err.Error(), "nats: maximum consumers limit reached") + _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur5", AckPolicy: nats.AckExplicitPolicy}) + require_Error(t, err) + require_Equal(t, err.Error(), "nats: maximum consumers limit reached") + + // test tiered storage limit + msg := [512]byte{} + _, err = js.Publish("testR1-1", msg[:]) + require_NoError(t, err) + _, err = js.Publish("testR3-1", msg[:]) + require_NoError(t, err) + _, err = js.Publish("testR3-1", msg[:]) + require_NoError(t, err) + _, err = js.Publish("testR1-2", msg[:]) + require_NoError(t, err) + + time.Sleep(2000 * time.Millisecond) // wait for update timer to synchronize totals + + // test exceeding tiered storage limit + _, err = js.Publish("testR1-1", []byte("1")) + require_Error(t, err) + require_Equal(t, err.Error(), "nats: resource limits exceeded for account") + _, err = js.Publish("testR3-1", []byte("fail this message!")) + require_Error(t, err) + require_Equal(t, err.Error(), "nats: resource limits exceeded for account") + + // retrieve limits + var info JSApiAccountInfoResponse + m, err := nc.Request("$JS.API.INFO", nil, time.Second) + require_NoError(t, err) + err = json.Unmarshal(m.Data, &info) + require_NoError(t, err) + + require_True(t, info.Memory == 0) + // R1 streams fail message with an add followed by remove, if the update was sent in between, the count is > limit + // Alternative to checking both values is, prior to the info request, wait for another update + require_True(t, info.Store == 4400 || info.Store == 4439) + require_True(t, info.Streams == 3) + require_True(t, info.Consumers == 3) + require_True(t, info.Limits == JetStreamAccountLimits{}) + r1 := info.Tiers["R1"] + require_True(t, r1.Streams == 2) + require_True(t, r1.Consumers == 2) + // R1 streams fail message with an add followed by remove, if the update was sent in between, the count is > limit + // Alternative to checking both values is, prior to the info request, wait for another update + require_True(t, r1.Store == 1100 || r1.Store == 1139) + require_True(t, r1.Memory == 0) + require_True(t, r1.Limits == JetStreamAccountLimits{ + MaxMemory: 0, + MaxStore: 1100, + MaxStreams: 2, + MaxConsumers: 2, + MaxAckPending: -1, + MemoryMaxStreamBytes: -1, + StoreMaxStreamBytes: -1, + MaxBytesRequired: false, + }) + r3 := info.Tiers["R3"] + require_True(t, r3.Streams == 1) + require_True(t, r3.Consumers == 1) + require_True(t, r3.Store == 3300) + require_True(t, r3.Memory == 0) + require_True(t, r3.Limits == JetStreamAccountLimits{ + MaxMemory: 0, + MaxStore: 1100, + MaxStreams: 1, + MaxConsumers: 1, + MaxAckPending: -1, + MemoryMaxStreamBytes: -1, + StoreMaxStreamBytes: -1, + MaxBytesRequired: false, + }) +} + +func TestJetStreamJWTClusteredTiersChange(t *testing.T) { + sysKp, syspub := createKey(t) + sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) + sysCreds := newUser(t, sysKp) + + accKp, aExpPub := createKey(t) + accClaim := jwt.NewAccountClaims(aExpPub) + accClaim.Name = "acc" + accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ + DiskStorage: 1000, MemoryStorage: 0, Consumer: 1, Streams: 1} + accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ + DiskStorage: 500, MemoryStorage: 0, Consumer: 1, Streams: 1} + accJwt1 := encodeClaim(t, accClaim, aExpPub) + accCreds := newUser(t, accKp) + start := time.Now() + + tmlp := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + leaf { + listen: 127.0.0.1:-1 + } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + ` + c := createJetStreamClusterWithTemplateAndModHook(t, tmlp, "cluster", 3, + func(serverName, clustername, storeDir, conf string) string { + return conf + fmt.Sprintf(` + operator: %s + system_account: %s + resolver: { + type: full + dir: '%s/jwt' + }`, ojwt, syspub, storeDir) + }) + defer c.shutdown() + + updateJwt(t, c.randomServer().ClientURL(), sysCreds, sysJwt, 3) + updateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt1, 3) + + c.waitOnAccount(aExpPub) + + nc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds)) + defer nc.Close() + + js, err := nc.JetStream() + require_NoError(t, err) + + // Prevent 'nats: JetStream not enabled for account' when creating the first stream. + c.waitOnAccount(aExpPub) + + // Test tiers up to stream limits + cfg := &nats.StreamConfig{Name: "testR1-1", Replicas: 1, Subjects: []string{"testR1-1"}, MaxBytes: 1000} + _, err = js.AddStream(cfg) + require_NoError(t, err) + + cfg.Replicas = 3 + _, err = js.UpdateStream(cfg) + require_Error(t, err, errors.New("nats: insufficient storage resources available")) + + time.Sleep(time.Second - time.Since(start)) // make sure the time stamp changes + accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ + DiskStorage: 1000, MemoryStorage: 0, Consumer: 1, Streams: 1} + accJwt2 := encodeClaim(t, accClaim, aExpPub) + + updateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt2, 3) + + var rBefore, rAfter JSApiAccountInfoResponse + m, err := nc.Request("$JS.API.INFO", nil, time.Second) + require_NoError(t, err) + err = json.Unmarshal(m.Data, &rBefore) + require_NoError(t, err) + _, err = js.UpdateStream(cfg) + require_NoError(t, err) + + m, err = nc.Request("$JS.API.INFO", nil, time.Second) + require_NoError(t, err) + err = json.Unmarshal(m.Data, &rAfter) + require_NoError(t, err) + require_True(t, rBefore.Tiers["R1"].Streams == 1) + require_True(t, rBefore.Tiers["R1"].Streams == rAfter.Tiers["R3"].Streams) + require_True(t, rBefore.Tiers["R3"].Streams == 0) + require_True(t, rAfter.Tiers["R1"].Streams == 0) +} + +func TestJetStreamJWTClusteredDeleteTierWithStreamAndMove(t *testing.T) { + sysKp, syspub := createKey(t) + sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) + sysCreds := newUser(t, sysKp) + + accKp, aExpPub := createKey(t) + accClaim := jwt.NewAccountClaims(aExpPub) + accClaim.Name = "acc" + accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ + DiskStorage: 1000, MemoryStorage: 0, Consumer: 1, Streams: 1} + accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ + DiskStorage: 3000, MemoryStorage: 0, Consumer: 1, Streams: 1} + accJwt1 := encodeClaim(t, accClaim, aExpPub) + accCreds := newUser(t, accKp) + start := time.Now() + + tmlp := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + leaf { + listen: 127.0.0.1:-1 + } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + ` + c := createJetStreamClusterWithTemplateAndModHook(t, tmlp, "cluster", 3, + func(serverName, clustername, storeDir, conf string) string { + return conf + fmt.Sprintf(` + operator: %s + system_account: %s + resolver: { + type: full + dir: '%s/jwt' + }`, ojwt, syspub, storeDir) + }) + defer c.shutdown() + + updateJwt(t, c.randomServer().ClientURL(), sysCreds, sysJwt, 3) + updateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt1, 3) + + c.waitOnAccount(aExpPub) + + nc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds)) + defer nc.Close() + + js, err := nc.JetStream() + require_NoError(t, err) + + // Prevent 'nats: JetStream not enabled for account' when creating the first stream. + c.waitOnAccount(aExpPub) + + // Test tiers up to stream limits + cfg := &nats.StreamConfig{Name: "testR1-1", Replicas: 1, Subjects: []string{"testR1-1"}, MaxBytes: 1000} + _, err = js.AddStream(cfg) + require_NoError(t, err) + + sl := c.streamLeader(aExpPub, "testR1-1") + require_NotNil(t, sl) + acc, err := sl.lookupAccount(aExpPub) + require_NoError(t, err) + mset, err := acc.lookupStream("testR1-1") + require_NoError(t, err) + require_Equal(t, mset.lastSeq(), 0) + + _, err = js.Publish("testR1-1", nil) + require_NoError(t, err) + require_Equal(t, mset.lastSeq(), 1) + + time.Sleep(time.Second - time.Since(start)) // make sure the time stamp changes + delete(accClaim.Limits.JetStreamTieredLimits, "R1") + accJwt2 := encodeClaim(t, accClaim, aExpPub) + updateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt2, 3) + + var respBefore JSApiAccountInfoResponse + m, err := nc.Request("$JS.API.INFO", nil, time.Second) + require_NoError(t, err) + err = json.Unmarshal(m.Data, &respBefore) + require_NoError(t, err) + + require_True(t, respBefore.JetStreamAccountStats.Tiers["R3"].Streams == 0) + require_True(t, respBefore.JetStreamAccountStats.Tiers["R1"].Streams == 1) + + _, err = js.Publish("testR1-1", nil) + require_Error(t, err) + require_Equal(t, err.Error(), "nats: no JetStream default or applicable tiered limit present") + require_Equal(t, mset.lastSeq(), 1) + + cfg.Replicas = 3 + _, err = js.UpdateStream(cfg) + require_NoError(t, err) + + // I noticed this taking > 5 seconds + checkFor(t, 10*time.Second, 250*time.Millisecond, func() error { + _, err = js.Publish("testR1-1", nil) + return err + }) + + var respAfter JSApiAccountInfoResponse + m, err = nc.Request("$JS.API.INFO", nil, time.Second) + require_NoError(t, err) + err = json.Unmarshal(m.Data, &respAfter) + require_NoError(t, err) + + require_True(t, respAfter.JetStreamAccountStats.Tiers["R3"].Streams == 1) + require_True(t, respAfter.JetStreamAccountStats.Tiers["R3"].Store > 0) + + _, ok := respAfter.JetStreamAccountStats.Tiers["R1"] + require_True(t, !ok) +} + +func TestJetStreamJWTSysAccUpdateMixedMode(t *testing.T) { + skp, spub := createKey(t) + sUsr := createUserCreds(t, nil, skp) + sysClaim := jwt.NewAccountClaims(spub) + sysClaim.Name = "SYS" + sysJwt := encodeClaim(t, sysClaim, spub) + encodeJwt1Time := time.Now() + + akp, apub := createKey(t) + aUsr := createUserCreds(t, nil, akp) + claim := jwt.NewAccountClaims(apub) + claim.Limits.JetStreamLimits.DiskStorage = 1024 * 1024 + claim.Limits.JetStreamLimits.Streams = 1 + jwt1 := encodeClaim(t, claim, apub) + + basePath := "/ngs/v1/accounts/jwt/" + reqCount := int32(0) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == basePath { + w.Write([]byte("ok")) + } else if strings.HasSuffix(r.URL.Path, spub) { + w.Write([]byte(sysJwt)) + } else if strings.HasSuffix(r.URL.Path, apub) { + w.Write([]byte(jwt1)) + } else { + // only count requests that could be filled + return + } + atomic.AddInt32(&reqCount, 1) + })) + defer ts.Close() + + tmpl := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + ` + + sc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, 3, 2, + func(serverName, clusterName, storeDir, conf string) string { + // create an ngs like setup, with connection and non connection server + if clusterName == "C1" { + conf = strings.ReplaceAll(conf, "jetstream", "#jetstream") + } + return fmt.Sprintf(`%s + operator: %s + system_account: %s + resolver: URL("%s%s")`, conf, ojwt, spub, ts.URL, basePath) + }, nil) + defer sc.shutdown() + disconnectChan := make(chan struct{}, 100) + defer close(disconnectChan) + disconnectCb := nats.DisconnectErrHandler(func(conn *nats.Conn, err error) { + disconnectChan <- struct{}{} + }) + + s := sc.clusterForName("C1").randomServer() + + sysNc := natsConnect(t, s.ClientURL(), sUsr, disconnectCb, nats.NoCallbacksAfterClientClose()) + defer sysNc.Close() + aNc := natsConnect(t, s.ClientURL(), aUsr, disconnectCb, nats.NoCallbacksAfterClientClose()) + defer aNc.Close() + + js, err := aNc.JetStream() + require_NoError(t, err) + + // Prevent 'nats: JetStream not enabled for account' when creating the first stream. + sc.waitOnAccount(apub) + + si, err := js.AddStream(&nats.StreamConfig{Name: "bar", Subjects: []string{"bar"}, Replicas: 3}) + require_NoError(t, err) + require_Equal(t, si.Cluster.Name, "C2") + _, err = js.AccountInfo() + require_NoError(t, err) + + r, err := sysNc.Request(fmt.Sprintf(serverPingReqSubj, "ACCOUNTZ"), + []byte(fmt.Sprintf(`{"account":"%s"}`, spub)), time.Second) + require_NoError(t, err) + respb := ServerAPIResponse{Data: &Accountz{}} + require_NoError(t, json.Unmarshal(r.Data, &respb)) + + hasJSExp := func(resp *ServerAPIResponse) bool { + found := false + for _, e := range resp.Data.(*Accountz).Account.Exports { + if e.Subject == jsAllAPI { + found = true + break + } + } + return found + } + require_True(t, hasJSExp(&respb)) + + // make sure jti increased + time.Sleep(time.Second - time.Since(encodeJwt1Time)) + sysJwt2 := encodeClaim(t, sysClaim, spub) + + oldRcount := atomic.LoadInt32(&reqCount) + _, err = sysNc.Request(fmt.Sprintf(accUpdateEventSubjNew, spub), []byte(sysJwt2), time.Second) + require_NoError(t, err) + // test to make sure connected client (aNc) was not kicked + time.Sleep(200 * time.Millisecond) + require_True(t, len(disconnectChan) == 0) + + // ensure nothing new has happened, lookup for account not found is skipped during inc + require_True(t, atomic.LoadInt32(&reqCount) == oldRcount) + // no responders + _, err = aNc.Request("foo", nil, time.Second) + require_Error(t, err) + require_Equal(t, err.Error(), "nats: no responders available for request") + + nc2, js2 := jsClientConnect(t, sc.clusterForName("C2").randomServer(), aUsr) + defer nc2.Close() + _, err = js2.AccountInfo() + require_NoError(t, err) + + r, err = sysNc.Request(fmt.Sprintf(serverPingReqSubj, "ACCOUNTZ"), + []byte(fmt.Sprintf(`{"account":"%s"}`, spub)), time.Second) + require_NoError(t, err) + respa := ServerAPIResponse{Data: &Accountz{}} + require_NoError(t, json.Unmarshal(r.Data, &respa)) + require_True(t, hasJSExp(&respa)) + + _, err = js.AccountInfo() + require_NoError(t, err) +} + +func TestJetStreamJWTExpiredAccountNotCountedTowardLimits(t *testing.T) { + op, _ := nkeys.CreateOperator() + opPk, _ := op.PublicKey() + sk, _ := nkeys.CreateOperator() + skPk, _ := sk.PublicKey() + opClaim := jwt.NewOperatorClaims(opPk) + opClaim.SigningKeys.Add(skPk) + opJwt, err := opClaim.Encode(op) + require_NoError(t, err) + createAccountAndUser := func(pubKey, jwt1, creds1 *string) { + t.Helper() + kp, _ := nkeys.CreateAccount() + *pubKey, _ = kp.PublicKey() + claim := jwt.NewAccountClaims(*pubKey) + claim.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: 7 * 1024 * 1024, DiskStorage: 7 * 1024 * 1024, Streams: 10} + var err error + *jwt1, err = claim.Encode(sk) + require_NoError(t, err) + + ukp, _ := nkeys.CreateUser() + seed, _ := ukp.Seed() + upub, _ := ukp.PublicKey() + uclaim := newJWTTestUserClaims() + uclaim.Subject = upub + + ujwt1, err := uclaim.Encode(kp) + require_NoError(t, err) + *creds1 = genCredsFile(t, ujwt1, seed) + } + generateRequest := func(accs []string, kp nkeys.KeyPair) []byte { + t.Helper() + opk, _ := kp.PublicKey() + c := jwt.NewGenericClaims(opk) + c.Data["accounts"] = accs + cJwt, err := c.Encode(kp) + if err != nil { + t.Fatalf("Expected no error %v", err) + } + return []byte(cJwt) + } + + var syspub, sysjwt, sysCreds string + createAccountAndUser(&syspub, &sysjwt, &sysCreds) + + dirSrv := t.TempDir() + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:-1 + operator: %s + jetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: "%s"} + system_account: %s + resolver: { + type: full + allow_delete: true + dir: '%s' + timeout: "500ms" + } + `, opJwt, dirSrv, syspub, dirSrv))) + + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + // update system account jwt + updateJwt(t, s.ClientURL(), sysCreds, sysjwt, 1) + + var apub, ajwt1, aCreds1 string + createAccountAndUser(&apub, &ajwt1, &aCreds1) + // push jwt (for full resolver) + updateJwt(t, s.ClientURL(), sysCreds, ajwt1, 1) + + ncA, jsA := jsClientConnect(t, s, nats.UserCredentials(aCreds1)) + defer ncA.Close() + + ai, err := jsA.AccountInfo() + require_NoError(t, err) + require_True(t, ai.Limits.MaxMemory == 7*1024*1024) + ncA.Close() + + nc := natsConnect(t, s.ClientURL(), nats.UserCredentials(sysCreds)) + defer nc.Close() + resp, err := nc.Request(accDeleteReqSubj, generateRequest([]string{apub}, sk), time.Second) + require_NoError(t, err) + require_True(t, strings.Contains(string(resp.Data), `"message":"deleted 1 accounts"`)) + + var apub2, ajwt2, aCreds2 string + createAccountAndUser(&apub2, &ajwt2, &aCreds2) + // push jwt (for full resolver) + updateJwt(t, s.ClientURL(), sysCreds, ajwt2, 1) + + ncB, jsB := jsClientConnect(t, s, nats.UserCredentials(aCreds2)) + defer ncB.Close() + + ai, err = jsB.AccountInfo() + require_NoError(t, err) + require_True(t, ai.Limits.MaxMemory == 7*1024*1024) +} + +func TestJetStreamJWTExpiredAccountWorksAfterExpirationUpdated(t *testing.T) { + sysKp, spub := createKey(t) + sysClaim := jwt.NewAccountClaims(spub) + sysClaim.Name = "$SYS" + sysJwt, err := sysClaim.Encode(oKp) + require_NoError(t, err) + sysCreds := newUser(t, sysKp) + + akp, apub := createKey(t) + require_NoError(t, err) + accClaim := jwt.NewAccountClaims(apub) + accClaim.Name = "TEST" + accClaim.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: 1024 * 1024, DiskStorage: 2048 * 1024, Streams: 1} + accJwt, err := accClaim.Encode(oKp) + require_NoError(t, err) + + dirSrv := t.TempDir() + dir := t.TempDir() + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:-1 + jetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: "%s"} + operator: %s + system_account: %s + resolver: { + type: full + allow_delete: true + dir: '%s' + } + `, dirSrv, ojwt, spub, dir))) + defer removeFile(t, conf) + + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + updateJwt(t, s.ClientURL(), sysCreds, sysJwt, 1) + updateJwt(t, s.ClientURL(), sysCreds, accJwt, 1) + + userCreds := newUser(t, akp) + nc, js := jsClientConnect(t, s, nats.UserCredentials(userCreds), nats.NoReconnect(), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + // Default handler would print to stderr, silence it + })) + defer nc.Close() + + _, err = js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo"}, + }) + require_NoError(t, err) + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + for { + if _, err := js.Publish("foo", []byte("hello")); err != nil { + return + } + time.Sleep(250 * time.Millisecond) + } + }() + + // Update the account so it expires in 2 seconds. + expires := time.Now().Add(2 * time.Second) + accClaim.Expires = expires.Unix() + accJwt = encodeClaim(t, accClaim, apub) + updateJwt(t, s.ClientURL(), sysCreds, accJwt, 1) + + // Wait for the publishing routine to fail. + wg.Wait() + + // Close this client, we will create a new one. + nc.Close() + + // Verify that we can't connect anymore. Because of rounding, we may have + // to try to avoid flapping. + for range 5 { + nc, err = nats.Connect(s.ClientURL(), nats.UserCredentials(userCreds), nats.NoReconnect(), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + // Default handler would print to stderr, silence it + })) + if err != nil { + // ok! + break + } + nc.Close() + time.Sleep(100 * time.Millisecond) + } + + // Check that the user account is marked as expired. + az, err := s.Accountz(&AccountzOptions{Account: apub}) + require_NoError(t, err) + require_True(t, az.Account != nil) + require_True(t, az.Account.Expired) + // But should still be true + require_True(t, az.Account.JetStream) + + // Update the expiration to 1 hour + expires = time.Now().Add(time.Hour) + accClaim.Expires = expires.Unix() + accJwt = encodeClaim(t, accClaim, apub) + updateJwt(t, s.ClientURL(), sysCreds, accJwt, 1) + + // Check that its is no longer expired and has still JetStream enabled. + az, err = s.Accountz(&AccountzOptions{Account: apub}) + require_NoError(t, err) + require_True(t, az.Account != nil) + require_False(t, az.Account.Expired) + require_True(t, az.Account.JetStream) + + // Create a connection and ensure we connect ok and can send a message. + nc, js = jsClientConnect(t, s, nats.UserCredentials(userCreds), nats.NoReconnect()) + defer nc.Close() + + _, err = js.Publish("foo", []byte("hello")) + require_NoError(t, err) +} + +// TestJetStreamJWTHealthzIgnoresExpiredAccounts verifies that a general +// healthz scan does not report unavailable solely because a JWT account with +// JetStream assets has expired (issue #8084). +func TestJetStreamJWTHealthzIgnoresExpiredAccounts(t *testing.T) { + sysKp, spub := createKey(t) + sysClaim := jwt.NewAccountClaims(spub) + sysClaim.Name = "$SYS" + sysJwt, err := sysClaim.Encode(oKp) + require_NoError(t, err) + sysCreds := newUser(t, sysKp) + + akp, apub := createKey(t) + accClaim := jwt.NewAccountClaims(apub) + accClaim.Name = "TEST" + accClaim.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: 1024 * 1024, DiskStorage: 2048 * 1024, Streams: 1} + accJwt, err := accClaim.Encode(oKp) + require_NoError(t, err) + + dirSrv := filepath.ToSlash(t.TempDir()) + dir := filepath.ToSlash(t.TempDir()) + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:-1 + jetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: "%s"} + operator: %s + system_account: %s + resolver: { + type: full + allow_delete: true + dir: '%s' + } + `, dirSrv, ojwt, spub, dir))) + defer removeFile(t, conf) + + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + updateJwt(t, s.ClientURL(), sysCreds, sysJwt, 1) + updateJwt(t, s.ClientURL(), sysCreds, accJwt, 1) + + userCreds := newUser(t, akp) + nc, js := jsClientConnect(t, s, nats.UserCredentials(userCreds), nats.NoReconnect()) + defer nc.Close() + + _, err = js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo"}, + }) + require_NoError(t, err) + nc.Close() + + // Expire the account and wait until LookupAccount sees it as expired. + expires := time.Now().Add(2 * time.Second) + accClaim.Expires = expires.Unix() + accJwt = encodeClaim(t, accClaim, apub) + updateJwt(t, s.ClientURL(), sysCreds, accJwt, 1) + + checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { + acc, err := s.LookupAccount(apub) + if err != nil { + if errors.Is(err, ErrAccountExpired) { + return nil + } + return err + } + if acc.IsExpired() { + return nil + } + return fmt.Errorf("account %s not expired yet", apub) + }) + + status := s.healthz(nil) + require_True(t, status.Status == "ok") + require_True(t, status.StatusCode == http.StatusOK) + + // Explicit account healthz must still report unavailable for an expired account. + explicit := s.healthz(&HealthzOptions{Account: apub}) + require_True(t, explicit.Status == "unavailable") + require_True(t, explicit.StatusCode == http.StatusServiceUnavailable) + + // With details + stream, keep the expired-account error instead of a 404 for + // a stream that was never examined because the account was skipped. + detailed := s.healthz(&HealthzOptions{Account: apub, Stream: "TEST", Details: true}) + require_True(t, detailed.Status == "error") + require_True(t, detailed.StatusCode == http.StatusServiceUnavailable) + require_True(t, len(detailed.Errors) == 1) + require_True(t, detailed.Errors[0].Type == HealthzErrorAccount) +} + +func TestJetStreamJWTDeletedAccountDoesNotLeakSubscriptions(t *testing.T) { + op, _ := nkeys.CreateOperator() + opPk, _ := op.PublicKey() + sk, _ := nkeys.CreateOperator() + skPk, _ := sk.PublicKey() + opClaim := jwt.NewOperatorClaims(opPk) + opClaim.SigningKeys.Add(skPk) + opJwt, err := opClaim.Encode(op) + require_NoError(t, err) + createAccountAndUser := func(pubKey, jwt1, creds1 *string) { + t.Helper() + kp, _ := nkeys.CreateAccount() + *pubKey, _ = kp.PublicKey() + claim := jwt.NewAccountClaims(*pubKey) + claim.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: 7 * 1024 * 1024, DiskStorage: 7 * 1024 * 1024, Streams: 10} + var err error + *jwt1, err = claim.Encode(sk) + require_NoError(t, err) + + ukp, _ := nkeys.CreateUser() + seed, _ := ukp.Seed() + upub, _ := ukp.PublicKey() + uclaim := newJWTTestUserClaims() + uclaim.Subject = upub + + ujwt1, err := uclaim.Encode(kp) + require_NoError(t, err) + *creds1 = genCredsFile(t, ujwt1, seed) + } + generateRequest := func(accs []string, kp nkeys.KeyPair) []byte { + t.Helper() + opk, _ := kp.PublicKey() + c := jwt.NewGenericClaims(opk) + c.Data["accounts"] = accs + cJwt, err := c.Encode(kp) + if err != nil { + t.Fatalf("Expected no error %v", err) + } + return []byte(cJwt) + } + + var syspub, sysjwt, sysCreds string + createAccountAndUser(&syspub, &sysjwt, &sysCreds) + + dirSrv := t.TempDir() + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:-1 + operator: %s + jetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: %v} + system_account: %s + resolver: { + type: full + allow_delete: true + dir: '%s' + timeout: "500ms" + } + `, opJwt, dirSrv, syspub, dirSrv))) + + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + checkNumSubs := func(expected uint32) uint32 { + t.Helper() + // Wait a bit before capturing number of subs... + time.Sleep(250 * time.Millisecond) + + var ns uint32 + checkFor(t, time.Second, 50*time.Millisecond, func() error { + subsz, err := s.Subsz(nil) + if err != nil { + return err + } + ns = subsz.NumSubs + if expected > 0 && ns > expected { + return fmt.Errorf("Expected num subs to be back at %v, got %v", + expected, ns) + } + return nil + }) + return ns + } + beforeCreate := checkNumSubs(0) + + // update system account jwt + updateJwt(t, s.ClientURL(), sysCreds, sysjwt, 1) + + createAndDelete := func() { + t.Helper() + + var apub, ajwt1, aCreds1 string + createAccountAndUser(&apub, &ajwt1, &aCreds1) + // push jwt (for full resolver) + updateJwt(t, s.ClientURL(), sysCreds, ajwt1, 1) + + ncA, jsA := jsClientConnect(t, s, nats.UserCredentials(aCreds1)) + defer ncA.Close() + + ai, err := jsA.AccountInfo() + require_NoError(t, err) + require_True(t, ai.Limits.MaxMemory == 7*1024*1024) + ncA.Close() + + nc := natsConnect(t, s.ClientURL(), nats.UserCredentials(sysCreds)) + defer nc.Close() + + resp, err := nc.Request(accDeleteReqSubj, generateRequest([]string{apub}, sk), time.Second) + require_NoError(t, err) + require_True(t, strings.Contains(string(resp.Data), `"message":"deleted 1 accounts"`)) + } + + // Create and delete multiple accounts + for i := 0; i < 10; i++ { + createAndDelete() + } + + // There is a subscription on `_R_.>` that is created on the system account + // and that will not go away, so discount it. + checkNumSubs(beforeCreate + 1) +} + +func TestJetStreamJWTDeletedAccountIsReEnabled(t *testing.T) { + op, _ := nkeys.CreateOperator() + opPk, _ := op.PublicKey() + sk, _ := nkeys.CreateOperator() + skPk, _ := sk.PublicKey() + opClaim := jwt.NewOperatorClaims(opPk) + opClaim.SigningKeys.Add(skPk) + opJwt, err := opClaim.Encode(op) + require_NoError(t, err) + createAccountAndUser := func(pubKey, jwt1, creds1 *string) { + t.Helper() + kp, _ := nkeys.CreateAccount() + *pubKey, _ = kp.PublicKey() + claim := jwt.NewAccountClaims(*pubKey) + claim.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: 7 * 1024 * 1024, DiskStorage: 7 * 1024 * 1024, Streams: 10} + var err error + *jwt1, err = claim.Encode(sk) + require_NoError(t, err) + + ukp, _ := nkeys.CreateUser() + seed, _ := ukp.Seed() + upub, _ := ukp.PublicKey() + uclaim := newJWTTestUserClaims() + uclaim.Subject = upub + + ujwt1, err := uclaim.Encode(kp) + require_NoError(t, err) + *creds1 = genCredsFile(t, ujwt1, seed) + } + generateRequest := func(accs []string, kp nkeys.KeyPair) []byte { + t.Helper() + opk, _ := kp.PublicKey() + c := jwt.NewGenericClaims(opk) + c.Data["accounts"] = accs + cJwt, err := c.Encode(kp) + if err != nil { + t.Fatalf("Expected no error %v", err) + } + return []byte(cJwt) + } + + // admin user + var syspub, sysjwt, sysCreds string + createAccountAndUser(&syspub, &sysjwt, &sysCreds) + + dirSrv := t.TempDir() + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:-1 + operator: %s + jetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: "%s"} + system_account: %s + resolver: { + type: full + allow_delete: true + dir: '%s' + timeout: "500ms" + } + `, opJwt, dirSrv, syspub, dirSrv))) + + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + // update system account jwt + updateJwt(t, s.ClientURL(), sysCreds, sysjwt, 1) + + // create account + var apub, ajwt1, aCreds1 string + kp, _ := nkeys.CreateAccount() + apub, _ = kp.PublicKey() + claim := jwt.NewAccountClaims(apub) + claim.Limits.JetStreamLimits = jwt.JetStreamLimits{ + MemoryStorage: 7 * 1024 * 1024, + DiskStorage: 7 * 1024 * 1024, + Streams: 10, + } + ajwt1, err = claim.Encode(sk) + require_NoError(t, err) + + // user + ukp, _ := nkeys.CreateUser() + seed, _ := ukp.Seed() + upub, _ := ukp.PublicKey() + uclaim := newJWTTestUserClaims() + uclaim.Subject = upub + + ujwt1, err := uclaim.Encode(kp) + require_NoError(t, err) + aCreds1 = genCredsFile(t, ujwt1, seed) + + // push user account + updateJwt(t, s.ClientURL(), sysCreds, ajwt1, 1) + + ncA, jsA := jsClientConnect(t, s, nats.UserCredentials(aCreds1)) + defer ncA.Close() + + jsA.AddStream(&nats.StreamConfig{Name: "foo"}) + jsA.Publish("foo", []byte("Hello World")) + jsA.Publish("foo", []byte("Hello Again")) + + // JS should be working + ai, err := jsA.AccountInfo() + require_NoError(t, err) + require_True(t, ai.Limits.MaxMemory == 7*1024*1024) + require_True(t, ai.Limits.MaxStore == 7*1024*1024) + require_True(t, ai.Tier.Streams == 1) + + // connect with a different connection and delete the account. + nc := natsConnect(t, s.ClientURL(), nats.UserCredentials(sysCreds)) + defer nc.Close() + + // delete account + resp, err := nc.Request(accDeleteReqSubj, generateRequest([]string{apub}, sk), time.Second) + require_NoError(t, err) + require_True(t, strings.Contains(string(resp.Data), `"message":"deleted 1 accounts"`)) + + // account was disabled and now disconnected, this should get a connection is closed error. + _, err = jsA.AccountInfo() + if err == nil || !errors.Is(err, nats.ErrConnectionClosed) { + t.Errorf("Expected connection closed error, got: %v", err) + } + ncA.Close() + + // re-enable, same claims would be detected + updateJwt(t, s.ClientURL(), sysCreds, ajwt1, 1) + + // expected to get authorization timeout at this time + _, err = nats.Connect(s.ClientURL(), nats.UserCredentials(aCreds1)) + if !errors.Is(err, nats.ErrAuthorization) { + t.Errorf("Expected authorization issue on connect, got: %v", err) + } + + // edit the account and push again with updated claims to same account + claim = jwt.NewAccountClaims(apub) + claim.Limits.JetStreamLimits = jwt.JetStreamLimits{ + MemoryStorage: -1, + DiskStorage: 10 * 1024 * 1024, + Streams: 10, + } + ajwt1, err = claim.Encode(sk) + require_NoError(t, err) + updateJwt(t, s.ClientURL(), sysCreds, ajwt1, 1) + + // reconnect with the updated account + ncA, jsA = jsClientConnect(t, s, nats.UserCredentials(aCreds1)) + defer ncA.Close() + ai, err = jsA.AccountInfo() + if err != nil { + t.Fatal(err) + } + require_True(t, ai.Limits.MaxMemory == -1) + require_True(t, ai.Limits.MaxStore == 10*1024*1024) + require_True(t, ai.Tier.Streams == 1) + + // should be possible to get stream info again + si, err := jsA.StreamInfo("foo") + if err != nil { + t.Fatal(err) + } + if si.State.Msgs != 2 { + t.Fatal("Unexpected number of messages from recovered stream") + } + msg, err := jsA.GetMsg("foo", 1) + if err != nil { + t.Fatal(err) + } + if string(msg.Data) != "Hello World" { + t.Error("Unexpected message") + } + ncA.Close() +} + +// Make sure 100MB HA means 100MB of R3, not 33.3MB. +func TestJetStreamJWTHAStorageLimitsAndAccounting(t *testing.T) { + sysKp, syspub := createKey(t) + sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) + newUser(t, sysKp) + + maxFileStorage := int64(100 * 1024 * 1024) + maxMemStorage := int64(2 * 1024 * 1024) + + accKp, aExpPub := createKey(t) + accClaim := jwt.NewAccountClaims(aExpPub) + accClaim.Name = "acc" + accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{DiskStorage: maxFileStorage, MemoryStorage: maxMemStorage} + accJwt := encodeClaim(t, accClaim, aExpPub) + accCreds := newUser(t, accKp) + tmlp := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + leaf { listen: 127.0.0.1:-1 } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + ` + fmt.Sprintf(` + operator: %s + system_account: %s + resolver = MEMORY + resolver_preload = { + %s : %s + %s : %s + } + `, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt) + + c := createJetStreamClusterWithTemplate(t, tmlp, "cluster", 3) + defer c.shutdown() + + nc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds)) + defer nc.Close() + + js, err := nc.JetStream() + require_NoError(t, err) + + // Prevent 'nats: JetStream not enabled for account' when creating the first stream. + c.waitOnAccount(aExpPub) + + // Test max bytes first. + _, err = js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3, MaxBytes: maxFileStorage, Subjects: []string{"foo"}}) + require_NoError(t, err) + + require_NoError(t, js.DeleteStream("TEST")) + + _, err = js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3, Subjects: []string{"foo"}}) + require_NoError(t, err) + + // Now test actual usage. + // We should be able to send just over 200 of these. + msg := [500 * 1024]byte{} + for i := 0; i < 250; i++ { + if _, err := js.Publish("foo", msg[:]); err != nil { + require_Error(t, err, NewJSAccountResourcesExceededError()) + require_True(t, i > 200) + break + } + } + + si, err := js.StreamInfo("TEST") + require_NoError(t, err) + // Make sure we are no more then 1 msg below our max in terms of size. + delta := maxFileStorage - int64(si.State.Bytes) + require_True(t, int(delta) < len(msg)) + + // Now memory as well. + require_NoError(t, js.DeleteStream("TEST")) + + // Test max bytes first. + _, err = js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3, MaxBytes: maxMemStorage, Storage: nats.MemoryStorage, Subjects: []string{"foo"}}) + require_NoError(t, err) + + require_NoError(t, js.DeleteStream("TEST")) + + _, err = js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3, Storage: nats.MemoryStorage, Subjects: []string{"foo"}}) + require_NoError(t, err) + + // This is much smaller, so should only be able to send 4. + for i := 0; i < 5; i++ { + if _, err := js.Publish("foo", msg[:]); err != nil { + require_Error(t, err, NewJSAccountResourcesExceededError()) + require_Equal(t, i, 4) + break + } + } + + si, err = js.StreamInfo("TEST") + require_NoError(t, err) + // Make sure we are no more then 1 msg below our max in terms of size. + delta = maxMemStorage - int64(si.State.Bytes) + require_True(t, int(delta) < len(msg)) +} + +func TestJetStreamJWTHAStorageLimitsOnScaleAndUpdate(t *testing.T) { + sysKp, syspub := createKey(t) + sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) + newUser(t, sysKp) + + maxFileStorage := int64(5 * 1024 * 1024) + maxMemStorage := int64(1 * 1024 * 1024) + + accKp, aExpPub := createKey(t) + accClaim := jwt.NewAccountClaims(aExpPub) + accClaim.Name = "acc" + accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{DiskStorage: maxFileStorage, MemoryStorage: maxMemStorage} + accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{DiskStorage: maxFileStorage, MemoryStorage: maxMemStorage} + + accJwt := encodeClaim(t, accClaim, aExpPub) + accCreds := newUser(t, accKp) + tmlp := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + leaf { listen: 127.0.0.1:-1 } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + ` + fmt.Sprintf(` + operator: %s + system_account: %s + resolver = MEMORY + resolver_preload = { + %s : %s + %s : %s + } + `, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt) + + c := createJetStreamClusterWithTemplate(t, tmlp, "cluster", 3) + defer c.shutdown() + + nc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds)) + defer nc.Close() + + js, err := nc.JetStream() + require_NoError(t, err) + + // Prevent 'nats: JetStream not enabled for account' when creating the first stream. + c.waitOnAccount(aExpPub) + + // Test max bytes first. + _, err = js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3, MaxBytes: maxFileStorage, Subjects: []string{"foo"}}) + require_NoError(t, err) + // Now delete + require_NoError(t, js.DeleteStream("TEST")) + // Now do 5 1MB streams. + for i := 1; i <= 5; i++ { + sname := fmt.Sprintf("TEST%d", i) + _, err = js.AddStream(&nats.StreamConfig{Name: sname, Replicas: 3, MaxBytes: 1 * 1024 * 1024}) + require_NoError(t, err) + } + // Should fail. + _, err = js.AddStream(&nats.StreamConfig{Name: "TEST6", Replicas: 3, MaxBytes: 1 * 1024 * 1024}) + require_Error(t, err, errors.New("insufficient storage resources")) + + // Update Test1 and Test2 to smaller reservations. + _, err = js.UpdateStream(&nats.StreamConfig{Name: "TEST1", Replicas: 3, MaxBytes: 512 * 1024}) + require_NoError(t, err) + _, err = js.UpdateStream(&nats.StreamConfig{Name: "TEST2", Replicas: 3, MaxBytes: 512 * 1024}) + require_NoError(t, err) + // Now make sure TEST6 succeeds. + checkFor(t, 1*time.Second, 500*time.Millisecond, func() error { + _, err = js.AddStream(&nats.StreamConfig{Name: "TEST6", Replicas: 3, MaxBytes: 1 * 1024 * 1024}) + // Since the stream leader answers the stream update, and the meta leader determines resources, + // we could hit a race condition here. Simply retry if hit. + if err != nil && strings.Contains(err.Error(), "insufficient storage resources") { + return err + } + require_NoError(t, err) + return nil + }) + // Now delete the R3 version. + require_NoError(t, js.DeleteStream("TEST6")) + // Now do R1 version and then we will scale up. + _, err = js.AddStream(&nats.StreamConfig{Name: "TEST6", Replicas: 1, MaxBytes: 1 * 1024 * 1024}) + require_NoError(t, err) + // Now make sure scale up works. + checkFor(t, 1*time.Second, 500*time.Millisecond, func() error { + _, err = js.UpdateStream(&nats.StreamConfig{Name: "TEST6", Replicas: 3, MaxBytes: 1 * 1024 * 1024}) + // Since the stream leader answers the stream add, and the meta leader determines stream not found, + // we could hit a race condition here. Simply retry if hit. + if err != nil && strings.Contains(err.Error(), "stream not found") { + return err + } + require_NoError(t, err) + return nil + }) + // Add in a few more streams to check reserved reporting in account info. + _, err = js.AddStream(&nats.StreamConfig{Name: "TEST7", Replicas: 1, MaxBytes: 2 * 1024 * 1024}) + require_NoError(t, err) + _, err = js.AddStream(&nats.StreamConfig{Name: "TEST8", Replicas: 1, MaxBytes: 256 * 1024, Storage: nats.MemoryStorage}) + require_NoError(t, err) + _, err = js.AddStream(&nats.StreamConfig{Name: "TEST9", Replicas: 3, MaxBytes: 22 * 1024, Storage: nats.MemoryStorage}) + require_NoError(t, err) + + // Now make sure we report reserved correctly. + // Do this direct to server since client does not support it yet. + var info JSApiAccountInfoResponse + resp, err := nc.Request("$JS.API.INFO", nil, time.Second) + require_NoError(t, err) + require_NoError(t, json.Unmarshal(resp.Data, &info)) + stats := info.JetStreamAccountStats + r1, r3 := stats.Tiers["R1"], stats.Tiers["R3"] + + require_Equal(t, r1.ReservedMemory, 256*1024) // TEST8 + require_Equal(t, r1.ReservedStore, 2*1024*1024) // TEST7 + require_Equal(t, r3.ReservedMemory, 22*1024) // TEST9 + require_Equal(t, r3.ReservedStore, 5*1024*1024) // TEST1-TEST6 +} + +func TestJetStreamJWTClusteredTiersR3StreamWithR1ConsumersAndAccounting(t *testing.T) { + sysKp, syspub := createKey(t) + sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) + newUser(t, sysKp) + + accKp, aExpPub := createKey(t) + accClaim := jwt.NewAccountClaims(aExpPub) + accClaim.Name = "acc" + accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ + DiskStorage: 1100, Consumer: 10, Streams: 1} + accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ + DiskStorage: 1100, Consumer: 1, Streams: 1} + accJwt := encodeClaim(t, accClaim, aExpPub) + accCreds := newUser(t, accKp) + tmlp := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + leaf { + listen: 127.0.0.1:-1 + } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + ` + fmt.Sprintf(` + operator: %s + system_account: %s + resolver = MEMORY + resolver_preload = { + %s : %s + %s : %s + } + `, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt) + + c := createJetStreamClusterWithTemplate(t, tmlp, "cluster", 3) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer(), nats.UserCredentials(accCreds)) + defer nc.Close() + + // Prevent 'nats: JetStream not enabled for account' when creating the first stream. + c.waitOnAccount(aExpPub) + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo.*"}, + Replicas: 3, + }) + require_NoError(t, err) + + // Now make sure we can add in 10 R1 consumers. + for i := 1; i <= 10; i++ { + _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ + Name: fmt.Sprintf("C-%d", i), + AckPolicy: nats.AckExplicitPolicy, + Replicas: 1, + }) + require_NoError(t, err) + } + + info, err := js.AccountInfo() + require_NoError(t, err) + + // Make sure we account for these properly. + r1 := info.Tiers["R1"] + r3 := info.Tiers["R3"] + + require_Equal(t, r1.Streams, 0) + require_Equal(t, r1.Consumers, 10) + require_Equal(t, r3.Streams, 1) + require_Equal(t, r3.Consumers, 0) +} + +func TestJetStreamJWTClusteredTieredR3StreamWithMixedReplicaConsumers(t *testing.T) { + sysKp, syspub := createKey(t) + sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) + newUser(t, sysKp) + + accKp, aExpPub := createKey(t) + accClaim := jwt.NewAccountClaims(aExpPub) + accClaim.Name = "acc" + // The R1 tier allows no stream and 2 consumers. + accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{DiskStorage: 1100, Consumer: 2, Streams: 0} + // The R3 tier allows 1 stream and 3 consumers. + accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{DiskStorage: 1100, Consumer: 3, Streams: 1} + accJwt := encodeClaim(t, accClaim, aExpPub) + accCreds := newUser(t, accKp) + + tmlp := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + leaf { + listen: 127.0.0.1:-1 + } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + ` + fmt.Sprintf(` + operator: %s + system_account: %s + resolver = MEMORY + resolver_preload = { + %s : %s + %s : %s + } + `, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt) + + c := createJetStreamClusterWithTemplate(t, tmlp, "cluster", 3) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer(), nats.UserCredentials(accCreds)) + defer nc.Close() + + // Prevent 'nats: JetStream not enabled for account' when creating the first stream. + c.waitOnAccount(aExpPub) + + // Add the one R3 stream the tier allows. + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo.*"}, + Replicas: 3, + }) + require_NoError(t, err) + + // Add 2 R3 consumers + for i := range 2 { + _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ + Name: fmt.Sprintf("R3-%d", i+1), + AckPolicy: nats.AckExplicitPolicy, + Replicas: 3, + }) + require_NoError(t, err) + } + + // Add 1 R1 consumer + _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ + Name: "R1-1", + AckPolicy: nats.AckExplicitPolicy, + Replicas: 1, + }) + require_NoError(t, err) + + // Add the 3rd R3 consumer (allowed by limits) + _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ + Name: "R3-3", + AckPolicy: nats.AckExplicitPolicy, + Replicas: 3, + }) + require_NoError(t, err) + + // Add the 2nd R1 consumer (allowed by limits) + _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ + Name: "R1-2", + AckPolicy: nats.AckExplicitPolicy, + Replicas: 1, + }) + require_NoError(t, err) + + // Both tiers are full now, so one more consumer of each tier must fail. + _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ + Name: "R3-4", + AckPolicy: nats.AckExplicitPolicy, + Replicas: 3, + }) + require_Error(t, err, NewJSMaximumConsumersLimitError()) + + _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ + Name: "R1-3", + AckPolicy: nats.AckExplicitPolicy, + Replicas: 1, + }) + require_Error(t, err, NewJSMaximumConsumersLimitError()) + + // Make sure each tier accounts for its own assets. + info, err := js.AccountInfo() + require_NoError(t, err) + + r1 := info.Tiers["R1"] + r3 := info.Tiers["R3"] + + require_Equal(t, r1.Streams, 0) + require_Equal(t, r1.Consumers, 2) + require_Equal(t, r3.Streams, 1) + require_Equal(t, r3.Consumers, 3) +} + +func TestJetStreamJWTClusterAccountNRG(t *testing.T) { + _, syspub := createKey(t) + sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) + + aExpKp, aExpPub := createKey(t) + accClaim := jwt.NewAccountClaims(aExpPub) + accClaim.Name = "acc" + accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{DiskStorage: 1100, Consumer: 10, Streams: 1} + accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{DiskStorage: 1100, Consumer: 1, Streams: 1} + accJwt := encodeClaim(t, accClaim, aExpPub) + accCreds := newUser(t, aExpKp) + + _, aExpPub2 := createKey(t) + accClaim2 := jwt.NewAccountClaims(aExpPub2) + accClaim2.Name = "another_acc" + accJwt2 := encodeClaim(t, accClaim2, aExpPub2) + + tmlp := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + leaf { + listen: 127.0.0.1:-1 + } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + ` + fmt.Sprintf(` + operator: %s + system_account: %s + resolver = MEMORY + resolver_preload = { + %s : %s + %s : %s + %s : %s + } + `, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt, aExpPub2, accJwt2) + + c := createJetStreamClusterWithTemplate(t, tmlp, "cluster", 3) + defer c.shutdown() + + nc, _ := jsClientConnect(t, c.randomServer(), nats.UserCredentials(accCreds)) + jsStreamCreate(t, nc, &StreamConfig{ + Name: "TEST", + Replicas: 3, + Storage: FileStorage, + }) + + // We'll try flipping the state a few times and then do some sanity + // checks to check that it took effect. + thirdAcc := jwt.ClusterTraffic(fmt.Sprintf("account:%s", aExpPub2)) + // TODO: Not currently testing thirdAcc because we haven't enabled this + // functionality yet. If/when we do enable, this test is ready just by + // uncommenting the third state below. + for _, state := range []jwt.ClusterTraffic{"system", "owner" /*, thirdAcc */} { + accClaim.ClusterTraffic = state + accJwt = encodeClaim(t, accClaim, aExpPub) + + for _, s := range c.servers { + // Update the account claim for our "third account". + acc, err := s.lookupAccount(aExpPub2) + require_NoError(t, err) + require_NoError(t, s.updateAccountWithClaimJWT(acc, accJwt2)) + + // Then update the account claim for the asset account. + acc, err = s.lookupAccount(aExpPub) + require_NoError(t, err) + require_NoError(t, s.updateAccountWithClaimJWT(acc, accJwt)) + + // Check that everything looks like it should. + require_True(t, acc != nil) + require_True(t, acc.js != nil) + switch state { + case "system": + require_Equal(t, acc.nrgAccount, _EMPTY_) + case "owner": + require_Equal(t, acc.nrgAccount, aExpPub) + case thirdAcc: + require_Equal(t, acc.nrgAccount, aExpPub2) + } + + // Now get a list of all of the Raft nodes that should + // have been updated by now. + s.rnMu.Lock() + raftNodes := make([]*raft, 0, len(s.raftNodes)) + for _, n := range s.raftNodes { + rg := n.(*raft) + if rg.accName != acc.Name { + continue + } + raftNodes = append(raftNodes, rg) + } + s.rnMu.Unlock() + + // The Raft nodes move to the new traffic account asynchronously, + // once statsz has propagated every peer's account-NRG capability, + // so wait for them to converge before checking. + checkFor(t, 2*time.Second, 200*time.Millisecond, func() error { + for _, rg := range raftNodes { + rg.Lock() + rgAcc := rg.t.Account() + rg.Unlock() + want := syspub + if state == "owner" { + want = aExpPub + } else if state == thirdAcc { + want = aExpPub2 + } + if rgAcc.Name != want { + return fmt.Errorf("group %q: traffic account is %q, want %q", rg.group, rgAcc.Name, want) + } + } + return nil + }) + + // Get the Raftz state also. + rz := s.Raftz(&RaftzOptions{AccountFilter: aExpPub}) + require_NotNil(t, rz) + rza := (*rz)[aExpPub] + require_NotNil(t, rza) + + // Check whether each of the Raft nodes reports being + // in-account or not. + for _, rg := range raftNodes { + rg.Lock() + rgAcc := rg.t.Account() + rg.Unlock() + switch state { + case "system": + require_Equal(t, rgAcc.Name, syspub) + require_Equal(t, rza[rg.group].SystemAcc, true) + require_Equal(t, rza[rg.group].TrafficAcc, syspub) + case "owner": + require_Equal(t, rgAcc.Name, aExpPub) + require_Equal(t, rza[rg.group].SystemAcc, false) + require_Equal(t, rza[rg.group].TrafficAcc, aExpPub) + case thirdAcc: + require_Equal(t, rgAcc.Name, aExpPub2) + require_Equal(t, rza[rg.group].SystemAcc, false) + require_Equal(t, rza[rg.group].TrafficAcc, aExpPub2) + } + } + } + } +} + +func TestJetStreamJWTClusterAccountNRGPersistsAfterRestart(t *testing.T) { + _, syspub := createKey(t) + sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) + + aExpKp, aExpPub := createKey(t) + accClaim := jwt.NewAccountClaims(aExpPub) + accClaim.Name = "acc" + accClaim.ClusterTraffic = jwt.ClusterTrafficOwner + accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{DiskStorage: 1100, Consumer: 10, Streams: 1} + accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{DiskStorage: 1100, Consumer: 1, Streams: 1} + accJwt := encodeClaim(t, accClaim, aExpPub) + accCreds := newUser(t, aExpKp) + + tmlp := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + leaf { + listen: 127.0.0.1:-1 + } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + ` + fmt.Sprintf(` + operator: %s + system_account: %s + resolver = MEMORY + resolver_preload = { + %s : %s + %s : %s + } + `, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt) + + c := createJetStreamClusterWithTemplate(t, tmlp, "cluster", 3) + defer c.shutdown() + + nc, _ := jsClientConnect(t, c.randomServer(), nats.UserCredentials(accCreds)) + + // Prevent 'nats: JetStream not enabled for account' when creating the first stream. + c.waitOnAccount(aExpPub) + + _, err := jsStreamCreate(t, nc, &StreamConfig{ + Name: "TEST", + Replicas: 3, + Storage: FileStorage, + }) + require_NoError(t, err) + + // The account had cluster traffic set to "owner" already. Restarting servers should remember this setting. + for _, s := range c.servers { + acc, err := s.lookupAccount(aExpPub) + require_NoError(t, err) + + // Check that everything looks like it should. + require_True(t, acc != nil) + require_True(t, acc.js != nil) + require_Equal(t, acc.nrgAccount, aExpPub) + + // Now get a list of all the Raft nodes that should have the correct cluster traffic set. + s.rnMu.Lock() + raftNodes := make([]*raft, 0, len(s.raftNodes)) + for _, n := range s.raftNodes { + rg := n.(*raft) + if rg.accName != acc.Name { + continue + } + raftNodes = append(raftNodes, rg) + } + s.rnMu.Unlock() + + // Get the Raftz state also. + rz := s.Raftz(&RaftzOptions{AccountFilter: aExpPub}) + require_NotNil(t, rz) + rza := (*rz)[aExpPub] + require_NotNil(t, rza) + + for _, rg := range raftNodes { + rg.Lock() + rgAcc := rg.t.Account() + rg.Unlock() + require_Equal(t, rgAcc.Name, aExpPub) + require_Equal(t, rza[rg.group].SystemAcc, false) + require_Equal(t, rza[rg.group].TrafficAcc, aExpPub) + } + } +} + +func TestJetStreamJWTUpdateWithPreExistingStream(t *testing.T) { + updateJwt := func(url string, creds string, pubKey string, jwt string) { + t.Helper() + c := natsConnect(t, url, nats.UserCredentials(creds)) + defer c.Close() + if msg, err := c.Request(fmt.Sprintf(accUpdateEventSubjNew, pubKey), []byte(jwt), time.Second); err != nil { + t.Fatal("error not expected in this test", err) + } else { + content := make(map[string]any) + if err := json.Unmarshal(msg.Data, &content); err != nil { + t.Fatalf("%v", err) + } else if _, ok := content["data"]; !ok { + t.Fatalf("did not get an ok response got: %v", content) + } + } + } + createUserCreds := func(akp nkeys.KeyPair) string { + uKp1, _ := nkeys.CreateUser() + uSeed1, _ := uKp1.Seed() + uclaim := newJWTTestUserClaims() + uclaim.Subject, _ = uKp1.PublicKey() + userJwt1, err := uclaim.Encode(akp) + require_NoError(t, err) + return genCredsFile(t, userJwt1, uSeed1) + } + // Create system account. + sysKp, _ := nkeys.CreateAccount() + sysPub, _ := sysKp.PublicKey() + sysUKp, _ := nkeys.CreateUser() + sysUSeed, _ := sysUKp.Seed() + uclaim := newJWTTestUserClaims() + uclaim.Subject, _ = sysUKp.PublicKey() + sysUserJwt, err := uclaim.Encode(sysKp) + require_NoError(t, err) + sysKp.Seed() + sysCreds := genCredsFile(t, sysUserJwt, sysUSeed) + // Create exporting account. + akpE, _ := nkeys.CreateAccount() + aPubE, _ := akpE.PublicKey() + claimE := jwt.NewAccountClaims(aPubE) + aJwtE, err := claimE.Encode(oKp) + require_NoError(t, err) + // Create importing account. + akpI, _ := nkeys.CreateAccount() + aPubI, _ := akpI.PublicKey() + claimI := jwt.NewAccountClaims(aPubI) + claimI.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: 1024 * 1024, DiskStorage: 1024 * 1024} + claimI.Imports.Add(&jwt.Import{ + Name: "import", + Subject: "foo", + Account: aPubE, + Type: jwt.Stream, + }) + aJwtI, err := claimI.Encode(oKp) + require_NoError(t, err) + // Create users. + userCredsE := createUserCreds(akpE) + userCredsI := createUserCreds(akpI) + // Start server and update JWTs. + dir := t.TempDir() + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:-1 + jetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: "%s"} + operator: %s + resolver: { + type: full + dir: '%s' + } + system_account: %s + `, dir, ojwt, dir, sysPub))) + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + updateJwt(s.ClientURL(), sysCreds, aPubI, aJwtI) + updateJwt(s.ClientURL(), sysCreds, aPubE, aJwtE) + + // Create stream on importing account before we restart. + nci, js := jsClientConnect(t, s, nats.UserCredentials(userCredsI)) + defer nci.Close() + _, err = js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo"}, + }) + require_NoError(t, err) + + // Restart server. + nci.Close() + s.Shutdown() + s.WaitForShutdown() + s, _ = RunServerWithConfig(conf) + defer s.Shutdown() + + // Reconnect and confirm stream is empty. + nci, js = jsClientConnect(t, s, nats.UserCredentials(userCredsI)) + defer nci.Close() + si, err := js.StreamInfo("TEST") + require_NoError(t, err) + require_Equal(t, si.State.Msgs, 0) + + // If an import/export gets added when the stream already existed on startup. + // We should still be able to route those messages. + claimE.Exports.Add(&jwt.Export{ + Name: "export", + Subject: "foo", + Type: jwt.Stream, + }) + aJwtE, err = claimE.Encode(oKp) + require_NoError(t, err) + updateJwt(s.ClientURL(), sysCreds, aPubE, aJwtE) + + // Connect to exporting account and publish a message that should be exported/imported. + nce := natsConnect(t, s.ClientURL(), nats.UserCredentials(userCredsE)) + defer nce.Close() + err = nce.Publish("foo", nil) + require_NoError(t, err) + + // Confirm the message was captured by the stream on the importing account. + checkFor(t, 2*time.Second, 500*time.Millisecond, func() error { + if si, err = js.StreamInfo("TEST"); err != nil { + return err + } else if si.State.Msgs != 1 { + return fmt.Errorf("expected 1 message in stream, got %d", si.State.Msgs) + } + return nil + }) +} + +func TestJetStreamAccountResolverNoFetchIfNotMember(t *testing.T) { + _, spub := createKey(t) + sysClaim := jwt.NewAccountClaims(spub) + sysClaim.Name = "SYS" + sysJwt := encodeClaim(t, sysClaim, spub) + kp, _ := nkeys.CreateAccount() + aPub, _ := kp.PublicKey() + + templ := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'} + + leaf { + listen: 127.0.0.1:-1 + } + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } +` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/" { + w.Write([]byte("ok")) + } else if strings.HasSuffix(r.URL.Path, spub) { + w.Write([]byte(sysJwt)) + } else { + // Simulate some time being spent, but doesn't respond. + time.Sleep(250 * time.Millisecond) + } + })) + defer ts.Close() + + c := createJetStreamClusterWithTemplateAndModHook(t, templ, "R3S", 3, + func(serverName, clusterName, storeDir, conf string) string { + return conf + fmt.Sprintf(` + operator: %s + system_account: %s + resolver: URL("%s")`, ojwt, spub, ts.URL) + }) + defer c.shutdown() + + s := c.leader() + js := s.getJetStream() + ci := &ClientInfo{Cluster: "R3S", Account: aPub} + cfg := &StreamConfig{Name: "TEST", Subjects: []string{"foo"}} + sa := &streamAssignment{Client: ci, Config: cfg} + start := time.Now() + // Simulate some meta operations where this server is not a member. + // The server should not fetch the account from the resolver. + for range 5 { + js.processStreamAssignment(sa) + } + require_LessThan(t, time.Since(start), 100*time.Millisecond) +} diff --git a/server/jetstream_leafnode_test.go b/server/jetstream_leafnode_test.go new file mode 100644 index 0000000..aba6d8f --- /dev/null +++ b/server/jetstream_leafnode_test.go @@ -0,0 +1,2170 @@ +// Copyright 2020-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !skip_js_tests + +package server + +import ( + "errors" + "fmt" + "os" + "regexp" + "runtime" + "strings" + "testing" + "time" + + jwt "github.com/nats-io/jwt/v2" + "github.com/nats-io/nats.go" + "github.com/nats-io/nkeys" +) + +func TestJetStreamLeafNodeUniqueServerNameCrossJSDomain(t *testing.T) { + name := "NOT-UNIQUE" + test := func(t *testing.T, s *Server, sIdExpected string, srvs ...*Server) { + ids := map[string]string{} + for _, srv := range srvs { + checkLeafNodeConnectedCount(t, srv, 2) + ids[srv.ID()] = srv.opts.JetStreamDomain + } + // ensure that an update for every server was received + sysNc := natsConnect(t, fmt.Sprintf("nats://admin:s3cr3t!@127.0.0.1:%d", s.opts.Port)) + defer sysNc.Close() + sub, err := sysNc.SubscribeSync(fmt.Sprintf(serverStatsSubj, "*")) + require_NoError(t, err) + for { + m, err := sub.NextMsg(time.Second) + require_NoError(t, err) + tk := strings.Split(m.Subject, ".") + if domain, ok := ids[tk[2]]; ok { + delete(ids, tk[2]) + require_Contains(t, string(m.Data), fmt.Sprintf(`"domain":"%s"`, domain)) + } + if len(ids) == 0 { + break + } + } + cnt := 0 + s.nodeToInfo.Range(func(key, value any) bool { + cnt++ + require_Equal(t, value.(nodeInfo).name, name) + require_Equal(t, value.(nodeInfo).id, sIdExpected) + return true + }) + require_Equal(t, cnt, 1) + } + tmplA := ` + listen: -1 + server_name: %s + jetstream { + max_mem_store: 256MB, + max_file_store: 2GB, + store_dir: '%s', + domain: hub + } + accounts { + JSY { users = [ { user: "y", pass: "p" } ]; jetstream: true } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } + leaf { + port: -1 + } + ` + tmplL := ` + listen: -1 + server_name: %s + jetstream { + max_mem_store: 256MB, + max_file_store: 2GB, + store_dir: '%s', + domain: %s + } + accounts { + JSY { users = [ { user: "y", pass: "p" } ]; jetstream: true } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } + leaf { + remotes [ + { urls: [ %s ], account: "JSY" } + { urls: [ %s ], account: "$SYS" } + ] + } + ` + t.Run("same-domain", func(t *testing.T) { + confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, name, t.TempDir()))) + sA, oA := RunServerWithConfig(confA) + defer sA.Shutdown() + // using same domain as sA + confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, name, t.TempDir(), "hub", + fmt.Sprintf("nats://y:p@127.0.0.1:%d", oA.LeafNode.Port), + fmt.Sprintf("nats://admin:s3cr3t!@127.0.0.1:%d", oA.LeafNode.Port)))) + sL, _ := RunServerWithConfig(confL) + defer sL.Shutdown() + // as server name uniqueness is violates, sL.ID() is the expected value + test(t, sA, sL.ID(), sA, sL) + }) + t.Run("different-domain", func(t *testing.T) { + confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, name, t.TempDir()))) + sA, oA := RunServerWithConfig(confA) + defer sA.Shutdown() + // using different domain as sA + confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, name, t.TempDir(), "spoke", + fmt.Sprintf("nats://y:p@127.0.0.1:%d", oA.LeafNode.Port), + fmt.Sprintf("nats://admin:s3cr3t!@127.0.0.1:%d", oA.LeafNode.Port)))) + sL, _ := RunServerWithConfig(confL) + defer sL.Shutdown() + checkLeafNodeConnectedCount(t, sL, 2) + checkLeafNodeConnectedCount(t, sA, 2) + // ensure sA contains only sA.ID + test(t, sA, sA.ID(), sA, sL) + }) +} + +func TestJetStreamLeafNodeJwtPermsAndJSDomains(t *testing.T) { + createAcc := func(js bool) (string, string, nkeys.KeyPair) { + kp, _ := nkeys.CreateAccount() + aPub, _ := kp.PublicKey() + claim := jwt.NewAccountClaims(aPub) + if js { + claim.Limits.JetStreamLimits = jwt.JetStreamLimits{ + MemoryStorage: 1024 * 1024, + DiskStorage: 1024 * 1024, + Streams: 1, Consumer: 2} + } + aJwt, err := claim.Encode(oKp) + require_NoError(t, err) + return aPub, aJwt, kp + } + sysPub, sysJwt, sysKp := createAcc(false) + accPub, accJwt, accKp := createAcc(true) + noExpiration := time.Now().Add(time.Hour) + // create user for acc to be used in leaf node. + lnCreds := createUserWithLimit(t, accKp, noExpiration, func(j *jwt.UserPermissionLimits) { + j.Sub.Deny.Add("subdeny") + j.Pub.Deny.Add("pubdeny") + }) + unlimitedCreds := createUserWithLimit(t, accKp, noExpiration, nil) + + sysCreds := createUserWithLimit(t, sysKp, noExpiration, nil) + + tmplA := ` +operator: %s +system_account: %s +resolver: MEMORY +resolver_preload: { + %s: %s + %s: %s +} +listen: 127.0.0.1:-1 +leafnodes: { + listen: 127.0.0.1:-1 +} +jetstream :{ + domain: "cluster" + store_dir: '%s' + max_mem: 100Mb + max_file: 100Mb +} +` + + tmplL := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account = SYS +jetstream: { + domain: ln1 + store_dir: '%s' + max_mem: 50Mb + max_file: 50Mb +} +leafnodes:{ + remotes:[{ url:nats://127.0.0.1:%d, account: A, credentials: '%s'}, + { url:nats://127.0.0.1:%d, account: SYS, credentials: '%s'}] +} +` + + confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, ojwt, sysPub, + sysPub, sysJwt, accPub, accJwt, + t.TempDir()))) + sA, _ := RunServerWithConfig(confA) + defer sA.Shutdown() + + confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, t.TempDir(), + sA.opts.LeafNode.Port, lnCreds, sA.opts.LeafNode.Port, sysCreds))) + sL, _ := RunServerWithConfig(confL) + defer sL.Shutdown() + + checkLeafNodeConnectedCount(t, sA, 2) + checkLeafNodeConnectedCount(t, sL, 2) + + ncA := natsConnect(t, sA.ClientURL(), nats.UserCredentials(unlimitedCreds)) + defer ncA.Close() + + ncL := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sL.opts.Port)) + defer ncL.Close() + + test := func(subject string, cSub, cPub *nats.Conn, remoteServerForSub *Server, accName string, pass bool) { + t.Helper() + sub, err := cSub.SubscribeSync(subject) + require_NoError(t, err) + require_NoError(t, cSub.Flush()) + // ensure the subscription made it across, or if not sent due to sub deny, make sure it could have made it. + if remoteServerForSub == nil { + time.Sleep(200 * time.Millisecond) + } else { + checkSubInterest(t, remoteServerForSub, accName, subject, time.Second) + } + require_NoError(t, cPub.Publish(subject, []byte("hello world"))) + require_NoError(t, cPub.Flush()) + m, err := sub.NextMsg(500 * time.Millisecond) + if pass { + require_NoError(t, err) + require_True(t, m.Subject == subject) + require_Equal(t, string(m.Data), "hello world") + } else { + require_True(t, err == nats.ErrTimeout) + } + } + + t.Run("sub-on-ln-pass", func(t *testing.T) { + test("sub", ncL, ncA, sA, accPub, true) + }) + t.Run("sub-on-ln-fail", func(t *testing.T) { + test("subdeny", ncL, ncA, nil, "", false) + }) + t.Run("pub-on-ln-pass", func(t *testing.T) { + test("pub", ncA, ncL, sL, "A", true) + }) + t.Run("pub-on-ln-fail", func(t *testing.T) { + test("pubdeny", ncA, ncL, nil, "A", false) + }) +} + +func TestJetStreamLeafNodeClusterExtensionWithSystemAccount(t *testing.T) { + /* + Topologies tested here + same == true + A <-> B + ^ |\ + | \ + | proxy + | \ + LA <-> LB + + same == false + A <-> B + ^ ^ + | | + | proxy + | | + LA <-> LB + + The proxy is turned on later, such that the system account connection can be started later, in a controlled way + This explicitly tests the system state before and after this happens. + */ + + tmplA := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account: SYS +leafnodes: { + listen: 127.0.0.1:-1 + no_advertise: true + authorization: { + timeout: 0.5 + } +} +jetstream :{ + domain: "cluster" + store_dir: '%s' + max_mem: 100Mb + max_file: 100Mb +} +server_name: A +cluster: { + name: clust1 + listen: 127.0.0.1:20104 + routes=[nats-route://127.0.0.1:20105] + no_advertise: true +} +` + + tmplB := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account: SYS +leafnodes: { + listen: 127.0.0.1:-1 + no_advertise: true + authorization: { + timeout: 0.5 + } +} +jetstream: { + domain: "cluster" + store_dir: '%s' + max_mem: 100Mb + max_file: 100Mb +} +server_name: B +cluster: { + name: clust1 + listen: 127.0.0.1:20105 + routes=[nats-route://127.0.0.1:20104] + no_advertise: true +} +` + + tmplLA := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account = SYS +jetstream: { + domain: "cluster" + store_dir: '%s' + max_mem: 50Mb + max_file: 50Mb + %s +} +server_name: LA +cluster: { + name: clustL + listen: 127.0.0.1:20106 + routes=[nats-route://127.0.0.1:20107] + no_advertise: true +} +leafnodes:{ + no_advertise: true + remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A}, + {url:nats://s1:s1@127.0.0.1:%d, account: SYS}] +} +` + + tmplLB := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account = SYS +jetstream: { + domain: "cluster" + store_dir: '%s' + max_mem: 50Mb + max_file: 50Mb + %s +} +server_name: LB +cluster: { + name: clustL + listen: 127.0.0.1:20107 + routes=[nats-route://127.0.0.1:20106] + no_advertise: true +} +leafnodes:{ + no_advertise: true + remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A}, + {url:nats://s1:s1@127.0.0.1:%d, account: SYS}] +} +` + + for _, testCase := range []struct { + // which topology to pick + same bool + // If leaf server should be operational and form a Js cluster prior to joining. + // In this setup this would be an error as you give the wrong hint. + // But this should work itself out regardless + leafFunctionPreJoin bool + }{ + {true, true}, + {true, false}, + {false, true}, + {false, false}} { + t.Run(fmt.Sprintf("%t-%t", testCase.same, testCase.leafFunctionPreJoin), func(t *testing.T) { + sd1 := t.TempDir() + confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, sd1))) + sA, _ := RunServerWithConfig(confA) + defer sA.Shutdown() + + sd2 := t.TempDir() + confB := createConfFile(t, []byte(fmt.Sprintf(tmplB, sd2))) + sB, _ := RunServerWithConfig(confB) + defer sB.Shutdown() + + checkClusterFormed(t, sA, sB) + + c := cluster{t: t, servers: []*Server{sA, sB}} + c.waitOnLeader() + + // starting this will allow the second remote in tmplL to successfully connect. + port := sB.opts.LeafNode.Port + if testCase.same { + port = sA.opts.LeafNode.Port + } + p := &proxyAcceptDetectFailureLate{acceptPort: port} + defer p.close() + lPort := p.runEx(t, true) + + hint := "" + if testCase.leafFunctionPreJoin { + hint = fmt.Sprintf("extension_hint: %s", strings.ToUpper(jsNoExtend)) + } + + sd3 := t.TempDir() + // deliberately pick server sA and proxy + confLA := createConfFile(t, []byte(fmt.Sprintf(tmplLA, sd3, hint, sA.opts.LeafNode.Port, lPort))) + sLA, _ := RunServerWithConfig(confLA) + defer sLA.Shutdown() + + sd4 := t.TempDir() + // deliberately pick server sA and proxy + confLB := createConfFile(t, []byte(fmt.Sprintf(tmplLB, sd4, hint, sA.opts.LeafNode.Port, lPort))) + sLB, _ := RunServerWithConfig(confLB) + defer sLB.Shutdown() + + checkClusterFormed(t, sLA, sLB) + + strmCfg := func(name, placementCluster string) *nats.StreamConfig { + if placementCluster == "" { + return &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}} + } + return &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}, + Placement: &nats.Placement{Cluster: placementCluster}} + } + // Only after the system account is fully connected can streams be placed anywhere. + testJSFunctions := func(pass bool) { + ncA := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sA.opts.Port)) + defer ncA.Close() + jsA, err := ncA.JetStream() + require_NoError(t, err) + _, err = jsA.AddStream(strmCfg(fmt.Sprintf("fooA1-%t", pass), "")) + require_NoError(t, err) + _, err = jsA.AddStream(strmCfg(fmt.Sprintf("fooA2-%t", pass), "clust1")) + require_NoError(t, err) + _, err = jsA.AddStream(strmCfg(fmt.Sprintf("fooA3-%t", pass), "clustL")) + if pass { + require_NoError(t, err) + } else { + require_Error(t, err) + require_Contains(t, err.Error(), "no suitable peers for placement") + } + ncL := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sLA.opts.Port)) + defer ncL.Close() + jsL, err := ncL.JetStream() + require_NoError(t, err) + _, err = jsL.AddStream(strmCfg(fmt.Sprintf("fooL1-%t", pass), "")) + require_NoError(t, err) + _, err = jsL.AddStream(strmCfg(fmt.Sprintf("fooL2-%t", pass), "clustL")) + require_NoError(t, err) + _, err = jsL.AddStream(strmCfg(fmt.Sprintf("fooL3-%t", pass), "clust1")) + if pass { + require_NoError(t, err) + } else { + require_Error(t, err) + require_Contains(t, err.Error(), "no suitable peers for placement") + } + } + clusterLnCnt := func(expected int) error { + cnt := 0 + for _, s := range c.servers { + cnt += s.NumLeafNodes() + } + if cnt == expected { + return nil + } + return fmt.Errorf("not enought leaf node connections, got %d needed %d", cnt, expected) + } + + // Even though there are two remotes defined in tmplL, only one will be able to connect. + checkFor(t, 10*time.Second, time.Second/4, func() error { return clusterLnCnt(2) }) + checkLeafNodeConnectedCount(t, sLA, 1) + checkLeafNodeConnectedCount(t, sLB, 1) + c.waitOnPeerCount(2) + + if testCase.leafFunctionPreJoin { + cl := cluster{t: t, servers: []*Server{sLA, sLB}} + cl.waitOnLeader() + cl.waitOnPeerCount(2) + testJSFunctions(false) + } else { + // In cases where the leaf nodes have to wait for the system account to connect, + // JetStream should not be operational during that time + ncA := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sLA.opts.Port)) + defer ncA.Close() + jsA, err := ncA.JetStream() + require_NoError(t, err) + _, err = jsA.AddStream(strmCfg("fail-false", "")) + require_Error(t, err) + } + // Starting the proxy will connect the system accounts. + // After they are connected the clusters are merged. + // Once this happened, all streams in test can be placed anywhere in the cluster. + // Before that only the cluster the client is connected to can be used for placement + p.start() + + // Even though there are two remotes defined in tmplL, only one will be able to connect. + checkFor(t, 10*time.Second, time.Second/4, func() error { return clusterLnCnt(4) }) + checkLeafNodeConnectedCount(t, sLA, 2) + checkLeafNodeConnectedCount(t, sLB, 2) + + // The leader will reside in the main cluster only + c.waitOnPeerCount(4) + testJSFunctions(true) + }) + } +} + +func TestJetStreamLeafNodeClusterMixedModeExtensionWithSystemAccount(t *testing.T) { + /* Topology used in this test: + CLUSTER(A <-> B <-> C (NO JS)) + ^ + | + LA + */ + + // once every server is up, we expect these peers to be part of the JetStream meta cluster + expectedJetStreamPeers := map[string]struct{}{ + "A": {}, + "B": {}, + "LA": {}, + } + + tmplA := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account: SYS +leafnodes: { + listen: 127.0.0.1:-1 + no_advertise: true + authorization: { + timeout: 0.5 + } +} +jetstream: { %s store_dir: '%s'; max_mem: 50Mb, max_file: 50Mb } +server_name: A +cluster: { + name: clust1 + listen: 127.0.0.1:20114 + routes=[nats-route://127.0.0.1:20115,nats-route://127.0.0.1:20116] + no_advertise: true +} +` + + tmplB := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account: SYS +leafnodes: { + listen: 127.0.0.1:-1 + no_advertise: true + authorization: { + timeout: 0.5 + } +} +jetstream: { %s store_dir: '%s'; max_mem: 50Mb, max_file: 50Mb } +server_name: B +cluster: { + name: clust1 + listen: 127.0.0.1:20115 + routes=[nats-route://127.0.0.1:20114,nats-route://127.0.0.1:20116] + no_advertise: true +} +` + + tmplC := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account: SYS +leafnodes: { + listen: 127.0.0.1:-1 + no_advertise: true + authorization: { + timeout: 0.5 + } +} +jetstream: { + enabled: false + %s +} +server_name: C +cluster: { + name: clust1 + listen: 127.0.0.1:20116 + routes=[nats-route://127.0.0.1:20114,nats-route://127.0.0.1:20115] + no_advertise: true +} +` + + tmplLA := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account = SYS +# the extension hint is to simplify this test. without it present we would need a cluster of size 2 +jetstream: { %s store_dir: '%s'; max_mem: 50Mb, max_file: 50Mb, extension_hint: will_extend } +server_name: LA +leafnodes:{ + no_advertise: true + remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A}, + {url:nats://s1:s1@127.0.0.1:%d, account: SYS}] +} +# add the cluster here so we can test placement +cluster: { name: clustL } +` + for _, withDomain := range []bool{true, false} { + t.Run(fmt.Sprintf("with-domain:%t", withDomain), func(t *testing.T) { + var jsDisabledDomainString string + var jsEnabledDomainString string + if withDomain { + jsEnabledDomainString = `domain: "domain", ` + jsDisabledDomainString = `domain: "domain"` + } else { + // in case no domain name is set, fall back to the extension hint. + // since JS is disabled, the value of this does not clash with other uses. + jsDisabledDomainString = "extension_hint: will_extend" + } + + sd1 := t.TempDir() + confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, jsEnabledDomainString, sd1))) + sA, _ := RunServerWithConfig(confA) + defer sA.Shutdown() + + sd2 := t.TempDir() + confB := createConfFile(t, []byte(fmt.Sprintf(tmplB, jsEnabledDomainString, sd2))) + sB, _ := RunServerWithConfig(confB) + defer sB.Shutdown() + + confC := createConfFile(t, []byte(fmt.Sprintf(tmplC, jsDisabledDomainString))) + sC, _ := RunServerWithConfig(confC) + defer sC.Shutdown() + + checkClusterFormed(t, sA, sB, sC) + c := cluster{t: t, servers: []*Server{sA, sB, sC}} + c.waitOnPeerCount(2) + + sd3 := t.TempDir() + // deliberately pick server sC (no JS) to connect to + confLA := createConfFile(t, []byte(fmt.Sprintf(tmplLA, jsEnabledDomainString, sd3, sC.opts.LeafNode.Port, sC.opts.LeafNode.Port))) + sLA, _ := RunServerWithConfig(confLA) + defer sLA.Shutdown() + + checkLeafNodeConnectedCount(t, sC, 2) + checkLeafNodeConnectedCount(t, sLA, 2) + c.waitOnPeerCount(3) + peers := c.leader().JetStreamClusterPeers() + for _, peer := range peers { + if _, ok := expectedJetStreamPeers[peer]; !ok { + t.Fatalf("Found unexpected peer %q", peer) + } + } + + // helper to create stream config with uniqe name and subject + cnt := 0 + strmCfg := func(placementCluster string) *nats.StreamConfig { + name := fmt.Sprintf("s-%d", cnt) + cnt++ + if placementCluster == "" { + return &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}} + } + return &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}, + Placement: &nats.Placement{Cluster: placementCluster}} + } + + test := func(port int, expectedDefPlacement string) { + ncA := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", port)) + defer ncA.Close() + jsA, err := ncA.JetStream() + require_NoError(t, err) + si, err := jsA.AddStream(strmCfg("")) + require_NoError(t, err) + require_Contains(t, si.Cluster.Name, expectedDefPlacement) + si, err = jsA.AddStream(strmCfg("clust1")) + require_NoError(t, err) + require_Contains(t, si.Cluster.Name, "clust1") + si, err = jsA.AddStream(strmCfg("clustL")) + require_NoError(t, err) + require_Contains(t, si.Cluster.Name, "clustL") + } + + test(sA.opts.Port, "clust1") + test(sB.opts.Port, "clust1") + test(sC.opts.Port, "clust1") + test(sLA.opts.Port, "clustL") + }) + } +} + +func TestJetStreamLeafNodeCredsDenies(t *testing.T) { + tmplL := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account = SYS +jetstream: { + domain: "cluster" + store_dir: '%s' + max_mem: 50Mb + max_file: 50Mb +} +leafnodes:{ + remotes:[{url:nats://a1:a1@127.0.0.1:20125, account: A, credentials: '%s' }, + {url:nats://s1:s1@127.0.0.1:20125, account: SYS, credentials: '%s', deny_imports: foo, deny_exports: bar}] +} +` + akp, err := nkeys.CreateAccount() + require_NoError(t, err) + creds := createUserWithLimit(t, akp, time.Time{}, func(pl *jwt.UserPermissionLimits) { + pl.Pub.Deny.Add(jsAllAPI) + pl.Sub.Deny.Add(jsAllAPI) + }) + + sd := t.TempDir() + + confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, sd, creds, creds))) + opts := LoadConfig(confL) + sL, err := NewServer(opts) + require_NoError(t, err) + + l := captureNoticeLogger{} + sL.SetLogger(&l, false, false) + + go sL.Start() + defer sL.Shutdown() + + // wait till the notices got printed +UNTIL_READY: + for { + <-time.After(50 * time.Millisecond) + l.Lock() + for _, n := range l.notices { + if strings.Contains(n, "Server is ready") { + l.Unlock() + break UNTIL_READY + } + } + l.Unlock() + } + + l.Lock() + cnt := 0 + for _, n := range l.notices { + if strings.Contains(n, "LeafNode Remote for Account A uses credentials file") || + strings.Contains(n, "LeafNode Remote for System Account uses") || + strings.Contains(n, "Remote for System Account uses restricted export permissions") || + strings.Contains(n, "Remote for System Account uses restricted import permissions") { + cnt++ + } + } + l.Unlock() + require_True(t, cnt == 4) +} + +func TestJetStreamLeafNodeDefaultDomainCfg(t *testing.T) { + tmplHub := ` +listen: 127.0.0.1:%d +accounts :{ + A:{ jetstream: %s, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account: SYS +jetstream : %s +server_name: HUB +leafnodes: { + listen: 127.0.0.1:%d +} +%s +` + + tmplL := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account: SYS +jetstream: { domain: "%s", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } +server_name: LEAF +leafnodes: { + remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},%s] +} +%s +` + + test := func(domain string, sysShared bool) { + confHub := createConfFile(t, []byte(fmt.Sprintf(tmplHub, -1, "disabled", "disabled", -1, ""))) + sHub, _ := RunServerWithConfig(confHub) + defer sHub.Shutdown() + + noDomainFix := "" + if domain == _EMPTY_ { + noDomainFix = `default_js_domain:{A:""}` + } + + sys := "" + if sysShared { + sys = fmt.Sprintf(`{url:nats://s1:s1@127.0.0.1:%d, account: SYS}`, sHub.opts.LeafNode.Port) + } + + sdLeaf := t.TempDir() + confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, domain, sdLeaf, sHub.opts.LeafNode.Port, sys, noDomainFix))) + sLeaf, _ := RunServerWithConfig(confL) + defer sLeaf.Shutdown() + + lnCnt := 1 + if sysShared { + lnCnt++ + } + + checkLeafNodeConnectedCount(t, sHub, lnCnt) + checkLeafNodeConnectedCount(t, sLeaf, lnCnt) + + ncA := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sHub.opts.Port)) + defer ncA.Close() + jsA, err := ncA.JetStream() + require_NoError(t, err) + + _, err = jsA.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, Subjects: []string{"foo"}}) + require_True(t, err == nats.ErrNoResponders) + + // Add in default domain and restart server + require_NoError(t, os.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub, + sHub.opts.Port, + "disabled", + "disabled", + sHub.opts.LeafNode.Port, + fmt.Sprintf(`default_js_domain: {A:"%s"}`, domain))), 0664)) + + sHub.Shutdown() + sHub.WaitForShutdown() + checkLeafNodeConnectedCount(t, sLeaf, 0) + sHubUpd1, _ := RunServerWithConfig(confHub) + defer sHubUpd1.Shutdown() + + checkLeafNodeConnectedCount(t, sHubUpd1, lnCnt) + checkLeafNodeConnectedCount(t, sLeaf, lnCnt) + + _, err = jsA.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, Subjects: []string{"foo"}}) + require_NoError(t, err) + + // Enable jetstream in hub. + sdHub := t.TempDir() + jsEnabled := fmt.Sprintf(`{ domain: "%s", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb }`, domain, sdHub) + require_NoError(t, os.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub, + sHubUpd1.opts.Port, + "disabled", + jsEnabled, + sHubUpd1.opts.LeafNode.Port, + fmt.Sprintf(`default_js_domain: {A:"%s"}`, domain))), 0664)) + + sHubUpd1.Shutdown() + sHubUpd1.WaitForShutdown() + checkLeafNodeConnectedCount(t, sLeaf, 0) + sHubUpd2, _ := RunServerWithConfig(confHub) + defer sHubUpd2.Shutdown() + + checkLeafNodeConnectedCount(t, sHubUpd2, lnCnt) + checkLeafNodeConnectedCount(t, sLeaf, lnCnt) + + _, err = jsA.AddStream(&nats.StreamConfig{Name: "bar", Replicas: 1, Subjects: []string{"bar"}}) + require_NoError(t, err) + + // Enable jetstream in account A of hub + // This is a mis config, as you can't have it both ways, local jetstream but default to another one + require_NoError(t, os.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub, + sHubUpd2.opts.Port, + "enabled", + jsEnabled, + sHubUpd2.opts.LeafNode.Port, + fmt.Sprintf(`default_js_domain: {A:"%s"}`, domain))), 0664)) + + if domain != _EMPTY_ { + // in case no domain name exists there are no additional guard rails, hence no error + // It is the users responsibility to get this edge case right + sHubUpd2.Shutdown() + sHubUpd2.WaitForShutdown() + checkLeafNodeConnectedCount(t, sLeaf, 0) + sHubUpd3, err := NewServer(LoadConfig(confHub)) + sHubUpd3.Shutdown() + + require_Error(t, err) + require_Contains(t, err.Error(), `default_js_domain contains account name "A" with enabled JetStream`) + } + } + + t.Run("with-domain-sys", func(t *testing.T) { + test("domain", true) + }) + t.Run("with-domain-nosys", func(t *testing.T) { + test("domain", false) + }) + t.Run("no-domain", func(t *testing.T) { + test("", true) + }) + t.Run("no-domain", func(t *testing.T) { + test("", false) + }) +} + +func TestJetStreamLeafNodeDefaultDomainJwtExplicit(t *testing.T) { + tmplHub := ` +listen: 127.0.0.1:%d +operator: %s +system_account: %s +resolver: MEM +resolver_preload: { + %s:%s + %s:%s +} +jetstream : disabled +server_name: HUB +leafnodes: { + listen: 127.0.0.1:%d +} +%s +` + + tmplL := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account: SYS +jetstream: { domain: "%s", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } +server_name: LEAF +leafnodes: { + remotes:[{url:nats://127.0.0.1:%d, account: A, credentials: '%s'}, + {url:nats://127.0.0.1:%d, account: SYS, credentials: '%s'}] +} +%s +` + + test := func(domain string) { + noDomainFix := "" + if domain == _EMPTY_ { + noDomainFix = `default_js_domain:{A:""}` + } + + sysKp, syspub := createKey(t) + sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) + sysCreds := newUser(t, sysKp) + + aKp, aPub := createKey(t) + aClaim := jwt.NewAccountClaims(aPub) + aJwt := encodeClaim(t, aClaim, aPub) + aCreds := newUser(t, aKp) + + confHub := createConfFile(t, []byte(fmt.Sprintf(tmplHub, -1, ojwt, syspub, syspub, sysJwt, aPub, aJwt, -1, ""))) + sHub, _ := RunServerWithConfig(confHub) + defer sHub.Shutdown() + + sdLeaf := t.TempDir() + confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, + domain, + sdLeaf, + sHub.opts.LeafNode.Port, + aCreds, + sHub.opts.LeafNode.Port, + sysCreds, + noDomainFix))) + sLeaf, _ := RunServerWithConfig(confL) + defer sLeaf.Shutdown() + + checkLeafNodeConnectedCount(t, sHub, 2) + checkLeafNodeConnectedCount(t, sLeaf, 2) + + ncA := natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", sHub.opts.Port), createUserCreds(t, nil, aKp)) + defer ncA.Close() + jsA, err := ncA.JetStream() + require_NoError(t, err) + + _, err = jsA.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, Subjects: []string{"foo"}}) + require_True(t, err == nats.ErrNoResponders) + + // Add in default domain and restart server + require_NoError(t, os.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub, + sHub.opts.Port, ojwt, syspub, syspub, sysJwt, aPub, aJwt, sHub.opts.LeafNode.Port, + fmt.Sprintf(`default_js_domain: {%s:"%s"}`, aPub, domain))), 0664)) + + sHub.Shutdown() + sHub.WaitForShutdown() + checkLeafNodeConnectedCount(t, sLeaf, 0) + sHubUpd1, _ := RunServerWithConfig(confHub) + defer sHubUpd1.Shutdown() + + checkLeafNodeConnectedCount(t, sHubUpd1, 2) + checkLeafNodeConnectedCount(t, sLeaf, 2) + + _, err = jsA.AddStream(&nats.StreamConfig{Name: "bar", Replicas: 1, Subjects: []string{"bar"}}) + require_NoError(t, err) + } + t.Run("with-domain", func(t *testing.T) { + test("domain") + }) + t.Run("no-domain", func(t *testing.T) { + test("") + }) +} + +func TestJetStreamLeafNodeDefaultDomainClusterBothEnds(t *testing.T) { + // test to ensure that default domain functions when both ends of the leaf node connection are clusters + tmplHub1 := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enabled, users:[ {user:a1,password:a1}]}, + B:{ jetstream: enabled, users:[ {user:b1,password:b1}]} +} +jetstream : { domain: "DHUB", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } +server_name: HUB1 +cluster: { + name: HUB + listen: 127.0.0.1:20134 + routes=[nats-route://127.0.0.1:20135] +} +leafnodes: { + listen:127.0.0.1:-1 +} +` + + tmplHub2 := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enabled, users:[ {user:a1,password:a1}]}, + B:{ jetstream: enabled, users:[ {user:b1,password:b1}]} +} +jetstream : { domain: "DHUB", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } +server_name: HUB2 +cluster: { + name: HUB + listen: 127.0.0.1:20135 + routes=[nats-route://127.0.0.1:20134] +} +leafnodes: { + listen:127.0.0.1:-1 +} +` + + tmplL1 := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enabled, users:[ {user:a1,password:a1}]}, + B:{ jetstream: disabled, users:[ {user:b1,password:b1}]} +} +jetstream: { domain: "DLEAF", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } +server_name: LEAF1 +cluster: { + name: LEAF + listen: 127.0.0.1:20136 + routes=[nats-route://127.0.0.1:20137] +} +leafnodes: { + remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},{url:nats://b1:b1@127.0.0.1:%d, account: B}] +} +default_js_domain: {B:"DHUB"} +` + + tmplL2 := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enabled, users:[ {user:a1,password:a1}]}, + B:{ jetstream: disabled, users:[ {user:b1,password:b1}]} +} +jetstream: { domain: "DLEAF", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } +server_name: LEAF2 +cluster: { + name: LEAF + listen: 127.0.0.1:20137 + routes=[nats-route://127.0.0.1:20136] +} +leafnodes: { + remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},{url:nats://b1:b1@127.0.0.1:%d, account: B}] +} +default_js_domain: {B:"DHUB"} +` + + sd1 := t.TempDir() + confHub1 := createConfFile(t, []byte(fmt.Sprintf(tmplHub1, sd1))) + sHub1, _ := RunServerWithConfig(confHub1) + defer sHub1.Shutdown() + + sd2 := t.TempDir() + confHub2 := createConfFile(t, []byte(fmt.Sprintf(tmplHub2, sd2))) + sHub2, _ := RunServerWithConfig(confHub2) + defer sHub2.Shutdown() + + checkClusterFormed(t, sHub1, sHub2) + c1 := cluster{t: t, servers: []*Server{sHub1, sHub2}} + c1.waitOnPeerCount(2) + + sd3 := t.TempDir() + confLeaf1 := createConfFile(t, []byte(fmt.Sprintf(tmplL1, sd3, sHub1.getOpts().LeafNode.Port, sHub1.getOpts().LeafNode.Port))) + sLeaf1, _ := RunServerWithConfig(confLeaf1) + defer sLeaf1.Shutdown() + + sd4 := t.TempDir() + confLeaf2 := createConfFile(t, []byte(fmt.Sprintf(tmplL2, sd4, sHub1.getOpts().LeafNode.Port, sHub1.getOpts().LeafNode.Port))) + sLeaf2, _ := RunServerWithConfig(confLeaf2) + defer sLeaf2.Shutdown() + + checkClusterFormed(t, sLeaf1, sLeaf2) + c2 := cluster{t: t, servers: []*Server{sLeaf1, sLeaf2}} + c2.waitOnPeerCount(2) + + checkLeafNodeConnectedCount(t, sHub1, 4) + checkLeafNodeConnectedCount(t, sLeaf1, 2) + checkLeafNodeConnectedCount(t, sLeaf2, 2) + + ncB := natsConnect(t, fmt.Sprintf("nats://b1:b1@127.0.0.1:%d", sLeaf1.getOpts().Port)) + defer ncB.Close() + jsB1, err := ncB.JetStream() + require_NoError(t, err) + si, err := jsB1.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, Subjects: []string{"foo"}}) + require_NoError(t, err) + require_Equal(t, si.Cluster.Name, "HUB") + + jsB2, err := ncB.JetStream(nats.Domain("DHUB")) + require_NoError(t, err) + si, err = jsB2.AddStream(&nats.StreamConfig{Name: "bar", Replicas: 1, Subjects: []string{"bar"}}) + require_NoError(t, err) + require_Equal(t, si.Cluster.Name, "HUB") +} + +func TestJetStreamLeafNodeSvcImportExportCycle(t *testing.T) { + accounts := ` + accounts { + SYS: { + users: [{user: admin, password: admin}] + } + LEAF_USER: { + users: [{user: leaf_user, password: leaf_user}] + imports: [ + {service: {account: LEAF_INGRESS, subject: "foo"}} + {service: {account: LEAF_INGRESS, subject: "_INBOX.>"}} + {service: {account: LEAF_INGRESS, subject: "$JS.leaf.API.>"}, to: "JS.leaf_ingress@leaf.API.>" } + ] + jetstream: enabled + } + LEAF_INGRESS: { + users: [{user: leaf_ingress, password: leaf_ingress}] + exports: [ + {service: "foo", accounts: [LEAF_USER]} + {service: "_INBOX.>", accounts: [LEAF_USER]} + {service: "$JS.leaf.API.>", response_type: "stream", accounts: [LEAF_USER]} + ] + imports: [ + ] + jetstream: enabled + } + } + system_account: SYS + ` + + hconf := createConfFile(t, []byte(fmt.Sprintf(` + %s + listen: "127.0.0.1:-1" + leafnodes { + listen: "127.0.0.1:-1" + } + `, accounts))) + defer os.Remove(hconf) + s, o := RunServerWithConfig(hconf) + defer s.Shutdown() + + lconf := createConfFile(t, []byte(fmt.Sprintf(` + %s + server_name: leaf-server + jetstream { + store_dir: '%s' + domain=leaf + } + + listen: "127.0.0.1:-1" + leafnodes { + remotes = [ + { + urls: ["nats-leaf://leaf_ingress:leaf_ingress@127.0.0.1:%v"] + account: "LEAF_INGRESS" + } + ] + } + `, accounts, t.TempDir(), o.LeafNode.Port))) + defer os.Remove(lconf) + sl, so := RunServerWithConfig(lconf) + defer sl.Shutdown() + + checkLeafNodeConnected(t, sl) + + nc := natsConnect(t, fmt.Sprintf("nats://leaf_user:leaf_user@127.0.0.1:%v", so.Port)) + defer nc.Close() + + js, _ := nc.JetStream(nats.APIPrefix("JS.leaf_ingress@leaf.API.")) + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo"}, + Storage: nats.FileStorage, + }) + require_NoError(t, err) + + _, err = js.Publish("foo", []byte("msg")) + require_NoError(t, err) +} + +func TestJetStreamLeafNodeJSClusterMigrateRecovery(t *testing.T) { + tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: hub, store_dir:", 1) + c := createJetStreamCluster(t, tmpl, "hub", _EMPTY_, 3, 12232, true) + defer c.shutdown() + + tmpl = strings.Replace(jsClusterTemplWithLeafNode, "store_dir:", "domain: leaf, store_dir:", 1) + lnc := c.createLeafNodesWithTemplateAndStartPort(tmpl, "leaf", 3, 23913) + defer lnc.shutdown() + + lnc.waitOnClusterReady() + for _, s := range lnc.servers { + s.setJetStreamMigrateOnRemoteLeaf() + } + + nc, _ := jsClientConnect(t, lnc.randomServer()) + defer nc.Close() + + ljs, err := nc.JetStream(nats.Domain("leaf")) + require_NoError(t, err) + + // Create an asset in the leafnode cluster. + si, err := ljs.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo"}, + Replicas: 3, + }) + require_NoError(t, err) + require_Equal(t, si.Cluster.Name, "leaf") + require_NotEqual(t, si.Cluster.Leader, noLeader) + require_Equal(t, len(si.Cluster.Replicas), 2) + + // Count how many remotes each server in the leafnode cluster is + // supposed to have and then take them down. + remotes := map[*Server]int{} + for _, s := range lnc.servers { + s.mu.RLock() + count := len(s.leafRemoteCfgs) + s.mu.RUnlock() + remotes[s] += count + s.closeAndDisableLeafnodes() + checkLeafNodeConnectedCount(t, s, 0) + } + + // The Raft nodes in the leafnode cluster now need some time to + // notice that they're no longer receiving AEs from a leader, as + // they should have been forced into observer mode. Check that + // this is the case. + time.Sleep(maxElectionTimeout) + for _, s := range lnc.servers { + s.rnMu.RLock() + for name, n := range s.raftNodes { + // We don't expect the metagroup to have turned into an + // observer but all other assets should have done. + if name == defaultMetaGroupName { + require_False(t, n.IsObserver()) + } else { + require_True(t, n.IsObserver()) + } + } + s.rnMu.RUnlock() + } + + // Bring the leafnode connections back up. + for _, s := range lnc.servers { + s.reEnableLeafnodes() + checkLeafNodeConnectedCount(t, s, remotes[s]) + } + + // Wait for nodes to notice they are no longer in observer mode + // and to leave observer mode. + time.Sleep(maxElectionTimeout) + for _, s := range lnc.servers { + s.rnMu.RLock() + for _, n := range s.raftNodes { + require_False(t, n.IsObserver()) + } + s.rnMu.RUnlock() + } + + // Previously nodes would have left observer mode but then would + // have failed to elect a stream leader as they were stuck on a + // long election timer. Now this should work reliably. + lnc.waitOnStreamLeader(globalAccountName, "TEST") +} + +func TestJetStreamLeafNodeJSClusterMigrateRecoveryWithDelay(t *testing.T) { + tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: hub, store_dir:", 1) + c := createJetStreamCluster(t, tmpl, "hub", _EMPTY_, 3, 12232, true) + defer c.shutdown() + + tmpl = strings.Replace(jsClusterTemplWithLeafNode, "store_dir:", "domain: leaf, store_dir:", 1) + lnc := c.createLeafNodesWithTemplateAndStartPort(tmpl, "leaf", 3, 23913) + defer lnc.shutdown() + + lnc.waitOnClusterReady() + delay := 5 * time.Second + for _, s := range lnc.servers { + s.setJetStreamMigrateOnRemoteLeafWithDelay(delay) + } + + nc, _ := jsClientConnect(t, lnc.randomServer()) + defer nc.Close() + + ljs, err := nc.JetStream(nats.Domain("leaf")) + require_NoError(t, err) + + // Create an asset in the leafnode cluster. + si, err := ljs.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo"}, + Replicas: 3, + }) + require_NoError(t, err) + require_Equal(t, si.Cluster.Name, "leaf") + require_NotEqual(t, si.Cluster.Leader, noLeader) + require_Equal(t, len(si.Cluster.Replicas), 2) + + // Count how many remotes each server in the leafnode cluster is + // supposed to have and then take them down. + remotes := map[*Server]int{} + for _, s := range lnc.servers { + s.mu.RLock() + count := len(s.leafRemoteCfgs) + s.mu.RUnlock() + remotes[s] += count + s.closeAndDisableLeafnodes() + checkLeafNodeConnectedCount(t, s, 0) + } + + // The Raft nodes in the leafnode cluster now need some time to + // notice that they're no longer receiving AEs from a leader, as + // they should have been forced into observer mode. Check that + // this is the case. + // We expect the nodes to become observers after the delay time. + now := time.Now() + timeout := maxElectionTimeout + delay + success := false + for time.Since(now) <= timeout { + allObservers := true + for _, s := range lnc.servers { + s.rnMu.RLock() + for name, n := range s.raftNodes { + if name == defaultMetaGroupName { + require_False(t, n.IsObserver()) + } else if n.IsObserver() { + // Make sure the migration delay is respected. + require_True(t, time.Since(now) > time.Duration(float64(delay)*0.7)) + } else { + allObservers = false + } + } + s.rnMu.RUnlock() + } + if allObservers { + success = true + break + } + time.Sleep(100 * time.Millisecond) + } + require_True(t, success) + + // Bring the leafnode connections back up. + for _, s := range lnc.servers { + s.reEnableLeafnodes() + checkLeafNodeConnectedCount(t, s, remotes[s]) + } + + // Wait for nodes to notice they are no longer in observer mode + // and to leave observer mode. + time.Sleep(maxElectionTimeout) + for _, s := range lnc.servers { + s.rnMu.RLock() + for _, n := range s.raftNodes { + require_False(t, n.IsObserver()) + } + s.rnMu.RUnlock() + } + + // Make sure all delay timers in remotes are disabled + for _, s := range lnc.servers { + for r := range s.leafRemoteCfgs { + r.RLock() + ok := r.jsMigrateTimer == nil + r.RUnlock() + require_True(t, ok) + } + } + + // Previously nodes would have left observer mode but then would + // have failed to elect a stream leader as they were stuck on a + // long election timer. Now this should work reliably. + lnc.waitOnStreamLeader(globalAccountName, "TEST") +} + +func TestJetStreamLeafNodeJSClusterMigrateClearObserverOnRemoteRemoval(t *testing.T) { + tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: hub, store_dir:", 1) + c := createJetStreamCluster(t, tmpl, "hub", _EMPTY_, 3, 12232, true) + defer c.shutdown() + + tmpl = strings.Replace(jsClusterTemplWithLeafNode, "store_dir:", "domain: leaf, store_dir:", 1) + lnc := c.createLeafNodesWithTemplateAndStartPort(tmpl, "leaf", 3, 23913) + defer lnc.shutdown() + + lnc.waitOnClusterReady() + for _, s := range lnc.servers { + s.setJetStreamMigrateOnRemoteLeaf() + } + + nc, _ := jsClientConnect(t, lnc.randomServer()) + defer nc.Close() + + ljs, err := nc.JetStream(nats.Domain("leaf")) + require_NoError(t, err) + + // Create an asset in the leafnode cluster. + si, err := ljs.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo"}, + Replicas: 3, + }) + require_NoError(t, err) + require_Equal(t, si.Cluster.Name, "leaf") + + // Take down the leafnode connections of one of the leaf servers so + // that checkJetStreamMigrate kicks in and moves its assets' raft + // nodes into observer mode. + s := lnc.randomServer() + s.closeAndDisableLeafnodes() + checkLeafNodeConnectedCount(t, s, 0) + + checkFor(t, maxElectionTimeout, 200*time.Millisecond, func() error { + s.rnMu.RLock() + defer s.rnMu.RUnlock() + for name, n := range s.raftNodes { + // The metagroup is not expected to become an observer, + // but all other assets should. + if name != defaultMetaGroupName && !n.IsObserver() { + return fmt.Errorf("expected %q to be an observer", name) + } + } + return nil + }) + + // Now remove the leafnode remotes from the configuration of that + // server and reload. Since we will never reconnect, the observer + // state should be cleared so this server's assets can become + // leaders again. + content, err := os.ReadFile(s.configFile) + require_NoError(t, err) + re := regexp.MustCompile(`(?s)remotes \[.*?\n\t\t\]`) + newContent := re.ReplaceAllString(string(content), "remotes [ ]") + require_NotEqual(t, string(content), newContent) + changeCurrentConfigContentWithNewContent(t, s.configFile, []byte(newContent)) + require_NoError(t, s.Reload()) + + checkFor(t, 5*time.Second, 200*time.Millisecond, func() error { + s.rnMu.RLock() + defer s.rnMu.RUnlock() + for name, n := range s.raftNodes { + if n.IsObserver() { + return fmt.Errorf("expected %q to no longer be an observer", name) + } + } + return nil + }) +} + +// This will test that when a mirror or source construct is setup across a leafnode/domain +// that it will recover quickly once the LN is re-established regardless +// of backoff state of the internal consumer create. +func TestJetStreamLeafNodeAndMirrorResyncAfterConnectionDown(t *testing.T) { + tmplA := ` + listen: -1 + server_name: tcm + jetstream { + store_dir: '%s', + domain: TCM + } + accounts { + JS { users = [ { user: "y", pass: "p" } ]; jetstream: true } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } + leaf { port: -1 } + ` + confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, t.TempDir()))) + sA, oA := RunServerWithConfig(confA) + defer sA.Shutdown() + + // Create a proxy - we will use this to simulate a network down event. + rtt, bw := 10*time.Microsecond, 10*1024*1024*1024 + proxy := newNetProxy(rtt, bw, bw, fmt.Sprintf("nats://y:p@127.0.0.1:%d", oA.LeafNode.Port)) + defer proxy.stop() + + tmplB := ` + listen: -1 + server_name: xmm + jetstream { + store_dir: '%s', + domain: XMM + } + accounts { + JS { users = [ { user: "y", pass: "p" } ]; jetstream: true } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } + leaf { remotes [ { url: %s, account: "JS" } ], reconnect: "0.25s" } + ` + + confB := createConfFile(t, []byte(fmt.Sprintf(tmplB, t.TempDir(), proxy.leafURL()))) + sB, _ := RunServerWithConfig(confB) + defer sA.Shutdown() + + // Make sure we are connected ok. + checkLeafNodeConnectedCount(t, sA, 1) + checkLeafNodeConnectedCount(t, sB, 1) + + // We will have 3 streams that we will test for proper syncing after + // the network is restored. + // + // 1. Mirror A --> B + // 2. Mirror A <-- B + // 3. Source A <-> B + + // Connect to sA. + ncA, jsA := jsClientConnect(t, sA, nats.UserInfo("y", "p")) + defer ncA.Close() + + // Connect to sB. + ncB, jsB := jsClientConnect(t, sB, nats.UserInfo("y", "p")) + defer ncB.Close() + + // Add in TEST-A + _, err := jsA.AddStream(&nats.StreamConfig{Name: "TEST-A", Subjects: []string{"foo"}}) + require_NoError(t, err) + + // Add in TEST-B + _, err = jsB.AddStream(&nats.StreamConfig{Name: "TEST-B", Subjects: []string{"bar"}}) + require_NoError(t, err) + + // Now setup mirrors. + _, err = jsB.AddStream(&nats.StreamConfig{ + Name: "M-A", + Mirror: &nats.StreamSource{ + Name: "TEST-A", + External: &nats.ExternalStream{APIPrefix: "$JS.TCM.API"}, + }, + }) + require_NoError(t, err) + + _, err = jsA.AddStream(&nats.StreamConfig{ + Name: "M-B", + Mirror: &nats.StreamSource{ + Name: "TEST-B", + External: &nats.ExternalStream{APIPrefix: "$JS.XMM.API"}, + }, + }) + require_NoError(t, err) + + // Now add in the streams that will source from one another bi-directionally. + _, err = jsA.AddStream(&nats.StreamConfig{ + Name: "SRC-A", + Subjects: []string{"A.*"}, + Sources: []*nats.StreamSource{{ + Name: "SRC-B", + FilterSubject: "B.*", + External: &nats.ExternalStream{APIPrefix: "$JS.XMM.API"}, + }}, + }) + require_NoError(t, err) + + _, err = jsB.AddStream(&nats.StreamConfig{ + Name: "SRC-B", + Subjects: []string{"B.*"}, + Sources: []*nats.StreamSource{{ + Name: "SRC-A", + FilterSubject: "A.*", + External: &nats.ExternalStream{APIPrefix: "$JS.TCM.API"}, + }}, + }) + require_NoError(t, err) + + // Now load them up with 500 messages. + initMsgs := 500 + for i := 0; i < initMsgs; i++ { + // Individual Streams + jsA.PublishAsync("foo", []byte("PAYLOAD")) + jsB.PublishAsync("bar", []byte("PAYLOAD")) + // Bi-directional Sources + jsA.PublishAsync("A.foo", []byte("PAYLOAD")) + jsB.PublishAsync("B.bar", []byte("PAYLOAD")) + } + select { + case <-jsA.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + select { + case <-jsB.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + + // Utility to check the number of stream msgs. + checkStreamMsgs := func(js nats.JetStreamContext, sname string, expected int, perr error) error { + t.Helper() + if perr != nil { + return perr + } + si, err := js.StreamInfo(sname) + require_NoError(t, err) + if si.State.Msgs != uint64(expected) { + return fmt.Errorf("Expected %d msgs for %s, got state: %+v", expected, sname, si.State) + } + return nil + } + + // Wait til we see all messages. + checkFor(t, 2*time.Second, 250*time.Millisecond, func() error { + err := checkStreamMsgs(jsA, "TEST-A", initMsgs, nil) + err = checkStreamMsgs(jsB, "M-A", initMsgs, err) + err = checkStreamMsgs(jsB, "TEST-B", initMsgs, err) + err = checkStreamMsgs(jsA, "M-B", initMsgs, err) + err = checkStreamMsgs(jsA, "SRC-A", initMsgs*2, err) + err = checkStreamMsgs(jsB, "SRC-B", initMsgs*2, err) + return err + }) + + // Take down proxy. This will stop any propagation of messages between TEST and M streams. + proxy.stop() + + // Now add an additional 500 messages to originals on both sides. + for i := 0; i < initMsgs; i++ { + // Individual Streams + jsA.PublishAsync("foo", []byte("PAYLOAD")) + jsB.PublishAsync("bar", []byte("PAYLOAD")) + // Bi-directional Sources + jsA.PublishAsync("A.foo", []byte("PAYLOAD")) + jsB.PublishAsync("B.bar", []byte("PAYLOAD")) + } + select { + case <-jsA.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + select { + case <-jsB.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + + cancelAndDelayConsumer := func(s *Server, stream string) { + // Now make sure internal consumer is at max backoff. + acc, err := s.lookupAccount("JS") + require_NoError(t, err) + mset, err := acc.lookupStream(stream) + require_NoError(t, err) + + // Reset sourceInfo to have lots of failures and last attempt 2 minutes ago. + // Lock should be held on parent stream. + resetSourceInfo := func(si *sourceInfo) { + // Do not reset sip here to make sure that the internal logic clears. + si.fails = 100 + si.lreq = time.Now().Add(-2 * time.Minute) + } + + // Force the consumer to be canceled and we simulate 100 failed attempts + // such that the next time we will try will be a long way out. + mset.mu.Lock() + if mset.mirror != nil { + resetSourceInfo(mset.mirror) + mset.cancelSourceInfo(mset.mirror) + mset.scheduleSetupMirrorConsumerRetry() + } else if len(mset.sources) > 0 { + for iname, si := range mset.sources { + resetSourceInfo(si) + mset.cancelSourceInfo(si) + mset.setupSourceConsumer(iname, si.sseq+1, time.Time{}) + } + } + mset.mu.Unlock() + } + + // Mirrors + cancelAndDelayConsumer(sA, "M-B") + cancelAndDelayConsumer(sB, "M-A") + // Now bi-directional sourcing + cancelAndDelayConsumer(sA, "SRC-A") + cancelAndDelayConsumer(sB, "SRC-B") + + // Now restart the network proxy. + proxy.start() + + // Make sure we are connected ok. + checkLeafNodeConnectedCount(t, sA, 1) + checkLeafNodeConnectedCount(t, sB, 1) + + // These should be good before re-sync. + require_NoError(t, checkStreamMsgs(jsA, "TEST-A", initMsgs*2, nil)) + require_NoError(t, checkStreamMsgs(jsB, "TEST-B", initMsgs*2, nil)) + + start := time.Now() + // Wait til we see all messages. + checkFor(t, 2*time.Minute, 50*time.Millisecond, func() error { + err := checkStreamMsgs(jsA, "M-B", initMsgs*2, err) + err = checkStreamMsgs(jsB, "M-A", initMsgs*2, err) + err = checkStreamMsgs(jsA, "SRC-A", initMsgs*4, err) + err = checkStreamMsgs(jsB, "SRC-B", initMsgs*4, err) + return err + }) + if elapsed := time.Since(start); elapsed > 3*time.Second { + t.Fatalf("Expected to resync all streams <3s but got %v", elapsed) + } +} + +// This test will test a 3 node setup where we have a hub node, a gateway node, and a satellite node. +// This is specifically testing re-sync when there is not a direct Domain with JS match for the first +// hop connect LN that is signaling. +// +// HUB <---- GW(+JS/DOMAIN) -----> SAT1 +// ^ +// | +// +------- GW(-JS/NO DOMAIN) --> SAT2 +// +// The Gateway node will solicit the satellites but will act as a LN hub. +func TestJetStreamLeafNodeAndMirrorResyncAfterLeafEstablished(t *testing.T) { + accs := ` + accounts { + JS { users = [ { user: "u", pass: "p" } ]; jetstream: true } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } + ` + hubT := ` + listen: -1 + server_name: hub + jetstream { store_dir: '%s', domain: HUB } + %s + leaf { port: -1 } + ` + confA := createConfFile(t, []byte(fmt.Sprintf(hubT, t.TempDir(), accs))) + sHub, oHub := RunServerWithConfig(confA) + defer sHub.Shutdown() + + // We run the SAT node second to extract out info for solicitation from targeted GW. + sat1T := ` + listen: -1 + server_name: sat1 + jetstream { store_dir: '%s', domain: SAT1 } + %s + leaf { port: -1 } + ` + confB := createConfFile(t, []byte(fmt.Sprintf(sat1T, t.TempDir(), accs))) + sSat1, oSat1 := RunServerWithConfig(confB) + defer sSat1.Shutdown() + + sat2T := ` + listen: -1 + server_name: sat2 + jetstream { store_dir: '%s', domain: SAT2 } + %s + leaf { port: -1 } + ` + confC := createConfFile(t, []byte(fmt.Sprintf(sat2T, t.TempDir(), accs))) + sSat2, oSat2 := RunServerWithConfig(confC) + defer sSat2.Shutdown() + + hubLeafPort := fmt.Sprintf("nats://u:p@127.0.0.1:%d", oHub.LeafNode.Port) + sat1LeafPort := fmt.Sprintf("nats://u:p@127.0.0.1:%d", oSat1.LeafNode.Port) + sat2LeafPort := fmt.Sprintf("nats://u:p@127.0.0.1:%d", oSat2.LeafNode.Port) + + gw1T := ` + listen: -1 + server_name: gw1 + jetstream { store_dir: '%s', domain: GW } + %s + leaf { remotes [ { url: %s, account: "JS" }, { url: %s, account: "JS", hub: true } ], reconnect: "0.25s" } + ` + confD := createConfFile(t, []byte(fmt.Sprintf(gw1T, t.TempDir(), accs, hubLeafPort, sat1LeafPort))) + sGW1, _ := RunServerWithConfig(confD) + defer sGW1.Shutdown() + + gw2T := ` + listen: -1 + server_name: gw2 + accounts { + JS { users = [ { user: "u", pass: "p" } ] } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } + leaf { remotes [ { url: %s, account: "JS" }, { url: %s, account: "JS", hub: true } ], reconnect: "0.25s" } + ` + confE := createConfFile(t, []byte(fmt.Sprintf(gw2T, hubLeafPort, sat2LeafPort))) + sGW2, _ := RunServerWithConfig(confE) + defer sGW2.Shutdown() + + // Make sure we are connected ok. + checkLeafNodeConnectedCount(t, sHub, 2) + checkLeafNodeConnectedCount(t, sSat1, 1) + checkLeafNodeConnectedCount(t, sSat2, 1) + checkLeafNodeConnectedCount(t, sGW1, 2) + checkLeafNodeConnectedCount(t, sGW2, 2) + + // Let's place a muxed stream on the hub and have it source from a stream on the Satellite. + // Connect to Hub. + ncHub, jsHub := jsClientConnect(t, sHub, nats.UserInfo("u", "p")) + defer ncHub.Close() + + _, err := jsHub.AddStream(&nats.StreamConfig{Name: "HUB", Subjects: []string{"H.>"}}) + require_NoError(t, err) + + // Connect to Sat1. + ncSat1, jsSat1 := jsClientConnect(t, sSat1, nats.UserInfo("u", "p")) + defer ncSat1.Close() + + _, err = jsSat1.AddStream(&nats.StreamConfig{ + Name: "SAT-1", + Subjects: []string{"S1.*"}, + Sources: []*nats.StreamSource{{ + Name: "HUB", + FilterSubject: "H.SAT-1.>", + External: &nats.ExternalStream{APIPrefix: "$JS.HUB.API"}, + }}, + }) + require_NoError(t, err) + + // Connect to Sat2. + ncSat2, jsSat2 := jsClientConnect(t, sSat2, nats.UserInfo("u", "p")) + defer ncSat2.Close() + + _, err = jsSat2.AddStream(&nats.StreamConfig{ + Name: "SAT-2", + Subjects: []string{"S2.*"}, + Sources: []*nats.StreamSource{{ + Name: "HUB", + FilterSubject: "H.SAT-2.>", + External: &nats.ExternalStream{APIPrefix: "$JS.HUB.API"}, + }}, + }) + require_NoError(t, err) + + // Put in 10 msgs each in for each satellite. + for i := 0; i < 10; i++ { + jsHub.Publish("H.SAT-1.foo", []byte("CMD")) + jsHub.Publish("H.SAT-2.foo", []byte("CMD")) + } + // Make sure both are sync'd. + checkFor(t, time.Second, 100*time.Millisecond, func() error { + si, err := jsSat1.StreamInfo("SAT-1") + require_NoError(t, err) + if si.State.Msgs != 10 { + return errors.New("SAT-1 Not sync'd yet") + } + si, err = jsSat2.StreamInfo("SAT-2") + require_NoError(t, err) + if si.State.Msgs != 10 { + return errors.New("SAT-2 Not sync'd yet") + } + return nil + }) + + testReconnect := func(t *testing.T, delay time.Duration, expected uint64) { + // Now disconnect Sat1 and Sat2. In 2.12 we can do this with active: false, but since this will be + // pulled into 2.11.9 just shutdown both gateways. + sGW1.Shutdown() + checkLeafNodeConnectedCount(t, sSat1, 0) + checkLeafNodeConnectedCount(t, sHub, 1) + + sGW2.Shutdown() + checkLeafNodeConnectedCount(t, sSat2, 0) + checkLeafNodeConnectedCount(t, sHub, 0) + + // Send 10 more messages for each while GW1 and GW2 are down. + for i := 0; i < 10; i++ { + jsHub.Publish("H.SAT-1.foo", []byte("CMD")) + jsHub.Publish("H.SAT-2.foo", []byte("CMD")) + } + + // Keep GWs down for delay. + time.Sleep(delay) + + sGW1, _ = RunServerWithConfig(confD) + // Make sure we are connected ok. + checkLeafNodeConnectedCount(t, sHub, 1) + checkLeafNodeConnectedCount(t, sSat1, 1) + checkLeafNodeConnectedCount(t, sGW1, 2) + + sGW2, _ = RunServerWithConfig(confE) + // Make sure we are connected ok. + checkLeafNodeConnectedCount(t, sHub, 2) + checkLeafNodeConnectedCount(t, sSat2, 1) + checkLeafNodeConnectedCount(t, sGW2, 2) + + // Make sure sync'd in less than a second or two. + checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { + si, err := jsSat1.StreamInfo("SAT-1") + require_NoError(t, err) + if si.State.Msgs != expected { + return fmt.Errorf("SAT-1 not sync'd, expected %d got %d", expected, si.State.Msgs) + } + si, err = jsSat2.StreamInfo("SAT-2") + require_NoError(t, err) + if si.State.Msgs != expected { + return fmt.Errorf("SAT-2 not sync'd, expected %d got %d", expected, si.State.Msgs) + } + return nil + }) + } + + // We will test two scenarios with amount of time the GWs (link) is down. + // 1. Just a second, we will not have detected the consumer is offline as of yet. + // 2. Just over sourceHealthCheckInterval, meaning we detect it is down and schedule for another try. + t.Run(fmt.Sprintf("reconnect-%v", time.Second), func(t *testing.T) { + testReconnect(t, time.Second, 20) + }) + t.Run(fmt.Sprintf("reconnect-%v", sourceHealthCheckInterval+time.Second), func(t *testing.T) { + testReconnect(t, sourceHealthCheckInterval+time.Second, 30) + }) + defer sGW1.Shutdown() + defer sGW2.Shutdown() +} + +func TestJetStreamSourceConsumerLeafReconnectStorm(t *testing.T) { + hubT := ` + listen: -1 + server_name: hub + jetstream { store_dir: '%s', domain: HUB } + leaf { port: -1 } + ` + hubConf := createConfFile(t, []byte(fmt.Sprintf(hubT, t.TempDir()))) + sHub, oHub := RunServerWithConfig(hubConf) + defer sHub.Shutdown() + hubLeafURL := fmt.Sprintf("nats://u:p@127.0.0.1:%d", oHub.LeafNode.Port) + + leafT := ` + listen: -1 + server_name: %s + jetstream { store_dir: '%s', domain: %s } + leaf { remotes [ { url: %s } ], reconnect: "0.25s" } + ` + + type leafSpec struct { + name string + domain string + srcStream string + conf string + srv *Server + } + leafs := []*leafSpec{ + {name: "leaf1", domain: "L1", srcStream: "S1"}, + {name: "leaf2", domain: "L2", srcStream: "S2"}, + {name: "leaf3", domain: "L3", srcStream: "S3"}, + } + for _, lf := range leafs { + lf.conf = createConfFile(t, []byte(fmt.Sprintf(leafT, lf.name, t.TempDir(), lf.domain, hubLeafURL))) + lf.srv, _ = RunServerWithConfig(lf.conf) + } + defer func() { + for _, lf := range leafs { + if lf.srv != nil { + lf.srv.Shutdown() + } + } + }() + + // All three leaves connected to the hub. + checkLeafNodeConnectedCount(t, sHub, 3) + for _, lf := range leafs { + checkLeafNodeConnectedCount(t, lf.srv, 1) + } + + // Source streams on each leaf. + for _, lf := range leafs { + nc, js := jsClientConnect(t, lf.srv, nats.UserInfo("u", "p")) + _, err := js.AddStream(&nats.StreamConfig{ + Name: lf.srcStream, + Subjects: []string{lf.srcStream + ".>"}, + }) + require_NoError(t, err) + nc.Close() + } + + // Aggregate stream on the hub sourcing from each leaf via that leaf's + // external API prefix. + ncHub, jsHub := jsClientConnect(t, sHub, nats.UserInfo("u", "p")) + defer ncHub.Close() + + var sources []*nats.StreamSource + for _, lf := range leafs { + sources = append(sources, &nats.StreamSource{ + Name: lf.srcStream, + External: &nats.ExternalStream{APIPrefix: fmt.Sprintf("$JS.%s.API", lf.domain)}, + }) + } + _, err := jsHub.AddStream(&nats.StreamConfig{Name: "AGG", Sources: sources}) + require_NoError(t, err) + mset, err := sHub.globalAccount().lookupStream("AGG") + require_NoError(t, err) + + checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { + mset.mu.RLock() + defer mset.mu.RUnlock() + if got := len(mset.sources); got != len(leafs) { + return fmt.Errorf("expected %d sources, have %d", len(leafs), got) + } + for _, si := range mset.sources { + if si.cname == _EMPTY_ || si.sub == nil { + return fmt.Errorf("source %s not established", si.name) + } + } + return nil + }) + + // Force the unrelated sources into the setup-in-progress state that the + // buggy shouldRetry latches onto. Correct scoping must ignore them; + // buggy scoping force-cancels their healthy subs. + type snap struct { + cname string + sub *subscription + } + pre := map[string]snap{} + mset.mu.Lock() + for _, si := range mset.sources { + pre[si.name] = snap{cname: si.cname, sub: si.sub} + if si.name == "S1" || si.name == "S3" { + si.sip = true + } + } + mset.mu.Unlock() + + leafs[1].srv.Shutdown() + checkLeafNodeConnectedCount(t, sHub, 2) + leafs[1].srv, _ = RunServerWithConfig(leafs[1].conf) + checkLeafNodeConnectedCount(t, sHub, 3) + checkLeafNodeConnectedCount(t, leafs[1].srv, 1) + + // processLeafNodeConnect → checkInternalSyncConsumers → + // retryDisconnectedSyncConsumers runs synchronously on the hub for the + // new connection; give it time to do (or fail to do) any damage. + time.Sleep(500 * time.Millisecond) + + // Pointer-equality on *subscription is timing-robust: + // cancelSourceInfo() nils si.sub immediately, and any fresh setup + // produces a new *subscription, so disturbance shows up regardless of + // when we sample within the post-bounce window. + mset.mu.RLock() + post := map[string]snap{} + for _, si := range mset.sources { + post[si.name] = snap{cname: si.cname, sub: si.sub} + } + mset.mu.RUnlock() + + for _, name := range []string{"S1", "S3"} { + if post[name].sub != pre[name].sub { + t.Errorf("source %q was recreated by leaf2 reconnect (different domain): "+ + "pre.sub=%p post.sub=%p (cname pre=%q post=%q)", + name, pre[name].sub, post[name].sub, + pre[name].cname, post[name].cname) + } + } +} + +func TestJetStreamSourceConsumerSetupTimerGoroutineLeak(t *testing.T) { + s := RunBasicJetStreamServer(t) + defer s.Shutdown() + + nc, js := jsClientConnect(t, s) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "AGG", + Sources: []*nats.StreamSource{{ + Name: "SRC", + External: &nats.ExternalStream{APIPrefix: "$JS.NONEXISTENT.API"}, + }}, + }) + require_NoError(t, err) + + mset, err := s.globalAccount().lookupStream("AGG") + require_NoError(t, err) + + // Wait for the source's sourceInfo to be present. + var iname string + checkFor(t, 5*time.Second, 50*time.Millisecond, func() error { + mset.mu.RLock() + defer mset.mu.RUnlock() + for in := range mset.sources { + iname = in + return nil + } + return errors.New("no source yet") + }) + + // Let the initial setup quiesce so its inner goroutine is the + // pre-existing baseline rather than being attributed to the leak. + time.Sleep(300 * time.Millisecond) + baseline := runtime.NumGoroutine() + + // Per cycle: call retry, then wait one AfterFunc delay (100ms+rand(100ms)) + // so the body fires and parks its inner goroutine in select. + const N = 30 + for range N { + mset.mu.Lock() + if si := mset.sources[iname]; si != nil { + si.sip = true + // bypass the 2s retry throttle in setupSourceConsumer. + si.lreq = time.Time{} + } + mset.mu.Unlock() + mset.retryDisconnectedSyncConsumers() + time.Sleep(250 * time.Millisecond) + } + time.Sleep(300 * time.Millisecond) + + leaked := runtime.NumGoroutine() - baseline + const tolerance = 5 + if leaked > tolerance { + t.Fatalf("AfterFunc-driven retry leaked %d goroutines after %d "+ + "retryDisconnectedSyncConsumers cycles for a single iname "+ + "(baseline=%d). Each cycle's AfterFunc fires a fresh "+ + "trySetupSourceConsumer which spawns a new inner goroutine; "+ + "the previous inner goroutine is orphaned in select, "+ + "allowing linear ramp-up.", + leaked, N, baseline) + } +} diff --git a/server/jetstream_meta_benchmark_test.go b/server/jetstream_meta_benchmark_test.go new file mode 100644 index 0000000..24ce2b0 --- /dev/null +++ b/server/jetstream_meta_benchmark_test.go @@ -0,0 +1,350 @@ +// Copyright 2024-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !skip_js_tests && !skip_js_cluster_tests && !skip_js_cluster_tests_2 + +package server + +import ( + "fmt" + "sync" + "sync/atomic" + "testing" + + "github.com/nats-io/nats.go" +) + +func BenchmarkJetStreamCreate(b *testing.B) { + + const ( + verbose = false + resourcePrefix = "S" + concurrency = 12 + ) + + // Types of resource that this benchmark creates + type ResourceType string + const ( + Stream ResourceType = "Stream" + KVBucket ResourceType = "KVBucket" + ObjectStore ResourceType = "ObjStore" + ) + + resourceTypeCases := []ResourceType{ + Stream, + KVBucket, + ObjectStore, + } + + benchmarksCases := []struct { + clusterSize int + replicas int + storage nats.StorageType + }{ + {1, 1, nats.MemoryStorage}, + {3, 3, nats.MemoryStorage}, + {3, 3, nats.FileStorage}, + } + + for _, bc := range benchmarksCases { + bName := fmt.Sprintf( + "N=%d,R=%d,storage=%s,C=%d", + bc.clusterSize, + bc.replicas, + bc.storage.String(), + concurrency, + ) + + b.Run( + bName, + func(b *testing.B) { + for _, rt := range resourceTypeCases { + //for _, bc := range benchmarksCases { + rName := fmt.Sprintf("resource=%s", rt) + b.Run( + rName, + func(b *testing.B) { + + if verbose { + b.Logf( + "Creating %d %s resources in cluster with %d nodes, R=%d, %s storage", + b.N, + string(rt), + bc.clusterSize, + bc.replicas, + bc.storage, + ) + } + + // Setup server or cluster + _, leaderServer, shutdown, nc, _ := startJSClusterAndConnect(b, bc.clusterSize) + defer shutdown() + defer nc.Close() + + // All clients connect to cluster (meta) leader for lower variability + connectURL := leaderServer.ClientURL() + + // Wait for all clients and main routine to be ready + wgReady := sync.WaitGroup{} + wgReady.Add(concurrency + 1) + // Wait for all routines to complete + wgComplete := sync.WaitGroup{} + wgComplete.Add(concurrency) + + // Number of operations (divided amongst clients) + opsLeft := atomic.Int64{} + opsLeft.Store(int64(b.N)) + totalErrors := atomic.Int64{} + + // Pre-create connections and JS contexts + for i := 1; i <= concurrency; i++ { + nc, js := jsClientConnectURL(b, connectURL) + defer nc.Close() + go func(clientId int, nc *nats.Conn, js nats.JetStreamContext) { + defer wgComplete.Done() + + // Config struct (reused and modified in place for each call) + streamConfig := nats.StreamConfig{ + Name: "?", + Storage: bc.storage, + Replicas: bc.replicas, + } + kvConfig := nats.KeyValueConfig{ + Bucket: "?", + Storage: bc.storage, + Replicas: bc.replicas, + } + objConfig := nats.ObjectStoreConfig{ + Bucket: "?", + Storage: bc.storage, + Replicas: bc.replicas, + } + + // Block until everyone is ready + wgReady.Done() + wgReady.Wait() + + errCount := int64(0) + defer func() { + // Roll up error count on completion + totalErrors.Add(errCount) + }() + + // Track per-client opCount (just for logging/debugging) + opCount := 0 + for opsLeft.Add(-1) >= 0 { + var err error + // Create unique resource name + resourceName := fmt.Sprintf("%s_%d_%d", resourcePrefix, clientId, opCount) + switch rt { + case Stream: + streamConfig.Name = resourceName + _, err = js.AddStream(&streamConfig) + case KVBucket: + kvConfig.Bucket = resourceName + _, err = js.CreateKeyValue(&kvConfig) + case ObjectStore: + objConfig.Bucket = resourceName + _, err = js.CreateObjectStore(&objConfig) + } + opCount += 1 + if err != nil { + b.Logf("Error creating %s (%s): %s", rt, resourceName, err) + errCount += 1 + } + } + + if verbose { + b.Logf("Client %d completed %d operations", clientId, opCount) + } + + }(i, nc, js) + } + + // Wait for all clients to be ready + wgReady.Done() + wgReady.Wait() + + // Start benchmark clock + b.ResetTimer() + + wgComplete.Wait() + b.StopTimer() + + b.ReportMetric(float64(100*(totalErrors.Load()))/float64(b.N), "%error") + }, + ) + } + }, + ) + } +} + +func BenchmarkJetStreamCreateConsumers(b *testing.B) { + + const ( + verbose = false + streamName = "S" + consumerPrefix = "C" + concurrency = 12 + ) + + benchmarksCases := []struct { + clusterSize int + consumerReplicas int + consumerStorage nats.StorageType + }{ + {1, 1, nats.MemoryStorage}, + {3, 3, nats.MemoryStorage}, + {3, 3, nats.FileStorage}, + } + + type ConsumerType string + const ( + Ephemeral ConsumerType = "Ephemeral" + Durable ConsumerType = "Durable" + ) + + consumerTypeCases := []ConsumerType{ + Ephemeral, + Durable, + } + + for _, bc := range benchmarksCases { + + bName := fmt.Sprintf( + "N=%d,R=%d,storage=%s,C=%d", + bc.clusterSize, + bc.consumerReplicas, + bc.consumerStorage.String(), + concurrency, + ) + + b.Run( + bName, + func(b *testing.B) { + + for _, ct := range consumerTypeCases { + + cName := fmt.Sprintf("Consumer=%s", ct) + + b.Run( + cName, + func(b *testing.B) { + if verbose { + b.Logf( + "Creating %d consumers in cluster with %d nodes, R=%d, %s storage", + b.N, + bc.clusterSize, + bc.consumerReplicas, + bc.consumerStorage, + ) + } + + // Setup server or cluster + _, leaderServer, shutdown, nc, js := startJSClusterAndConnect(b, bc.clusterSize) + defer shutdown() + defer nc.Close() + + // All clients connect to cluster (meta) leader for lower variability + connectURL := leaderServer.ClientURL() + + // Create stream + streamConfig := nats.StreamConfig{ + Name: streamName, + Storage: nats.FileStorage, + Replicas: bc.clusterSize, + } + + _, err := js.AddStream(&streamConfig) + if err != nil { + b.Fatalf("Failed to create stream: %s", err) + } + + // Wait for all clients and main routine to be ready + wgReady := sync.WaitGroup{} + wgReady.Add(concurrency + 1) + // Wait for all routines to complete + wgComplete := sync.WaitGroup{} + wgComplete.Add(concurrency) + + // Number of operations (divided amongst clients) + opsLeft := atomic.Int64{} + opsLeft.Store(int64(b.N)) + // Total number of errors + totalErrors := atomic.Int64{} + + // Pre-create connections and JS contexts + for i := 1; i <= concurrency; i++ { + nc, js := jsClientConnectURL(b, connectURL) + defer nc.Close() + + go func(clientId int, nc *nats.Conn, js nats.JetStreamContext) { + defer wgComplete.Done() + + // Config struct (reused and modified in place for each call) + cfg := nats.ConsumerConfig{ + Durable: "", + Name: "", + Replicas: bc.consumerReplicas, + MemoryStorage: bc.consumerStorage == nats.MemoryStorage, + } + + // Block until everyone is ready + wgReady.Done() + wgReady.Wait() + + errCount := int64(0) + opCount := 0 + for opsLeft.Add(-1) >= 0 { + var err error + // Set unique consumer name + cfg.Name = fmt.Sprintf("%s_%d_%d", consumerPrefix, clientId, opCount) + if ct == Durable { + cfg.Durable = cfg.Name + } + _, err = js.AddConsumer(streamName, &cfg) + if err != nil { + b.Logf("Failed to add consumer: %s", err) + errCount += 1 + } + opCount += 1 + } + + if verbose { + b.Logf("Client %d completed %d operations", clientId, opCount) + } + + totalErrors.Add(errCount) + + }(i, nc, js) + } + + // Wait for all clients to be ready + wgReady.Done() + wgReady.Wait() + + // Start benchmark clock + b.ResetTimer() + + wgComplete.Wait() + b.StopTimer() + + b.ReportMetric(float64(100*(totalErrors.Load()))/float64(b.N), "%error") + }, + ) + } + }, + ) + } +} diff --git a/server/jetstream_sourcing_scaling_test.go b/server/jetstream_sourcing_scaling_test.go new file mode 100644 index 0000000..5dca5b1 --- /dev/null +++ b/server/jetstream_sourcing_scaling_test.go @@ -0,0 +1,283 @@ +// Copyright 2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "fmt" + "strconv" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +var serverConfig1 = ` + server_name: server1 + listen: 127.0.0.1:4222 + http: 8222 + + prof_port = 18222 + + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + cluster { + name: my_cluster + listen: 127.0.0.1:4248 + routes: [nats://127.0.0.1:4249,nats://127.0.0.1:4250] + } + + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] }} +` + +var serverConfig2 = ` + server_name: server2 + listen: 127.0.0.1:5222 + http: 8223 + + prof_port = 18223 + + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + cluster { + name: my_cluster + listen: 127.0.0.1:4249 + routes: [nats://127.0.0.1:4248,nats://127.0.0.1:4250] + } + + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] }} +` + +var serverConfig3 = ` + server_name: server3 + listen: 127.0.0.1:6222 + http: 8224 + + prof_port = 18224 + + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + cluster { + name: my_cluster + listen: 127.0.0.1:4250 + routes: [nats://127.0.0.1:4248,nats://127.0.0.1:4249] + } + + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] }} +` + +var connectURL = "nats://127.0.0.1:4222,nats://127.0.0.1:4223,nats://127.0.0.1:4224" + +func createMyLocalCluster(t *testing.T) *cluster { + c := &cluster{servers: make([]*Server, 0, 3), opts: make([]*Options, 0, 3), name: "C3"} + + storeDir1 := t.TempDir() + s1, o := RunServerWithConfig(createConfFile(t, []byte(fmt.Sprintf(serverConfig1, storeDir1)))) + c.servers = append(c.servers, s1) + c.opts = append(c.opts, o) + + storeDir2 := t.TempDir() + s2, o := RunServerWithConfig(createConfFile(t, []byte(fmt.Sprintf(serverConfig2, storeDir2)))) + c.servers = append(c.servers, s2) + c.opts = append(c.opts, o) + + storeDir3 := t.TempDir() + s3, o := RunServerWithConfig(createConfFile(t, []byte(fmt.Sprintf(serverConfig3, storeDir3)))) + c.servers = append(c.servers, s3) + c.opts = append(c.opts, o) + + c.t = t + + // Wait til we are formed and have a leader. + c.checkClusterFormed() + c.waitOnClusterReady() + + return c +} + +// This test is being skipped by CI as it takes too long to run and is meant to test the scalability of sourcing +// rather than being a unit test. +func TestStreamSourcingScalingSourcingManyBenchmark(t *testing.T) { + t.Skip() + + var numSourced = 1000 + var numMsgPerSource = 10_000 + var batchSize = 200 + var retries int + + var err error + + c := createMyLocalCluster(t) + defer c.shutdown() + + nc, err := nats.Connect(connectURL) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + js, err := nc.JetStream(nats.MaxWait(10 * time.Second)) + if err != nil { + t.Fatalf("Unexpected error getting JetStream context: %v", err) + } + defer nc.Close() + + // create n streams to source from + for i := 0; i < numSourced; i++ { + _, err := js.AddStream(&nats.StreamConfig{ + Name: fmt.Sprintf("sourced-%d", i), + Subjects: []string{strconv.Itoa(i)}, + Retention: nats.LimitsPolicy, + Storage: nats.FileStorage, + Discard: nats.DiscardOld, + Replicas: 1, + AllowDirect: true, + MirrorDirect: false, + DiscardNewPerSubject: false, + }) + require_NoError(t, err) + } + + fmt.Printf("Streams created\n") + + // publish n messages for each sourced stream + for j := 0; j < numMsgPerSource; j++ { + start := time.Now() + var pafs = make([]nats.PubAckFuture, numSourced) + for i := 0; i < numSourced; i++ { + var err error + + for { + pafs[i], err = js.PublishAsync(strconv.Itoa(i), []byte(strconv.Itoa(j))) + if err != nil { + fmt.Printf("Error async publishing: %v, retrying\n", err) + retries++ + time.Sleep(10 * time.Millisecond) + } else { + break + } + } + + if i != 0 && i%batchSize == 0 { + <-js.PublishAsyncComplete() + } + + } + + <-js.PublishAsyncComplete() + + for i := 0; i < numSourced; i++ { + select { + case <-pafs[i].Ok(): + case psae := <-pafs[i].Err(): + fmt.Printf("Error on PubAckFuture: %v, retrying sync...\n", psae) + retries++ + _, err = js.Publish(strconv.Itoa(i), []byte(strconv.Itoa(j))) + require_NoError(t, err) + } + } + + end := time.Now() + if j%1000 == 0 { + fmt.Printf("Published round %d, avg pub latency %v\n", j, end.Sub(start)/time.Duration(numSourced)) + } + } + + fmt.Printf("Messages published\n") + + // create the StreamSources + streamSources := make([]*nats.StreamSource, numSourced) + + for i := 0; i < numSourced; i++ { + streamSources[i] = &nats.StreamSource{Name: fmt.Sprintf("sourced-%d", i), FilterSubject: strconv.Itoa(i)} + } + + // create a stream that sources from them + _, err = js.AddStream(&nats.StreamConfig{ + Name: "sourcing", + Subjects: []string{"foo"}, + Sources: streamSources, + Retention: nats.LimitsPolicy, + Storage: nats.FileStorage, + Discard: nats.DiscardOld, + Replicas: 3, + AllowDirect: true, + MirrorDirect: false, + DiscardNewPerSubject: false, + }) + require_NoError(t, err) + c.waitOnStreamLeader(globalAccountName, "sourcing") + sl := c.streamLeader(globalAccountName, "sourcing") + fmt.Printf("Sourcing stream created leader is *** %s ***\n", sl.Name()) + + expectedSeq := make([]int, numSourced) + + start := time.Now() + + var lastMsgs uint64 + mset, err := sl.GlobalAccount().lookupStream("sourcing") + require_NoError(t, err) + + checkFor(t, 5*time.Minute, 1000*time.Millisecond, func() error { + mset.mu.RLock() + var state StreamState + mset.store.FastState(&state) + mset.mu.RUnlock() + + if state.Msgs == uint64(numMsgPerSource*numSourced) { + fmt.Printf("👍 Test passed: expected %d messages, got %d and took %v\n", uint64(numMsgPerSource*numSourced), state.Msgs, time.Since(start)) + return nil + } else if state.Msgs < uint64(numMsgPerSource*numSourced) { + fmt.Printf("Current Rate %d per second - Received %d\n", state.Msgs-lastMsgs, state.Msgs) + lastMsgs = state.Msgs + return fmt.Errorf("Expected %d messages, got %d", uint64(numMsgPerSource*numSourced), state.Msgs) + } else { + fmt.Printf("Too many messages! expected %d (retries=%d), got %d\n", uint64(numMsgPerSource*numSourced), retries, state.Msgs) + return fmt.Errorf("Too many messages: expected %d (retries=%d), got %d", uint64(numMsgPerSource*numSourced), retries, state.Msgs) + } + }) + + // Check that all the messages sourced in the stream are correct + // Note: expects to see exactly increasing matching sequence numbers, so could theoretically fail if some messages + // get recorded 'out of order' (according to the payload value) because asynchronous JS publication is used to + // publish the messages. + // However, that should not happen if the publish 'batch size' is not more than the number of streams being sourced. + + // create a consumer on sourcing + _, err = js.AddConsumer("sourcing", &nats.ConsumerConfig{AckPolicy: nats.AckExplicitPolicy}) + require_NoError(t, err) + syncSub, err := js.SubscribeSync("", nats.BindStream("sourcing")) + require_NoError(t, err) + + start = time.Now() + + print("Checking the messages\n") + for i := 0; i < numSourced*numMsgPerSource; i++ { + msg, err := syncSub.NextMsg(30 * time.Second) + require_NoError(t, err) + sId, err := strconv.Atoi(msg.Subject) + require_NoError(t, err) + seq, err := strconv.Atoi(string(msg.Data)) + require_NoError(t, err) + if expectedSeq[sId] == seq { + expectedSeq[sId]++ + } else { + t.Fatalf("Expected seq number %d got %d for source %d\n", expectedSeq[sId], seq, sId) + } + msg.Ack() + if i%100_000 == 0 { + now := time.Now() + fmt.Printf("[%v] Checked %d messages: %f msgs/sec \n", now, i, 100_000/now.Sub(start).Seconds()) + start = now + } + } + print("👍 Done. \n") +} diff --git a/server/jetstream_tpm_test.go b/server/jetstream_tpm_test.go new file mode 100644 index 0000000..94c82c3 --- /dev/null +++ b/server/jetstream_tpm_test.go @@ -0,0 +1,248 @@ +// Copyright 2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package server + +import ( + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +// The tests in this file are not complete, but are a start to test the TPM +// with default settings. For a complete test we would need to use a TPM +// configured with an owner password and a SRK password. This is not +// typically the default or setup in CI/CD environments. + +var jsTPMConfigPassword = ` + listen: 127.0.0.1:-1 + jetstream: { + store_dir: %q + tpm { + keys_file: %q + encryption_password: "s3cr3t!" + } + }` + +var jsTPMConfigBadPassword = ` + listen: 127.0.0.1:-1 + jetstream: { + store_dir: %q + tpm { + keys_file: %q + encryption_password: "garbage" + } + }` + +var jsTPMConfigPasswordPcr = ` + listen: 127.0.0.1:-1 + jetstream: { + store_dir: %q + tpm { + keys_file: %q + encryption_password: "s3cr3t!" + pcr: %d + } + }` + +var jsTPMConfigAllFields = ` + listen: 127.0.0.1:-1 + jetstream: { + store_dir: %q + tpm { + keys_file: %q + encryption_password: "s3cr3t!" + pcr: %d + srk_password: %q + cipher: "aes" + } + }` + +var jsTPMConfigInvalidBothOptionsSet = ` + listen: 127.0.0.1:-1 + jetstream: { + store_dir: %q + encryption_key: "foo" + tpm { + keys_file: "bar.json" + encryption_password: "s3cr3t!" + } + }` + +func getTPMFileName(t *testing.T) string { + return t.TempDir() + "/" + t.Name() + "/jskeys.json" +} + +func checkSendMessage(t *testing.T, s *Server) { + nc, js := jsClientConnect(t, s) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "tpm_test", + Subjects: []string{"tpm_test"}, + }) + require_NoError(t, err) + + _, err = js.Publish("tpm_test", []byte("hello")) + require_NoError(t, err) +} + +func checkReceiveMessage(t *testing.T, s *Server) { + // reconnect and get the message + nc, js := jsClientConnect(t, s) + defer nc.Close() + + // get the message + sub, err := js.PullSubscribe("tpm_test", "cls") + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + m := fetchMsgs(t, sub, 1, 5*time.Second) + if m == nil { + t.Fatalf("Did not receive the message") + } + if len(m) != 1 { + t.Fatalf("Expected 1 message, got %d", len(m)) + } +} + +func TestJetStreamTPMBasic(t *testing.T) { + fileName := getTPMFileName(t) + cf := createConfFile(t, []byte(fmt.Sprintf(jsTPMConfigPassword, t.TempDir(), fileName))) + + s, _ := RunServerWithConfig(cf) + defer s.Shutdown() + + // Note, actual encryption is tested elsewhere. This just tests that the key is generated. + key := s.opts.JetStreamKey + if !strings.HasPrefix(key, "SU") { + t.Fatalf("expected a TPM key to be generated. Key = %q", key) + } + + if _, err := os.Stat(fileName); err != nil { + t.Fatalf("keys file was not created") + } + + checkSendMessage(t, s) + + // restart the server. + s.Shutdown() + s, _ = RunServerWithConfig(cf) + if s.Running() == false { + t.Fatalf("Server should be running") + } + defer s.Shutdown() + + // double check we're encrypted with the same key. + if s.opts.JetStreamKey != key { + t.Fatalf("expected same key") + } + checkReceiveMessage(t, s) +} + +// Warning: Running this test too frequently will considerably slow down +// these tests and eventually lock out the TPM. +// +// Depending on the environment this varies from 10 failures per day with +// a 24 hour lockout (Dell) to 32 failures per day with a 10 +// minute healing time (decrementing one failure every 10 minutes). +func TestJetStreamTPMKeyBadPassword(t *testing.T) { + fileName := getTPMFileName(t) + cf := createConfFile(t, []byte(fmt.Sprintf(jsTPMConfigPassword, + t.TempDir(), fileName))) + + s, _ := RunServerWithConfig(cf) + defer s.Shutdown() + + checkSendMessage(t, s) + s.Shutdown() + + // restart the server. + defer func() { + r := recover() + if r == nil { + t.Fatalf("Expected server panic, unexpected start.") + } + }() + cf2 := createConfFile(t, []byte(fmt.Sprintf(jsTPMConfigBadPassword, + t.TempDir(), fileName))) + s, _ = RunServerWithConfig(cf2) + time.Sleep(3 * time.Second) + if s.Running() { + s.Shutdown() + t.Fatalf("Server should NOT be running") + } +} + +func TestJetStreamTPMKeyWithPCR(t *testing.T) { + cf := createConfFile(t, []byte(fmt.Sprintf(jsTPMConfigPasswordPcr, + t.TempDir(), getTPMFileName(t), 18))) + + s, _ := RunServerWithConfig(cf) + defer s.Shutdown() + + checkSendMessage(t, s) + s.Shutdown() + + // restart the server + s, _ = RunServerWithConfig(cf) + if !s.Running() { + t.Fatalf("Server should be running") + } +} + +func TestJetStreamTPMAll(t *testing.T) { + fileName := getTPMFileName(t) + + // TODO: When the CI/CD environment supports updating the TPM, + // expand this test with the SRK password. + cf := createConfFile(t, []byte(fmt.Sprintf(jsTPMConfigAllFields, + getTPMFileName(t), fileName, 19, ""))) + + s, _ := RunServerWithConfig(cf) + defer s.Shutdown() + + checkSendMessage(t, s) + s.Shutdown() + + // restart the server. + s, _ = RunServerWithConfig(cf) + if s.Running() == false { + t.Fatalf("Server should be running") + } + defer s.Shutdown() + checkReceiveMessage(t, s) +} + +func TestJetStreamInvalidConfig(t *testing.T) { + cf := createConfFile(t, []byte(fmt.Sprintf(jsTPMConfigInvalidBothOptionsSet, + getTPMFileName(t)))) + + defer func() { + r := recover() + if r == nil { + t.Fatalf("Expected server panic, unexpected start.") + } + }() + s, _ := RunServerWithConfig(cf) + if s.Running() == true { + s.Shutdown() + t.Fatalf("Server should NOT be running") + } +} diff --git a/server/jetstream_versioning.go b/server/jetstream_versioning.go new file mode 100644 index 0000000..bcf710d --- /dev/null +++ b/server/jetstream_versioning.go @@ -0,0 +1,246 @@ +// Copyright 2024-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import "strconv" + +const ( + // JSApiLevel is the maximum supported JetStream API level for this server. + JSApiLevel int = 5 + + JSRequiredLevelMetadataKey = "_nats.req.level" + JSServerVersionMetadataKey = "_nats.ver" + JSServerLevelMetadataKey = "_nats.level" +) + +// getRequiredApiLevel returns the required API level for the JetStream asset. +func getRequiredApiLevel(metadata map[string]string) string { + if l, ok := metadata[JSRequiredLevelMetadataKey]; ok && l != _EMPTY_ { + return l + } + return _EMPTY_ +} + +// supportsRequiredApiLevel returns whether the required API level for the JetStream asset is supported. +func supportsRequiredApiLevel(metadata map[string]string) bool { + if l := getRequiredApiLevel(metadata); l != _EMPTY_ { + li, err := strconv.Atoi(l) + return err == nil && li <= JSApiLevel + } + return true +} + +// setStaticStreamMetadata sets JetStream stream metadata, like the server version and API level. +// Any dynamic metadata is removed, it must not be stored and only be added for responses. +func setStaticStreamMetadata(cfg *StreamConfig) { + if cfg.Metadata == nil { + cfg.Metadata = make(map[string]string) + } else { + deleteDynamicMetadata(cfg.Metadata) + } + + var requiredApiLevel int + requires := func(level int) { + if level > requiredApiLevel { + requiredApiLevel = level + } + } + + // TTLs were added in v2.11 and require API level 1. + if cfg.AllowMsgTTL || cfg.SubjectDeleteMarkerTTL > 0 { + requires(1) + } + + // Counter CRDTs were added in v2.12 and require API level 2. + if cfg.AllowMsgCounter { + requires(2) + } + + // Atomic batch publishing was added in v2.12 and require API level 2. + if cfg.AllowAtomicPublish { + requires(2) + } + + // Message scheduling was added in v2.12 and require API level 2. + if cfg.AllowMsgSchedules { + requires(2) + } + + // Async persist mode was added in v2.12 and requires API level 2. + if cfg.PersistMode == AsyncPersistMode { + requires(2) + } + + // Fast batch publishing was added in v2.14 and requires API level 4. + if cfg.AllowBatchPublish { + requires(4) + } + + cfg.Metadata[JSRequiredLevelMetadataKey] = strconv.Itoa(requiredApiLevel) +} + +// setDynamicStreamMetadata adds dynamic fields into the (copied) metadata. +func setDynamicStreamMetadata(cfg *StreamConfig) *StreamConfig { + var newCfg StreamConfig + if cfg != nil { + newCfg = *cfg + } + newCfg.Metadata = make(map[string]string) + if cfg != nil { + for key, value := range cfg.Metadata { + newCfg.Metadata[key] = value + } + } + newCfg.Metadata[JSServerVersionMetadataKey] = VERSION + newCfg.Metadata[JSServerLevelMetadataKey] = strconv.Itoa(JSApiLevel) + return &newCfg +} + +// copyConsumerMetadata copies versioning fields from metadata of prevCfg into cfg. +// Removes versioning fields if no previous metadata, updates if set, and removes fields if it doesn't exist in prevCfg. +// Any dynamic metadata is removed, it must not be stored and only be added for responses. +// +// Note: useful when doing equality checks on cfg and prevCfg, but ignoring any versioning metadata differences. +func copyStreamMetadata(cfg *StreamConfig, prevCfg *StreamConfig) { + if cfg.Metadata != nil { + deleteDynamicMetadata(cfg.Metadata) + } + setOrDeleteInStreamMetadata(cfg, prevCfg, JSRequiredLevelMetadataKey) +} + +// setOrDeleteInConsumerMetadata sets field with key/value in metadata of cfg if set, deletes otherwise. +func setOrDeleteInStreamMetadata(cfg *StreamConfig, prevCfg *StreamConfig, key string) { + if prevCfg != nil && prevCfg.Metadata != nil { + if value, ok := prevCfg.Metadata[key]; ok { + if cfg.Metadata == nil { + cfg.Metadata = make(map[string]string) + } + cfg.Metadata[key] = value + return + } + } + delete(cfg.Metadata, key) + if len(cfg.Metadata) == 0 { + cfg.Metadata = nil + } +} + +// setStaticConsumerMetadata sets JetStream consumer metadata, like the server version and API level. +// Any dynamic metadata is removed, it must not be stored and only be added for responses. +func setStaticConsumerMetadata(cfg *ConsumerConfig) { + if cfg.Metadata == nil { + cfg.Metadata = make(map[string]string) + } else { + deleteDynamicMetadata(cfg.Metadata) + } + + var requiredApiLevel int + requires := func(level int) { + if level > requiredApiLevel { + requiredApiLevel = level + } + } + + // Added in 2.11, absent | zero is the feature is not used. + // one could be stricter and say even if its set but the time + // has already passed it is also not needed to restore the consumer + if cfg.PauseUntil != nil && !cfg.PauseUntil.IsZero() { + requires(1) + } + + if cfg.PriorityPolicy != PriorityNone || cfg.PinnedTTL != 0 || len(cfg.PriorityGroups) > 0 { + requires(1) + } + + // Added in 2.14 + if cfg.AckPolicy == AckFlowControl { + requires(4) + } + + cfg.Metadata[JSRequiredLevelMetadataKey] = strconv.Itoa(requiredApiLevel) +} + +// setDynamicConsumerMetadata adds dynamic fields into the (copied) metadata. +func setDynamicConsumerMetadata(cfg *ConsumerConfig) *ConsumerConfig { + var newCfg ConsumerConfig + if cfg != nil { + newCfg = *cfg + } + newCfg.Metadata = make(map[string]string) + if cfg != nil { + for key, value := range cfg.Metadata { + newCfg.Metadata[key] = value + } + } + newCfg.Metadata[JSServerVersionMetadataKey] = VERSION + newCfg.Metadata[JSServerLevelMetadataKey] = strconv.Itoa(JSApiLevel) + return &newCfg +} + +// setDynamicConsumerInfoMetadata adds dynamic fields into the (copied) metadata. +func setDynamicConsumerInfoMetadata(info *ConsumerInfo) *ConsumerInfo { + if info == nil { + return nil + } + + newInfo := *info + cfg := setDynamicConsumerMetadata(info.Config) + newInfo.Config = cfg + return &newInfo +} + +// copyConsumerMetadata copies versioning fields from metadata of prevCfg into cfg. +// Removes versioning fields if no previous metadata, updates if set, and removes fields if it doesn't exist in prevCfg. +// Any dynamic metadata is removed, it must not be stored and only be added for responses. +// +// Note: useful when doing equality checks on cfg and prevCfg, but ignoring any versioning metadata differences. +func copyConsumerMetadata(cfg *ConsumerConfig, prevCfg *ConsumerConfig) { + if cfg.Metadata != nil { + deleteDynamicMetadata(cfg.Metadata) + } + setOrDeleteInConsumerMetadata(cfg, prevCfg, JSRequiredLevelMetadataKey) +} + +// setOrDeleteInConsumerMetadata sets field with key/value in metadata of cfg if set, deletes otherwise. +func setOrDeleteInConsumerMetadata(cfg *ConsumerConfig, prevCfg *ConsumerConfig, key string) { + if prevCfg != nil && prevCfg.Metadata != nil { + if value, ok := prevCfg.Metadata[key]; ok { + if cfg.Metadata == nil { + cfg.Metadata = make(map[string]string) + } + cfg.Metadata[key] = value + return + } + } + delete(cfg.Metadata, key) + if len(cfg.Metadata) == 0 { + cfg.Metadata = nil + } +} + +// deleteDynamicMetadata deletes dynamic fields from the metadata. +func deleteDynamicMetadata(metadata map[string]string) { + delete(metadata, JSServerVersionMetadataKey) + delete(metadata, JSServerLevelMetadataKey) +} + +// errorOnRequiredApiLevel returns whether a request should be rejected based on the JSRequiredApiLevel header. +func errorOnRequiredApiLevel(hdr []byte) bool { + reqApiLevel := sliceHeader(JSRequiredApiLevel, hdr) + if len(reqApiLevel) == 0 { + return false + } + minLevel, err := strconv.Atoi(string(reqApiLevel)) + return err != nil || JSApiLevel < minLevel +} diff --git a/server/jetstream_versioning_test.go b/server/jetstream_versioning_test.go new file mode 100644 index 0000000..904172b --- /dev/null +++ b/server/jetstream_versioning_test.go @@ -0,0 +1,734 @@ +// Copyright 2024-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !skip_js_tests + +package server + +import ( + "archive/tar" + "bytes" + "encoding/json" + "fmt" + "math" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/klauspost/compress/s2" + "github.com/nats-io/nats.go" +) + +func TestGetAndSupportsRequiredApiLevel(t *testing.T) { + require_Equal(t, getRequiredApiLevel(nil), _EMPTY_) + require_Equal(t, getRequiredApiLevel(map[string]string{}), _EMPTY_) + require_Equal(t, getRequiredApiLevel(map[string]string{JSRequiredLevelMetadataKey: "1"}), "1") + require_Equal(t, getRequiredApiLevel(map[string]string{JSRequiredLevelMetadataKey: "text"}), "text") + + require_True(t, supportsRequiredApiLevel(nil)) + require_True(t, supportsRequiredApiLevel(map[string]string{})) + require_True(t, supportsRequiredApiLevel(map[string]string{JSRequiredLevelMetadataKey: "1"})) + require_True(t, supportsRequiredApiLevel(map[string]string{JSRequiredLevelMetadataKey: strconv.Itoa(JSApiLevel)})) + require_False(t, supportsRequiredApiLevel(map[string]string{JSRequiredLevelMetadataKey: "text"})) +} + +func metadataAtLevel(featureLevel string) map[string]string { + return map[string]string{ + JSRequiredLevelMetadataKey: featureLevel, + } +} + +func metadataPrevious() map[string]string { + return map[string]string{ + JSRequiredLevelMetadataKey: "previous-level", + } +} + +func TestJetStreamSetStaticStreamMetadata(t *testing.T) { + for _, test := range []struct { + desc string + cfg *StreamConfig + expectedMetadata map[string]string + }{ + { + desc: "empty", + cfg: &StreamConfig{}, + expectedMetadata: metadataAtLevel("0"), + }, + { + desc: "overwrite-user-provided", + cfg: &StreamConfig{Metadata: metadataPrevious()}, + expectedMetadata: metadataAtLevel("0"), + }, + { + desc: "empty-prev-metadata/delete-user-provided", + cfg: &StreamConfig{Metadata: metadataPrevious()}, + expectedMetadata: metadataAtLevel("0"), + }, + { + desc: "AllowMsgTTL", + cfg: &StreamConfig{AllowMsgTTL: true}, + expectedMetadata: metadataAtLevel("1"), + }, + { + desc: "SubjectDeleteMarkerTTL", + cfg: &StreamConfig{SubjectDeleteMarkerTTL: time.Second}, + expectedMetadata: metadataAtLevel("1"), + }, + { + desc: "AllowMsgCounter", + cfg: &StreamConfig{AllowMsgCounter: true}, + expectedMetadata: metadataAtLevel("2"), + }, + { + desc: "AllowAtomicPublish", + cfg: &StreamConfig{AllowAtomicPublish: true}, + expectedMetadata: metadataAtLevel("2"), + }, + { + desc: "AllowMsgSchedules", + cfg: &StreamConfig{AllowMsgSchedules: true}, + expectedMetadata: metadataAtLevel("2"), + }, + { + desc: "AsyncPersistMode", + cfg: &StreamConfig{PersistMode: AsyncPersistMode}, + expectedMetadata: metadataAtLevel("2"), + }, + { + desc: "AllowBatchPublish", + cfg: &StreamConfig{AllowBatchPublish: true}, + expectedMetadata: metadataAtLevel("4"), + }, + } { + t.Run(test.desc, func(t *testing.T) { + setStaticStreamMetadata(test.cfg) + level := test.cfg.Metadata[JSRequiredLevelMetadataKey] + require_Equal(t, level, test.expectedMetadata[JSRequiredLevelMetadataKey]) + + // Ensure we up the server API level if we introduced a feature that requires it. + l, err := strconv.Atoi(level) + require_NoError(t, err) + require_True(t, l <= JSApiLevel) + }) + } +} + +func TestJetStreamSetStaticStreamMetadataRemoveDynamicFields(t *testing.T) { + dynamicMetadata := func() map[string]string { + return map[string]string{ + JSServerVersionMetadataKey: "dynamic-version", + JSServerLevelMetadataKey: "dynamic-version", + } + } + + cfg := StreamConfig{Metadata: dynamicMetadata()} + setStaticStreamMetadata(&cfg) + require_True(t, reflect.DeepEqual(cfg.Metadata, metadataAtLevel("0"))) +} + +func TestJetStreamSetDynamicStreamMetadata(t *testing.T) { + cfg := StreamConfig{Metadata: metadataAtLevel("0")} + newCfg := setDynamicStreamMetadata(&cfg) + + // Only new metadata must contain dynamic fields. + metadata := metadataAtLevel("0") + require_True(t, reflect.DeepEqual(cfg.Metadata, metadata)) + metadata[JSServerVersionMetadataKey] = VERSION + metadata[JSServerLevelMetadataKey] = strconv.Itoa(JSApiLevel) + require_True(t, reflect.DeepEqual(newCfg.Metadata, metadata)) +} + +func TestJetStreamCopyStreamMetadata(t *testing.T) { + for _, test := range []struct { + desc string + cfg *StreamConfig + prev *StreamConfig + }{ + { + desc: "no-previous-ignore", + cfg: &StreamConfig{Metadata: metadataAtLevel("-1")}, + prev: nil, + }, + { + desc: "nil-previous-metadata-ignore", + cfg: &StreamConfig{Metadata: metadataAtLevel("-1")}, + prev: &StreamConfig{Metadata: nil}, + }, + { + desc: "nil-current-metadata-ignore", + cfg: &StreamConfig{Metadata: nil}, + prev: &StreamConfig{Metadata: metadataPrevious()}, + }, + { + desc: "copy-previous", + cfg: &StreamConfig{Metadata: metadataAtLevel("-1")}, + prev: &StreamConfig{Metadata: metadataPrevious()}, + }, + { + desc: "delete-missing-fields", + cfg: &StreamConfig{Metadata: metadataAtLevel("-1")}, + prev: &StreamConfig{Metadata: make(map[string]string)}, + }, + } { + t.Run(test.desc, func(t *testing.T) { + copyStreamMetadata(test.cfg, test.prev) + + var expectedMetadata map[string]string + if test.prev != nil { + expectedMetadata = test.prev.Metadata + } + + value, ok := expectedMetadata[JSRequiredLevelMetadataKey] + if ok { + require_Equal(t, test.cfg.Metadata[JSRequiredLevelMetadataKey], value) + } else { + // Key shouldn't exist. + _, ok = test.cfg.Metadata[JSRequiredLevelMetadataKey] + require_False(t, ok) + } + }) + } +} + +func TestJetStreamCopyStreamMetadataRemoveDynamicFields(t *testing.T) { + dynamicMetadata := func() map[string]string { + return map[string]string{ + JSServerVersionMetadataKey: "dynamic-version", + JSServerLevelMetadataKey: "dynamic-version", + } + } + + cfg := StreamConfig{Metadata: dynamicMetadata()} + copyStreamMetadata(&cfg, nil) + require_Equal(t, len(cfg.Metadata), 0) + + cfg = StreamConfig{Metadata: dynamicMetadata()} + prevCfg := StreamConfig{Metadata: metadataAtLevel("0")} + copyStreamMetadata(&cfg, &prevCfg) + require_True(t, reflect.DeepEqual(cfg.Metadata, metadataAtLevel("0"))) +} + +func TestJetStreamSetStaticConsumerMetadata(t *testing.T) { + pauseUntil := time.Unix(0, 0) + pauseUntilZero := time.Time{} + for _, test := range []struct { + desc string + cfg *ConsumerConfig + expectedMetadata map[string]string + }{ + { + desc: "empty", + cfg: &ConsumerConfig{}, + expectedMetadata: metadataAtLevel("0"), + }, + { + desc: "overwrite-user-provided", + cfg: &ConsumerConfig{Metadata: metadataPrevious()}, + expectedMetadata: metadataAtLevel("0"), + }, + { + desc: "PauseUntil/zero", + cfg: &ConsumerConfig{PauseUntil: &pauseUntilZero}, + expectedMetadata: metadataAtLevel("0"), + }, + { + desc: "PauseUntil", + cfg: &ConsumerConfig{PauseUntil: &pauseUntil}, + expectedMetadata: metadataAtLevel("1"), + }, + { + desc: "Pinned", + cfg: &ConsumerConfig{PriorityPolicy: PriorityPinnedClient, PriorityGroups: []string{"a"}}, + expectedMetadata: metadataAtLevel("1"), + }, + { + desc: "AckFlowControl", + cfg: &ConsumerConfig{AckPolicy: AckFlowControl}, + expectedMetadata: metadataAtLevel("4"), + }, + } { + t.Run(test.desc, func(t *testing.T) { + setStaticConsumerMetadata(test.cfg) + level := test.cfg.Metadata[JSRequiredLevelMetadataKey] + require_Equal(t, level, test.expectedMetadata[JSRequiredLevelMetadataKey]) + + // Ensure we up the server API level if we introduced a feature that requires it. + l, err := strconv.Atoi(level) + require_NoError(t, err) + require_True(t, l <= JSApiLevel) + }) + } +} + +func TestJetStreamSetStaticConsumerMetadataRemoveDynamicFields(t *testing.T) { + dynamicMetadata := func() map[string]string { + return map[string]string{ + JSServerVersionMetadataKey: "dynamic-version", + JSServerLevelMetadataKey: "dynamic-version", + } + } + + cfg := ConsumerConfig{Metadata: dynamicMetadata()} + setStaticConsumerMetadata(&cfg) + require_True(t, reflect.DeepEqual(cfg.Metadata, metadataAtLevel("0"))) +} + +func TestJetStreamSetDynamicConsumerMetadata(t *testing.T) { + cfg := ConsumerConfig{Metadata: metadataAtLevel("0")} + newCfg := setDynamicConsumerMetadata(&cfg) + + // Only new metadata must contain dynamic fields. + metadata := metadataAtLevel("0") + require_True(t, reflect.DeepEqual(cfg.Metadata, metadata)) + metadata[JSServerVersionMetadataKey] = VERSION + metadata[JSServerLevelMetadataKey] = strconv.Itoa(JSApiLevel) + require_True(t, reflect.DeepEqual(newCfg.Metadata, metadata)) +} + +func TestJetStreamSetDynamicConsumerInfoMetadata(t *testing.T) { + ci := ConsumerInfo{Config: &ConsumerConfig{Metadata: metadataAtLevel("0")}} + newCi := setDynamicConsumerInfoMetadata(&ci) + + // Configs should not equal, as that would mean we've overwritten the original ConsumerInfo. + require_False(t, reflect.DeepEqual(ci, newCi)) + + // Only new metadata must contain dynamic fields. + metadata := metadataAtLevel("0") + require_True(t, reflect.DeepEqual(ci.Config.Metadata, metadata)) + metadata[JSServerVersionMetadataKey] = VERSION + metadata[JSServerLevelMetadataKey] = strconv.Itoa(JSApiLevel) + require_True(t, reflect.DeepEqual(newCi.Config.Metadata, metadata)) +} + +func TestJetStreamCopyConsumerMetadata(t *testing.T) { + for _, test := range []struct { + desc string + cfg *ConsumerConfig + prev *ConsumerConfig + }{ + { + desc: "no-previous-ignore", + cfg: &ConsumerConfig{Metadata: metadataAtLevel("-1")}, + prev: nil, + }, + { + desc: "nil-previous-metadata-ignore", + cfg: &ConsumerConfig{Metadata: metadataAtLevel("-1")}, + prev: &ConsumerConfig{Metadata: nil}, + }, + { + desc: "nil-current-metadata-ignore", + cfg: &ConsumerConfig{Metadata: nil}, + prev: &ConsumerConfig{Metadata: metadataPrevious()}, + }, + { + desc: "copy-previous", + cfg: &ConsumerConfig{Metadata: metadataAtLevel("-1")}, + prev: &ConsumerConfig{Metadata: metadataPrevious()}, + }, + { + desc: "delete-missing-fields", + cfg: &ConsumerConfig{Metadata: metadataAtLevel("-1")}, + prev: &ConsumerConfig{Metadata: make(map[string]string)}, + }, + } { + t.Run(test.desc, func(t *testing.T) { + copyConsumerMetadata(test.cfg, test.prev) + + var expectedMetadata map[string]string + if test.prev != nil { + expectedMetadata = test.prev.Metadata + } + + value, ok := expectedMetadata[JSRequiredLevelMetadataKey] + if ok { + require_Equal(t, test.cfg.Metadata[JSRequiredLevelMetadataKey], value) + } else { + // Key shouldn't exist. + _, ok = test.cfg.Metadata[JSRequiredLevelMetadataKey] + require_False(t, ok) + } + }) + } +} + +func TestJetStreamCopyConsumerMetadataRemoveDynamicFields(t *testing.T) { + dynamicMetadata := func() map[string]string { + return map[string]string{ + JSServerVersionMetadataKey: "dynamic-version", + JSServerLevelMetadataKey: "dynamic-version", + } + } + + cfg := ConsumerConfig{Metadata: dynamicMetadata()} + copyConsumerMetadata(&cfg, nil) + require_Equal(t, len(cfg.Metadata), 0) + + cfg = ConsumerConfig{Metadata: dynamicMetadata()} + prevCfg := ConsumerConfig{Metadata: metadataAtLevel("0")} + copyConsumerMetadata(&cfg, &prevCfg) + require_True(t, reflect.DeepEqual(cfg.Metadata, metadataAtLevel("0"))) +} + +type server struct { + replicas int + js nats.JetStreamContext + nc *nats.Conn +} + +const ( + streamName = "STREAM" + consumerName = "CONSUMER" +) + +func TestJetStreamMetadataMutations(t *testing.T) { + single := RunBasicJetStreamServer(t) + defer single.Shutdown() + nc, js := jsClientConnect(t, single) + defer nc.Close() + + cluster := createJetStreamClusterExplicit(t, "R3S", 3) + defer cluster.shutdown() + cnc, cjs := jsClientConnect(t, cluster.randomServer()) + defer cnc.Close() + + // Test for both single server and clustered mode. + for _, s := range []server{ + {1, js, nc}, + {3, cjs, cnc}, + } { + t.Run(fmt.Sprintf("R%d", s.replicas), func(t *testing.T) { + streamMetadataChecks(t, s) + consumerMetadataChecks(t, s) + }) + } +} + +func validateMetadata(metadata map[string]string, expectedFeatureLevel string) bool { + return metadata[JSRequiredLevelMetadataKey] == expectedFeatureLevel || + metadata[JSServerVersionMetadataKey] == VERSION || + metadata[JSServerLevelMetadataKey] == strconv.Itoa(JSApiLevel) +} + +func streamMetadataChecks(t *testing.T, s server) { + // Add stream. + sc := nats.StreamConfig{Name: streamName, Replicas: s.replicas} + si, err := s.js.AddStream(&sc) + require_NoError(t, err) + require_True(t, validateMetadata(si.Config.Metadata, "0")) + + // (Double) add stream, has different code path for clustered streams. + sc = nats.StreamConfig{Name: streamName, Replicas: s.replicas} + si, err = s.js.AddStream(&sc) + require_NoError(t, err) + require_True(t, validateMetadata(si.Config.Metadata, "0")) + + // Stream info. + si, err = s.js.StreamInfo(streamName) + require_NoError(t, err) + require_True(t, validateMetadata(si.Config.Metadata, "0")) + + // Update stream. + // Metadata set on creation should be preserved, even if not included in update. + si, err = s.js.UpdateStream(&sc) + require_NoError(t, err) + require_True(t, validateMetadata(si.Config.Metadata, "0")) +} + +func consumerMetadataChecks(t *testing.T, s server) { + // Add consumer. + cc := nats.ConsumerConfig{Name: consumerName, Replicas: s.replicas} + ci, err := s.js.AddConsumer(streamName, &cc) + require_NoError(t, err) + require_True(t, validateMetadata(ci.Config.Metadata, "0")) + + // Consumer info. + ci, err = s.js.ConsumerInfo(streamName, consumerName) + require_NoError(t, err) + require_True(t, validateMetadata(ci.Config.Metadata, "0")) + + // Update consumer. + // Metadata set on creation should be preserved, even if not included in update. + ci, err = s.js.UpdateConsumer(streamName, &cc) + require_NoError(t, err) + require_True(t, validateMetadata(ci.Config.Metadata, "0")) + + // Use pause advisories to know when pause/resume is applied. + pauseCh := make(chan *nats.Msg, 10) + _, err = s.nc.ChanSubscribe(JSAdvisoryConsumerPausePre+".STREAM.CONSUMER", pauseCh) + require_NoError(t, err) + + // Pause consumer, should up required API level. + jsTestPause_PauseConsumer(t, s.nc, streamName, consumerName, time.Now().Add(time.Second*3)) + require_ChanRead(t, pauseCh, time.Second*2) + require_Len(t, len(pauseCh), 0) + + ci, err = s.js.ConsumerInfo(streamName, consumerName) + require_NoError(t, err) + require_True(t, validateMetadata(ci.Config.Metadata, "1")) + + // Unpause consumer, should lower required API level. + subj := fmt.Sprintf("$JS.API.CONSUMER.PAUSE.%s.%s", streamName, consumerName) + _, err = s.nc.Request(subj, nil, time.Second) + require_NoError(t, err) + require_ChanRead(t, pauseCh, time.Second*2) + require_Len(t, len(pauseCh), 0) + + ci, err = s.js.ConsumerInfo(streamName, consumerName) + require_NoError(t, err) + require_True(t, validateMetadata(ci.Config.Metadata, "0")) + + // Test scaling up/down. + if s.replicas == 3 { + // Scale down. + cc.Replicas = 1 + ci, err = s.js.UpdateConsumer(streamName, &cc) + require_NoError(t, err) + require_True(t, validateMetadata(ci.Config.Metadata, "0")) + + ci, err = s.js.ConsumerInfo(streamName, consumerName) + require_NoError(t, err) + require_True(t, validateMetadata(ci.Config.Metadata, "0")) + + // Scale up. + cc.Replicas = 3 + ci, err = s.js.UpdateConsumer(streamName, &cc) + require_NoError(t, err) + require_True(t, validateMetadata(ci.Config.Metadata, "0")) + + ci, err = s.js.ConsumerInfo(streamName, consumerName) + require_NoError(t, err) + require_True(t, validateMetadata(ci.Config.Metadata, "0")) + } +} + +func TestJetStreamMetadataStreamRestoreAndRestart(t *testing.T) { + s := RunBasicJetStreamServer(t) + defer s.Shutdown() + nc, js := jsClientConnect(t, s) + defer nc.Close() + + restoreEmptyStream(t, nc, 1) + + expectedMetadata := map[string]string{ + JSServerVersionMetadataKey: VERSION, + JSServerLevelMetadataKey: strconv.Itoa(JSApiLevel), + } + + // Stream restore should result in empty metadata to be preserved, only adding dynamic metadata. + si, err := js.StreamInfo(streamName) + require_NoError(t, err) + require_True(t, reflect.DeepEqual(si.Config.Metadata, expectedMetadata)) + + // Restart server. + port := s.opts.Port + sd := s.StoreDir() + nc.Close() + s.Shutdown() + s = RunJetStreamServerOnPort(port, sd) + defer s.Shutdown() + nc, js = jsClientConnect(t, s) + defer nc.Close() + + // After restart (or upgrade) metadata data should remain empty, only adding dynamic metadata. + si, err = js.StreamInfo(streamName) + require_NoError(t, err) + require_True(t, reflect.DeepEqual(si.Config.Metadata, expectedMetadata)) +} + +func TestJetStreamMetadataStreamRestoreAndRestartCluster(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + restoreEmptyStream(t, nc, 3) + + expectedMetadata := map[string]string{ + JSServerVersionMetadataKey: VERSION, + JSServerLevelMetadataKey: strconv.Itoa(JSApiLevel), + } + + // Stream restore should result in empty metadata to be preserved, only adding dynamic metadata. + si, err := js.StreamInfo(streamName) + require_NoError(t, err) + require_True(t, reflect.DeepEqual(si.Config.Metadata, expectedMetadata)) + + // Restart cluster. + c.stopAll() + c.restartAllSamePorts() + defer c.shutdown() + c.waitOnAllCurrent() + c.waitOnStreamLeader("$G", streamName) + nc, js = jsClientConnect(t, c.randomServer()) + defer nc.Close() + + // After restart (or upgrade) metadata data should remain empty, only adding dynamic metadata. + si, err = js.StreamInfo(streamName) + require_NoError(t, err) + require_True(t, reflect.DeepEqual(si.Config.Metadata, expectedMetadata)) +} + +func restoreEmptyStream(t *testing.T, nc *nats.Conn, replicas int) { + rreq := JSApiStreamRestoreRequest{ + Config: StreamConfig{ + Name: "STREAM", + Retention: LimitsPolicy, + Storage: FileStorage, + Replicas: replicas, + }, + } + buf, err := json.Marshal(rreq) + require_NoError(t, err) + + var rresp JSApiStreamRestoreResponse + msg, err := nc.Request(fmt.Sprintf(JSApiStreamRestoreT, rreq.Config.Name), buf, 5*time.Second) + require_NoError(t, err) + json.Unmarshal(msg.Data, &rresp) + if rresp.Error != nil { + t.Fatalf("Error on restore: %+v", rresp.Error) + } + + // Construct empty stream.tar.s2 (only containing meta.inf). + fsi := FileStreamInfo{StreamConfig: rreq.Config} + fsij, err := json.Marshal(fsi) + require_NoError(t, err) + + hdr := &tar.Header{ + Name: JetStreamMetaFile, + Mode: 0600, + Uname: "nats", + Gname: "nats", + Size: int64(len(fsij)), + Format: tar.FormatPAX, + } + var buffer bytes.Buffer + enc := s2.NewWriter(&buffer) + tw := tar.NewWriter(enc) + err = tw.WriteHeader(hdr) + require_NoError(t, err) + _, err = tw.Write(fsij) + require_NoError(t, err) + err = tw.Close() + require_NoError(t, err) + err = enc.Close() + require_NoError(t, err) + + data := buffer.Bytes() + msg, err = nc.Request(rresp.DeliverSubject, data, 5*time.Second) + require_NoError(t, err) + json.Unmarshal(msg.Data, &rresp) + if rresp.Error != nil { + t.Fatalf("Error on restore: %+v", rresp.Error) + } + + msg, err = nc.Request(rresp.DeliverSubject, nil, 5*time.Second) + require_NoError(t, err) + + expectedMetadata := map[string]string{ + JSServerVersionMetadataKey: VERSION, + JSServerLevelMetadataKey: strconv.Itoa(JSApiLevel), + } + + var cresp JSApiStreamCreateResponse + err = json.Unmarshal(msg.Data, &cresp) + require_NoError(t, err) + require_True(t, reflect.DeepEqual(cresp.Config.Metadata, expectedMetadata)) +} + +func TestJetStreamApiErrorOnRequiredApiLevel(t *testing.T) { + s := RunBasicJetStreamServer(t) + defer s.Shutdown() + nc := clientConnectToServer(t, s) + defer nc.Close() + + var subs []*subscription + s.getJetStream().apiSubs.All(&subs) + require_True(t, len(subs) > 0) + for _, sub := range subs { + apiSubject := string(sub.subject) + t.Run(apiSubject, func(t *testing.T) { + req := nats.NewMsg(apiSubject) + req.Header.Set("Nats-Required-Api-Level", strconv.Itoa(math.MaxInt)) + msg, err := nc.RequestMsg(req, time.Second) + require_NoError(t, err) + + var resp ApiResponse + require_NoError(t, json.Unmarshal(msg.Data, &resp)) + require_True(t, resp.Error != nil) + // Peer remove, stepdown or cancel move is not supported if not clustered. + if strings.Contains(apiSubject, ".STEPDOWN.") || + strings.Contains(apiSubject, ".PEER.") || + strings.Contains(apiSubject, ".CANCEL_MOVE.") { + require_Error(t, resp.Error, NewJSClusterRequiredError()) + } else { + require_Error(t, resp.Error, NewJSRequiredApiLevelError()) + } + }) + } +} + +func TestJetStreamApiErrorOnRequiredApiLevelDirectGet(t *testing.T) { + s := RunBasicJetStreamServer(t) + defer s.Shutdown() + nc, js := jsClientConnect(t, s) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo"}, + AllowDirect: true, + }) + require_NoError(t, err) + + req := nats.NewMsg(fmt.Sprintf(JSDirectMsgGetT, "TEST")) + req.Header.Set("Nats-Required-Api-Level", strconv.Itoa(math.MaxInt)) + msg, err := nc.RequestMsg(req, time.Second) + require_NoError(t, err) + require_Equal(t, msg.Header.Get("Status"), "412") + require_Equal(t, msg.Header.Get("Description"), "Required Api Level") + + req = nats.NewMsg(fmt.Sprintf(JSDirectGetLastBySubjectT, "TEST", "foo")) + req.Header.Set("Nats-Required-Api-Level", strconv.Itoa(math.MaxInt)) + msg, err = nc.RequestMsg(req, time.Second) + require_NoError(t, err) + require_Equal(t, msg.Header.Get("Status"), "412") + require_Equal(t, msg.Header.Get("Description"), "Required Api Level") +} + +func TestJetStreamApiErrorOnRequiredApiLevelPullConsumerNextMsg(t *testing.T) { + s := RunBasicJetStreamServer(t) + defer s.Shutdown() + nc, js := jsClientConnect(t, s) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo"}, + AllowDirect: true, + }) + require_NoError(t, err) + + _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "CONSUMER"}) + require_NoError(t, err) + + req := nats.NewMsg(fmt.Sprintf(JSApiRequestNextT, "TEST", "CONSUMER")) + req.Header.Set("Nats-Required-Api-Level", strconv.Itoa(math.MaxInt)) + msg, err := nc.RequestMsg(req, time.Second) + require_NoError(t, err) + require_Equal(t, msg.Header.Get("Status"), "412") + require_Equal(t, msg.Header.Get("Description"), "Required Api Level") +} diff --git a/server/jwt.go b/server/jwt.go new file mode 100644 index 0000000..7e83235 --- /dev/null +++ b/server/jwt.go @@ -0,0 +1,275 @@ +// Copyright 2018-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "errors" + "fmt" + "net" + "os" + "strings" + "time" + + "github.com/nats-io/jwt/v2" + "github.com/nats-io/nkeys" +) + +// All JWTs once encoded start with this +const jwtPrefix = "eyJ" + +// ReadOperatorJWT will read a jwt file for an operator claim. This can be a decorated file. +func ReadOperatorJWT(jwtfile string) (*jwt.OperatorClaims, error) { + _, claim, err := readOperatorJWT(jwtfile) + return claim, err +} + +func readOperatorJWT(jwtfile string) (string, *jwt.OperatorClaims, error) { + contents, err := os.ReadFile(jwtfile) + if err != nil { + // Check to see if the JWT has been inlined. + if !strings.HasPrefix(jwtfile, jwtPrefix) { + return "", nil, err + } + // We may have an inline jwt here. + contents = []byte(jwtfile) + } + defer wipeSlice(contents) + + theJWT, err := jwt.ParseDecoratedJWT(contents) + if err != nil { + return "", nil, err + } + opc, err := jwt.DecodeOperatorClaims(theJWT) + if err != nil { + return "", nil, err + } + return theJWT, opc, nil +} + +// Just wipe slice with 'x', for clearing contents of nkey seed file. +func wipeSlice(buf []byte) { + for i := range buf { + buf[i] = 'x' + } +} + +// validateTrustedOperators will check that we do not have conflicts with +// assigned trusted keys and trusted operators. If operators are defined we +// will expand the trusted keys in options. +func validateTrustedOperators(o *Options) error { + if len(o.TrustedOperators) == 0 { + // if we have no operator, default sentinel shouldn't be set + if o.DefaultSentinel != _EMPTY_ { + return fmt.Errorf("default sentinel requires operators and accounts") + } + return nil + } + if o.DefaultSentinel != _EMPTY_ { + juc, err := jwt.DecodeUserClaims(o.DefaultSentinel) + if err != nil { + return fmt.Errorf("default sentinel JWT not valid") + } + + if !juc.BearerToken && juc.IssuerAccount != "" && juc.HasEmptyPermissions() { + // we cannot resolve the account yet - but this looks like a scoped user + // it will be rejected at runtime if not valid + } else if !juc.BearerToken { + return fmt.Errorf("default sentinel must be a bearer token") + } + } + if o.AccountResolver == nil { + return fmt.Errorf("operators require an account resolver to be configured") + } + if len(o.Accounts) > 0 { + return fmt.Errorf("operators do not allow Accounts to be configured directly") + } + if len(o.Users) > 0 || len(o.Nkeys) > 0 { + return fmt.Errorf("operators do not allow users to be configured directly") + } + if len(o.TrustedOperators) > 0 && len(o.TrustedKeys) > 0 { + return fmt.Errorf("conflicting options for 'TrustedKeys' and 'TrustedOperators'") + } + if o.SystemAccount != _EMPTY_ { + foundSys := false + foundNonEmpty := false + for _, op := range o.TrustedOperators { + if op.SystemAccount != _EMPTY_ { + foundNonEmpty = true + } + if op.SystemAccount == o.SystemAccount { + foundSys = true + break + } + } + if foundNonEmpty && !foundSys { + return fmt.Errorf("system_account in config and operator JWT must be identical") + } + } else if o.TrustedOperators[0].SystemAccount == _EMPTY_ { + // In case the system account is neither defined in config nor in the first operator. + // If it would be needed due to the nats account resolver, raise an error. + switch o.AccountResolver.(type) { + case *DirAccResolver, *CacheDirAccResolver: + return fmt.Errorf("using nats based account resolver - the system account needs to be specified in configuration or the operator jwt") + } + } + + srvMajor, srvMinor, srvUpdate, _ := versionComponents(VERSION) + for _, opc := range o.TrustedOperators { + if major, minor, update, err := jwt.ParseServerVersion(opc.AssertServerVersion); err != nil { + return fmt.Errorf("operator %s expects version %s got error instead: %s", + opc.Subject, opc.AssertServerVersion, err) + } else if major > srvMajor { + return fmt.Errorf("operator %s expected major version %d > server major version %d", + opc.Subject, major, srvMajor) + } else if srvMajor > major { + } else if minor > srvMinor { + return fmt.Errorf("operator %s expected minor version %d > server minor version %d", + opc.Subject, minor, srvMinor) + } else if srvMinor > minor { + } else if update > srvUpdate { + return fmt.Errorf("operator %s expected update version %d > server update version %d", + opc.Subject, update, srvUpdate) + } + } + // If we have operators, fill in the trusted keys. + // FIXME(dlc) - We had TrustedKeys before TrustedOperators. The jwt.OperatorClaims + // has a DidSign(). Use that longer term. For now we can expand in place. + for _, opc := range o.TrustedOperators { + if o.TrustedKeys == nil { + o.TrustedKeys = make([]string, 0, 4) + } + if !opc.StrictSigningKeyUsage { + o.TrustedKeys = append(o.TrustedKeys, opc.Subject) + } + o.TrustedKeys = append(o.TrustedKeys, opc.SigningKeys...) + } + for _, key := range o.TrustedKeys { + if !nkeys.IsValidPublicOperatorKey(key) { + return fmt.Errorf("trusted Keys %q are required to be a valid public operator nkey", key) + } + } + if len(o.resolverPinnedAccounts) > 0 { + for key := range o.resolverPinnedAccounts { + if !nkeys.IsValidPublicAccountKey(key) { + return fmt.Errorf("pinned account key %q is not a valid public account nkey", key) + } + } + // ensure the system account (belonging to the operator can always connect) + if o.SystemAccount != _EMPTY_ { + o.resolverPinnedAccounts[o.SystemAccount] = struct{}{} + } + } + + // If we have an auth callout defined make sure we are not in operator mode. + if o.AuthCallout != nil { + return errors.New("operators do not allow authorization callouts to be configured directly") + } + + return nil +} + +func validateSrc(claims *jwt.UserClaims, host string) bool { + if claims == nil { + return false + } else if len(claims.Src) == 0 { + return true + } else if host == "" { + return false + } + ip := net.ParseIP(host) + if ip == nil { + return false + } + for _, cidr := range claims.Src { + if _, net, err := net.ParseCIDR(cidr); err != nil { + return false // should not happen as this jwt is invalid + } else if net.Contains(ip) { + return true + } + } + return false +} + +func validateTimes(claims *jwt.UserClaims) (bool, time.Duration) { + return validateTimesAt(claims, time.Now()) +} + +func validateTimesAt(claims *jwt.UserClaims, now time.Time) (bool, time.Duration) { + if claims == nil { + return false, time.Duration(0) + } else if len(claims.Times) == 0 { + return true, time.Duration(0) + } + loc := time.Local + if claims.Locale != "" { + var err error + if loc, err = time.LoadLocation(claims.Locale); err != nil { + return false, time.Duration(0) // parsing not expected to fail at this point + } + now = now.In(loc) + } + + var ok bool + var validFor time.Duration + + for _, timeRange := range claims.Times { + start, err := time.ParseInLocation("15:04:05", timeRange.Start, loc) + if err != nil { + return false, time.Duration(0) // parsing not expected to fail at this point + } + end, err := time.ParseInLocation("15:04:05", timeRange.End, loc) + if err != nil { + return false, time.Duration(0) // parsing not expected to fail at this point + } + + y, m, d := now.Date() + start = time.Date(y, m, d, start.Hour(), start.Minute(), start.Second(), 0, loc) + end = time.Date(y, m, d, end.Hour(), end.Minute(), end.Second(), 0, loc) + + inRange, expires := validateTimeRangeAt(start, end, now) + if inRange && (!ok || expires > validFor) { + ok = true + validFor = expires + } + } + return ok, validFor +} + +// Returns true if now is within `start` and `end`, and +// how much time is left until `end`. +// False if `now` is not within range. +func validateTimeRangeAt(start, end, now time.Time) (bool, time.Duration) { + // Now falls within range. + // For example 11:00-22:00 at 13:00 + if start.Before(now) && end.After(now) { + return true, end.Sub(now) + } + + // Range crosses midnight. + if start.After(end) { + // Now is after midnight. + // For example 22:00-06:00 at 05:00. + if end.After(now) { + return true, end.Sub(now) + } + + // Now is before midnight. + // For example 22:00-06:00 at 23:30. + end = end.AddDate(0, 0, 1) + if start.Before(now) && end.After(now) { + return true, end.Sub(now) + } + } + return false, time.Duration(0) +} diff --git a/server/leafnode.go b/server/leafnode.go new file mode 100644 index 0000000..f495885 --- /dev/null +++ b/server/leafnode.go @@ -0,0 +1,3825 @@ +// Copyright 2019-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bufio" + "bytes" + "crypto/tls" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "math/rand/v2" + "net" + "net/http" + "net/url" + "os" + "path" + "regexp" + "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/klauspost/compress/s2" + "github.com/nats-io/jwt/v2" + "github.com/nats-io/nkeys" + "github.com/nats-io/nuid" +) + +const ( + // Warning when user configures leafnode TLS insecure + leafnodeTLSInsecureWarning = "TLS certificate chain and hostname of solicited leafnodes will not be verified. DO NOT USE IN PRODUCTION!" + + // When a loop is detected, delay the reconnect of solicited connection. + leafNodeReconnectDelayAfterLoopDetected = 30 * time.Second + + // When a server receives a message causing a permission violation, the + // connection is closed and it won't attempt to reconnect for that long. + leafNodeReconnectAfterPermViolation = 30 * time.Second + + // When we have the same cluster name as the hub. + leafNodeReconnectDelayAfterClusterNameSame = 30 * time.Second + + // Prefix for loop detection subject + leafNodeLoopDetectionSubjectPrefix = "$LDS." + + // Path added to URL to indicate to WS server that the connection is a + // LEAF connection as opposed to a CLIENT. + leafNodeWSPath = "/leafnode" + + // When a soliciting leafnode is rejected because it does not meet the + // configured minimum version, delay the next reconnect attempt by this long. + leafNodeMinVersionReconnectDelay = 5 * time.Second +) + +type leaf struct { + // We have any auth stuff here for solicited connections. + remote *leafNodeCfg + // isSpoke tells us what role we are playing. + // Used when we receive a connection but otherside tells us they are a hub. + isSpoke bool + // remoteCluster is when we are a hub but the spoke leafnode is part of a cluster. + remoteCluster string + // remoteServer holds onto the remote server's name or ID. + remoteServer string + // domain name of remote server + remoteDomain string + // account name of remote server + remoteAccName string + // Whether or not we want to propagate east-west interest from other LNs. + isolated bool + // Used to suppress sub and unsub interest. Same as routes but our audience + // here is tied to this leaf node. This will hold all subscriptions except this + // leaf nodes. This represents all the interest we want to send to the other side. + smap map[string]int32 + // This map will contain all the subscriptions that have been added to the smap + // during initLeafNodeSmapAndSendSubs. It is short lived and is there to avoid + // race between processing of a sub where sub is added to account sublist but + // updateSmap has not be called on that "thread", while in the LN readloop, + // when processing CONNECT, initLeafNodeSmapAndSendSubs is invoked and add + // this subscription to smap. When processing of the sub then calls updateSmap, + // we would add it a second time in the smap causing later unsub to suppress the LS-. + tsub map[*subscription]struct{} + tsubt *time.Timer + // Selected compression mode, which may be different from the server configured mode. + compression string + // This is for GW map replies. + gwSub *subscription +} + +// Used for remote (solicited) leafnodes. +type leafNodeCfg struct { + sync.RWMutex + *RemoteLeafOpts + urls []*url.URL + curURL *url.URL + tlsName string + username string + password string + perms *Permissions + connDelay time.Duration // Delay before a connect, could be used while detecting loop condition, etc.. + jsMigrateTimer *time.Timer + quitCh chan struct{} + removed bool + connInProgress bool +} + +// Check to see if this is a solicited leafnode. We do special processing for solicited. +func (c *client) isSolicitedLeafNode() bool { + return c.kind == LEAF && c.leaf != nil && c.leaf.remote != nil +} + +// Returns true if this is a solicited leafnode and is not configured to be treated as a hub or a receiving +// connection leafnode where the otherside has declared itself to be the hub. +func (c *client) isSpokeLeafNode() bool { + return c.kind == LEAF && c.leaf != nil && c.leaf.isSpoke +} + +func (c *client) isHubLeafNode() bool { + return c.kind == LEAF && c.leaf != nil && !c.leaf.isSpoke +} + +func (c *client) isIsolatedLeafNode() bool { + // TODO(nat): In future we may want to pass in and consider an isolation + // group name here, which the hub and/or leaf could provide, so that we + // can isolate away certain LNs but not others on an opt-in basis. For + // now we will just isolate all LN interest until then. + return c.kind == LEAF && c.leaf != nil && c.leaf.isolated +} + +// This will spin up go routines to solicit the remote leaf node connections. +func (s *Server) solicitLeafNodeRemotes(remotes []*RemoteLeafOpts) { + sysAccName := _EMPTY_ + sAcc := s.SystemAccount() + if sAcc != nil { + sysAccName = sAcc.Name + } + addRemote := func(r *RemoteLeafOpts, isSysAccRemote bool) *leafNodeCfg { + s.mu.Lock() + remote := newLeafNodeCfg(r) + creds := remote.Credentials + accName := remote.LocalAccount + if s.leafRemoteCfgs == nil { + s.leafRemoteCfgs = make(map[*leafNodeCfg]struct{}) + } + s.leafRemoteCfgs[remote] = struct{}{} + // Print notice if + if isSysAccRemote { + if len(remote.DenyExports) > 0 { + s.Noticef("Remote for System Account uses restricted export permissions") + } + if len(remote.DenyImports) > 0 { + s.Noticef("Remote for System Account uses restricted import permissions") + } + } + s.mu.Unlock() + if creds != _EMPTY_ { + contents, err := os.ReadFile(creds) + defer wipeSlice(contents) + if err != nil { + s.Errorf("Error reading LeafNode Remote Credentials file %q: %v", creds, err) + } else if items := credsRe.FindAllSubmatch(contents, -1); len(items) < 2 { + s.Errorf("LeafNode Remote Credentials file %q malformed", creds) + } else if _, err := nkeys.FromSeed(items[1][1]); err != nil { + s.Errorf("LeafNode Remote Credentials file %q has malformed seed", creds) + } else if uc, err := jwt.DecodeUserClaims(string(items[0][1])); err != nil { + s.Errorf("LeafNode Remote Credentials file %q has malformed user jwt", creds) + } else if isSysAccRemote { + if !uc.Permissions.Pub.Empty() || !uc.Permissions.Sub.Empty() || uc.Permissions.Resp != nil { + s.Noticef("LeafNode Remote for System Account uses credentials file %q with restricted permissions", creds) + } + } else { + if !uc.Permissions.Pub.Empty() || !uc.Permissions.Sub.Empty() || uc.Permissions.Resp != nil { + s.Noticef("LeafNode Remote for Account %s uses credentials file %q with restricted permissions", accName, creds) + } + } + } + return remote + } + for _, r := range remotes { + // We need to call this, even if the leaf is disabled. This is so that + // the number of internal configuration matches the options' remote leaf + // configuration required for configuration reload. + remote := addRemote(r, r.LocalAccount == sysAccName) + if !r.Disabled { + s.connectToRemoteLeafNodeAsynchronously(remote, true) + } + } +} + +// Ensure that leafnode is properly configured. +func validateLeafNode(o *Options) error { + if err := validateLeafNodeAuthOptions(o); err != nil { + return err + } + + if len(o.LeafNode.Remotes) > 0 { + names := make(map[string]struct{}) + // Check for duplicate remotes, also, users can bind to any local account, + // if its empty we will assume the $G account. + for _, r := range o.LeafNode.Remotes { + if r.LocalAccount == _EMPTY_ { + r.LocalAccount = globalAccountName + } + if err := checkPermSubjectArray(r.DenyImports, false); err != nil { + return fmt.Errorf("invalid deny_imports for remote %s: %w", r.safeName(), err) + } + if err := checkPermSubjectArray(r.DenyExports, false); err != nil { + return fmt.Errorf("invalid deny_exports for remote %s: %w", r.safeName(), err) + } + rn := r.name() + if _, dup := names[rn]; dup { + return fmt.Errorf("duplicate remote %s", r.safeName()) + } + names[rn] = struct{}{} + } + } + + // In local config mode, check that leafnode configuration refers to accounts that exist. + if len(o.TrustedOperators) == 0 { + accNames := map[string]struct{}{} + for _, a := range o.Accounts { + accNames[a.Name] = struct{}{} + } + // global account is always created + accNames[DEFAULT_GLOBAL_ACCOUNT] = struct{}{} + // in the context of leaf nodes, empty account means global account + accNames[_EMPTY_] = struct{}{} + // system account either exists or, if not disabled, will be created + if o.SystemAccount == _EMPTY_ && !o.NoSystemAccount { + accNames[DEFAULT_SYSTEM_ACCOUNT] = struct{}{} + } + checkAccountExists := func(accName string, cfgType string) error { + if _, ok := accNames[accName]; !ok { + return fmt.Errorf("cannot find local account %q specified in leafnode %s", accName, cfgType) + } + return nil + } + if err := checkAccountExists(o.LeafNode.Account, "authorization"); err != nil { + return err + } + for _, lu := range o.LeafNode.Users { + if lu.Account == nil { // means global account + continue + } + if err := checkAccountExists(lu.Account.Name, "authorization"); err != nil { + return err + } + } + for _, r := range o.LeafNode.Remotes { + if err := checkAccountExists(r.LocalAccount, "remote"); err != nil { + return err + } + } + } else { + if len(o.LeafNode.Users) != 0 { + return fmt.Errorf("operator mode does not allow specifying users in leafnode config") + } + for _, r := range o.LeafNode.Remotes { + if !nkeys.IsValidPublicAccountKey(r.LocalAccount) { + return fmt.Errorf( + "operator mode requires account nkeys in remotes. " + + "Please add an `account` key to each remote in your `leafnodes` section, to assign it to an account. " + + "Each account value should be a 56 character public key, starting with the letter 'A'") + } + } + if o.LeafNode.Port != 0 && o.LeafNode.Account != "" && !nkeys.IsValidPublicAccountKey(o.LeafNode.Account) { + return fmt.Errorf("operator mode and non account nkeys are incompatible") + } + } + + // Validate compression settings + if o.LeafNode.Compression.Mode != _EMPTY_ { + if err := validateAndNormalizeCompressionOption(&o.LeafNode.Compression, CompressionS2Auto); err != nil { + return err + } + } + + // If a remote has a websocket scheme, all need to have it. + for _, rcfg := range o.LeafNode.Remotes { + // Validate proxy configuration + if _, err := validateLeafNodeProxyOptions(rcfg); err != nil { + return err + } + + if len(rcfg.URLs) >= 2 { + firstIsWS, ok := isWSURL(rcfg.URLs[0]), true + for i := 1; i < len(rcfg.URLs); i++ { + u := rcfg.URLs[i] + if isWS := isWSURL(u); isWS && !firstIsWS || !isWS && firstIsWS { + ok = false + break + } + } + if !ok { + return fmt.Errorf("remote leaf node configuration cannot have a mix of websocket and non-websocket urls: %q", redactURLList(rcfg.URLs)) + } + } + // Validate compression settings + if rcfg.Compression.Mode != _EMPTY_ { + if err := validateAndNormalizeCompressionOption(&rcfg.Compression, CompressionS2Auto); err != nil { + return err + } + } + } + + if o.LeafNode.Port == 0 { + return nil + } + + // If MinVersion is defined, check that it is valid. + if mv := o.LeafNode.MinVersion; mv != _EMPTY_ { + if err := checkLeafMinVersionConfig(mv); err != nil { + return err + } + } + + // The checks below will be done only when detecting that we are configured + // with gateways. So if an option validation needs to be done regardless, + // it MUST be done before this point! + + if o.Gateway.Name == _EMPTY_ && o.Gateway.Port == 0 { + return nil + } + // If we are here we have both leaf nodes and gateways defined, make sure there + // is a system account defined. + if o.SystemAccount == _EMPTY_ { + return fmt.Errorf("leaf nodes and gateways (both being defined) require a system account to also be configured") + } + if err := validatePinnedCerts(o.LeafNode.TLSPinnedCerts); err != nil { + return fmt.Errorf("leafnode: %v", err) + } + return nil +} + +func checkLeafMinVersionConfig(mv string) error { + if ok, err := versionAtLeastCheckError(mv, 2, 8, 0); !ok || err != nil { + if err != nil { + return fmt.Errorf("invalid leafnode's minimum version: %v", err) + } else { + return fmt.Errorf("the minimum version should be at least 2.8.0") + } + } + return nil +} + +// Used to validate user names in LeafNode configuration. +// - rejects mix of single and multiple users. +// - rejects duplicate user names. +func validateLeafNodeAuthOptions(o *Options) error { + if len(o.LeafNode.Users) == 0 { + return nil + } + if o.LeafNode.Username != _EMPTY_ { + return fmt.Errorf("can not have a single user/pass and a users array") + } + if o.LeafNode.Nkey != _EMPTY_ { + return fmt.Errorf("can not have a single nkey and a users array") + } + users := map[string]struct{}{} + for _, u := range o.LeafNode.Users { + if _, exists := users[u.Username]; exists { + return fmt.Errorf("duplicate user %q detected in leafnode authorization", u.Username) + } + users[u.Username] = struct{}{} + } + return nil +} + +func validateLeafNodeProxyOptions(remote *RemoteLeafOpts) ([]string, error) { + var warnings []string + + if remote.Proxy.URL == _EMPTY_ { + return warnings, nil + } + + proxyURL, err := url.Parse(remote.Proxy.URL) + if err != nil { + return warnings, fmt.Errorf("invalid proxy URL: %v", err) + } + + if proxyURL.Scheme != "http" && proxyURL.Scheme != "https" { + return warnings, fmt.Errorf("proxy URL scheme must be http or https, got: %s", proxyURL.Scheme) + } + + if proxyURL.Host == _EMPTY_ { + return warnings, fmt.Errorf("proxy URL must specify a host") + } + + if remote.Proxy.Timeout < 0 { + return warnings, fmt.Errorf("proxy timeout must be >= 0") + } + + if (remote.Proxy.Username == _EMPTY_) != (remote.Proxy.Password == _EMPTY_) { + return warnings, fmt.Errorf("proxy username and password must both be specified or both be empty") + } + + if len(remote.URLs) > 0 { + hasWebSocketURL := false + hasNonWebSocketURL := false + + for _, remoteURL := range remote.URLs { + if remoteURL.Scheme == wsSchemePrefix || remoteURL.Scheme == wsSchemePrefixTLS { + hasWebSocketURL = true + if (remoteURL.Scheme == wsSchemePrefixTLS) && + remote.TLSConfig == nil && !remote.TLS { + return warnings, fmt.Errorf("proxy is configured but remote URL %s requires TLS and no TLS configuration is provided. When using proxy with TLS endpoints, ensure TLS is properly configured for the leafnode remote", remoteURL.String()) + } + } else { + hasNonWebSocketURL = true + } + } + + if !hasWebSocketURL { + warnings = append(warnings, "proxy configuration will be ignored: proxy settings only apply to WebSocket connections (ws:// or wss://), but all configured URLs use TCP connections (nats://)") + } else if hasNonWebSocketURL { + warnings = append(warnings, "proxy configuration will only be used for WebSocket URLs: proxy settings do not apply to TCP connections (nats://)") + } + } + + return warnings, nil +} + +// Wait for the configured reconnect interval before attempting to connect +// again to the remote leafnode. +func (s *Server) reConnectToRemoteLeafNode(remote *leafNodeCfg) { + clearInProgress := true + defer func() { + s.grWG.Done() + if clearInProgress { + remote.setConnectInProgress(false) + } + }() + delay := s.getOpts().LeafNode.ReconnectInterval + select { + case <-time.After(delay): + case <-remote.quitCh: + return + case <-s.quitCh: + return + } + clearInProgress = !connectToRemoteLeafNode(s, remote, false) +} + +// Creates a leafNodeCfg object that wraps the RemoteLeafOpts. +func newLeafNodeCfg(remote *RemoteLeafOpts) *leafNodeCfg { + cfg := &leafNodeCfg{ + RemoteLeafOpts: remote, + urls: make([]*url.URL, 0, len(remote.URLs)), + quitCh: make(chan struct{}, 1), + } + if len(remote.DenyExports) > 0 || len(remote.DenyImports) > 0 { + perms := &Permissions{} + if len(remote.DenyExports) > 0 { + perms.Publish = &SubjectPermission{Deny: remote.DenyExports} + } + if len(remote.DenyImports) > 0 { + perms.Subscribe = &SubjectPermission{Deny: remote.DenyImports} + } + cfg.perms = perms + } + // Start with the one that is configured. We will add to this + // array when receiving async leafnode INFOs. + cfg.urls = append(cfg.urls, cfg.URLs...) + // If allowed to randomize, do it on our copy of URLs + if !remote.NoRandomize { + rand.Shuffle(len(cfg.urls), func(i, j int) { + cfg.urls[i], cfg.urls[j] = cfg.urls[j], cfg.urls[i] + }) + } + // If we are TLS make sure we save off a proper servername if possible. + // Do same for user/password since we may need them to connect to + // a bare URL that we get from INFO protocol. + for _, u := range cfg.urls { + cfg.saveTLSHostname(u) + cfg.saveUserPassword(u) + // If the url(s) have the "wss://" scheme, and we don't have a TLS + // config, mark that we should be using TLS anyway. + if !cfg.TLS && isWSSURL(u) { + cfg.TLS = true + } + } + return cfg +} + +// Notifies the quit channel without blocking. +// No lock is needed to invoke this function. +func (cfg *leafNodeCfg) notifyQuitChannel() { + select { + case cfg.quitCh <- struct{}{}: + default: + } +} + +// Sets the connect-in-progress status for this remote leaf configuration. +func (cfg *leafNodeCfg) setConnectInProgress(inProgress bool) { + cfg.Lock() + defer cfg.Unlock() + // In both cases we want to drain the "quit" channel. + select { + case <-cfg.quitCh: + default: + } + cfg.connInProgress = inProgress +} + +// Returns `true` if this remote is in the middle of a connect, `false` otherwise. +func (cfg *leafNodeCfg) isConnectInProgress() bool { + cfg.RLock() + defer cfg.RUnlock() + return cfg.connInProgress +} + +// Mark that this remote is being removed from the configuration. +func (cfg *leafNodeCfg) markAsRemoved() { + cfg.Lock() + defer cfg.Unlock() + // This function should be invoked only once, but protect. + if cfg.removed { + return + } + cfg.removed = true + cfg.notifyQuitChannel() +} + +// Returns false if it has been disabled or removed. +func (cfg *leafNodeCfg) stillValid() bool { + cfg.RLock() + defer cfg.RUnlock() + return !cfg.Disabled && !cfg.removed +} + +// Will pick an URL from the list of available URLs. +func (cfg *leafNodeCfg) pickNextURL() *url.URL { + cfg.Lock() + defer cfg.Unlock() + // If the current URL is the first in the list and we have more than + // one URL, then move that one to end of the list. + if cfg.curURL != nil && len(cfg.urls) > 1 && urlsAreEqual(cfg.curURL, cfg.urls[0]) { + first := cfg.urls[0] + copy(cfg.urls, cfg.urls[1:]) + cfg.urls[len(cfg.urls)-1] = first + } + cfg.curURL = cfg.urls[0] + return cfg.curURL +} + +// Returns the current URL +func (cfg *leafNodeCfg) getCurrentURL() *url.URL { + cfg.RLock() + defer cfg.RUnlock() + return cfg.curURL +} + +// Returns how long the server should wait before attempting +// to solicit a remote leafnode connection. +func (cfg *leafNodeCfg) getConnectDelay() time.Duration { + cfg.RLock() + delay := cfg.connDelay + cfg.RUnlock() + return delay +} + +// Sets the connect delay. +func (cfg *leafNodeCfg) setConnectDelay(delay time.Duration) { + cfg.Lock() + cfg.connDelay = delay + cfg.Unlock() +} + +// leafNodeDialer establishes the TCP connection to a remote leafnode server. +// It has the same signature as natsDialTimeout(), which is what is used in +// production. Tests can substitute their own implementation via +// LeafNodeOpts.dialer in order to observe the resolved dial timeout, or to +// simulate a slow/unreachable remote, without needing to manipulate the +// network stack. +type leafNodeDialer func(network, address string, timeout time.Duration) (net.Conn, error) + +// Ensure that non-exported options (used in tests) have +// been properly set. +func (s *Server) setLeafNodeNonExportedOptions() { + opts := s.getOpts() + s.leafNodeOpts.dialTimeout = opts.LeafNode.DialTimeout + if s.leafNodeOpts.dialTimeout <= 0 { + // Use same timeouts as routes for now. + s.leafNodeOpts.dialTimeout = DEFAULT_ROUTE_DIAL + } + s.leafNodeOpts.resolver = opts.LeafNode.resolver + if s.leafNodeOpts.resolver == nil { + s.leafNodeOpts.resolver = net.DefaultResolver + } + s.leafNodeOpts.dialer = opts.LeafNode.dialer + if s.leafNodeOpts.dialer == nil { + s.leafNodeOpts.dialer = natsDialTimeout + } +} + +const sharedSysAccDelay = 250 * time.Millisecond + +// establishHTTPProxyTunnel establishes an HTTP CONNECT tunnel through a proxy server +func establishHTTPProxyTunnel(proxyURL, targetHost string, timeout time.Duration, username, password string) (net.Conn, error) { + proxyAddr, err := url.Parse(proxyURL) + if err != nil { + // This should not happen since proxy URL is validated during configuration parsing + return nil, fmt.Errorf("unexpected proxy URL parse error (URL was pre-validated): %v", err) + } + + // Connect to the proxy server + conn, err := natsDialTimeout("tcp", proxyAddr.Host, timeout) + if err != nil { + return nil, fmt.Errorf("failed to connect to proxy: %v", err) + } + + // Set deadline for the entire proxy handshake + if err := conn.SetDeadline(time.Now().Add(timeout)); err != nil { + conn.Close() + return nil, fmt.Errorf("failed to set deadline: %v", err) + } + + req := &http.Request{ + Method: http.MethodConnect, + URL: &url.URL{Opaque: targetHost}, // Opaque is required for CONNECT + Host: targetHost, + Header: make(http.Header), + } + + // Add proxy authentication if provided + if username != "" && password != "" { + req.Header.Set("Proxy-Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(username+":"+password))) + } + + if err := req.Write(conn); err != nil { + conn.Close() + return nil, fmt.Errorf("failed to write CONNECT request: %v", err) + } + + resp, err := http.ReadResponse(bufio.NewReader(conn), req) + if err != nil { + conn.Close() + return nil, fmt.Errorf("failed to read proxy response: %v", err) + } + + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + conn.Close() + return nil, fmt.Errorf("proxy CONNECT failed: %s", resp.Status) + } + + // Close the response body + resp.Body.Close() + + // Clear the deadline now that we've finished the proxy handshake + if err := conn.SetDeadline(time.Time{}); err != nil { + conn.Close() + return nil, fmt.Errorf("failed to clear deadline: %v", err) + } + + return conn, nil +} + +// Connect to a remote leaf node asynchronously (that is, this function will do +// the connect in a go routine). +func (s *Server) connectToRemoteLeafNodeAsynchronously(remote *leafNodeCfg, firstConnect bool) { + remote.setConnectInProgress(true) + s.startGoRoutine(func() { + defer s.grWG.Done() + if !connectToRemoteLeafNode(s, remote, firstConnect) { + remote.setConnectInProgress(false) + } + }) +} + +// Connect to a remote leaf node. Should only be invoked from +// `s.connectToRemoteLeafNodeAsynchronously()` or `s.reConnectToRemoteLeafNode()`. +// Returns `true` if this function invoked `s.createLeafNode()`, false otherwise. +func connectToRemoteLeafNode(s *Server, remote *leafNodeCfg, firstConnect bool) bool { + + if remote == nil || len(remote.URLs) == 0 { + s.Debugf("Empty remote leafnode definition, nothing to connect") + return false + } + + // If this remote is no longer valid by the time we return (e.g. it was + // removed or disabled through a configuration reload), we will never + // reconnect, so clear any JetStream observer state. Otherwise, the raft + // nodes of this account's assets would remain observers. + defer func() { + if remote.stillValid() { + return + } + s.mu.RLock() + shouldMigrate := remote.JetStreamClusterMigrate + s.mu.RUnlock() + if shouldMigrate { + s.clearObserverState(remote) + } + }() + + opts := s.getOpts() + reconnectDelay := opts.LeafNode.ReconnectInterval + s.mu.RLock() + dialTimeout := s.leafNodeOpts.dialTimeout + resolver := s.leafNodeOpts.resolver + dialer := s.leafNodeOpts.dialer + var isSysAcc bool + if s.eventsEnabled() { + isSysAcc = remote.LocalAccount == s.sys.account.Name + } + jetstreamMigrateDelay := remote.JetStreamClusterMigrateDelay + s.mu.RUnlock() + + // If we are sharing a system account and we are not standalone delay to gather some info prior. + if firstConnect && isSysAcc && !s.standAloneMode() { + s.Debugf("Will delay first leafnode connect to shared system account due to clustering") + remote.setConnectDelay(sharedSysAccDelay) + } + + if connDelay := remote.getConnectDelay(); connDelay > 0 { + select { + case <-time.After(connDelay): + case <-remote.quitCh: + return false + case <-s.quitCh: + return false + } + remote.setConnectDelay(0) + } + + var conn net.Conn + + const connErrFmt = "Error trying to connect as leafnode to remote server %q (attempt %v): %v" + + // Capture proxy configuration once before the loop with proper locking + remote.RLock() + proxyURL := remote.Proxy.URL + proxyUsername := remote.Proxy.Username + proxyPassword := remote.Proxy.Password + proxyTimeout := remote.Proxy.Timeout + // A remote can override the server-wide dial timeout. This is useful on + // high latency links where the default is too short for the TCP handshake + // to complete. + if remote.DialTimeout > 0 { + dialTimeout = remote.DialTimeout + } + remote.RUnlock() + + // Set default proxy timeout if not specified + if proxyTimeout == 0 { + proxyTimeout = dialTimeout + } + + attempts := 0 + + // In case the migrate timer was created but not canceled, do it when + // this function exits. Note that the timer would not be created if + // `jetstreamMigrateDelay == 0`. + if jetstreamMigrateDelay > 0 { + defer remote.cancelMigrateTimer() + } + + reconnectTimer := time.NewTimer(reconnectDelay) + reconnectTimer.Stop() + defer stopAndClearTimer(&reconnectTimer) + + for s.isRunning() && remote.stillValid() { + rURL := remote.pickNextURL() + url, err := s.getRandomIP(resolver, rURL.Host, nil) + if err == nil { + var ipStr string + if url != rURL.Host { + ipStr = fmt.Sprintf(" (%s)", url) + } + // Some test may want to disable remotes from connecting + if s.isLeafConnectDisabled() { + s.Debugf("Will not attempt to connect to remote server on %q%s, leafnodes currently disabled", rURL.Host, ipStr) + err = ErrLeafNodeDisabled + } else { + s.Debugf("Trying to connect as leafnode to remote server on %q%s", rURL.Host, ipStr) + + // Check if proxy is configured + if proxyURL != _EMPTY_ { + targetHost := rURL.Host + // If URL doesn't include port, add the default port for the scheme + if rURL.Port() == _EMPTY_ { + defaultPort := "80" + if rURL.Scheme == wsSchemePrefixTLS { + defaultPort = "443" + } + targetHost = net.JoinHostPort(rURL.Hostname(), defaultPort) + } + + conn, err = establishHTTPProxyTunnel(proxyURL, targetHost, proxyTimeout, proxyUsername, proxyPassword) + } else { + // Direct connection + conn, err = dialer("tcp", url, dialTimeout) + } + } + } + if err != nil { + jitter := time.Duration(rand.Int64N(int64(reconnectDelay))) + delay := reconnectDelay + jitter + attempts++ + if s.shouldReportConnectErr(firstConnect, attempts) { + s.Errorf(connErrFmt, rURL.Host, attempts, err) + } else { + s.Debugf(connErrFmt, rURL.Host, attempts, err) + } + remote.Lock() + // if we are using a delay to start migrating assets, kick off a migrate timer. + if remote.jsMigrateTimer == nil && jetstreamMigrateDelay > 0 { + remote.jsMigrateTimer = time.AfterFunc(jetstreamMigrateDelay, func() { + s.checkJetStreamMigrate(remote) + }) + } + remote.Unlock() + reconnectTimer.Reset(delay) + select { + case <-s.quitCh: + return false + case <-remote.quitCh: + return false + case <-reconnectTimer.C: + // Check if we should migrate any JetStream assets immediately while this remote is down. + // This will be used if JetStreamClusterMigrateDelay was not set + if jetstreamMigrateDelay == 0 { + s.checkJetStreamMigrate(remote) + } + continue + } + } + remote.cancelMigrateTimer() + // We can check here, but really we will have to check again when the server + // is about to add to the `s.leafs` map later in the process. + if !remote.stillValid() { + conn.Close() + return false + } + + // We have a connection here to a remote server. + // Go ahead and create our leaf node and return. + s.createLeafNode(conn, rURL, remote, nil) + + // Clear any observer states if we had them. + s.clearObserverState(remote) + + return true + } + + return false +} + +func (cfg *leafNodeCfg) cancelMigrateTimer() { + cfg.Lock() + stopAndClearTimer(&cfg.jsMigrateTimer) + cfg.Unlock() +} + +// This will clear any observer state such that stream or consumer assets on this server can become leaders again. +func (s *Server) clearObserverState(remote *leafNodeCfg) { + s.mu.RLock() + accName := remote.LocalAccount + s.mu.RUnlock() + + acc, err := s.LookupAccount(accName) + if err != nil { + s.Warnf("Error looking up account [%s] checking for JetStream clear observer state on a leafnode", accName) + return + } + + acc.jscmMu.Lock() + defer acc.jscmMu.Unlock() + + // Walk all streams looking for any clustered stream, skip otherwise. + for _, mset := range acc.streams() { + node := mset.raftNode() + if node == nil { + // Not R>1 + continue + } + // Check consumers + for _, o := range mset.getConsumers() { + if n := o.raftNode(); n != nil { + // Ensure we can become a leader again. + n.SetObserver(false) + } + } + // Ensure we can not become a leader again. + node.SetObserver(false) + } +} + +// Check to see if we should migrate any assets from this account. +func (s *Server) checkJetStreamMigrate(remote *leafNodeCfg) { + s.mu.RLock() + accName, shouldMigrate := remote.LocalAccount, remote.JetStreamClusterMigrate + s.mu.RUnlock() + + if !shouldMigrate { + return + } + + acc, err := s.LookupAccount(accName) + if err != nil { + s.Warnf("Error looking up account [%s] checking for JetStream migration on a leafnode", accName) + return + } + + acc.jscmMu.Lock() + defer acc.jscmMu.Unlock() + + // Walk all streams looking for any clustered stream, skip otherwise. + // If we are the leader force stepdown. + for _, mset := range acc.streams() { + node := mset.raftNode() + if node == nil { + // Not R>1 + continue + } + // Collect any consumers + for _, o := range mset.getConsumers() { + if n := o.raftNode(); n != nil { + n.StepDown() + // Ensure we can not become a leader while in this state. + n.SetObserver(true) + } + } + // Stepdown if this stream was leader. + node.StepDown() + // Ensure we can not become a leader while in this state. + node.SetObserver(true) + } +} + +// Helper for checking. +func (s *Server) isLeafConnectDisabled() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.leafDisableConnect +} + +// Save off the tlsName for when we use TLS and mix hostnames and IPs. IPs usually +// come from the server we connect to. +// +// We used to save the name only if there was a TLSConfig or scheme equal to "tls". +// However, this was causing failures for users that did not set the scheme (and +// their remote connections did not have a tls{} block). +// We now save the host name regardless in case the remote returns an INFO indicating +// that TLS is required. +// +// Lock held on entry. +func (cfg *leafNodeCfg) saveTLSHostname(u *url.URL) { + if cfg.tlsName == _EMPTY_ && net.ParseIP(u.Hostname()) == nil { + cfg.tlsName = u.Hostname() + } +} + +// Save off the username/password for when we connect using a bare URL +// that we get from the INFO protocol. +// +// Lock held on entry. +func (cfg *leafNodeCfg) saveUserPassword(u *url.URL) { + if cfg.username == _EMPTY_ && u.User != nil { + cfg.username = u.User.Username() + cfg.password, _ = u.User.Password() + } +} + +// This starts the leafnode accept loop in a go routine, unless it +// is detected that the server has already been shutdown. +func (s *Server) startLeafNodeAcceptLoop() { + // Snapshot server options. + opts := s.getOpts() + + port := opts.LeafNode.Port + if port == -1 { + port = 0 + } + + if s.isShuttingDown() { + return + } + + s.mu.Lock() + hp := net.JoinHostPort(opts.LeafNode.Host, strconv.Itoa(port)) + l, e := natsListen("tcp", hp) + s.leafNodeListenerErr = e + if e != nil { + s.mu.Unlock() + s.Fatalf("Error listening on leafnode port: %d - %v", opts.LeafNode.Port, e) + return + } + + s.Noticef("Listening for leafnode connections on %s", + net.JoinHostPort(opts.LeafNode.Host, strconv.Itoa(l.Addr().(*net.TCPAddr).Port))) + + tlsRequired := opts.LeafNode.TLSConfig != nil + tlsVerify := tlsRequired && opts.LeafNode.TLSConfig.ClientAuth == tls.RequireAndVerifyClientCert + // Do not set compression in this Info object, it would possibly cause + // issues when sending asynchronous INFO to the remote. + info := Info{ + ID: s.info.ID, + Name: s.info.Name, + Version: s.info.Version, + GitCommit: gitCommit, + GoVersion: runtime.Version(), + AuthRequired: true, + TLSRequired: tlsRequired, + TLSVerify: tlsVerify, + MaxPayload: s.info.MaxPayload, // TODO(dlc) - Allow override? + Headers: s.supportsHeaders(), + JetStream: opts.JetStream, + Domain: opts.JetStreamDomain, + Proto: s.getServerProto(), + InfoOnConnect: true, + JSApiLevel: JSApiLevel, + } + // If we have selected a random port... + if port == 0 { + // Write resolved port back to options. + opts.LeafNode.Port = l.Addr().(*net.TCPAddr).Port + } + + s.leafNodeInfo = info + // Possibly override Host/Port and set IP based on Cluster.Advertise + if err := s.setLeafNodeInfoHostPortAndIP(); err != nil { + s.Fatalf("Error setting leafnode INFO with LeafNode.Advertise value of %s, err=%v", opts.LeafNode.Advertise, err) + l.Close() + s.mu.Unlock() + return + } + if !opts.LeafNode.NoAdvertise { + s.leafURLsMap[s.leafNodeInfo.IP]++ + } + s.generateLeafNodeInfoJSON() + + // Setup state that can enable shutdown + s.leafNodeListener = l + + // As of now, a server that does not have remotes configured would + // never solicit a connection, so we should not have to warn if + // InsecureSkipVerify is set in main LeafNodes config (since + // this TLS setting matters only when soliciting a connection). + // Still, warn if insecure is set in any of LeafNode block. + // We need to check remotes, even if tls is not required on accept. + warn := tlsRequired && opts.LeafNode.TLSConfig.InsecureSkipVerify + if !warn { + for _, r := range opts.LeafNode.Remotes { + if r.TLSConfig != nil && r.TLSConfig.InsecureSkipVerify { + warn = true + break + } + } + } + if warn { + s.Warnf(leafnodeTLSInsecureWarning) + } + go s.acceptConnections(l, "Leafnode", func(conn net.Conn) { s.createLeafNode(conn, nil, nil, nil) }, nil) + s.mu.Unlock() +} + +// RegEx to match a creds file with user JWT and Seed. +var credsRe = regexp.MustCompile(`\s*(?:(?:[-]{3,}.*[-]{3,}\r?\n)([\w\-.=]+)(?:\r?\n[-]{3,}.*[-]{3,}(\r?\n|\z)))`) + +// clusterName is provided as argument to avoid lock ordering issues with the locked client c +// Lock should be held entering here. +func (c *client) sendLeafConnect(clusterName string, headers bool) error { + // We support basic user/pass and operator based user JWT with signatures. + cinfo := leafConnectInfo{ + Version: VERSION, + ID: c.srv.info.ID, + Domain: c.srv.info.Domain, + Name: c.srv.info.Name, + Hub: c.leaf.remote.Hub, + Cluster: clusterName, + Headers: headers, + JetStream: c.acc.jetStreamConfigured(), + DenyPub: c.leaf.remote.DenyImports, + Compression: c.leaf.compression, + RemoteAccount: c.acc.GetName(), + Proto: c.srv.getServerProto(), + Isolate: c.leaf.remote.RequestIsolation, + } + + // If a signature callback is specified, this takes precedence over anything else. + if cb := c.leaf.remote.SignatureCB; cb != nil { + nonce := c.nonce + c.mu.Unlock() + jwt, sigraw, err := cb(nonce) + c.mu.Lock() + if err == nil && c.isClosed() { + err = ErrConnectionClosed + } + if err != nil { + c.Errorf("Error signing the nonce: %v", err) + return err + } + sig := base64.RawURLEncoding.EncodeToString(sigraw) + cinfo.JWT, cinfo.Sig = jwt, sig + + } else if creds := c.leaf.remote.Credentials; creds != _EMPTY_ { + // Check for credentials first, that will take precedence.. + c.Debugf("Authenticating with credentials file %q", c.leaf.remote.Credentials) + contents, err := os.ReadFile(creds) + if err != nil { + c.Errorf("%v", err) + return err + } + defer wipeSlice(contents) + items := credsRe.FindAllSubmatch(contents, -1) + if len(items) < 2 { + c.Errorf("Credentials file malformed") + return err + } + // First result should be the user JWT. + // We copy here so that the file containing the seed will be wiped appropriately. + raw := items[0][1] + tmp := make([]byte, len(raw)) + copy(tmp, raw) + // Seed is second item. + kp, err := nkeys.FromSeed(items[1][1]) + if err != nil { + c.Errorf("Credentials file has malformed seed") + return err + } + // Wipe our key on exit. + defer kp.Wipe() + + sigraw, _ := kp.Sign(c.nonce) + sig := base64.RawURLEncoding.EncodeToString(sigraw) + cinfo.JWT = bytesToString(tmp) + cinfo.Sig = sig + } else if nkey := c.leaf.remote.Nkey; nkey != _EMPTY_ { + kp, err := nkeys.FromSeed([]byte(nkey)) + if err != nil { + c.Errorf("Remote nkey has malformed seed") + return err + } + // Wipe our key on exit. + defer kp.Wipe() + sigraw, _ := kp.Sign(c.nonce) + sig := base64.RawURLEncoding.EncodeToString(sigraw) + pkey, _ := kp.PublicKey() + cinfo.Nkey = pkey + cinfo.Sig = sig + } + // In addition, and this is to allow auth callout, set user/password or + // token if applicable. + if userInfo := c.leaf.remote.curURL.User; userInfo != nil { + cinfo.User = userInfo.Username() + var ok bool + cinfo.Pass, ok = userInfo.Password() + // For backward compatibility, if only username is provided, set both + // Token and User, not just Token. + if !ok { + cinfo.Token = cinfo.User + } + } else if c.leaf.remote.username != _EMPTY_ { + cinfo.User = c.leaf.remote.username + cinfo.Pass = c.leaf.remote.password + // For backward compatibility, if only username is provided, set both + // Token and User, not just Token. + if cinfo.Pass == _EMPTY_ { + cinfo.Token = cinfo.User + } + } + b, err := json.Marshal(cinfo) + if err != nil { + c.Errorf("Error marshaling CONNECT to remote leafnode: %v\n", err) + return err + } + // Although this call is made before the writeLoop is created, + // we don't really need to send in place. The protocol will be + // sent out by the writeLoop. + c.enqueueProto([]byte(fmt.Sprintf(ConProto, b))) + return nil +} + +// Makes a deep copy of the LeafNode Info structure. +// The server lock is held on entry. +func (s *Server) copyLeafNodeInfo() *Info { + clone := s.leafNodeInfo + // Copy the array of urls. + if len(s.leafNodeInfo.LeafNodeURLs) > 0 { + clone.LeafNodeURLs = append([]string(nil), s.leafNodeInfo.LeafNodeURLs...) + } + return &clone +} + +// Adds a LeafNode URL that we get when a route connects to the Info structure. +// Regenerates the JSON byte array so that it can be sent to LeafNode connections. +// Returns a boolean indicating if the URL was added or not. +// Server lock is held on entry +func (s *Server) addLeafNodeURL(urlStr string) bool { + if s.leafURLsMap.addUrl(urlStr) { + s.generateLeafNodeInfoJSON() + return true + } + return false +} + +// Removes a LeafNode URL of the route that is disconnecting from the Info structure. +// Regenerates the JSON byte array so that it can be sent to LeafNode connections. +// Returns a boolean indicating if the URL was removed or not. +// Server lock is held on entry. +func (s *Server) removeLeafNodeURL(urlStr string) bool { + // Don't need to do this if we are removing the route connection because + // we are shuting down... + if s.isShuttingDown() { + return false + } + if s.leafURLsMap.removeUrl(urlStr) { + s.generateLeafNodeInfoJSON() + return true + } + return false +} + +// Server lock is held on entry +func (s *Server) generateLeafNodeInfoJSON() { + s.leafNodeInfo.Cluster = s.cachedClusterName() + s.leafNodeInfo.LeafNodeURLs = s.leafURLsMap.getAsStringSlice() + s.leafNodeInfo.WSConnectURLs = s.websocket.connectURLsMap.getAsStringSlice() + s.leafNodeInfoJSON = generateInfoJSON(&s.leafNodeInfo) +} + +// Sends an async INFO protocol so that the connected servers can update +// their list of LeafNode urls. +func (s *Server) sendAsyncLeafNodeInfo() { + for _, c := range s.leafs { + c.mu.Lock() + c.enqueueProto(s.leafNodeInfoJSON) + c.mu.Unlock() + } +} + +// Called when an inbound leafnode connection is accepted or we create one for a solicited leafnode. +func (s *Server) createLeafNode(conn net.Conn, rURL *url.URL, remote *leafNodeCfg, ws *websocket) *client { + // Snapshot server options. + opts := s.getOpts() + + maxPay := int32(opts.MaxPayload) + maxSubs := int32(opts.MaxSubs) + // For system, maxSubs of 0 means unlimited, so re-adjust here. + if maxSubs == 0 { + maxSubs = -1 + } + now := time.Now().UTC() + + c := &client{srv: s, nc: conn, kind: LEAF, opts: defaultOpts, mpay: maxPay, msubs: maxSubs, start: now, last: now} + // Do not update the smap here, we need to do it in initLeafNodeSmapAndSendSubs + c.leaf = &leaf{} + + // If the leafnode subject interest should be isolated, flag it here. + s.optsMu.RLock() + if c.leaf.isolated = s.opts.LeafNode.IsolateLeafnodeInterest; !c.leaf.isolated && remote != nil { + c.leaf.isolated = remote.LocalIsolation + } + s.optsMu.RUnlock() + + // For accepted LN connections, ws will be != nil if it was accepted + // through the Websocket port. + c.ws = ws + + // For remote, check if the scheme starts with "ws", if so, we will initiate + // a remote Leaf Node connection as a websocket connection. + if remote != nil && rURL != nil && isWSURL(rURL) { + remote.RLock() + c.ws = &websocket{compress: remote.Websocket.Compression, maskwrite: !remote.Websocket.NoMasking} + remote.RUnlock() + } + + // Determines if we are soliciting the connection or not. + var solicited bool + var acc *Account + var remoteSuffix string + if remote != nil { + // For now, if lookup fails, we will constantly try + // to recreate this LN connection. + lacc := remote.LocalAccount + var err error + acc, err = s.LookupAccount(lacc) + if err != nil { + // An account not existing is something that can happen with nats/http account resolver and the account + // has not yet been pushed, or the request failed for other reasons. + // remote needs to be set or retry won't happen + c.leaf.remote = remote + c.closeConnection(MissingAccount) + s.Errorf("Unable to lookup account %s for solicited leafnode connection: %v", lacc, err) + return nil + } + remoteSuffix = fmt.Sprintf(" for account: %s", acc.traceLabel()) + } + + c.mu.Lock() + c.initClient() + c.Noticef("Leafnode connection created%s %s", remoteSuffix, c.opts.Name) + + var ( + tlsFirst bool + tlsFirstFallback time.Duration + infoTimeout time.Duration + ) + if remote != nil { + solicited = true + remote.Lock() + c.leaf.remote = remote + c.setPermissions(remote.perms) + if !c.leaf.remote.Hub { + c.leaf.isSpoke = true + } + tlsFirst = remote.TLSHandshakeFirst + infoTimeout = remote.FirstInfoTimeout + remote.Unlock() + c.acc = acc + } else { + c.flags.set(expectConnect) + if ws != nil { + c.Debugf("Leafnode compression=%v", c.ws.compress) + } + tlsFirst = opts.LeafNode.TLSHandshakeFirst + if f := opts.LeafNode.TLSHandshakeFirstFallback; f > 0 { + tlsFirstFallback = f + } + } + c.mu.Unlock() + + var nonce [nonceLen]byte + var info *Info + + // Grab this before the client lock below. + if !solicited { + // Grab server variables + s.mu.Lock() + info = s.copyLeafNodeInfo() + // For tests that want to simulate old servers, do not set the compression + // on the INFO protocol if configured with CompressionNotSupported. + // Also suppress it if WebSocket compression is already in use, otherwise + // an old soliciting peer would honor the advertised mode, switch to S2, + // and then wait forever for a compressed INFO response from us. + if cm := opts.LeafNode.Compression.Mode; cm != CompressionNotSupported && (ws == nil || !ws.compress) { + info.Compression = cm + } + // We always send a nonce for LEAF connections. Do not change that without + // taking into account presence of proxy trusted keys. + s.generateNonce(nonce[:]) + s.mu.Unlock() + } + + // Grab lock + c.mu.Lock() + + var preBuf []byte + if solicited { + // For websocket connection, we need to send an HTTP request, + // and get the response before starting the readLoop to get + // the INFO, etc.. + if c.isWebsocket() { + var err error + var closeReason ClosedState + + preBuf, closeReason, err = c.leafNodeSolicitWSConnection(opts, rURL, remote) + if err != nil { + c.Errorf("Error soliciting websocket connection: %v", err) + c.mu.Unlock() + if closeReason != 0 { + c.closeConnection(closeReason) + } + return nil + } + } else { + // If configured to do TLS handshake first + if tlsFirst { + if _, err := c.leafClientHandshakeIfNeeded(remote, opts); err != nil { + c.mu.Unlock() + return nil + } + } + // We need to wait for the info, but not for too long. + c.nc.SetReadDeadline(time.Now().Add(infoTimeout)) + } + + // We will process the INFO from the readloop and finish by + // sending the CONNECT and finish registration later. + } else { + // Send our info to the other side. + // Remember the nonce we sent here for signatures, etc. + c.nonce = make([]byte, nonceLen) + copy(c.nonce, nonce[:]) + info.Nonce = bytesToString(c.nonce) + info.CID = c.cid + proto := generateInfoJSON(info) + + var pre []byte + // We need first to check for "TLS First" fallback delay. + if tlsFirstFallback > 0 { + // We wait and see if we are getting any data. Since we did not send + // the INFO protocol yet, only clients that use TLS first should be + // sending data (the TLS handshake). We don't really check the content: + // if it is a rogue agent and not an actual client performing the + // TLS handshake, the error will be detected when performing the + // handshake on our side. + pre = make([]byte, 4) + c.nc.SetReadDeadline(time.Now().Add(tlsFirstFallback)) + n, _ := io.ReadFull(c.nc, pre[:]) + c.nc.SetReadDeadline(time.Time{}) + // If we get any data (regardless of possible timeout), we will proceed + // with the TLS handshake. + if n > 0 { + pre = pre[:n] + } else { + // We did not get anything so we will send the INFO protocol. + pre = nil + // Set the boolean to false for the rest of the function. + tlsFirst = false + } + } + + if !tlsFirst { + // We have to send from this go routine because we may + // have to block for TLS handshake before we start our + // writeLoop go routine. The other side needs to receive + // this before it can initiate the TLS handshake.. + c.sendProtoNow(proto) + + // The above call could have marked the connection as closed (due to TCP error). + if c.isClosed() { + c.mu.Unlock() + c.closeConnection(WriteError) + return nil + } + } + + // Check to see if we need to spin up TLS. + if !c.isWebsocket() && info.TLSRequired { + // If we have a prebuffer create a multi-reader. + if len(pre) > 0 { + c.nc = &tlsMixConn{c.nc, bytes.NewBuffer(pre)} + } + // Perform server-side TLS handshake. + if err := c.doTLSServerHandshake(tlsHandshakeLeaf, opts.LeafNode.TLSConfig, opts.LeafNode.TLSTimeout, opts.LeafNode.TLSPinnedCerts); err != nil { + c.mu.Unlock() + return nil + } + } + + // If the user wants the TLS handshake to occur first, now that it is + // done, send the INFO protocol. + if tlsFirst { + c.flags.set(didTLSFirst) + c.sendProtoNow(proto) + if c.isClosed() { + c.mu.Unlock() + c.closeConnection(WriteError) + return nil + } + } + + // Leaf nodes will always require a CONNECT to let us know + // when we are properly bound to an account. + // + // If compression is configured, we can't set the authTimer here because + // it would cause the parser to fail any incoming protocol that is not a + // CONNECT (and we need to exchange INFO protocols for compression + // negotiation). So instead, use the ping timer until we are done with + // negotiation and can set the auth timer. + timeout := secondsToDuration(opts.LeafNode.AuthTimeout) + if needsCompression(opts.LeafNode.Compression.Mode) { + c.ping.tmr = time.AfterFunc(timeout, func() { + c.authTimeout() + }) + } else { + c.setAuthTimer(timeout) + } + } + + // Keep track in case server is shutdown before we can successfully register. + if !s.addToTempClients(c.cid, c) { + c.mu.Unlock() + c.setNoReconnect() + c.closeConnection(ServerShutdown) + return nil + } + + // Spin up the read loop. + s.startGoRoutine(func() { c.readLoop(preBuf) }) + + // We will spin the write loop for solicited connections only + // when processing the INFO and after switching to TLS if needed. + if !solicited { + s.startGoRoutine(func() { c.writeLoop() }) + } + + c.mu.Unlock() + + return c +} + +// Will perform the client-side TLS handshake if needed. Assumes that this +// is called by the solicit side (remote will be non nil). Returns `true` +// if TLS is required, `false` otherwise. +// Lock held on entry. +func (c *client) leafClientHandshakeIfNeeded(remote *leafNodeCfg, opts *Options) (bool, error) { + // Check if TLS is required and gather TLS config variables. + tlsRequired, tlsConfig, tlsName, tlsTimeout := c.leafNodeGetTLSConfigForSolicit(remote) + if !tlsRequired { + return false, nil + } + + // If TLS required, peform handshake. + // Get the URL that was used to connect to the remote server. + rURL := remote.getCurrentURL() + + // Perform the client-side TLS handshake. + if resetTLSName, err := c.doTLSClientHandshake(tlsHandshakeLeaf, rURL, tlsConfig, tlsName, tlsTimeout, opts.LeafNode.TLSPinnedCerts); err != nil { + // Check if we need to reset the remote's TLS name. + if resetTLSName { + remote.Lock() + remote.tlsName = _EMPTY_ + remote.Unlock() + } + return false, err + } + return true, nil +} + +func (c *client) processLeafnodeInfo(info *Info) { + c.mu.Lock() + if c.leaf == nil || c.isClosed() { + c.mu.Unlock() + return + } + s := c.srv + opts := s.getOpts() + remote := c.leaf.remote + didSolicit := remote != nil + firstINFO := !c.flags.isSet(infoReceived) + + // In case of websocket, the TLS handshake has been already done. + // So check only for non websocket connections and for configurations + // where the TLS Handshake was not done first. + if didSolicit && !c.flags.isSet(handshakeComplete) && !c.isWebsocket() && !remote.TLSHandshakeFirst { + // If the server requires TLS, we need to set this in the remote + // otherwise if there is no TLS configuration block for the remote, + // the solicit side will not attempt to perform the TLS handshake. + if firstINFO && info.TLSRequired { + // Check for TLS/proxy configuration mismatch + if remote.Proxy.URL != _EMPTY_ && !remote.TLS && remote.TLSConfig == nil { + c.mu.Unlock() + c.Errorf("TLS configuration mismatch: Hub requires TLS but leafnode remote is not configured for TLS. When using a proxy, ensure TLS leafnode configuration matches the Hub requirements.") + c.closeConnection(TLSHandshakeError) + return + } + remote.TLS = true + } + if _, err := c.leafClientHandshakeIfNeeded(remote, opts); err != nil { + c.mu.Unlock() + return + } + } + + // Check for compression, unless already done. + if firstINFO && !c.flags.isSet(compressionNegotiated) { + // A solicited leafnode connection must first receive a leafnode INFO. + // Classify wrong-port connections before any leaf-specific negotiation. + if didSolicit && (info.CID == 0 || (info.LeafNodeURLs == nil && !info.InfoOnConnect)) { + c.mu.Unlock() + c.Errorf(ErrConnectedToWrongPort.Error()) + c.closeConnection(WrongPort) + return + } + + // Prevent from getting back here. + c.flags.set(compressionNegotiated) + + var co *CompressionOpts + if !didSolicit { + co = &opts.LeafNode.Compression + } else { + co = &remote.Compression + } + if needsCompression(co.Mode) { + // Release client lock since following function will need server lock. + c.mu.Unlock() + compress, err := s.negotiateLeafCompression(c, didSolicit, info.Compression, co) + if err != nil { + c.sendErrAndErr(err.Error()) + c.closeConnection(ProtocolViolation) + return + } + if compress { + // Done for now, will get back another INFO protocol... + return + } + // No compression because one side does not want/can't, so proceed. + c.mu.Lock() + // Check that the connection did not close if the lock was released. + if c.isClosed() { + c.mu.Unlock() + return + } + } else { + // Coming from an old server, the Compression field would be the empty + // string. For servers that are configured with CompressionNotSupported, + // this makes them behave as old servers. + if info.Compression == _EMPTY_ || co.Mode == CompressionNotSupported { + c.leaf.compression = CompressionNotSupported + } else { + c.leaf.compression = CompressionOff + } + } + // Accepting side does not normally process an INFO protocol during + // initial connection handshake. So we keep it consistent by returning + // if we are not soliciting. + if !didSolicit { + // If we had created the ping timer instead of the auth timer, we will + // clear the ping timer and set the auth timer now that the compression + // negotiation is done. + if info.Compression != _EMPTY_ && c.ping.tmr != nil { + clearTimer(&c.ping.tmr) + c.setAuthTimer(secondsToDuration(opts.LeafNode.AuthTimeout)) + } + c.mu.Unlock() + return + } + // Fall through and process the INFO protocol as usual. + } + + // Note: For now, only the initial INFO has a nonce. We + // will probably do auto key rotation at some point. + if firstINFO { + // Mark that the INFO protocol has been received. + c.flags.set(infoReceived) + // Prevent connecting to non leafnode port. Need to do this only for + // the first INFO, not for async INFO updates... + // + // Content of INFO sent by the server when accepting a tcp connection. + // ------------------------------------------------------------------- + // Listen Port Of | CID | ClientConnectURLs | LeafNodeURLs | Gateway | + // ------------------------------------------------------------------- + // CLIENT | X* | X** | | | + // ROUTE | | X** | X*** | | + // GATEWAY | | | | X | + // LEAFNODE | X | | X | | + // ------------------------------------------------------------------- + // * Not on older servers. + // ** Not if "no advertise" is enabled. + // *** Not if leafnode's "no advertise" is enabled. + // + // Reject a cluster that contains spaces. + if info.Cluster != _EMPTY_ && strings.Contains(info.Cluster, " ") { + c.mu.Unlock() + c.sendErrAndErr(ErrClusterNameHasSpaces.Error()) + c.closeConnection(ProtocolViolation) + return + } + if info.Cluster == leafNoOriginCluster { + c.mu.Unlock() + c.sendErrAndErr(ErrClusterNameReserved.Error()) + c.closeConnection(ProtocolViolation) + return + } + // For solicited outbound leaf connections, capture the remote's nonce. + // For inbound leaf connections, keep using the server-issued nonce that + // was sent in our initial INFO and must be signed in CONNECT. + if didSolicit { + c.nonce = []byte(info.Nonce) + } + if info.TLSRequired && didSolicit { + remote.TLS = true + } + supportsHeaders := c.srv.supportsHeaders() + c.headers = supportsHeaders && info.Headers + + // Remember the remote server. + // Pre 2.2.0 servers are not sending their server name. + // In that case, use info.ID, which, for those servers, matches + // the content of the field `Name` in the leafnode CONNECT protocol. + if info.Name == _EMPTY_ { + c.leaf.remoteServer = info.ID + } else { + c.leaf.remoteServer = info.Name + } + c.leaf.remoteDomain = info.Domain + c.leaf.remoteCluster = info.Cluster + // We send the protocol version in the INFO protocol. + // Keep track of it, so we know if this connection supports message + // tracing for instance. + c.opts.Protocol = info.Proto + } + + // For both initial INFO and async INFO protocols, Possibly + // update our list of remote leafnode URLs we can connect to, + // unless we are instructed not to. + if didSolicit && !remote.IgnoreDiscoveredServers && + (len(info.LeafNodeURLs) > 0 || len(info.WSConnectURLs) > 0) { + // Consider the incoming array as the most up-to-date + // representation of the remote cluster's list of URLs. + c.updateLeafNodeURLs(info) + } + + // Only solicited leafnode connections trust permission updates from INFO. + if didSolicit && (info.Import != nil || info.Export != nil) { + perms := &Permissions{ + Publish: info.Export, + Subscribe: info.Import, + } + // Check if we have local deny clauses that we need to merge. + if remote := c.leaf.remote; remote != nil { + if len(remote.DenyExports) > 0 { + if perms.Publish == nil { + perms.Publish = &SubjectPermission{} + } + perms.Publish.Deny = append(perms.Publish.Deny, remote.DenyExports...) + } + if len(remote.DenyImports) > 0 { + if perms.Subscribe == nil { + perms.Subscribe = &SubjectPermission{} + } + perms.Subscribe.Deny = append(perms.Subscribe.Deny, remote.DenyImports...) + } + } + c.setPermissions(perms) + } + + var resumeConnect bool + + // If this is a remote connection and this is the first INFO protocol, + // then we need to finish the connect process by sending CONNECT, etc.. + if firstINFO && didSolicit { + // Clear deadline that was set in createLeafNode while waiting for the INFO. + c.nc.SetDeadline(time.Time{}) + resumeConnect = true + } else if !firstINFO && didSolicit { + c.leaf.remoteAccName = info.RemoteAccount + } + + // Check if we have the remote account information and if so make sure it's stored. + if info.RemoteAccount != _EMPTY_ { + if c.acc == nil { + c.mu.Unlock() + c.sendErr("Authorization Violation") + c.closeConnection(ProtocolViolation) + return + } + s.leafRemoteAccounts.Store(c.acc.Name, info.RemoteAccount) + } + c.mu.Unlock() + + finishConnect := info.ConnectInfo + if resumeConnect && s != nil { + s.leafNodeResumeConnectProcess(c) + if !info.InfoOnConnect { + finishConnect = true + } + } + if finishConnect { + s.leafNodeFinishConnectProcess(c) + } + + // Check to see if we need to kick any internal source or mirror consumers. + // This will be a no-op if JetStream not enabled for this server or if the bound account + // does not have jetstream. + s.checkInternalSyncConsumers(c.acc) +} + +func (s *Server) negotiateLeafCompression(c *client, didSolicit bool, infoCompression string, co *CompressionOpts) (bool, error) { + // If WebSocket compression is already negotiated on this connection then + // we shouldn't layer S2 compression on top of it. + c.mu.Lock() + if c.ws != nil && c.ws.compress { + c.leaf.compression = CompressionOff + c.mu.Unlock() + return false, nil + } + c.mu.Unlock() + // Negotiate the appropriate compression mode (or no compression) + cm, err := selectCompressionMode(co.Mode, infoCompression) + if err != nil { + return false, err + } + c.mu.Lock() + // For "auto" mode, set the initial compression mode based on RTT + if cm == CompressionS2Auto { + if c.rttStart.IsZero() { + c.rtt = computeRTT(c.start) + } + cm = selectS2AutoModeBasedOnRTT(c.rtt, co.RTTThresholds) + } + // Keep track of the negotiated compression mode. + c.leaf.compression = cm + cid := c.cid + var nonce string + if !didSolicit { + nonce = bytesToString(c.nonce) + } + c.mu.Unlock() + + if !needsCompression(cm) { + return false, nil + } + + // If we end-up doing compression... + + // Generate an INFO with the chosen compression mode. + s.mu.Lock() + info := s.copyLeafNodeInfo() + info.Compression, info.CID, info.Nonce = compressionModeForInfoProtocol(co, cm), cid, nonce + infoProto := generateInfoJSON(info) + s.mu.Unlock() + + // If we solicited, then send this INFO protocol BEFORE switching + // to compression writer. However, if we did not, we send it after. + c.mu.Lock() + if didSolicit { + c.enqueueProto(infoProto) + // Make sure it is completely flushed (the pending bytes goes to + // 0) before proceeding. + for c.out.pb > 0 && !c.isClosed() { + c.flushOutbound() + } + } + // This is to notify the readLoop that it should switch to a + // (de)compression reader. + c.in.flags.set(switchToCompression) + // Create the compress writer before queueing the INFO protocol for + // a route that did not solicit. It will make sure that that proto + // is sent with compression on. + c.out.cw = s2.NewWriter(nil, s2WriterOptions(cm)...) + if !didSolicit { + c.enqueueProto(infoProto) + } + c.mu.Unlock() + return true, nil +} + +// When getting a leaf node INFO protocol, use the provided +// array of urls to update the list of possible endpoints. +func (c *client) updateLeafNodeURLs(info *Info) { + cfg := c.leaf.remote + cfg.Lock() + defer cfg.Unlock() + + // We have ensured that if a remote has a WS scheme, then all are. + // So check if first is WS, then add WS URLs, otherwise, add non WS ones. + if len(cfg.URLs) > 0 && isWSURL(cfg.URLs[0]) { + // It does not really matter if we use "ws://" or "wss://" here since + // we will have already marked that the remote should use TLS anyway. + // But use proper scheme for log statements, etc... + proto := wsSchemePrefix + if cfg.TLS { + proto = wsSchemePrefixTLS + } + c.doUpdateLNURLs(cfg, proto, info.WSConnectURLs) + return + } + c.doUpdateLNURLs(cfg, "nats-leaf", info.LeafNodeURLs) +} + +func (c *client) doUpdateLNURLs(cfg *leafNodeCfg, scheme string, URLs []string) { + cfg.urls = make([]*url.URL, 0, 1+len(URLs)) + // Add the ones we receive in the protocol + for _, surl := range URLs { + url, err := url.Parse(fmt.Sprintf("%s://%s", scheme, surl)) + if err != nil { + // As per below, the URLs we receive should not have contained URL info, so this should be safe to log. + c.Errorf("Error parsing url %q: %v", surl, err) + continue + } + // Do not add if it's the same as what we already have configured. + var dup bool + for _, u := range cfg.URLs { + // URLs that we receive never have user info, but the + // ones that were configured may have. Simply compare + // host and port to decide if they are equal or not. + if url.Host == u.Host && url.Port() == u.Port() { + dup = true + break + } + } + if !dup { + cfg.urls = append(cfg.urls, url) + cfg.saveTLSHostname(url) + } + } + // Add the configured one + cfg.urls = append(cfg.urls, cfg.URLs...) +} + +// Similar to setInfoHostPortAndGenerateJSON, but for leafNodeInfo. +func (s *Server) setLeafNodeInfoHostPortAndIP() error { + opts := s.getOpts() + if opts.LeafNode.Advertise != _EMPTY_ { + advHost, advPort, err := parseHostPort(opts.LeafNode.Advertise, opts.LeafNode.Port) + if err != nil { + return err + } + s.leafNodeInfo.Host = advHost + s.leafNodeInfo.Port = advPort + } else { + s.leafNodeInfo.Host = opts.LeafNode.Host + s.leafNodeInfo.Port = opts.LeafNode.Port + // If the host is "0.0.0.0" or "::" we need to resolve to a public IP. + // This will return at most 1 IP. + hostIsIPAny, ips, err := s.getNonLocalIPsIfHostIsIPAny(s.leafNodeInfo.Host, false) + if err != nil { + return err + } + if hostIsIPAny { + if len(ips) == 0 { + s.Errorf("Could not find any non-local IP for leafnode's listen specification %q", + s.leafNodeInfo.Host) + } else { + // Take the first from the list... + s.leafNodeInfo.Host = ips[0] + } + } + } + // Use just host:port for the IP + s.leafNodeInfo.IP = net.JoinHostPort(s.leafNodeInfo.Host, strconv.Itoa(s.leafNodeInfo.Port)) + if opts.LeafNode.Advertise != _EMPTY_ { + s.Noticef("Advertise address for leafnode is set to %s", s.leafNodeInfo.IP) + } + return nil +} + +// Add the connection to the map of leaf nodes. +// If `checkForDup` is true (invoked when a leafnode is accepted), then we check +// if a connection already exists for the same server name and account. +// That can happen when the remote is attempting to reconnect while the accepting +// side did not detect the connection as broken yet. +// But it can also happen when there is a misconfiguration and the remote is +// creating two (or more) connections that bind to the same account on the accept +// side. +// When a duplicate is found, the new connection is accepted and the old is closed +// (this solves the stale connection situation). An error is returned to help the +// remote detect the misconfiguration when the duplicate is the result of that +// misconfiguration. +func (s *Server) addLeafNodeConnection(c *client, srvName, clusterName string, checkForDup bool) bool { + var accName string + c.mu.Lock() + cid := c.cid + acc := c.acc + if acc != nil { + accName = acc.Name + } + myRemoteDomain := c.leaf.remoteDomain + mySrvName := c.leaf.remoteServer + remoteAccName := c.leaf.remoteAccName + myClustName := c.leaf.remoteCluster + remote := c.leaf.remote + solicited := remote != nil + c.mu.Unlock() + + var old *client + s.mu.Lock() + // We check for empty because in some test we may send empty CONNECT{} + if checkForDup && srvName != _EMPTY_ { + for _, ol := range s.leafs { + ol.mu.Lock() + // We care here only about non solicited Leafnode. This function + // is more about replacing stale connections than detecting loops. + // We have code for the loop detection elsewhere, which also delays + // attempt to reconnect. + if !ol.isSolicitedLeafNode() && ol.leaf.remoteServer == srvName && + ol.leaf.remoteCluster == clusterName && ol.acc.Name == accName && + remoteAccName != _EMPTY_ && ol.leaf.remoteAccName == remoteAccName { + old = ol + } + ol.mu.Unlock() + if old != nil { + break + } + } + } + // Now that we are under the server lock and before adding it to the map, + // for a solicited leaf, we need to make sure that it has not been removed + // from the config or disabled. + if solicited { + // If no longer valid, do not add to the server map. The connection + // should have been marked so that it can't reconnect. When the caller + // calls closeConnection(), cleanup (including clearing the connect- + // in-progress flag) will occur at the appropriate time. + if !remote.stillValid() { + // Prevent reconnect in case it was not yet done. + c.setNoReconnect() + s.mu.Unlock() + s.removeFromTempClients(cid) + return false + } + remote.setConnectInProgress(false) + } + // Store new connection in the map + s.leafs[cid] = c + s.mu.Unlock() + s.removeFromTempClients(cid) + + // If applicable, evict the old one. + if old != nil { + old.sendErrAndErr(DuplicateRemoteLeafnodeConnection.String()) + old.closeConnection(DuplicateRemoteLeafnodeConnection) + c.Warnf("Replacing connection from same server") + } + + srvDecorated := func() string { + if myClustName == _EMPTY_ { + return mySrvName + } + return fmt.Sprintf("%s/%s", mySrvName, myClustName) + } + + opts := s.getOpts() + sysAcc := s.SystemAccount() + js := s.getJetStream() + var meta *raft + if js != nil { + if mg := js.getMetaGroup(); mg != nil { + meta = mg.(*raft) + } + } + blockMappingOutgoing := false + // Deny (non domain) JetStream API traffic unless system account is shared + // and domain names are identical and extending is not disabled + + // Check if backwards compatibility has been enabled and needs to be acted on + forceSysAccDeny := false + if len(opts.JsAccDefaultDomain) > 0 { + if acc == sysAcc { + for _, d := range opts.JsAccDefaultDomain { + if d == _EMPTY_ { + // Extending JetStream via leaf node is mutually exclusive with a domain mapping to the empty/default domain. + // As soon as one mapping to "" is found, disable the ability to extend JS via a leaf node. + c.Noticef("Not extending remote JetStream domain %q due to presence of empty default domain", myRemoteDomain) + forceSysAccDeny = true + break + } + } + } else if domain, ok := opts.JsAccDefaultDomain[accName]; ok && domain == _EMPTY_ { + // for backwards compatibility with old setups that do not have a domain name set + c.Debugf("Skipping deny %q for account %q due to default domain", jsAllAPI, accName) + return true + } + } + + // If the server has JS disabled, it may still be part of a JetStream that could be extended. + // This is either signaled by js being disabled and a domain set, + // or in cases where no domain name exists, an extension hint is set. + // However, this is only relevant in mixed setups. + // + // If the system account connects but default domains are present, JetStream can't be extended. + if opts.JetStreamDomain != myRemoteDomain || (!opts.JetStream && (opts.JetStreamDomain == _EMPTY_ && opts.JetStreamExtHint != jsWillExtend)) || + sysAcc == nil || acc == nil || forceSysAccDeny { + // If domain names mismatch always deny. This applies to system accounts as well as non system accounts. + // Not having a system account, account or JetStream disabled is considered a mismatch as well. + if acc != nil && acc == sysAcc { + c.Noticef("System account connected from %s", srvDecorated()) + c.Noticef("JetStream not extended, domains differ") + c.mergeDenyPermissionsLocked(both, denyAllJs) + // When a remote with a system account is present in a server, unless otherwise disabled, the server will be + // started in observer mode. Now that it is clear that this not used, turn the observer mode off. + if solicited && meta != nil && meta.IsObserver() { + meta.setObserver(false, extNotExtended) + c.Debugf("Turning JetStream metadata controller Observer Mode off") + // Take note that the domain was not extended to avoid this state from startup. + writePeerState(c.srv.diskIOSemaphore(), js.config.StoreDir, meta.currentPeerState()) + // Meta controller can't be leader yet. + // Yet it is possible that due to observer mode every server already stopped campaigning. + // Therefore this server needs to be kicked into campaigning gear explicitly. + meta.Campaign() + } + } else { + c.Noticef("JetStream using domains: local %q, remote %q", opts.JetStreamDomain, myRemoteDomain) + c.mergeDenyPermissionsLocked(both, denyAllClientJs) + } + blockMappingOutgoing = true + } else if acc == sysAcc { + // system account and same domain + s.sys.client.Noticef("Extending JetStream domain %q as System Account connected from server %s", + myRemoteDomain, srvDecorated()) + // In an extension use case, pin leadership to server remotes connect to. + // Therefore, server with a remote that are not already in observer mode, need to be put into it. + if solicited && meta != nil && !meta.IsObserver() { + c.Debugf("Turning JetStream metadata controller Observer Mode on - System Account Connected") + // Discard any local metagroup state accumulated before the SYS-account + // leaf came up (e.g. the wrong-hint case where this server bootstrapped + // its own metagroup). The parent's view is now authoritative; without + // this reset the two raft logs stay forked because the standalone log's + // commit prefix short-circuits the follower's AE handling. + meta.setObserver(true, extExtended) + meta.Reset() + } + } else { + // This deny is needed in all cases (system account shared or not) + // If the system account is shared, jsAllAPI traffic will go through the system account. + // So in order to prevent duplicate delivery (from system and actual account) suppress it on the account. + // If the system account is NOT shared, jsAllAPI traffic has no business + c.Debugf("Adding deny %+v for account %q", denyAllClientJs, accName) + c.mergeDenyPermissionsLocked(both, denyAllClientJs) + } + // If we have a specified JetStream domain we will want to add a mapping to + // allow access cross domain for each non-system account. The system account + // is mapped in setupJetStreamExports, it only needs the outgoing block here. + if opts.JetStreamDomain != _EMPTY_ && opts.JetStream && acc != nil { + if acc != sysAcc { + for src, dest := range generateJSMappingTable(opts.JetStreamDomain) { + if err := acc.AddMapping(src, dest); err != nil { + c.Debugf("Error adding JetStream domain mapping: %s", err.Error()) + } else { + c.Debugf("Adding JetStream Domain Mapping %q -> %s to account %q", src, dest, accName) + } + } + } + if blockMappingOutgoing { + src := fmt.Sprintf(jsDomainAPI, opts.JetStreamDomain) + // make sure that messages intended for this domain, do not leave the cluster via this leaf node connection + // This is a guard against a miss-config with two identical domain names and will only cover some forms + // of this issue, not all of them. + // This guards against a hub and a spoke having the same domain name. + // But not two spokes having the same one and the request coming from the hub. + c.mergeDenyPermissionsLocked(pub, []string{src}) + c.Debugf("Adding deny %q for outgoing messages to account %q", src, accName) + } + } + return true +} + +func (s *Server) removeLeafNodeConnection(c *client) { + s.mu.Lock() + c.mu.Lock() + cid := c.cid + if c.leaf != nil { + if c.leaf.tsubt != nil { + c.leaf.tsubt.Stop() + c.leaf.tsubt = nil + } + if c.leaf.gwSub != nil { + s.gwLeafSubs.Remove(c.leaf.gwSub) + // We need to set this to nil for GC to release the connection + c.leaf.gwSub = nil + } + if remote := c.leaf.remote; remote != nil { + // If "noReconnect" is true, then we won't attempt to reconnect, so + // we will clear the "connect-in-progress" flag. However, if we can + // reconnect, then we should set "connect-in-progress" to true while + // we are under the server/client lock. The go routine that performs + // the reconnect will be started later and there would be a gap with + // the wrong flag value otherwise. + remote.setConnectInProgress(!c.flags.isSet(noReconnect)) + } + } + proxyKey := c.proxyKey + c.mu.Unlock() + delete(s.leafs, cid) + if proxyKey != _EMPTY_ { + s.removeProxiedConn(proxyKey, cid) + } + s.mu.Unlock() + s.removeFromTempClients(cid) +} + +// Connect information for solicited leafnodes. +type leafConnectInfo struct { + Version string `json:"version,omitempty"` + Nkey string `json:"nkey,omitempty"` + JWT string `json:"jwt,omitempty"` + Sig string `json:"sig,omitempty"` + User string `json:"user,omitempty"` + Pass string `json:"pass,omitempty"` + Token string `json:"auth_token,omitempty"` + ID string `json:"server_id,omitempty"` + Domain string `json:"domain,omitempty"` + Name string `json:"name,omitempty"` + Hub bool `json:"is_hub,omitempty"` + Cluster string `json:"cluster,omitempty"` + Headers bool `json:"headers,omitempty"` + JetStream bool `json:"jetstream,omitempty"` + DenyPub []string `json:"deny_pub,omitempty"` + Isolate bool `json:"isolate,omitempty"` + + // There was an existing field called: + // >> Comp bool `json:"compression,omitempty"` + // that has never been used. With support for compression, we now need + // a field that is a string. So we use a different json tag: + Compression string `json:"compress_mode,omitempty"` + + // Just used to detect wrong connection attempts. + Gateway string `json:"gateway,omitempty"` + + // Tells the accept side which account the remote is binding to. + RemoteAccount string `json:"remote_account,omitempty"` + + // The accept side of a LEAF connection, unlike ROUTER and GATEWAY, receives + // only the CONNECT protocol, and no INFO. So we need to send the protocol + // version as part of the CONNECT. It will indicate if a connection supports + // some features, such as message tracing. + // We use `protocol` as the JSON tag, so this is automatically unmarshal'ed + // in the low level process CONNECT. + Proto int `json:"protocol,omitempty"` +} + +// processLeafNodeConnect will process the inbound connect args. +// Once we are here we are bound to an account, so can send any interest that +// we would have to the other side. +func (c *client) processLeafNodeConnect(s *Server, arg []byte, lang string) error { + // Way to detect clients that incorrectly connect to the route listen + // port. Client provided "lang" in the CONNECT protocol while LEAFNODEs don't. + if lang != _EMPTY_ { + c.sendErrAndErr(ErrClientConnectedToLeafNodePort.Error()) + c.closeConnection(WrongPort) + return ErrClientConnectedToLeafNodePort + } + + // Unmarshal as a leaf node connect protocol + proto := &leafConnectInfo{} + if err := json.Unmarshal(arg, proto); err != nil { + return err + } + + // Reject a cluster that contains spaces. + if proto.Cluster != _EMPTY_ && strings.Contains(proto.Cluster, " ") { + c.sendErrAndErr(ErrClusterNameHasSpaces.Error()) + c.closeConnection(ProtocolViolation) + return ErrClusterNameHasSpaces + } + if proto.Cluster == leafNoOriginCluster { + c.sendErrAndErr(ErrClusterNameReserved.Error()) + c.closeConnection(ProtocolViolation) + return ErrClusterNameReserved + } + + // Check for cluster name collisions. + if cn := s.cachedClusterName(); cn != _EMPTY_ && proto.Cluster != _EMPTY_ && proto.Cluster == cn { + c.sendErrAndErr(ErrLeafNodeHasSameClusterName.Error()) + c.closeConnection(ClusterNamesIdentical) + return ErrLeafNodeHasSameClusterName + } + + // Reject if this has Gateway which means that it would be from a gateway + // connection that incorrectly connects to the leafnode port. + if proto.Gateway != _EMPTY_ { + errTxt := fmt.Sprintf("Rejecting connection from gateway %q on the leafnode port", proto.Gateway) + c.Errorf(errTxt) + c.sendErr(errTxt) + c.closeConnection(WrongGateway) + return ErrWrongGateway + } + + if mv := s.getOpts().LeafNode.MinVersion; mv != _EMPTY_ { + major, minor, update, _ := versionComponents(mv) + if !versionAtLeast(proto.Version, major, minor, update) { + // Send back an INFO so recent remote servers process the rejection + // cleanly, then close immediately. The soliciting side applies the + // reconnect delay when it processes the error. + s.sendPermsAndAccountInfo(c) + c.sendErrAndErr(fmt.Sprintf("%s %q", ErrLeafNodeMinVersionRejected, mv)) + c.closeConnection(MinimumVersionRequired) + return ErrMinimumVersionRequired + } + } + + // Check if this server supports headers. + supportHeaders := c.srv.supportsHeaders() + + c.mu.Lock() + // Leaf Nodes do not do echo or verbose or pedantic. + c.opts.Verbose = false + c.opts.Echo = false + c.opts.Pedantic = false + // This inbound connection will be marked as supporting headers if this server + // support headers and the remote has sent in the CONNECT protocol that it does + // support headers too. + c.headers = supportHeaders && proto.Headers + // If the compression level is still not set, set it based on what has been + // given to us in the CONNECT protocol. + if c.leaf.compression == _EMPTY_ { + // But if proto.Compression is _EMPTY_, set it to CompressionNotSupported + if proto.Compression == _EMPTY_ { + c.leaf.compression = CompressionNotSupported + } else { + c.leaf.compression = proto.Compression + } + } + + // Remember the remote server. + c.leaf.remoteServer = proto.Name + // Remember the remote account name + c.leaf.remoteAccName = proto.RemoteAccount + // Remember if the leafnode requested isolation. + c.leaf.isolated = c.leaf.isolated || proto.Isolate + + // If the other side has declared itself a hub, so we will take on the spoke role. + if proto.Hub { + c.leaf.isSpoke = true + } + + // The soliciting side is part of a cluster. + if proto.Cluster != _EMPTY_ { + c.leaf.remoteCluster = proto.Cluster + } + + c.leaf.remoteDomain = proto.Domain + + // When a leaf solicits a connection to a hub, the perms that it will use on the soliciting leafnode's + // behalf are correct for them, but inside the hub need to be reversed since data is flowing in the opposite direction. + if !c.isSolicitedLeafNode() && c.perms != nil { + sp, pp := c.perms.sub, c.perms.pub + c.perms.sub, c.perms.pub = pp, sp + // setPermissions populated darray from the subscribe permissions, + // which are the import permissions advertised to the spoke. Keep + // those parsed denies after reversing the live permission directions. + if c.opts.Import == nil { + c.darray = nil + } + } + + // Set the Ping timer + c.setFirstPingTimer() + + // If we received pub deny permissions from the other end, merge with existing ones. + c.mergeDenyPermissions(pub, proto.DenyPub) + + acc := c.acc + c.mu.Unlock() + + // If the account is not set (e.g. connection was closed due to auth + // timeout while still being processed), bail out to avoid a panic. + if acc == nil { + c.closeConnection(MissingAccount) + return ErrMissingAccount + } + + // Register the cluster, even if empty, as long as we are acting as a hub. + if !proto.Hub { + acc.registerLeafNodeCluster(proto.Cluster) + } + + // Add in the leafnode here since we passed through auth at this point. + s.addLeafNodeConnection(c, proto.Name, proto.Cluster, true) + + // If we have permissions bound to this leafnode we need to send then back to the + // origin server for local enforcement. + s.sendPermsAndAccountInfo(c) + + // Create and initialize the smap since we know our bound account now. + // This will send all registered subs too. + s.initLeafNodeSmapAndSendSubs(c) + + // Announce the account connect event for a leaf node. + // This will be a no-op as needed. + s.sendLeafNodeConnect(c.acc) + + // Check to see if we need to kick any internal source or mirror consumers. + // This will be a no-op if JetStream not enabled for this server or if the bound account + // does not have jetstream. + s.checkInternalSyncConsumers(acc) + + return nil +} + +// checkInternalSyncConsumers +func (s *Server) checkInternalSyncConsumers(acc *Account) { + // Grab our js + js := s.getJetStream() + + // Only applicable if we have JS and the leafnode has JS as well. + // We check for remote JS outside. + if !js.isEnabled() || acc == nil { + return + } + + // We will check all streams in our local account. They must be a leader and + // be sourcing or mirroring. We will check the external config on the stream itself + // if this is cross domain, or if the remote domain is empty, meaning we might be + // extending the system across this leafnode connection and hence we would be extending + // our own domain. + jsa := js.lookupAccount(acc) + if jsa == nil { + return + } + + var streams []*stream + jsa.mu.RLock() + for _, mset := range jsa.streams { + mset.cfgMu.RLock() + // We need to have a mirror or source defined. + // We do not want to force another lock here to look for leader status, + // so collect and after we release jsa will make sure. + if mset.cfg.Mirror != nil || len(mset.cfg.Sources) > 0 { + streams = append(streams, mset) + } + mset.cfgMu.RUnlock() + } + jsa.mu.RUnlock() + + // Now loop through all candidates and check if we are the leader and have NOT + // created the sync up consumer. + for _, mset := range streams { + mset.retryDisconnectedSyncConsumers() + } +} + +// Returns the remote cluster name. This is set only once so does not require a lock. +func (c *client) remoteCluster() string { + if c.leaf == nil { + return _EMPTY_ + } + return c.leaf.remoteCluster +} + +// Sends back an info block to the soliciting leafnode to let it know about +// its permission settings for local enforcement. +func (s *Server) sendPermsAndAccountInfo(c *client) { + // Copy + s.mu.Lock() + info := s.copyLeafNodeInfo() + s.mu.Unlock() + c.mu.Lock() + info.CID = c.cid + info.Import = c.opts.Import + info.Export = c.opts.Export + info.RemoteAccount = c.acc.Name + // s.SystemAccount() uses an atomic operation and does not get the server lock, so this is safe. + info.IsSystemAccount = c.acc == s.SystemAccount() + info.ConnectInfo = true + c.enqueueProto(generateInfoJSON(info)) + c.mu.Unlock() +} + +// Snapshot the current subscriptions from the sublist into our smap which +// we will keep updated from now on. +// Also send the registered subscriptions. +func (s *Server) initLeafNodeSmapAndSendSubs(c *client) { + acc := c.acc + if acc == nil { + c.Debugf("Leafnode does not have an account bound") + return + } + // Collect all account subs here. + _subs := [1024]*subscription{} + subs := _subs[:0] + ims := []string{} + + // Hold the client lock otherwise there can be a race and miss some subs. + c.mu.Lock() + defer c.mu.Unlock() + + acc.mu.RLock() + accName := acc.Name + accNTag := acc.nameTag + + // To make printing look better when no friendly name present. + if accNTag != _EMPTY_ { + accNTag = "/" + accNTag + } + + // If we are solicited we only send interest for local clients. + if c.isSpokeLeafNode() { + acc.sl.localSubs(&subs, true) + } else { + acc.sl.All(&subs) + } + + // Check if we have an existing service import reply. + siReply := copyBytes(acc.siReply) + + // Since leaf nodes only send on interest, if the bound + // account has import services we need to send those over. + for isubj := range acc.imports.services { + if c.isSpokeLeafNode() && !c.canSubscribe(isubj) { + c.Debugf("Not permitted to import service %q on behalf of %s%s", isubj, accName, accNTag) + continue + } + ims = append(ims, isubj) + } + // Likewise for mappings. + for _, m := range acc.mappings { + if c.isSpokeLeafNode() && !c.canSubscribe(m.src) { + c.Debugf("Not permitted to import mapping %q on behalf of %s%s", m.src, accName, accNTag) + continue + } + ims = append(ims, m.src) + } + + // Create a unique subject that will be used for loop detection. + lds := acc.lds + acc.mu.RUnlock() + + // Check if we have to create the LDS. + if lds == _EMPTY_ { + lds = leafNodeLoopDetectionSubjectPrefix + nuid.Next() + acc.mu.Lock() + acc.lds = lds + acc.mu.Unlock() + } + + // Now check for gateway interest. Leafnodes will put this into + // the proper mode to propagate, but they are not held in the account. + gwsa := [16]*client{} + gws := gwsa[:0] + s.getOutboundGatewayConnections(&gws) + for _, cgw := range gws { + cgw.mu.Lock() + gw := cgw.gw + cgw.mu.Unlock() + if gw != nil { + if ei, _ := gw.outsim.Load(accName); ei != nil { + if e := ei.(*outsie); e != nil && e.sl != nil { + e.sl.All(&subs) + } + } + } + } + + applyGlobalRouting := s.gateway.enabled + if c.isSpokeLeafNode() { + // Add a fake subscription for this solicited leafnode connection + // so that we can send back directly for mapped GW replies. + // We need to keep track of this subscription so it can be removed + // when the connection is closed so that the GC can release it. + c.leaf.gwSub = &subscription{client: c, subject: []byte(gwReplyPrefix + ">")} + c.srv.gwLeafSubs.Insert(c.leaf.gwSub) + } + + // Now walk the results and add them to our smap + rc := c.leaf.remoteCluster + c.leaf.smap = make(map[string]int32) + for _, sub := range subs { + // Check perms regardless of role. + if c.perms != nil && !c.canSubscribe(string(sub.subject)) { + c.Debugf("Not permitted to subscribe to %q on behalf of %s%s", sub.subject, accName, accNTag) + continue + } + // Don't advertise interest from leafnodes to other isolated leafnodes. + if (sub.client.kind == LEAF || sub.leaf) && c.isIsolatedLeafNode() { + continue + } + // We ignore ourselves here. + // Also don't add the subscription if it has a origin cluster and the + // cluster name matches the one of the client we are sending to. + if c != sub.client && (sub.origin == nil || (bytesToString(sub.origin) != rc)) { + count := int32(1) + if len(sub.queue) > 0 && sub.qw > 0 { + count = sub.qw + } + c.leaf.smap[keyFromSub(sub)] += count + if c.leaf.tsub == nil { + c.leaf.tsub = make(map[*subscription]struct{}) + } + c.leaf.tsub[sub] = struct{}{} + } + } + // FIXME(dlc) - We need to update appropriately on an account claims update. + for _, isubj := range ims { + c.leaf.smap[isubj]++ + } + // If we have gateways enabled we need to make sure the other side sends us responses + // that have been augmented from the original subscription. + // TODO(dlc) - Should we lock this down more? + if applyGlobalRouting { + c.leaf.smap[oldGWReplyPrefix+"*.>"]++ + c.leaf.smap[gwReplyPrefix+">"]++ + } + // Detect loops by subscribing to a specific subject and checking + // if this sub is coming back to us. + c.leaf.smap[lds]++ + + // Check if we need to add an existing siReply to our map. + // This will be a prefix so add on the wildcard. + if siReply != nil { + wcsub := append(siReply, '>') + c.leaf.smap[string(wcsub)]++ + } + // Queue all protocols. There is no max pending limit for LN connection, + // so we don't need chunking. The writes will happen from the writeLoop. + var b bytes.Buffer + for key, n := range c.leaf.smap { + c.writeLeafSub(&b, key, n) + } + if b.Len() > 0 { + c.enqueueProto(b.Bytes()) + } + if c.leaf.tsub != nil { + // Clear the tsub map after 5 seconds. + c.leaf.tsubt = time.AfterFunc(5*time.Second, func() { + c.mu.Lock() + if c.leaf != nil { + c.leaf.tsub = nil + c.leaf.tsubt = nil + } + c.mu.Unlock() + }) + } +} + +// updateInterestForAccountOnGateway called from gateway code when processing RS+ and RS-. +func (s *Server) updateInterestForAccountOnGateway(accName string, sub *subscription, delta int32) { + // Since we're in the gateway's readLoop, and we would otherwise block, don't allow fetching. + acc, err := s.lookupOrFetchAccount(accName, false) + if acc == nil || err != nil { + s.Debugf("No or bad account for %q, failed to update interest from gateway", accName) + return + } + acc.updateLeafNodes(sub, delta) +} + +// updateLeafNodesEx will make sure to update the account smap for the subscription. +// Will also forward to all leaf nodes as needed. +// If `hubOnly` is true, then will update only leaf nodes that connect to this server +// (that is, for which this server acts as a hub to them). +func (acc *Account) updateLeafNodesEx(sub *subscription, delta int32, hubOnly bool) { + if acc == nil || sub == nil { + return + } + + // We will do checks for no leafnodes and same cluster here inline and under the + // general account read lock. + // If we feel we need to update the leafnodes we will do that out of line to avoid + // blocking routes or GWs. + + acc.mu.RLock() + // First check if we even have leafnodes here. + if acc.nleafs == 0 { + acc.mu.RUnlock() + return + } + + // Is this a loop detection subject. + isLDS := bytes.HasPrefix(sub.subject, []byte(leafNodeLoopDetectionSubjectPrefix)) + + // Capture the cluster even if its empty. + var cluster string + if sub.origin != nil { + cluster = bytesToString(sub.origin) + } + + // If we have an isolated cluster we can return early, as long as it is not a loop detection subject. + // Empty clusters will return false for the check. + if !isLDS && acc.isLeafNodeClusterIsolated(cluster) { + acc.mu.RUnlock() + return + } + + // We can release the general account lock. + acc.mu.RUnlock() + + // We can hold the list lock here to avoid having to copy a large slice. + acc.lmu.RLock() + defer acc.lmu.RUnlock() + + // Do this once. + subject := string(sub.subject) + + // Walk the connected leafnodes from a random starting point to avoid + // concurrent callers all contending over leafs in the same order. + nleafs := len(acc.lleafs) + start := 0 + if nleafs > 1 { + start = rand.IntN(nleafs) + } + for i := 0; i < nleafs; i++ { + ln := acc.lleafs[(start+i)%nleafs] + if ln == sub.client { + continue + } + ln.mu.RLock() + // Don't advertise interest from leafnodes to other isolated leafnodes. + if (sub.client.kind == LEAF || sub.leaf) && ln.isIsolatedLeafNode() { + ln.mu.RUnlock() + continue + } + // If `hubOnly` is true, it means that we want to update only leafnodes + // that connect to this server (so isHubLeafNode() would return `true`). + if hubOnly && !ln.isHubLeafNode() { + ln.mu.RUnlock() + continue + } + // Check to make sure this sub does not have an origin cluster that matches the leafnode. + // If skipped, make sure that we still let go the "$LDS." subscription that allows + // the detection of loops as long as different cluster. + clusterDifferent := cluster != ln.remoteCluster() + update := (isLDS && clusterDifferent) || + ((cluster == _EMPTY_ || clusterDifferent) && (delta <= 0 || ln.canSubscribeInternal(subject))) + ln.mu.RUnlock() + if update { + ln.mu.Lock() + // The leaf role, isolation mode, and remote cluster are stable + // for the connection. Recheck canSubscribe here since permissions + // can change, and to initializes mperms for wildcard subscriptions + // that collide with deny rules. + if isLDS || delta <= 0 || ln.canSubscribe(subject) { + ln.updateSmap(sub, delta, isLDS) + } + ln.mu.Unlock() + } + } +} + +// updateLeafNodes will make sure to update the account smap for the subscription. +// Will also forward to all leaf nodes as needed. +func (acc *Account) updateLeafNodes(sub *subscription, delta int32) { + acc.updateLeafNodesEx(sub, delta, false) +} + +// This will make an update to our internal smap and determine if we should send out +// an interest update to the remote side. +// Lock should be held. +func (c *client) updateSmap(sub *subscription, delta int32, isLDS bool) { + if c.leaf.smap == nil { + return + } + + // If we are solicited make sure this is a local client or a non-solicited leaf node + skind := sub.client.kind + updateClient := skind == CLIENT || skind == SYSTEM || skind == JETSTREAM || skind == ACCOUNT + if !isLDS && c.isSpokeLeafNode() && !(updateClient || (skind == LEAF && !sub.client.isSpokeLeafNode())) { + return + } + + // For additions, check if that sub has just been processed during initLeafNodeSmapAndSendSubs + if delta > 0 && c.leaf.tsub != nil { + if _, present := c.leaf.tsub[sub]; present { + delete(c.leaf.tsub, sub) + if len(c.leaf.tsub) == 0 { + c.leaf.tsub = nil + c.leaf.tsubt.Stop() + c.leaf.tsubt = nil + } + return + } + } + + key := keyFromSub(sub) + n, ok := c.leaf.smap[key] + if delta < 0 && !ok { + return + } + + // We will update if its a queue, if count is zero (or negative), or we were 0 and are N > 0. + update := sub.queue != nil || (n <= 0 && n+delta > 0) || (n > 0 && n+delta <= 0) + n += delta + if n > 0 { + c.leaf.smap[key] = n + } else { + delete(c.leaf.smap, key) + } + if update { + c.sendLeafNodeSubUpdate(key, n) + } +} + +// Used to force add subjects to the subject map. +func (c *client) forceAddToSmap(subj string) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.leaf.smap == nil { + return + } + n := c.leaf.smap[subj] + if n != 0 { + return + } + // Place into the map since it was not there. + c.leaf.smap[subj] = 1 + c.sendLeafNodeSubUpdate(subj, 1) +} + +// Used to force remove a subject from the subject map. +func (c *client) forceRemoveFromSmap(subj string) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.leaf.smap == nil { + return + } + n := c.leaf.smap[subj] + if n == 0 { + return + } + n-- + if n == 0 { + // Remove is now zero + delete(c.leaf.smap, subj) + c.sendLeafNodeSubUpdate(subj, 0) + } else { + c.leaf.smap[subj] = n + } +} + +// Send the subscription interest change to the other side. +// Lock should be held. +func (c *client) sendLeafNodeSubUpdate(key string, n int32) { + // If we are a spoke, we need to check if we are allowed to send this subscription over to the hub. + if c.isSpokeLeafNode() { + checkPerms := true + if len(key) > 0 && (key[0] == '$' || key[0] == '_') { + if strings.HasPrefix(key, leafNodeLoopDetectionSubjectPrefix) || + strings.HasPrefix(key, oldGWReplyPrefix) || + strings.HasPrefix(key, gwReplyPrefix) { + checkPerms = false + } + } + if checkPerms { + var subject string + if sep := strings.IndexByte(key, ' '); sep != -1 { + subject = key[:sep] + } else { + subject = key + } + if !c.canSubscribe(subject) { + return + } + } + } + // If we are here we can send over to the other side. + _b := [64]byte{} + b := bytes.NewBuffer(_b[:0]) + c.writeLeafSub(b, key, n) + c.enqueueProto(b.Bytes()) +} + +// Helper function to build the key. +func keyFromSub(sub *subscription) string { + var sb strings.Builder + sb.Grow(len(sub.subject) + len(sub.queue) + 1) + sb.Write(sub.subject) + if sub.queue != nil { + // Just make the key subject spc group, e.g. 'foo bar' + sb.WriteByte(' ') + sb.Write(sub.queue) + } + return sb.String() +} + +const ( + keyRoutedSub = "R" + keyRoutedSubByte = 'R' + keyRoutedLeafSub = "L" + keyRoutedLeafSubByte = 'L' + keyRoutedLeafNoOriginSub = "N" + keyRoutedLeafNoOriginSubByte = 'N' +) + +// Helper function to build the key that prevents collisions between normal +// routed subscriptions and routed subscriptions on behalf of a leafnode. +// Keys will look like this: +// "R foo" -> plain routed sub on "foo" +// "R foo bar" -> queue routed sub on "foo", queue "bar" +// "N foo" -> plain routed leaf sub on "foo" without an origin +// "N foo bar" -> queue routed leaf sub on "foo", queue "bar", without an origin +// "L foo bar" -> plain routed leaf sub on "foo", leaf "bar" +// "L foo bar baz" -> queue routed sub on "foo", queue "bar", leaf "baz" +func keyFromSubWithOrigin(sub *subscription) string { + var sb strings.Builder + sb.Grow(2 + len(sub.origin) + 1 + len(sub.subject) + 1 + len(sub.queue)) + hasOrigin := len(sub.origin) > 0 + if hasOrigin { + sb.WriteByte(keyRoutedLeafSubByte) + } else if sub.leaf { + sb.WriteByte(keyRoutedLeafNoOriginSubByte) + } else { + sb.WriteByte(keyRoutedSubByte) + } + sb.WriteByte(' ') + sb.Write(sub.subject) + if sub.queue != nil { + sb.WriteByte(' ') + sb.Write(sub.queue) + } + if hasOrigin { + sb.WriteByte(' ') + sb.Write(sub.origin) + } + return sb.String() +} + +// Lock should be held. +func (c *client) writeLeafSub(w *bytes.Buffer, key string, n int32) { + if key == _EMPTY_ { + return + } + if n > 0 { + w.WriteString("LS+ " + key) + // Check for queue semantics, if found write n. + if strings.Contains(key, " ") { + w.WriteString(" ") + var b [12]byte + var i = len(b) + for l := n; l > 0; l /= 10 { + i-- + b[i] = digits[l%10] + } + w.Write(b[i:]) + if c.trace { + arg := fmt.Sprintf("%s %d", key, n) + c.traceOutOp("LS+", []byte(arg)) + } + } else if c.trace { + c.traceOutOp("LS+", []byte(key)) + } + } else { + w.WriteString("LS- " + key) + if c.trace { + c.traceOutOp("LS-", []byte(key)) + } + } + w.WriteString(CR_LF) +} + +// processLeafSub will process an inbound sub request for the remote leaf node. +func (c *client) processLeafSub(argo []byte) (err error) { + // Indicate activity. + c.in.subs++ + + srv := c.srv + if srv == nil { + return nil + } + + // Copy so we do not reference a potentially large buffer + arg := make([]byte, len(argo)) + copy(arg, argo) + + args := splitArg(arg) + sub := &subscription{client: c, leaf: true} + + delta := int32(1) + switch len(args) { + case 1: + sub.queue = nil + case 3: + sub.queue = args[1] + sub.qw = int32(parseSize(args[2])) + // TODO: (ik) We should have a non empty queue name and a queue + // weight >= 1. For 2.11, we may want to return an error if that + // is not the case, but for now just overwrite `delta` if queue + // weight is greater than 1 (it is possible after a reconnect/ + // server restart to receive a queue weight > 1 for a new sub). + if sub.qw > 1 { + delta = sub.qw + } + default: + return fmt.Errorf("processLeafSub Parse Error: '%s'", arg) + } + sub.subject = args[0] + + c.mu.Lock() + if c.isClosed() { + c.mu.Unlock() + return nil + } + + acc := c.acc + // Guard against LS+ arriving before CONNECT has been processed, which + // can happen when compression is enabled. + if acc == nil { + c.mu.Unlock() + c.sendErr("Authorization Violation") + c.closeConnection(ProtocolViolation) + return nil + } + // Check if we have a loop. + ldsPrefix := bytes.HasPrefix(sub.subject, []byte(leafNodeLoopDetectionSubjectPrefix)) + + if ldsPrefix && bytesToString(sub.subject) == acc.getLDSubject() { + c.mu.Unlock() + c.handleLeafNodeLoop(true) + return nil + } + + // Check permissions if applicable. (but exclude the $LDS, $GR and _GR_) + checkPerms := true + if sub.subject[0] == '$' || sub.subject[0] == '_' { + if ldsPrefix || + bytes.HasPrefix(sub.subject, []byte(oldGWReplyPrefix)) || + bytes.HasPrefix(sub.subject, []byte(gwReplyPrefix)) { + checkPerms = false + } + } + + // If we are a hub check that we can publish to this subject. + if checkPerms { + subj := string(sub.subject) + if subjectIsLiteral(subj) && !c.pubAllowedFullCheck(subj, true, true) { + c.mu.Unlock() + c.leafSubPermViolation(sub.subject) + c.Debugf(fmt.Sprintf("Permissions Violation for Subscription to %q", sub.subject)) + return nil + } + c.loadMsgDenyFilterIfNeeded(subj, len(sub.queue) > 0) + } + + // Check if we have a maximum on the number of subscriptions. + if c.subsAtLimit() { + c.mu.Unlock() + c.maxSubsExceeded() + return nil + } + + // If we have an origin cluster associated mark that in the sub. + if rc := c.remoteCluster(); rc != _EMPTY_ { + sub.origin = []byte(rc) + } + + // Like Routes, we store local subs by account and subject and optionally queue name. + // If we have a queue it will have a trailing weight which we do not want. + if sub.queue != nil { + sub.sid = arg[:len(arg)-len(args[2])-1] + } else { + sub.sid = arg + } + key := bytesToString(sub.sid) + osub := c.subs[key] + if osub == nil { + c.subs[key] = sub + // Now place into the account sl. + if err := acc.sl.Insert(sub); err != nil { + delete(c.subs, key) + c.mu.Unlock() + c.Errorf("Could not insert subscription: %v", err) + c.sendErr("Invalid Subscription") + return nil + } + } else if sub.queue != nil { + // For a queue we need to update the weight. + delta = sub.qw - atomic.LoadInt32(&osub.qw) + atomic.StoreInt32(&osub.qw, sub.qw) + acc.sl.UpdateRemoteQSub(osub) + } + spoke := c.isSpokeLeafNode() + c.mu.Unlock() + + // Only add in shadow subs if a new sub or qsub. + if osub == nil { + if err := c.addShadowSubscriptions(acc, sub); err != nil { + c.Errorf(err.Error()) + } + } + + // If we are not solicited, treat leaf node subscriptions similar to a + // client subscription, meaning we forward them to routes, gateways and + // other leaf nodes as needed. + if !spoke { + // If we are routing add to the route map for the associated account. + srv.updateRouteSubscriptionMap(acc, sub, delta) + if srv.gateway.enabled { + srv.gatewayUpdateSubInterest(acc.Name, sub, delta) + } + } + // Now check on leafnode updates for other leaf nodes. We understand solicited + // and non-solicited state in this call so we will do the right thing. + acc.updateLeafNodes(sub, delta) + + return nil +} + +// If the leafnode is a solicited, set the connect delay based on default +// or private option (for tests). Sends the error to the other side, log and +// close the connection. +func (c *client) handleLeafNodeLoop(sendErr bool) { + accName, delay := c.setLeafConnectDelayIfSoliciting(leafNodeReconnectDelayAfterLoopDetected) + errTxt := fmt.Sprintf("Loop detected for leafnode account=%q. Delaying attempt to reconnect for %v", accName, delay) + if sendErr { + c.sendErr(errTxt) + } + + c.Errorf(errTxt) + // If we are here with "sendErr" false, it means that this is the server + // that received the error. The other side will have closed the connection, + // but does not hurt to close here too. + c.closeConnection(ProtocolViolation) +} + +// processLeafUnsub will process an inbound unsub request for the remote leaf node. +func (c *client) processLeafUnsub(arg []byte) error { + // Indicate any activity, so pub and sub or unsubs. + c.in.subs++ + + srv := c.srv + + c.mu.Lock() + if c.isClosed() { + c.mu.Unlock() + return nil + } + + acc := c.acc + // Guard against LS- arriving before CONNECT has been processed. + if acc == nil { + c.mu.Unlock() + c.sendErr("Authorization Violation") + c.closeConnection(ProtocolViolation) + return nil + } + + spoke := c.isSpokeLeafNode() + // We store local subs by account and subject and optionally queue name. + // LS- will have the arg exactly as the key. + sub, ok := c.subs[string(arg)] + if !ok { + // If not found, don't try to update routes/gws/leaf nodes. + c.mu.Unlock() + return nil + } + delta := int32(1) + if len(sub.queue) > 0 { + delta = sub.qw + } + c.mu.Unlock() + + c.unsubscribe(acc, sub, true, true) + if !spoke { + // If we are routing subtract from the route map for the associated account. + srv.updateRouteSubscriptionMap(acc, sub, -delta) + // Gateways + if srv.gateway.enabled { + srv.gatewayUpdateSubInterest(acc.Name, sub, -delta) + } + } + // Now check on leafnode updates for other leaf nodes. + acc.updateLeafNodes(sub, -delta) + return nil +} + +func (c *client) processLeafHeaderMsgArgs(arg []byte) error { + // Unroll splitArgs to avoid runtime/heap issues + args := c.argsa[:0] + start := -1 + for i, b := range arg { + switch b { + case ' ', '\t', '\r', '\n': + if start >= 0 { + args = append(args, arg[start:i]) + start = -1 + } + default: + if start < 0 { + start = i + } + } + } + if start >= 0 { + args = append(args, arg[start:]) + } + + c.pa.arg = arg + switch len(args) { + case 0, 1, 2: + return fmt.Errorf("processLeafHeaderMsgArgs Parse Error: '%s'", args) + case 3: + c.pa.reply = nil + c.pa.queues = nil + c.pa.hdb = args[1] + c.pa.hdr = parseSize(args[1]) + c.pa.szb = args[2] + c.pa.size = parseSize(args[2]) + case 4: + c.pa.reply = args[1] + c.pa.queues = nil + c.pa.hdb = args[2] + c.pa.hdr = parseSize(args[2]) + c.pa.szb = args[3] + c.pa.size = parseSize(args[3]) + default: + // args[1] is our reply indicator. Should be + or | normally. + if len(args[1]) != 1 { + return fmt.Errorf("processLeafHeaderMsgArgs Bad or Missing Reply Indicator: '%s'", args[1]) + } + switch args[1][0] { + case '+': + c.pa.reply = args[2] + case '|': + c.pa.reply = nil + default: + return fmt.Errorf("processLeafHeaderMsgArgs Bad or Missing Reply Indicator: '%s'", args[1]) + } + // Grab header size. + c.pa.hdb = args[len(args)-2] + c.pa.hdr = parseSize(c.pa.hdb) + + // Grab size. + c.pa.szb = args[len(args)-1] + c.pa.size = parseSize(c.pa.szb) + + // Grab queue names. + if c.pa.reply != nil { + c.pa.queues = args[3 : len(args)-2] + } else { + c.pa.queues = args[2 : len(args)-2] + } + } + if c.pa.hdr < 0 { + return fmt.Errorf("processLeafHeaderMsgArgs Bad or Missing Header Size: '%s'", arg) + } + if c.pa.size < 0 { + return fmt.Errorf("processLeafHeaderMsgArgs Bad or Missing Size: '%s'", args) + } + if c.pa.hdr > c.pa.size { + return fmt.Errorf("processLeafHeaderMsgArgs Header Size larger then TotalSize: '%s'", arg) + } + maxPayload := atomic.LoadInt32(&c.mpay) + if maxPayload != jwt.NoLimit && int64(c.pa.size) > int64(maxPayload) { + c.maxPayloadViolation(c.pa.size, maxPayload) + return ErrMaxPayload + } + + // Common ones processed after check for arg length + c.pa.subject = args[0] + + return nil +} + +func (c *client) processLeafMsgArgs(arg []byte) error { + // Unroll splitArgs to avoid runtime/heap issues + args := c.argsa[:0] + start := -1 + for i, b := range arg { + switch b { + case ' ', '\t', '\r', '\n': + if start >= 0 { + args = append(args, arg[start:i]) + start = -1 + } + default: + if start < 0 { + start = i + } + } + } + if start >= 0 { + args = append(args, arg[start:]) + } + + c.pa.arg = arg + switch len(args) { + case 0, 1: + return fmt.Errorf("processLeafMsgArgs Parse Error: '%s'", args) + case 2: + c.pa.reply = nil + c.pa.queues = nil + c.pa.szb = args[1] + c.pa.size = parseSize(args[1]) + case 3: + c.pa.reply = args[1] + c.pa.queues = nil + c.pa.szb = args[2] + c.pa.size = parseSize(args[2]) + default: + // args[1] is our reply indicator. Should be + or | normally. + if len(args[1]) != 1 { + return fmt.Errorf("processLeafMsgArgs Bad or Missing Reply Indicator: '%s'", args[1]) + } + switch args[1][0] { + case '+': + c.pa.reply = args[2] + case '|': + c.pa.reply = nil + default: + return fmt.Errorf("processLeafMsgArgs Bad or Missing Reply Indicator: '%s'", args[1]) + } + // Grab size. + c.pa.szb = args[len(args)-1] + c.pa.size = parseSize(c.pa.szb) + + // Grab queue names. + if c.pa.reply != nil { + c.pa.queues = args[3 : len(args)-1] + } else { + c.pa.queues = args[2 : len(args)-1] + } + } + if c.pa.size < 0 { + return fmt.Errorf("processLeafMsgArgs Bad or Missing Size: '%s'", args) + } + maxPayload := atomic.LoadInt32(&c.mpay) + if maxPayload != jwt.NoLimit && int64(c.pa.size) > int64(maxPayload) { + c.maxPayloadViolation(c.pa.size, maxPayload) + return ErrMaxPayload + } + + // Common ones processed after check for arg length + c.pa.subject = args[0] + + return nil +} + +// processInboundLeafMsg is called to process an inbound msg from a leaf node. +func (c *client) processInboundLeafMsg(msg []byte) { + // Update statistics + // The msg includes the CR_LF, so pull back out for accounting. + c.in.msgs++ + c.in.bytes += int32(len(msg) - LEN_CR_LF) + + srv, acc, subject := c.srv, c.acc, string(c.pa.subject) + + // Mostly under testing scenarios. + if srv == nil || acc == nil { + return + } + + // Check that leaf messages respect the subject permissions. + if c.perms != nil && !c.leafMsgAllowed() { + c.leafPubPermViolation(c.pa.subject) + return + } + + // Match the subscriptions. We will use our own L1 map if + // it's still valid, avoiding contention on the shared sublist. + var r *SublistResult + var ok bool + + genid := atomic.LoadUint64(&c.acc.sl.genid) + if genid == c.in.genid && c.in.results != nil { + r, ok = c.in.results[subject] + } else { + // Reset our L1 completely. + c.in.results = make(map[string]*SublistResult) + c.in.genid = genid + } + + // Go back to the sublist data structure. + if !ok { + r = c.acc.sl.Match(subject) + // Prune the results cache. Keeps us from unbounded growth. Random delete. + if len(c.in.results) >= maxResultCacheSize { + n := 0 + for subj := range c.in.results { + delete(c.in.results, subj) + if n++; n > pruneSize { + break + } + } + } + // Then add the new cache entry. + c.in.results[subject] = r + } + + // Collect queue names if needed. + var qnames [][]byte + + // Check for no interest, short circuit if so. + // This is the fanout scale. + if len(r.psubs)+len(r.qsubs) > 0 { + flag := pmrNoFlag + // If we have queue subs in this cluster, then if we run in gateway + // mode and the remote gateways have queue subs, then we need to + // collect the queue groups this message was sent to so that we + // exclude them when sending to gateways. + if len(r.qsubs) > 0 && c.srv.gateway.enabled && + atomic.LoadInt64(&c.srv.gateway.totalQSubs) > 0 { + flag |= pmrCollectQueueNames + } + // If this is a mapped subject that means the mapped interest + // is what got us here, but this might not have a queue designation + // If that is the case, make sure we ignore to process local queue subscribers. + if len(c.pa.mapped) > 0 && len(c.pa.queues) == 0 { + flag |= pmrIgnoreEmptyQueueFilter + } + _, qnames = c.processMsgResults(acc, r, msg, nil, c.pa.subject, c.pa.reply, flag) + } + + // Now deal with gateways + if c.srv.gateway.enabled { + c.sendMsgToGateways(acc, msg, c.pa.subject, c.pa.reply, qnames, true) + } +} + +// Checks whether the inbound leaf message is allowed by the +// connection's permissions. On the hub side this enforces what +// the remote leaf may publish. On the spoke side this enforces +// import restrictions such as deny_imports. +func (c *client) leafMsgAllowed() bool { + wireSubject := c.pa.subject + if len(c.pa.mapped) > 0 { + // Mappings rewrite c.pa.subject to the internal + // destination. For leaf ACLs, need to check + // the original wire subject from the remote side. + wireSubject = c.pa.mapped + } + // Strip any gateway routing prefix for the permission check. + subjectToCheck, isGW := getGWRoutedSubjectOrSelf(wireSubject) + + // Service-import replies (_R_), JS ack subjects ($JS.ACK.) + // are internal routing subjects forwarded via LS+ without + // permission checks. + if isServiceReply(subjectToCheck) || isJSAckSubject(subjectToCheck) { + return true + } + + c.mu.RLock() + if c.isSpokeLeafNode() { + // Gateway routed replies are forwarded without + // permission checks. + if isGW || c.leafReceiveAllowed(subjectToCheck) { + c.mu.RUnlock() + return true + } + } else if c.leafSendAllowed(subjectToCheck) { + c.mu.RUnlock() + return true + } + + // If allow_responses is not configured, or there is no tracked reply for + // this subject, the answer is "denied" and we can return it while still + // holding only the read lock. + replySubject := bytesToString(wireSubject) + if c.perms == nil || c.perms.resp == nil || c.replies[replySubject] == nil { + c.mu.RUnlock() + return false + } + c.mu.RUnlock() + + // Check tracked reply permissions (allow_responses). + // Use the pre-strip subject since deliverMsg tracks + // replies under the original form, which includes + // the GW routing prefix for routed requests. + c.mu.Lock() + defer c.mu.Unlock() + return c.responseAllowed(replySubject) +} + +// Returns true if the leaf side ACLs allow importing this subject, +// based on the permissions received over INFO and any local deny_imports. +// At least a read lock must be held. +func (c *client) leafReceiveAllowed(subject []byte) bool { + return c.canSubscribeInternal(bytesToString(subject)) +} + +// Returns true if the hub side ACLs allow the remote leaf to send +// this subject. +// At least a read lock must be held. +func (c *client) leafSendAllowed(bsubject []byte) bool { + // Use the original export ACL captured for this accepted leaf. + // The live perms also contain additional JetStream denies used by + // the normal forwarding path, and applying them here would reject + // legitimate inbound JS API requests. + subject := bytesToString(bsubject) + perms := c.opts.Export + if perms == nil || (perms.Allow == nil && perms.Deny == nil) { + return true + } + + allowed := true + if perms.Allow != nil && !strings.HasPrefix(subject, mqttPrefix) { + allowed = false + for _, allowSubj := range perms.Allow { + if matchLiteral(subject, allowSubj) { + allowed = true + break + } + } + } + + if allowed && len(perms.Deny) > 0 { + for _, denySubj := range perms.Deny { + if matchLiteral(subject, denySubj) { + allowed = false + break + } + } + } + return allowed +} + +// Handles a subscription permission violation. +// See leafPermViolation() for details. +func (c *client) leafSubPermViolation(subj []byte) { + c.leafPermViolation(false, subj) +} + +// Handles a publish permission violation. +// See leafPermViolation() for details. +func (c *client) leafPubPermViolation(subj []byte) { + c.leafPermViolation(true, subj) +} + +// Common function to process publish or subscribe leafnode permission violation. +// Sends the permission violation error to the remote, logs it and closes the connection. +// If this is from a server soliciting, the reconnection will be delayed. +func (c *client) leafPermViolation(pub bool, subj []byte) { + if c.isSpokeLeafNode() { + // For spokes these are no-ops since the hub server told us our permissions. + // We just need to not send these over to the other side since we will get cutoff. + return + } + // FIXME(dlc) ? + c.setLeafConnectDelayIfSoliciting(leafNodeReconnectAfterPermViolation) + var action string + if pub { + c.sendErr(fmt.Sprintf("Permissions Violation for Publish to %q", subj)) + action = "Publish" + } else { + c.sendErr(fmt.Sprintf("Permissions Violation for Subscription to %q", subj)) + action = "Subscription" + } + c.Errorf("%s Violation on %q - Check other side configuration", action, subj) + // TODO: add a new close reason that is more appropriate? + c.closeConnection(ProtocolViolation) +} + +// Invoked from generic processErr() for LEAF connections. +func (c *client) leafProcessErr(errStr string) { + // Check if we got a cluster name collision. + if strings.Contains(errStr, ErrLeafNodeHasSameClusterName.Error()) { + _, delay := c.setLeafConnectDelayIfSoliciting(leafNodeReconnectDelayAfterClusterNameSame) + c.Errorf("Leafnode connection dropped with same cluster name error. Delaying attempt to reconnect for %v", delay) + return + } + if strings.Contains(errStr, ErrLeafNodeMinVersionRejected.Error()) { + _, delay := c.setLeafConnectDelayIfSoliciting(leafNodeMinVersionReconnectDelay) + c.Errorf("Leafnode connection dropped due to minimum version requirement. Delaying attempt to reconnect for %v", delay) + return + } + + // We will look for Loop detected error coming from the other side. + // If we solicit, set the connect delay. + if !strings.Contains(errStr, "Loop detected") { + return + } + c.handleLeafNodeLoop(false) +} + +// If this leaf connection solicits, sets the connect delay to the given value, +// or the one from the server option's LeafNode.connDelay if one is set (for tests). +// Returns the connection's account name and delay. +func (c *client) setLeafConnectDelayIfSoliciting(delay time.Duration) (string, time.Duration) { + c.mu.Lock() + if c.isSolicitedLeafNode() { + if s := c.srv; s != nil { + if srvdelay := s.getOpts().LeafNode.connDelay; srvdelay != 0 { + delay = srvdelay + } + } + c.leaf.remote.setConnectDelay(delay) + } + var accName string + if c.acc != nil { + accName = c.acc.Name + } + c.mu.Unlock() + return accName, delay +} + +// For the given remote Leafnode configuration, this function returns +// if TLS is required, and if so, will return a clone of the TLS Config +// (since some fields will be changed during handshake), the TLS server +// name that is remembered, and the TLS timeout. +func (c *client) leafNodeGetTLSConfigForSolicit(remote *leafNodeCfg) (bool, *tls.Config, string, float64) { + var ( + tlsConfig *tls.Config + tlsName string + tlsTimeout float64 + ) + + remote.RLock() + defer remote.RUnlock() + + tlsRequired := remote.TLS || remote.TLSConfig != nil + if tlsRequired { + if remote.TLSConfig != nil { + tlsConfig = remote.TLSConfig.Clone() + } else { + tlsConfig = &tls.Config{MinVersion: tls.VersionTLS12} + } + tlsName = remote.tlsName + tlsTimeout = remote.TLSTimeout + if tlsTimeout == 0 { + tlsTimeout = float64(TLS_TIMEOUT / time.Second) + } + } + + return tlsRequired, tlsConfig, tlsName, tlsTimeout +} + +// Initiates the LeafNode Websocket connection by: +// - doing the TLS handshake if needed +// - sending the HTTP request +// - waiting for the HTTP response +// +// Since some bufio reader is used to consume the HTTP response, this function +// returns the slice of buffered bytes (if any) so that the readLoop that will +// be started after that consume those first before reading from the socket. +// The boolean +// +// Lock held on entry. +func (c *client) leafNodeSolicitWSConnection(opts *Options, rURL *url.URL, remote *leafNodeCfg) ([]byte, ClosedState, error) { + remote.RLock() + compress := remote.Websocket.Compression + // By default the server will mask outbound frames, but it can be disabled with this option. + noMasking := remote.Websocket.NoMasking + infoTimeout := remote.FirstInfoTimeout + remote.RUnlock() + // Will do the client-side TLS handshake if needed. + tlsRequired, err := c.leafClientHandshakeIfNeeded(remote, opts) + if err != nil { + // 0 will indicate that the connection was already closed + return nil, 0, err + } + + // For http request, we need the passed URL to contain either http or https scheme. + scheme := "http" + if tlsRequired { + scheme = "https" + } + // We will use the `/leafnode` path to tell the accepting WS server that it should + // create a LEAF connection, not a CLIENT. + // In case we use the user's URL path in the future, make sure we append the user's + // path to our `/leafnode` path. + lpath := leafNodeWSPath + if curPath := rURL.EscapedPath(); curPath != _EMPTY_ { + if curPath[0] == '/' { + curPath = curPath[1:] + } + lpath = path.Join(curPath, lpath) + } else { + lpath = lpath[1:] + } + ustr := fmt.Sprintf("%s://%s/%s", scheme, rURL.Host, lpath) + u, _ := url.Parse(ustr) + req := &http.Request{ + Method: "GET", + URL: u, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: make(http.Header), + Host: u.Host, + } + wsKey, err := wsMakeChallengeKey() + if err != nil { + return nil, WriteError, err + } + + req.Header["Upgrade"] = []string{"websocket"} + req.Header["Connection"] = []string{"Upgrade"} + req.Header["Sec-WebSocket-Key"] = []string{wsKey} + req.Header["Sec-WebSocket-Version"] = []string{"13"} + if compress { + req.Header.Add("Sec-WebSocket-Extensions", wsPMCReqHeaderValue) + } + if noMasking { + req.Header.Add(wsNoMaskingHeader, wsNoMaskingValue) + } + c.nc.SetDeadline(time.Now().Add(infoTimeout)) + if err := req.Write(c.nc); err != nil { + return nil, WriteError, err + } + + var resp *http.Response + + br := bufio.NewReaderSize(c.nc, MAX_CONTROL_LINE_SIZE) + resp, err = http.ReadResponse(br, req) + if err == nil && + (resp.StatusCode != 101 || + !strings.EqualFold(resp.Header.Get("Upgrade"), "websocket") || + !strings.EqualFold(resp.Header.Get("Connection"), "upgrade") || + resp.Header.Get("Sec-Websocket-Accept") != wsAcceptKey(wsKey)) { + + err = fmt.Errorf("invalid websocket connection") + } + // Check compression extension... + if err == nil && c.ws.compress { + // Check that not only permessage-deflate extension is present, but that + // we also have server and client no context take over. + srvCompress, noCtxTakeover := wsPMCExtensionSupport(resp.Header, false) + + // If server does not support compression, then simply disable it in our side. + if !srvCompress { + c.ws.compress = false + } else if !noCtxTakeover { + err = fmt.Errorf("compression negotiation error") + } + } + // Same for no masking... + if err == nil && noMasking { + // Check if server accepts no masking + if resp.Header.Get(wsNoMaskingHeader) != wsNoMaskingValue { + // Nope, need to mask our writes as any client would do. + c.ws.maskwrite = true + } + } + if resp != nil { + resp.Body.Close() + } + if err != nil { + return nil, ReadError, err + } + c.Debugf("Leafnode compression=%v masking=%v", c.ws.compress, c.ws.maskwrite) + + var preBuf []byte + // We have to slurp whatever is in the bufio reader and pass that to the readloop. + if n := br.Buffered(); n != 0 { + preBuf, _ = br.Peek(n) + } + return preBuf, 0, nil +} + +const connectProcessTimeout = 2 * time.Second + +// This is invoked for remote LEAF remote connections after processing the INFO +// protocol. +func (s *Server) leafNodeResumeConnectProcess(c *client) { + clusterName := s.ClusterName() + + c.mu.Lock() + if c.isClosed() { + c.mu.Unlock() + return + } + if err := c.sendLeafConnect(clusterName, c.headers); err != nil { + c.mu.Unlock() + c.closeConnection(WriteError) + return + } + + // Spin up the write loop. + s.startGoRoutine(func() { c.writeLoop() }) + + // timeout leafNodeFinishConnectProcess + c.ping.tmr = time.AfterFunc(connectProcessTimeout, func() { + c.mu.Lock() + // check if leafNodeFinishConnectProcess was called and prevent later leafNodeFinishConnectProcess + if !c.flags.setIfNotSet(connectProcessFinished) { + c.mu.Unlock() + return + } + clearTimer(&c.ping.tmr) + closed := c.isClosed() + c.mu.Unlock() + if !closed { + c.sendErrAndDebug("Stale Leaf Node Connection - Closing") + c.closeConnection(StaleConnection) + } + }) + c.mu.Unlock() + c.Debugf("Remote leafnode connect msg sent") +} + +// This is invoked for remote LEAF connections after processing the INFO +// protocol and leafNodeResumeConnectProcess. +// This will send LS+ the CONNECT protocol and register the leaf node. +func (s *Server) leafNodeFinishConnectProcess(c *client) { + c.mu.Lock() + if !c.flags.setIfNotSet(connectProcessFinished) { + c.mu.Unlock() + return + } + if c.isClosed() { + c.mu.Unlock() + s.removeLeafNodeConnection(c) + return + } + remote := c.leaf.remote + if remote == nil || c.acc == nil { + c.mu.Unlock() + c.sendErr("Authorization Violation") + c.closeConnection(ProtocolViolation) + return + } + // Check if we will need to send the system connect event. + remote.RLock() + sendSysConnectEvent := remote.Hub + remote.RUnlock() + + // Capture account before releasing lock + acc := c.acc + // cancel connectProcessTimeout + clearTimer(&c.ping.tmr) + c.mu.Unlock() + + // Make sure we register with the account here. + if err := c.registerWithAccount(acc); err != nil { + if err == ErrTooManyAccountConnections { + c.maxAccountConnExceeded() + return + } else if err == ErrLeafNodeLoop { + c.handleLeafNodeLoop(true) + return + } + c.Errorf("Registering leaf with account %s resulted in error: %v", acc.Name, err) + c.closeConnection(ProtocolViolation) + return + } + if !s.addLeafNodeConnection(c, _EMPTY_, _EMPTY_, false) { + // Was not added, could be because the remote configuration has been removed. + c.closeConnection(ClientClosed) + return + } + s.initLeafNodeSmapAndSendSubs(c) + if sendSysConnectEvent { + s.sendLeafNodeConnect(acc) + } + s.accountConnectEvent(c) + + // The above functions are not running under the client lock, so it is + // possible that between the time we have started the read/write loops + // and now, that the connection was closed. This would leave the closed + // LN connection possibly registered with the account and/or the server's + // leafs map. So check if connection is closed, and if so, manually cleanup. + c.mu.Lock() + closed := c.isClosed() + if !closed { + c.setFirstPingTimer() + } + c.mu.Unlock() + if closed { + s.removeLeafNodeConnection(c) + if prev := acc.removeClient(c); prev == 1 { + s.decActiveAccounts() + } + } +} diff --git a/server/leafnode_proxy_test.go b/server/leafnode_proxy_test.go new file mode 100644 index 0000000..78755c5 --- /dev/null +++ b/server/leafnode_proxy_test.go @@ -0,0 +1,929 @@ +// Copyright 2025-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "errors" + "fmt" + "net" + "net/url" + "strings" + "testing" + "time" +) + +// Basic HTTP proxy for testing +type testHTTPProxy struct { + listener net.Listener + port int + username string + password string + started bool + closeDelay time.Duration // Delay before closing connections for robustness + connections []net.Conn // Track connections for cleanup +} + +func createTestHTTPProxy(username, password string) *testHTTPProxy { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + panic(err) + } + port := l.Addr().(*net.TCPAddr).Port + + proxy := &testHTTPProxy{ + listener: l, + port: port, + username: username, + password: password, + closeDelay: 100 * time.Millisecond, // Default delay for test robustness + connections: make([]net.Conn, 0), + } + + return proxy +} + +func (p *testHTTPProxy) setCloseDelay(delay time.Duration) { + p.closeDelay = delay +} + +func (p *testHTTPProxy) start() { + if p.started { + return + } + p.started = true + + go func() { + for { + conn, err := p.listener.Accept() + if err != nil { + return + } + p.connections = append(p.connections, conn) + go p.handleConnection(conn) + } + }() +} + +func (p *testHTTPProxy) handleConnection(conn net.Conn) { + defer func() { + if p.closeDelay > 0 { + time.Sleep(p.closeDelay) + } + conn.Close() + }() + + // Set read timeout to prevent hanging on malformed requests + conn.SetReadDeadline(time.Now().Add(10 * time.Second)) + + // Read the CONNECT request + buffer := make([]byte, 4096) + n, err := conn.Read(buffer) + if err != nil { + return + } + + request := string(buffer[:n]) + lines := strings.Split(request, "\r\n") + + if len(lines) == 0 || !strings.HasPrefix(lines[0], "CONNECT ") { + conn.Write([]byte("HTTP/1.1 400 Bad Request\r\n\r\n")) + return + } + + // Check authentication if required + if p.username != _EMPTY_ || p.password != _EMPTY_ { + authFound := false + for _, line := range lines { + if strings.HasPrefix(line, "Proxy-Authorization: Basic ") { + authFound = true + break + } + } + if !authFound { + conn.Write([]byte("HTTP/1.1 407 Proxy Authentication Required\r\n\r\n")) + return + } + } + + // Extract target host from CONNECT line + parts := strings.Fields(lines[0]) + if len(parts) < 3 { + conn.Write([]byte("HTTP/1.1 400 Bad Request\r\n\r\n")) + return + } + + targetHost := parts[1] + + // Connect to target with timeout + target, err := net.DialTimeout("tcp", targetHost, 5*time.Second) + if err != nil { + conn.Write([]byte("HTTP/1.1 502 Bad Gateway\r\n\r\n")) + return + } + defer target.Close() + + // Send success response + conn.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n")) + + // Clear read deadline for ongoing connection + conn.SetReadDeadline(time.Time{}) + + // Relay data between client and target with proper error handling + done := make(chan bool, 2) + + // Client to target + go func() { + defer func() { + done <- true + target.Close() + }() + buffer := make([]byte, 32*1024) + for { + conn.SetReadDeadline(time.Now().Add(30 * time.Second)) + n, err := conn.Read(buffer) + if err != nil { + return + } + target.SetWriteDeadline(time.Now().Add(10 * time.Second)) + _, err = target.Write(buffer[:n]) + if err != nil { + return + } + } + }() + + // Target to client + go func() { + defer func() { + done <- true + conn.Close() + }() + buffer := make([]byte, 32*1024) + for { + target.SetReadDeadline(time.Now().Add(30 * time.Second)) + n, err := target.Read(buffer) + if err != nil { + return + } + conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) + _, err = conn.Write(buffer[:n]) + if err != nil { + return + } + } + }() + + // Wait for either direction to finish + <-done +} + +func (p *testHTTPProxy) stop() { + if p.listener != nil { + p.listener.Close() + } + // Close all tracked connections with delay for robustness + for _, conn := range p.connections { + go func(c net.Conn) { + if p.closeDelay > 0 { + time.Sleep(p.closeDelay) + } + c.Close() + }(conn) + } +} + +func (p *testHTTPProxy) url() string { + return fmt.Sprintf("http://127.0.0.1:%d", p.port) +} + +func TestLeafNodeHttpProxyConfigParsing(t *testing.T) { + // Test valid proxy configuration + conf := ` + leafnodes { + remotes = [ + { + url: "ws://127.0.0.1:7422" + proxy { + url: "http://proxy.example.com:8080" + username: "user" + password: "pass" + timeout: "10s" + } + } + ] + } + ` + + configFile := createConfFile(t, []byte(conf)) + + opts, err := ProcessConfigFile(configFile) + if err != nil { + t.Fatalf("Error parsing config: %v", err) + } + + if len(opts.LeafNode.Remotes) != 1 { + t.Fatalf("Expected 1 remote, got %d", len(opts.LeafNode.Remotes)) + } + + remote := opts.LeafNode.Remotes[0] + if remote.Proxy.URL != "http://proxy.example.com:8080" { + t.Errorf("Expected proxy URL 'http://proxy.example.com:8080', got '%s'", remote.Proxy.URL) + } + if remote.Proxy.Username != "user" { + t.Errorf("Expected proxy username 'user', got '%s'", remote.Proxy.Username) + } + if remote.Proxy.Password != "pass" { + t.Errorf("Expected proxy password 'pass', got '%s'", remote.Proxy.Password) + } + if remote.Proxy.Timeout != 10*time.Second { + t.Errorf("Expected proxy timeout 10s, got %v", remote.Proxy.Timeout) + } +} + +func TestLeafNodeHttpProxyConfigWarnings(t *testing.T) { + testCases := []struct { + name string + config string + expectWarning bool + warningMatch string + }{ + { + name: "proxy with only TCP URLs", + config: ` + leafnodes { + remotes = [ + { + url: "nats://127.0.0.1:7422" + proxy { + url: "http://proxy.example.com:8080" + } + } + ] + } + `, + expectWarning: true, + warningMatch: "proxy configuration will be ignored", + }, + { + name: "proxy with mixed TCP and WebSocket URLs", + config: ` + leafnodes { + remotes = [ + { + urls: ["nats://127.0.0.1:7422", "ws://127.0.0.1:8080"] + proxy { + url: "http://proxy.example.com:8080" + } + } + ] + } + `, + expectWarning: true, + warningMatch: "proxy configuration will only be used for WebSocket URLs", + }, + { + name: "proxy with only WebSocket URLs", + config: ` + leafnodes { + remotes = [ + { + url: "ws://127.0.0.1:7422" + proxy { + url: "http://proxy.example.com:8080" + } + } + ] + } + `, + expectWarning: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + configFile := createConfFile(t, []byte(tc.config)) + + opts, err := ProcessConfigFile(configFile) + + if tc.expectWarning { + // With ProcessConfigFile, warnings don't cause errors + // The configuration should be valid but might log warnings + if err != nil { + t.Fatalf("Expected valid configuration with warnings, but got error: %v", err) + } + if opts == nil { + t.Fatal("Expected valid options but got nil") + } + // Note: With ProcessConfigFile, warnings are filtered out and not returned as errors + // The test verifies that the configuration is valid despite having warning conditions + } else { + // No warnings expected - should parse successfully + if err != nil { + t.Fatalf("Expected no error but got: %v", err) + } + if opts == nil { + t.Fatal("Expected valid options but got nil") + } + } + }) + } +} + +func TestLeafNodeHttpProxyConnection(t *testing.T) { + // Create a hub server with WebSocket support using config file + hubConfig := createConfFile(t, []byte(` + listen: "127.0.0.1:-1" + websocket { + listen: "127.0.0.1:-1" + no_tls: true + } + leafnodes { + listen: "127.0.0.1:-1" + } + `)) + + hub, hubOpts := RunServerWithConfig(hubConfig) + defer hub.Shutdown() + + // Create HTTP proxy + proxy := createTestHTTPProxy(_EMPTY_, _EMPTY_) + proxy.start() + defer proxy.stop() + + // Create spoke server with proxy configuration via config file + configContent := fmt.Sprintf(` + listen: "127.0.0.1:-1" + leafnodes { + reconnect_interval: "50ms" + remotes = [ + { + url: "ws://127.0.0.1:%d" + proxy { + url: "%s" + timeout: 5s + } + } + ] + } + `, hubOpts.Websocket.Port, proxy.url()) + + configFile := createConfFile(t, []byte(configContent)) + + spoke, _ := RunServerWithConfig(configFile) + defer spoke.Shutdown() + + // Verify leafnode connections are established + checkLeafNodeConnected(t, spoke) + checkLeafNodeConnected(t, hub) +} + +func TestLeafNodeHttpProxyConnectionToTCP(t *testing.T) { + // Create a hub server using config file + hubConfig := createConfFile(t, []byte(` + listen: "127.0.0.1:-1" + leafnodes { + listen: "127.0.0.1:-1" + } + `)) + + hub, hubOpts := RunServerWithConfig(hubConfig) + defer hub.Shutdown() + + // Create HTTP proxy + proxy := createTestHTTPProxy(_EMPTY_, _EMPTY_) + proxy.start() + defer proxy.stop() + + // Create spoke server with proxy configuration via config file + configContent := fmt.Sprintf(` + listen: "127.0.0.1:-1" + leafnodes { + reconnect_interval: "50ms" + remotes = [ + { + url: "nats://127.0.0.1:%d" + proxy { + url: "%s" + timeout: 5s + } + } + ] + } + `, hubOpts.LeafNode.Port, proxy.url()) + + configFile := createConfFile(t, []byte(configContent)) + + spoke, _ := RunServerWithConfig(configFile) + defer spoke.Shutdown() + + // Verify leafnode connections are established + checkLeafNodeConnected(t, spoke) + checkLeafNodeConnected(t, hub) +} + +func TestLeafNodeHttpProxyWithAuthentication(t *testing.T) { + // Create a hub server with WebSocket support using config file + hubConfig := createConfFile(t, []byte(` + listen: "127.0.0.1:-1" + websocket { + listen: "127.0.0.1:-1" + no_tls: true + } + leafnodes { + listen: "127.0.0.1:-1" + } + `)) + + hub, hubOpts := RunServerWithConfig(hubConfig) + defer hub.Shutdown() + + // Create HTTP proxy with authentication + proxy := createTestHTTPProxy("testuser", "testpass") + proxy.start() + defer proxy.stop() + + // Create spoke server with proxy configuration including auth via config file + configContent := fmt.Sprintf(` + listen: "127.0.0.1:-1" + leafnodes { + reconnect_interval: "50ms" + remotes = [ + { + url: "ws://127.0.0.1:%d" + proxy { + url: "%s" + username: "testuser" + password: "testpass" + timeout: 5s + } + } + ] + } + `, hubOpts.Websocket.Port, proxy.url()) + + configFile := createConfFile(t, []byte(configContent)) + + spoke, _ := RunServerWithConfig(configFile) + defer spoke.Shutdown() + + // Verify leafnode connections are established + checkLeafNodeConnected(t, spoke) + checkLeafNodeConnected(t, hub) +} + +func TestLeafNodeHttpProxyTLSMismatchDetection(t *testing.T) { + // This test simulates the TLS mismatch scenario described in the feedback: + // - Leafnode configured with proxy but no TLS + // - Hub requires TLS + // - Connection should fail with appropriate error message + + // Create hub server with TLS required using config file + hubConfig := createConfFile(t, []byte(` + listen: "127.0.0.1:-1" + leafnodes { + listen: "127.0.0.1:-1" + tls: { + cert_file: "../test/configs/certs/server-cert.pem" + key_file: "../test/configs/certs/server-key.pem" + } + } + `)) + + hub, hubOpts := RunServerWithConfig(hubConfig) + defer hub.Shutdown() + + // Create HTTP proxy + proxy := createTestHTTPProxy(_EMPTY_, _EMPTY_) + proxy.start() + defer proxy.stop() + + // Create spoke server with proxy but no TLS configuration (intentional mismatch) + spokeConfigContent := fmt.Sprintf(` + listen: "127.0.0.1:-1" + leafnodes { + reconnect_interval: "50ms" + remotes = [ + { + url: "ws://127.0.0.1:%d" + proxy { + url: "%s" + timeout: 5s + } + # Intentionally no TLS configuration to create mismatch + } + ] + } + `, hubOpts.LeafNode.Port, proxy.url()) + + spokeConfig := createConfFile(t, []byte(spokeConfigContent)) + spoke, _ := RunServerWithConfig(spokeConfig) + defer spoke.Shutdown() + + // Wait and verify that connection was NOT established due to TLS mismatch + // First attempt happens during RunServerWithConfig(), then retries every 50ms + time.Sleep(250 * time.Millisecond) + + if spoke.NumLeafNodes() != 0 { + t.Errorf("Expected 0 leafnode connections due to TLS mismatch, got %d", spoke.NumLeafNodes()) + } +} + +func TestLeafNodeHttpProxyTunnelBasic(t *testing.T) { + // Create HTTP proxy with longer delay for robustness + proxy := createTestHTTPProxy(_EMPTY_, _EMPTY_) + proxy.setCloseDelay(200 * time.Millisecond) + proxy.start() + defer proxy.stop() + + // Create a simple TCP server to connect to through proxy + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Failed to create test server: %v", err) + } + defer listener.Close() + + targetPort := listener.Addr().(*net.TCPAddr).Port + targetHost := fmt.Sprintf("127.0.0.1:%d", targetPort) + + errCh := make(chan error, 1) + + // Accept one connection with proper timeout handling + go func() { + conn, err := listener.Accept() + if err != nil { + errCh <- fmt.Errorf("unable to accept: %v", err) + return + } + defer conn.Close() + + // Set read deadline to prevent hanging forever + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + + // Read the incoming data first + buffer := make([]byte, 1024) + n, err := conn.Read(buffer) + if err != nil { + errCh <- fmt.Errorf("server failed to read: %v", err) + return + } + + receivedMsg := string(buffer[:n]) + if receivedMsg != "Hello" { + errCh <- fmt.Errorf("server expected 'Hello', got '%s'", receivedMsg) + return + } + + // Send response + conn.SetWriteDeadline(time.Now().Add(5 * time.Second)) + _, err = conn.Write([]byte("Hello from target server")) + if err != nil { + errCh <- fmt.Errorf("server failed to write: %v", err) + return + } + + // Wait a bit to ensure the client has time to read before closing + time.Sleep(50 * time.Millisecond) + errCh <- nil + }() + + // Test establishing proxy tunnel with timeout + conn, err := establishHTTPProxyTunnel(proxy.url(), targetHost, 10*time.Second, _EMPTY_, _EMPTY_) + if err != nil { + t.Fatalf("Failed to establish proxy tunnel: %v", err) + } + defer conn.Close() + + // Test that we can communicate through the tunnel with deadlines + conn.SetWriteDeadline(time.Now().Add(5 * time.Second)) + _, err = conn.Write([]byte("Hello")) + if err != nil { + t.Fatalf("Failed to write to proxy tunnel: %v", err) + } + + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + buffer := make([]byte, 1024) + n, err := conn.Read(buffer) + if err != nil { + t.Fatalf("Failed to read from proxy tunnel: %v", err) + } + + response := string(buffer[:n]) + if response != "Hello from target server" { + t.Errorf("Unexpected response: '%s', expected 'Hello from target server'", response) + } + + select { + case err := <-errCh: + if err != nil { + t.Fatalf("%v", err) + } + case <-time.After(3 * time.Second): + t.Fatal("Server goroutine didn't complete in time, but test data was exchanged successfully") + } +} + +func TestLeafNodeHttpProxyTunnelWithAuth(t *testing.T) { + // Create HTTP proxy with authentication and delay for robustness + proxy := createTestHTTPProxy("testuser", "testpass") + proxy.setCloseDelay(200 * time.Millisecond) + proxy.start() + defer proxy.stop() + + // Create a simple TCP server to connect to through proxy + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Failed to create test server: %v", err) + } + defer listener.Close() + + targetPort := listener.Addr().(*net.TCPAddr).Port + targetHost := fmt.Sprintf("127.0.0.1:%d", targetPort) + + errCh := make(chan error, 1) + + // Accept one connection with proper timeout handling + go func() { + conn, err := listener.Accept() + if err != nil { + errCh <- fmt.Errorf("unable to accept: %v", err) + return + } + defer conn.Close() + + // Set read deadline to prevent hanging + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + + // Read the incoming data first + buffer := make([]byte, 1024) + _, err = conn.Read(buffer) + if err != nil { + errCh <- fmt.Errorf("unable to read: %v", err) + return + } + + // Send response + conn.SetWriteDeadline(time.Now().Add(5 * time.Second)) + if _, err := conn.Write([]byte("Hello from authenticated server")); err != nil { + errCh <- fmt.Errorf("unable to write: %v", err) + return + } + + errCh <- nil + }() + + // Test establishing proxy tunnel with authentication and timeout + conn, err := establishHTTPProxyTunnel(proxy.url(), targetHost, 10*time.Second, "testuser", "testpass") + if err != nil { + t.Fatalf("Failed to establish proxy tunnel with auth: %v", err) + } + defer conn.Close() + + // Test that we can communicate through the tunnel with deadlines + conn.SetWriteDeadline(time.Now().Add(5 * time.Second)) + _, err = conn.Write([]byte("Hello")) + if err != nil { + t.Fatalf("Failed to write to proxy tunnel: %v", err) + } + + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + buffer := make([]byte, 1024) + n, err := conn.Read(buffer) + if err != nil { + t.Fatalf("Failed to read from proxy tunnel: %v", err) + } + + response := string(buffer[:n]) + if response != "Hello from authenticated server" { + t.Errorf("Unexpected response: %s", response) + } + + select { + case err := <-errCh: + if err != nil { + t.Fatalf("%v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Server goroutine didn't complete in time, but test data was exchanged successfully") + } +} + +func TestLeafNodeHttpProxyTunnelFailsWithoutAuth(t *testing.T) { + // Create HTTP proxy with authentication required and delay for robustness + proxy := createTestHTTPProxy("testuser", "testpass") + proxy.setCloseDelay(200 * time.Millisecond) + proxy.start() + defer proxy.stop() + + // Try to establish tunnel without providing credentials (should fail quickly) + _, err := establishHTTPProxyTunnel(proxy.url(), "127.0.0.1:80", 10*time.Second, _EMPTY_, _EMPTY_) + if err == nil { + t.Fatal("Expected error when connecting without authentication") + } + + if !strings.Contains(err.Error(), "proxy CONNECT failed") { + t.Errorf("Expected proxy authentication error, got: %v", err) + } + + // Verify the error contains the expected HTTP response code + if !strings.Contains(err.Error(), "407") { + t.Errorf("Expected HTTP 407 error in response, got: %v", err) + } +} + +// TestLeafNodeProxyValidationProgrammatic tests proxy validation when configuring server programmatically +func TestLeafNodeHttpProxyValidationProgrammatic(t *testing.T) { + tests := []struct { + // name is the name of the test. + name string + + // setupOptions creates the Options configuration for the test. + setupOptions func() *Options + + // err is the expected error. nil means no error expected. + err error + }{ + { + name: "invalid proxy scheme", + setupOptions: func() *Options { + opts := &Options{} + opts.LeafNode.Remotes = []*RemoteLeafOpts{ + { + URLs: []*url.URL{{Scheme: wsSchemePrefix, Host: "127.0.0.1:7422"}}, + }, + } + opts.LeafNode.Remotes[0].Proxy.URL = "ftp://proxy.example.com:21" + return opts + }, + err: errors.New("proxy URL scheme must be http or https"), + }, + { + name: "empty proxy URL - no validation performed", + setupOptions: func() *Options { + opts := &Options{} + opts.LeafNode.Remotes = []*RemoteLeafOpts{ + { + URLs: []*url.URL{{Scheme: wsSchemePrefix, Host: "127.0.0.1:7422"}}, + }, + } + opts.LeafNode.Remotes[0].Proxy.URL = _EMPTY_ + return opts + }, + err: nil, // No error expected for empty URL + }, + { + name: "invalid proxy URL parse failure", + setupOptions: func() *Options { + opts := &Options{} + opts.LeafNode.Remotes = []*RemoteLeafOpts{ + { + URLs: []*url.URL{{Scheme: wsSchemePrefix, Host: "127.0.0.1:7422"}}, + }, + } + opts.LeafNode.Remotes[0].Proxy.URL = "ht!tp://invalid-url-with-bad-characters" + return opts + }, + err: errors.New("invalid proxy URL"), + }, + { + name: "missing proxy host", + setupOptions: func() *Options { + opts := &Options{} + opts.LeafNode.Remotes = []*RemoteLeafOpts{ + { + URLs: []*url.URL{{Scheme: wsSchemePrefix, Host: "127.0.0.1:7422"}}, + }, + } + opts.LeafNode.Remotes[0].Proxy.URL = "http://" + return opts + }, + err: errors.New("proxy URL must specify a host"), + }, + { + name: "username without password", + setupOptions: func() *Options { + opts := &Options{} + opts.LeafNode.Remotes = []*RemoteLeafOpts{ + { + URLs: []*url.URL{{Scheme: wsSchemePrefix, Host: "127.0.0.1:7422"}}, + }, + } + opts.LeafNode.Remotes[0].Proxy.URL = "http://proxy.example.com:8080" + opts.LeafNode.Remotes[0].Proxy.Username = "user" + return opts + }, + err: errors.New("proxy username and password must both be specified"), + }, + { + name: "password without username", + setupOptions: func() *Options { + opts := &Options{} + opts.LeafNode.Remotes = []*RemoteLeafOpts{ + { + URLs: []*url.URL{{Scheme: wsSchemePrefix, Host: "127.0.0.1:7422"}}, + }, + } + opts.LeafNode.Remotes[0].Proxy.URL = "http://proxy.example.com:8080" + opts.LeafNode.Remotes[0].Proxy.Password = "pass" + return opts + }, + err: errors.New("proxy username and password must both be specified"), + }, + { + name: "negative timeout value", + setupOptions: func() *Options { + opts := &Options{} + opts.LeafNode.Remotes = []*RemoteLeafOpts{ + { + URLs: []*url.URL{{Scheme: wsSchemePrefix, Host: "127.0.0.1:7422"}}, + }, + } + opts.LeafNode.Remotes[0].Proxy.URL = "http://proxy.example.com:8080" + opts.LeafNode.Remotes[0].Proxy.Timeout = -5 * time.Second + return opts + }, + err: errors.New("proxy timeout must be >= 0"), + }, + { + name: "zero timeout value - valid", + setupOptions: func() *Options { + opts := &Options{} + opts.LeafNode.Remotes = []*RemoteLeafOpts{ + { + URLs: []*url.URL{{Scheme: wsSchemePrefix, Host: "127.0.0.1:7422"}}, + }, + } + opts.LeafNode.Remotes[0].Proxy.URL = "http://proxy.example.com:8080" + opts.LeafNode.Remotes[0].Proxy.Timeout = 0 + return opts + }, + err: nil, // No error expected for zero timeout + }, + { + name: "valid proxy configuration with authentication", + setupOptions: func() *Options { + opts := &Options{} + opts.LeafNode.Remotes = []*RemoteLeafOpts{ + { + URLs: []*url.URL{{Scheme: wsSchemePrefix, Host: "127.0.0.1:7422"}}, + }, + } + opts.LeafNode.Remotes[0].Proxy.URL = "http://proxy.example.com:8080" + opts.LeafNode.Remotes[0].Proxy.Username = "user" + opts.LeafNode.Remotes[0].Proxy.Password = "pass" + opts.LeafNode.Remotes[0].Proxy.Timeout = 10 * time.Second + return opts + }, + err: nil, // No error expected + }, + { + name: "valid proxy configuration without authentication", + setupOptions: func() *Options { + opts := &Options{} + opts.LeafNode.Remotes = []*RemoteLeafOpts{ + { + URLs: []*url.URL{{Scheme: wsSchemePrefix, Host: "127.0.0.1:7422"}}, + }, + } + opts.LeafNode.Remotes[0].Proxy.URL = "https://proxy.example.com:3128" + opts.LeafNode.Remotes[0].Proxy.Timeout = 30 * time.Second + return opts + }, + err: nil, // No error expected + }, + } + + checkErr := func(t *testing.T, err, expectedErr error) { + t.Helper() + switch { + case err == nil && expectedErr == nil: + // OK + case err != nil && expectedErr == nil: + t.Errorf("Unexpected error after validating options: %s", err) + case err == nil && expectedErr != nil: + t.Errorf("Expected %q error after validating invalid options but got nothing", expectedErr) + case err != nil && expectedErr != nil: + if !strings.Contains(err.Error(), expectedErr.Error()) { + t.Errorf("Expected error containing %q, got: %q", expectedErr.Error(), err.Error()) + } + } + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + opts := test.setupOptions() + err := validateLeafNode(opts) + checkErr(t, err, test.err) + }) + } +} diff --git a/server/log.go b/server/log.go new file mode 100644 index 0000000..0df8a5a --- /dev/null +++ b/server/log.go @@ -0,0 +1,298 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "fmt" + "io" + "os" + "sync/atomic" + "time" + + srvlog "github.com/nats-io/nats-server/v2/logger" +) + +// Logger interface of the NATS Server +type Logger interface { + + // Log a notice statement + Noticef(format string, v ...any) + + // Log a warning statement + Warnf(format string, v ...any) + + // Log a fatal error + Fatalf(format string, v ...any) + + // Log an error + Errorf(format string, v ...any) + + // Log a debug statement + Debugf(format string, v ...any) + + // Log a trace statement + Tracef(format string, v ...any) +} + +// ConfigureLogger configures and sets the logger for the server. +func (s *Server) ConfigureLogger() { + var ( + log Logger + + // Snapshot server options. + opts = s.getOpts() + ) + + if opts.NoLog { + return + } + + syslog := opts.Syslog + if isWindowsService() && opts.LogFile == "" { + // Enable syslog if no log file is specified and we're running as a + // Windows service so that logs are written to the Windows event log. + syslog = true + } + + if opts.LogFile != "" { + log = srvlog.NewFileLogger(opts.LogFile, opts.Logtime, opts.Debug, opts.Trace, true, srvlog.LogUTC(opts.LogtimeUTC)) + if opts.LogSizeLimit > 0 { + if l, ok := log.(*srvlog.Logger); ok { + l.SetSizeLimit(opts.LogSizeLimit) + } + } + if opts.LogMaxFiles > 0 { + if l, ok := log.(*srvlog.Logger); ok { + al := int(opts.LogMaxFiles) + if int64(al) != opts.LogMaxFiles { + // set to default (no max) on overflow + al = 0 + } + l.SetMaxNumFiles(al) + } + } + } else if opts.RemoteSyslog != "" { + log = srvlog.NewRemoteSysLogger(opts.RemoteSyslog, opts.Debug, opts.Trace) + } else if syslog { + log = srvlog.NewSysLogger(opts.Debug, opts.Trace) + } else { + colors := true + // Check to see if stderr is being redirected and if so turn off color + // Also turn off colors if we're running on Windows where os.Stderr.Stat() returns an invalid handle-error + stat, err := os.Stderr.Stat() + if err != nil || (stat.Mode()&os.ModeCharDevice) == 0 { + colors = false + } + log = srvlog.NewStdLogger(opts.Logtime, opts.Debug, opts.Trace, colors, true, srvlog.LogUTC(opts.LogtimeUTC)) + } + + s.SetLoggerV2(log, opts.Debug, opts.Trace, opts.TraceVerbose) +} + +// Returns our current logger. +func (s *Server) Logger() Logger { + s.logging.Lock() + defer s.logging.Unlock() + return s.logging.logger +} + +// SetLogger sets the logger of the server +func (s *Server) SetLogger(logger Logger, debugFlag, traceFlag bool) { + s.SetLoggerV2(logger, debugFlag, traceFlag, false) +} + +// SetLogger sets the logger of the server +func (s *Server) SetLoggerV2(logger Logger, debugFlag, traceFlag, sysTrace bool) { + if debugFlag { + atomic.StoreInt32(&s.logging.debug, 1) + } else { + atomic.StoreInt32(&s.logging.debug, 0) + } + if traceFlag { + atomic.StoreInt32(&s.logging.trace, 1) + } else { + atomic.StoreInt32(&s.logging.trace, 0) + } + if sysTrace { + atomic.StoreInt32(&s.logging.traceSysAcc, 1) + } else { + atomic.StoreInt32(&s.logging.traceSysAcc, 0) + } + var prevLoggerErr error + s.logging.Lock() + if s.logging.logger != nil { + // Check to see if the logger implements io.Closer. This could be a + // logger from another process embedding the NATS server or a dummy + // test logger that may not implement that interface. + if l, ok := s.logging.logger.(io.Closer); ok { + prevLoggerErr = l.Close() + } + } + s.logging.logger = logger + s.logging.Unlock() + + if prevLoggerErr != nil { + s.Errorf("Error closing logger: %v", prevLoggerErr) + } +} + +// ReOpenLogFile if the logger is a file based logger, close and re-open the file. +// This allows for file rotation by 'mv'ing the file then signaling +// the process to trigger this function. +func (s *Server) ReOpenLogFile() { + // Check to make sure this is a file logger. + s.logging.RLock() + ll := s.logging.logger + s.logging.RUnlock() + + if ll == nil { + s.Noticef("File log re-open ignored, no logger") + return + } + + // Snapshot server options. + opts := s.getOpts() + + if opts.LogFile == "" { + s.Noticef("File log re-open ignored, not a file logger") + } else { + fileLog := srvlog.NewFileLogger( + opts.LogFile, opts.Logtime, + opts.Debug, opts.Trace, true, + srvlog.LogUTC(opts.LogtimeUTC), + ) + s.SetLogger(fileLog, opts.Debug, opts.Trace) + if opts.LogSizeLimit > 0 { + fileLog.SetSizeLimit(opts.LogSizeLimit) + } + s.Noticef("File log re-opened") + } +} + +// Noticef logs a notice statement +func (s *Server) Noticef(format string, v ...any) { + s.executeLogCall(func(logger Logger, format string, v ...any) { + logger.Noticef(format, v...) + }, format, v...) +} + +// Errorf logs an error +func (s *Server) Errorf(format string, v ...any) { + s.executeLogCall(func(logger Logger, format string, v ...any) { + logger.Errorf(format, v...) + }, format, v...) +} + +// Error logs an error with a scope +func (s *Server) Errors(scope any, e error) { + s.executeLogCall(func(logger Logger, format string, v ...any) { + logger.Errorf(format, v...) + }, "%s - %s", scope, UnpackIfErrorCtx(e)) +} + +// Error logs an error with a context +func (s *Server) Errorc(ctx string, e error) { + s.executeLogCall(func(logger Logger, format string, v ...any) { + logger.Errorf(format, v...) + }, "%s: %s", ctx, UnpackIfErrorCtx(e)) +} + +// Error logs an error with a scope and context +func (s *Server) Errorsc(scope any, ctx string, e error) { + s.executeLogCall(func(logger Logger, format string, v ...any) { + logger.Errorf(format, v...) + }, "%s - %s: %s", scope, ctx, UnpackIfErrorCtx(e)) +} + +// Warnf logs a warning error +func (s *Server) Warnf(format string, v ...any) { + s.executeLogCall(func(logger Logger, format string, v ...any) { + logger.Warnf(format, v...) + }, format, v...) +} + +func (s *Server) rateLimitFormatWarnf(format string, v ...any) { + if _, loaded := s.rateLimitLogging.LoadOrStore(format, time.Now()); loaded { + return + } + statement := fmt.Sprintf(format, v...) + s.Warnf("%s", statement) +} + +func (s *Server) RateLimitErrorf(format string, v ...any) { + statement := fmt.Sprintf(format, v...) + if _, loaded := s.rateLimitLogging.LoadOrStore(statement, time.Now()); loaded { + return + } + s.Errorf("%s", statement) +} + +func (s *Server) RateLimitWarnf(format string, v ...any) { + statement := fmt.Sprintf(format, v...) + if _, loaded := s.rateLimitLogging.LoadOrStore(statement, time.Now()); loaded { + return + } + s.Warnf("%s", statement) +} + +func (s *Server) RateLimitDebugf(format string, v ...any) { + statement := fmt.Sprintf(format, v...) + if _, loaded := s.rateLimitLogging.LoadOrStore(statement, time.Now()); loaded { + return + } + s.Debugf("%s", statement) +} + +// Fatalf logs a fatal error +func (s *Server) Fatalf(format string, v ...any) { + if s.isShuttingDown() { + s.Errorf(format, v) + return + } + s.executeLogCall(func(logger Logger, format string, v ...any) { + logger.Fatalf(format, v...) + }, format, v...) +} + +// Debugf logs a debug statement +func (s *Server) Debugf(format string, v ...any) { + if atomic.LoadInt32(&s.logging.debug) == 0 { + return + } + + s.executeLogCall(func(logger Logger, format string, v ...any) { + logger.Debugf(format, v...) + }, format, v...) +} + +// Tracef logs a trace statement +func (s *Server) Tracef(format string, v ...any) { + if atomic.LoadInt32(&s.logging.trace) == 0 { + return + } + + s.executeLogCall(func(logger Logger, format string, v ...any) { + logger.Tracef(format, v...) + }, format, v...) +} + +func (s *Server) executeLogCall(f func(logger Logger, format string, v ...any), format string, args ...any) { + s.logging.RLock() + defer s.logging.RUnlock() + if s.logging.logger == nil { + return + } + + f(s.logging.logger, format, args...) +} diff --git a/server/log_test.go b/server/log_test.go new file mode 100644 index 0000000..761bef4 --- /dev/null +++ b/server/log_test.go @@ -0,0 +1,536 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/internal/testhelper" + "github.com/nats-io/nats-server/v2/logger" +) + +func TestSetLogger(t *testing.T) { + server := &Server{} + defer server.SetLogger(nil, false, false) + dl := &DummyLogger{} + server.SetLogger(dl, true, true) + + // We assert that the logger has change to the DummyLogger + _ = server.logging.logger.(*DummyLogger) + + if server.logging.debug != 1 { + t.Fatalf("Expected debug 1, received value %d\n", server.logging.debug) + } + + if server.logging.trace != 1 { + t.Fatalf("Expected trace 1, received value %d\n", server.logging.trace) + } + + // Check traces + expectedStr := "This is a Notice" + server.Noticef(expectedStr) + dl.CheckContent(t, expectedStr) + expectedStr = "This is an Error" + server.Errorf(expectedStr) + dl.CheckContent(t, expectedStr) + expectedStr = "This is a Fatal" + server.Fatalf(expectedStr) + dl.CheckContent(t, expectedStr) + expectedStr = "This is a Debug" + server.Debugf(expectedStr) + dl.CheckContent(t, expectedStr) + expectedStr = "This is a Trace" + server.Tracef(expectedStr) + dl.CheckContent(t, expectedStr) + expectedStr = "This is a Warning" + server.Tracef(expectedStr) + dl.CheckContent(t, expectedStr) + + // Make sure that we can reset to fal + server.SetLogger(dl, false, false) + if server.logging.debug != 0 { + t.Fatalf("Expected debug 0, got %v", server.logging.debug) + } + if server.logging.trace != 0 { + t.Fatalf("Expected trace 0, got %v", server.logging.trace) + } + // Now, Debug and Trace should not produce anything + dl.Msg = "" + server.Debugf("This Debug should not be traced") + dl.CheckContent(t, "") + server.Tracef("This Trace should not be traced") + dl.CheckContent(t, "") +} + +type DummyLogger = testhelper.DummyLogger + +func TestReOpenLogFile(t *testing.T) { + // We can't rename the file log when still opened on Windows, so skip + if runtime.GOOS == "windows" { + t.SkipNow() + } + s := &Server{opts: &Options{}} + defer s.SetLogger(nil, false, false) + + // First check with no logger + s.SetLogger(nil, false, false) + s.ReOpenLogFile() + + // Then when LogFile is not provided. + dl := &DummyLogger{} + s.SetLogger(dl, false, false) + s.ReOpenLogFile() + dl.CheckContent(t, "File log re-open ignored, not a file logger") + + // Set a File log + s.opts.LogFile = filepath.Join(t.TempDir(), "test.log") + fileLog := logger.NewFileLogger(s.opts.LogFile, s.opts.Logtime, s.opts.Debug, s.opts.Trace, true, logger.LogUTC(s.opts.LogtimeUTC)) + s.SetLogger(fileLog, false, false) + // Add some log + expectedStr := "This is a Notice" + s.Noticef(expectedStr) + // Check content of log + buf, err := os.ReadFile(s.opts.LogFile) + if err != nil { + t.Fatalf("Error reading file: %v", err) + } + if !strings.Contains(string(buf), expectedStr) { + t.Fatalf("Expected log to contain: %q, got %q", expectedStr, string(buf)) + } + // Close the file and rename it + if err := os.Rename(s.opts.LogFile, s.opts.LogFile+".bak"); err != nil { + t.Fatalf("Unable to rename log file: %v", err) + } + // Now re-open LogFile + s.ReOpenLogFile() + // Content should indicate that we have re-opened the log + buf, err = os.ReadFile(s.opts.LogFile) + if err != nil { + t.Fatalf("Error reading file: %v", err) + } + if strings.HasSuffix(string(buf), "File log-reopened") { + t.Fatalf("File should indicate that file log was re-opened, got: %v", string(buf)) + } + // Make sure we can append to the log + s.Noticef("New message") + buf, err = os.ReadFile(s.opts.LogFile) + if err != nil { + t.Fatalf("Error reading file: %v", err) + } + if strings.HasSuffix(string(buf), "New message") { + t.Fatalf("New message was not appended after file was re-opened, got: %v", string(buf)) + } +} + +func TestFileLoggerSizeLimitAndReopen(t *testing.T) { + file := createTempFile(t, "log_") + file.Close() + + s := &Server{opts: &Options{}} + defer s.SetLogger(nil, false, false) + + // Set a File log + s.opts.LogFile = file.Name() + s.opts.Logtime = true + s.opts.LogSizeLimit = 1000 + s.ConfigureLogger() + + // Add a trace + s.Noticef("this is a notice") + + // Do a re-open... + s.ReOpenLogFile() + + // Content should indicate that we have re-opened the log + buf, err := os.ReadFile(s.opts.LogFile) + if err != nil { + t.Fatalf("Error reading file: %v", err) + } + if strings.HasSuffix(string(buf), "File log-reopened") { + t.Fatalf("File should indicate that file log was re-opened, got: %v", string(buf)) + } + + // Now make sure that the limit is still honored. + txt := make([]byte, 800) + for i := 0; i < len(txt); i++ { + txt[i] = 'A' + } + s.Noticef(string(txt)) + for i := 0; i < len(txt); i++ { + txt[i] = 'B' + } + s.Noticef(string(txt)) + + buf, err = os.ReadFile(s.opts.LogFile) + if err != nil { + t.Fatalf("Error reading file: %v", err) + } + sbuf := string(buf) + if strings.Contains(sbuf, "AAAAA") || strings.Contains(sbuf, "BBBBB") { + t.Fatalf("Looks like file was not rotated: %s", sbuf) + } + if !strings.Contains(sbuf, "Rotated log, backup saved") { + t.Fatalf("File should have been rotated, was not: %s", sbuf) + } +} + +func TestNoPasswordsFromConnectTrace(t *testing.T) { + opts := DefaultOptions() + opts.NoLog = false + opts.Trace = true + opts.Username = "derek" + opts.Password = "s3cr3t" + opts.PingInterval = 2 * time.Minute + setBaselineOptions(opts) + s := &Server{opts: opts} + dl := testhelper.NewDummyLogger(100) + s.SetLogger(dl, false, true) + + _ = s.logging.logger.(*DummyLogger) + if s.logging.trace != 1 { + t.Fatalf("Expected trace 1, received value %d\n", s.logging.trace) + } + defer s.SetLogger(nil, false, false) + + c, _, _ := newClientForServer(s) + defer c.close() + + connectOp := []byte("CONNECT {\"user\":\"derek\",\"pass\":\"s3cr3t\"}\r\n") + err := c.parse(connectOp) + if err != nil { + t.Fatalf("Received error: %v\n", err) + } + + dl.CheckForProhibited(t, "password found", "s3cr3t") +} + +func TestRemovePassFromTrace(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + "user and pass", + "CONNECT {\"user\":\"derek\",\"pass\":\"s3cr3t\"}\r\n", + "CONNECT {\"user\":\"derek\",\"pass\":\"[REDACTED]\"}\r\n", + }, + { + "user and pass extra space", + "CONNECT {\"user\":\"derek\",\"pass\": \"s3cr3t\"}\r\n", + "CONNECT {\"user\":\"derek\",\"pass\": \"[REDACTED]\"}\r\n", + }, + { + "user and pass is empty", + "CONNECT {\"user\":\"derek\",\"pass\":\"\"}\r\n", + "CONNECT {\"user\":\"derek\",\"pass\":\"[REDACTED]\"}\r\n", + }, + { + "user and pass is empty whitespace", + "CONNECT {\"user\":\"derek\",\"pass\":\" \"}\r\n", + "CONNECT {\"user\":\"derek\",\"pass\":\"[REDACTED]\"}\r\n", + }, + { + "user and pass whitespace", + "CONNECT {\"user\":\"derek\",\"pass\": \"s3cr3t\" }\r\n", + "CONNECT {\"user\":\"derek\",\"pass\": \"[REDACTED]\" }\r\n", + }, + { + "only pass", + "CONNECT {\"pass\":\"s3cr3t\",}\r\n", + "CONNECT {\"pass\":\"[REDACTED]\",}\r\n", + }, + { + "invalid json", + "CONNECT {pass:s3cr3t , password = s3cr3t}", + "CONNECT {pass:[REDACTED], password = [REDACTED]}", + }, + { + "invalid json no whitespace after key", + "CONNECT {pass:s3cr3t , password= s3cr3t}", + "CONNECT {pass:[REDACTED], password= [REDACTED]}", + }, + { + "both pass and wrong password key", + `CONNECT {"pass":"s3cr3t4", "password": "s3cr3t4"}`, + `CONNECT {"pass":"[REDACTED]", "password": "[REDACTED]"}`, + }, + { + "invalid json", + "CONNECT {user = hello, password = s3cr3t}", + "CONNECT {user = hello, password = [REDACTED]}", + }, + { + "complete connect", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"foo\",\"pass\":\"s3cr3t\",\"tls_required\":false,\"name\":\"APM7JU94z77YzP6WTBEiuw\"}\r\n", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"foo\",\"pass\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"APM7JU94z77YzP6WTBEiuw\"}\r\n", + }, + { + "invalid json with only pass key", + "CONNECT {pass:s3cr3t\r\n", + "CONNECT {pass:[REDACTED]\r\n", + }, + { + "invalid password key also filtered", + "CONNECT {\"password\":\"s3cr3t\",}\r\n", + "CONNECT {\"password\":\"[REDACTED]\",}\r\n", + }, + { + "single long password with whitespace", + "CONNECT {\"pass\":\"secret password which is very long\",}\r\n", + "CONNECT {\"pass\":\"[REDACTED]\",}\r\n", + }, + { + "single long pass key is filtered", + "CONNECT {\"pass\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"}\r\n", + "CONNECT {\"pass\":\"[REDACTED]\"}\r\n", + }, + { + "duplicate keys all filtered", + "CONNECT {\"pass\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\",\"pass\":\"BBBBBBBBBBBBBBBBBBBB\",\"password\":\"CCCCCCCCCCCCCCCC\"}\r\n", + "CONNECT {\"pass\":\"[REDACTED]\",\"pass\":\"[REDACTED]\",\"password\":\"[REDACTED]\"}\r\n", + }, + { + "invalid json with multiple keys all filtered", + "CONNECT {pass = \"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\",pass= \"BBBBBBBBBBBBBBBBBBBB\",password =\"CCCCCCCCCCCCCCCC\"}\r\n", + "CONNECT {pass = \"[REDACTED]\",pass= \"[REDACTED]\",password =\"[REDACTED]\"}\r\n", + }, + { + "complete connect protocol", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"foo\",\"pass\":\"s3cr3t\",\"tls_required\":false,\"name\":\"APM7JU94z77YzP6WTBEiuw\"}\r\n", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"foo\",\"pass\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"APM7JU94z77YzP6WTBEiuw\"}\r\n", + }, + { + "user and pass are filterered", + "CONNECT {\"user\":\"s3cr3t\",\"pass\":\"s3cr3t\"}\r\n", + "CONNECT {\"user\":\"s3cr3t\",\"pass\":\"[REDACTED]\"}\r\n", + }, + { + "complete connect using password key with user and password being the same", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"pass\":\"s3cr3t\",\"tls_required\":false,\"name\":\"...\"}\r\n", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"pass\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"...\"}\r\n", + }, + { + "complete connect with user password and name all the same", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"pass\":\"s3cr3t\",\"tls_required\":false,\"name\":\"s3cr3t\"}\r\n", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"pass\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"s3cr3t\"}\r\n", + }, + { + "complete connect extra white space at the beginning", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"pass\":\"s3cr3t\",\"tls_required\":false,\"name\":\"foo\"}\r\n", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"pass\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"foo\"}\r\n", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + output := removeSecretsFromTrace([]byte(test.input)) + if !bytes.Equal(output, []byte(test.expected)) { + t.Errorf("\nExpected %q\n got: %q", test.expected, string(output)) + } + }) + } +} + +func TestRemoveAuthTokenFromTrace(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + "user and auth_token", + "CONNECT {\"user\":\"derek\",\"auth_token\":\"s3cr3t\"}\r\n", + "CONNECT {\"user\":\"derek\",\"auth_token\":\"[REDACTED]\"}\r\n", + }, + { + "user and pass extra space", + "CONNECT {\"user\":\"derek\",\"auth_token\": \"s3cr3t\"}\r\n", + "CONNECT {\"user\":\"derek\",\"auth_token\": \"[REDACTED]\"}\r\n", + }, + { + "user and pass is empty", + "CONNECT {\"user\":\"derek\",\"auth_token\":\"\"}\r\n", + "CONNECT {\"user\":\"derek\",\"auth_token\":\"[REDACTED]\"}\r\n", + }, + { + "user and pass is empty whitespace", + "CONNECT {\"user\":\"derek\",\"auth_token\":\" \"}\r\n", + "CONNECT {\"user\":\"derek\",\"auth_token\":\"[REDACTED]\"}\r\n", + }, + { + "user and pass whitespace", + "CONNECT {\"user\":\"derek\",\"auth_token\": \"s3cr3t\" }\r\n", + "CONNECT {\"user\":\"derek\",\"auth_token\": \"[REDACTED]\" }\r\n", + }, + { + "only pass", + "CONNECT {\"auth_token\":\"s3cr3t\",}\r\n", + "CONNECT {\"auth_token\":\"[REDACTED]\",}\r\n", + }, + { + "invalid json", + "CONNECT {auth_token:s3cr3t , password = s3cr3t}", + "CONNECT {auth_token:[REDACTED], password = [REDACTED]}", + }, + { + "invalid json no whitespace after key", + "CONNECT {auth_token:s3cr3t , password= s3cr3t}", + "CONNECT {auth_token:[REDACTED], password= [REDACTED]}", + }, + { + "both pass and wrong password key", + `CONNECT {"auth_token":"s3cr3t4", "password": "s3cr3t4"}`, + `CONNECT {"auth_token":"[REDACTED]", "password": "[REDACTED]"}`, + }, + { + "invalid json", + "CONNECT {user = hello, auth_token = s3cr3t}", + "CONNECT {user = hello, auth_token = [REDACTED]}", + }, + { + "complete connect", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"auth_token\":\"s3cr3t\",\"tls_required\":false,\"name\":\"APM7JU94z77YzP6WTBEiuw\"}\r\n", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"auth_token\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"APM7JU94z77YzP6WTBEiuw\"}\r\n", + }, + { + "invalid json with only pass key", + "CONNECT {auth_token:s3cr3t\r\n", + "CONNECT {auth_token:[REDACTED]\r\n", + }, + { + "invalid password key also filtered", + "CONNECT {\"auth_token\":\"s3cr3t\",}\r\n", + "CONNECT {\"auth_token\":\"[REDACTED]\",}\r\n", + }, + { + "single long password with whitespace", + "CONNECT {\"auth_token\":\"secret password which is very long\",}\r\n", + "CONNECT {\"auth_token\":\"[REDACTED]\",}\r\n", + }, + { + "single long pass key is filtered", + "CONNECT {\"auth_token\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"}\r\n", + "CONNECT {\"auth_token\":\"[REDACTED]\"}\r\n", + }, + { + "duplicate keys all filtered", + "CONNECT {\"auth_token\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\",\"auth_token\":\"BBB\",\"pass\":\"BBBBBBBBBBBBBBBBBBBB\",\"password\":\"CCCCCCCCCCCCCCCC\"}\r\n", + "CONNECT {\"auth_token\":\"[REDACTED]\",\"auth_token\":\"[REDACTED]\",\"pass\":\"[REDACTED]\",\"password\":\"[REDACTED]\"}\r\n", + }, + { + "invalid json with multiple keys all filtered", + "CONNECT {auth_token = \"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\",pass= \"BBBBBBBBBBBBBBBBBBBB\",password =\"CCCCCCCCCCCCCCCC\"}\r\n", + "CONNECT {auth_token = \"[REDACTED]\",pass= \"[REDACTED]\",password =\"[REDACTED]\"}\r\n", + }, + { + "complete connect protocol", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"foo\",\"auth_token\":\"s3cr3t\",\"tls_required\":false,\"name\":\"APM7JU94z77YzP6WTBEiuw\"}\r\n", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"foo\",\"auth_token\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"APM7JU94z77YzP6WTBEiuw\"}\r\n", + }, + { + "user and token are filterered", + "CONNECT {\"user\":\"s3cr3t\",\"auth_token\":\"s3cr3t\"}\r\n", + "CONNECT {\"user\":\"s3cr3t\",\"auth_token\":\"[REDACTED]\"}\r\n", + }, + { + "complete connect using token key with user and token being the same", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"auth_token\":\"s3cr3t\",\"tls_required\":false,\"name\":\"...\"}\r\n", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"auth_token\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"...\"}\r\n", + }, + { + "complete connect with user, token and name all the same", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"auth_token\":\"s3cr3t\",\"tls_required\":false,\"name\":\"s3cr3t\"}\r\n", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"auth_token\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"s3cr3t\"}\r\n", + }, + { + "complete connect extra white space at the beginning", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"auth_token\":\"s3cr3t\",\"tls_required\":false,\"name\":\"foo\"}\r\n", + "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"auth_token\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"foo\"}\r\n", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + output := removeSecretsFromTrace([]byte(test.input)) + if !bytes.Equal(output, []byte(test.expected)) { + t.Errorf("\nExpected %q\n got: %q", test.expected, string(output)) + } + }) + } +} + +func TestRemoveAdditionalAuthFieldsFromTrace(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "sig", + input: "CONNECT {\"sig\":\"signature\",\"user\":\"foo\"}\r\n", + expected: "CONNECT {\"sig\":\"[REDACTED]\",\"user\":\"foo\"}\r\n", + }, + { + name: "nkey", + input: "CONNECT {\"nkey\":\"nkey\",\"user\":\"foo\"}\r\n", + expected: "CONNECT {\"nkey\":\"[REDACTED]\",\"user\":\"foo\"}\r\n", + }, + { + name: "proxy sig", + input: "CONNECT {\"proxy_sig\":\"proxy-signature\",\"user\":\"foo\"}\r\n", + expected: "CONNECT {\"proxy_sig\":\"[REDACTED]\",\"user\":\"foo\"}\r\n", + }, + { + name: "proxy sig followed by sig", + input: "CONNECT {\"proxy_sig\":\"proxy-signature\",\"sig\":\"sig\",\"user\":\"foo\"}\r\n", + expected: "CONNECT {\"proxy_sig\":\"[REDACTED]\",\"sig\":\"[REDACTED]\",\"user\":\"foo\"}\r\n", + }, + { + name: "sig followed by proxy sig", + input: "CONNECT {\"sig\":\"sig\",\"proxy_sig\":\"proxy-signature\",\"user\":\"foo\"}\r\n", + expected: "CONNECT {\"sig\":\"[REDACTED]\",\"proxy_sig\":\"[REDACTED]\",\"user\":\"foo\"}\r\n", + }, + { + name: "sig marker inside another string still redacts real sig", + input: "CONNECT {\"name\":\"foo sig:bar\",\"sig\":\"REAL\",\"user\":\"foo\"}\r\n", + expected: "CONNECT {\"name\":\"foo sig:[REDACTED]\",\"sig\":\"[REDACTED]\",\"user\":\"foo\"}\r\n", + }, + { + name: "malformed sig without object delimiters", + input: "CONNECT sig=abc\r\n", + expected: "CONNECT sig=[REDACTED]\r\n", + }, + { + name: "sig after malformed quoted field is redacted", + input: "CONNECT {\"name\":\"unterminated,\"sig\":\"abc\"}\r\n", + expected: "CONNECT {\"name\":\"unterminated,\"sig\":\"[REDACTED]\"}\r\n", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + output := removeSecretsFromTrace([]byte(test.input)) + if !bytes.Equal(output, []byte(test.expected)) { + t.Errorf("\nExpected %q\n got: %q", test.expected, string(output)) + } + }) + } +} diff --git a/server/memstore.go b/server/memstore.go new file mode 100644 index 0000000..2f7d078 --- /dev/null +++ b/server/memstore.go @@ -0,0 +1,3138 @@ +// Copyright 2019-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + crand "crypto/rand" + "encoding/binary" + "fmt" + "iter" + "math" + "slices" + "sync" + "time" + + "github.com/antithesishq/antithesis-sdk-go/assert" + "github.com/nats-io/nats-server/v2/server/ats" + "github.com/nats-io/nats-server/v2/server/avl" + "github.com/nats-io/nats-server/v2/server/gsl" + "github.com/nats-io/nats-server/v2/server/stree" + "github.com/nats-io/nats-server/v2/server/thw" +) + +// TODO(dlc) - This is a fairly simplistic approach but should do for now. +type memStore struct { + mu sync.RWMutex + cfg StreamConfig + state StreamState + msgs map[uint64]*StoreMsg + fss *stree.SubjectTree[SimpleState] + dmap avl.SequenceSet + maxp int64 + scb StorageUpdateHandler + rmcb StorageRemoveMsgHandler + pmsgcb ProcessJetStreamMsgHandler + ageChk *time.Timer // Timer to expire messages. + ageChkRun bool // Whether message expiration is currently running. + ageChkTime int64 // When the message expiration is scheduled to run. + recovering bool // Timers, schedules, TTLs etc won't fire while set. + consumers []ConsumerStore + receivedAny bool + ttls *thw.HashWheel + scheduling *MsgScheduling + sdm *SDMMeta + sources map[string]*StreamSourceState +} + +func newMemStore(cfg *StreamConfig) (*memStore, error) { + return newMemStoreWithMode(cfg, false) +} + +func newMemStoreWithMode(cfg *StreamConfig, recovering bool) (*memStore, error) { + if cfg == nil { + return nil, fmt.Errorf("config required") + } + if cfg.Storage != MemoryStorage { + return nil, fmt.Errorf("memStore requires memory storage type in config") + } + ms := &memStore{ + msgs: make(map[uint64]*StoreMsg), + fss: stree.NewSubjectTree[SimpleState](), + maxp: cfg.MaxMsgsPer, + cfg: *cfg, + recovering: recovering, + } + // Only create a THW if we're going to allow TTLs. + if cfg.AllowMsgTTL { + ms.ttls = thw.NewHashWheel() + } + if cfg.AllowMsgSchedules { + ms.scheduling = newMsgScheduling(ms.runMsgScheduling) + ms.scheduling.paused = recovering + } + if cfg.FirstSeq > 0 { + if _, err := ms.purge(cfg.FirstSeq); err != nil { + return nil, err + } + } + // Seed configured sources (zero sequence == configured but not yet observed), + // so storeRawMsg can validate headers against the configured set. + if len(cfg.Sources) > 0 { + ms.sources = make(map[string]*StreamSourceState, len(cfg.Sources)) + for _, ssi := range cfg.Sources { + ms.sources[ssi.composeIName()] = &StreamSourceState{} + } + } + + // Register with access time service. + ats.Register() + + return ms, nil +} + +func (ms *memStore) UpdateConfig(cfg *StreamConfig) error { + if cfg == nil { + return fmt.Errorf("config required") + } + if cfg.Storage != MemoryStorage { + return fmt.Errorf("memStore requires memory storage type in config") + } + + ms.mu.Lock() + ms.cfg = *cfg + // Create or delete the THW if needed. + if cfg.AllowMsgTTL && ms.ttls == nil { + ms.recoverTTLState() + } else if !cfg.AllowMsgTTL && ms.ttls != nil { + ms.ttls = nil + } + if cfg.AllowMsgSchedules && ms.scheduling == nil { + ms.recoverMsgSchedulingState() + } else if !cfg.AllowMsgSchedules && ms.scheduling != nil { + ms.scheduling = nil + } + + // Refresh the source tracking to match the new config. + if len(cfg.Sources) == 0 { + // If no sources remain, clear. + ms.sources = nil + } else { + // Ensure the map exists even if sources were just added, so storeRawMsg + // can validate headers against the configured set. + if ms.sources == nil { + ms.sources = make(map[string]*StreamSourceState, len(cfg.Sources)) + } + sources := make(map[string]*StreamSource, len(cfg.Sources)) + for _, ssi := range cfg.Sources { + sources[ssi.composeIName()] = ssi + } + // Remove sources that are tracked but no longer exist. + for k := range ms.sources { + if _, ok := sources[k]; !ok { + delete(ms.sources, k) + } + } + // Seed sources that aren't in the map yet. Anything newly seeded needs its + // sequence recovered, otherwise it would re-source from the very beginning. + scan := make(map[string]*StreamSource) + for k, ssi := range sources { + if _, ok := ms.sources[k]; !ok { + ms.sources[k] = &StreamSourceState{} + scan[k] = ssi + } + } + if len(scan) > 0 && ms.state.Msgs > 0 { + ms.recoverSourcesBackwardScan(scan) + } + } + + // Limits checks and enforcement. + ms.enforceMsgLimit() + ms.enforceBytesLimit() + // Do age timers. + if ms.ageChk == nil && ms.cfg.MaxAge != 0 { + ms.startAgeChk() + } + if ms.ageChk != nil && ms.cfg.MaxAge == 0 { + ms.ageChk.Stop() + ms.ageChk = nil + ms.ageChkTime = 0 + } + // Make sure to update MaxMsgsPer + if cfg.MaxMsgsPer < -1 { + cfg.MaxMsgsPer = -1 + } + maxp := ms.maxp + ms.maxp = cfg.MaxMsgsPer + // If the value is smaller, or was unset before, we need to enforce that. + if ms.maxp > 0 && (maxp <= 0 || ms.maxp < maxp) { + lm := uint64(ms.maxp) + ms.fss.IterFast(func(subj []byte, ss *SimpleState) bool { + if ss.Msgs > lm { + ms.enforcePerSubjectLimit(bytesToString(subj), ss) + } + return true + }) + } + ms.mu.Unlock() + + if cfg.MaxAge != 0 || cfg.AllowMsgTTL { + ms.expireMsgs() + } + if cfg.AllowMsgSchedules { + ms.runMsgScheduling() + } + return nil +} + +// Lock should be held. +func (ms *memStore) recoverTTLState() { + ms.ttls = thw.NewHashWheel() + if ms.state.Msgs == 0 { + return + } + + var ( + seq uint64 + smv StoreMsg + sm *StoreMsg + ) + defer ms.resetAgeChk(0) + for sm, seq, _ = ms.loadNextMsgLocked(fwcs, true, 0, &smv); sm != nil; sm, seq, _ = ms.loadNextMsgLocked(fwcs, true, seq+1, &smv) { + if len(sm.hdr) == 0 { + continue + } + if ttl, _ := getMessageTTL(sm.hdr); ttl > 0 { + expires := time.Duration(sm.ts) + (time.Second * time.Duration(ttl)) + ms.ttls.Add(seq, int64(expires)) + } + } +} + +// Lock should be held. +func (ms *memStore) recoverMsgSchedulingState() { + ms.scheduling = newMsgScheduling(ms.runMsgScheduling) + ms.scheduling.paused = ms.recovering + if ms.state.Msgs == 0 { + return + } + + var ( + seq uint64 + smv StoreMsg + sm *StoreMsg + ) + defer ms.scheduling.resetTimer() + for sm, seq, _ = ms.loadNextMsgLocked(fwcs, true, 0, &smv); sm != nil; sm, seq, _ = ms.loadNextMsgLocked(fwcs, true, seq+1, &smv) { + if len(sm.hdr) == 0 { + continue + } + if schedule, apiErr := nextMessageSchedule(sm.hdr, sm.ts); apiErr == nil && !schedule.IsZero() { + ms.scheduling.init(seq, sm.subj, schedule.UnixNano()) + } + } +} + +// Stores a raw message with expected sequence number and timestamp. +// Lock should be held. +func (ms *memStore) storeRawMsg(subj string, hdr, msg []byte, seq uint64, ts, ttl int64, discardNewCheck bool) error { + if ms.msgs == nil { + return ErrStoreClosed + } + + // Tracking by subject. + var ss *SimpleState + var asl bool + if len(subj) > 0 { + var ok bool + if ss, ok = ms.fss.Find(stringToBytes(subj)); ok { + asl = ms.maxp > 0 && ss.Msgs >= uint64(ms.maxp) + } + } + + // Check if we are discarding new messages when we reach the limit. + // If we are clustered, we do the enforcement above and should not disqualify + // the message here since it could cause replicas to drift. + if discardNewCheck && ms.cfg.Discard == DiscardNew { + // Allow rollup messages through since they will purge old + // messages for the subject after storing, restoring the limit. + if asl && ms.cfg.DiscardNewPer && len(sliceHeader(JSMsgRollup, hdr)) == 0 { + return ErrMaxMsgsPerSubject + } + if ms.cfg.MaxMsgs > 0 && ms.state.Msgs >= uint64(ms.cfg.MaxMsgs) { + // If we are tracking max messages per subject and are at the limit we will replace, so this is ok. + if !asl { + return ErrMaxMsgs + } + } + if ms.cfg.MaxBytes > 0 && ms.state.Bytes+memStoreMsgSize(subj, hdr, msg) > uint64(ms.cfg.MaxBytes) { + if !asl { + return ErrMaxBytes + } + // If we are here we are at a subject maximum, need to determine if dropping last message gives us enough room. + if ss.firstNeedsUpdate || ss.lastNeedsUpdate { + ms.recalculateForSubj(subj, ss) + } + sm, ok := ms.msgs[ss.First] + if !ok || memStoreMsgSize(sm.subj, sm.hdr, sm.msg) < memStoreMsgSize(subj, hdr, msg) { + return ErrMaxBytes + } + } + } + + if seq != ms.state.LastSeq+1 { + if seq > 0 { + return ErrSequenceMismatch + } + seq = ms.state.LastSeq + 1 + } + + // Adjust first if needed. + now := time.Unix(0, ts).UTC() + if ms.state.Msgs == 0 { + ms.state.FirstSeq = seq + ms.state.FirstTime = now + } + + // Make copies + // TODO(dlc) - Maybe be smarter here. + if len(msg) > 0 { + msg = copyBytes(msg) + } + if len(hdr) > 0 { + hdr = copyBytes(hdr) + } + + // FIXME(dlc) - Could pool at this level? + sm := &StoreMsg{subj, nil, nil, make([]byte, 0, len(hdr)+len(msg)), seq, ts} + sm.buf = append(sm.buf, hdr...) + sm.buf = append(sm.buf, msg...) + if len(hdr) > 0 { + sm.hdr = sm.buf[:len(hdr)] + } + sm.msg = sm.buf[len(hdr):] + ms.msgs[seq] = sm + ms.state.Msgs++ + ms.state.Bytes += memStoreMsgSize(subj, hdr, msg) + ms.state.LastSeq = seq + ms.state.LastTime = now + + // Track per subject. + if len(subj) > 0 { + if ss != nil { + ss.Msgs++ + ss.Last = seq + ss.lastNeedsUpdate = false + // Check per subject limits. + if ms.maxp > 0 && ss.Msgs > uint64(ms.maxp) { + ms.enforcePerSubjectLimit(subj, ss) + } + } else { + ms.fss.Insert([]byte(subj), SimpleState{Msgs: 1, First: seq, Last: seq}) + } + } + + // Limits checks and enforcement. + ms.enforceMsgLimit() + ms.enforceBytesLimit() + + // Per-message TTL. + if ms.ttls != nil && ttl > 0 { + expires := time.Duration(ts) + (time.Second * time.Duration(ttl)) + ms.ttls.Add(seq, int64(expires)) + } + + // Check if we have and need the age expiration timer running. + switch { + case ms.ttls != nil && ttl > 0: + ms.resetAgeChk(0) + case ms.ageChk == nil && (ms.cfg.MaxAge > 0 || ms.ttls != nil): + ms.startAgeChk() + } + + // Message scheduling. + if ms.scheduling != nil { + if schedule, apiErr := nextMessageSchedule(hdr, ts); apiErr == nil && !schedule.IsZero() { + ms.scheduling.add(seq, subj, schedule.UnixNano()) + } else if getMessageScheduler(hdr) == _EMPTY_ { + ms.scheduling.removeSubject(subj) + } + + // Check for a repeating schedule and update such that it triggers again. + if scheduleNext := bytesToString(sliceHeader(JSScheduleNext, hdr)); scheduleNext != _EMPTY_ && scheduleNext != JSScheduleNextPurge { + scheduler := getMessageScheduler(hdr) + if next, err := time.Parse(time.RFC3339Nano, scheduleNext); err == nil && scheduler != _EMPTY_ { + ms.scheduling.update(scheduler, next.UnixNano()) + } + } + } + + // Track sources. + if len(ms.cfg.Sources) > 0 { + if ss := sliceHeader(JSStreamSource, hdr); len(ss) > 0 { + _, indexName, sseq, ident := streamAndSeq(bytesToString(ss)) + if sss, ok := ms.sources[indexName]; ok { + sss.Seq, sss.Ident = sseq, ident + } + } + } + + return nil +} + +// StoreRawMsg stores a raw message with expected sequence number and timestamp. +func (ms *memStore) StoreRawMsg(subj string, hdr, msg []byte, seq uint64, ts, ttl int64, discardNewCheck bool) error { + ms.mu.Lock() + err := ms.storeRawMsg(subj, hdr, msg, seq, ts, ttl, discardNewCheck) + cb := ms.scb + // Check if first message timestamp requires expiry + // sooner than initial replica expiry timer set to MaxAge when initializing. + if !ms.receivedAny && ms.cfg.MaxAge != 0 && ts > 0 { + ms.receivedAny = true + // Calculate duration when the next expireMsgs should be called. + ms.resetAgeChk(int64(time.Millisecond) * 50) + } + ms.mu.Unlock() + + if err == nil && cb != nil { + cb(1, int64(memStoreMsgSize(subj, hdr, msg)), seq, subj) + } + + return err +} + +// Store stores a message. +func (ms *memStore) StoreMsg(subj string, hdr, msg []byte, ttl int64) (uint64, int64, error) { + ms.mu.Lock() + seq, ts := ms.state.LastSeq+1, time.Now().UnixNano() + // This is called for a R1 with no expected sequence number, so perform DiscardNew checks on the store-level. + err := ms.storeRawMsg(subj, hdr, msg, seq, ts, ttl, true) + cb := ms.scb + ms.mu.Unlock() + + if err != nil { + seq, ts = 0, 0 + } else if cb != nil { + cb(1, int64(memStoreMsgSize(subj, hdr, msg)), seq, subj) + } + + return seq, ts, err +} + +// SkipMsg will use the next sequence number but not store anything. +func (ms *memStore) SkipMsg(seq uint64) (uint64, error) { + return ms.skipMsg(seq, false) +} + +// SkipMsgNoInterest will use the next sequence number but not store anything. +// Unlike SkipMsg it also advances LastTime, which is used for Interest retention. +func (ms *memStore) SkipMsgNoInterest(seq uint64) (uint64, error) { + return ms.skipMsg(seq, true) +} + +func (ms *memStore) skipMsg(seq uint64, noInterest bool) (uint64, error) { + ms.mu.Lock() + defer ms.mu.Unlock() + + // Check sequence matches our last sequence. + if seq != ms.state.LastSeq+1 { + if seq > 0 { + return 0, ErrSequenceMismatch + } + seq = ms.state.LastSeq + 1 + } + + ms.state.LastSeq = seq + // Only update time if not already set or for Interest retention. + if noInterest || ms.state.LastTime.IsZero() { + ms.state.LastTime = time.Unix(0, ats.AccessTime()).UTC() + } + if ms.state.Msgs == 0 { + ms.state.FirstSeq = seq + 1 + ms.state.FirstTime = time.Time{} + } else { + ms.dmap.Insert(seq) + } + return seq, nil +} + +// Skip multiple msgs. +func (ms *memStore) SkipMsgs(seq uint64, num uint64) error { + ms.mu.Lock() + defer ms.mu.Unlock() + + // Check sequence matches our last sequence. + if seq != ms.state.LastSeq+1 { + if seq > 0 { + return ErrSequenceMismatch + } + seq = ms.state.LastSeq + 1 + } + lseq := seq + num - 1 + + ms.state.LastSeq = lseq + // Only update time if not already set. + if ms.state.LastTime.IsZero() { + ms.state.LastTime = time.Unix(0, ats.AccessTime()).UTC() + } + if ms.state.Msgs == 0 { + ms.state.FirstSeq, ms.state.FirstTime = lseq+1, time.Time{} + } else { + for ; seq <= lseq; seq++ { + ms.dmap.Insert(seq) + } + } + return nil +} + +// FlushAllPending flushes all data that was still pending to be written. +func (ms *memStore) FlushAllPending() error { + // Noop, in-memory store doesn't use async applying. + return nil +} + +// RegisterStorageUpdates registers a callback for updates to storage changes. +// It will present number of messages and bytes as a signed integer and an +// optional sequence number of the message if a single. +func (ms *memStore) RegisterStorageUpdates(cb StorageUpdateHandler) { + ms.mu.Lock() + ms.scb = cb + ms.mu.Unlock() +} + +// RegisterStorageRemoveMsg registers a callback to remove messages. +// Replicated streams should propose removals, R1 can remove inline. +func (ms *memStore) RegisterStorageRemoveMsg(cb StorageRemoveMsgHandler) { + ms.mu.Lock() + ms.rmcb = cb + ms.mu.Unlock() +} + +// RegisterProcessJetStreamMsg registers a callback to process new JetStream messages. +func (ms *memStore) RegisterProcessJetStreamMsg(cb ProcessJetStreamMsgHandler) { + ms.mu.Lock() + ms.pmsgcb = cb + ms.mu.Unlock() +} + +// GetSeqFromTime looks for the first sequence number that has the message +// with >= timestamp. +func (ms *memStore) GetSeqFromTime(t time.Time) uint64 { + ts := t.UnixNano() + ms.mu.RLock() + defer ms.mu.RUnlock() + if len(ms.msgs) == 0 { + return ms.state.LastSeq + 1 + } + if ts <= ms.msgs[ms.state.FirstSeq].ts { + return ms.state.FirstSeq + } + // LastSeq is not guaranteed to be present since last does not go backwards. + var lmsg *StoreMsg + for lseq := ms.state.LastSeq; lseq > ms.state.FirstSeq; lseq-- { + if lmsg = ms.msgs[lseq]; lmsg != nil { + break + } + } + if lmsg == nil { + return ms.state.LastSeq + 1 + } + + last := lmsg.ts + if ts == last { + return lmsg.seq + } + if ts > last { + return ms.state.LastSeq + 1 + } + + var ( + cts int64 + cseq uint64 + off uint64 + ) + + // Using a binary search, but need to be aware of interior deletes. + fseq := ms.state.FirstSeq + lseq := ms.state.LastSeq + seq := lseq + 1 +loop: + for fseq <= lseq { + mid := fseq + (lseq-fseq)/2 + off = 0 + // Potentially skip over gaps. We keep the original middle but keep track of a + // potential delete range with an offset. + for { + msg := ms.msgs[mid+off] + if msg == nil { + off++ + if mid+off <= lseq { + continue + } else { + // Continue search to the left. Purposely ignore the skipped deletes here. + lseq = mid - 1 + continue loop + } + } + cts = msg.ts + cseq = msg.seq + break + } + if cts >= ts { + seq = cseq + if mid == fseq { + break + } + // Continue search to the left. + lseq = mid - 1 + } else { + // Continue search to the right (potentially skipping over interior deletes). + fseq = mid + off + 1 + } + } + return seq +} + +// FilteredState will return the SimpleState associated with the filtered subject and a proposed starting sequence. +func (ms *memStore) FilteredState(sseq uint64, subj string) (SimpleState, error) { + // This needs to be a write lock, as filteredStateLocked can + // mutate the per-subject state. + ms.mu.Lock() + defer ms.mu.Unlock() + + return ms.filteredStateLocked(sseq, subj, false), nil +} + +func (ms *memStore) filteredStateLocked(sseq uint64, filter string, lastPerSubject bool) SimpleState { + if sseq < ms.state.FirstSeq { + sseq = ms.state.FirstSeq + } + + // If past the end no results. + if sseq > ms.state.LastSeq { + return SimpleState{} + } + + if filter == _EMPTY_ { + filter = fwcs + } + isAll := filter == fwcs + + // First check if we can optimize this part. + // This means we want all and the starting sequence was before this block. + if isAll && sseq <= ms.state.FirstSeq { + total := ms.state.Msgs + if lastPerSubject { + total = uint64(ms.fss.Size()) + } + return SimpleState{ + Msgs: total, + First: ms.state.FirstSeq, + Last: ms.state.LastSeq, + } + } + + _tsa, _fsa := [32]string{}, [32]string{} + tsa, fsa := _tsa[:0], _fsa[:0] + wc := subjectHasWildcard(filter) + if wc { + fsa = tokenizeSubjectIntoSlice(fsa[:0], filter) + } + // 1. See if we match any subs from fss. + // 2. If we match and the sseq is past ss.Last then we can use meta only. + // 3. If we match we need to do a partial, break and clear any totals and do a full scan like num pending. + + isMatch := func(subj string) bool { + if isAll { + return true + } + if !wc { + return subj == filter + } + tsa = tokenizeSubjectIntoSlice(tsa[:0], subj) + return isSubsetMatchTokenized(tsa, fsa) + } + + var ss SimpleState + update := func(fss *SimpleState) { + msgs, first, last := fss.Msgs, fss.First, fss.Last + if lastPerSubject { + msgs, first = 1, last + } + ss.Msgs += msgs + if ss.First == 0 || first < ss.First { + ss.First = first + } + if last > ss.Last { + ss.Last = last + } + } + + var havePartial bool + var totalSkipped uint64 + // We will track start and end sequences as we go. + ms.fss.Match(stringToBytes(filter), func(subj []byte, fss *SimpleState) { + if fss.firstNeedsUpdate || fss.lastNeedsUpdate { + ms.recalculateForSubj(bytesToString(subj), fss) + } + if sseq <= fss.First { + update(fss) + } else if sseq <= fss.Last { + // We matched but it is a partial. + havePartial = true + // Don't break here, we will update to keep tracking last. + update(fss) + } else { + totalSkipped += fss.Msgs + } + }) + + // If we did not encounter any partials we can return here. + if !havePartial { + return ss + } + + // If we are here we need to scan the msgs. + // Capture first and last sequences for scan and then clear what we had. + first, last := ss.First, ss.Last + // To track if we decide to exclude we need to calculate first. + var needScanFirst bool + if first < sseq { + first = sseq + needScanFirst = true + } + + // Now we want to check if it is better to scan inclusive and recalculate that way + // or leave and scan exclusive and adjust our totals. + // ss.Last is always correct here. + toScan, toExclude := last-first, first-ms.state.FirstSeq+ms.state.LastSeq-ss.Last + var seen map[string]bool + if lastPerSubject { + seen = make(map[string]bool) + } + if toScan < toExclude { + ss.Msgs, ss.First = 0, 0 + + update := func(sm *StoreMsg) { + ss.Msgs++ + if ss.First == 0 { + ss.First = sm.seq + } + if seen != nil { + seen[sm.subj] = true + } + } + // Check if easier to just scan msgs vs the sequence range. + // This can happen with lots of interior deletes. + if last-first > uint64(len(ms.msgs)) { + for _, sm := range ms.msgs { + if sm.seq >= first && sm.seq <= last && !seen[sm.subj] && isMatch(sm.subj) { + update(sm) + } + } + } else { + for seq := first; seq <= last; seq++ { + if sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && isMatch(sm.subj) { + update(sm) + } + } + } + } else { + // We will adjust from the totals above by scanning what we need to exclude. + ss.First = first + ss.Msgs += totalSkipped + var adjust uint64 + var tss *SimpleState + + update := func(sm *StoreMsg) { + if lastPerSubject { + tss, _ = ms.fss.Find(stringToBytes(sm.subj)) + } + if tss == nil { + adjust++ + } else if tss.Last < first { + // If we are last per subject, make sure to only adjust if all messages are before our first. + adjust += tss.Msgs + } + if seen != nil { + seen[sm.subj] = true + } + } + // Check if easier to just scan msgs vs the sequence range. + if first-ms.state.FirstSeq > uint64(len(ms.msgs)) { + for _, sm := range ms.msgs { + if sm.seq < first && !seen[sm.subj] && isMatch(sm.subj) { + update(sm) + } + } + } else { + for seq := ms.state.FirstSeq; seq < first; seq++ { + if sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && isMatch(sm.subj) { + update(sm) + } + } + } + // Now do range at end. + for seq := last + 1; seq < ms.state.LastSeq; seq++ { + if sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && isMatch(sm.subj) { + adjust++ + if seen != nil { + seen[sm.subj] = true + } + } + } + ss.Msgs -= adjust + if needScanFirst { + // Check if easier to just scan msgs vs the sequence range. + // Since we will need to scan all of the msgs vs below where we break on the first match, + // we will only do so if a few orders of magnitude lower. + if last-first > 100*uint64(len(ms.msgs)) { + low := ms.state.LastSeq + for _, sm := range ms.msgs { + if sm.seq >= first && sm.seq < last && isMatch(sm.subj) { + if sm.seq < low { + low = sm.seq + } + } + } + if low < ms.state.LastSeq { + ss.First = low + } + } else { + for seq := first; seq < last; seq++ { + if sm, ok := ms.msgs[seq]; ok && isMatch(sm.subj) { + ss.First = seq + break + } + } + } + } + } + + return ss +} + +// SubjectsState returns a map of SimpleState for all matching subjects. +func (ms *memStore) SubjectsState(subject string) map[string]SimpleState { + // This needs to be a write lock, as we can mutate the per-subject state. + ms.mu.Lock() + defer ms.mu.Unlock() + + if ms.fss.Size() == 0 { + return nil + } + + if subject == _EMPTY_ { + subject = fwcs + } + + fss := make(map[string]SimpleState) + ms.fss.Match(stringToBytes(subject), func(subj []byte, ss *SimpleState) { + subjs := string(subj) + if ss.firstNeedsUpdate || ss.lastNeedsUpdate { + ms.recalculateForSubj(subjs, ss) + } + oss := fss[subjs] + if oss.First == 0 { // New + fss[subjs] = *ss + } else { + // Merge here. + oss.Last, oss.Msgs = ss.Last, oss.Msgs+ss.Msgs + fss[subjs] = oss + } + }) + return fss +} + +// AllLastSeqs will return a sorted list of last sequences for all subjects. +func (ms *memStore) AllLastSeqs() ([]uint64, error) { + ms.mu.RLock() + defer ms.mu.RUnlock() + return ms.allLastSeqsLocked() +} + +// allLastSeqsLocked will return a sorted list of last sequences for all +// subjects, but won't take the lock to do it, to avoid the issue of compounding +// read locks causing a deadlock with a write lock. +func (ms *memStore) allLastSeqsLocked() ([]uint64, error) { + if len(ms.msgs) == 0 { + return nil, nil + } + + seqs := make([]uint64, 0, ms.fss.Size()) + ms.fss.IterFast(func(subj []byte, ss *SimpleState) bool { + // Check if we need to recalculate. We only care about the last sequence. + if ss.lastNeedsUpdate { + ms.recalculateForSubj(bytesToString(subj), ss) + } + seqs = append(seqs, ss.Last) + return true + }) + + slices.Sort(seqs) + return seqs, nil +} + +// Helper to determine if the filter(s) represent all the subjects. +// Most clients send in subjects even if they match the stream's ingest subjects. +// Lock should be held. +func (ms *memStore) filterIsAll(filters []string) bool { + if len(filters) == 1 && filters[0] == fwcs { + return true + } + if len(filters) != len(ms.cfg.Subjects) { + return false + } + // Sort so we can compare. + subjects := copyStrings(ms.cfg.Subjects) + slices.Sort(filters) + slices.Sort(subjects) + for i, subj := range filters { + if !subjectIsSubsetMatch(subjects[i], subj) { + return false + } + } + return true +} + +// MultiLastSeqs will return a sorted list of sequences that match all subjects presented in filters. +// We will not exceed the maxSeq, which if 0 becomes the store's last sequence. +func (ms *memStore) MultiLastSeqs(filters []string, maxSeq uint64, maxAllowed int) ([]uint64, error) { + ms.mu.Lock() + defer ms.mu.Unlock() + return ms.multiLastSeqsLocked(filters, maxSeq, maxAllowed) +} + +// Write lock should be held, for recalculateForSubj. +func (ms *memStore) multiLastSeqsLocked(filters []string, maxSeq uint64, maxAllowed int) ([]uint64, error) { + if len(ms.msgs) == 0 { + return nil, nil + } + + // See if we can short circuit if we think they are asking for all last sequences and have no maxSeq or maxAllowed set. + if maxSeq == 0 && maxAllowed <= 0 && ms.filterIsAll(filters) { + return ms.allLastSeqsLocked() + } + + // Implied last sequence. + if maxSeq == 0 { + maxSeq = ms.state.LastSeq + } + + seqs := make([]uint64, 0, 64) + seen := make(map[uint64]struct{}) + + addIfNotDupe := func(seq uint64) { + if _, ok := seen[seq]; !ok { + seqs = append(seqs, seq) + seen[seq] = struct{}{} + } + } + + for _, filter := range filters { + ms.fss.Match(stringToBytes(filter), func(subj []byte, ss *SimpleState) { + if ss.lastNeedsUpdate { + ms.recalculateForSubj(bytesToString(subj), ss) + } + if ss.Last <= maxSeq { + addIfNotDupe(ss.Last) + } else if ss.Msgs > 1 { + // The last is greater than maxSeq. + s := bytesToString(subj) + for seq := maxSeq; seq > 0; seq-- { + if sm, ok := ms.msgs[seq]; ok && sm.subj == s { + addIfNotDupe(seq) + break + } + } + } + }) + // If maxAllowed was sepcified check that we will not exceed that. + if maxAllowed > 0 && len(seqs) > maxAllowed { + return nil, ErrTooManyResults + } + } + slices.Sort(seqs) + return seqs, nil +} + +// MultiLastMsgs delivers the last message per subject matching the filters, +// up to maxSeq and skipping sequences below minSeq, in ascending sequence +// order through cb, all within a single read section so the batch is a +// point-in-time snapshot of the store. +func (ms *memStore) MultiLastMsgs(filters []string, minSeq, maxSeq uint64, maxAllowed int, cb func(sm *StoreMsg, np uint64) bool) (uint64, uint64, error) { + ms.mu.Lock() + defer ms.mu.Unlock() + + seqs, err := ms.multiLastSeqsLocked(filters, maxSeq, maxAllowed) + if err != nil || len(seqs) == 0 { + return 0, 0, err + } + total := uint64(len(seqs)) + np := total + for _, seq := range seqs { + if np > 0 { + np-- + } + if seq < minSeq { + continue + } + var svp StoreMsg + sm, err := ms.loadMsgLocked(seq, &svp, false) + if err != nil { + return total, np, err + } + if !cb(sm, np) { + break + } + } + return total, np, nil +} + +// SubjectsTotals return message totals per subject. +func (ms *memStore) SubjectsTotals(filterSubject string) map[string]uint64 { + ms.mu.RLock() + defer ms.mu.RUnlock() + return ms.subjectsTotalsLocked(filterSubject) +} + +// Lock should be held. +func (ms *memStore) subjectsTotalsLocked(filterSubject string) map[string]uint64 { + if ms.fss.Size() == 0 { + return nil + } + + _tsa, _fsa := [32]string{}, [32]string{} + tsa, fsa := _tsa[:0], _fsa[:0] + fsa = tokenizeSubjectIntoSlice(fsa[:0], filterSubject) + isAll := filterSubject == _EMPTY_ || filterSubject == fwcs + + fst := make(map[string]uint64) + ms.fss.Match(stringToBytes(filterSubject), func(subj []byte, ss *SimpleState) { + subjs := string(subj) + if isAll { + fst[subjs] = ss.Msgs + } else { + if tsa = tokenizeSubjectIntoSlice(tsa[:0], subjs); isSubsetMatchTokenized(tsa, fsa) { + fst[subjs] = ss.Msgs + } + } + }) + return fst +} + +// ApplySourcesState replaces the tracked sourcing state with the leader's. +func (ms *memStore) ApplySourcesState(sources map[string]StreamSourceState) { + if len(sources) == 0 { + return + } + ms.mu.Lock() + defer ms.mu.Unlock() + if ms.sources == nil { + return + } + clear(ms.sources) + for name, ss := range sources { + ms.sources[name] = &StreamSourceState{Seq: ss.Seq, Ident: ss.Ident} + } + for _, ssi := range ms.cfg.Sources { + if iname := ssi.composeIName(); ms.sources[iname] == nil { + ms.sources[iname] = &StreamSourceState{} + } + } +} + +// SourcesState return sources keys and their last known state. +func (ms *memStore) SourcesState() map[string]StreamSourceState { + ms.mu.RLock() + defer ms.mu.RUnlock() + if len(ms.sources) == 0 { + return nil + } + dst := make(map[string]StreamSourceState, len(ms.sources)) + for k, ss := range ms.sources { + dst[k] = *ss + } + return dst +} + +// NumPending will return the number of pending messages matching the filter subject starting at sequence. +func (ms *memStore) NumPending(sseq uint64, filter string, lastPerSubject bool) (total, validThrough uint64, err error) { + // This needs to be a write lock, as filteredStateLocked can mutate the per-subject state. + ms.mu.Lock() + defer ms.mu.Unlock() + + ss := ms.filteredStateLocked(sseq, filter, lastPerSubject) + return ss.Msgs, ms.state.LastSeq, nil +} + +// NumPending will return the number of pending messages matching any subject in the sublist starting at sequence. +func (ms *memStore) NumPendingMulti(sseq uint64, sl *gsl.SimpleSublist, lastPerSubject bool) (total, validThrough uint64, err error) { + if sl == nil { + return ms.NumPending(sseq, fwcs, lastPerSubject) + } + + // This needs to be a write lock, as we can mutate the per-subject state. + ms.mu.Lock() + defer ms.mu.Unlock() + + var ss SimpleState + if sseq < ms.state.FirstSeq { + sseq = ms.state.FirstSeq + } + // If past the end no results. + if sseq > ms.state.LastSeq { + return 0, ms.state.LastSeq, nil + } + + update := func(fss *SimpleState) { + msgs, first, last := fss.Msgs, fss.First, fss.Last + if lastPerSubject { + msgs, first = 1, last + } + ss.Msgs += msgs + if ss.First == 0 || first < ss.First { + ss.First = first + } + if last > ss.Last { + ss.Last = last + } + } + + var havePartial bool + var totalSkipped uint64 + // We will track start and end sequences as we go. + stree.IntersectGSL[SimpleState](ms.fss, sl, func(subj []byte, fss *SimpleState) bool { + if fss.firstNeedsUpdate || fss.lastNeedsUpdate { + ms.recalculateForSubj(bytesToString(subj), fss) + } + if sseq <= fss.First { + update(fss) + } else if sseq <= fss.Last { + // We matched but it is a partial. + havePartial = true + // Don't break here, we will update to keep tracking last. + update(fss) + } else { + totalSkipped += fss.Msgs + } + return true + }) + + // If we did not encounter any partials we can return here. + if !havePartial { + return ss.Msgs, ms.state.LastSeq, nil + } + + // If we are here we need to scan the msgs. + // Capture first and last sequences for scan and then clear what we had. + first, last := ss.First, ss.Last + // To track if we decide to exclude we need to calculate first. + if first < sseq { + first = sseq + } + + // Now we want to check if it is better to scan inclusive and recalculate that way + // or leave and scan exclusive and adjust our totals. + // ss.Last is always correct here. + toScan, toExclude := last-first, first-ms.state.FirstSeq+ms.state.LastSeq-ss.Last + var seen map[string]bool + if lastPerSubject { + seen = make(map[string]bool) + } + if toScan < toExclude { + ss.Msgs, ss.First = 0, 0 + + update := func(sm *StoreMsg) { + ss.Msgs++ + if ss.First == 0 { + ss.First = sm.seq + } + if seen != nil { + seen[sm.subj] = true + } + } + // Check if easier to just scan msgs vs the sequence range. + // This can happen with lots of interior deletes. + if last-first > uint64(len(ms.msgs)) { + for _, sm := range ms.msgs { + if sm.seq >= first && sm.seq <= last && !seen[sm.subj] && sl.HasInterest(sm.subj) { + update(sm) + } + } + } else { + for seq := first; seq <= last; seq++ { + if sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && sl.HasInterest(sm.subj) { + update(sm) + } + } + } + } else { + // We will adjust from the totals above by scanning what we need to exclude. + ss.First = first + ss.Msgs += totalSkipped + var adjust uint64 + var tss *SimpleState + + update := func(sm *StoreMsg) { + if lastPerSubject { + tss, _ = ms.fss.Find(stringToBytes(sm.subj)) + } + if tss == nil { + adjust++ + } else if tss.Last < first { + // If we are last per subject, make sure to only adjust if all messages are before our first. + adjust += tss.Msgs + } + if seen != nil { + seen[sm.subj] = true + } + } + // Check if easier to just scan msgs vs the sequence range. + if first-ms.state.FirstSeq > uint64(len(ms.msgs)) { + for _, sm := range ms.msgs { + if sm.seq < first && !seen[sm.subj] && sl.HasInterest(sm.subj) { + update(sm) + } + } + } else { + for seq := ms.state.FirstSeq; seq < first; seq++ { + if sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && sl.HasInterest(sm.subj) { + update(sm) + } + } + } + // Now do range at end. + for seq := last + 1; seq < ms.state.LastSeq; seq++ { + if sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && sl.HasInterest(sm.subj) { + adjust++ + if seen != nil { + seen[sm.subj] = true + } + } + } + ss.Msgs -= adjust + } + + return ss.Msgs, ms.state.LastSeq, nil +} + +// Will check the msg limit for this tracked subject. +// Lock should be held. +func (ms *memStore) enforcePerSubjectLimit(subj string, ss *SimpleState) { + if ms.maxp <= 0 { + return + } + for nmsgs := ss.Msgs; nmsgs > uint64(ms.maxp); nmsgs = ss.Msgs { + if ss.firstNeedsUpdate || ss.lastNeedsUpdate { + ms.recalculateForSubj(subj, ss) + } + if !ms.removeMsg(ss.First, false) { + break + } + } +} + +// Will check the msg limit and drop firstSeq msg if needed. +// Lock should be held. +func (ms *memStore) enforceMsgLimit() { + if ms.cfg.Discard != DiscardOld { + return + } + if ms.cfg.MaxMsgs <= 0 || ms.state.Msgs <= uint64(ms.cfg.MaxMsgs) { + return + } + for nmsgs := ms.state.Msgs; nmsgs > uint64(ms.cfg.MaxMsgs); nmsgs = ms.state.Msgs { + ms.deleteFirstMsgOrPanic() + } +} + +// Will check the bytes limit and drop msgs if needed. +// Lock should be held. +func (ms *memStore) enforceBytesLimit() { + if ms.cfg.Discard != DiscardOld { + return + } + if ms.cfg.MaxBytes <= 0 || ms.state.Bytes <= uint64(ms.cfg.MaxBytes) { + return + } + for bs := ms.state.Bytes; bs > uint64(ms.cfg.MaxBytes); bs = ms.state.Bytes { + ms.deleteFirstMsgOrPanic() + } +} + +// Will start the age check timer. +// Lock should be held. +func (ms *memStore) startAgeChk() { + if ms.recovering || ms.ageChk != nil { + return + } + if ms.cfg.MaxAge != 0 || ms.ttls != nil { + ms.ageChk = time.AfterFunc(ms.cfg.MaxAge, ms.expireMsgs) + } +} + +// Lock should be held. +func (ms *memStore) resetAgeChk(delta int64) { + if ms.recovering { + return + } + // If we're already expiring messages, it will make sure to reset. + // Don't trigger again, as that could result in many expire goroutines. + if ms.ageChkRun { + return + } + + var next int64 = math.MaxInt64 + if ms.ttls != nil { + next = ms.ttls.GetNextExpiration(next) + } + + // If there's no MaxAge and there's nothing waiting to be expired then + // don't bother continuing. The next storeRawMsg() will wake us up if + // needs be. + if ms.cfg.MaxAge <= 0 && next == math.MaxInt64 { + clearTimer(&ms.ageChk) + return + } + + // Check to see if we should be firing sooner than MaxAge for an expiring TTL. + fireIn := ms.cfg.MaxAge + + // If delta for next-to-expire message is unset, but we still have messages to remove. + // Assume messages are removed through proposals, and we need to speed up subsequent age check. + if delta == 0 && ms.state.Msgs > 0 { + if until := 2 * time.Second; until < fireIn { + fireIn = until + } + } + + if next < math.MaxInt64 { + // Looks like there's a next expiration, use it either if there's no + // MaxAge set or if it looks to be sooner than MaxAge is. + if until := time.Until(time.Unix(0, next)); fireIn == 0 || until < fireIn { + fireIn = until + } + } + + // If not then look at the delta provided (usually gap to next age expiry). + if delta > 0 { + if fireIn == 0 || time.Duration(delta) < fireIn { + fireIn = time.Duration(delta) + } + } + + // Make sure we aren't firing too often either way, otherwise we can + // negatively impact stream ingest performance. + if fireIn < 250*time.Millisecond { + fireIn = 250 * time.Millisecond + } + + // If we want to kick the timer to run later than what was assigned before, don't reset it. + // Otherwise, we could get in a situation where the timer is continuously reset, and it never runs. + expires := ats.AccessTime() + fireIn.Nanoseconds() + if ms.ageChkTime > 0 && expires > ms.ageChkTime { + return + } + + ms.ageChkTime = expires + if ms.ageChk != nil { + ms.ageChk.Reset(fireIn) + } else { + ms.ageChk = time.AfterFunc(fireIn, ms.expireMsgs) + } +} + +// Lock should be held. +func (ms *memStore) cancelAgeChk() { + if ms.ageChk != nil { + ms.ageChk.Stop() + ms.ageChk = nil + ms.ageChkTime = 0 + } +} + +// Will expire msgs that are too old. +func (ms *memStore) expireMsgs() { + var smv StoreMsg + var sm *StoreMsg + ms.mu.Lock() + if ms.recovering { + ms.cancelAgeChk() + ms.mu.Unlock() + return + } + maxAge := int64(ms.cfg.MaxAge) + minAge := time.Now().UnixNano() - maxAge + rmcb := ms.rmcb + pmsgcb := ms.pmsgcb + sdmTTL := int64(ms.cfg.SubjectDeleteMarkerTTL.Seconds()) + sdmEnabled := sdmTTL > 0 + + // If SDM is enabled, but handlers aren't set up yet. Try again later. + if sdmEnabled && (rmcb == nil || pmsgcb == nil) { + ms.resetAgeChk(0) + ms.mu.Unlock() + return + } + ms.ageChkRun = true + ms.mu.Unlock() + + if maxAge > 0 { + var seq uint64 + for sm, seq, _ = ms.LoadNextMsg(fwcs, true, 0, &smv); sm != nil && sm.ts <= minAge; sm, seq, _ = ms.LoadNextMsg(fwcs, true, seq+1, &smv) { + if len(sm.hdr) > 0 { + if ttl, err := getMessageTTL(sm.hdr); err == nil && ttl < 0 { + // The message has a negative TTL, therefore it must "never expire". + minAge = time.Now().UnixNano() - maxAge + continue + } + } + if sdmEnabled { + if last, ok := ms.shouldProcessSdm(seq, sm.subj); ok { + sdm := last && !isSubjectDeleteMarker(sm.hdr) + ms.handleRemovalOrSdm(seq, sm.subj, sdm, sdmTTL) + } + } else { + ms.mu.Lock() + ms.removeMsg(seq, false) + ms.mu.Unlock() + } + // Recalculate in case we are expiring a bunch. + minAge = time.Now().UnixNano() - maxAge + } + } + + ms.mu.Lock() + defer ms.mu.Unlock() + + // TODO: Not great that we're holding the lock here, but the timed hash wheel isn't thread-safe. + nextTTL := int64(math.MaxInt64) + var rmSeqs []thw.HashWheelEntry + if ms.ttls != nil { + ms.ttls.ExpireTasks(func(seq uint64, ts int64) bool { + rmSeqs = append(rmSeqs, thw.HashWheelEntry{Seq: seq, Expires: ts}) + // We might need to remove messages out of band, those can fail, and we can be shutdown halfway + // through so don't remove from THW just yet. + return false + }) + if maxAge > 0 { + // Only check if we're expiring something in the next MaxAge interval, saves us a bit + // of work if MaxAge will beat us to the next expiry anyway. + nextTTL = ms.ttls.GetNextExpiration(time.Now().Add(time.Duration(maxAge)).UnixNano()) + } else { + nextTTL = ms.ttls.GetNextExpiration(math.MaxInt64) + } + } + + // Remove messages collected by THW. + if !sdmEnabled { + for _, rm := range rmSeqs { + ms.removeMsg(rm.Seq, false) + } + } else { + // THW is unordered, so must sort by sequence and must not be holding the lock. + ms.mu.Unlock() + slices.SortFunc(rmSeqs, func(a, b thw.HashWheelEntry) int { + if a.Seq == b.Seq { + return 0 + } else if a.Seq < b.Seq { + return -1 + } else { + return 1 + } + }) + for _, rm := range rmSeqs { + // Need to grab subject for the specified sequence if for SDM, and check + // if the message hasn't been removed in the meantime. + // We need to grab the message and check if we should process SDM while holding the lock, + // otherwise we can race if a deletion of this message is in progress. + ms.mu.Lock() + sm, _ = ms.loadMsgLocked(rm.Seq, &smv, false) + if sm == nil { + ms.ttls.Remove(rm.Seq, rm.Expires) + ms.mu.Unlock() + continue + } + last, ok := ms.shouldProcessSdmLocked(rm.Seq, sm.subj) + ms.mu.Unlock() + if ok { + sdm := last && !isSubjectDeleteMarker(sm.hdr) + ms.handleRemovalOrSdm(rm.Seq, sm.subj, sdm, sdmTTL) + } + } + ms.mu.Lock() + } + + // Only cancel if no message left, not on potential lookup error that would result in sm == nil. + ms.ageChkRun, ms.ageChkTime = false, 0 + if ms.state.Msgs == 0 && nextTTL == math.MaxInt64 { + ms.cancelAgeChk() + } else { + if sm == nil { + ms.resetAgeChk(0) + } else { + ms.resetAgeChk(sm.ts - minAge) + } + } +} + +func (ms *memStore) shouldProcessSdm(seq uint64, subj string) (bool, bool) { + ms.mu.Lock() + defer ms.mu.Unlock() + return ms.shouldProcessSdmLocked(seq, subj) +} + +// Lock should be held. +func (ms *memStore) shouldProcessSdmLocked(seq uint64, subj string) (bool, bool) { + if ms.sdm == nil { + ms.sdm = newSDMMeta() + } + + if p, ok := ms.sdm.pending[seq]; ok { + // Don't allow more proposals for the same sequence if we already did recently. + if time.Since(time.Unix(0, p.ts)) < 2*time.Second { + return p.last, false + } + // If we're about to use the cached value, and we knew it was last before, + // quickly check that we don't have more remaining messages for the subject now. + // Which means we are not the last anymore and must reset to not remove later data. + if p.last { + msgs := ms.subjectsTotalsLocked(subj)[subj] + numPending := ms.sdm.totals[subj] + if remaining := msgs - numPending; remaining > 0 { + p.last = false + } + } + ms.sdm.pending[seq] = SDMBySeq{p.last, time.Now().UnixNano()} + return p.last, true + } + + msgs := ms.subjectsTotalsLocked(subj)[subj] + if msgs == 0 { + return false, true + } + numPending := ms.sdm.totals[subj] + remaining := msgs - numPending + return ms.sdm.trackPending(seq, subj, remaining == 1), true +} + +func (ms *memStore) handleRemovalOrSdm(seq uint64, subj string, sdm bool, sdmTTL int64) { + if sdm { + var _hdr [128]byte + hdr := fmt.Appendf( + _hdr[:0], + "NATS/1.0\r\n%s: %s\r\n%s: %s\r\n%s: %s\r\n\r\n", + JSMarkerReason, JSMarkerReasonMaxAge, + JSMessageTTL, time.Duration(sdmTTL)*time.Second, + JSMsgRollup, JSMsgRollupSubject, + ) + msg := &inMsg{ + subj: subj, + hdr: hdr, + } + ms.pmsgcb(msg) + } else { + ms.rmcb(seq) + } +} + +// Will run through scheduled messages. +func (ms *memStore) runMsgScheduling() { + // TODO: Not great that we're holding the lock here, but the timed hash wheel and message scheduling isn't thread-safe. + ms.mu.Lock() + defer ms.mu.Unlock() + + // If scheduling is enabled, but handler isn't set up yet. Try again later. + if ms.scheduling == nil || ms.scheduling.paused { + return + } + if ms.pmsgcb == nil { + ms.scheduling.resetTimer() + return + } + ms.scheduling.running = true + + scheduledMsgs := ms.scheduling.getScheduledMessages( + func(seq uint64, smv *StoreMsg) *StoreMsg { + sm, _ := ms.loadMsgLocked(seq, smv, false) + return sm + }, + func(subj string, smv *StoreMsg) *StoreMsg { + sm, _ := ms.loadLastLocked(subj, smv) + return sm + }, + ) + if len(scheduledMsgs) > 0 { + ms.mu.Unlock() + for _, msg := range scheduledMsgs { + ms.pmsgcb(msg) + } + ms.mu.Lock() + } + + ms.scheduling.running, ms.scheduling.deadline = false, 0 + ms.scheduling.resetTimer() +} + +// PurgeEx will remove messages based on subject filters, sequence and number of messages to keep. +// Will return the number of purged messages. +func (ms *memStore) PurgeEx(subject string, sequence, keep uint64) (purged uint64, err error) { + // sequence == 1 means "purge up to but not including 1", a no-op. + if sequence == 1 { + return 0, nil + } + if subject == _EMPTY_ || subject == fwcs { + if keep == 0 && sequence == 0 { + return ms.purge(0) + } + if sequence > 1 { + return ms.compact(sequence) + } + // Make sure to not leave subject if empty. + if subject == _EMPTY_ { + subject = fwcs + } + } + eq := compareFn(subject) + + // FilteredState narrows the search range. + ss, _ := ms.FilteredState(1, subject) + if ss.Msgs == 0 { + return 0, nil + } + // If we have a "keep" designation need to know how many to purge. + var maxp uint64 + if keep > 0 { + if keep >= ss.Msgs { + return 0, nil + } + maxp = ss.Msgs - keep + } + // "Purge up to but not including sequence": sequence == 0 means no + // sequence filter; sequence >= 1 clamps the upper bound to sequence-1 + // (so sequence == 1 purges nothing). + last := ss.Last + if sequence >= 1 { + last = sequence - 1 + } + var bytes, lowSeq uint64 + var lowSubj string + ms.mu.Lock() + for seq := ss.First; seq <= last; seq++ { + if sm, ok := ms.msgs[seq]; ok && eq(sm.subj, subject) { + if subj, sz, ok := ms.removeMsgNoCB(sm.seq, false); ok { + purged++ + bytes += sz + if lowSeq == 0 { + lowSeq, lowSubj = sm.seq, subj + } + if maxp > 0 && purged >= maxp { + break + } + } + } + } + cb := ms.scb + ms.mu.Unlock() + + if cb != nil && purged > 0 { + if purged == 1 { + cb(-1, -int64(bytes), lowSeq, lowSubj) + } else { + cb(-int64(purged), -int64(bytes), 0, _EMPTY_) + } + } + return purged, nil +} + +// Purge will remove all messages from this store. +// Will return the number of purged messages. +func (ms *memStore) Purge() (uint64, error) { + return ms.purge(0) +} + +func (ms *memStore) purge(fseq uint64) (uint64, error) { + ms.mu.Lock() + purged := uint64(len(ms.msgs)) + cb := ms.scb + bytes := int64(ms.state.Bytes) + if fseq == 0 { + fseq = ms.state.LastSeq + 1 + } else if fseq < ms.state.LastSeq { + ms.mu.Unlock() + return 0, fmt.Errorf("partial purges not supported on memory store") + } + ms.state.FirstSeq = fseq + ms.state.LastSeq = fseq - 1 + ms.state.FirstTime = time.Time{} + ms.state.Bytes = 0 + ms.state.Msgs = 0 + if ms.msgs != nil { + ms.msgs = make(map[uint64]*StoreMsg) + } + ms.fss = stree.NewSubjectTree[SimpleState]() + ms.dmap.Empty() + ms.sdm.empty() + ms.mu.Unlock() + + if cb != nil { + cb(-int64(purged), -bytes, 0, _EMPTY_) + } + + return purged, nil +} + +// Compact will remove all messages from this store up to +// but not including the seq parameter. +// Will return the number of purged messages. +func (ms *memStore) Compact(seq uint64) (uint64, error) { + return ms.compact(seq) +} + +func (ms *memStore) compact(seq uint64) (uint64, error) { + if seq == 0 { + return ms.Purge() + } + + var purged, bytes uint64 + + ms.mu.Lock() + // Short-circuit if the store was already compacted past this point. + if ms.state.FirstSeq > seq { + ms.mu.Unlock() + return purged, nil + } + + cb := ms.scb + if seq <= ms.state.LastSeq { + fseq := ms.state.FirstSeq + // Determine new first sequence. + for ; seq <= ms.state.LastSeq; seq++ { + if sm, ok := ms.msgs[seq]; ok { + ms.state.FirstSeq = seq + ms.state.FirstTime = time.Unix(0, sm.ts).UTC() + break + } + } + for seq := seq - 1; seq >= fseq; seq-- { + if sm := ms.msgs[seq]; sm != nil { + bytes += memStoreMsgSize(sm.subj, sm.hdr, sm.msg) + purged++ + ms.removeSeqPerSubject(sm.subj, seq) + // Must delete message after updating per-subject info, to be consistent with file store. + delete(ms.msgs, seq) + } else if !ms.dmap.IsEmpty() { + ms.dmap.Delete(seq) + } + } + if purged > ms.state.Msgs { + purged = ms.state.Msgs + } + ms.state.Msgs -= purged + if ms.state.Msgs == 0 { + ms.state.FirstSeq = ms.state.LastSeq + 1 + ms.state.FirstTime = time.Time{} + } + if bytes > ms.state.Bytes { + bytes = ms.state.Bytes + } + ms.state.Bytes -= bytes + } else { + // We are compacting past the end of our range. Do purge and set sequences correctly + // such that the next message placed will have seq. + purged = uint64(len(ms.msgs)) + bytes = ms.state.Bytes + ms.state.Bytes = 0 + ms.state.Msgs = 0 + ms.state.FirstSeq = seq + ms.state.FirstTime = time.Time{} + ms.state.LastSeq = seq - 1 + // Reset msgs, fss and dmap. + ms.msgs = make(map[uint64]*StoreMsg) + ms.fss = stree.NewSubjectTree[SimpleState]() + ms.dmap.Empty() + ms.sdm.empty() + } + ms.mu.Unlock() + + if cb != nil { + cb(-int64(purged), -int64(bytes), 0, _EMPTY_) + } + + return purged, nil +} + +// Will completely reset our store. +func (ms *memStore) reset() error { + ms.mu.Lock() + var purged, bytes uint64 + cb := ms.scb + if cb != nil { + for _, sm := range ms.msgs { + purged++ + bytes += memStoreMsgSize(sm.subj, sm.hdr, sm.msg) + } + } + + // Reset + ms.state.FirstSeq = 0 + ms.state.FirstTime = time.Time{} + ms.state.LastSeq = 0 + ms.state.LastTime = time.Now().UTC() + // Update msgs and bytes. + ms.state.Msgs = 0 + ms.state.Bytes = 0 + // Reset msgs, fss and dmap. + ms.msgs = make(map[uint64]*StoreMsg) + ms.fss = stree.NewSubjectTree[SimpleState]() + ms.dmap.Empty() + ms.sdm.empty() + + ms.mu.Unlock() + + if cb != nil { + cb(-int64(purged), -int64(bytes), 0, _EMPTY_) + } + + return nil +} + +// Truncate will truncate a stream store up to seq. Sequence needs to be valid. +func (ms *memStore) Truncate(seq uint64) error { + // Check for request to reset. + if seq == 0 { + return ms.reset() + } + + var purged, bytes uint64 + + ms.mu.Lock() + lsm, ok := ms.msgs[seq] + lastTime := ms.state.LastTime + if ok && lsm != nil { + lastTime = time.Unix(0, lsm.ts).UTC() + } + + for i := ms.state.LastSeq; i > seq; i-- { + if sm := ms.msgs[i]; sm != nil { + purged++ + bytes += memStoreMsgSize(sm.subj, sm.hdr, sm.msg) + ms.removeSeqPerSubject(sm.subj, i) + // Must delete message after updating per-subject info, to be consistent with file store. + delete(ms.msgs, i) + } else if !ms.dmap.IsEmpty() { + ms.dmap.Delete(i) + } + } + // Reset last. + ms.state.LastSeq = seq + ms.state.LastTime = lastTime + // Update msgs and bytes. + if purged > ms.state.Msgs { + purged = ms.state.Msgs + } + ms.state.Msgs -= purged + if bytes > ms.state.Bytes { + bytes = ms.state.Bytes + } + ms.state.Bytes -= bytes + + cb := ms.scb + ms.mu.Unlock() + + if cb != nil { + cb(-int64(purged), -int64(bytes), 0, _EMPTY_) + } + + return nil +} + +func (ms *memStore) deleteFirstMsgOrPanic() { + if !ms.deleteFirstMsg() { + panic("jetstream memstore has inconsistent state, can't find first seq msg") + } +} + +func (ms *memStore) deleteFirstMsg() bool { + return ms.removeMsg(ms.state.FirstSeq, false) +} + +// SubjectForSeq will return what the subject is for this sequence if found. +func (ms *memStore) SubjectForSeq(seq uint64) (string, error) { + ms.mu.RLock() + defer ms.mu.RUnlock() + if seq < ms.state.FirstSeq { + return _EMPTY_, ErrStoreMsgNotFound + } + if sm, ok := ms.msgs[seq]; ok { + // Copy the subject, as it's used elsewhere, and we've released the lock in the meantime. + return copyString(sm.subj), nil + } + return _EMPTY_, ErrStoreMsgNotFound +} + +// LoadMsg will lookup the message by sequence number and return it if found. +func (ms *memStore) LoadMsg(seq uint64, smp *StoreMsg) (*StoreMsg, error) { + return ms.loadMsgLocked(seq, smp, true) +} + +// loadMsgLocked will lookup the message by sequence number and return it if found. +func (ms *memStore) loadMsgLocked(seq uint64, smp *StoreMsg, needMSLock bool) (*StoreMsg, error) { + if needMSLock { + ms.mu.RLock() + } + sm, ok := ms.msgs[seq] + last := ms.state.LastSeq + if needMSLock { + ms.mu.RUnlock() + } + + if !ok || sm == nil { + var err = ErrStoreEOF + if seq <= last { + err = ErrStoreMsgNotFound + } + return nil, err + } + + if smp == nil { + smp = new(StoreMsg) + } + sm.copy(smp) + return smp, nil +} + +// LoadLastMsg will return the last message we have that matches a given subject. +// The subject can be a wildcard. +func (ms *memStore) LoadLastMsg(subject string, smp *StoreMsg) (*StoreMsg, error) { + // This needs to be a write lock, as filteredStateLocked can + // mutate the per-subject state. + ms.mu.Lock() + defer ms.mu.Unlock() + return ms.loadLastLocked(subject, smp) +} + +// Lock should be held. +func (ms *memStore) loadLastLocked(subject string, smp *StoreMsg) (*StoreMsg, error) { + var sm *StoreMsg + var ok bool + + if subject == _EMPTY_ || subject == fwcs { + sm, ok = ms.msgs[ms.state.LastSeq] + } else if subjectIsLiteral(subject) { + var ss *SimpleState + if ss, ok = ms.fss.Find(stringToBytes(subject)); ok && ss.Msgs > 0 { + // Check if we need to recalculate. We only care about the last sequence. + if ss.lastNeedsUpdate { + ms.recalculateForSubj(subject, ss) + } + sm, ok = ms.msgs[ss.Last] + } + } else if ss := ms.filteredStateLocked(1, subject, true); ss.Msgs > 0 { + sm, ok = ms.msgs[ss.Last] + } + if !ok || sm == nil { + return nil, ErrStoreMsgNotFound + } + + if smp == nil { + smp = new(StoreMsg) + } + sm.copy(smp) + return smp, nil +} + +// LoadNextMsgMulti will find the next message matching any entry in the sublist. +func (ms *memStore) LoadNextMsgMulti(sl *gsl.SimpleSublist, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error) { + // TODO(dlc) - for now simple linear walk to get started. + ms.mu.RLock() + defer ms.mu.RUnlock() + + if start < ms.state.FirstSeq { + start = ms.state.FirstSeq + } + + // If past the end no results. + if start > ms.state.LastSeq || ms.state.Msgs == 0 { + return nil, ms.state.LastSeq, ErrStoreEOF + } + + // Initial setup. + fseq, lseq := start, ms.state.LastSeq + + for nseq := fseq; nseq <= lseq; nseq++ { + sm, ok := ms.msgs[nseq] + if !ok { + continue + } + if sl.HasInterest(sm.subj) { + if smp == nil { + smp = new(StoreMsg) + } + sm.copy(smp) + return smp, nseq, nil + } + } + return nil, ms.state.LastSeq, ErrStoreEOF +} + +// LoadNextMsg will find the next message matching the filter subject starting at the start sequence. +// The filter subject can be a wildcard. +func (ms *memStore) LoadNextMsg(filter string, wc bool, start uint64, smp *StoreMsg) (*StoreMsg, uint64, error) { + ms.mu.Lock() + defer ms.mu.Unlock() + return ms.loadNextMsgLocked(filter, wc, start, smp) +} + +// Find sequence bounds matching a wildcard filter from ms.fss. +// Returns (first, last, true) if there is at least one matching +// subject at or after start (start <= first <= last). +// Returns (0, 0, false) if the subject does not exist or has no +// messages at or after start. +// Lock should be held. +func (ms *memStore) nextWildcardMatchLocked(filter string, start uint64) (uint64, uint64, bool) { + found := false + first, last := ms.state.LastSeq, uint64(0) + ms.fss.MatchUntil(stringToBytes(filter), func(subj []byte, ss *SimpleState) bool { + ms.recalculateForSubj(string(subj), ss) + + // Skip matches that are below our starting sequence + if start > ss.Last { + return true + } + + // A match was found, adjust the bounds accordingly + found = true + if ss.First < first { + first = ss.First + } + if ss.Last > last { + last = ss.Last + } + + // If first > start, there may be more matches between + // start and first, in which case we keep searching. + // If not, we have a match between start and last, we + // can break out of the search. + // This could be further optimized: if first and start + // are "close", we could just extend the linear search, + // especially if we know that the remaining ms.fss to + // explore is large. + return first > start + }) + if !found { + return 0, 0, false + } + return max(first, start), last, found +} + +// Find sequence bounds matching a literal filter from ms.fss. +// Returns (first, last, true) if there is a matching literal +// subject at or after start (start <= first <= last). +// Returns (0, 0, false) if the subject does not exist or has no +// messages at or after start. +// Lock should be held. +func (ms *memStore) nextLiteralMatchLocked(filter string, start uint64) (uint64, uint64, bool) { + ss, ok := ms.fss.Find(stringToBytes(filter)) + if !ok { + return 0, 0, false + } + ms.recalculateForSubj(filter, ss) + if start > ss.Last { + return 0, 0, false + } + return max(start, ss.First), ss.Last, true +} + +// Returns true if LoadNextMsg should perform a linear scan, +// false if it should use the subject tree to try to reduce +// the search space. +// Lock should be held. +func (ms *memStore) shouldLinearScan(filter string, wc bool, start uint64) bool { + // Skip scan of ms.fss if number of messages in the block are less than + // 1/2 the number of subjects in ms.fss. Or we have a wc and lots of fss entries. + const linearScanMaxFSS = 256 + isAll := filter == fwcs + return isAll || 2*int(ms.state.LastSeq-start) < ms.fss.Size() || (wc && ms.fss.Size() > linearScanMaxFSS) +} + +// Lock should be held. +func (ms *memStore) loadNextMsgLocked(filter string, wc bool, start uint64, smp *StoreMsg) (*StoreMsg, uint64, error) { + if start < ms.state.FirstSeq { + start = ms.state.FirstSeq + } + + // If past the end no results. + if start > ms.state.LastSeq || ms.state.Msgs == 0 { + return nil, ms.state.LastSeq, ErrStoreEOF + } + + if filter == _EMPTY_ { + filter = fwcs + } + isAll := filter == fwcs + + // Initial setup. + fseq, lseq := start, ms.state.LastSeq + + if !ms.shouldLinearScan(filter, wc, start) { + var found bool + if wc { + fseq, lseq, found = ms.nextWildcardMatchLocked(filter, start) + } else { + fseq, lseq, found = ms.nextLiteralMatchLocked(filter, start) + } + if !found { + return nil, ms.state.LastSeq, ErrStoreEOF + } + } + + eq := subjectsEqual + if wc { + eq = matchLiteral + } + + for nseq := fseq; nseq <= lseq; nseq++ { + if sm, ok := ms.msgs[nseq]; ok && (isAll || eq(sm.subj, filter)) { + if smp == nil { + smp = new(StoreMsg) + } + sm.copy(smp) + return smp, nseq, nil + } + } + return nil, ms.state.LastSeq, ErrStoreEOF +} + +// Will load the previous message matching the filter subject, starting at the start sequence and walking backwards. +func (ms *memStore) LoadPrevMsg(filter string, wc bool, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() + + if ms.msgs == nil { + return nil, 0, ErrStoreClosed + } + if ms.state.Msgs == 0 || start < ms.state.FirstSeq { + return nil, ms.state.FirstSeq, ErrStoreEOF + } + if start > ms.state.LastSeq { + start = ms.state.LastSeq + } + + if filter == _EMPTY_ { + filter = fwcs + wc = true + } + isAll := filter == fwcs + eq := subjectsEqual + if wc { + eq = matchLiteral + } + + for seq := start; seq >= ms.state.FirstSeq; seq-- { + if sm, ok := ms.msgs[seq]; ok && (isAll || eq(sm.subj, filter)) { + if smp == nil { + smp = new(StoreMsg) + } + sm.copy(smp) + return smp, seq, nil + } + } + return nil, ms.state.FirstSeq, ErrStoreEOF +} + +// recoverSourcesBackwardScan recovers ms.sources by scanning the stream backwards. +// Lock should be held. +func (ms *memStore) recoverSourcesBackwardScan(sources map[string]*StreamSource) { + var sl *gsl.SimpleSublist + refreshSublist := func() { + sl = gsl.NewSimpleSublist() + for _, src := range sources { + if src.FilterSubject == _EMPTY_ { + sl.Insert(fwcs, struct{}{}) + } else { + sl.Insert(src.FilterSubject, struct{}{}) + } + for _, tr := range src.SubjectTransforms { + if tr.Destination == _EMPTY_ { + sl.Insert(fwcs, struct{}{}) + } else { + sl.Insert(tr.Destination, struct{}{}) + } + } + } + } + refreshSublist() + + update := func(iName string, sseq uint64, ident string) { + // Only consider sources that are configured and not yet found. The backward + // scan visits the highest matching sequence first, so the first hit per source + // is the latest. + if _, ok := sources[iName]; !ok { + return + } + ms.sources[iName] = &StreamSourceState{Seq: sseq, Ident: ident} + delete(sources, iName) + refreshSublist() + } + + var smv StoreMsg + for last := ms.state.LastSeq; ; { + sm, seq, err := ms.loadPrevMsgMultiLocked(sl, last, &smv) + if err != nil { + // Done, including ErrStoreEOF. + return + } + if len(sm.hdr) > 0 { + if ss := sliceHeader(JSStreamSource, sm.hdr); len(ss) > 0 { + streamName, iName, sseq, ident := streamAndSeq(bytesToString(ss)) + if iName == _EMPTY_ { + // Pre-2.10 message header means it's a match for any source using that + // stream name. Such headers carry no identity, so it is left unset. + for _, ssi := range ms.cfg.Sources { + if streamName == ssi.Name || (ssi.External != nil && streamName == ssi.Name+":"+getHash(ssi.External.ApiPrefix)) { + update(ssi.composeIName(), sseq, _EMPTY_) + } + } + } else { + update(iName, sseq, ident) + } + } + } + // Done once every source has been resolved, or we've reached the floor. + if len(sources) == 0 || seq <= ms.state.FirstSeq { + return + } + last = seq - 1 + } +} + +// LoadPrevMsgMulti will find the previous message matching any entry in the sublist. +func (ms *memStore) LoadPrevMsgMulti(sl *gsl.SimpleSublist, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() + return ms.loadPrevMsgMultiLocked(sl, start, smp) +} + +// loadPrevMsgMultiLocked will find the previous message matching any entry in the sublist. +// Lock should be held. +func (ms *memStore) loadPrevMsgMultiLocked(sl *gsl.SimpleSublist, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error) { + // TODO(dlc) - for now simple linear walk to get started. + if start > ms.state.LastSeq { + start = ms.state.LastSeq + } + + // If past the start no results. + if start < ms.state.FirstSeq || ms.state.Msgs == 0 { + return nil, ms.state.FirstSeq, ErrStoreEOF + } + + // Initial setup. + fseq, lseq := start, ms.state.FirstSeq + + for nseq := fseq; nseq >= lseq; nseq-- { + sm, ok := ms.msgs[nseq] + if !ok { + continue + } + if sl.HasInterest(sm.subj) { + if smp == nil { + smp = new(StoreMsg) + } + sm.copy(smp) + return smp, nseq, nil + } + } + return nil, ms.state.FirstSeq, ErrStoreEOF +} + +// RemoveMsg will remove the message from this store. +// Will return the number of bytes removed. +func (ms *memStore) RemoveMsg(seq uint64) (bool, error) { + ms.mu.Lock() + removed := ms.removeMsg(seq, false) + ms.mu.Unlock() + return removed, nil +} + +// EraseMsg will remove the message and rewrite its contents. +func (ms *memStore) EraseMsg(seq uint64) (bool, error) { + ms.mu.Lock() + removed := ms.removeMsg(seq, true) + ms.mu.Unlock() + return removed, nil +} + +// Performs logic to update first sequence number. +// Lock should be held. +func (ms *memStore) updateFirstSeq(seq uint64) { + if seq != ms.state.FirstSeq { + // Interior delete. + return + } + var nsm *StoreMsg + var ok bool + for nseq := ms.state.FirstSeq + 1; nseq <= ms.state.LastSeq; nseq++ { + if nsm, ok = ms.msgs[nseq]; ok { + break + } + } + oldFirst := ms.state.FirstSeq + if nsm != nil { + ms.state.FirstSeq = nsm.seq + ms.state.FirstTime = time.Unix(0, nsm.ts).UTC() + } else { + // Like purge. + ms.state.FirstSeq = ms.state.LastSeq + 1 + ms.state.FirstTime = time.Time{} + } + + if oldFirst == ms.state.FirstSeq-1 { + ms.dmap.Delete(oldFirst) + } else { + for seq := oldFirst; seq < ms.state.FirstSeq; seq++ { + ms.dmap.Delete(seq) + } + } +} + +// Remove a seq from the fss and select new first. +// Lock should be held. +func (ms *memStore) removeSeqPerSubject(subj string, seq uint64) { + ss, ok := ms.fss.Find(stringToBytes(subj)) + if !ok { + return + } + ms.sdm.removeSeqAndSubject(seq, subj) + if ss.Msgs == 1 { + ms.fss.Delete(stringToBytes(subj)) + return + } + ss.Msgs-- + + // Only one left + if ss.Msgs == 1 { + if !ss.lastNeedsUpdate && seq != ss.Last { + ss.First = ss.Last + ss.firstNeedsUpdate = false + return + } + if !ss.firstNeedsUpdate && seq != ss.First { + ss.Last = ss.First + ss.lastNeedsUpdate = false + return + } + } + + // We can lazily calculate the first/last sequence when needed. + ss.firstNeedsUpdate = seq == ss.First || ss.firstNeedsUpdate + ss.lastNeedsUpdate = seq == ss.Last || ss.lastNeedsUpdate +} + +// Will recalculate the first and/or last sequence for this subject. +// Lock should be held. +func (ms *memStore) recalculateForSubj(subj string, ss *SimpleState) { + if ss.firstNeedsUpdate { + tseq := ss.First + 1 + if tseq < ms.state.FirstSeq { + tseq = ms.state.FirstSeq + } + for ; tseq <= ss.Last; tseq++ { + if sm := ms.msgs[tseq]; sm != nil && sm.subj == subj { + ss.First = tseq + ss.firstNeedsUpdate = false + if ss.Msgs == 1 { + ss.Last = tseq + ss.lastNeedsUpdate = false + return + } + break + } + } + } + if ss.lastNeedsUpdate { + tseq := ss.Last - 1 + if tseq > ms.state.LastSeq { + tseq = ms.state.LastSeq + } + for ; tseq >= ss.First; tseq-- { + if sm := ms.msgs[tseq]; sm != nil && sm.subj == subj { + ss.Last = tseq + ss.lastNeedsUpdate = false + if ss.Msgs == 1 { + ss.First = tseq + ss.firstNeedsUpdate = false + } + return + } + } + } +} + +// Removes the message referenced by seq. +// Lock should be held. +func (ms *memStore) removeMsg(seq uint64, secure bool) bool { + subj, size, ok := ms.removeMsgNoCB(seq, secure) + if !ok { + return false + } + if ms.scb != nil { + // We do not want to hold any locks here. + ms.mu.Unlock() + if ms.scb != nil { + ms.scb(-1, -int64(size), seq, subj) + } + ms.mu.Lock() + } + return true +} + +// Removes the message referenced by seq, but without calling the storage callback. +// Returns the removed message's subject and size. +// Lock should be held. +func (ms *memStore) removeMsgNoCB(seq uint64, secure bool) (subj string, size uint64, ok bool) { + sm, ok := ms.msgs[seq] + if !ok { + return _EMPTY_, 0, false + } + + size = memStoreMsgSize(sm.subj, sm.hdr, sm.msg) + + if ms.state.Msgs > 0 { + ms.state.Msgs-- + if size > ms.state.Bytes { + size = ms.state.Bytes + } + ms.state.Bytes -= size + } + ms.dmap.Insert(seq) + ms.updateFirstSeq(seq) + + // Remove any per subject tracking. + ms.removeSeqPerSubject(sm.subj, seq) + if ms.ttls != nil { + if ttl, err := getMessageTTL(sm.hdr); err == nil { + expires := time.Duration(sm.ts) + (time.Second * time.Duration(ttl)) + ms.ttls.Remove(seq, int64(expires)) + } + } + + if secure { + if len(sm.hdr) > 0 { + sm.hdr = make([]byte, len(sm.hdr)) + crand.Read(sm.hdr) + } + if len(sm.msg) > 0 { + sm.msg = make([]byte, len(sm.msg)) + crand.Read(sm.msg) + } + sm.seq, sm.ts = 0, 0 + } + + // Must delete message after updating per-subject info, to be consistent with file store. + delete(ms.msgs, seq) + + return sm.subj, size, true +} + +// Type returns the type of the underlying store. +func (ms *memStore) Type() StorageType { + return MemoryStorage +} + +// FastState will fill in state with only the following. +// Msgs, Bytes, First and Last Sequence and Time and NumDeleted. +func (ms *memStore) FastState(state *StreamState) { + ms.mu.RLock() + state.Msgs = ms.state.Msgs + state.Bytes = ms.state.Bytes + state.FirstSeq = ms.state.FirstSeq + state.FirstTime = ms.state.FirstTime + state.LastSeq = ms.state.LastSeq + state.LastTime = ms.state.LastTime + if state.LastSeq > state.FirstSeq { + state.NumDeleted = int((state.LastSeq - state.FirstSeq + 1) - state.Msgs) + if state.NumDeleted < 0 { + state.NumDeleted = 0 + } + } + state.Consumers = len(ms.consumers) + state.NumSubjects = ms.fss.Size() + ms.mu.RUnlock() +} + +func (ms *memStore) State() StreamState { + ms.mu.Lock() + defer ms.mu.Unlock() + + state := ms.state + state.Consumers = len(ms.consumers) + state.NumSubjects = ms.fss.Size() + state.Deleted = nil + + // Calculate interior delete details. + if numDeleted := int((state.LastSeq - state.FirstSeq + 1) - state.Msgs); numDeleted > 0 { + state.Deleted = make([]uint64, 0, numDeleted) + fseq, lseq := state.FirstSeq, state.LastSeq + ms.dmap.Range(func(seq uint64) bool { + if seq < fseq || seq > lseq { + ms.dmap.Delete(seq) + } else { + state.Deleted = append(state.Deleted, seq) + } + return true + }) + } + if len(state.Deleted) > 0 { + state.NumDeleted = len(state.Deleted) + } + + return state +} + +func (ms *memStore) Utilization() (total, reported uint64, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() + return ms.state.Bytes, ms.state.Bytes, nil +} + +func memStoreMsgSizeRaw(slen, hlen, mlen int) uint64 { + return uint64(slen + hlen + mlen + 16) // 8*2 for seq + age +} + +func memStoreMsgSize(subj string, hdr, msg []byte) uint64 { + return memStoreMsgSizeRaw(len(subj), len(hdr), len(msg)) +} + +// ResetState resets any state that's temporary. For example when changing leaders. +func (ms *memStore) ResetState() { + ms.mu.Lock() + defer ms.mu.Unlock() + if ms.scheduling != nil { + ms.scheduling.clearInflight() + } +} + +func (ms *memStore) Ready() { + ms.mu.Lock() + defer ms.mu.Unlock() + if !ms.recovering { + return + } + ms.recovering = false + var ageDelta int64 + if ms.cfg.MaxAge > 0 && !ms.state.FirstTime.IsZero() { + ageDelta = time.Until(ms.state.FirstTime.Add(ms.cfg.MaxAge)).Nanoseconds() + if ageDelta <= 0 { + ageDelta = 1 + } + } + ms.resetAgeChk(ageDelta) + if ms.scheduling != nil { + ms.scheduling.paused = false + ms.scheduling.resetTimer() + } +} + +// Delete is same as Stop for memory store. +func (ms *memStore) Delete(_ bool) error { + return ms.Stop() +} + +func (ms *memStore) Stop() error { + ms.mu.Lock() + if ms.msgs == nil { + ms.mu.Unlock() + return nil + } + if ms.ageChk != nil { + ms.ageChk.Stop() + ms.ageChk = nil + ms.ageChkTime = 0 + } + ms.msgs = nil + ms.mu.Unlock() + + // These can't come back, so stop is same as Delete. + ms.Purge() + + // Unregister from the access time service. + ats.Unregister() + + return nil +} + +func (ms *memStore) isClosed() bool { + ms.mu.RLock() + defer ms.mu.RUnlock() + return ms.msgs == nil +} + +type consumerMemStore struct { + mu sync.Mutex + ms StreamStore + name string + cfg ConsumerConfig + state ConsumerState + closed bool +} + +func (ms *memStore) ConsumerStore(name string, _ time.Time, cfg *ConsumerConfig) (ConsumerStore, error) { + if ms == nil { + return nil, fmt.Errorf("memstore is nil") + } + if ms.isClosed() { + return nil, ErrStoreClosed + } + if cfg == nil || name == _EMPTY_ { + return nil, fmt.Errorf("bad consumer config") + } + o := &consumerMemStore{ms: ms, name: name, cfg: *cfg} + ms.AddConsumer(o) + return o, nil +} + +func (ms *memStore) AddConsumer(o ConsumerStore) error { + ms.mu.Lock() + ms.consumers = append(ms.consumers, o) + ms.mu.Unlock() + return nil +} + +func (ms *memStore) RemoveConsumer(o ConsumerStore) error { + ms.mu.Lock() + for i, cfs := range ms.consumers { + if o == cfs { + ms.consumers = append(ms.consumers[:i], ms.consumers[i+1:]...) + break + } + } + ms.mu.Unlock() + return nil +} + +func (ms *memStore) Consumers() iter.Seq[ConsumerStore] { + return func(yield func(ConsumerStore) bool) { + ms.mu.RLock() + defer ms.mu.RUnlock() + + for _, v := range ms.consumers { + if !yield(v) { + return + } + } + } +} + +func (ms *memStore) Snapshot(_ time.Duration, _, _ bool) (*SnapshotResult, error) { + return nil, fmt.Errorf("no impl") +} + +// Binary encoded state snapshot, >= v2.10 server. +func (ms *memStore) EncodedStreamState(failed uint64, withSources bool) ([]byte, error) { + ms.mu.RLock() + defer ms.mu.RUnlock() + + withSources = withSources && len(ms.sources) > 0 + version := streamStateVersion + + // Quick calculate num deleted. + numDeleted := int((ms.state.LastSeq - ms.state.FirstSeq + 1) - ms.state.Msgs) + if numDeleted < 0 { + numDeleted = 0 + } + + // Encoded is Msgs, Bytes, FirstSeq, LastSeq, Failed, NumDeleted and optional DeletedBlocks. + // Calculate the exact encoded size up front so the buffer is allocated once. + total := hdrLen + uvarintLen(ms.state.Msgs) + uvarintLen(ms.state.Bytes) + + uvarintLen(ms.state.FirstSeq) + uvarintLen(ms.state.LastSeq) + + uvarintLen(failed) + uvarintLen(uint64(numDeleted)) + + var numSources uint64 + var sourcesLen int + if withSources { + for name, ss := range ms.sources { + if ss.Seq == 0 { + continue + } + numSources++ + // Length prefix + source bytes + seq varint + + // length prefix + identity bytes. + sourcesLen += uvarintLen(uint64(len(name))) + len(name) + uvarintLen(ss.Seq) + + uvarintLen(uint64(len(ss.Ident))) + len(ss.Ident) + } + // Nothing observed yet, then there is nothing to carry. + if withSources = numSources > 0; withSources { + total += uvarintLen(numSources) + sourcesLen + version = streamStateVersionSources + } + } + if numDeleted > 0 { + total += ms.dmap.EncodeLen() + } + + b := make([]byte, 0, total) + b = append(b, streamStateMagic, version) + b = binary.AppendUvarint(b, ms.state.Msgs) + b = binary.AppendUvarint(b, ms.state.Bytes) + b = binary.AppendUvarint(b, ms.state.FirstSeq) + b = binary.AppendUvarint(b, ms.state.LastSeq) + b = binary.AppendUvarint(b, failed) + if withSources { + b = binary.AppendUvarint(b, numSources) + for name, ss := range ms.sources { + if ss.Seq == 0 { + continue + } + b = binary.AppendUvarint(b, uint64(len(name))) + b = append(b, name...) + b = binary.AppendUvarint(b, ss.Seq) + b = binary.AppendUvarint(b, uint64(len(ss.Ident))) + b = append(b, ss.Ident...) + } + } + b = binary.AppendUvarint(b, uint64(numDeleted)) + + if numDeleted > 0 { + enc := ms.dmap.Encode(b[len(b):]) + if n := len(b) + len(enc); n <= cap(b) { + b = b[:n] + } else { + // Fallback if the buffer didn't have spare capacity. + b = append(b, enc...) + } + } + + if len(b) != total { + assert.Unreachable("Memstore EncodedStreamState size accounting mismatch", map[string]any{ + "name": ms.cfg.Name, + "total": total, + "length": len(b), + }) + } + return b, nil +} + +// SyncDeleted will make sure this stream has same deleted state as dbs. +func (ms *memStore) SyncDeleted(dbs DeleteBlocks) error { + if len(dbs) == 0 { + return nil + } + + ms.mu.Lock() + defer ms.mu.Unlock() + + // For now we share one dmap, so if we have one entry here check if states are the same. + // Note this will work for any DeleteBlock type, but we expect this to be a dmap too. + if len(dbs) == 1 { + min, max, num := ms.dmap.State() + if pmin, pmax, pnum := dbs[0].State(); pmin == min && pmax == max && pnum == num { + return nil + } + } + lseq := ms.state.LastSeq + for _, db := range dbs { + // Skip if beyond our current state. + if first, _, _ := db.State(); first > lseq { + continue + } + db.Range(func(seq uint64) bool { + ms.removeMsg(seq, false) + return true + }) + } + return nil +} + +func (o *consumerMemStore) Update(state *ConsumerState) error { + // Sanity checks. + if state.AckFloor.Consumer > state.Delivered.Consumer { + return fmt.Errorf("bad ack floor for consumer") + } + if state.AckFloor.Stream > state.Delivered.Stream { + return fmt.Errorf("bad ack floor for stream") + } + + // Copy to our state. + var pending map[uint64]*Pending + var redelivered map[uint64]uint64 + if len(state.Pending) > 0 { + pending = make(map[uint64]*Pending, len(state.Pending)) + for seq, p := range state.Pending { + pending[seq] = &Pending{p.Sequence, p.Timestamp} + if seq <= state.AckFloor.Stream || seq > state.Delivered.Stream { + return fmt.Errorf("bad pending entry, sequence [%d] out of range", seq) + } + } + } + if len(state.Redelivered) > 0 { + redelivered = make(map[uint64]uint64, len(state.Redelivered)) + for seq, dc := range state.Redelivered { + redelivered[seq] = dc + } + } + + // Replace our state. + o.mu.Lock() + defer o.mu.Unlock() + + // Check to see if this is an outdated update. + if state.Delivered.Consumer < o.state.Delivered.Consumer || state.AckFloor.Stream < o.state.AckFloor.Stream { + return ErrStoreOldUpdate + } + + o.state.Delivered = state.Delivered + o.state.AckFloor = state.AckFloor + o.state.Pending = pending + o.state.Redelivered = redelivered + + return nil +} + +func (o *consumerMemStore) ForceUpdate(state *ConsumerState) error { + // Sanity checks. + if state.AckFloor.Consumer > state.Delivered.Consumer { + return fmt.Errorf("bad ack floor for consumer") + } + if state.AckFloor.Stream > state.Delivered.Stream { + return fmt.Errorf("bad ack floor for stream") + } + + // Copy to our state. + var pending map[uint64]*Pending + var redelivered map[uint64]uint64 + if len(state.Pending) > 0 { + pending = make(map[uint64]*Pending, len(state.Pending)) + for seq, p := range state.Pending { + pending[seq] = &Pending{p.Sequence, p.Timestamp} + if seq <= state.AckFloor.Stream || seq > state.Delivered.Stream { + return fmt.Errorf("bad pending entry, sequence [%d] out of range", seq) + } + } + } + if len(state.Redelivered) > 0 { + redelivered = make(map[uint64]uint64, len(state.Redelivered)) + for seq, dc := range state.Redelivered { + redelivered[seq] = dc + } + } + + // Replace our state. + o.mu.Lock() + defer o.mu.Unlock() + + o.state.Delivered = state.Delivered + o.state.AckFloor = state.AckFloor + o.state.Pending = pending + o.state.Redelivered = redelivered + + return nil +} + +// SetStarting sets our starting stream sequence. +func (o *consumerMemStore) SetStarting(sseq uint64) error { + o.mu.Lock() + o.state.Delivered.Stream = sseq + o.state.AckFloor.Stream = sseq + o.mu.Unlock() + return nil +} + +// UpdateStarting updates our starting stream sequence. +func (o *consumerMemStore) UpdateStarting(sseq uint64) { + o.mu.Lock() + defer o.mu.Unlock() + + if sseq > o.state.Delivered.Stream { + o.state.Delivered.Stream = sseq + // For AckNone just update delivered and ackfloor at the same time. + if o.cfg.AckPolicy == AckNone { + o.state.AckFloor.Stream = sseq + } + } +} + +// Reset all values in the store, and reset the starting sequence. +func (o *consumerMemStore) Reset(sseq uint64) error { + o.mu.Lock() + o.state = ConsumerState{} + o.mu.Unlock() + return o.SetStarting(sseq) +} + +// HasState returns if this store has a recorded state. +func (o *consumerMemStore) HasState() bool { + o.mu.Lock() + defer o.mu.Unlock() + // We have a running state. + return o.state.Delivered.Consumer != 0 || o.state.Delivered.Stream != 0 +} + +func (o *consumerMemStore) UpdateDelivered(dseq, sseq, dc uint64, ts int64) error { + o.mu.Lock() + defer o.mu.Unlock() + + if dc != 1 && o.cfg.AckPolicy == AckNone { + return ErrNoAckPolicy + } + + // On restarts the old leader may get a replay from the raft logs that are old. + if dseq <= o.state.AckFloor.Consumer { + return nil + } + + // See if we expect an ack for this. + if o.cfg.AckPolicy != AckNone { + // Need to create pending records here. + if o.state.Pending == nil { + o.state.Pending = make(map[uint64]*Pending) + } + var p *Pending + // Check for an update to a message already delivered. + if sseq <= o.state.Delivered.Stream { + if p = o.state.Pending[sseq]; p != nil { + // Do not update p.Sequence, that should be the original delivery sequence. + p.Timestamp = ts + } + } else { + // Add to pending. + o.state.Pending[sseq] = &Pending{dseq, ts} + } + // Update delivered as needed. + if dseq > o.state.Delivered.Consumer { + o.state.Delivered.Consumer = dseq + } + if sseq > o.state.Delivered.Stream { + o.state.Delivered.Stream = sseq + } + + if dc > 1 { + if maxdc := uint64(o.cfg.MaxDeliver); maxdc > 0 && dc > maxdc { + // Make sure to remove from pending. + delete(o.state.Pending, sseq) + } + if o.state.Redelivered == nil { + o.state.Redelivered = make(map[uint64]uint64) + } + // Only update if greater than what we already have. + if o.state.Redelivered[sseq] < dc-1 { + o.state.Redelivered[sseq] = dc - 1 + } + } + } else { + // For AckNone just update delivered and ackfloor at the same time. + if dseq > o.state.Delivered.Consumer { + o.state.Delivered.Consumer = dseq + o.state.AckFloor.Consumer = dseq + } + if sseq > o.state.Delivered.Stream { + o.state.Delivered.Stream = sseq + o.state.AckFloor.Stream = sseq + } + } + + return nil +} + +func (o *consumerMemStore) UpdateAcks(dseq, sseq uint64) error { + o.mu.Lock() + defer o.mu.Unlock() + + if o.cfg.AckPolicy == AckNone { + return ErrNoAckPolicy + } + + // We do this regardless. + delete(o.state.Redelivered, sseq) + + // On restarts the old leader may get a replay from the raft logs that are old. + if dseq <= o.state.AckFloor.Consumer { + return nil + } + + if len(o.state.Pending) == 0 || o.state.Pending[sseq] == nil { + return ErrStoreMsgNotFound + } + + // Check for AckAll here (or AckFlowControl which functions like AckAll). + if o.cfg.AckPolicy == AckAll || o.cfg.AckPolicy == AckFlowControl { + sgap := sseq - o.state.AckFloor.Stream + o.state.AckFloor.Consumer = dseq + o.state.AckFloor.Stream = sseq + if sgap > uint64(len(o.state.Pending)) { + for seq := range o.state.Pending { + if seq <= sseq { + delete(o.state.Pending, seq) + delete(o.state.Redelivered, seq) + } + } + } else { + for seq := sseq; seq > sseq-sgap && len(o.state.Pending) > 0; seq-- { + delete(o.state.Pending, seq) + delete(o.state.Redelivered, seq) + } + } + return nil + } + + // AckExplicit + + // First delete from our pending state. + if p, ok := o.state.Pending[sseq]; ok { + delete(o.state.Pending, sseq) + if dseq > p.Sequence && p.Sequence > 0 { + dseq = p.Sequence // Use the original. + } + } + + if len(o.state.Pending) == 0 { + o.state.AckFloor.Consumer = o.state.Delivered.Consumer + o.state.AckFloor.Stream = o.state.Delivered.Stream + } else if dseq == o.state.AckFloor.Consumer+1 { + o.state.AckFloor.Consumer = dseq + o.state.AckFloor.Stream = sseq + + if o.state.Delivered.Consumer > dseq { + for ss := sseq + 1; ss <= o.state.Delivered.Stream; ss++ { + if p, ok := o.state.Pending[ss]; ok { + if p.Sequence > 0 { + o.state.AckFloor.Consumer = p.Sequence - 1 + o.state.AckFloor.Stream = ss - 1 + } + break + } + } + } + } + + return nil +} + +func (o *consumerMemStore) RemoveRedeliveredBelow(seq uint64) { + if seq == 0 { + return + } + o.mu.Lock() + defer o.mu.Unlock() + for s := range o.state.Redelivered { + if s < seq { + delete(o.state.Redelivered, s) + } + } +} + +func (o *consumerMemStore) GetConfig() *ConsumerConfig { + o.mu.Lock() + defer o.mu.Unlock() + clone := o.cfg.clone() + clone.Name = o.name + return clone +} + +func (o *consumerMemStore) UpdateConfig(cfg *ConsumerConfig) error { + o.mu.Lock() + defer o.mu.Unlock() + + // This is mostly unchecked here. We are assuming the upper layers have done sanity checking. + o.cfg = *cfg + return nil +} + +func (o *consumerMemStore) Stop() error { + o.mu.Lock() + o.closed = true + ms := o.ms + o.mu.Unlock() + ms.RemoveConsumer(o) + return nil +} + +func (o *consumerMemStore) Delete() error { + return o.Stop() +} + +func (o *consumerMemStore) StreamDelete() error { + return o.Stop() +} + +func (o *consumerMemStore) State() (*ConsumerState, error) { + return o.stateWithCopy(true) +} + +// This will not copy pending or redelivered, so should only be done under the +// consumer owner's lock. +func (o *consumerMemStore) BorrowState() (*ConsumerState, error) { + return o.stateWithCopy(false) +} + +func (o *consumerMemStore) stateWithCopy(doCopy bool) (*ConsumerState, error) { + o.mu.Lock() + defer o.mu.Unlock() + + if o.closed { + return nil, ErrStoreClosed + } + + state := &ConsumerState{} + + state.Delivered = o.state.Delivered + state.AckFloor = o.state.AckFloor + if len(o.state.Pending) > 0 { + if doCopy { + state.Pending = o.copyPending() + } else { + state.Pending = o.state.Pending + } + } + if len(o.state.Redelivered) > 0 { + if doCopy { + state.Redelivered = o.copyRedelivered() + } else { + state.Redelivered = o.state.Redelivered + } + } + return state, nil +} + +// EncodedState for this consumer store. +func (o *consumerMemStore) EncodedState() ([]byte, error) { + o.mu.Lock() + defer o.mu.Unlock() + + if o.closed { + return nil, ErrStoreClosed + } + + return encodeConsumerState(&o.state), nil +} + +func (o *consumerMemStore) copyPending() map[uint64]*Pending { + pending := make(map[uint64]*Pending, len(o.state.Pending)) + for seq, p := range o.state.Pending { + pending[seq] = &Pending{p.Sequence, p.Timestamp} + } + return pending +} + +func (o *consumerMemStore) copyRedelivered() map[uint64]uint64 { + redelivered := make(map[uint64]uint64, len(o.state.Redelivered)) + for seq, dc := range o.state.Redelivered { + redelivered[seq] = dc + } + return redelivered +} + +// Type returns the type of the underlying store. +func (o *consumerMemStore) Type() StorageType { return MemoryStorage } diff --git a/server/memstore_test.go b/server/memstore_test.go new file mode 100644 index 0000000..383964c --- /dev/null +++ b/server/memstore_test.go @@ -0,0 +1,1837 @@ +// Copyright 2019-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !skip_store_tests + +package server + +import ( + "bytes" + "errors" + "fmt" + "math/rand/v2" + "reflect" + "slices" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server/gsl" + "github.com/nats-io/nuid" +) + +func TestMemStoreBasics(t *testing.T) { + ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) + require_NoError(t, err) + defer ms.Stop() + + subj, msg := "foo", []byte("Hello World") + now := time.Now().UnixNano() + if seq, ts, err := ms.StoreMsg(subj, nil, msg, 0); err != nil { + t.Fatalf("Error storing msg: %v", err) + } else if seq != 1 { + t.Fatalf("Expected sequence to be 1, got %d", seq) + } else if ts < now || ts > now+int64(time.Millisecond) { + t.Fatalf("Expected timestamp to be current, got %v", ts-now) + } + + state := ms.State() + if state.Msgs != 1 { + t.Fatalf("Expected 1 msg, got %d", state.Msgs) + } + expectedSize := memStoreMsgSize(subj, nil, msg) + if state.Bytes != expectedSize { + t.Fatalf("Expected %d bytes, got %d", expectedSize, state.Bytes) + } + sm, err := ms.LoadMsg(1, nil) + if err != nil { + t.Fatalf("Unexpected error looking up msg: %v", err) + } + if sm.subj != subj { + t.Fatalf("Subjects don't match, original %q vs %q", subj, sm.subj) + } + if !bytes.Equal(sm.msg, msg) { + t.Fatalf("Msgs don't match, original %q vs %q", msg, sm.msg) + } +} + +func TestMemStoreMsgLimit(t *testing.T) { + ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage, MaxMsgs: 10}) + require_NoError(t, err) + defer ms.Stop() + + subj, msg := "foo", []byte("Hello World") + for i := 0; i < 10; i++ { + ms.StoreMsg(subj, nil, msg, 0) + } + state := ms.State() + if state.Msgs != 10 { + t.Fatalf("Expected %d msgs, got %d", 10, state.Msgs) + } + if _, _, err := ms.StoreMsg(subj, nil, msg, 0); err != nil { + t.Fatalf("Error storing msg: %v", err) + } + state = ms.State() + if state.Msgs != 10 { + t.Fatalf("Expected %d msgs, got %d", 10, state.Msgs) + } + if state.LastSeq != 11 { + t.Fatalf("Expected the last sequence to be 11 now, but got %d", state.LastSeq) + } + if state.FirstSeq != 2 { + t.Fatalf("Expected the first sequence to be 2 now, but got %d", state.FirstSeq) + } + // Make sure we can not lookup seq 1. + if _, err := ms.LoadMsg(1, nil); err == nil { + t.Fatalf("Expected error looking up seq 1 but got none") + } +} + +func TestMemStoreBytesLimit(t *testing.T) { + subj, msg := "foo", make([]byte, 512) + storedMsgSize := memStoreMsgSize(subj, nil, msg) + + toStore := uint64(1024) + maxBytes := storedMsgSize * toStore + + ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage, MaxBytes: int64(maxBytes)}) + require_NoError(t, err) + defer ms.Stop() + + for i := uint64(0); i < toStore; i++ { + ms.StoreMsg(subj, nil, msg, 0) + } + state := ms.State() + if state.Msgs != toStore { + t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) + } + if state.Bytes != storedMsgSize*toStore { + t.Fatalf("Expected bytes to be %d, got %d", storedMsgSize*toStore, state.Bytes) + } + + // Now send 10 more and check that bytes limit enforced. + for i := 0; i < 10; i++ { + if _, _, err := ms.StoreMsg(subj, nil, msg, 0); err != nil { + t.Fatalf("Error storing msg: %v", err) + } + } + state = ms.State() + if state.Msgs != toStore { + t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) + } + if state.Bytes != storedMsgSize*toStore { + t.Fatalf("Expected bytes to be %d, got %d", storedMsgSize*toStore, state.Bytes) + } + if state.FirstSeq != 11 { + t.Fatalf("Expected first sequence to be 11, got %d", state.FirstSeq) + } + if state.LastSeq != toStore+10 { + t.Fatalf("Expected last sequence to be %d, got %d", toStore+10, state.LastSeq) + } +} + +// https://github.com/nats-io/nats-server/issues/4771 +func TestMemStoreBytesLimitWithDiscardNew(t *testing.T) { + subj, msg := "tiny", make([]byte, 7) + storedMsgSize := memStoreMsgSize(subj, nil, msg) + + toStore := uint64(3) + maxBytes := 100 + + ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage, MaxBytes: int64(maxBytes), Discard: DiscardNew}) + require_NoError(t, err) + defer ms.Stop() + + // Now send 10 messages and check that bytes limit enforced. + for i := 0; i < 10; i++ { + _, _, err := ms.StoreMsg(subj, nil, msg, 0) + if i < int(toStore) { + if err != nil { + t.Fatalf("Error storing msg: %v", err) + } + } else if !errors.Is(err, ErrMaxBytes) { + t.Fatalf("Storing msg should result in: %v", ErrMaxBytes) + } + } + state := ms.State() + if state.Msgs != toStore { + t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) + } + if state.Bytes != storedMsgSize*toStore { + t.Fatalf("Expected bytes to be %d, got %d", storedMsgSize*toStore, state.Bytes) + } +} + +func TestMemStoreAgeLimit(t *testing.T) { + maxAge := 10 * time.Millisecond + ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage, MaxAge: maxAge}) + require_NoError(t, err) + defer ms.Stop() + + // Store some messages. Does not really matter how many. + subj, msg := "foo", []byte("Hello World") + toStore := 100 + for i := 0; i < toStore; i++ { + ms.StoreMsg(subj, nil, msg, 0) + } + state := ms.State() + if state.Msgs != uint64(toStore) { + t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) + } + checkExpired := func(t *testing.T) { + t.Helper() + checkFor(t, time.Second, maxAge, func() error { + state = ms.State() + if state.Msgs != 0 { + return fmt.Errorf("Expected no msgs, got %d", state.Msgs) + } + if state.Bytes != 0 { + return fmt.Errorf("Expected no bytes, got %d", state.Bytes) + } + return nil + }) + } + // Let them expire + checkExpired(t) + // Now add some more and make sure that timer will fire again. + for i := 0; i < toStore; i++ { + ms.StoreMsg(subj, nil, msg, 0) + } + state = ms.State() + if state.Msgs != uint64(toStore) { + t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) + } + checkExpired(t) +} + +func TestMemStoreTimeStamps(t *testing.T) { + ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) + require_NoError(t, err) + defer ms.Stop() + + last := time.Now().UnixNano() + subj, msg := "foo", []byte("Hello World") + for i := 0; i < 10; i++ { + time.Sleep(5 * time.Microsecond) + ms.StoreMsg(subj, nil, msg, 0) + } + var smv StoreMsg + for seq := uint64(1); seq <= 10; seq++ { + sm, err := ms.LoadMsg(seq, &smv) + if err != nil { + t.Fatalf("Unexpected error looking up msg: %v", err) + } + // These should be different + if sm.ts <= last { + t.Fatalf("Expected different timestamps, got %v", sm.ts) + } + last = sm.ts + } +} + +func TestMemStorePurge(t *testing.T) { + ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) + require_NoError(t, err) + defer ms.Stop() + + subj, msg := "foo", []byte("Hello World") + for i := 0; i < 10; i++ { + ms.StoreMsg(subj, nil, msg, 0) + } + if state := ms.State(); state.Msgs != 10 { + t.Fatalf("Expected 10 msgs, got %d", state.Msgs) + } + ms.Purge() + if state := ms.State(); state.Msgs != 0 { + t.Fatalf("Expected no msgs, got %d", state.Msgs) + } +} + +func TestMemStoreCompact(t *testing.T) { + ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) + require_NoError(t, err) + defer ms.Stop() + + subj, msg := "foo", []byte("Hello World") + for i := 0; i < 10; i++ { + ms.StoreMsg(subj, nil, msg, 0) + } + if state := ms.State(); state.Msgs != 10 { + t.Fatalf("Expected 10 msgs, got %d", state.Msgs) + } + n, err := ms.Compact(6) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if n != 5 { + t.Fatalf("Expected to have purged 5 msgs, got %d", n) + } + state := ms.State() + if state.Msgs != 5 { + t.Fatalf("Expected 5 msgs, got %d", state.Msgs) + } + if state.FirstSeq != 6 { + t.Fatalf("Expected first seq of 6, got %d", state.FirstSeq) + } + // Now test that compact will also reset first if seq > last + n, err = ms.Compact(100) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if n != 5 { + t.Fatalf("Expected to have purged 5 msgs, got %d", n) + } + if state = ms.State(); state.FirstSeq != 100 { + t.Fatalf("Expected first seq of 100, got %d", state.FirstSeq) + } +} + +func TestMemStoreEraseMsg(t *testing.T) { + ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) + require_NoError(t, err) + defer ms.Stop() + + subj, msg := "foo", []byte("Hello World") + ms.StoreMsg(subj, nil, msg, 0) + sm, err := ms.LoadMsg(1, nil) + if err != nil { + t.Fatalf("Unexpected error looking up msg: %v", err) + } + if !bytes.Equal(msg, sm.msg) { + t.Fatalf("Expected same msg, got %q vs %q", sm.msg, msg) + } + if removed, _ := ms.EraseMsg(1); !removed { + t.Fatalf("Expected erase msg to return success") + } +} + +func TestMemStoreMsgHeaders(t *testing.T) { + ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) + require_NoError(t, err) + defer ms.Stop() + + subj, hdr, msg := "foo", []byte("name:derek"), []byte("Hello World") + if sz := int(memStoreMsgSize(subj, hdr, msg)); sz != (len(subj) + len(hdr) + len(msg) + 16) { + t.Fatalf("Wrong size for stored msg with header") + } + ms.StoreMsg(subj, hdr, msg, 0) + sm, err := ms.LoadMsg(1, nil) + if err != nil { + t.Fatalf("Unexpected error looking up msg: %v", err) + } + if !bytes.Equal(msg, sm.msg) { + t.Fatalf("Expected same msg, got %q vs %q", sm.msg, msg) + } + if !bytes.Equal(hdr, sm.hdr) { + t.Fatalf("Expected same hdr, got %q vs %q", sm.hdr, hdr) + } + if removed, _ := ms.EraseMsg(1); !removed { + t.Fatalf("Expected erase msg to return success") + } +} + +func TestMemStoreStreamStateDeleted(t *testing.T) { + ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) + require_NoError(t, err) + defer ms.Stop() + + subj, toStore := "foo", uint64(10) + for i := uint64(1); i <= toStore; i++ { + msg := []byte(fmt.Sprintf("[%08d] Hello World!", i)) + if _, _, err := ms.StoreMsg(subj, nil, msg, 0); err != nil { + t.Fatalf("Error storing msg: %v", err) + } + } + state := ms.State() + if len(state.Deleted) != 0 { + t.Fatalf("Expected deleted to be empty") + } + // Now remove some interior messages. + var expected []uint64 + for seq := uint64(2); seq < toStore; seq += 2 { + ms.RemoveMsg(seq) + expected = append(expected, seq) + } + state = ms.State() + if !reflect.DeepEqual(state.Deleted, expected) { + t.Fatalf("Expected deleted to be %+v, got %+v\n", expected, state.Deleted) + } + // Now fill the gap by deleting 1 and 3 + ms.RemoveMsg(1) + ms.RemoveMsg(3) + expected = expected[2:] + state = ms.State() + if !reflect.DeepEqual(state.Deleted, expected) { + t.Fatalf("Expected deleted to be %+v, got %+v\n", expected, state.Deleted) + } + if state.FirstSeq != 5 { + t.Fatalf("Expected first seq to be 5, got %d", state.FirstSeq) + } + ms.Purge() + if state = ms.State(); len(state.Deleted) != 0 { + t.Fatalf("Expected no deleted after purge, got %+v\n", state.Deleted) + } +} + +func TestMemStoreStreamTruncate(t *testing.T) { + ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) + require_NoError(t, err) + defer ms.Stop() + + tseq := uint64(50) + + subj, toStore := "foo", uint64(100) + for i := uint64(1); i < tseq; i++ { + _, _, err := ms.StoreMsg(subj, nil, []byte("ok"), 0) + require_NoError(t, err) + } + subj = "bar" + for i := tseq; i <= toStore; i++ { + _, _, err := ms.StoreMsg(subj, nil, []byte("ok"), 0) + require_NoError(t, err) + } + + if state := ms.State(); state.Msgs != toStore { + t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) + } + + if err := ms.Truncate(tseq); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if state := ms.State(); state.Msgs != tseq { + t.Fatalf("Expected %d msgs, got %d", tseq, state.Msgs) + } + + // Now make sure we report properly if we have some deleted interior messages. + ms.RemoveMsg(10) + ms.RemoveMsg(20) + ms.RemoveMsg(30) + ms.RemoveMsg(40) + + tseq = uint64(25) + if err := ms.Truncate(tseq); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + state := ms.State() + if state.Msgs != tseq-2 { + t.Fatalf("Expected %d msgs, got %d", tseq-2, state.Msgs) + } + if state.NumSubjects != 1 { + t.Fatalf("Expected only 1 subject, got %d", state.NumSubjects) + } + expected := []uint64{10, 20} + if !reflect.DeepEqual(state.Deleted, expected) { + t.Fatalf("Expected deleted to be %+v, got %+v\n", expected, state.Deleted) + } +} + +func TestMemStorePurgeExWithSubject(t *testing.T) { + ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) + require_NoError(t, err) + defer ms.Stop() + + for i := 0; i < 100; i++ { + _, _, err = ms.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + } + + // This should purge all. + ms.PurgeEx("foo", 0, 0) + require_True(t, ms.State().Msgs == 0) +} + +func TestMemStoreUpdateMaxMsgsPerSubject(t *testing.T) { + cfg := &StreamConfig{ + Name: "TEST", + Storage: MemoryStorage, + Subjects: []string{"foo"}, + MaxMsgsPer: 10, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + // Make sure this is honored on an update. + cfg.MaxMsgsPer = 50 + err = ms.UpdateConfig(cfg) + require_NoError(t, err) + + numStored := 22 + for i := 0; i < numStored; i++ { + _, _, err = ms.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + } + + ss := ms.SubjectsState("foo")["foo"] + if ss.Msgs != uint64(numStored) { + t.Fatalf("Expected to have %d stored, got %d", numStored, ss.Msgs) + } + + // Now make sure we trunk if setting to lower value. + cfg.MaxMsgsPer = 10 + err = ms.UpdateConfig(cfg) + require_NoError(t, err) + + ss = ms.SubjectsState("foo")["foo"] + if ss.Msgs != 10 { + t.Fatalf("Expected to have %d stored, got %d", 10, ss.Msgs) + } +} + +func TestMemStoreStreamTruncateReset(t *testing.T) { + cfg := &StreamConfig{ + Name: "TEST", + Storage: MemoryStorage, + Subjects: []string{"foo"}, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + subj, msg := "foo", []byte("Hello World") + for i := 0; i < 1000; i++ { + _, _, err := ms.StoreMsg(subj, nil, msg, 0) + require_NoError(t, err) + } + + // Reset everything + require_NoError(t, ms.Truncate(0)) + + state := ms.State() + require_True(t, state.Msgs == 0) + require_True(t, state.Bytes == 0) + require_True(t, state.FirstSeq == 0) + require_True(t, state.LastSeq == 0) + require_True(t, state.NumSubjects == 0) + require_True(t, state.NumDeleted == 0) + + for i := 0; i < 1000; i++ { + _, _, err := ms.StoreMsg(subj, nil, msg, 0) + require_NoError(t, err) + } + + state = ms.State() + require_True(t, state.Msgs == 1000) + require_True(t, state.Bytes == 30000) + require_True(t, state.FirstSeq == 1) + require_True(t, state.LastSeq == 1000) + require_True(t, state.NumSubjects == 1) + require_True(t, state.NumDeleted == 0) +} + +func TestMemStoreStreamCompactMultiBlockSubjectInfo(t *testing.T) { + cfg := &StreamConfig{ + Name: "TEST", + Storage: MemoryStorage, + Subjects: []string{"foo.*"}, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + for i := 0; i < 1000; i++ { + subj := fmt.Sprintf("foo.%d", i) + _, _, err := ms.StoreMsg(subj, nil, []byte("Hello World"), 0) + require_NoError(t, err) + } + + // Compact such that we know we throw blocks away from the beginning. + deleted, err := ms.Compact(501) + require_NoError(t, err) + require_True(t, deleted == 500) + + // Make sure we adjusted for subjects etc. + state := ms.State() + require_True(t, state.NumSubjects == 500) +} + +func TestMemStoreSubjectsTotals(t *testing.T) { + cfg := &StreamConfig{ + Name: "TEST", + Storage: MemoryStorage, + Subjects: []string{"*.*"}, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + fmap := make(map[int]int) + bmap := make(map[int]int) + + var m map[int]int + var ft string + + for i := 0; i < 10_000; i++ { + // Flip coin for prefix + if rand.IntN(2) == 0 { + ft, m = "foo", fmap + } else { + ft, m = "bar", bmap + } + dt := rand.IntN(100) + subj := fmt.Sprintf("%s.%d", ft, dt) + m[dt]++ + + _, _, err := ms.StoreMsg(subj, nil, []byte("Hello World"), 0) + require_NoError(t, err) + } + + // Now test SubjectsTotal + for dt, total := range fmap { + subj := fmt.Sprintf("foo.%d", dt) + m := ms.SubjectsTotals(subj) + if m[subj] != uint64(total) { + t.Fatalf("Expected %q to have %d total, got %d", subj, total, m[subj]) + } + } + + // Check fmap. + if st := ms.SubjectsTotals("foo.*"); len(st) != len(fmap) { + t.Fatalf("Expected %d subjects for %q, got %d", len(fmap), "foo.*", len(st)) + } else { + expected := 0 + for _, n := range fmap { + expected += n + } + received := uint64(0) + for _, n := range st { + received += n + } + if received != uint64(expected) { + t.Fatalf("Expected %d total but got %d", expected, received) + } + } + + // Check bmap. + if st := ms.SubjectsTotals("bar.*"); len(st) != len(bmap) { + t.Fatalf("Expected %d subjects for %q, got %d", len(bmap), "bar.*", len(st)) + } else { + expected := 0 + for _, n := range bmap { + expected += n + } + received := uint64(0) + for _, n := range st { + received += n + } + if received != uint64(expected) { + t.Fatalf("Expected %d total but got %d", expected, received) + } + } + + // All with pwc match. + if st, expected := ms.SubjectsTotals("*.*"), len(bmap)+len(fmap); len(st) != expected { + t.Fatalf("Expected %d subjects for %q, got %d", expected, "*.*", len(st)) + } +} + +func TestMemStoreNumPending(t *testing.T) { + cfg := &StreamConfig{ + Name: "TEST", + Storage: MemoryStorage, + Subjects: []string{"*.*.*.*"}, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + tokens := []string{"foo", "bar", "baz"} + genSubj := func() string { + return fmt.Sprintf("%s.%s.%s.%s", + tokens[rand.IntN(len(tokens))], + tokens[rand.IntN(len(tokens))], + tokens[rand.IntN(len(tokens))], + tokens[rand.IntN(len(tokens))], + ) + } + + for i := 0; i < 50_000; i++ { + subj := genSubj() + _, _, err := ms.StoreMsg(subj, nil, []byte("Hello World"), 0) + require_NoError(t, err) + } + + state := ms.State() + + // Scan one by one for sanity check against other calculations. + sanityCheck := func(sseq uint64, filter string) SimpleState { + t.Helper() + var ss SimpleState + var smv StoreMsg + // For here we know 0 is invalid, set to 1. + if sseq == 0 { + sseq = 1 + } + for seq := sseq; seq <= state.LastSeq; seq++ { + sm, err := ms.LoadMsg(seq, &smv) + if err != nil { + t.Logf("Encountered error %v loading sequence: %d", err, seq) + continue + } + if subjectIsSubsetMatch(sm.subj, filter) { + ss.Msgs++ + ss.Last = seq + if ss.First == 0 || seq < ss.First { + ss.First = seq + } + } + } + return ss + } + + check := func(sseq uint64, filter string) { + t.Helper() + np, lvs, err := ms.NumPending(sseq, filter, false) + if err != nil { + t.Fatalf("NumPending error: %v", err) + } + ss, err := ms.FilteredState(sseq, filter) + require_NoError(t, err) + sss := sanityCheck(sseq, filter) + if lvs != state.LastSeq { + t.Fatalf("Expected NumPending to return valid through last of %d but got %d", state.LastSeq, lvs) + } + if ss.Msgs != np { + t.Fatalf("NumPending of %d did not match ss.Msgs of %d", np, ss.Msgs) + } + if ss != sss { + t.Fatalf("Failed sanity check, expected %+v got %+v", sss, ss) + } + } + + sanityCheckLastOnly := func(sseq uint64, filter string) SimpleState { + t.Helper() + var ss SimpleState + var smv StoreMsg + // For here we know 0 is invalid, set to 1. + if sseq == 0 { + sseq = 1 + } + seen := make(map[string]bool) + for seq := state.LastSeq; seq >= sseq; seq-- { + sm, err := ms.LoadMsg(seq, &smv) + if err != nil { + t.Logf("Encountered error %v loading sequence: %d", err, seq) + continue + } + if !seen[sm.subj] && subjectIsSubsetMatch(sm.subj, filter) { + ss.Msgs++ + if ss.Last == 0 { + ss.Last = seq + } + if ss.First == 0 || seq < ss.First { + ss.First = seq + } + seen[sm.subj] = true + } + } + return ss + } + + checkLastOnly := func(sseq uint64, filter string) { + t.Helper() + np, lvs, err := ms.NumPending(sseq, filter, true) + if err != nil { + t.Fatalf("NumPending error: %v", err) + } + ss := sanityCheckLastOnly(sseq, filter) + if lvs != state.LastSeq { + t.Fatalf("Expected NumPending to return valid through last of %d but got %d", state.LastSeq, lvs) + } + if ss.Msgs != np { + t.Fatalf("NumPending of %d did not match ss.Msgs of %d", np, ss.Msgs) + } + } + + startSeqs := []uint64{0, 1, 2, 200, 444, 555, 2222, 8888, 12_345, 28_222, 33_456, 44_400, 49_999} + checkSubs := []string{"foo.>", "*.bar.>", "foo.bar.*.baz", "*.bar.>", "*.foo.bar.*", "foo.foo.bar.baz"} + + for _, filter := range checkSubs { + for _, start := range startSeqs { + check(start, filter) + checkLastOnly(start, filter) + } + } +} + +func TestMemStoreInitialFirstSeq(t *testing.T) { + cfg := &StreamConfig{ + Name: "zzz", + Storage: MemoryStorage, + FirstSeq: 1000, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + seq, _, err := ms.StoreMsg("A", nil, []byte("OK"), 0) + require_NoError(t, err) + if seq != 1000 { + t.Fatalf("Message should have been sequence 1000 but was %d", seq) + } + + seq, _, err = ms.StoreMsg("B", nil, []byte("OK"), 0) + require_NoError(t, err) + if seq != 1001 { + t.Fatalf("Message should have been sequence 1001 but was %d", seq) + } + + var state StreamState + ms.FastState(&state) + switch { + case state.Msgs != 2: + t.Fatalf("Expected 2 messages, got %d", state.Msgs) + case state.FirstSeq != 1000: + t.Fatalf("Expected first seq 1000, got %d", state.FirstSeq) + case state.LastSeq != 1001: + t.Fatalf("Expected last seq 1001, got %d", state.LastSeq) + } +} + +func TestMemStoreDeleteBlocks(t *testing.T) { + cfg := &StreamConfig{ + Name: "zzz", + Subjects: []string{"*"}, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + // Put in 10_000 msgs. + total := 10_000 + for i := 0; i < total; i++ { + _, _, err := ms.StoreMsg("A", nil, []byte("OK"), 0) + require_NoError(t, err) + } + + // Now pick 5k random sequences. + delete := 5000 + deleteMap := make(map[int]struct{}, delete) + for len(deleteMap) < delete { + deleteMap[rand.IntN(total)+1] = struct{}{} + } + // Now remove? + for seq := range deleteMap { + ms.RemoveMsg(uint64(seq)) + } + + var state StreamState + ms.FastState(&state) + + // For now we just track via one dmap. + ms.mu.RLock() + dmap := ms.dmap.Clone() + ms.mu.RUnlock() + + require_True(t, dmap.Size() == state.NumDeleted) +} + +// https://github.com/nats-io/nats-server/issues/4850 +func TestMemStoreGetSeqFromTimeWithLastDeleted(t *testing.T) { + cfg := &StreamConfig{ + Name: "zzz", + Subjects: []string{"*"}, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + // Put in 1000 msgs. + total := 1000 + var st time.Time + for i := 1; i <= total; i++ { + _, _, err := ms.StoreMsg("A", nil, []byte("OK"), 0) + require_NoError(t, err) + if i == total/2 { + time.Sleep(100 * time.Millisecond) + st = time.Now() + } + } + // Delete last 100 + for seq := total - 100; seq <= total; seq++ { + ms.RemoveMsg(uint64(seq)) + } + + // Make sure this does not panic with last sequence no longer accessible. + seq := ms.GetSeqFromTime(st) + // Make sure we get the right value. + require_Equal(t, seq, 501) +} + +func TestMemStoreSkipMsgs(t *testing.T) { + cfg := &StreamConfig{ + Name: "zzz", + Subjects: []string{"*"}, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + // Test on empty FS first. + // Make sure wrong starting sequence fails. + err = ms.SkipMsgs(10, 100) + require_Error(t, err, ErrSequenceMismatch) + + err = ms.SkipMsgs(1, 100) + require_NoError(t, err) + + state := ms.State() + require_Equal(t, state.FirstSeq, 101) + require_Equal(t, state.LastSeq, 100) + + // Now add alot. + err = ms.SkipMsgs(101, 100_000) + require_NoError(t, err) + state = ms.State() + require_Equal(t, state.FirstSeq, 100_101) + require_Equal(t, state.LastSeq, 100_100) + + // Now add in a message, and then skip to check dmap. + ms, err = newMemStore(cfg) + require_NoError(t, err) + ms.StoreMsg("foo", nil, nil, 0) + + err = ms.SkipMsgs(2, 10) + require_NoError(t, err) + state = ms.State() + require_Equal(t, state.FirstSeq, 1) + require_Equal(t, state.LastSeq, 11) + require_Equal(t, state.Msgs, 1) + require_Equal(t, state.NumDeleted, 10) + require_Equal(t, len(state.Deleted), 10) + + // Check Fast State too. + state.Deleted = nil + ms.FastState(&state) + require_Equal(t, state.FirstSeq, 1) + require_Equal(t, state.LastSeq, 11) + require_Equal(t, state.Msgs, 1) + require_Equal(t, state.NumDeleted, 10) +} + +func TestMemStoreMultiLastSeqs(t *testing.T) { + cfg := &StreamConfig{ + Name: "zzz", + Subjects: []string{"foo.*", "bar.*"}, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + msg := []byte("abc") + for i := 0; i < 33; i++ { + ms.StoreMsg("foo.foo", nil, msg, 0) + ms.StoreMsg("foo.bar", nil, msg, 0) + ms.StoreMsg("foo.baz", nil, msg, 0) + } + for i := 0; i < 33; i++ { + ms.StoreMsg("bar.foo", nil, msg, 0) + ms.StoreMsg("bar.bar", nil, msg, 0) + ms.StoreMsg("bar.baz", nil, msg, 0) + } + + checkResults := func(seqs, expected []uint64) { + t.Helper() + if len(seqs) != len(expected) { + t.Fatalf("Expected %+v got %+v", expected, seqs) + } + for i := range seqs { + if seqs[i] != expected[i] { + t.Fatalf("Expected %+v got %+v", expected, seqs) + } + } + } + + // UpTo sequence 3. Tests block split. + seqs, err := ms.MultiLastSeqs([]string{"foo.*"}, 3, -1) + require_NoError(t, err) + checkResults(seqs, []uint64{1, 2, 3}) + // Up to last sequence of the stream. + seqs, err = ms.MultiLastSeqs([]string{"foo.*"}, 0, -1) + require_NoError(t, err) + checkResults(seqs, []uint64{97, 98, 99}) + // Check for bar.* at the end. + seqs, err = ms.MultiLastSeqs([]string{"bar.*"}, 0, -1) + require_NoError(t, err) + checkResults(seqs, []uint64{196, 197, 198}) + // This should find nothing. + seqs, err = ms.MultiLastSeqs([]string{"bar.*"}, 99, -1) + require_NoError(t, err) + checkResults(seqs, nil) + + // Do multiple subjects explicitly. + seqs, err = ms.MultiLastSeqs([]string{"foo.foo", "foo.bar", "foo.baz"}, 3, -1) + require_NoError(t, err) + checkResults(seqs, []uint64{1, 2, 3}) + seqs, err = ms.MultiLastSeqs([]string{"foo.foo", "foo.bar", "foo.baz"}, 0, -1) + require_NoError(t, err) + checkResults(seqs, []uint64{97, 98, 99}) + seqs, err = ms.MultiLastSeqs([]string{"bar.foo", "bar.bar", "bar.baz"}, 0, -1) + require_NoError(t, err) + checkResults(seqs, []uint64{196, 197, 198}) + seqs, err = ms.MultiLastSeqs([]string{"bar.foo", "bar.bar", "bar.baz"}, 99, -1) + require_NoError(t, err) + checkResults(seqs, nil) + + // Check single works + seqs, err = ms.MultiLastSeqs([]string{"foo.foo"}, 3, -1) + require_NoError(t, err) + checkResults(seqs, []uint64{1}) + + // Now test that we properly de-duplicate between filters. + seqs, err = ms.MultiLastSeqs([]string{"foo.*", "foo.bar"}, 3, -1) + require_NoError(t, err) + checkResults(seqs, []uint64{1, 2, 3}) + seqs, err = ms.MultiLastSeqs([]string{"bar.>", "bar.bar", "bar.baz"}, 0, -1) + require_NoError(t, err) + checkResults(seqs, []uint64{196, 197, 198}) + + // All + seqs, err = ms.MultiLastSeqs([]string{">"}, 0, -1) + require_NoError(t, err) + checkResults(seqs, []uint64{97, 98, 99, 196, 197, 198}) + seqs, err = ms.MultiLastSeqs([]string{">"}, 99, -1) + require_NoError(t, err) + checkResults(seqs, []uint64{97, 98, 99}) +} + +func TestMemStoreMultiLastSeqsMaxAllowed(t *testing.T) { + cfg := &StreamConfig{ + Name: "zzz", + Subjects: []string{"foo.*"}, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + msg := []byte("abc") + for i := 1; i <= 100; i++ { + ms.StoreMsg(fmt.Sprintf("foo.%d", i), nil, msg, 0) + } + // Test that if we specify maxAllowed that we get the correct error. + seqs, err := ms.MultiLastSeqs([]string{"foo.*"}, 0, 10) + require_True(t, seqs == nil) + require_Error(t, err, ErrTooManyResults) +} + +// Bug would cause PurgeEx to fail if it encountered a deleted msg at sequence to delete up to. +func TestMemStorePurgeExWithDeletedMsgs(t *testing.T) { + cfg := &StreamConfig{ + Name: "zzz", + Subjects: []string{"foo"}, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + msg := []byte("abc") + for i := 1; i <= 10; i++ { + ms.StoreMsg("foo", nil, msg, 0) + } + ms.RemoveMsg(2) + ms.RemoveMsg(9) // This was the bug + + n, err := ms.PurgeEx(_EMPTY_, 9, 0) + require_NoError(t, err) + require_Equal(t, n, 7) + + var state StreamState + ms.FastState(&state) + require_Equal(t, state.FirstSeq, 10) + require_Equal(t, state.LastSeq, 10) + require_Equal(t, state.Msgs, 1) +} + +// When all messages are deleted we should have a state of first = last + 1. +func TestMemStoreDeleteAllFirstSequenceCheck(t *testing.T) { + cfg := &StreamConfig{ + Name: "zzz", + Subjects: []string{"foo"}, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + msg := []byte("abc") + for i := 1; i <= 10; i++ { + ms.StoreMsg("foo", nil, msg, 0) + } + for seq := uint64(1); seq <= 10; seq++ { + ms.RemoveMsg(seq) + } + var state StreamState + ms.FastState(&state) + require_Equal(t, state.FirstSeq, 11) + require_Equal(t, state.LastSeq, 10) + require_Equal(t, state.Msgs, 0) +} + +func TestMemStoreNumPendingMulti(t *testing.T) { + cfg := &StreamConfig{ + Name: "zzz", + Subjects: []string{"ev.*"}, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + totalMsgs := 100_000 + totalSubjects := 10_000 + numFiltered := 5000 + startSeq := uint64(5_000 + rand.IntN(90_000)) + + subjects := make([]string, 0, totalSubjects) + for i := 0; i < totalSubjects; i++ { + subjects = append(subjects, fmt.Sprintf("ev.%s", nuid.Next())) + } + + // Put in 100k msgs with random subjects. + msg := bytes.Repeat([]byte("ZZZ"), 333) + for i := 0; i < totalMsgs; i++ { + _, _, err = ms.StoreMsg(subjects[rand.IntN(totalSubjects)], nil, msg, 0) + require_NoError(t, err) + } + + // Now we want to do a calculate NumPendingMulti. + filters := gsl.NewSublist[struct{}]() + for filters.Count() < uint32(numFiltered) { + filter := subjects[rand.IntN(totalSubjects)] + if !filters.HasInterest(filter) { + filters.Insert(filter, struct{}{}) + } + } + + // Use new function. + total, _, err := ms.NumPendingMulti(startSeq, filters, false) + require_NoError(t, err) + + // Check our results. + var checkTotal uint64 + var smv StoreMsg + for seq := startSeq; seq <= uint64(totalMsgs); seq++ { + sm, err := ms.LoadMsg(seq, &smv) + require_NoError(t, err) + if filters.HasInterest(sm.subj) { + checkTotal++ + } + } + require_Equal(t, total, checkTotal) +} + +func TestMemStoreNumPendingBug(t *testing.T) { + cfg := &StreamConfig{ + Name: "zzz", + Subjects: []string{"foo.*"}, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + // 12 msgs total + for _, subj := range []string{"foo.foo", "foo.bar", "foo.baz", "foo.zzz"} { + ms.StoreMsg("foo.aaa", nil, nil, 0) + ms.StoreMsg(subj, nil, nil, 0) + ms.StoreMsg(subj, nil, nil, 0) + } + total, _, err := ms.NumPending(4, "foo.*", false) + require_NoError(t, err) + + var checkTotal uint64 + var smv StoreMsg + for seq := 4; seq <= 12; seq++ { + sm, err := ms.LoadMsg(uint64(seq), &smv) + require_NoError(t, err) + if subjectIsSubsetMatch(sm.subj, "foo.*") { + checkTotal++ + } + } + require_Equal(t, total, checkTotal) +} + +func TestMemStorePurgeLeaksDmap(t *testing.T) { + cfg := &StreamConfig{ + Name: "TEST", + Subjects: []string{"foo"}, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + for i := 0; i < 10; i++ { + _, _, err = ms.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + } + + for i := uint64(2); i <= 9; i++ { + _, err = ms.RemoveMsg(i) + require_NoError(t, err) + } + ms.mu.Lock() + dmaps := ms.dmap.Size() + ms.mu.Unlock() + require_Equal(t, dmaps, 8) + + purged, err := ms.Purge() + require_NoError(t, err) + require_Equal(t, purged, 2) + + ms.mu.Lock() + dmaps = ms.dmap.Size() + ms.mu.Unlock() + require_Equal(t, dmaps, 0) +} + +func TestMemStoreMessageTTL(t *testing.T) { + fs, err := newMemStore( + &StreamConfig{Name: "zzz", Subjects: []string{"test"}, Storage: MemoryStorage, AllowMsgTTL: true}, + ) + require_NoError(t, err) + defer fs.Stop() + + ttl := int64(1) // 1 second + + for i := 1; i <= 10; i++ { + _, _, err = fs.StoreMsg("test", nil, nil, ttl) + require_NoError(t, err) + } + + var ss StreamState + fs.FastState(&ss) + require_Equal(t, ss.FirstSeq, 1) + require_Equal(t, ss.LastSeq, 10) + require_Equal(t, ss.Msgs, 10) + + time.Sleep(time.Second * 2) + + fs.FastState(&ss) + require_Equal(t, ss.FirstSeq, 11) + require_Equal(t, ss.LastSeq, 10) + require_Equal(t, ss.Msgs, 0) +} + +func TestMemStoreSubjectDeleteMarkers(t *testing.T) { + fs, err := newMemStore( + &StreamConfig{ + Name: "zzz", Subjects: []string{"test"}, Storage: MemoryStorage, + MaxAge: time.Second, AllowMsgTTL: true, + SubjectDeleteMarkerTTL: time.Second, + }, + ) + require_NoError(t, err) + defer fs.Stop() + + // Capture subject delete marker proposals. + ch := make(chan *inMsg, 1) + fs.rmcb = func(seq uint64) { + _, err := fs.RemoveMsg(seq) + require_NoError(t, err) + } + fs.pmsgcb = func(im *inMsg) { + ch <- im + } + + // Store three messages that will expire because of MaxAge. + for i := 0; i < 3; i++ { + _, _, err = fs.StoreMsg("test", nil, nil, 0) + require_NoError(t, err) + } + + // Wait for MaxAge to pass. + time.Sleep(time.Second + time.Millisecond*500) + + // We should have placed a subject delete marker. + im := require_ChanRead(t, ch, time.Second*5) + require_Equal(t, bytesToString(getHeader(JSMarkerReason, im.hdr)), JSMarkerReasonMaxAge) + require_Equal(t, bytesToString(getHeader(JSMessageTTL, im.hdr)), "1s") +} + +func TestMemStoreAllLastSeqs(t *testing.T) { + cfg := &StreamConfig{ + Name: "zzz", + Subjects: []string{"*.*"}, + MaxMsgsPer: 50, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + subjs := []string{"foo.foo", "foo.bar", "foo.baz", "bar.foo", "bar.bar", "bar.baz"} + msg := []byte("abc") + + for i := 0; i < 100_000; i++ { + subj := subjs[rand.IntN(len(subjs))] + ms.StoreMsg(subj, nil, msg, 0) + } + + expected := make([]uint64, 0, len(subjs)) + var smv StoreMsg + for _, subj := range subjs { + sm, err := ms.LoadLastMsg(subj, &smv) + require_NoError(t, err) + expected = append(expected, sm.seq) + } + slices.Sort(expected) + + seqs, err := ms.AllLastSeqs() + require_NoError(t, err) + require_True(t, reflect.DeepEqual(seqs, expected)) +} + +func TestMemStoreUpdateConfigTTLState(t *testing.T) { + cfg := &StreamConfig{ + Name: "zzz", + Subjects: []string{">"}, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + require_Equal(t, ms.ttls, nil) + + cfg.AllowMsgTTL = true + require_NoError(t, ms.UpdateConfig(cfg)) + require_NotEqual(t, ms.ttls, nil) + + cfg.AllowMsgTTL = false + require_NoError(t, ms.UpdateConfig(cfg)) + require_Equal(t, ms.ttls, nil) +} + +func TestMemStoreSubjectForSeq(t *testing.T) { + cfg := StreamConfig{ + Name: "foo", + Subjects: []string{"foo.>"}, + Storage: MemoryStorage, + } + ms, err := newMemStore(&cfg) + require_NoError(t, err) + + seq, _, err := ms.StoreMsg("foo.bar", nil, nil, 0) + require_NoError(t, err) + require_Equal(t, seq, 1) + + _, err = ms.SubjectForSeq(0) + require_Error(t, err, ErrStoreMsgNotFound) + + subj, err := ms.SubjectForSeq(1) + require_NoError(t, err) + require_Equal(t, subj, "foo.bar") + + _, err = ms.SubjectForSeq(2) + require_Error(t, err, ErrStoreMsgNotFound) +} + +func TestMemStoreMessageSchedule(t *testing.T) { + fs, err := newMemStore( + &StreamConfig{ + Name: "TEST", Subjects: []string{"foo.*"}, Storage: MemoryStorage, + AllowMsgSchedules: true, + }, + ) + require_NoError(t, err) + defer fs.Stop() + + // Capture message schedule proposals. + ch := make(chan *inMsg, 1) + fs.pmsgcb = func(im *inMsg) { + ch <- im + } + + // Store a single message schedule. + schedule := time.Now().Add(time.Second).Format(time.RFC3339Nano) + hdr := genHeader(nil, JSSchedulePattern, fmt.Sprintf("@at %s", schedule)) + hdr = genHeader(hdr, JSScheduleTarget, "foo.target") + _, _, err = fs.StoreMsg("foo.schedule", hdr, nil, 0) + require_NoError(t, err) + + // We should have published a scheduled message. + im := require_ChanRead(t, ch, time.Second*5) + require_Equal(t, im.subj, "foo.target") + require_Equal(t, bytesToString(getHeader(JSScheduler, im.hdr)), "foo.schedule") + require_Equal(t, bytesToString(getHeader(JSScheduleNext, im.hdr)), JSScheduleNextPurge) +} + +func TestMemStoreMessageScheduleEverySubSecondPrecision(t *testing.T) { + fs, err := newMemStore( + &StreamConfig{ + Name: "TEST", Subjects: []string{"foo.*"}, Storage: MemoryStorage, + AllowMsgSchedules: true, + }, + ) + require_NoError(t, err) + defer fs.Stop() + + // Capture message schedule proposals. + ch := make(chan *inMsg, 1) + fs.pmsgcb = func(im *inMsg) { + ch <- im + } + + // Store a repeating message schedule with a sub-second remainder. + hdr := genHeader(nil, JSSchedulePattern, "@every 1500ms") + hdr = genHeader(hdr, JSScheduleTarget, "foo.target") + _, ts, err := fs.StoreMsg("foo.schedule", hdr, nil, 0) + require_NoError(t, err) + + // We should have published a scheduled message. + im := require_ChanRead(t, ch, time.Second*3) + require_Equal(t, im.subj, "foo.target") + require_Equal(t, bytesToString(getHeader(JSScheduler, im.hdr)), "foo.schedule") + + // The re-arm header is written with RFC3339Nano so the sub-second component + // of the next fire time survives the round trip through the header. Each + // fire time anchors on the previous one rounded to the nearest second, plus + // the interval: the first fire is round(store time) + 1500ms, and the + // re-arm rounds that again (up, .5s rounds away from zero) before adding + // another 1500ms. + const interval = 1500 * time.Millisecond + first := time.Unix(0, ts).UTC().Round(time.Second).Add(interval) + expected := first.Round(time.Second).Add(interval) + scheduleNext := bytesToString(getHeader(JSScheduleNext, im.hdr)) + next, err := time.Parse(time.RFC3339Nano, scheduleNext) + require_NoError(t, err) + require_Equal(t, next.UnixNano(), expected.UnixNano()) +} + +func TestMemStoreNextWildcardMatch(t *testing.T) { + cfg := &StreamConfig{ + Name: "zzz", + Subjects: []string{"foo.>"}, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + msg := []byte("msg") + storeN := func(subj string, n int) { + t.Helper() + for range n { + _, _, err := ms.StoreMsg(subj, nil, msg, 0) + require_NoError(t, err) + } + } + storeN("foo.bar.a", 1) + storeN("foo.baz.bar", 10) + storeN("foo.bar.b", 1) + storeN("foo.baz.bar", 10) + storeN("foo.baz.bar.no.match", 10) + + ms.mu.Lock() + defer ms.mu.Unlock() + + first, last, found := ms.nextWildcardMatchLocked("foo.bar.*", 0) + require_True(t, found) + require_Equal(t, first, 1) + require_Equal(t, last, 12) + + first, last, found = ms.nextWildcardMatchLocked("foo.bar.*", 1) + require_True(t, found) + require_Equal(t, first, 1) + require_Equal(t, last, 12) + + first, last, found = ms.nextWildcardMatchLocked("foo.bar.*", 2) + require_True(t, found) + require_Equal(t, first, 12) + require_Equal(t, last, 12) + + _, _, found = ms.nextWildcardMatchLocked("foo.bar.*", first+1) + require_False(t, found) + + first, last, found = ms.nextWildcardMatchLocked("foo.baz.*", 1) + require_True(t, found) + require_Equal(t, first, 2) + require_Equal(t, last, 22) + + first, last, found = ms.nextWildcardMatchLocked("foo.baz.*", 11) + require_True(t, found) + require_Equal(t, first, 11) + require_Equal(t, last, 22) + + first, last, found = ms.nextWildcardMatchLocked("foo.baz.*", 12) + require_True(t, found) + require_Equal(t, first, 12) + require_Equal(t, last, 22) + + first, last, found = ms.nextWildcardMatchLocked("foo.baz.*", 22) + require_True(t, found) + require_Equal(t, first, 22) + require_Equal(t, last, 22) + + first, last, found = ms.nextWildcardMatchLocked("foo.baz.*", 23) + require_False(t, found) + require_Equal(t, first, 0) + require_Equal(t, last, 0) + + first, last, found = ms.nextWildcardMatchLocked("foo.nope.*", 1) + require_False(t, found) + require_Equal(t, first, 0) + require_Equal(t, last, 0) + + first, last, found = ms.nextWildcardMatchLocked("foo.>", 1) + require_True(t, found) + require_Equal(t, first, 1) + require_Equal(t, last, 32) +} + +func TestMemStoreNextLiteralMatch(t *testing.T) { + cfg := &StreamConfig{ + Name: "zzz", + Subjects: []string{"foo.>"}, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + msg := []byte("msg") + storeN := func(subj string, n int) { + t.Helper() + for range n { + _, _, err := ms.StoreMsg(subj, nil, msg, 0) + require_NoError(t, err) + } + } + storeN("foo.bar.a", 1) // seq 1 + storeN("foo.baz.bar", 10) // seqs 2-11 + storeN("foo.bar.b", 1) // seq 12 + storeN("foo.baz.bar", 10) // seqs 13-22 + storeN("foo.baz.bar.no.match", 10) // seqs 23-32 + + ms.mu.Lock() + defer ms.mu.Unlock() + + first, last, found := ms.nextLiteralMatchLocked("foo.bar.a", 0) + require_True(t, found) + require_Equal(t, first, 1) + require_Equal(t, last, 1) + + _, _, found = ms.nextLiteralMatchLocked("foo.bar.a", 2) + require_False(t, found) + + first, last, found = ms.nextLiteralMatchLocked("foo.baz.bar", 1) + require_True(t, found) + require_Equal(t, first, 2) + require_Equal(t, last, 22) + + first, last, found = ms.nextLiteralMatchLocked("foo.baz.bar", 11) + require_True(t, found) + require_Equal(t, first, 11) + require_Equal(t, last, 22) + + first, last, found = ms.nextLiteralMatchLocked("foo.baz.bar", 22) + require_True(t, found) + require_Equal(t, first, 22) + require_Equal(t, last, 22) + + first, last, found = ms.nextLiteralMatchLocked("foo.baz.bar", 23) + require_False(t, found) + require_Equal(t, first, 0) + require_Equal(t, last, 0) + + first, last, found = ms.nextLiteralMatchLocked("foo.nope", 1) + require_False(t, found) + require_Equal(t, first, 0) + require_Equal(t, last, 0) + +} + +func TestMemStoreMultiLastSeqsDoesNotUseStaleLastValue(t *testing.T) { + cfg := &StreamConfig{ + Name: "zzz", + Subjects: []string{"foo", "bar", "baz"}, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + subjects := []string{"foo", "bar", "foo", " baz", "foo", "foo", "bar", "foo", "bar", "baz"} + for _, s := range subjects { + _, _, err := ms.StoreMsg(s, nil, nil, 0) + require_NoError(t, err) + } + + // Initially we expect last seq for `foo` to be 8 + seqs, err := ms.MultiLastSeqs([]string{"foo", "bar"}, 0, 0) + require_NoError(t, err) + require_Equal(t, len(seqs), 2) + require_Equal(t, seqs[0], uint64(8)) + + // Remove latest foo message + removed, err := ms.RemoveMsg(8) + require_NoError(t, err) + require_True(t, removed) + + // If bug is present: MultiLastSeqs returns last sequence + // 8 for subject `foo`. + // After removal, expect last sequence 6 for subject `foo`. + seqs, err = ms.MultiLastSeqs([]string{"foo", "bar"}, 0, 0) + require_NoError(t, err) + require_Equal(t, len(seqs), 2) + require_Equal(t, seqs[0], uint64(6)) +} + +/////////////////////////////////////////////////////////////////////////// +// Benchmarks +/////////////////////////////////////////////////////////////////////////// + +func Benchmark_MemStoreNumPendingWithLargeInteriorDeletesScan(b *testing.B) { + cfg := &StreamConfig{ + Name: "zzz", + Subjects: []string{"foo.*.*"}, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(b, err) + defer ms.Stop() + + msg := []byte("abc") + ms.StoreMsg("foo.bar.baz", nil, msg, 0) + for i := 1; i <= 1_000_000; i++ { + ms.SkipMsg(0) + } + ms.StoreMsg("foo.bar.baz", nil, msg, 0) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + total, _, err := ms.NumPending(600_000, "foo.*.baz", false) + require_NoError(b, err) + if total != 1 { + b.Fatalf("Expected total of 2 got %d", total) + } + } +} + +func Benchmark_MemStoreNumPendingWithLargeInteriorDeletesExclude(b *testing.B) { + cfg := &StreamConfig{ + Name: "zzz", + Subjects: []string{"foo.*.*"}, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(b, err) + defer ms.Stop() + + msg := []byte("abc") + ms.StoreMsg("foo.bar.baz", nil, msg, 0) + for i := 1; i <= 1_000_000; i++ { + ms.SkipMsg(0) + } + ms.StoreMsg("foo.bar.baz", nil, msg, 0) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + total, _, err := ms.NumPending(400_000, "foo.*.baz", false) + require_NoError(b, err) + if total != 1 { + b.Fatalf("Expected total of 2 got %d", total) + } + } +} + +func Benchmark_MemStoreSubjectStateConsistencyOptimizationPerf(b *testing.B) { + cfg := &StreamConfig{Name: "TEST", Subjects: []string{"foo.*"}, Storage: MemoryStorage, MaxMsgsPer: 1} + ms, err := newMemStore(cfg) + require_NoError(b, err) + defer ms.Stop() + + // Do R rounds of storing N messages. + // MaxMsgsPer=1, so every unique subject that's placed only exists in the stream once. + // If R=2, N=3 that means we'd place foo.0, foo.1, foo.2 in the first round, and the second + // round we'd place foo.2, foo.1, foo.0, etc. This is intentional so that without any + // optimizations we'd need to scan either 1 in the optimal case or N in the worst case. + // Which is way more expensive than always knowing what the sequences are and it being O(1). + r := max(2, b.N) + n := 40_000 + b.ResetTimer() + for i := 0; i < r; i++ { + for j := 0; j < n; j++ { + d := j + if i%2 == 0 { + d = n - j - 1 + } + subject := fmt.Sprintf("foo.%d", d) + _, _, err = ms.StoreMsg(subject, nil, nil, 0) + require_NoError(b, err) + } + } +} + +// This benchmark populates a memstore and then measures +// the time it takes to load the entire store using +// LoadNextMsg repeatedly until the store returns ErrStoreEOF. +func Benchmark_MemStoreLoadNextMsgFiltered(b *testing.B) { + cases := []struct { + name string + msgs int + matchingMsgEvery int + filter string + wc bool + matchingSubject func(i int) string + expectLinear bool + }{ + { + name: "wildcard_linear_scan", + msgs: 10_000_000, + matchingMsgEvery: 10_000, + filter: "foo.baz.*", + wc: true, + matchingSubject: func(i int) string { + return fmt.Sprintf("foo.baz.%d", i) + }, + expectLinear: true, + }, + { + name: "wildcard_bounded_scan", + msgs: 10_000_000, + matchingMsgEvery: 100_000, + filter: "foo.baz.*", + wc: true, + matchingSubject: func(i int) string { + return fmt.Sprintf("foo.baz.%d", i) + }, + expectLinear: false, + }, + { + name: "literal_bounded_scan", + msgs: 10_000_000, + matchingMsgEvery: 100_000, + filter: "foo.baz", + wc: false, + matchingSubject: func(i int) string { + return "foo.baz" + }, + expectLinear: false, + }, + } + + for _, tc := range cases { + b.Run(tc.name, func(b *testing.B) { + cfg := &StreamConfig{Name: "TEST", Subjects: []string{"foo.>"}, Storage: MemoryStorage} + ms, err := newMemStore(cfg) + require_NoError(b, err) + defer ms.Stop() + + msg := []byte("ok") + for i := range tc.msgs { + subject := "foo.bar" + if i%tc.matchingMsgEvery == 0 { + subject = tc.matchingSubject(i) + } + _, _, err = ms.StoreMsg(subject, nil, msg, 0) + require_NoError(b, err) + } + + ms.mu.Lock() + require_Equal(b, ms.shouldLinearScan(tc.filter, tc.wc, 1), tc.expectLinear) + ms.mu.Unlock() + + var smv StoreMsg + expectedMatches := uint64(tc.msgs / tc.matchingMsgEvery) + + b.ResetTimer() + for b.Loop() { + var start uint64 + var count uint64 + for { + _, start, err = ms.LoadNextMsg(tc.filter, tc.wc, start, &smv) + start++ + if err == ErrStoreEOF { + require_Equal(b, count, expectedMatches) + break + } + require_NoError(b, err) + count++ + } + } + }) + } +} + +func TestMemStoreMultiLastSeqsDoesNotReorderConfigSubjects(t *testing.T) { + subjects := []string{"orders.*", "billing.*"} + ms, err := newMemStore(&StreamConfig{Name: "zzz", Storage: MemoryStorage, Subjects: subjects}) + require_NoError(t, err) + + _, _, err = ms.StoreMsg("orders.1", nil, []byte("x"), 0) + require_NoError(t, err) + _, _, err = ms.StoreMsg("billing.1", nil, []byte("x"), 0) + require_NoError(t, err) + + // Filter count == subject count drives the filterIsAll path. + _, err = ms.MultiLastSeqs([]string{"orders.*", "billing.*"}, 0, 0) + require_NoError(t, err) + + // The advertised config subjects must keep their original order. + require_Equal(t, ms.cfg.Subjects[0], "orders.*") + require_Equal(t, ms.cfg.Subjects[1], "billing.*") +} + +func TestMemStoreSourcesAddedSourceKeepsExistingState(t *testing.T) { + origin1 := &StreamSource{Name: "ORIGIN1"} + origin2 := &StreamSource{Name: "ORIGIN2"} + iName1, iName2 := origin1.composeIName(), origin2.composeIName() + + cfg := StreamConfig{ + Name: "SOURCE", + Subjects: []string{"foo"}, + Storage: MemoryStorage, + Sources: []*StreamSource{origin1}, + } + ms, err := newMemStore(&cfg) + require_NoError(t, err) + defer ms.Stop() + + for i := range 10 { + hdr := genHeader(nil, JSStreamSource, fmt.Sprintf("ORIGIN1 %d > > foo", i+1)) + _, _, err = ms.StoreMsg("foo", hdr, nil, 0) + require_NoError(t, err) + } + state := ms.SourcesState() + require_Len(t, len(state), 1) + require_Equal(t, state[iName1].Seq, 10) + + // Remove the newest message, a scan would now only be able to derive + // ORIGIN1 up to sequence 9. + removed, err := ms.RemoveMsg(10) + require_NoError(t, err) + require_True(t, removed) + + // Adding a source must only recover the sequence of the source that was just + // added. Sources that are already tracked must keep their state, even if a + // scan derives a lower sequence for them, since that would result in + // the source consumer re-delivering messages it had already stored. + ucfg := cfg + ucfg.Sources = []*StreamSource{origin1, origin2} + require_NoError(t, ms.UpdateConfig(&ucfg)) + + state = ms.SourcesState() + require_Len(t, len(state), 2) + // ORIGIN1 must not have moved backward. + require_Equal(t, state[iName1].Seq, 10) + // ORIGIN2 is newly seeded and has nothing to recover. + require_Equal(t, state[iName2].Seq, 0) +} diff --git a/server/monitor_sort_opts.go b/server/monitor_sort_opts.go new file mode 100644 index 0000000..79152f8 --- /dev/null +++ b/server/monitor_sort_opts.go @@ -0,0 +1,156 @@ +// Copyright 2013-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "time" +) + +// ConnInfos represents a connection info list. We use pointers since it will be sorted. +type ConnInfos []*ConnInfo + +// For sorting +// Len returns length for sorting. +func (cl ConnInfos) Len() int { return len(cl) } + +// Swap will sawap the elements. +func (cl ConnInfos) Swap(i, j int) { cl[i], cl[j] = cl[j], cl[i] } + +// SortOpt is a helper type to sort clients +type SortOpt string + +// Possible sort options +const ( + ByCid SortOpt = "cid" // By connection ID + ByStart SortOpt = "start" // By connection start time, same as CID + BySubs SortOpt = "subs" // By number of subscriptions + ByPending SortOpt = "pending" // By amount of data in bytes waiting to be sent to client + ByOutMsgs SortOpt = "msgs_to" // By number of messages sent + ByInMsgs SortOpt = "msgs_from" // By number of messages received + ByOutBytes SortOpt = "bytes_to" // By amount of bytes sent + ByInBytes SortOpt = "bytes_from" // By amount of bytes received + ByLast SortOpt = "last" // By the last activity + ByIdle SortOpt = "idle" // By the amount of inactivity + ByUptime SortOpt = "uptime" // By the amount of time connections exist + ByStop SortOpt = "stop" // By the stop time for a closed connection + ByReason SortOpt = "reason" // By the reason for a closed connection + ByRTT SortOpt = "rtt" // By the round trip time +) + +// Individual sort options provide the Less for sort.Interface. Len and Swap are on cList. +// CID +type SortByCid struct{ ConnInfos } + +func (l SortByCid) Less(i, j int) bool { return l.ConnInfos[i].Cid < l.ConnInfos[j].Cid } + +// Number of Subscriptions +type SortBySubs struct{ ConnInfos } + +func (l SortBySubs) Less(i, j int) bool { return l.ConnInfos[i].NumSubs < l.ConnInfos[j].NumSubs } + +// Pending Bytes +type SortByPending struct{ ConnInfos } + +func (l SortByPending) Less(i, j int) bool { return l.ConnInfos[i].Pending < l.ConnInfos[j].Pending } + +// Outbound Msgs +type SortByOutMsgs struct{ ConnInfos } + +func (l SortByOutMsgs) Less(i, j int) bool { return l.ConnInfos[i].OutMsgs < l.ConnInfos[j].OutMsgs } + +// Inbound Msgs +type SortByInMsgs struct{ ConnInfos } + +func (l SortByInMsgs) Less(i, j int) bool { return l.ConnInfos[i].InMsgs < l.ConnInfos[j].InMsgs } + +// Outbound Bytes +type SortByOutBytes struct{ ConnInfos } + +func (l SortByOutBytes) Less(i, j int) bool { return l.ConnInfos[i].OutBytes < l.ConnInfos[j].OutBytes } + +// Inbound Bytes +type SortByInBytes struct{ ConnInfos } + +func (l SortByInBytes) Less(i, j int) bool { return l.ConnInfos[i].InBytes < l.ConnInfos[j].InBytes } + +// Last Activity +type SortByLast struct{ ConnInfos } + +func (l SortByLast) Less(i, j int) bool { + return l.ConnInfos[i].LastActivity.UnixNano() < l.ConnInfos[j].LastActivity.UnixNano() +} + +// Idle time +type SortByIdle struct { + ConnInfos + now time.Time +} + +func (l SortByIdle) Less(i, j int) bool { + return l.now.Sub(l.ConnInfos[i].LastActivity) < l.now.Sub(l.ConnInfos[j].LastActivity) +} + +// Uptime +type SortByUptime struct { + ConnInfos + now time.Time +} + +func (l SortByUptime) Less(i, j int) bool { + ci := l.ConnInfos[i] + cj := l.ConnInfos[j] + var upi, upj time.Duration + if ci.Stop == nil || ci.Stop.IsZero() { + upi = l.now.Sub(ci.Start) + } else { + upi = ci.Stop.Sub(ci.Start) + } + if cj.Stop == nil || cj.Stop.IsZero() { + upj = l.now.Sub(cj.Start) + } else { + upj = cj.Stop.Sub(cj.Start) + } + return upi < upj +} + +// Stop +type SortByStop struct{ ConnInfos } + +func (l SortByStop) Less(i, j int) bool { + ciStop := l.ConnInfos[i].Stop + cjStop := l.ConnInfos[j].Stop + return ciStop.Before(*cjStop) +} + +// Reason +type SortByReason struct{ ConnInfos } + +func (l SortByReason) Less(i, j int) bool { + return l.ConnInfos[i].Reason < l.ConnInfos[j].Reason +} + +// RTT - Default is descending +type SortByRTT struct{ ConnInfos } + +func (l SortByRTT) Less(i, j int) bool { return l.ConnInfos[i].rtt < l.ConnInfos[j].rtt } + +// IsValid determines if a sort option is valid +func (s SortOpt) IsValid() bool { + switch s { + case _EMPTY_, ByCid, ByStart, BySubs, ByPending, ByOutMsgs, ByInMsgs, ByOutBytes, ByInBytes, ByLast, ByIdle, ByUptime, ByStop, ByReason, ByRTT: + return true + default: + return false + } +} diff --git a/server/mqtt_ex_bench_test.go b/server/mqtt_ex_bench_test.go new file mode 100644 index 0000000..2e63785 --- /dev/null +++ b/server/mqtt_ex_bench_test.go @@ -0,0 +1,353 @@ +// Copyright 2024-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !skip_mqtt_tests + +package server + +import ( + "fmt" + "strconv" + "testing" + "time" +) + +const ( + KB = 1024 +) + +type mqttBenchMatrix struct { + QOS []int + MessageSize []int + Topics []int + Publishers []int + Subscribers []int +} + +type mqttBenchContext struct { + QOS int + MessageSize int + Topics int + Publishers int + Subscribers int + + Host string + Port int +} + +var mqttBenchDefaultMatrix = mqttBenchMatrix{ + QOS: []int{0, 1, 2}, + MessageSize: []int{100, 1 * KB, 10 * KB}, + Topics: []int{100}, + Publishers: []int{1}, + Subscribers: []int{1}, +} + +type MQTTBenchmarkResult struct { + Ops int `json:"ops"` + NS map[string]time.Duration `json:"ns"` + Bytes int64 `json:"bytes"` +} + +func BenchmarkXMQTT(b *testing.B) { + if mqttTestCommandPath == "" { + b.Skip(`"mqtt-test" command is not found in $PATH.`) + } + + bc := mqttBenchContext{} + b.Run("Server", func(b *testing.B) { + b.Cleanup(bc.startServer(b, false)) + bc.runAll(b) + }) + + b.Run("Cluster", func(b *testing.B) { + b.Cleanup(bc.startCluster(b, false)) + bc.runAll(b) + }) + + b.Run("Server-no-RMSCache", func(b *testing.B) { + b.Cleanup(bc.startServer(b, true)) + + bc.benchmarkSubRet(b) + }) + + b.Run("Cluster-no-RMSCache", func(b *testing.B) { + b.Cleanup(bc.startCluster(b, true)) + + bc.benchmarkSubRet(b) + }) +} + +func (bc mqttBenchContext) runAll(b *testing.B) { + bc.benchmarkPub(b) + bc.benchmarkPubPipelined(b) + bc.benchmarkPubRetained(b) + bc.benchmarkPubSub(b) + bc.benchmarkSubRet(b) +} + +// makes a copy of bc +func (bc mqttBenchContext) benchmarkPub(b *testing.B) { + m := mqttBenchDefaultMatrix. + NoSubscribers(). + NoTopics() + + b.Run("PUB", func(b *testing.B) { + m.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) { + bc.runAndReport(b, "pub", + "--qos", strconv.Itoa(bc.QOS), + "--messages", strconv.Itoa(b.N), + "--size", strconv.Itoa(bc.MessageSize), + "--publishers", strconv.Itoa(bc.Publishers), + "--mps", "0", // no rate throttle + ) + }) + }) +} + +// makes a copy of bc +// +// Like PUB, but with up to 256 messages in flight: measures per-connection +// throughput rather than round-trip latency. +func (bc mqttBenchContext) benchmarkPubPipelined(b *testing.B) { + m := mqttBenchDefaultMatrix. + NoSubscribers(). + NoTopics(). + QOS1Only() + + b.Run("PUBX", func(b *testing.B) { + m.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) { + bc.runAndReport(b, "pub", + "--qos", strconv.Itoa(bc.QOS), + "--messages", strconv.Itoa(b.N), + "--size", strconv.Itoa(bc.MessageSize), + "--publishers", strconv.Itoa(bc.Publishers), + "--mps", "0", // no rate throttle + "--pipeline", "256", + ) + }) + }) +} + +// makes a copy of bc +func (bc mqttBenchContext) benchmarkPubRetained(b *testing.B) { + // This bench is meaningless for QOS0 since the client considers the message + // sent as soon as it's written out. It is also useless for QOS2 since the + // flow takes a lot longer, and the difference of publishing as retained or + // not is lost in the noise. + m := mqttBenchDefaultMatrix. + NoSubscribers(). + NoTopics(). + QOS1Only() + + b.Run("PUBRET", func(b *testing.B) { + m.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) { + bc.runAndReport(b, "pub", "--retain", + "--qos", strconv.Itoa(bc.QOS), + "--messages", strconv.Itoa(b.N), + "--size", strconv.Itoa(bc.MessageSize), + "--publishers", strconv.Itoa(bc.Publishers), + ) + }) + }) +} + +// makes a copy of bc +func (bc mqttBenchContext) benchmarkPubSub(b *testing.B) { + // This test uses a single built-in topic, and a built-in publisher, so no + // reason to run it for topics and publishers. + m := mqttBenchDefaultMatrix. + NoTopics(). + NoPublishers() + + b.Run("PUBSUB", func(b *testing.B) { + m.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) { + bc.runAndReport(b, "pubsub", + "--qos", strconv.Itoa(bc.QOS), + "--messages", strconv.Itoa(b.N), + "--size", strconv.Itoa(bc.MessageSize), + "--subscribers", strconv.Itoa(bc.Subscribers), + ) + }) + }) +} + +// makes a copy of bc +func (bc mqttBenchContext) benchmarkSubRet(b *testing.B) { + // This test uses a built-in publisher, and it makes most sense to measure + // the retained message delivery "overhead" on a QoS0 subscription; without + // the extra time involved in actually subscribing. + m := mqttBenchDefaultMatrix. + NoPublishers(). + QOS0Only() + + b.Run("SUBRET", func(b *testing.B) { + m.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) { + bc.runAndReport(b, "subret", + "--qos", strconv.Itoa(bc.QOS), + "--topics", strconv.Itoa(bc.Topics), // number of retained messages + "--subscribers", strconv.Itoa(bc.Subscribers), + "--size", strconv.Itoa(bc.MessageSize), + "--repeat", strconv.Itoa(b.N), // number of subscribe requests + ) + }) + }) +} + +func (bc mqttBenchContext) runAndReport(b *testing.B, name string, extraArgs ...string) { + b.Helper() + r := mqttRunExCommandTest(b, name, mqttNewDial("", "", bc.Host, bc.Port, ""), extraArgs...) + r.report(b) +} + +func (bc *mqttBenchContext) startServer(b *testing.B, disableRMSCache bool) func() { + b.Helper() + b.StopTimer() + o := testMQTTDefaultOptions() + s := testMQTTRunServer(b, o) + + o = s.getOpts() + bc.Host = o.MQTT.Host + bc.Port = o.MQTT.Port + mqttInitTestServer(b, mqttNewDial("", "", bc.Host, bc.Port, "")) + return func() { + testMQTTShutdownServer(s) + } +} + +func (bc *mqttBenchContext) startCluster(b *testing.B, disableRMSCache bool) func() { + b.Helper() + b.StopTimer() + conf := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + mqtt { + listen: 127.0.0.1:-1 + stream_replicas: 3 + } + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } + ` + + cl := createJetStreamClusterWithTemplate(b, conf, "MQTT", 3) + o := cl.randomNonLeader().getOpts() + bc.Host = o.MQTT.Host + bc.Port = o.MQTT.Port + mqttInitTestServer(b, mqttNewDial("", "", bc.Host, bc.Port, "")) + return func() { + cl.shutdown() + } +} + +func mqttBenchWrapForMatrixField( + vFieldPtr *int, + arr []int, + f func(b *testing.B, bc *mqttBenchContext), + namef func(int) string, +) func(b *testing.B, bc *mqttBenchContext) { + if len(arr) == 0 { + return f + } + return func(b *testing.B, bc *mqttBenchContext) { + for _, value := range arr { + *vFieldPtr = value + b.Run(namef(value), func(b *testing.B) { + f(b, bc) + }) + } + } +} + +func (m mqttBenchMatrix) runMatrix(b *testing.B, bc mqttBenchContext, f func(*testing.B, *mqttBenchContext)) { + b.Helper() + f = mqttBenchWrapForMatrixField(&bc.MessageSize, m.MessageSize, f, func(size int) string { + return sizeKB(size) + }) + f = mqttBenchWrapForMatrixField(&bc.Topics, m.Topics, f, func(n int) string { + return fmt.Sprintf("%dtopics", n) + }) + f = mqttBenchWrapForMatrixField(&bc.Publishers, m.Publishers, f, func(n int) string { + return fmt.Sprintf("%dpubc", n) + }) + f = mqttBenchWrapForMatrixField(&bc.Subscribers, m.Subscribers, f, func(n int) string { + return fmt.Sprintf("%dsubc", n) + }) + f = mqttBenchWrapForMatrixField(&bc.QOS, m.QOS, f, func(qos int) string { + return fmt.Sprintf("QOS%d", qos) + }) + b.ResetTimer() + b.StartTimer() + f(b, &bc) +} + +func (m mqttBenchMatrix) NoSubscribers() mqttBenchMatrix { + m.Subscribers = nil + return m +} + +func (m mqttBenchMatrix) NoTopics() mqttBenchMatrix { + m.Topics = nil + return m +} + +func (m mqttBenchMatrix) NoPublishers() mqttBenchMatrix { + m.Publishers = nil + return m +} + +func (m mqttBenchMatrix) QOS0Only() mqttBenchMatrix { + m.QOS = []int{0} + return m +} + +func (m mqttBenchMatrix) QOS1Only() mqttBenchMatrix { + m.QOS = []int{1} + return m +} + +func sizeKB(size int) string { + unit := "B" + N := size + if size >= KB { + unit = "KB" + N = (N + KB/2) / KB + } + return fmt.Sprintf("%d%s", N, unit) +} + +func (r MQTTBenchmarkResult) report(b *testing.B) { + // Disable the default ns metric in favor of custom X_ms/op. + b.ReportMetric(0, "ns/op") + + // Disable MB/s since the github benchmarking action misinterprets the sign + // of the result (treats it as less is better). + b.SetBytes(0) + // b.SetBytes(r.Bytes) + + for unit, ns := range r.NS { + nsOp := float64(ns) / float64(r.Ops) + b.ReportMetric(nsOp/1000000, unit+"_ms/op") + } + // Diable ReportAllocs() since it confuses the github benchmarking action + // with the noise. + // b.ReportAllocs() +} diff --git a/server/mqtt_ex_test_test.go b/server/mqtt_ex_test_test.go new file mode 100644 index 0000000..c5880f3 --- /dev/null +++ b/server/mqtt_ex_test_test.go @@ -0,0 +1,382 @@ +// Copyright 2024-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !skip_mqtt_tests + +package server + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "strconv" + "strings" + "testing" + + "github.com/nats-io/nuid" +) + +type mqttDial string + +type mqttTarget struct { + singleServers []*Server + clusters []*cluster + configs []mqttTestConfig + all []mqttDial +} + +type mqttTestConfig struct { + name string + pub []mqttDial + sub []mqttDial +} + +func TestXMQTTCompliance(t *testing.T) { + if mqttCLICommandPath == _EMPTY_ { + t.Skip(`"mqtt" command is not found in $PATH nor $MQTT_CLI. See https://hivemq.github.io/mqtt-cli/docs/installation/#debian-package for installation instructions`) + } + + o := testMQTTDefaultOptions() + s := testMQTTRunServer(t, o) + o = s.getOpts() + defer testMQTTShutdownServer(s) + + cmd := exec.Command(mqttCLICommandPath, "test", "-V", "3", "-p", strconv.Itoa(o.MQTT.Port)) + + output, err := cmd.CombinedOutput() + t.Log(string(output)) + if err != nil { + if exitError, ok := err.(*exec.ExitError); ok { + t.Fatalf("mqtt cli exited with error: %v", exitError) + } + } +} + +func TestXMQTTRetainedMessages(t *testing.T) { + if mqttTestCommandPath == _EMPTY_ { + t.Skip(`"mqtt-test" command is not found in $PATH.`) + } + + for _, topo := range []struct { + name string + makef func(testing.TB) *mqttTarget + }{ + { + name: "single server", + makef: mqttMakeTestServer, + }, + { + name: "cluster", + makef: mqttMakeTestCluster(4, ""), + }, + } { + t.Run(topo.name, func(t *testing.T) { + target := topo.makef(t) + t.Cleanup(target.Shutdown) + + // initialize the MQTT assets by "touching" all nodes in the + // cluster, but then reload to start with fresh server state. + for _, dial := range target.all { + mqttInitTestServer(t, dial) + } + + numRMS := 100 + strNumRMS := strconv.Itoa(numRMS) + topics := make([]string, len(target.configs)) + + for i, tc := range target.configs { + // Publish numRMS retained messages one at a time, + // round-robin across pub nodes. Remember the topic for each + // test config to check the subs after reload. + topic := "subret_" + nuid.Next() + topics[i] = topic + iNode := 0 + for i := 0; i < numRMS; i++ { + pubTopic := fmt.Sprintf("%s/%d", topic, i) + dial := tc.pub[iNode%len(tc.pub)] + mqttRunExCommandTest(t, "pub", dial, + "--retain", + "--topic", pubTopic, + "--qos", "0", + "--size", "128", // message size 128 bytes + ) + iNode++ + } + } + + // Check all sub nodes for retained messages + for i, tc := range target.configs { + for _, dial := range tc.sub { + mqttRunExCommandTest(t, "sub", dial, + "--retained", strNumRMS, + "--qos", "0", + "--topic", topics[i], + ) + } + } + + // Reload the target + target.Reload(t) + + // Now check again + for i, tc := range target.configs { + for _, dial := range tc.sub { + mqttRunExCommandTestRetry(t, 1, "sub", dial, + "--retained", strNumRMS, + "--qos", "0", + "--topic", topics[i], + ) + } + } + }) + } +} + +func mqttInitTestServer(tb testing.TB, dial mqttDial) { + tb.Helper() + mqttRunExCommandTestRetry(tb, 5, "pub", dial) +} + +func mqttNewDialForServer(s *Server, username, password string) mqttDial { + o := s.getOpts().MQTT + return mqttNewDial(username, password, o.Host, o.Port, s.Name()) +} + +func mqttNewDial(username, password, host string, port int, comment string) mqttDial { + d := "" + switch { + case username != "" && password != "": + d = fmt.Sprintf("%s:%s@%s:%d", username, password, host, port) + case username != "": + d = fmt.Sprintf("%s@%s:%d", username, host, port) + default: + d = fmt.Sprintf("%s:%d", host, port) + } + if comment != "" { + d += "#" + comment + } + return mqttDial(d) +} + +func (d mqttDial) Get() (u, p, s, c string) { + if d == "" { + return "", "", "127.0.0.1:1883", "" + } + in := string(d) + if i := strings.LastIndex(in, "#"); i != -1 { + c = in[i+1:] + in = in[:i] + } + if i := strings.LastIndex(in, "@"); i != -1 { + up := in[:i] + in = in[i+1:] + u = up + if i := strings.Index(up, ":"); i != -1 { + u = up[:i] + p = up[i+1:] + } + } + s = in + return u, p, s, c +} + +func (d mqttDial) Name() string { + _, _, _, c := d.Get() + return c +} + +func (t *mqttTarget) Reload(tb testing.TB) { + tb.Helper() + + for _, c := range t.clusters { + c.stopAll() + c.restartAllSamePorts() + } + + for i, s := range t.singleServers { + o := s.getOpts() + s.Shutdown() + t.singleServers[i] = testMQTTRunServer(tb, o) + } + + for _, dial := range t.all { + mqttInitTestServer(tb, dial) + } +} + +func (t *mqttTarget) Shutdown() { + for _, c := range t.clusters { + c.shutdown() + } + for _, s := range t.singleServers { + testMQTTShutdownServer(s) + } +} + +func mqttMakeTestServer(tb testing.TB) *mqttTarget { + tb.Helper() + o := testMQTTDefaultOptions() + s := testMQTTRunServer(tb, o) + all := []mqttDial{mqttNewDialForServer(s, "", "")} + return &mqttTarget{ + singleServers: []*Server{s}, + all: all, + configs: []mqttTestConfig{ + { + name: "single server", + pub: all, + sub: all, + }, + }, + } +} + +func mqttMakeTestCluster(size int, domain string) func(tb testing.TB) *mqttTarget { + return func(tb testing.TB) *mqttTarget { + tb.Helper() + if size < 3 { + tb.Fatal("cluster size must be at least 3") + } + + if domain != "" { + domain = "domain: " + domain + ", " + } + clusterConf := ` + listen: 127.0.0.1:-1 + + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, ` + domain + `store_dir: '%s'} + + leafnodes { + listen: 127.0.0.1:-1 + } + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + mqtt { + listen: 127.0.0.1:-1 + stream_replicas: 3 + } + + accounts { + ONE { users = [ { user: "one", pass: "p" } ]; jetstream: enabled } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } +` + cl := createJetStreamClusterWithTemplate(tb, clusterConf, "MQTT", size) + cl.waitOnLeader() + + all := []mqttDial{} + for _, s := range cl.servers { + all = append(all, mqttNewDialForServer(s, "one", "p")) + } + + return &mqttTarget{ + clusters: []*cluster{cl}, + all: all, + configs: []mqttTestConfig{ + { + name: "publish to one", + pub: []mqttDial{ + mqttNewDialForServer(cl.randomServer(), "one", "p"), + }, + sub: all, + }, + { + name: "publish to all", + pub: all, + sub: all, + }, + }, + } + } +} + +var mqttCLICommandPath = func() string { + p := os.Getenv("MQTT_CLI") + if p == "" { + p, _ = exec.LookPath("mqtt") + } + return p +}() + +var mqttTestCommandPath = func() string { + p, _ := exec.LookPath("mqtt-test") + return p +}() + +func mqttRunExCommandTest(tb testing.TB, subCommand string, dial mqttDial, extraArgs ...string) *MQTTBenchmarkResult { + tb.Helper() + return mqttRunExCommandTestRetry(tb, 1, subCommand, dial, extraArgs...) +} + +func mqttRunExCommandTestRetry(tb testing.TB, n int, subCommand string, dial mqttDial, extraArgs ...string) (r *MQTTBenchmarkResult) { + tb.Helper() + var err error + for i := 0; i < n; i++ { + if r, err = mqttTryExCommandTest(tb, subCommand, dial, extraArgs...); err == nil { + return r + } + + if i < (n - 1) { + tb.Logf("failed to %q %s to %q on attempt %v, will retry.", subCommand, extraArgs, dial.Name(), i) + } else { + tb.Fatal(err) + } + } + return nil +} + +func mqttTryExCommandTest(tb testing.TB, subCommand string, dial mqttDial, extraArgs ...string) (r *MQTTBenchmarkResult, err error) { + tb.Helper() + if mqttTestCommandPath == "" { + tb.Skip(`"mqtt-test" command is not found in $PATH.`) + } + + args := []string{subCommand, // "-q", + "-s", string(dial), + } + args = append(args, extraArgs...) + cmd := exec.Command(mqttTestCommandPath, args...) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("error executing %q: %v", cmd.String(), err) + } + defer stdout.Close() + errbuf := bytes.Buffer{} + cmd.Stderr = &errbuf + if err = cmd.Start(); err != nil { + return nil, fmt.Errorf("error executing %q: %v", cmd.String(), err) + } + out, err := io.ReadAll(stdout) + if err != nil { + return nil, fmt.Errorf("error executing %q: failed to read output: %v", cmd.String(), err) + } + if err = cmd.Wait(); err != nil { + return nil, fmt.Errorf("error executing %q: %v\n\n%s\n\n%s", cmd.String(), err, string(out), errbuf.String()) + } + + r = &MQTTBenchmarkResult{} + if err := json.Unmarshal(out, r); err != nil { + tb.Fatalf("error executing %q: failed to decode output: %v\n\n%s\n\n%s", cmd.String(), err, string(out), errbuf.String()) + } + return r, nil +} diff --git a/server/msgtrace.go b/server/msgtrace.go new file mode 100644 index 0000000..e23c800 --- /dev/null +++ b/server/msgtrace.go @@ -0,0 +1,903 @@ +// Copyright 2024-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "encoding/json" + "fmt" + "math/rand/v2" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +const ( + MsgTraceDest = "Nats-Trace-Dest" + MsgTraceDestDisabled = "trace disabled" // This must be an invalid NATS subject + MsgTraceHop = "Nats-Trace-Hop" + MsgTraceOriginAccount = "Nats-Trace-Origin-Account" + MsgTraceOnly = "Nats-Trace-Only" + + // External trace header. Note that this header is normally in lower + // case (https://www.w3.org/TR/trace-context/#header-name). Vendors + // MUST expect the header in any case (upper, lower, mixed), and + // SHOULD send the header name in lowercase. We used to change it + // to lower case, but no longer do that in 2.14. + traceParentHdr = "traceparent" +) + +var ( + traceDestHdrAsBytes = stringToBytes(MsgTraceDest) + traceDestDisabledAsBytes = stringToBytes(MsgTraceDestDisabled) + traceParentHdrAsBytes = stringToBytes(traceParentHdr) + crLFAsBytes = stringToBytes(CR_LF) + dashAsBytes = stringToBytes("-") +) + +type MsgTraceType string + +// Type of message trace events in the MsgTraceEvents list. +// This is needed to unmarshal the list. +const ( + MsgTraceIngressType = "in" + MsgTraceSubjectMappingType = "sm" + MsgTraceStreamExportType = "se" + MsgTraceServiceImportType = "si" + MsgTraceJetStreamType = "js" + MsgTraceEgressType = "eg" +) + +type MsgTraceEvent struct { + Server ServerInfo `json:"server"` + Request MsgTraceRequest `json:"request"` + Hops int `json:"hops,omitempty"` + Events MsgTraceEvents `json:"events"` +} + +type MsgTraceRequest struct { + // We are not making this an http.Header so that header name case is preserved. + Header map[string][]string `json:"header,omitempty"` + MsgSize int `json:"msgsize,omitempty"` +} + +type MsgTraceEvents []MsgTrace + +type MsgTrace interface { + new() MsgTrace + typ() MsgTraceType +} + +type MsgTraceBase struct { + Type MsgTraceType `json:"type"` + Timestamp time.Time `json:"ts"` +} + +type MsgTraceIngress struct { + MsgTraceBase + Kind int `json:"kind"` + CID uint64 `json:"cid"` + Name string `json:"name,omitempty"` + Account string `json:"acc"` + Subject string `json:"subj"` + Error string `json:"error,omitempty"` +} + +type MsgTraceSubjectMapping struct { + MsgTraceBase + MappedTo string `json:"to"` +} + +type MsgTraceStreamExport struct { + MsgTraceBase + Account string `json:"acc"` + To string `json:"to"` +} + +type MsgTraceServiceImport struct { + MsgTraceBase + Account string `json:"acc"` + From string `json:"from"` + To string `json:"to"` +} + +type MsgTraceJetStream struct { + MsgTraceBase + Stream string `json:"stream"` + Subject string `json:"subject,omitempty"` + NoInterest bool `json:"nointerest,omitempty"` + Error string `json:"error,omitempty"` +} + +type MsgTraceEgress struct { + MsgTraceBase + Kind int `json:"kind"` + CID uint64 `json:"cid"` + Name string `json:"name,omitempty"` + Hop string `json:"hop,omitempty"` + Account string `json:"acc,omitempty"` + Subscription string `json:"sub,omitempty"` + Queue string `json:"queue,omitempty"` + Error string `json:"error,omitempty"` + + // This is for applications that unmarshal the trace events + // and want to link an egress to route/leaf/gateway with + // the MsgTraceEvent from that server. + Link *MsgTraceEvent `json:"-"` +} + +// ------------------------------------------------------------- + +func (t MsgTraceBase) typ() MsgTraceType { return t.Type } +func (MsgTraceIngress) new() MsgTrace { return &MsgTraceIngress{} } +func (MsgTraceSubjectMapping) new() MsgTrace { return &MsgTraceSubjectMapping{} } +func (MsgTraceStreamExport) new() MsgTrace { return &MsgTraceStreamExport{} } +func (MsgTraceServiceImport) new() MsgTrace { return &MsgTraceServiceImport{} } +func (MsgTraceJetStream) new() MsgTrace { return &MsgTraceJetStream{} } +func (MsgTraceEgress) new() MsgTrace { return &MsgTraceEgress{} } + +var msgTraceInterfaces = map[MsgTraceType]MsgTrace{ + MsgTraceIngressType: MsgTraceIngress{}, + MsgTraceSubjectMappingType: MsgTraceSubjectMapping{}, + MsgTraceStreamExportType: MsgTraceStreamExport{}, + MsgTraceServiceImportType: MsgTraceServiceImport{}, + MsgTraceJetStreamType: MsgTraceJetStream{}, + MsgTraceEgressType: MsgTraceEgress{}, +} + +func (t *MsgTraceEvents) UnmarshalJSON(data []byte) error { + var raw []json.RawMessage + err := json.Unmarshal(data, &raw) + if err != nil { + return err + } + *t = make(MsgTraceEvents, len(raw)) + var tt MsgTraceBase + for i, r := range raw { + if err = json.Unmarshal(r, &tt); err != nil { + return err + } + tr, ok := msgTraceInterfaces[tt.Type] + if !ok { + return fmt.Errorf("unknown trace type %v", tt.Type) + } + te := tr.new() + if err := json.Unmarshal(r, te); err != nil { + return err + } + (*t)[i] = te + } + return nil +} + +func getTraceAs[T MsgTrace](e any) *T { + v, ok := e.(*T) + if ok { + return v + } + return nil +} + +func (t *MsgTraceEvent) Ingress() *MsgTraceIngress { + if len(t.Events) < 1 { + return nil + } + return getTraceAs[MsgTraceIngress](t.Events[0]) +} + +func (t *MsgTraceEvent) SubjectMapping() *MsgTraceSubjectMapping { + for _, e := range t.Events { + if e.typ() == MsgTraceSubjectMappingType { + return getTraceAs[MsgTraceSubjectMapping](e) + } + } + return nil +} + +func (t *MsgTraceEvent) StreamExports() []*MsgTraceStreamExport { + var se []*MsgTraceStreamExport + for _, e := range t.Events { + if e.typ() == MsgTraceStreamExportType { + se = append(se, getTraceAs[MsgTraceStreamExport](e)) + } + } + return se +} + +func (t *MsgTraceEvent) ServiceImports() []*MsgTraceServiceImport { + var si []*MsgTraceServiceImport + for _, e := range t.Events { + if e.typ() == MsgTraceServiceImportType { + si = append(si, getTraceAs[MsgTraceServiceImport](e)) + } + } + return si +} + +func (t *MsgTraceEvent) JetStream() *MsgTraceJetStream { + for _, e := range t.Events { + if e.typ() == MsgTraceJetStreamType { + return getTraceAs[MsgTraceJetStream](e) + } + } + return nil +} + +func (t *MsgTraceEvent) Egresses() []*MsgTraceEgress { + var eg []*MsgTraceEgress + for _, e := range t.Events { + if e.typ() == MsgTraceEgressType { + eg = append(eg, getTraceAs[MsgTraceEgress](e)) + } + } + return eg +} + +const ( + errMsgTraceOnlyNoSupport = "Not delivered because remote does not support message tracing" + errMsgTraceNoSupport = "Message delivered but remote does not support message tracing so no trace event generated from there" + errMsgTraceNoEcho = "Not delivered because of no echo" + errMsgTracePubViolation = "Not delivered because publish denied for this subject" + errMsgTraceSubDeny = "Not delivered because subscription denies this subject" + errMsgTraceSubClosed = "Not delivered because subscription is closed" + errMsgTraceClientClosed = "Not delivered because client is closed" + errMsgTraceAutoSubExceeded = "Not delivered because auto-unsubscribe exceeded" + errMsgTraceFastProdNoStall = "Not delivered because fast producer not stalled and consumer is slow" +) + +type msgTrace struct { + kind int + ready int32 + srv *Server + acc *Account + dest string + event *MsgTraceEvent + js *MsgTraceJetStream + siMu sync.RWMutex + si []*msgTraceServiceImport + rsi *msgTraceRespServiceImport + hop string + nhop string + tonly bool // Will only trace the message, not do delivery. + ct compressionType +} + +type msgTraceServiceImport struct { + si *serviceImport + acc *Account +} + +type msgTraceRespServiceImport struct { + hops int + received int + tree map[string]*msgTraceRespServiceImport +} + +// This will be false outside of the tests, so when building the server binary, +// any code where you see `if msgTraceRunInTests` statement will be compiled +// out, so this will have no performance penalty. +var ( + msgTraceRunInTests bool + msgTraceCheckSupport bool +) + +// Returns the message trace object, if message is being traced, +// and `true` if we want to only trace, not actually deliver the message. +func (c *client) isMsgTraceEnabled() (*msgTrace, bool) { + t := c.pa.trace + if t == nil { + return nil, false + } + return t, t.tonly +} + +// For LEAF/ROUTER/GATEWAY, return false if the remote does not support +// message tracing (important if the tracing requests trace-only). +func (c *client) msgTraceSupport() bool { + // Exclude client connection from the protocol check. + return c.kind == CLIENT || c.opts.Protocol >= MsgTraceProto +} + +func getConnName(c *client) string { + switch c.kind { + case ROUTER: + if n := c.route.remoteName; n != _EMPTY_ { + return n + } + case GATEWAY: + if n := c.gw.remoteName; n != _EMPTY_ { + return n + } + case LEAF: + if n := c.leaf.remoteServer; n != _EMPTY_ { + return n + } + } + return c.opts.Name +} + +func getCompressionType(cts string) compressionType { + if cts == _EMPTY_ { + return noCompression + } + cts = strings.ToLower(cts) + if strings.Contains(cts, "snappy") || strings.Contains(cts, "s2") { + return snappyCompression + } + if strings.Contains(cts, "gzip") { + return gzipCompression + } + return unsupportedCompression +} + +// Possibly initialize message tracing if appropriate headers are found in the +// given `hdr` byte slice. +// If there is an error and the header `MsgTraceOnly` is set to `true` (indicating +// that it should be a "trace only" message, without message delivery), then this +// function will return `true, nil`, so that the caller can skip message processing. +func (c *client) initMsgTrace(hdr []byte, ingressError error) (bool, *msgTrace) { + // The code in the "if" statement is only running in test mode. + if msgTraceRunInTests { + // Check the type of client that tries to initialize a trace struct. + if !(c.kind == CLIENT || c.kind == ROUTER || c.kind == GATEWAY || c.kind == LEAF) { + panic(fmt.Sprintf("Unexpected client type %q trying to initialize msgTrace", c.kindString())) + } + // In some tests, we want to make a server behave like an old server + // and so even if a trace header is received, we want the server to + // simply ignore it. + if msgTraceCheckSupport { + if c.srv == nil || c.srv.getServerProto() < MsgTraceProto { + return false, nil + } + } + } + // Get the headers if we find `Nats-Trace-Dest` or `traceparent` header. + // For `traceparent`, `external` will be true indicating that we need + // to get the destination and sampling from the account. + headers, external := genHeaderMapIfTraceHeadersPresent(hdr) + if len(headers) == 0 { + return false, nil + } + // Little helper to give us the first value of a given header, or _EMPTY_ + // if key is not present. + getHdrVal := func(key string) string { + vv, ok := headers[key] + if !ok { + return _EMPTY_ + } + return vv[0] + } + var ( + dest string // The destination the trace message should be sent to + traceOnly bool // True if this is "trace only" and no message delivery should occur + hop string // The hop "id", taken from headers only when not from CLIENT + kind = c.kind // The type of connection this originates from + ) + // Check for traceOnly only if not external. + if !external { + if to := getHdrVal(MsgTraceOnly); to != _EMPTY_ { + tos := strings.ToLower(to) + switch tos { + case "1", "true", "on": + traceOnly = true + } + } + dest = getHdrVal(MsgTraceDest) + // If no dest, bail out. + if dest == _EMPTY_ { + return traceOnly, nil + } + } + // Now that we know if there is `traceOnly` set or not, if we were to fail + // here by returning `nil` for `*msgTrace`, we will return `true, nil` to + // indicate to the caller that it should skip message processing (so that + // the failed trace message is not delivered to regular subscriptions). + + // First, disable tracing if the message contains the old MsgTraceOriginAccount header. + if v := getHdrVal(MsgTraceOriginAccount); v != _EMPTY_ { + c.Errorf("Message tracing disabled because of header %q with value=%q", + MsgTraceOriginAccount, v) + // Skip this if we are invoked with ingressError, or if we are going to skip + // message processing anyway (if `traceOnly` is true). + if ingressError == nil && !traceOnly { + // Disable tracing by changing the `MsgTraceDest` header to `MsgTraceDestDisabled`. + c.msgBuf = c.setHeader(MsgTraceDest, MsgTraceDestDisabled, c.msgBuf) + } + // If `traceOnly` is false, the message will be delivered and possibly routed, + // but it will not be traced. If it is `true`, then there will be no tracing + // and the message will not be processed (so no chance to be routed). + return traceOnly, nil + } + + // Get the account and server. + c.mu.Lock() + acc := c.acc + srv := c.srv + c.mu.Unlock() + // There should always be a server object, and for CLIENT and LEAF, the + // account should be available. + if srv == nil || (acc == nil && (kind == CLIENT || kind == LEAF)) { + // Make the caller skip message procesing if `traceOnly` is true. + return traceOnly, nil + } + // For non CLIENT connection, get the account and hop header. + if kind != CLIENT { + hop = getHdrVal(MsgTraceHop) + if kind != LEAF { + // Lookup the account, if not present, bail out. + if acci, ok := c.srv.accounts.Load(string(c.pa.account)); ok { + acc = acci.(*Account) + } else { + c.Errorf("Account %q was not found, won't be able to trace events", c.pa.account) + return traceOnly, nil + } + } + } + // If external, we need to have the account's trace destination set, + // otherwise, we are not enabling tracing. + if external { + var sampling int + dest, sampling = acc.getTraceDestAndSampling() + if dest == _EMPTY_ { + // No account destination, no tracing for external trace headers. + // This is not an error, so return false so that we don't skip + // message processing. + return false, nil + } + // Check sampling, but only from origin server. + if kind == CLIENT && !sample(sampling) { + // Need to disable tracing so that if the message is routed, it won't + // trigger a trace there. Skip this if we have an ingress error. + if ingressError == nil { + c.msgBuf = c.setHeader(MsgTraceDest, MsgTraceDestDisabled, c.msgBuf) + } + return false, nil + } + } + // Check the destination to see if this is a valid publish subject. + if !IsValidPublishSubject(dest) { + c.Errorf("Destination %q is not valid, won't be able to trace events", dest) + // Return `traceOnly` to cause the caller to skip message processing if + // this was supposed to be a trace only message. + return traceOnly, nil + } + // Now that we have a valid `dest`, make sure this connection is allowed + // to publish to it. + if !c.allowedToPublishOnMsgTraceDest(srv, acc, dest) { + // Send the error back only if CLIENT or LEAF, otherwise just log. + if kind == CLIENT || kind == LEAF { + c.pubPermissionViolation(stringToBytes(dest)) + } else { + c.Errorf("Publish Violation - Subject %q", dest) + } + // Return `true, nil` to force skipping of message processing. + return true, nil + } + c.pa.trace = &msgTrace{ + kind: kind, + srv: c.srv, + acc: acc, + dest: dest, + ct: getCompressionType(getHdrVal(acceptEncodingHeader)), + hop: hop, + event: &MsgTraceEvent{ + Request: MsgTraceRequest{ + Header: headers, + MsgSize: c.pa.size, + }, + Events: append(MsgTraceEvents(nil), &MsgTraceIngress{ + MsgTraceBase: MsgTraceBase{ + Type: MsgTraceIngressType, + Timestamp: time.Now(), + }, + Kind: kind, + CID: c.cid, + Name: getConnName(c), + Account: acc.GetName(), + Subject: string(c.pa.subject), + }), + }, + tonly: traceOnly, + } + // If we are invoked with an ingress error, set it now. + if ingressError != nil { + c.pa.trace.event.Events[0].(*MsgTraceIngress).Error = ingressError.Error() + } + return false, c.pa.trace +} + +func (c *client) allowedToPublishOnMsgTraceDest(s *Server, acc *Account, dest string) bool { + td := stringToBytes(dest) + if hasGWRoutedReplyPrefix(td) { + return false + } + if bytes.HasPrefix(td, clientNRGPrefix) && acc != s.SystemAccount() { + return false + } + allowed := true + c.mu.Lock() + if c.kind == LEAF { + if c.isSpokeLeafNode() { + allowed = c.leafReceiveAllowed(td) + } else { + allowed = c.leafSendAllowed(td) + } + } else if c.perms != nil && (c.perms.pub.allow != nil || c.perms.pub.deny != nil) && !c.pubAllowedFullCheck(dest, false, true) { + allowed = false + } + c.mu.Unlock() + return allowed +} + +func sample(sampling int) bool { + // Option parsing should ensure that sampling is [1..100], but consider + // any value outside of this range to be 100%. + if sampling <= 0 || sampling >= 100 { + return true + } + return rand.Int32N(100) <= int32(sampling) +} + +// This function will return the header as a map (instead of http.Header because +// we want to preserve the header names' case) and a boolean that indicates if +// the headers have been lifted due to the presence of the external trace header +// only. +// Note that because of the traceParentHdr, the search is done in a case +// insensitive way. We used to rewrite it in lower case but no longer do since v2.14. +func genHeaderMapIfTraceHeadersPresent(hdr []byte) (map[string][]string, bool) { + + var ( + _keys = [64][]byte{} + _vals = [64][]byte{} + m map[string][]string + traceDestHdrFound bool + traceParentHdrFound bool + ) + // Skip the hdrLine + if !bytes.HasPrefix(hdr, stringToBytes(hdrLine)) { + return nil, false + } + + keys := _keys[:0] + vals := _vals[:0] + + for i := len(hdrLine); i < len(hdr); { + // Search for key/val delimiter + del := bytes.IndexByte(hdr[i:], ':') + if del < 0 { + break + } + keyStart := i + key := hdr[keyStart : keyStart+del] + i += del + 1 + for i < len(hdr) && (hdr[i] == ' ' || hdr[i] == '\t') { + i++ + } + valStart := i + nl := bytes.Index(hdr[valStart:], crLFAsBytes) + if nl < 0 { + break + } + valEnd := valStart + nl + for valEnd > valStart && (hdr[valEnd-1] == ' ' || hdr[valEnd-1] == '\t') { + valEnd-- + } + val := hdr[valStart:valEnd] + if len(key) > 0 && len(val) > 0 { + vals = append(vals, val) + + // We search for our special keys only if not already found. + + // Check for the external trace header. + // Search needs to be case insensitive. + if !traceParentHdrFound && bytes.EqualFold(key, traceParentHdrAsBytes) { + // We will now check if the value has sampling or not. + // TODO(ik): Not sure if this header can have multiple values + // or not, and if so, what would be the rule to check for + // sampling. What is done here is to check them all until we + // found one with sampling. + tk := bytes.Split(val, dashAsBytes) + if len(tk) == 4 && len([]byte(tk[3])) == 2 { + if hexVal, err := strconv.ParseInt(bytesToString(tk[3]), 16, 8); err == nil { + if hexVal&0x1 == 0x1 { + traceParentHdrFound = true + } + } + } + } else if !traceDestHdrFound && bytes.Equal(key, traceDestHdrAsBytes) { + // This is the Nats-Trace-Dest header, check the value to see + // if it indicates that the trace was disabled. + if bytes.Equal(val, traceDestDisabledAsBytes) { + return nil, false + } + traceDestHdrFound = true + } + // Add to the keys and preserve the key's case + keys = append(keys, key) + } + i += nl + 2 + } + if !traceDestHdrFound && !traceParentHdrFound { + return nil, false + } + m = make(map[string][]string, len(keys)) + for i, k := range keys { + hname := string(k) + m[hname] = append(m[hname], string(vals[i])) + } + return m, !traceDestHdrFound && traceParentHdrFound +} + +// Special case where we create a trace event before parsing the message. +// This is for cases where the connection will be closed when detecting +// an error during early message processing (for instance max payload). +func (c *client) sendMsgTraceIngressErrEvent(hdr []byte, ingressError error) { + if ingressError == nil { + return + } + if _, t := c.initMsgTrace(hdr, ingressError); t != nil { + t.sendEvent() + } +} + +// Returns `true` if message tracing is enabled and we are tracing only, +// that is, we are not going to deliver the inbound message, returns +// `false` otherwise (no tracing, or tracing and message delivery). +func (t *msgTrace) traceOnly() bool { + return t != nil && t.tonly +} + +func (t *msgTrace) setHopHeader(c *client, msg []byte) []byte { + e := t.event + e.Hops++ + if len(t.hop) > 0 { + t.nhop = fmt.Sprintf("%s.%d", t.hop, e.Hops) + } else { + t.nhop = fmt.Sprintf("%d", e.Hops) + } + if t.kind == CLIENT && strings.HasPrefix(t.dest, replyPrefix) { + t.siMu.Lock() + if t.rsi == nil { + t.rsi = &msgTraceRespServiceImport{} + } + t.rsi.hops++ + t.siMu.Unlock() + } + return c.setHeader(MsgTraceHop, t.nhop, msg) +} + +func (t *msgTrace) setIngressError(err string) { + if i := t.event.Ingress(); i != nil { + i.Error = err + } +} + +func (t *msgTrace) addSubjectMappingEvent(subj []byte) { + if t == nil { + return + } + t.event.Events = append(t.event.Events, &MsgTraceSubjectMapping{ + MsgTraceBase: MsgTraceBase{ + Type: MsgTraceSubjectMappingType, + Timestamp: time.Now(), + }, + MappedTo: string(subj), + }) +} + +func (t *msgTrace) addEgressEvent(dc *client, sub *subscription, err string) { + if t == nil { + return + } + e := &MsgTraceEgress{ + MsgTraceBase: MsgTraceBase{ + Type: MsgTraceEgressType, + Timestamp: time.Now(), + }, + Kind: dc.kind, + CID: dc.cid, + Name: getConnName(dc), + Hop: t.nhop, + Error: err, + } + t.nhop = _EMPTY_ + // Specific to CLIENT connections... + if dc.kind == CLIENT { + // Set the subscription's subject and possibly queue name. + e.Subscription = string(sub.subject) + if len(sub.queue) > 0 { + e.Queue = string(sub.queue) + } + } + if dc.kind == CLIENT || dc.kind == LEAF { + if i := t.event.Ingress(); i != nil { + // If the Ingress' account is different from the destination's + // account, add the account name into the Egress trace event. + // This would happen with service imports. + if dcAccName := dc.acc.GetName(); dcAccName != i.Account { + e.Account = dcAccName + } + } + } + t.event.Events = append(t.event.Events, e) +} + +func (t *msgTrace) addStreamExportEvent(dc *client, to []byte) { + if t == nil { + return + } + dc.mu.Lock() + accName := dc.acc.GetName() + dc.mu.Unlock() + t.event.Events = append(t.event.Events, &MsgTraceStreamExport{ + MsgTraceBase: MsgTraceBase{ + Type: MsgTraceStreamExportType, + Timestamp: time.Now(), + }, + Account: accName, + To: string(to), + }) +} + +func (t *msgTrace) addServiceImportEvent(accName, from, to string) { + if t == nil { + return + } + t.event.Events = append(t.event.Events, &MsgTraceServiceImport{ + MsgTraceBase: MsgTraceBase{ + Type: MsgTraceServiceImportType, + Timestamp: time.Now(), + }, + Account: accName, + From: from, + To: to, + }) +} + +func (t *msgTrace) addJetStreamEvent(streamName string) { + if t == nil { + return + } + t.js = &MsgTraceJetStream{ + MsgTraceBase: MsgTraceBase{ + Type: MsgTraceJetStreamType, + Timestamp: time.Now(), + }, + Stream: streamName, + } + t.event.Events = append(t.event.Events, t.js) +} + +func (t *msgTrace) updateJetStreamEvent(subject string, noInterest bool) { + if t == nil { + return + } + // JetStream event should have been created in addJetStreamEvent + if t.js == nil { + return + } + t.js.Subject = subject + t.js.NoInterest = noInterest + // Update the timestamp since this is more accurate than when it + // was first added in addJetStreamEvent(). + t.js.Timestamp = time.Now() +} + +func (t *msgTrace) sendEventFromJetStream(err error) { + if t == nil { + return + } + // JetStream event should have been created in addJetStreamEvent + if t.js == nil { + return + } + if err != nil { + t.js.Error = err.Error() + } + t.sendEvent() +} + +func (t *msgTrace) sendEvent() { + if t == nil { + return + } + if t.js != nil { + ready := atomic.AddInt32(&t.ready, 1) == 2 + if !ready { + return + } + } + t.srv.sendInternalAccountSysMsg(t.acc, t.dest, &t.event.Server, t.event, t.ct) +} + +func (t *msgTrace) setupResponseServiceImport(c *client, acc *Account, si *serviceImport, msg []byte) (*serviceImport, []byte) { + rsi := si.acc.addRespServiceImport(acc, t.dest, si, false, nil, t) + t.dest = rsi.from + t.siMu.Lock() + t.si = append(t.si, &msgTraceServiceImport{rsi, si.acc}) + t.siMu.Unlock() + return rsi, c.setHeader(MsgTraceDest, t.dest, msg) +} + +func (t *msgTrace) handleRespServiceImport(e *MsgTraceEvent) { + t.siMu.Lock() + defer t.siMu.Unlock() + + // If response service import were created and routed for message traces, + // we should have t.rsi created. If it is not, we are done. Note that + // if for any reason we bail out because we are not in a state that we + // expect, the response service imports will be cleaned-up on a timer based. + if t.rsi == nil { + return + } + t.updateRespServiceImport(e) + + if t.allRespServiceImportReceived(t.rsi) { + for _, rsi := range t.si { + rsi.acc.removeRespServiceImport(rsi.si, rsiOk) + } + t.rsi, t.si = nil, nil + } +} + +// For a given trace message event, update the tree of response service import +// trace messages. +// Lock is held on entry. +func (t *msgTrace) updateRespServiceImport(e *MsgTraceEvent) { + // Check for the hop header. + hop, ok := e.Request.Header[MsgTraceHop] + if !ok || len(hop) != 1 { + return + } + hops := strings.Split(hop[0], ".") + prsi := t.rsi + for i, h := range hops { + rsi := prsi.tree[h] + if rsi == nil { + if prsi.tree == nil { + prsi.tree = make(map[string]*msgTraceRespServiceImport) + } + // Create an entry and initializes the hops count to -1. It will be set + // when receiving the corresponding trace message. + rsi = &msgTraceRespServiceImport{hops: -1} + // Bind it to the parent tree for this "hop" id. + prsi.tree[h] = rsi + } + // When dealing with the last section of the `hop` string, we set the + // expected hops count based on the event's `Hops` field and bump the number + // of received trace messages on the parent's node. We do this only once + // per event (use rsi.hops == -1 as the indicator). + if rsi.hops == -1 && i == len(hops)-1 { + rsi.hops = e.Hops + prsi.received++ + } + prsi = rsi + } +} + +// Determine if all response service import traces have been received. +// Lock held on entry. +func (t *msgTrace) allRespServiceImportReceived(prsi *msgTraceRespServiceImport) bool { + if prsi.hops != prsi.received { + return false + } + for _, rsi := range prsi.tree { + if !t.allRespServiceImportReceived(rsi) { + return false + } + } + return true +} diff --git a/server/nkey.go b/server/nkey.go new file mode 100644 index 0000000..97195b0 --- /dev/null +++ b/server/nkey.go @@ -0,0 +1,47 @@ +// Copyright 2018-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + crand "crypto/rand" + "encoding/base64" +) + +// Raw length of the nonce challenge +const ( + nonceRawLen = 11 + nonceLen = 15 // base64.RawURLEncoding.EncodedLen(nonceRawLen) +) + +// NonceRequired tells us if we should send a nonce. +func (s *Server) NonceRequired() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.nonceRequired() +} + +// nonceRequired tells us if we should send a nonce. +// Lock should be held on entry. +func (s *Server) nonceRequired() bool { + return s.getOpts().AlwaysEnableNonce || len(s.nkeys) > 0 || s.trustedKeys != nil || len(s.proxiesKeyPairs) > 0 +} + +// Generate a nonce for INFO challenge. +// Assumes server lock is held +func (s *Server) generateNonce(n []byte) { + var raw [nonceRawLen]byte + data := raw[:] + crand.Read(data) + base64.RawURLEncoding.Encode(n, data) +} diff --git a/server/nkey_test.go b/server/nkey_test.go new file mode 100644 index 0000000..3f7eda0 --- /dev/null +++ b/server/nkey_test.go @@ -0,0 +1,303 @@ +// Copyright 2018-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bufio" + "encoding/base64" + "encoding/json" + "fmt" + "strings" + "testing" + + crand "crypto/rand" + mrand "math/rand/v2" + + "github.com/nats-io/nkeys" +) + +// Nonce has to be a string since we used different encoding by default than json.Unmarshal. +type nonceInfo struct { + Id string `json:"server_id"` + CID uint64 `json:"client_id,omitempty"` + Nonce string `json:"nonce,omitempty"` +} + +// This is a seed for a user. We can extract public and private keys from this for testing. +var seed = []byte("SUAKYRHVIOREXV7EUZTBHUHL7NUMHPMAS7QMDU3GTIUWEI5LDNOXD43IZY") + +func nkeyBasicSetup() (*Server, *testAsyncClient, *bufio.Reader, string) { + kp, _ := nkeys.FromSeed(seed) + pub, _ := kp.PublicKey() + opts := defaultServerOptions + opts.Nkeys = []*NkeyUser{{Nkey: string(pub)}} + return rawSetup(opts) +} + +func mixedSetup() (*Server, *testAsyncClient, *bufio.Reader, string) { + kp, _ := nkeys.FromSeed(seed) + pub, _ := kp.PublicKey() + opts := defaultServerOptions + opts.Nkeys = []*NkeyUser{{Nkey: string(pub)}} + opts.Users = []*User{{Username: "derek", Password: "foo"}} + return rawSetup(opts) +} + +func TestServerInfoNonceAlwaysEnabled(t *testing.T) { + opts := defaultServerOptions + opts.AlwaysEnableNonce = true + s, c, _, l := rawSetup(opts) + defer s.WaitForShutdown() + defer s.Shutdown() + defer c.close() + + if !strings.HasPrefix(l, "INFO ") { + t.Fatalf("INFO response incorrect: %s\n", l) + } + + var info nonceInfo + err := json.Unmarshal([]byte(l[5:]), &info) + if err != nil { + t.Fatalf("Could not parse INFO json: %v\n", err) + } + if info.Nonce == "" { + t.Fatalf("Expected a non-empty nonce with AlwaysEnableNonce set") + } +} + +func TestServerInfoNonce(t *testing.T) { + c, l := setUpClientWithResponse() + defer c.close() + if !strings.HasPrefix(l, "INFO ") { + t.Fatalf("INFO response incorrect: %s\n", l) + } + // Make sure payload is proper json + var info nonceInfo + err := json.Unmarshal([]byte(l[5:]), &info) + if err != nil { + t.Fatalf("Could not parse INFO json: %v\n", err) + } + if info.Nonce != "" { + t.Fatalf("Expected an empty nonce with no nkeys defined") + } + + // Now setup server with auth and nkeys to trigger nonce generation + s, c, _, l := nkeyBasicSetup() + defer c.close() + + if !strings.HasPrefix(l, "INFO ") { + t.Fatalf("INFO response incorrect: %s\n", l) + } + // Make sure payload is proper json + err = json.Unmarshal([]byte(l[5:]), &info) + if err != nil { + t.Fatalf("Could not parse INFO json: %v\n", err) + } + if info.Nonce == "" { + t.Fatalf("Expected a non-empty nonce with nkeys defined") + } + + // Make sure new clients get new nonces + oldNonce := info.Nonce + + c, _, l = newClientForServer(s) + defer c.close() + + err = json.Unmarshal([]byte(l[5:]), &info) + if err != nil { + t.Fatalf("Could not parse INFO json: %v\n", err) + } + if info.Nonce == "" { + t.Fatalf("Expected a non-empty nonce") + } + if strings.Compare(oldNonce, info.Nonce) == 0 { + t.Fatalf("Expected subsequent nonces to be different\n") + } +} + +func TestNkeyClientConnect(t *testing.T) { + s, c, cr, _ := nkeyBasicSetup() + defer c.close() + // Send CONNECT with no signature or nkey, should fail. + connectOp := "CONNECT {\"verbose\":true,\"pedantic\":true}\r\n" + c.parseAsync(connectOp) + l, _ := cr.ReadString('\n') + if !strings.HasPrefix(l, "-ERR ") { + t.Fatalf("Expected an error") + } + + kp, _ := nkeys.FromSeed(seed) + pubKey, _ := kp.PublicKey() + + // Send nkey but no signature + c, cr, _ = newClientForServer(s) + defer c.close() + cs := fmt.Sprintf("CONNECT {\"nkey\":%q, \"verbose\":true,\"pedantic\":true}\r\n", pubKey) + c.parseAsync(cs) + l, _ = cr.ReadString('\n') + if !strings.HasPrefix(l, "-ERR ") { + t.Fatalf("Expected an error") + } + + // Now improperly sign etc. + c, cr, _ = newClientForServer(s) + defer c.close() + cs = fmt.Sprintf("CONNECT {\"nkey\":%q,\"sig\":%q,\"verbose\":true,\"pedantic\":true}\r\n", pubKey, "bad_sig") + c.parseAsync(cs) + l, _ = cr.ReadString('\n') + if !strings.HasPrefix(l, "-ERR ") { + t.Fatalf("Expected an error") + } + + // Now properly sign the nonce + c, cr, l = newClientForServer(s) + defer c.close() + // Check for Nonce + var info nonceInfo + err := json.Unmarshal([]byte(l[5:]), &info) + if err != nil { + t.Fatalf("Could not parse INFO json: %v\n", err) + } + if info.Nonce == "" { + t.Fatalf("Expected a non-empty nonce with nkeys defined") + } + sigraw, err := kp.Sign([]byte(info.Nonce)) + if err != nil { + t.Fatalf("Failed signing nonce: %v", err) + } + sig := base64.RawURLEncoding.EncodeToString(sigraw) + + // PING needed to flush the +OK to us. + cs = fmt.Sprintf("CONNECT {\"nkey\":%q,\"sig\":\"%s\",\"verbose\":true,\"pedantic\":true}\r\nPING\r\n", pubKey, sig) + c.parseAsync(cs) + l, _ = cr.ReadString('\n') + if !strings.HasPrefix(l, "+OK") { + t.Fatalf("Expected an OK, got: %v", l) + } +} + +func TestMixedClientConnect(t *testing.T) { + s, c, cr, _ := mixedSetup() + defer c.close() + // Normal user/pass + c.parseAsync("CONNECT {\"user\":\"derek\",\"pass\":\"foo\",\"verbose\":true,\"pedantic\":true}\r\nPING\r\n") + l, _ := cr.ReadString('\n') + if !strings.HasPrefix(l, "+OK") { + t.Fatalf("Expected an OK, got: %v", l) + } + + kp, _ := nkeys.FromSeed(seed) + pubKey, _ := kp.PublicKey() + + c, cr, l = newClientForServer(s) + defer c.close() + // Check for Nonce + var info nonceInfo + err := json.Unmarshal([]byte(l[5:]), &info) + if err != nil { + t.Fatalf("Could not parse INFO json: %v\n", err) + } + if info.Nonce == "" { + t.Fatalf("Expected a non-empty nonce with nkeys defined") + } + sigraw, err := kp.Sign([]byte(info.Nonce)) + if err != nil { + t.Fatalf("Failed signing nonce: %v", err) + } + sig := base64.RawURLEncoding.EncodeToString(sigraw) + + // PING needed to flush the +OK to us. + cs := fmt.Sprintf("CONNECT {\"nkey\":%q,\"sig\":\"%s\",\"verbose\":true,\"pedantic\":true}\r\nPING\r\n", pubKey, sig) + c.parseAsync(cs) + l, _ = cr.ReadString('\n') + if !strings.HasPrefix(l, "+OK") { + t.Fatalf("Expected an OK, got: %v", l) + } +} + +func TestMixedClientConfig(t *testing.T) { + confFileName := createConfFile(t, []byte(` + authorization { + users = [ + {nkey: "UDKTV7HZVYJFJN64LLMYQBUR6MTNNYCDC3LAZH4VHURW3GZLL3FULBXV"} + {user: alice, password: foo} + ] + }`)) + opts, err := ProcessConfigFile(confFileName) + if err != nil { + t.Fatalf("Received an error processing config file: %v", err) + } + if len(opts.Nkeys) != 1 { + t.Fatalf("Expected 1 nkey, got %d", len(opts.Nkeys)) + } + if len(opts.Users) != 1 { + t.Fatalf("Expected 1 user, got %d", len(opts.Users)) + } +} + +func BenchmarkCryptoRandGeneration(b *testing.B) { + data := make([]byte, 16) + for i := 0; i < b.N; i++ { + crand.Read(data) + } +} + +func BenchmarkMathRandGeneration(b *testing.B) { + data := make([]byte, 16) + prng := mrand.NewChaCha8([32]byte{42}) + for i := 0; i < b.N; i++ { + prng.Read(data) + } +} + +func BenchmarkNonceGeneration(b *testing.B) { + data := make([]byte, nonceRawLen) + b64 := make([]byte, nonceLen) + prand := mrand.NewChaCha8([32]byte{42}) + for i := 0; i < b.N; i++ { + prand.Read(data) + base64.RawURLEncoding.Encode(b64, data) + } +} + +func BenchmarkPublicVerify(b *testing.B) { + data := make([]byte, nonceRawLen) + nonce := make([]byte, nonceLen) + crand.Read(data) + base64.RawURLEncoding.Encode(nonce, data) + + user, err := nkeys.CreateUser() + if err != nil { + b.Fatalf("Error creating User Nkey: %v", err) + } + sig, err := user.Sign(nonce) + if err != nil { + b.Fatalf("Error sigining nonce: %v", err) + } + pk, err := user.PublicKey() + if err != nil { + b.Fatalf("Could not extract public key from user: %v", err) + } + pub, err := nkeys.FromPublicKey(pk) + if err != nil { + b.Fatalf("Could not create public key pair from public key string: %v", err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := pub.Verify(nonce, sig); err != nil { + b.Fatalf("Error verifying nonce: %v", err) + } + } +} diff --git a/server/norace_2_test.go b/server/norace_2_test.go new file mode 100644 index 0000000..bd3294c --- /dev/null +++ b/server/norace_2_test.go @@ -0,0 +1,3927 @@ +// Copyright 2018-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !race && !skip_no_race_tests && !skip_no_race_2_tests + +package server + +import ( + "bytes" + "compress/flate" + "context" + "encoding/json" + "errors" + "fmt" + "math/rand/v2" + "net" + "os" + "path/filepath" + "reflect" + "runtime" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + crand "crypto/rand" + + "github.com/nats-io/nats-server/v2/server/gsl" + "github.com/nats-io/nats.go" + "github.com/nats-io/nuid" +) + +// IMPORTANT: Tests in this file are not executed when running with the -race flag. +// The test name should be prefixed with TestNoRace so we can run only +// those tests: go test -run=TestNoRace ... + +func TestNoRaceJetStreamClusterLeafnodeConnectPerf(t *testing.T) { + // Uncomment to run. Needs to be on a big machine. Do not want as part of Travis tests atm. + skip(t) + + tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: cloud, store_dir:", 1) + c := createJetStreamCluster(t, tmpl, "CLOUD", _EMPTY_, 3, 18033, true) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "STATE", + Subjects: []string{"STATE.GLOBAL.CELL1.*.>"}, + Replicas: 3, + }) + require_NoError(t, err) + + tmpl = strings.Replace(jsClusterTemplWithSingleFleetLeafNode, "store_dir:", "domain: vehicle, store_dir:", 1) + + var vinSerial int + genVIN := func() string { + vinSerial++ + return fmt.Sprintf("7PDSGAALXNN%06d", vinSerial) + } + + numVehicles := 500 + for i := 0; i < numVehicles; i++ { + start := time.Now() + vin := genVIN() + ln := c.createLeafNodeWithTemplateNoSystemWithProto(vin, tmpl, "ws") + nc, js := jsClientConnect(t, ln) + _, err := js.AddStream(&nats.StreamConfig{ + Name: "VEHICLE", + Subjects: []string{"STATE.GLOBAL.LOCAL.>"}, + Sources: []*nats.StreamSource{{ + Name: "STATE", + FilterSubject: fmt.Sprintf("STATE.GLOBAL.CELL1.%s.>", vin), + External: &nats.ExternalStream{ + APIPrefix: "$JS.cloud.API", + DeliverPrefix: fmt.Sprintf("DELIVER.STATE.GLOBAL.CELL1.%s", vin), + }, + }}, + }) + require_NoError(t, err) + // Create the sourced stream. + checkLeafNodeConnectedCount(t, ln, 1) + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Fatalf("Took too long to create leafnode %d connection: %v", i+1, elapsed) + } + nc.Close() + } +} + +func TestNoRaceJetStreamClusterDifferentRTTInterestBasedStreamPreAck(t *testing.T) { + tmpl := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + cluster { + name: "F3" + listen: 127.0.0.1:%d + routes = [%s] + } + + accounts { + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } + ` + + // Route Ports + // "S1": 14622, + // "S2": 15622, + // "S3": 16622, + + // S2 (stream leader) will have a slow path to S1 (via proxy) and S3 (consumer leader) will have a fast path. + + // Do these in order, S1, S2 (proxy) then S3. + c := &cluster{t: t, servers: make([]*Server, 3), opts: make([]*Options, 3), name: "F3"} + + // S1 + // The route connection to S2 must be through a slow proxy + np12 := createNetProxy(10*time.Millisecond, 1024*1024*1024, 1024*1024*1024, "route://127.0.0.1:15622", true) + routes := fmt.Sprintf("%s, route://127.0.0.1:16622", np12.routeURL()) + conf := fmt.Sprintf(tmpl, "S1", t.TempDir(), 14622, routes) + c.servers[0], c.opts[0] = RunServerWithConfig(createConfFile(t, []byte(conf))) + + // S2 + // The route connection to S1 must be through a slow proxy + np21 := createNetProxy(10*time.Millisecond, 1024*1024*1024, 1024*1024*1024, "route://127.0.0.1:14622", true) + routes = fmt.Sprintf("%s, route://127.0.0.1:16622", np21.routeURL()) + conf = fmt.Sprintf(tmpl, "S2", t.TempDir(), 15622, routes) + c.servers[1], c.opts[1] = RunServerWithConfig(createConfFile(t, []byte(conf))) + + // S3 + conf = fmt.Sprintf(tmpl, "S3", t.TempDir(), 16622, "route://127.0.0.1:14622, route://127.0.0.1:15622") + c.servers[2], c.opts[2] = RunServerWithConfig(createConfFile(t, []byte(conf))) + + c.checkClusterFormed() + c.waitOnClusterReady() + defer c.shutdown() + defer np12.stop() + defer np21.stop() + + slow := c.servers[0] // Expecting pre-acks here. + sl := c.servers[1] // Stream leader, will publish here. + cl := c.servers[2] // Consumer leader, will consume & ack here. + + snc, sjs := jsClientConnect(t, sl) + defer snc.Close() + + cnc, cjs := jsClientConnect(t, cl) + defer cnc.Close() + + // Now create the stream. + _, err := sjs.AddStream(&nats.StreamConfig{ + Name: "EVENTS", + Subjects: []string{"EV.>"}, + Replicas: 3, + Retention: nats.InterestPolicy, + }) + require_NoError(t, err) + + // Make sure it's leader is on S2. + checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { + c.waitOnStreamLeader(globalAccountName, "EVENTS") + if s := c.streamLeader(globalAccountName, "EVENTS"); s != sl { + s.JetStreamStepdownStream(globalAccountName, "EVENTS") + return fmt.Errorf("Server %s is not stream leader yet", sl) + } + return nil + }) + + // Now create the consumer. + _, err = sjs.AddConsumer("EVENTS", &nats.ConsumerConfig{ + Durable: "C", + AckPolicy: nats.AckExplicitPolicy, + DeliverSubject: "dx", + }) + require_NoError(t, err) + + // Make sure the consumer leader is on S3. + checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { + c.waitOnConsumerLeader(globalAccountName, "EVENTS", "C") + if s := c.consumerLeader(globalAccountName, "EVENTS", "C"); s != cl { + s.JetStreamStepdownConsumer(globalAccountName, "EVENTS", "C") + return fmt.Errorf("Server %s is not consumer leader yet", sl) + } + return nil + }) + + _, err = cjs.Subscribe(_EMPTY_, func(msg *nats.Msg) { + msg.Ack() + }, nats.BindStream("EVENTS"), nats.Durable("C"), nats.ManualAck()) + require_NoError(t, err) + + // Publish directly on the stream leader to make it efficient. + for i := 0; i < 1_000; i++ { + _, err := sjs.PublishAsync("EV.PAID", []byte("ok")) + require_NoError(t, err) + } + select { + case <-sjs.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + + mset, err := slow.GlobalAccount().lookupStream("EVENTS") + require_NoError(t, err) + + checkFor(t, 10*time.Second, 200*time.Millisecond, func() error { + state := mset.state() + if state.LastSeq != 1000 { + return fmt.Errorf("Haven't received all messages yet (last seq %d)", state.LastSeq) + } + mset.mu.RLock() + preAcks := mset.preAcks + mset.mu.RUnlock() + if preAcks == nil { + return fmt.Errorf("Expected to have preAcks by now") + } + if state.Msgs == 0 { + mset.mu.RLock() + lp := len(mset.preAcks) + mset.mu.RUnlock() + if lp == 0 { + return nil + } else { + t.Fatalf("Expected no preAcks with no msgs, but got %d", lp) + } + } + return fmt.Errorf("Still have %d msgs left", state.Msgs) + }) + +} + +func TestNoRaceCheckAckFloorWithVeryLargeFirstSeqAndNewConsumers(t *testing.T) { + s := RunBasicJetStreamServer(t) + defer s.Shutdown() + + nc, _ := jsClientConnect(t, s) + defer nc.Close() + + // Make sure to time bound here for the acksync call below. + js, err := nc.JetStream(nats.MaxWait(200 * time.Millisecond)) + require_NoError(t, err) + + _, err = js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"wq-req"}, + Retention: nats.WorkQueuePolicy, + }) + require_NoError(t, err) + + largeFirstSeq := uint64(1_200_000_000) + err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Sequence: largeFirstSeq}) + require_NoError(t, err) + si, err := js.StreamInfo("TEST") + require_NoError(t, err) + require_True(t, si.State.FirstSeq == largeFirstSeq) + + // Add a simple request to the stream. + sendStreamMsg(t, nc, "wq-req", "HELP") + + sub, err := js.PullSubscribe("wq-req", "dlc") + require_NoError(t, err) + + msgs, err := sub.Fetch(1) + require_NoError(t, err) + require_True(t, len(msgs) == 1) + + // The bug is around the checkAckFloor walking the sequences from current ackfloor + // to the first sequence of the stream. We time bound the max wait with the js context + // to 200ms. Since checkAckFloor is spinning and holding up processing of acks this will fail. + // We will short circuit new consumers to fix this one. + require_NoError(t, msgs[0].AckSync()) + + // Now do again so we move past the new consumer with no ack floor situation. + err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Sequence: 2 * largeFirstSeq}) + require_NoError(t, err) + si, err = js.StreamInfo("TEST") + require_NoError(t, err) + require_True(t, si.State.FirstSeq == 2*largeFirstSeq) + + sendStreamMsg(t, nc, "wq-req", "MORE HELP") + + // We check this one directly for this use case. + mset, err := s.GlobalAccount().lookupStream("TEST") + require_NoError(t, err) + o := mset.lookupConsumer("dlc") + require_True(t, o != nil) + + // Purge will move the stream floor by default, so force into the situation where it is back to largeFirstSeq. + // This will not trigger the new consumer logic, but will trigger a walk of the sequence space. + // Fix will be to walk the lesser of the two linear spaces. + o.mu.Lock() + o.asflr = largeFirstSeq + o.mu.Unlock() + + done := make(chan bool) + go func() { + o.checkAckFloor() + done <- true + }() + + select { + case <-done: + return + case <-time.After(time.Second): + t.Fatalf("Check ack floor taking too long!") + } +} + +func TestNoRaceReplicatedMirrorWithLargeStartingSequenceOverLeafnode(t *testing.T) { + // Cluster B + tmpl := strings.Replace(jsClusterTempl, "store_dir:", "domain: B, store_dir:", 1) + c := createJetStreamCluster(t, tmpl, "B", _EMPTY_, 3, 22020, true) + defer c.shutdown() + + // Cluster A + // Domain is "A' + lc := c.createLeafNodesWithStartPortAndDomain("A", 3, 22110, "A") + defer lc.shutdown() + + lc.waitOnClusterReady() + + // Create a stream on B (HUB/CLOUD) and set its starting sequence very high. + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo"}, + Replicas: 3, + }) + require_NoError(t, err) + + err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Sequence: 1_000_000_000}) + require_NoError(t, err) + + // Send in a small amount of messages. + for i := 0; i < 1000; i++ { + sendStreamMsg(t, nc, "foo", "Hello") + } + + si, err := js.StreamInfo("TEST") + require_NoError(t, err) + require_True(t, si.State.FirstSeq == 1_000_000_000) + + // Now try to create a replicated mirror on the leaf cluster. + lnc, ljs := jsClientConnect(t, lc.randomServer()) + defer lnc.Close() + + _, err = ljs.AddStream(&nats.StreamConfig{ + Name: "TEST", + Mirror: &nats.StreamSource{ + Name: "TEST", + Domain: "B", + }, + }) + require_NoError(t, err) + + // Make sure we sync quickly. + checkFor(t, time.Second, 200*time.Millisecond, func() error { + si, err = ljs.StreamInfo("TEST") + require_NoError(t, err) + if si.State.Msgs == 1000 && si.State.FirstSeq == 1_000_000_000 { + return nil + } + return fmt.Errorf("Mirror state not correct: %+v", si.State) + }) +} + +func TestNoRaceBinaryStreamSnapshotEncodingBasic(t *testing.T) { + s := RunBasicJetStreamServer(t) + defer s.Shutdown() + + nc, js := jsClientConnect(t, s) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"*"}, + MaxMsgsPerSubject: 1, + }) + require_NoError(t, err) + + // Set first key + sendStreamMsg(t, nc, "key:1", "hello") + + // Set Second key but keep updating it, causing a laggard pattern. + value := bytes.Repeat([]byte("Z"), 8*1024) + + for i := 0; i <= 1000; i++ { + _, err := js.PublishAsync("key:2", value) + require_NoError(t, err) + } + select { + case <-js.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + + // Now do more of swiss cheese style. + for i := 3; i <= 1000; i++ { + key := fmt.Sprintf("key:%d", i) + _, err := js.PublishAsync(key, value) + require_NoError(t, err) + // Send it twice to create hole right behind it, like swiss cheese. + _, err = js.PublishAsync(key, value) + require_NoError(t, err) + } + select { + case <-js.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + + // Make for round numbers for stream state. + sendStreamMsg(t, nc, "key:2", "hello") + sendStreamMsg(t, nc, "key:2", "world") + + si, err := js.StreamInfo("TEST") + require_NoError(t, err) + require_True(t, si.State.FirstSeq == 1) + require_True(t, si.State.LastSeq == 3000) + require_True(t, si.State.Msgs == 1000) + require_True(t, si.State.NumDeleted == 2000) + + mset, err := s.GlobalAccount().lookupStream("TEST") + require_NoError(t, err) + + snap, err := mset.store.EncodedStreamState(0, false) + require_NoError(t, err) + + // Now decode the snapshot. + ss, err := DecodeStreamState(snap) + require_NoError(t, err) + + require_Equal(t, ss.FirstSeq, 1) + require_Equal(t, ss.LastSeq, 3000) + require_Equal(t, ss.Msgs, 1000) + require_Equal(t, ss.Deleted.NumDeleted(), 2000) +} + +func TestNoRaceFilestoreBinaryStreamSnapshotEncodingLargeGaps(t *testing.T) { + storeDir := t.TempDir() + fcfg := FileStoreConfig{ + StoreDir: storeDir, + BlockSize: 512, // Small on purpose to create a lot of blks. + } + fs, err := newFileStore(fcfg, StreamConfig{Name: "zzz", Subjects: []string{"zzz"}, Storage: FileStorage}) + require_NoError(t, err) + defer fs.Stop() + + subj, msg := "zzz", bytes.Repeat([]byte("X"), 128) + numMsgs := 20_000 + + fs.StoreMsg(subj, nil, msg, 0) + for i := 2; i < numMsgs; i++ { + seq, _, err := fs.StoreMsg(subj, nil, nil, 0) + require_NoError(t, err) + fs.RemoveMsg(seq) + } + fs.StoreMsg(subj, nil, msg, 0) + + // The tombstones from above will only be cleaned up when syncing blocks. + fs.syncBlocks() + + snap, err := fs.EncodedStreamState(0, false) + require_NoError(t, err) + require_LessThan(t, len(snap), 512) + + // Now decode the snapshot. + ss, err := DecodeStreamState(snap) + require_NoError(t, err) + + require_Equal(t, ss.FirstSeq, 1) + require_Equal(t, ss.LastSeq, 20_000) + require_Equal(t, ss.Msgs, 2) + require_Equal(t, len(ss.Deleted), 1) + require_Equal(t, ss.Deleted.NumDeleted(), 19_998) +} + +func TestNoRaceJetStreamClusterStreamSnapshotCatchup(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + // Client based API + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"*"}, + MaxMsgsPerSubject: 1, + Replicas: 3, + }) + require_NoError(t, err) + + msg := []byte("Hello World") + _, err = js.Publish("foo", msg) + require_NoError(t, err) + + for i := 1; i < 1000; i++ { + _, err := js.PublishAsync("bar", msg) + require_NoError(t, err) + } + select { + case <-js.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + + sr := c.randomNonStreamLeader(globalAccountName, "TEST") + sr.Shutdown() + + // In case we were connected to sr. + nc, js = jsClientConnect(t, c.randomServer()) + defer nc.Close() + + // Now create a large gap. + for i := 0; i < 50_000; i++ { + _, err := js.PublishAsync("bar", msg) + require_NoError(t, err) + } + select { + case <-js.PublishAsyncComplete(): + case <-time.After(10 * time.Second): + t.Fatalf("Did not receive completion signal") + } + + sl := c.streamLeader(globalAccountName, "TEST") + sl.JetStreamSnapshotStream(globalAccountName, "TEST") + + sr = c.restartServer(sr) + c.checkClusterFormed() + c.waitOnServerCurrent(sr) + c.waitOnStreamCurrent(sr, globalAccountName, "TEST") + + mset, err := sr.GlobalAccount().lookupStream("TEST") + require_NoError(t, err) + + // Make sure it's caught up + var state StreamState + mset.store.FastState(&state) + require_Equal(t, state.Msgs, 2) + require_Equal(t, state.FirstSeq, 1) + require_Equal(t, state.LastSeq, 51_000) + require_Equal(t, state.NumDeleted, 51_000-2) + + sr.Shutdown() + + _, err = js.Publish("baz", msg) + require_NoError(t, err) + + sl.JetStreamSnapshotStream(globalAccountName, "TEST") + + sr = c.restartServer(sr) + c.checkClusterFormed() + c.waitOnServerCurrent(sr) + c.waitOnStreamCurrent(sr, globalAccountName, "TEST") + + mset, err = sr.GlobalAccount().lookupStream("TEST") + require_NoError(t, err) + mset.store.FastState(&state) + + require_Equal(t, state.Msgs, 3) + require_Equal(t, state.FirstSeq, 1) + require_Equal(t, state.LastSeq, 51_001) + require_Equal(t, state.NumDeleted, 51_001-3) +} + +func TestNoRaceStoreStreamEncoderDecoder(t *testing.T) { + cfg := &StreamConfig{ + Name: "zzz", + Subjects: []string{"*"}, + MaxMsgsPer: 1, + Storage: MemoryStorage, + } + ms, err := newMemStore(cfg) + require_NoError(t, err) + defer ms.Stop() + + fs, err := newFileStore( + FileStoreConfig{StoreDir: t.TempDir()}, + StreamConfig{Name: "zzz", Subjects: []string{"*"}, MaxMsgsPer: 1, Storage: FileStorage}, + ) + require_NoError(t, err) + defer fs.Stop() + + const seed = 2222222 + msg := bytes.Repeat([]byte("ABC"), 33) // ~100bytes + + maxEncodeTime := 2 * time.Second + maxEncodeSize := 700 * 1024 + + test := func(t *testing.T, gs StreamStore) { + t.Parallel() + prand := rand.New(rand.NewPCG(uint64(seed), 0)) + tick := time.NewTicker(time.Second) + defer tick.Stop() + done := time.NewTimer(10 * time.Second) + + for running := true; running; { + select { + case <-tick.C: + var state StreamState + gs.FastState(&state) + if state.NumDeleted == 0 { + continue + } + start := time.Now() + snap, err := gs.EncodedStreamState(0, false) + require_NoError(t, err) + elapsed := time.Since(start) + // Should take <1ms without race but if CI/CD is slow we will give it a bit of room. + if elapsed > maxEncodeTime { + t.Logf("Encode took longer then expected: %v", elapsed) + } + if len(snap) > maxEncodeSize { + t.Fatalf("Expected snapshot size < %v got %v", friendlyBytes(maxEncodeSize), friendlyBytes(len(snap))) + } + ss, err := DecodeStreamState(snap) + require_True(t, len(ss.Deleted) > 0) + require_NoError(t, err) + case <-done.C: + running = false + default: + key := strconv.Itoa(prand.IntN(256_000)) + gs.StoreMsg(key, nil, msg, 0) + } + } + } + + for _, gs := range []StreamStore{ms, fs} { + switch gs.(type) { + case *memStore: + t.Run("MemStore", func(t *testing.T) { + test(t, gs) + }) + case *fileStore: + t.Run("FileStore", func(t *testing.T) { + test(t, gs) + }) + } + } +} + +func TestNoRaceJetStreamClusterKVWithServerKill(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + // Setup the KV bucket and use for making assertions. + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + _, err := js.CreateKeyValue(&nats.KeyValueConfig{ + Bucket: "TEST", + Replicas: 3, + History: 10, + }) + require_NoError(t, err) + + // Total number of keys to range over. + numKeys := 50 + + // ID is the server id to explicitly connect to. + work := func(ctx context.Context, wg *sync.WaitGroup, id int) { + defer wg.Done() + + nc, js := jsClientConnectEx(t, c.servers[id], []nats.JSOpt{nats.Context(ctx)}) + defer nc.Close() + + kv, err := js.KeyValue("TEST") + require_NoError(t, err) + + // 100 messages a second for each single client. + tk := time.NewTicker(10 * time.Millisecond) + defer tk.Stop() + + for { + select { + case <-ctx.Done(): + return + + case <-tk.C: + // Pick a random key within the range. + k := fmt.Sprintf("key.%d", rand.IntN(numKeys)) + // Attempt to get a key. + e, err := kv.Get(k) + // If found, attempt to update or delete. + if err == nil { + if rand.IntN(10) < 3 { + kv.Delete(k, nats.LastRevision(e.Revision())) + } else { + kv.Update(k, nil, e.Revision()) + } + } else if errors.Is(err, nats.ErrKeyNotFound) { + kv.Create(k, nil) + } + } + } + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var wg sync.WaitGroup + wg.Add(3) + + go work(ctx, &wg, 0) + go work(ctx, &wg, 1) + go work(ctx, &wg, 2) + + time.Sleep(time.Second) + + // Simulate server stop and restart. + for i := 0; i < 7; i++ { + s := c.randomServer() + s.Shutdown() + c.waitOnLeader() + c.waitOnStreamLeader(globalAccountName, "KV_TEST") + + // Wait for a bit and then start the server again. + time.Sleep(time.Duration(rand.IntN(1250)) * time.Millisecond) + s = c.restartServer(s) + c.waitOnServerCurrent(s) + c.waitOnLeader() + c.waitOnStreamLeader(globalAccountName, "KV_TEST") + c.waitOnPeerCount(3) + } + + // Stop the workload. + cancel() + wg.Wait() + + type fullState struct { + state StreamState + lseq uint64 + clfs uint64 + } + + grabState := func(mset *stream) *fullState { + mset.mu.RLock() + defer mset.mu.RUnlock() + var state StreamState + mset.store.FastState(&state) + return &fullState{state, mset.lseq, mset.clfs} + } + + grabStore := func(mset *stream) map[string][]uint64 { + mset.mu.RLock() + store := mset.store + mset.mu.RUnlock() + var state StreamState + store.FastState(&state) + storeMap := make(map[string][]uint64) + for seq := state.FirstSeq; seq <= state.LastSeq; seq++ { + if sm, err := store.LoadMsg(seq, nil); err == nil { + storeMap[sm.subj] = append(storeMap[sm.subj], sm.seq) + } + } + return storeMap + } + + checkFor(t, 10*time.Second, 500*time.Millisecond, func() error { + // Current stream leader. + sl := c.streamLeader(globalAccountName, "KV_TEST") + mset, err := sl.GlobalAccount().lookupStream("KV_TEST") + require_NoError(t, err) + lstate := grabState(mset) + golden := grabStore(mset) + + // Report messages per server. + for _, s := range c.servers { + if s == sl { + continue + } + mset, err := s.GlobalAccount().lookupStream("KV_TEST") + require_NoError(t, err) + state := grabState(mset) + if !reflect.DeepEqual(state, lstate) { + return fmt.Errorf("Expected follower state\n%+v\nto match leader's\n %+v", state, lstate) + } + sm := grabStore(mset) + if !reflect.DeepEqual(sm, golden) { + t.Fatalf("Expected follower store for %v\n%+v\nto match leader's %v\n %+v", s, sm, sl, golden) + } + } + return nil + }) +} + +func TestNoRaceFileStoreLargeMsgsAndFirstMatching(t *testing.T) { + sd := t.TempDir() + fs, err := newFileStore( + FileStoreConfig{StoreDir: sd, BlockSize: 8 * 1024 * 1024}, + StreamConfig{Name: "zzz", Subjects: []string{">"}, Storage: FileStorage}) + require_NoError(t, err) + defer fs.Stop() + + for i := 0; i < 150_000; i++ { + fs.StoreMsg(fmt.Sprintf("foo.bar.%d", i), nil, nil, 0) + } + for i := 0; i < 150_000; i++ { + fs.StoreMsg(fmt.Sprintf("foo.baz.%d", i), nil, nil, 0) + } + require_Equal(t, fs.numMsgBlocks(), 2) + fs.mu.RLock() + mb := fs.blks[1] + fs.mu.RUnlock() + fseq := atomic.LoadUint64(&mb.first.seq) + // The -40 leaves enough mb.fss entries to kick in linear scan. + for seq := fseq; seq < 300_000-40; seq++ { + fs.RemoveMsg(uint64(seq)) + } + start := time.Now() + fs.LoadNextMsg("*.baz.*", true, fseq, nil) + require_True(t, time.Since(start) < 200*time.Microsecond) + // Now remove more to kick into non-linear logic. + for seq := 300_000 - 40; seq < 300_000; seq++ { + fs.RemoveMsg(uint64(seq)) + } + start = time.Now() + fs.LoadNextMsg("*.baz.*", true, fseq, nil) + require_True(t, time.Since(start) < 200*time.Microsecond) +} + +func TestNoRaceWSNoCorruptionWithFrameSizeLimit(t *testing.T) { + testWSNoCorruptionWithFrameSizeLimit(t, 50000) +} + +func TestNoRaceWSCompressionGrownBufferRecycling(t *testing.T) { + opts := testWSOptions() + opts.MaxPending = MAX_PENDING_SIZE + s := &Server{opts: opts} + c := &client{srv: s, ws: &websocket{compress: true}} + c.initClient() + + // Incompressible payload filling the medium pool bucket exactly, so + // that the flate output (payload + block overhead) is guaranteed to + // outgrow the pooled buffer seeding the compression bytes.Buffer. + payload := make([]byte, nbPoolSizeMedium) + rnd := uint32(1) + for i := range payload { + rnd = rnd*1664525 + 1013904223 + payload[i] = byte(rnd >> 24) + } + + // Confirm the payload no longer fits the bucket once compressed. + var cb bytes.Buffer + cw, _ := flate.NewWriter(&cb, flate.BestSpeed) + cw.Write(payload) + cw.Flush() + if cb.Len() <= nbPoolSizeMedium { + t.Fatalf("Expected compressed payload to exceed %v bytes, got %v", nbPoolSizeMedium, cb.Len()) + } + + nbSlice := make(net.Buffers, 1) + collapse := func() { + c.mu.Lock() + data := nbPoolGet(len(payload)) + data = append(data, payload...) + nbSlice[0] = data + c.out.nb = nbSlice + c.out.pb = int64(len(payload)) + c.ws.fs = 0 + + // calls c.wsCollapsePtoNB internally. + bufs, _ := c.collapsePtoNB() + for _, buf := range bufs { + nbPoolPut(buf) + } + c.out.nb = nil + c.mu.Unlock() + } + // Populate the pool and initialize the compressor. + for i := 0; i < 10; i++ { + collapse() + } + + // Growing the bytes.Buffer beyond the seeded pool buffer costs a few + // allocations per iteration but the seed itself must be returned to the pool. + allocs := testing.AllocsPerRun(500, collapse) + if allocs > 3 { + t.Fatalf("Too many allocs per iteration (%.1f); the compression seed buffer is likely not recycled", allocs) + } +} + +func TestNoRaceJetStreamAPIDispatchQueuePending(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + // Setup the KV bucket and use for making assertions. + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo.*.*"}, + }) + require_NoError(t, err) + + // Queue up 500k messages all with different subjects. + // We want to make num pending for a consumer expensive, so a large subject + // space and wildcards for now does the trick. + toks := []string{"foo", "bar", "baz"} // for second token. + for i := 1; i <= 500_000; i++ { + subj := fmt.Sprintf("foo.%s.%d", toks[rand.IntN(len(toks))], i) + _, err := js.PublishAsync(subj, nil, nats.StallWait(time.Second)) + require_NoError(t, err) + } + select { + case <-js.PublishAsyncComplete(): + case <-time.After(20 * time.Second): + t.Fatalf("Did not receive completion signal") + } + + // To back up our pending queue we will create lots of filtered, with wildcards, R1 consumers + // from a different server then the one hosting the stream. + // ok to share this connection here. + sldr := c.streamLeader(globalAccountName, "TEST") + for _, s := range c.servers { + if s != sldr { + nc, js = jsClientConnect(t, s) + defer nc.Close() + break + } + } + + ngr, ncons := 100, 10 + startCh, errCh := make(chan bool), make(chan error, ngr) + var wg, swg sync.WaitGroup + wg.Add(ngr) + swg.Add(ngr) + + // The wildcard in the filter subject is the key. + cfg := &nats.ConsumerConfig{FilterSubject: "foo.*.22"} + var tt atomic.Int64 + + for i := 0; i < ngr; i++ { + go func() { + defer wg.Done() + swg.Done() + // Make them all fire at once. + <-startCh + + for i := 0; i < ncons; i++ { + start := time.Now() + if _, err := js.AddConsumer("TEST", cfg); err != nil { + errCh <- err + t.Logf("Got err creating consumer: %v", err) + } + elapsed := time.Since(start) + tt.Add(int64(elapsed)) + } + }() + } + swg.Wait() + close(startCh) + time.Sleep(time.Millisecond) + jsz, _ := sldr.Jsz(nil) + // This could be 0 legit, so just log, don't fail. + if jsz.JetStreamStats.API.Inflight == 0 { + t.Log("Expected a non-zero inflight") + } + wg.Wait() + + if len(errCh) > 0 { + t.Fatalf("Expected no errors, got %d: %v", len(errCh), <-errCh) + } +} + +func TestNoRaceJetStreamMirrorAndSourceConsumerFailBackoff(t *testing.T) { + owt := srcConsumerWaitTime + srcConsumerWaitTime = 2 * time.Second + defer func() { srcConsumerWaitTime = owt }() + + // Check calculations first. + for i := 1; i <= 20; i++ { + backoff := calculateRetryBackoff(i) + if i < 12 { + require_Equal(t, backoff, time.Duration(i)*10*time.Second) + } else { + require_Equal(t, backoff, retryMaximum) + } + } + + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo.*.*"}, + }) + require_NoError(t, err) + sl := c.streamLeader(globalAccountName, "TEST") + + // Create a mirror. + ml := sl + // Make sure not on the same server. Should not happened in general but possible. + for ml == sl { + js.DeleteStream("MIRROR") + _, err = js.AddStream(&nats.StreamConfig{ + Name: "MIRROR", + Mirror: &nats.StreamSource{Name: "TEST"}, + }) + require_NoError(t, err) + ml = c.streamLeader(globalAccountName, "MIRROR") + } + // Create a source. + srcl := sl + for srcl == sl { + js.DeleteStream("SOURCE") + _, err = js.AddStream(&nats.StreamConfig{ + Name: "SOURCE", + Sources: []*nats.StreamSource{{Name: "TEST"}}, + }) + require_NoError(t, err) + srcl = c.streamLeader(globalAccountName, "MIRROR") + } + + // Create sub to watch for the consumer create requests. + nc, _ = jsClientConnect(t, ml) + defer nc.Close() + sub := natsSubSync(t, nc, "$JS.API.CONSUMER.CREATE.>") + + // Kill the server where the source is.. + sldr := c.streamLeader(globalAccountName, "TEST") + sldr.Shutdown() + + // Wait for just greater than 5s. We should only see 1 request during this time. + time.Sleep(6 * time.Second) + // There should have been 2 requests, one for mirror, one for source + n, _, _ := sub.Pending() + require_Equal(t, n, 2) + var mreq, sreq int + for i := 0; i < 2; i++ { + msg := natsNexMsg(t, sub, time.Second) + if bytes.Contains(msg.Data, []byte("$JS.M.")) { + mreq++ + } else if bytes.Contains(msg.Data, []byte("$JS.S.")) { + sreq++ + } + } + if mreq != 1 || sreq != 1 { + t.Fatalf("Consumer create captures invalid: mreq=%v sreq=%v", mreq, sreq) + } + + // Now make sure that the fails is set properly. + mset, err := c.streamLeader(globalAccountName, "MIRROR").GlobalAccount().lookupStream("MIRROR") + require_NoError(t, err) + mset.mu.RLock() + fails := mset.mirror.fails + mset.mu.RUnlock() + require_Equal(t, fails, 1) + + mset, err = c.streamLeader(globalAccountName, "SOURCE").GlobalAccount().lookupStream("SOURCE") + require_NoError(t, err) + mset.mu.RLock() + si := mset.sources["TEST > >"] + mset.mu.RUnlock() + require_True(t, si != nil) + require_Equal(t, si.fails, 1) +} + +func TestNoRaceJetStreamClusterStreamCatchupLargeInteriorDeletes(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + cfg := &nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo.*"}, + MaxMsgsPerSubject: 100, + Replicas: 1, + } + + _, err := js.AddStream(cfg) + require_NoError(t, err) + + msg := bytes.Repeat([]byte("Z"), 2*1024) + // We will create lots of interior deletes on our R1 then scale up. + _, err = js.Publish("foo.0", msg) + require_NoError(t, err) + + // Create 50k messages randomly from 1-100 + for i := 0; i < 50_000; i++ { + subj := fmt.Sprintf("foo.%d", rand.IntN(100)+1) + publishAsync(t, js, subj, msg) + } + select { + case <-js.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + // Now create a large gap. + for i := 0; i < 100_000; i++ { + publishAsync(t, js, "foo.2", msg) + } + select { + case <-js.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + // Do 50k random again at end. + for i := 0; i < 50_000; i++ { + subj := fmt.Sprintf("foo.%d", rand.IntN(100)+1) + publishAsync(t, js, subj, msg) + } + select { + case <-js.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + + si, err := js.StreamInfo("TEST") + require_NoError(t, err) + + cfg.Replicas = 2 + _, err = js.UpdateStream(cfg) + require_NoError(t, err) + + // Let catchup start. + c.waitOnStreamLeader(globalAccountName, "TEST") + + nl := c.randomNonStreamLeader(globalAccountName, "TEST") + require_True(t, nl != nil) + mset, err := nl.GlobalAccount().lookupStream("TEST") + require_NoError(t, err) + + checkFor(t, 10*time.Second, 500*time.Millisecond, func() error { + state := mset.state() + if state.Msgs == si.State.Msgs { + return nil + } + return fmt.Errorf("Msgs not equal %d vs %d", state.Msgs, si.State.Msgs) + }) +} + +func TestNoRaceJetStreamClusterBadRestartsWithHealthzPolling(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + cfg := &nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo.>"}, + Replicas: 3, + } + _, err := js.AddStream(cfg) + require_NoError(t, err) + + // We will poll healthz at a decent clip and make sure any restart logic works + // correctly with assets coming and going. + ch := make(chan struct{}) + defer close(ch) + + go func() { + for { + select { + case <-ch: + return + case <-time.After(50 * time.Millisecond): + for _, s := range c.servers { + s.healthz(nil) + } + } + } + }() + + numConsumers := 500 + consumers := make([]string, 0, numConsumers) + + var wg sync.WaitGroup + + // Cap inflight consumer create requests to prevent timeout on slower CI machines. + inflight := make(chan struct{}, 50) + + for i := 0; i < numConsumers; i++ { + cname := fmt.Sprintf("CONS-%d", i+1) + consumers = append(consumers, cname) + wg.Add(1) + inflight <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-inflight }() + _, err := js.PullSubscribe("foo.>", cname, nats.BindStream("TEST")) + require_NoError(t, err) + }() + } + wg.Wait() + + // Make sure all are reported. + c.waitOnAllCurrent() + checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { + for _, s := range c.servers { + jsz, _ := s.Jsz(nil) + if jsz.Consumers != numConsumers { + return fmt.Errorf("%v wrong number of consumers: %d vs %d", s, jsz.Consumers, numConsumers) + } + } + return nil + }) + + // Now do same for streams. + numStreams := 200 + streams := make([]string, 0, numStreams) + + // Cap number of inflight stream request creates. + sinflight := make(chan struct{}, 25) + + for i := 0; i < numStreams; i++ { + sname := fmt.Sprintf("TEST-%d", i+1) + streams = append(streams, sname) + wg.Add(1) + sinflight <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sinflight }() + _, err := js.AddStream(&nats.StreamConfig{Name: sname, Replicas: 3}) + require_NoError(t, err) + }() + } + wg.Wait() + + // Make sure all are reported. + c.waitOnAllCurrent() + checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { + for _, s := range c.servers { + jsz, _ := s.Jsz(nil) + if jsz.Streams != numStreams+1 { + return fmt.Errorf("%v wrong number of streams: %d vs %d", s, jsz.Streams, numStreams+1) + } + } + return nil + }) + + // Delete consumers. + for _, cname := range consumers { + err := js.DeleteConsumer("TEST", cname) + require_NoError(t, err) + } + // Make sure reporting goes to zero. + c.waitOnAllCurrent() + checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { + for _, s := range c.servers { + jsz, _ := s.Jsz(nil) + if jsz.Consumers != 0 { + return fmt.Errorf("%v still has %d consumers", s, jsz.Consumers) + } + } + return nil + }) + + // Delete streams + for _, sname := range streams { + err := js.DeleteStream(sname) + require_NoError(t, err) + } + err = js.DeleteStream("TEST") + require_NoError(t, err) + + // Make sure reporting goes to zero. + c.waitOnAllCurrent() + checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { + for _, s := range c.servers { + jsz, _ := s.Jsz(nil) + if jsz.Streams != 0 { + return fmt.Errorf("%v still has %d streams", s, jsz.Streams) + } + } + return nil + }) +} + +func TestNoRaceJetStreamKVReplaceWithServerRestart(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + nc, _ := jsClientConnect(t, c.randomServer()) + defer nc.Close() + // Shorten wait time for disconnects. + js, err := nc.JetStream(nats.MaxWait(time.Second)) + require_NoError(t, err) + + kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ + Bucket: "TEST", + Replicas: 3, + }) + require_NoError(t, err) + + // Manually disable direct get on underlying stream, since this test + // relies on immediate consistency which we can't guarantee with direct gets + // until we provide a solution for this. + si, err := js.StreamInfo("KV_TEST") + require_NoError(t, err) + si.Config.AllowDirect = false + _, err = js.UpdateStream(&si.Config) + require_NoError(t, err) + + createData := func(n int) []byte { + const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + b := make([]byte, n) + for i := range b { + b[i] = letterBytes[rand.IntN(len(letterBytes))] + } + return b + } + + _, err = kv.Create("foo", createData(160)) + require_NoError(t, err) + + // Ensure all replicas have applied the key. + checkFor(t, 2*time.Second, 200*time.Millisecond, func() error { + return checkState(t, c, globalAccountName, "KV_TEST") + }) + + ch := make(chan struct{}) + wg := sync.WaitGroup{} + + // For counting errors that should not happen. + errCh := make(chan error, 1024) + + wg.Add(1) + go func() { + defer wg.Done() + + var lastData []byte + var revision uint64 + + for { + select { + case <-ch: + close(errCh) + return + default: + k, err := kv.Get("foo") + if err == nats.ErrKeyNotFound { + errCh <- err + } else if k != nil { + if lastData != nil && k.Revision() == revision && !bytes.Equal(lastData, k.Value()) { + errCh <- fmt.Errorf("data loss [%s][rev:%d] expected:[%q] is:[%q]\n", "foo", revision, lastData, k.Value()) + } + newData := createData(160) + if revision, err = kv.Update("foo", newData, k.Revision()); err == nil { + lastData = newData + } + } + } + } + }() + + // Wait a short bit. + time.Sleep(2 * time.Second) + for _, s := range c.servers { + s.Shutdown() + // Need to leave servers down for a while to trigger bug properly. + time.Sleep(5 * time.Second) + s = c.restartServer(s) + c.waitOnServerHealthz(s) + } + + // Shutdown the go routine above. + close(ch) + // Wait for it to finish. + wg.Wait() + + if len(errCh) != 0 { + for err := range errCh { + t.Logf("Received err %v during test", err) + } + t.Fatalf("Encountered errors") + } +} + +func TestNoRaceMemStoreCompactPerformance(t *testing.T) { + //Load MemStore so that it is full + subj, msg := "foo", make([]byte, 1000) + storedMsgSize := memStoreMsgSize(subj, nil, msg) + + toStore := uint64(10_000) + toStoreOnTop := uint64(1_000) + setSeqNo := uint64(10_000_000_000) + + expectedPurge := toStore - 1 + maxBytes := storedMsgSize * toStore + + ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage, MaxBytes: int64(maxBytes)}) + require_NoError(t, err) + defer ms.Stop() + + for i := uint64(0); i < toStore; i++ { + ms.StoreMsg(subj, nil, msg, 0) + } + state := ms.State() + require_Equal(t, toStore, state.Msgs) + require_Equal(t, state.Bytes, storedMsgSize*toStore) + + //1st run: Load additional messages then compact + for i := uint64(0); i < toStoreOnTop; i++ { + ms.StoreMsg(subj, nil, msg, 0) + } + startFirstRun := time.Now() + purgedFirstRun, _ := ms.Compact(toStore + toStoreOnTop) + elapsedFirstRun := time.Since(startFirstRun) + require_Equal(t, expectedPurge, purgedFirstRun) + + //set the seq number to a very high value by compacting with a too high seq number + purgedFull, _ := ms.Compact(setSeqNo) + require_Equal(t, 1, purgedFull) + + //2nd run: Compact again + for i := uint64(0); i < toStore; i++ { + ms.StoreMsg(subj, nil, msg, 0) + } + startSecondRun := time.Now() + purgedSecondRun, _ := ms.Compact(setSeqNo + toStore - 1) + elapsedSecondRun := time.Since(startSecondRun) + require_Equal(t, expectedPurge, purgedSecondRun) + + //Calculate delta between runs and fail if it is too high + require_LessThan(t, elapsedSecondRun-elapsedFirstRun, time.Duration(1)*time.Second) +} + +func TestNoRaceJetStreamSnapshotsWithSlowAckDontSlowConsumer(t *testing.T) { + // Test takes less time this way. + snapshotAckTimeout = 500 * time.Millisecond + t.Cleanup(func() { + snapshotAckTimeout = defaultSnapshotAckTimeout + }) + + s := RunBasicJetStreamServer(t) + defer s.Shutdown() + + ech := make(chan error) + ecb := func(_ *nats.Conn, _ *nats.Subscription, err error) { + if err != nil { + ech <- err + } + } + nc, js := jsClientConnect(t, s, nats.ErrorHandler(ecb)) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo"}, + }) + require_NoError(t, err) + + // Put in over 64MB. + msg, toSend := make([]byte, 1024*1024), 80 + crand.Read(msg) + + for i := 0; i < toSend; i++ { + _, err := js.Publish("foo", msg) + require_NoError(t, err) + } + + sreq := &JSApiStreamSnapshotRequest{ + DeliverSubject: nats.NewInbox(), + ChunkSize: 1024 * 1024, + } + req, _ := json.Marshal(sreq) + rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, "TEST"), req, time.Second) + require_NoError(t, err) + + var resp JSApiStreamSnapshotResponse + json.Unmarshal(rmsg.Data, &resp) + require_True(t, resp.Error == nil) + + done := make(chan *nats.Msg) + sub, _ := nc.Subscribe(sreq.DeliverSubject, func(m *nats.Msg) { + // EOF + if len(m.Data) == 0 { + done <- m + return + } + }) + defer sub.Unsubscribe() + + // Check that we do not get disconnected due to slow consumer. + select { + case msg := <-done: + require_Equal(t, msg.Header.Get("Status"), "408") + require_Equal(t, msg.Header.Get("Description"), "No Flow Response") + case <-ech: + t.Fatalf("Got disconnected: %v", err) + case <-time.After(snapshotAckTimeout * 2): + t.Fatalf("Should have received EOF with error status") + } +} + +func TestNoRaceJetStreamWQSkippedMsgsOnScaleUp(t *testing.T) { + checkInterestStateT = 4 * time.Second + checkInterestStateJ = 1 + defer func() { + checkInterestStateT = defaultCheckInterestStateT + checkInterestStateJ = defaultCheckInterestStateJ + }() + + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + const pre = "CORE_ENT_DR_OTP_22." + wcSubj := pre + ">" + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{wcSubj}, + Retention: nats.WorkQueuePolicy, + AllowDirect: true, + Replicas: 3, + }) + require_NoError(t, err) + + cfg := &nats.ConsumerConfig{ + Durable: "dlc", + FilterSubject: wcSubj, + DeliverPolicy: nats.DeliverAllPolicy, + AckPolicy: nats.AckExplicitPolicy, + MaxAckPending: 10_000, + AckWait: 500 * time.Millisecond, + MaxWaiting: 100, + MaxRequestExpires: 1050 * time.Millisecond, + } + _, err = js.AddConsumer("TEST", cfg) + require_NoError(t, err) + + pdone := make(chan bool) + cdone := make(chan bool) + + // We will have 51 consumer apps and a producer app. Make sure to wait for + // all go routines to end at the end of the test. + wg := sync.WaitGroup{} + wg.Add(52) + + // Publish routine + go func() { + defer wg.Done() + + publishSubjects := []string{ + "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.918886682066", + "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.918886682067", + "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543211", + "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543212", + "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543213", + "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543214", + "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543215", + "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543216", + "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543217", + } + // ~1.7kb + msg := bytes.Repeat([]byte("Z"), 1750) + + // 200 msgs/s + st := time.NewTicker(5 * time.Millisecond) + defer st.Stop() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + for { + select { + case <-st.C: + subj := publishSubjects[rand.IntN(len(publishSubjects))] + _, err = js.Publish(subj, msg) + require_NoError(t, err) + case <-pdone: + return + } + } + }() + + consumerApp := func() { + defer wg.Done() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + _, err := js.ConsumerInfo("TEST", "dlc") + require_NoError(t, err) + _, err = js.UpdateConsumer("TEST", cfg) + require_NoError(t, err) + + sub, err := js.PullSubscribe(wcSubj, "dlc") + require_NoError(t, err) + + st := time.NewTicker(100 * time.Millisecond) + defer st.Stop() + + for { + select { + case <-st.C: + msgs, err := sub.Fetch(1, nats.MaxWait(100*time.Millisecond)) + if err != nil { + continue + } + require_Equal(t, len(msgs), 1) + m := msgs[0] + if rand.IntN(10) == 1 { + m.Nak() + } else { + // Wait up to 20ms to ack. + time.Sleep(time.Duration(rand.IntN(20)) * time.Millisecond) + // This could fail and that is ok, system should recover due to low ack wait. + m.Ack() + } + case <-cdone: + return + } + } + } + + // Now consumer side single. + go consumerApp() + + // Wait for 2s + time.Sleep(2 * time.Second) + + // Now spin up 50 more. + for i := 1; i <= 50; i++ { + if i%5 == 0 { + time.Sleep(200 * time.Millisecond) + } + go consumerApp() + } + + timeout := time.Now().Add(8 * time.Second) + for time.Now().Before(timeout) { + time.Sleep(750 * time.Millisecond) + if s := c.consumerLeader(globalAccountName, "TEST", "dlc"); s != nil { + s.JetStreamStepdownConsumer(globalAccountName, "TEST", "dlc") + } + } + + // Close publishers and defer closing consumers. + close(pdone) + defer func() { + close(cdone) + wg.Wait() + }() + + checkFor(t, 30*time.Second, 50*time.Millisecond, func() error { + si, err := js.StreamInfo("TEST") + require_NoError(t, err) + if si.State.NumDeleted > 0 || si.State.Msgs > 0 { + return fmt.Errorf("State not correct: %+v", si.State) + } + return nil + }) +} + +func TestNoRaceConnectionObjectReleased(t *testing.T) { + ob1Conf := createConfFile(t, []byte(` + listen: "127.0.0.1:-1" + server_name: "B1" + accounts { + A { users: [{user: a, password: pwd}] } + SYS { users: [{user: sys, password: pwd}] } + } + cluster { + name: "B" + listen: "127.0.0.1:-1" + } + gateway { + name: "B" + listen: "127.0.0.1:-1" + } + leaf { + listen: "127.0.0.1:-1" + } + system_account: "SYS" + `)) + sb1, ob1 := RunServerWithConfig(ob1Conf) + defer sb1.Shutdown() + + oaConf := createConfFile(t, []byte(fmt.Sprintf(` + listen: "127.0.0.1:-1" + server_name: "A" + accounts { + A { users: [{user: a, password: pwd}] } + SYS { users: [{user: sys, password: pwd}] } + } + gateway { + name: "A" + listen: "127.0.0.1:-1" + gateways [ + { + name: "B" + url: "nats://a:pwd@127.0.0.1:%d" + } + ] + } + websocket { + listen: "127.0.0.1:-1" + no_tls: true + } + system_account: "SYS" + `, ob1.Gateway.Port))) + sa, oa := RunServerWithConfig(oaConf) + defer sa.Shutdown() + + waitForOutboundGateways(t, sa, 1, 2*time.Second) + waitForOutboundGateways(t, sb1, 1, 2*time.Second) + + ob2Conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: "127.0.0.1:-1" + server_name: "B2" + accounts { + A { users: [{user: a, password: pwd}] } + SYS { users: [{user: sys, password: pwd}] } + } + cluster { + name: "B" + listen: "127.0.0.1:-1" + routes: ["nats://127.0.0.1:%d"] + } + gateway { + name: "B" + listen: "127.0.0.1:-1" + } + system_account: "SYS" + `, ob1.Cluster.Port))) + sb2, _ := RunServerWithConfig(ob2Conf) + defer sb2.Shutdown() + + checkClusterFormed(t, sb1, sb2) + waitForOutboundGateways(t, sb2, 1, 2*time.Second) + waitForInboundGateways(t, sa, 2, 2*time.Second) + + leafConf := createConfFile(t, []byte(fmt.Sprintf(` + listen: "127.0.0.1:-1" + server_name: "C" + accounts { + A { users: [{user: a, password: pwd}] } + SYS { users: [{user: sys, password: pwd}] } + } + leafnodes { + remotes [ + { url: "nats://a:pwd@127.0.0.1:%d" } + ] + } + system_account: "SYS" + `, ob1.LeafNode.Port))) + leaf, _ := RunServerWithConfig(leafConf) + defer leaf.Shutdown() + + checkLeafNodeConnected(t, leaf) + + // Start an independent MQTT server to check MQTT client connection. + mo := testMQTTDefaultOptions() + mo.ServerName = "MQTTServer" + sm := testMQTTRunServer(t, mo) + defer testMQTTShutdownServer(sm) + + mc, mr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, mo.MQTT.Host, mo.MQTT.Port) + defer mc.Close() + testMQTTCheckConnAck(t, mr, mqttConnAckRCConnectionAccepted, false) + + nc := natsConnect(t, sb1.ClientURL(), nats.UserInfo("a", "pwd")) + defer nc.Close() + cid, err := nc.GetClientID() + require_NoError(t, err) + natsSubSync(t, nc, "foo") + natsFlush(t, nc) + + ncWS := natsConnect(t, fmt.Sprintf("ws://a:pwd@127.0.0.1:%d", oa.Websocket.Port)) + defer ncWS.Close() + cidWS, err := ncWS.GetClientID() + require_NoError(t, err) + + var conns []net.Conn + var total int + var ch chan string + + track := func(c *client) { + total++ + c.mu.Lock() + conns = append(conns, c.nc) + c.mu.Unlock() + runtime.SetFinalizer(c, func(c *client) { + ch <- fmt.Sprintf("Server=%s - Kind=%s - Conn=%v", c.srv, c.kindString(), c) + }) + } + // Track the connection for the MQTT client + sm.mu.RLock() + for _, c := range sm.clients { + track(c) + } + sm.mu.RUnlock() + + // Track the connection from the NATS client + track(sb1.getClient(cid)) + // The outbound connection to GW "A" + track(sb1.getOutboundGatewayConnection("A")) + // The inbound connection from GW "A" + var inGW []*client + sb1.getInboundGatewayConnections(&inGW) + track(inGW[0]) + // The routes from sb2 + sb1.forEachRoute(func(r *client) { + track(r) + }) + // The leaf form "LEAF" + sb1.mu.RLock() + for _, l := range sb1.leafs { + track(l) + } + sb1.mu.RUnlock() + + // Now from sb2, the routes to sb1 + sb2.forEachRoute(func(r *client) { + track(r) + }) + // The outbound connection to GW "A" + track(sb2.getOutboundGatewayConnection("A")) + + // From server "A", track the outbound GW + track(sa.getOutboundGatewayConnection("B")) + inGW = inGW[:0] + // Track the inbound GW connections + sa.getInboundGatewayConnections(&inGW) + for _, ig := range inGW { + track(ig) + } + // Track the websocket client + track(sa.getClient(cidWS)) + + // From the LEAF server, the connection to sb1 + leaf.mu.RLock() + for _, l := range leaf.leafs { + track(l) + } + leaf.mu.RUnlock() + + // Now close all connections and wait to see if all connections + // with the finalizer set is invoked. + ch = make(chan string, total) + // Close the clients and then all other connections to create a disconnect. + nc.Close() + mc.Close() + ncWS.Close() + for _, conn := range conns { + conn.Close() + } + // Wait and see if we get them all. + tm := time.NewTimer(10 * time.Second) + defer tm.Stop() + tk := time.NewTicker(10 * time.Millisecond) + for clients := make([]string, 0, total); len(clients) < total; { + select { + case <-tk.C: + runtime.GC() + case cs := <-ch: + clients = append(clients, cs) + case <-tm.C: + // Don't fail the test since there is no guarantee that + // finalizers are invoked. + t.Logf("Got %v out of %v finalizers", len(clients), total) + slices.Sort(clients) + for _, cs := range clients { + t.Logf(" => %s", cs) + } + return + } + } +} + +func TestNoRaceFileStoreMsgLoadNextMsgMultiPerf(t *testing.T) { + fs, err := newFileStore( + FileStoreConfig{StoreDir: t.TempDir()}, + StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) + require_NoError(t, err) + defer fs.Stop() + + // Put 1k msgs in + for i := 0; i < 1000; i++ { + subj := fmt.Sprintf("foo.%d", i) + _, _, err = fs.StoreMsg(subj, nil, []byte("ZZZ"), 0) + require_NoError(t, err) + } + + var smv StoreMsg + + // Now do normal load next with no filter. + // This is baseline. + start := time.Now() + for i, seq := 0, uint64(1); i < 1000; i++ { + sm, nseq, err := fs.LoadNextMsg(_EMPTY_, false, seq, &smv) + require_NoError(t, err) + require_Equal(t, sm.subj, fmt.Sprintf("foo.%d", i)) + require_Equal(t, nseq, seq) + seq++ + } + baseline := time.Since(start) + t.Logf("Single - No filter %v", baseline) + + // Allow some additional skew. + baseline += time.Millisecond + + // Now do normal load next with wc filter. + start = time.Now() + for i, seq := 0, uint64(1); i < 1000; i++ { + sm, nseq, err := fs.LoadNextMsg("foo.>", true, seq, &smv) + require_NoError(t, err) + require_Equal(t, sm.subj, fmt.Sprintf("foo.%d", i)) + require_Equal(t, nseq, seq) + seq++ + } + elapsed := time.Since(start) + t.Logf("Single - WC filter %v", elapsed) + require_LessThan(t, elapsed, 2*baseline) + + // Now do multi load next with 1 wc entry. + sl := gsl.NewSublist[struct{}]() + require_NoError(t, sl.Insert("foo.>", struct{}{})) + start = time.Now() + for i, seq := 0, uint64(1); i < 1000; i++ { + sm, nseq, err := fs.LoadNextMsgMulti(sl, seq, &smv) + require_NoError(t, err) + require_Equal(t, sm.subj, fmt.Sprintf("foo.%d", i)) + require_Equal(t, nseq, seq) + seq++ + } + elapsed = time.Since(start) + t.Logf("Multi - Single WC filter %v", elapsed) + require_LessThan(t, elapsed, 2*baseline) + + // Now do multi load next with 1000 literal subjects. + sl = gsl.NewSublist[struct{}]() + for i := 0; i < 1000; i++ { + subj := fmt.Sprintf("foo.%d", i) + require_NoError(t, sl.Insert(subj, struct{}{})) + } + start = time.Now() + for i, seq := 0, uint64(1); i < 1000; i++ { + sm, nseq, err := fs.LoadNextMsgMulti(sl, seq, &smv) + require_NoError(t, err) + require_Equal(t, sm.subj, fmt.Sprintf("foo.%d", i)) + require_Equal(t, nseq, seq) + seq++ + } + elapsed = time.Since(start) + t.Logf("Multi - 1000 filters %v", elapsed) + require_LessThan(t, elapsed, 3*baseline) +} + +func TestNoRaceWQAndMultiSubjectFilters(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"Z.>"}, + Retention: nats.WorkQueuePolicy, + }) + require_NoError(t, err) + + stopPubs := make(chan bool) + + publish := func(subject string) { + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + for { + select { + case <-stopPubs: + return + default: + _, _ = js.Publish(subject, []byte("hello")) + } + } + } + + go publish("Z.foo") + go publish("Z.bar") + go publish("Z.baz") + + // Cancel pubs after 10s. + time.AfterFunc(10*time.Second, func() { close(stopPubs) }) + + // Create a consumer + _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ + Durable: "zzz", + AckPolicy: nats.AckExplicitPolicy, + AckWait: 5 * time.Second, + FilterSubjects: []string{"Z.foo", "Z.bar", "Z.baz"}, + }) + require_NoError(t, err) + + sub, err := js.PullSubscribe(_EMPTY_, "zzz", nats.Bind("TEST", "zzz")) + require_NoError(t, err) + + received := make([]uint64, 0, 256_000) + batchSize := 10 + + for running := true; running; { + msgs, err := sub.Fetch(batchSize, nats.MaxWait(2*time.Second)) + if err == nats.ErrTimeout { + running = false + } + for _, m := range msgs { + meta, err := m.Metadata() + require_NoError(t, err) + received = append(received, meta.Sequence.Stream) + m.Ack() + } + } + + slices.Sort(received) + + var pseq, gaps uint64 + for _, seq := range received { + if pseq != 0 && pseq != seq-1 { + gaps += seq - pseq + 1 + } + pseq = seq + } + si, err := js.StreamInfo("TEST") + require_NoError(t, err) + + if si.State.Msgs != 0 || gaps > 0 { + t.Fatalf("Orphaned msgs %d with %d gaps detected", si.State.Msgs, gaps) + } +} + +// https://github.com/nats-io/nats-server/issues/4957 +func TestNoRaceWQAndMultiSubjectFiltersRace(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"Z.>"}, + Retention: nats.WorkQueuePolicy, + Replicas: 1, + }) + require_NoError(t, err) + + // The bug would happen when the stream was on same server as meta-leader. + // So make that so. + // Make sure stream leader is on S-1 + sl := c.streamLeader(globalAccountName, "TEST") + checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { + if sl == c.leader() { + return nil + } + // Move meta-leader since stream can be R1. + snc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo("admin", "s3cr3t!")) + defer snc.Close() + if _, err := snc.Request(JSApiLeaderStepDown, nil, time.Second); err != nil { + return err + } + return fmt.Errorf("stream leader on meta-leader") + }) + + start := make(chan struct{}) + var done, ready sync.WaitGroup + + // Create num go routines who will all race to create a consumer with the same filter subject but a different name. + num := 10 + ready.Add(num) + done.Add(num) + + for i := 0; i < num; i++ { + go func(n int) { + // Connect directly to the meta leader but with our own connection. + s := c.leader() + nc, js := jsClientConnect(t, s) + defer nc.Close() + + ready.Done() + defer done.Done() + <-start + + js.AddConsumer("TEST", &nats.ConsumerConfig{ + Name: fmt.Sprintf("C-%d", n), + FilterSubject: "Z.foo", + AckPolicy: nats.AckExplicitPolicy, + }) + }(i) + } + + // Wait for requestors to be ready + ready.Wait() + close(start) + done.Wait() + + checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { + si, err := js.StreamInfo("TEST") + require_NoError(t, err) + if si.State.Consumers != 1 { + return fmt.Errorf("Consumer count not correct: %d vs 1", si.State.Consumers) + } + return nil + }) +} + +func TestNoRaceFileStoreWriteFullStateUniqueSubjects(t *testing.T) { + fcfg := FileStoreConfig{StoreDir: t.TempDir()} + fs, err := newFileStore(fcfg, + StreamConfig{Name: "zzz", Subjects: []string{"records.>"}, Storage: FileStorage, MaxMsgsPer: 1, MaxBytes: 15 * 1024 * 1024 * 1024}) + require_NoError(t, err) + defer fs.Stop() + + qch := make(chan struct{}) + defer close(qch) + + go func() { + const numThreshold = 1_000_000 + tick := time.NewTicker(1 * time.Second) + for { + select { + case <-qch: + return + case <-tick.C: + err := fs.writeFullState() + var state StreamState + fs.FastState(&state) + if state.Msgs > numThreshold && err != nil { + require_Error(t, err, errStateTooBig) + } + } + } + }() + + labels := []string{"AAAAA", "BBBB", "CCCC", "DD", "EEEEE"} + msg := []byte(strings.Repeat("Z", 128)) + + for i := 0; i < 100; i++ { + partA := nuid.Next() + for j := 0; j < 100; j++ { + partB := nuid.Next() + for k := 0; k < 500; k++ { + partC := nuid.Next() + partD := labels[rand.IntN(len(labels)-1)] + subject := fmt.Sprintf("records.%s.%s.%s.%s.%s", partA, partB, partC, partD, nuid.Next()) + start := time.Now() + fs.StoreMsg(subject, nil, msg, 0) + elapsed := time.Since(start) + if elapsed > 500*time.Millisecond { + t.Fatalf("Slow store for %q: %v\n", subject, elapsed) + } + } + } + } + // Make sure we do write the full state on stop. + fs.Stop() + fi, err := os.Stat(filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile)) + require_NoError(t, err) + // ~500MB, could change if we tweak encodings.. + require_True(t, fi.Size() > 500*1024*1024) +} + +// When a catchup takes a long time and the ingest rate is high enough to cause us +// to drop append entries and move our first past out catchup window due to max bytes or max msgs, etc. +func TestNoRaceLargeStreamCatchups(t *testing.T) { + // This usually takes too long on Travis. + t.Skip() + + var jsMaxOutTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_file_store: 22GB, store_dir: '%s', max_outstanding_catchup: 128KB} + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } +` + + c := createJetStreamClusterWithTemplate(t, jsMaxOutTempl, "R3S", 3) + defer c.shutdown() + + nc, _ := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + js, err := nc.JetStream(nats.PublishAsyncMaxPending(64 * 1024)) + require_NoError(t, err) + + cfg := &nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"Z.>"}, + MaxBytes: 4 * 1024 * 1024 * 1024, + Replicas: 1, // Start at R1 + } + + _, err = js.AddStream(cfg) + require_NoError(t, err) + + // Load up to a decent size first. + num, msg := 25_000, bytes.Repeat([]byte("Z"), 256*1024) + for i := 0; i < num; i++ { + _, err := js.PublishAsync("Z.Z.Z", msg) + require_NoError(t, err) + } + select { + case <-js.PublishAsyncComplete(): + case <-time.After(20 * time.Second): + t.Fatalf("Did not receive completion signal") + } + + var sl *Server + time.AfterFunc(time.Second, func() { + cfg.Replicas = 3 + _, err = js.UpdateStream(cfg) + require_NoError(t, err) + c.waitOnStreamLeader(globalAccountName, "TEST") + sl = c.streamLeader(globalAccountName, "TEST") + }) + + // Run for 60 seconds sending new messages at a high rate. + timeout := time.Now().Add(60 * time.Second) + for time.Now().Before(timeout) { + for i := 0; i < 5_000; i++ { + // Not worried about each message, so not checking err here. + // Just generating high load of new traffic while trying to catch up. + js.PublishAsync("Z.Z.Z", msg) + } + // This will gate us waiting on a response. + js.Publish("Z.Z.Z", msg) + } + + // Make sure the leader has not changed. + require_Equal(t, sl, c.streamLeader(globalAccountName, "TEST")) + + // Grab the leader and its state. + mset, err := sl.GlobalAccount().lookupStream("TEST") + require_NoError(t, err) + expected := mset.state() + + checkFor(t, 45*time.Second, time.Second, func() error { + for _, s := range c.servers { + if s == sl { + continue + } + mset, err := s.GlobalAccount().lookupStream("TEST") + require_NoError(t, err) + state := mset.state() + if !reflect.DeepEqual(expected, state) { + return fmt.Errorf("Follower %v state does not match: %+v vs %+v", s, state, expected) + } + } + return nil + }) +} + +func TestNoRaceLargeNumDeletesStreamCatchups(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + nc, _ := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + js, err := nc.JetStream(nats.PublishAsyncMaxPending(16 * 1024)) + require_NoError(t, err) + + cfg := &nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo"}, + Replicas: 1, // Start at R1 + } + _, err = js.AddStream(cfg) + require_NoError(t, err) + + // We will manipulate the stream at the lower level to achieve large number of interior deletes. + // We will store only 2 msgs, but have 100M deletes in between. + sl := c.streamLeader(globalAccountName, "TEST") + mset, err := sl.GlobalAccount().lookupStream("TEST") + require_NoError(t, err) + mset.mu.Lock() + mset.store.StoreMsg("foo", nil, []byte("ok"), 0) + mset.store.SkipMsgs(2, 1_000_000_000) + mset.store.StoreMsg("foo", nil, []byte("ok"), 0) + mset.store.SkipMsgs(1_000_000_003, 1_000_000_000) + var state StreamState + mset.store.FastState(&state) + mset.lseq = state.LastSeq + mset.mu.Unlock() + + cfg.Replicas = 3 + _, err = js.UpdateStream(cfg) + require_NoError(t, err) + c.waitOnStreamLeader(globalAccountName, "TEST") + mset, err = sl.GlobalAccount().lookupStream("TEST") + require_NoError(t, err) + expected := mset.state() + + // This should happen fast and not spin on all interior deletes. + checkFor(t, 250*time.Millisecond, 50*time.Millisecond, func() error { + for _, s := range c.servers { + if s == sl { + continue + } + mset, err := s.GlobalAccount().lookupStream("TEST") + require_NoError(t, err) + state := mset.state() + // Ignore LastTime for this test since we send delete range at end. + state.LastTime = expected.LastTime + if !reflect.DeepEqual(expected, state) { + return fmt.Errorf("Follower %v state does not match: %+v vs %+v", s, state, expected) + } + } + return nil + }) +} + +func TestNoRaceJetStreamClusterMemoryStreamLastSequenceResetAfterRestart(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + numStreams := 250 + var wg sync.WaitGroup + wg.Add(numStreams) + + for i := 1; i <= numStreams; i++ { + go func(n int) { + defer wg.Done() + _, err := js.AddStream(&nats.StreamConfig{ + Name: fmt.Sprintf("TEST:%d", n), + Storage: nats.MemoryStorage, + Subjects: []string{fmt.Sprintf("foo.%d.*", n)}, + Replicas: 3, + }, nats.MaxWait(30*time.Second)) + require_NoError(t, err) + subj := fmt.Sprintf("foo.%d.bar", n) + for i := 0; i < 222; i++ { + js.Publish(subj, nil) + } + }(i) + } + wg.Wait() + + // Make sure all streams have a snapshot in place to stress the snapshot logic for memory based streams. + for _, s := range c.servers { + for i := 1; i <= numStreams; i++ { + stream := fmt.Sprintf("TEST:%d", i) + mset, err := s.GlobalAccount().lookupStream(stream) + require_NoError(t, err) + node := mset.raftNode() + require_NotNil(t, node) + node.InstallSnapshot(mset.stateSnapshot(), false) + } + } + + // Do 5 rolling restarts waiting on healthz in between. + for i := 0; i < 5; i++ { + // Walk the servers and shut each down, and wipe the storage directory. + for _, s := range c.servers { + s.Shutdown() + s.WaitForShutdown() + s = c.restartServer(s) + c.waitOnServerHealthz(s) + c.waitOnAllCurrent() + // Make sure all streams are current after healthz returns ok. + for i := 1; i <= numStreams; i++ { + stream := fmt.Sprintf("TEST:%d", i) + mset, err := s.GlobalAccount().lookupStream(stream) + require_NoError(t, err) + var state StreamState + checkFor(t, 30*time.Second, time.Second, func() error { + mset.store.FastState(&state) + if state.LastSeq != 222 { + return fmt.Errorf("%v Wrong last sequence %d for %q - State %+v", s, state.LastSeq, stream, state) + } + return nil + }) + } + } + } +} + +func TestNoRaceJetStreamClusterMemoryWorkQueueLastSequenceResetAfterRestart(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + numStreams := 50 + var wg sync.WaitGroup + wg.Add(numStreams) + + for i := 1; i <= numStreams; i++ { + go func(n int) { + defer wg.Done() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + checkFor(t, 5*time.Second, time.Second, func() error { + _, err := js.AddStream(&nats.StreamConfig{ + Name: fmt.Sprintf("TEST:%d", n), + Storage: nats.MemoryStorage, + Retention: nats.WorkQueuePolicy, + Subjects: []string{fmt.Sprintf("foo.%d.*", n)}, + Replicas: 3, + }, nats.MaxWait(time.Second)) + return err + }) + + subj := fmt.Sprintf("foo.%d.bar", n) + for i := 0; i < 22; i++ { + checkFor(t, 5*time.Second, time.Second, func() error { + _, err := js.Publish(subj, nil) + return err + }) + } + // Now consume them all as well. + var err error + var sub *nats.Subscription + checkFor(t, 5*time.Second, time.Second, func() error { + sub, err = js.PullSubscribe(subj, "wq") + return err + }) + + var msgs []*nats.Msg + checkFor(t, 5*time.Second, time.Second, func() error { + msgs, err = sub.Fetch(22, nats.MaxWait(time.Second)) + return err + }) + require_Equal(t, len(msgs), 22) + for _, m := range msgs { + checkFor(t, 5*time.Second, time.Second, func() error { + return m.AckSync() + }) + } + }(i) + } + wg.Wait() + + // Do 2 rolling restarts waiting on healthz in between. + for i := 0; i < 2; i++ { + // Walk the servers and shut each down, and wipe the storage directory. + for _, s := range c.servers { + s.Shutdown() + s.WaitForShutdown() + s = c.restartServer(s) + c.waitOnServerHealthz(s) + c.waitOnAllCurrent() + // Make sure all streams are current after healthz returns ok. + for i := 1; i <= numStreams; i++ { + stream := fmt.Sprintf("TEST:%d", i) + mset, err := s.GlobalAccount().lookupStream(stream) + require_NoError(t, err) + var state StreamState + checkFor(t, 20*time.Second, time.Second, func() error { + mset.store.FastState(&state) + if state.LastSeq != 22 { + return fmt.Errorf("%v Wrong last sequence %d for %q - State %+v", s, state.LastSeq, stream, state) + } + if state.FirstSeq != 23 { + return fmt.Errorf("%v Wrong first sequence %d for %q - State %+v", s, state.FirstSeq, stream, state) + } + return nil + }) + } + } + } +} + +func TestNoRaceJetStreamClusterMirrorSkipSequencingBug(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R5S", 5) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "ORIGIN", + Subjects: []string{"foo.*"}, + MaxMsgsPerSubject: 1, + Replicas: 1, + Storage: nats.MemoryStorage, + }) + require_NoError(t, err) + + // Create a mirror with R5 such that it will be much slower than the ORIGIN. + _, err = js.AddStream(&nats.StreamConfig{ + Name: "M", + Replicas: 5, + Mirror: &nats.StreamSource{Name: "ORIGIN"}, + }) + require_NoError(t, err) + + // Connect new directly to ORIGIN + s := c.streamLeader(globalAccountName, "ORIGIN") + nc, js = jsClientConnect(t, s) + defer nc.Close() + + // We are going to send at a high rate and also delete some along the way + // via the max msgs per limit. + for i := 0; i < 500_000; i++ { + subj := fmt.Sprintf("foo.%d", i) + publishAsync(t, js, subj, nil) + // Create sequence holes every 100k. + if i%100_000 == 0 { + publishAsync(t, js, subj, nil) + } + } + select { + case <-js.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + + checkFor(t, 20*time.Second, time.Second, func() error { + si, err := js.StreamInfo("M") + require_NoError(t, err) + if si.State.Msgs != 500_000 { + return fmt.Errorf("Expected 500k msgs, got state: %+v", si.State) + } + return nil + }) +} + +func TestNoRaceJetStreamStandaloneDontReplyToAckBeforeProcessingIt(t *testing.T) { + s := RunBasicJetStreamServer(t) + defer s.Shutdown() + + // Client for API requests. + nc, js := jsClientConnect(t, s) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "WQ", + Discard: nats.DiscardNew, + MaxMsgsPerSubject: 1, + DiscardNewPerSubject: true, + Retention: nats.WorkQueuePolicy, + Subjects: []string{"queue.>"}, + }) + require_NoError(t, err) + + // Keep this low since we are going to run as many go routines + // to consume, ack and republish the message. + total := 10000 + // Populate the queue, one message per subject. + for i := 0; i < total; i++ { + js.Publish(fmt.Sprintf("queue.%d", i), []byte("hello")) + } + + _, err = js.AddConsumer("WQ", &nats.ConsumerConfig{ + Durable: "cons", + AckPolicy: nats.AckExplicitPolicy, + MaxWaiting: 20000, + MaxAckPending: -1, + }) + require_NoError(t, err) + + sub, err := js.PullSubscribe("queue.>", "cons", nats.BindStream("WQ")) + require_NoError(t, err) + + errCh := make(chan error, total) + + var wg sync.WaitGroup + for iter := 0; iter < 3; iter++ { + wg.Add(total) + for i := 0; i < total; i++ { + go func() { + defer wg.Done() + for { + msgs, err := sub.Fetch(1) + if err != nil { + time.Sleep(5 * time.Millisecond) + continue + } + msg := msgs[0] + err = msg.AckSync() + if err != nil { + time.Sleep(5 * time.Millisecond) + continue + } + _, err = js.Publish(msg.Subject, []byte("hello")) + if err != nil { + errCh <- err + return + } + break + } + }() + } + wg.Wait() + select { + case err := <-errCh: + t.Fatalf("Test failed, first error was: %v", err) + default: + // OK! + } + } +} + +// Under certain scenarios an old index.db with a stream that has msg limits set will not restore properly +// due to an old index.db and compaction after the index.db took place which could lose per subject information. +func TestNoRaceFileStoreMsgLimitsAndOldRecoverState(t *testing.T) { + for _, test := range []struct { + name string + expectedFirstSeq uint64 + expectedLastSeq uint64 + expectedMsgs uint64 + transform func(StreamConfig) StreamConfig + }{ + { + name: "MaxMsgsPer", + expectedFirstSeq: 10_001, + expectedLastSeq: 1_010_001, + expectedMsgs: 1_000_001, + transform: func(config StreamConfig) StreamConfig { + config.MaxMsgsPer = 1 + return config + }, + }, + { + name: "MaxMsgs", + expectedFirstSeq: 10_001, + expectedLastSeq: 1_010_001, + expectedMsgs: 1_000_001, + transform: func(config StreamConfig) StreamConfig { + config.MaxMsgs = 1_000_001 + return config + }, + }, + { + name: "MaxBytes", + expectedFirstSeq: 8_624, + expectedLastSeq: 1_010_001, + expectedMsgs: 1_001_378, + transform: func(config StreamConfig) StreamConfig { + config.MaxBytes = 1_065_353_216 + return config + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + sd := t.TempDir() + fs, err := newFileStore( + FileStoreConfig{StoreDir: sd}, + test.transform(StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}), + ) + require_NoError(t, err) + defer fs.Stop() + + msg := make([]byte, 1024) + + for i := 0; i < 10_000; i++ { + subj := fmt.Sprintf("foo.%d", i) + fs.StoreMsg(subj, nil, msg, 0) + } + + // This will write the index.db file. We will capture this and use it to replace a new one. + sfile := filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile) + fs.Stop() + _, err = os.Stat(sfile) + require_NoError(t, err) + + // Read it in and make sure len > 0. + buf, err := os.ReadFile(sfile) + require_NoError(t, err) + require_True(t, len(buf) > 0) + + // Restart + fs, err = newFileStore( + FileStoreConfig{StoreDir: sd}, + test.transform(StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}), + ) + require_NoError(t, err) + defer fs.Stop() + + // Put in more messages with wider range. This will compact a bunch of the previous blocks. + for i := 0; i < 1_000_001; i++ { + subj := fmt.Sprintf("foo.%d", i) + fs.StoreMsg(subj, nil, msg, 0) + } + + var ss StreamState + fs.FastState(&ss) + require_Equal(t, ss.FirstSeq, test.expectedFirstSeq) + require_Equal(t, ss.LastSeq, test.expectedLastSeq) + require_Equal(t, ss.Msgs, test.expectedMsgs) + + // Now stop again, but replace index.db with old one. + fs.Stop() + // Put back old stream state. + require_NoError(t, os.WriteFile(sfile, buf, defaultFilePerms)) + + // Restart + fs, err = newFileStore( + FileStoreConfig{StoreDir: sd}, + test.transform(StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}), + ) + require_NoError(t, err) + defer fs.Stop() + + fs.FastState(&ss) + require_Equal(t, ss.FirstSeq, test.expectedFirstSeq) + require_Equal(t, ss.LastSeq, test.expectedLastSeq) + require_Equal(t, ss.Msgs, test.expectedMsgs) + }) + } +} + +func TestNoRaceJetStreamClusterCheckInterestStatePerformanceWQ(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3F", 3) + defer c.shutdown() + + s := c.randomServer() + nc, js := jsClientConnect(t, s) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo.*"}, + Retention: nats.WorkQueuePolicy, + }) + require_NoError(t, err) + + // Load up a bunch of messages for three different subjects. + msg := bytes.Repeat([]byte("Z"), 4096) + for i := 0; i < 100_000; i++ { + publishAsync(t, js, "foo.foo", msg) + } + for i := 0; i < 5_000; i++ { + publishAsync(t, js, "foo.bar", msg) + publishAsync(t, js, "foo.baz", msg) + } + select { + case <-js.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + + // We will not process this one and leave it as "offline". + _, err = js.PullSubscribe("foo.foo", "A") + require_NoError(t, err) + subB, err := js.PullSubscribe("foo.bar", "B") + require_NoError(t, err) + subC, err := js.PullSubscribe("foo.baz", "C") + require_NoError(t, err) + + // Now catch up both B and C but let A simulate being offline of very behind. + for i := 0; i < 5; i++ { + for _, sub := range []*nats.Subscription{subB, subC} { + msgs, err := sub.Fetch(1000) + require_NoError(t, err) + require_Equal(t, len(msgs), 1000) + for _, m := range msgs { + m.Ack() + } + } + } + // Let acks process. + nc.Flush() + time.Sleep(200 * time.Millisecond) + + // Now test the check checkInterestState() on the stream. + sl := c.streamLeader(globalAccountName, "TEST") + mset, err := sl.GlobalAccount().lookupStream("TEST") + require_NoError(t, err) + + expireAllBlks := func() { + mset.mu.RLock() + fs := mset.store.(*fileStore) + mset.mu.RUnlock() + fs.mu.RLock() + for _, mb := range fs.blks { + mb.tryForceExpireCache() + } + fs.mu.RUnlock() + } + + // First expire all the blocks. + expireAllBlks() + + start := time.Now() + mset.checkInterestState() + elapsed := time.Since(start) + // This is actually ~300 microseconds but due to travis and race flags etc. + // Was > 30 ms before fix for comparison, M2 macbook air. + require_LessThan(t, elapsed, 5*time.Millisecond) + + // Make sure we set the chkflr correctly. + // The chkflr should be equal to asflr+1 (unless the next message matching the filter is further ahead). + // Otherwise, if chkflr would be set higher a subsequent call to checkInterestState will be ineffective. + requireFloorsEqual := func(o *consumer) { + t.Helper() + require_True(t, o != nil) + o.mu.RLock() + defer o.mu.RUnlock() + _, seq, err := mset.store.LoadNextMsg(o.cfg.FilterSubject, false, o.asflr+1, nil) + if err == ErrStoreEOF { + seq++ + } + require_True(t, o.asflr+1 <= seq) + require_Equal(t, o.chkflr, seq) + } + + requireFloorsEqual(mset.lookupConsumer("A")) + requireFloorsEqual(mset.lookupConsumer("B")) + requireFloorsEqual(mset.lookupConsumer("C")) + + // Expire all the blocks again. + expireAllBlks() + + // This checks the chkflr state. + start = time.Now() + mset.checkInterestState() + elapsed = time.Since(start) + require_LessThan(t, elapsed, 5*time.Millisecond) +} + +func TestNoRaceJetStreamClusterCheckInterestStatePerformanceInterest(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3F", 3) + defer c.shutdown() + + s := c.randomServer() + nc, js := jsClientConnect(t, s) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo.*"}, + Retention: nats.InterestPolicy, + }) + require_NoError(t, err) + + // We will not process this one and leave it as "offline". + _, err = js.PullSubscribe("foo.foo", "A") + require_NoError(t, err) + _, err = js.PullSubscribe("foo.*", "B") + require_NoError(t, err) + // Make subC multi-subject. + _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ + Durable: "C", + FilterSubjects: []string{"foo.foo", "foo.bar", "foo.baz"}, + AckPolicy: nats.AckExplicitPolicy, + }) + require_NoError(t, err) + + // Load up a bunch of messages for three different subjects. + msg := bytes.Repeat([]byte("Z"), 4096) + for i := 0; i < 90_000; i++ { + publishAsync(t, js, "foo.foo", msg) + } + for i := 0; i < 5_000; i++ { + publishAsync(t, js, "foo.bar", msg) + publishAsync(t, js, "foo.baz", msg) + } + select { + case <-js.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + // This is so we do not asynchronously update our consumer state after we set the state due to notifications + // from new messages for the stream. + time.Sleep(250 * time.Millisecond) + + // Now catch up both B and C but let A simulate being offline of very behind. + // Will do this manually here to speed up tests. + sl := c.streamLeader(globalAccountName, "TEST") + mset, err := sl.GlobalAccount().lookupStream("TEST") + require_NoError(t, err) + + for _, cname := range []string{"B", "C"} { + o := mset.lookupConsumer(cname) + o.mu.Lock() + o.setStoreState(&ConsumerState{ + Delivered: SequencePair{100_000, 100_000}, + AckFloor: SequencePair{100_000, 100_000}, + }) + o.mu.Unlock() + } + + // Now test the check checkInterestState() on the stream. + start := time.Now() + mset.checkInterestState() + elapsed := time.Since(start) + + // Make sure we set the chkflr correctly. + checkFloor := func(o *consumer) uint64 { + require_True(t, o != nil) + o.mu.RLock() + defer o.mu.RUnlock() + return o.chkflr + } + + require_Equal(t, checkFloor(mset.lookupConsumer("A")), 1) + require_Equal(t, checkFloor(mset.lookupConsumer("B")), 90_001) + require_Equal(t, checkFloor(mset.lookupConsumer("C")), 100_001) + + // This checks the chkflr state. For this test this should be much faster, + // two orders of magnitude then the first time. + start = time.Now() + mset.checkInterestState() + require_True(t, time.Since(start) < elapsed/100) +} + +func TestNoRaceJetStreamClusterLargeMetaSnapshotTiming(t *testing.T) { + // This test was to show improvements in speed for marshaling the meta layer with lots of assets. + // Move to S2.Encode vs EncodeBetter which is 2x faster and actually better compression. + // Also moved to goccy json which is faster then the default and in my tests now always matches + // the default encoder byte for byte which last time I checked it did not. + t.Skip() + + c := createJetStreamClusterExplicit(t, "R3F", 3) + defer c.shutdown() + + // Create 200 streams, each with 500 consumers. + numStreams := 200 + numConsumers := 500 + wg := sync.WaitGroup{} + wg.Add(numStreams) + for i := 0; i < numStreams; i++ { + go func() { + defer wg.Done() + s := c.randomServer() + nc, js := jsClientConnect(t, s) + defer nc.Close() + sname := fmt.Sprintf("TEST-SNAPSHOT-%d", i) + subj := fmt.Sprintf("foo.%d", i) + _, err := js.AddStream(&nats.StreamConfig{ + Name: sname, + Subjects: []string{subj}, + Replicas: 3, + }) + require_NoError(t, err) + + // Now consumers. + for c := 0; c < numConsumers; c++ { + _, err = js.AddConsumer(sname, &nats.ConsumerConfig{ + Durable: fmt.Sprintf("C-%d", c), + FilterSubject: subj, + AckPolicy: nats.AckExplicitPolicy, + Replicas: 1, + }) + require_NoError(t, err) + } + }() + } + wg.Wait() + + s := c.leader() + js := s.getJetStream() + n := js.getMetaGroup() + // Now let's see how long it takes to create a meta snapshot and how big it is. + start := time.Now() + snap, _, _, err := js.metaSnapshot() + require_NoError(t, err) + require_NoError(t, n.InstallSnapshot(snap, false)) + t.Logf("Took %v to snap meta with size of %v\n", time.Since(start), friendlyBytes(len(snap))) +} + +func TestNoRaceStoreReverseWalkWithDeletesPerf(t *testing.T) { + cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage} + + fs, err := newFileStore(FileStoreConfig{StoreDir: t.TempDir()}, cfg) + require_NoError(t, err) + defer fs.Stop() + + cfg.Storage = MemoryStorage + ms, err := newMemStore(&cfg) + require_NoError(t, err) + defer ms.Stop() + + msg := []byte("Hello") + + for _, store := range []StreamStore{fs, ms} { + _, _, err = store.StoreMsg("foo.A", nil, msg, 0) + require_NoError(t, err) + for i := 0; i < 1_000_000; i++ { + _, _, err = store.StoreMsg("foo.B", nil, msg, 0) + require_NoError(t, err) + } + _, _, err = store.StoreMsg("foo.C", nil, msg, 0) + require_NoError(t, err) + + var ss StreamState + store.FastState(&ss) + require_Equal(t, ss.Msgs, 1_000_002) + + // Create a bunch of interior deletes. + p, err := store.PurgeEx("foo.B", 0, 0) + require_NoError(t, err) + require_Equal(t, p, 1_000_000) + + // Preload the caches so this test only measures the reverse walk. + preloadCaches := func() { + if fs, ok := store.(*fileStore); ok { + fs.mu.Lock() + defer fs.mu.Unlock() + for _, mb := range fs.blks { + require_NoError(t, mb.loadMsgs()) + } + } + } + + // Now simulate a walk backwards as we currently do when searching for starting sequence numbers in sourced streams. + preloadCaches() + start := time.Now() + var smv StoreMsg + for seq := ss.LastSeq; seq > 0; seq-- { + _, err := store.LoadMsg(seq, &smv) + if err == errDeletedMsg || err == ErrStoreMsgNotFound { + continue + } + require_NoError(t, err) + } + elapsed := time.Since(start) + + // Now use the optimized load prev. + preloadCaches() + seq, seen := ss.LastSeq, 0 + start = time.Now() + for { + sm, _, err := store.LoadPrevMsg(_EMPTY_, false, seq, &smv) + if err == ErrStoreEOF { + break + } + require_NoError(t, err) + seq = sm.seq - 1 + seen++ + } + elapsedNew := time.Since(start) + require_Equal(t, seen, 2) + + switch store.(type) { + case *memStore: + require_LessThan(t, elapsedNew, elapsed) + case *fileStore: + // Bigger gains for filestore. + require_LessThan(t, elapsedNew*10, elapsed) + } + } +} + +type fastProdLogger struct { + DummyLogger + gotIt chan struct{} +} + +func (l *fastProdLogger) Debugf(format string, args ...any) { + msg := fmt.Sprintf(format, args...) + if strings.Contains(msg, "fast producer") { + select { + case l.gotIt <- struct{}{}: + default: + } + } +} + +func TestNoRaceNoFastProducerStall(t *testing.T) { + tmpl := ` + listen: "127.0.0.1:-1" + no_fast_producer_stall: %s + ` + conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, "true"))) + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + l := &fastProdLogger{gotIt: make(chan struct{}, 1)} + s.SetLogger(l, true, false) + + ncSlow := natsConnect(t, s.ClientURL()) + defer ncSlow.Close() + natsSub(t, ncSlow, "foo", func(_ *nats.Msg) {}) + natsFlush(t, ncSlow) + + nc := natsConnect(t, s.ClientURL()) + defer nc.Close() + traceSub := natsSubSync(t, nc, "my.trace.subj") + natsFlush(t, nc) + + ncProd := natsConnect(t, s.ClientURL()) + defer ncProd.Close() + + payload := make([]byte, 256) + + cid, err := ncSlow.GetClientID() + require_NoError(t, err) + c := s.GetClient(cid) + require_True(t, c != nil) + + wg := sync.WaitGroup{} + pub := func() { + wg.Add(1) + go func() { + defer wg.Done() + + msg := nats.NewMsg("foo") + msg.Header.Set(MsgTraceDest, traceSub.Subject) + msg.Data = payload + ncProd.PublishMsg(msg) + }() + } + + checkTraceMsg := func(err string) { + t.Helper() + var e MsgTraceEvent + traceMsg := natsNexMsg(t, traceSub, time.Second) + json.Unmarshal(traceMsg.Data, &e) + egresses := e.Egresses() + require_Equal(t, len(egresses), 1) + eg := egresses[0] + require_Equal(t, eg.CID, cid) + if err != _EMPTY_ { + require_Contains(t, eg.Error, err) + } else { + require_Equal(t, eg.Error, _EMPTY_) + } + } + + // Artificially set a stall channel. + c.mu.Lock() + c.out.stc = make(chan struct{}) + c.mu.Unlock() + + // Publish a message, it should not stall the producer. + pub() + // Now make sure we did not get any fast producer debug statements. + select { + case <-l.gotIt: + t.Fatal("Got debug logs about fast producer") + case <-time.After(250 * time.Millisecond): + // OK! + } + wg.Wait() + + checkTraceMsg(errMsgTraceFastProdNoStall) + + // Now we will conf reload to enable fast producer stalling. + reloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, "false")) + + // Publish, this time the prod should be stalled. + pub() + select { + case <-l.gotIt: + // OK! + case <-time.After(500 * time.Millisecond): + t.Fatal("Timed-out waiting for a warning") + } + wg.Wait() + + // Should have been delivered to the trace subscription. + checkTraceMsg(_EMPTY_) +} + +func TestNoRaceProducerStallLimits(t *testing.T) { + tmpl := ` + listen: "127.0.0.1:-1" + ` + conf := createConfFile(t, []byte(tmpl)) + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + ncSlow := natsConnect(t, s.ClientURL(), nats.Name("slow")) + defer ncSlow.Close() + natsSub(t, ncSlow, "foo", func(m *nats.Msg) { m.Respond([]byte("42")) }) + natsFlush(t, ncSlow) + + ncProd := natsConnect(t, s.ClientURL(), nats.Name("fast")) + defer ncProd.Close() + + cid, err := ncSlow.GetClientID() + require_NoError(t, err) + c := s.GetClient(cid) + require_True(t, c != nil) + + // Artificially set a stall channel on the subscriber. + c.mu.Lock() + c.out.stc = make(chan struct{}) + c.mu.Unlock() + + start := time.Now() + _, err = ncProd.Request("foo", []byte("HELLO"), time.Second) + elapsed := time.Since(start) + require_NoError(t, err) + + // This should have not cleared on its own but should have between min and max pause. + require_True(t, elapsed >= stallClientMinDuration) + require_LessThan(t, elapsed, stallClientMaxDuration+5*time.Millisecond) + + // Now test total maximum by loading up a bunch of requests and measuring the last one. + // Artificially set a stall channel again on the subscriber. + c.mu.Lock() + c.out.stc = make(chan struct{}) + // This will prevent us from clearing the stc. + c.out.pb = c.out.mp/4*3 + 100 + c.mu.Unlock() + + for i := 0; i < 10; i++ { + err = ncProd.PublishRequest("foo", "bar", []byte("HELLO")) + require_NoError(t, err) + } + start = time.Now() + _, err = ncProd.Request("foo", []byte("HELLO"), time.Second) + elapsed = time.Since(start) + require_NoError(t, err) + + require_True(t, elapsed >= stallTotalAllowed) + // Should always be close to totalAllowed (e.g. 10ms), but if you run a lot of them in one go can bump up + // just past it, hence the Max setting below to avoid a flapper. + require_LessThan(t, elapsed, stallTotalAllowed+20*time.Millisecond) + + var tc *client + s.mu.Lock() + for _, client := range s.clients { + client.mu.Lock() + if client.opts.Name == "slow" { + tc = client + client.mu.Unlock() + break + } + client.mu.Unlock() + } + s.mu.Unlock() + + // Verify counter was incremented. + totalStalls := atomic.LoadInt64(&tc.stalls) + if totalStalls == 0 { + t.Fatalf("Expected stalls count to be incremented, got: %d", totalStalls) + } + + // Verify this shows up in Connz monitoring. + pconns := pollConnz(t, s, 1, "", nil) + var connzStalls int64 + for _, conn := range pconns.Conns { + if conn.Cid == tc.cid { + connzStalls = conn.Stalls + break + } + } + if connzStalls < 1 { + t.Fatalf("Expected connz stalls count to be incremented, got: %d", connzStalls) + } + if s.NumStalledClients() < 1 { + t.Fatalf("Expected total stalled clients to be incremented, got: %d", s.NumStalledClients()) + } +} + +func TestNoRaceJetStreamClusterConsumerDeleteInterestPolicyPerf(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Retention: nats.InterestPolicy, + Subjects: []string{"foo.*"}, + Replicas: 3, + }) + require_NoError(t, err) + + // Make the first sequence high. We already protect against it but for extra sanity. + err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Sequence: 100_000_000}) + require_NoError(t, err) + + // Create 3 consumers. 1 Ack explicit, 1 AckAll and 1 AckNone + _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ + Durable: "C1", + AckPolicy: nats.AckExplicitPolicy, + }) + require_NoError(t, err) + + _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ + Durable: "C2", + AckPolicy: nats.AckAllPolicy, + }) + require_NoError(t, err) + + _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ + Durable: "C3", + AckPolicy: nats.AckNonePolicy, + }) + require_NoError(t, err) + + for i := 0; i < 500_000; i++ { + js.PublishAsync("foo.bar", []byte("ok")) + + // Confirm batch. + if i%1000 == 0 { + select { + case <-js.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + } + } + select { + case <-js.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + + expectedStreamMsgs := func(msgs uint64) { + t.Helper() + checkFor(t, 2*time.Second, 200*time.Millisecond, func() error { + si, err := js.StreamInfo("TEST") + if err != nil { + return err + } + if si.State.Msgs != msgs { + return fmt.Errorf("require uint64 equal, but got: %d != %d", si.State.Msgs, msgs) + } + return nil + }) + } + expectedStreamMsgs(500_000) + + // For C1 grab 100 and ack evens. + sub, err := js.PullSubscribe("foo.bar", "C1") + require_NoError(t, err) + msgs, err := sub.Fetch(100) + require_NoError(t, err) + require_Equal(t, len(msgs), 100) + for _, m := range msgs { + meta, _ := m.Metadata() + if meta.Sequence.Stream%2 == 0 { + require_NoError(t, m.AckSync()) + } + } + + // For C2 grab 500 and ack 100. + sub, err = js.PullSubscribe("foo.bar", "C2") + require_NoError(t, err) + msgs, err = sub.Fetch(500) + require_NoError(t, err) + require_Equal(t, len(msgs), 500) + require_NoError(t, msgs[99].AckSync()) + + // Simulate stream viewer, get first 10 from C3 + sub, err = js.PullSubscribe("foo.bar", "C3") + require_NoError(t, err) + msgs, err = sub.Fetch(10) + require_NoError(t, err) + require_Equal(t, len(msgs), 10) + + expectedStreamMsgs(499_995) + + // This test would flake depending on if mset.checkInterestState already ran or not. + // Manually call it here, because consumers don't retry removing messages below their ack floor. + for _, s := range c.servers { + acc, err := s.lookupAccount(globalAccountName) + require_NoError(t, err) + mset, err := acc.lookupStream("TEST") + require_NoError(t, err) + mset.checkInterestState() + } + + // Before fix this was in the seconds. All the while the stream is locked. + // This should be short now, but messages might not be cleaned up. + start := time.Now() + err = js.DeleteConsumer("TEST", "C3") + require_NoError(t, err) + if elapsed := time.Since(start); elapsed > 100*time.Millisecond { + t.Fatalf("Deleting AckNone consumer took too long: %v", elapsed) + } + + expectedStreamMsgs(499_950) + + // Now do AckAll + start = time.Now() + err = js.DeleteConsumer("TEST", "C2") + require_NoError(t, err) + if elapsed := time.Since(start); elapsed > 100*time.Millisecond { + t.Fatalf("Deleting AckAll consumer took too long: %v", elapsed) + } + + expectedStreamMsgs(499_950) + + // Now do AckExplicit + start = time.Now() + err = js.DeleteConsumer("TEST", "C1") + require_NoError(t, err) + if elapsed := time.Since(start); elapsed > 100*time.Millisecond { + t.Fatalf("Deleting AckExplicit consumer took too long: %v", elapsed) + } + + expectedStreamMsgs(0) +} + +func TestNoRaceJetStreamClusterConsumerDeleteInterestPolicyUniqueFiltersPerf(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Retention: nats.InterestPolicy, + Subjects: []string{"foo.*"}, + Replicas: 3, + }) + require_NoError(t, err) + + // Create 2 consumers. 1 Ack explicit, 1 AckNone + _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ + Durable: "C0", + AckPolicy: nats.AckExplicitPolicy, + FilterSubjects: []string{"foo.0"}, + }) + require_NoError(t, err) + + _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ + Durable: "C1", + AckPolicy: nats.AckNonePolicy, + FilterSubjects: []string{"foo.1"}, + }) + require_NoError(t, err) + + for i := 0; i < 500_000; i++ { + subject := fmt.Sprintf("foo.%d", i%2) + js.PublishAsync(subject, []byte("ok")) + + // Confirm batch. + if i%1000 == 0 { + select { + case <-js.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + } + } + select { + case <-js.PublishAsyncComplete(): + case <-time.After(5 * time.Second): + t.Fatalf("Did not receive completion signal") + } + + expectedStreamMsgs := func(msgs uint64) { + t.Helper() + checkFor(t, 2*time.Second, 200*time.Millisecond, func() error { + si, err := js.StreamInfo("TEST") + if err != nil { + return err + } + if si.State.Msgs != msgs { + return fmt.Errorf("require uint64 equal, but got: %d != %d", si.State.Msgs, msgs) + } + return nil + }) + } + + expectedStreamMsgs(500_000) + + // For C0 grab 100 and ack them. + sub, err := js.PullSubscribe("foo.0", "C0") + require_NoError(t, err) + msgs, err := sub.Fetch(100) + require_NoError(t, err) + require_Equal(t, len(msgs), 100) + for _, msg := range msgs { + require_NoError(t, msg.AckSync()) + } + + expectedStreamMsgs(499_900) + + start := time.Now() + err = js.DeleteConsumer("TEST", "C1") + require_NoError(t, err) + if elapsed := time.Since(start); elapsed > 100*time.Millisecond { + t.Fatalf("Deleting AckNone consumer took too long: %v", elapsed) + } + + expectedStreamMsgs(499_800) + + start = time.Now() + err = js.DeleteConsumer("TEST", "C0") + require_NoError(t, err) + if elapsed := time.Since(start); elapsed > 100*time.Millisecond { + t.Fatalf("Deleting AckExplicit consumer took too long: %v", elapsed) + } + expectedStreamMsgs(0) +} + +func TestNoRaceFileStorePurgeExAsyncTombstones(t *testing.T) { + fs, err := newFileStore( + FileStoreConfig{StoreDir: t.TempDir()}, + StreamConfig{Name: "zzz", Subjects: []string{"*.*"}, Storage: FileStorage}) + require_NoError(t, err) + defer fs.Stop() + + msg := []byte("zzz") + + fs.StoreMsg("foo.A", nil, msg, 0) + fs.StoreMsg("foo.B", nil, msg, 0) + for i := 0; i < 500; i++ { + fs.StoreMsg("foo.C", nil, msg, 0) + } + fs.StoreMsg("foo.D", nil, msg, 0) + + // Load all blocks to avoid that being a factor in timing. + fs.mu.RLock() + for _, mb := range fs.blks { + mb.loadMsgs() + } + fs.mu.RUnlock() + + // Now purge 1 that is not the first message and take note of time. + // Since we are loaded this should mostly be the time to write / flush tombstones. + start := time.Now() + n, err := fs.PurgeEx("foo.B", 0, 0) + elapsed := time.Since(start) + require_NoError(t, err) + require_Equal(t, n, 1) + + start = time.Now() + n, err = fs.PurgeEx("foo.C", 0, 0) + elapsed2 := time.Since(start) + require_NoError(t, err) + require_Equal(t, n, 500) + + // If we are flushing for each tombstone the second elapsed time will be a larger multiple of the single message purge. + // In testing this is like >200x + // With async and flush for all tombstones will be ~30x + require_True(t, elapsed*50 > elapsed2) +} + +// Chck that we do not leak any go routines from access time optimization. +func TestNoRaceAccessTimeLeakCheck(t *testing.T) { + time.Sleep(time.Second) + ngrp := runtime.NumGoroutine() + + s := RunBasicJetStreamServer(t) + defer s.Shutdown() + + nc, js := jsClientConnect(t, s) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo"}, + }) + require_NoError(t, err) + + _, err = js.Publish("foo", []byte("Hello World")) + require_NoError(t, err) + + // Close down client and server. + nc.Close() + s.Shutdown() + s.WaitForShutdown() + + checkFor(t, 5*time.Second, time.Second, func() error { + if ngra := runtime.NumGoroutine(); ngrp != ngra { + return fmt.Errorf("expected %d, got %d", ngrp, ngra) + } + return nil + }) +} + +func TestNoRaceQuickMultipleConfigReloadRemoteLeafNoDuplicate(t *testing.T) { + // This test is to demonstrate that if we perform quicj disable/enable, + // or remove/add of remote leafnodes, we don't end up with multiple + // "connect to remote" go routines for the same remote, leading to duplicate + // remote leafnodes. + + conf1 := createConfFile(t, []byte(` + port: -1 + server_name: "A" + leafnodes { + listen: "127.0.0.1:-1" + } + `)) + s1, o1 := RunServerWithConfig(conf1) + defer s1.Shutdown() + l := &captureErrorLogger{errCh: make(chan string, 100)} + s1.SetLogger(l, false, false) + + tmpl2 := ` + port: -1 + server_name: "B" + leafnodes { + reconnect_interval: "50ms" + remotes [ + %s + ] + } + ` + + configWithAdded := fmt.Appendf(nil, tmpl2, + fmt.Sprintf(`{url: "nats://127.0.0.1:%d"}`, o1.LeafNode.Port)) + configWithRemoved := fmt.Appendf(nil, tmpl2, _EMPTY_) + configWithDisabled := fmt.Appendf(nil, tmpl2, + fmt.Sprintf(`{url: "nats://127.0.0.1:%d", disabled: true}`, o1.LeafNode.Port)) + configWithEnabled := fmt.Appendf(nil, tmpl2, + fmt.Sprintf(`{url: "nats://127.0.0.1:%d", disabled: false}`, o1.LeafNode.Port)) + + conf2 := createConfFile(t, configWithEnabled) + s2, _ := RunServerWithConfig(conf2) + defer s2.Shutdown() + + checkLeafNodeConnectedCount(t, s2, 1) + + for _, test := range []struct { + name string + disabled bool + }{ + {"disabled_enabled", true}, + {"removed_added", false}, + } { + t.Run(test.name, func(t *testing.T) { + var disabledOrRemoved bool + // We pick a number that will end the loop with enabled/added remote. + for range 20 { + disabledOrRemoved = !disabledOrRemoved + var config []byte + if test.disabled { + if disabledOrRemoved { + config = configWithDisabled + } else { + config = configWithEnabled + } + } else { + if disabledOrRemoved { + config = configWithRemoved + } else { + config = configWithAdded + } + } + err := os.WriteFile(conf2, config, 0666) + require_NoError(t, err) + deadline := time.Now().Add(1500 * time.Millisecond) + for time.Now().Before(deadline) { + err = s2.Reload() + if err == nil || !strings.Contains(err.Error(), "try again") { + break + } + } + require_NoError(t, err) + } + checkLeafNodeConnectedCount(t, s2, 1) + + for { + select { + case errTxt := <-l.errCh: + if strings.Contains(errTxt, DuplicateRemoteLeafnodeConnection.String()) { + t.Fatalf("Duplicates detected: %s", errTxt) + } + default: + // No error, all ok! + return + } + } + }) + } +} + +// This test observes buffers being returned to the block buffer sync.Pools. +// Under the race detector sync.Pool randomly drops puts by design, so it +// cannot run with -race. +func TestNoRaceFileStoreWeakCacheGCRecyclesBuf(t *testing.T) { + // Run everything on a single P so all sync.Pool puts/gets use the same + // P-local pool: buffers recycled by the GC cleanup goroutine would + // otherwise often land in another P's private slot, which Get on this + // goroutine can never steal from and the test couldn't observe them. + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1)) + + // All cache buffers in this test come from the tiny block pool, identify + // them by their backing array so we can spot them in the pool. + bufArr := func(buf []byte) *[defaultTinyBlockSize]byte { + return (*[defaultTinyBlockSize]byte)(buf[0:defaultTinyBlockSize]) + } + // Wait for the GC cleanup to recycle the buffer back into the pool. + // GC and polling must be interleaved carefully: every GC cycle also + // drains the sync.Pools, so polling Get while triggering GCs could + // evict the recycled buffer before we get a chance to observe it. + expectRecycled := func(t *testing.T, arr *[defaultTinyBlockSize]byte) { + t.Helper() + for attempts := 0; attempts < 10; attempts++ { + runtime.GC() + // Poll without triggering more GCs so the pool isn't drained + // while we look for the buffer. + for range 20 { + if blkPoolTiny.Get().(*[defaultTinyBlockSize]byte) == arr { + return + } + time.Sleep(5 * time.Millisecond) + } + } + t.Fatalf("buffer was not recycled into the pool") + } + // Ensure the buffer does not (re)appear in the pool through a GC cleanup. + expectNotRecycled := func(t *testing.T, arr *[defaultTinyBlockSize]byte) { + t.Helper() + for attempts := 0; attempts < 3; attempts++ { + runtime.GC() + // Give a (faulty) cleanup time to run, then poll without + // triggering more GCs so we would reliably see its recycle. + time.Sleep(20 * time.Millisecond) + for range 20 { + if blkPoolTiny.Get().(*[defaultTinyBlockSize]byte) == arr { + t.Fatalf("buffer was recycled into the pool, but should not have been") + } + time.Sleep(2 * time.Millisecond) + } + } + } + + newTestStore := func(t *testing.T) *msgBlock { + t.Helper() + fs, err := newFileStore( + FileStoreConfig{StoreDir: t.TempDir(), BlockSize: 8192}, + StreamConfig{Name: "zzz", Storage: FileStorage, Subjects: []string{"foo"}}, + ) + require_NoError(t, err) + t.Cleanup(func() { fs.Stop() }) + + for range 4 { + _, _, err = fs.StoreMsg("foo", nil, []byte("hello"), 0) + require_NoError(t, err) + } + require_NoError(t, fs.FlushAllPending()) + + fs.mu.Lock() + mb := fs.lmb + fs.mu.Unlock() + return mb + } + + t.Run("recycledOnGC", func(t *testing.T) { + arr := func() *[defaultTinyBlockSize]byte { + c := &cache{buf: getMsgBlockBuf(1)} + c.registerRecycle() + require_True(t, c.clean != (runtime.Cleanup{})) + return bufArr(c.buf) + }() + // The cache is unreachable now, GC should recycle its buffer. + expectRecycled(t, arr) + }) + + t.Run("stopRecycle", func(t *testing.T) { + arr := func() *[defaultTinyBlockSize]byte { + c := &cache{buf: getMsgBlockBuf(1)} + c.registerRecycle() + c.stopRecycle() + require_True(t, c.clean == (runtime.Cleanup{})) + return bufArr(c.buf) + }() + expectNotRecycled(t, arr) + }) + + t.Run("registerRecycleTwice", func(t *testing.T) { + arr := func() *[defaultTinyBlockSize]byte { + c := &cache{buf: getMsgBlockBuf(1)} + // Registering again must cancel the previous registration, + // otherwise the buffer would be recycled twice. + c.registerRecycle() + c.registerRecycle() + return bufArr(c.buf) + }() + expectRecycled(t, arr) + expectNotRecycled(t, arr) + }) + + t.Run("noRecycleNRA", func(t *testing.T) { + arr := func() *[defaultTinyBlockSize]byte { + c := &cache{buf: getMsgBlockBuf(1), nra: true} + c.registerRecycle() + require_True(t, c.clean == (runtime.Cleanup{})) + return bufArr(c.buf) + }() + expectNotRecycled(t, arr) + }) + + t.Run("noRecycleEmptyBuf", func(t *testing.T) { + c := &cache{} + c.registerRecycle() + require_True(t, c.clean == (runtime.Cleanup{})) + // Stopping without a registration, or twice, must not panic. + c.stopRecycle() + c.stopRecycle() + // Neither must registering on a nil cache. + (*cache)(nil).registerRecycle() + }) + + t.Run("weakWriteCacheGC", func(t *testing.T) { + mb := newTestStore(t) + arr := func() *[defaultTinyBlockSize]byte { + mb.mu.Lock() + defer mb.mu.Unlock() + // The write path registered the recycle cleanup when it grew + // the write cache buffer from the pool. + require_NotNil(t, mb.cache) + require_True(t, mb.cache.clean != (runtime.Cleanup{})) + require_Equal(t, cap(mb.cache.buf), defaultTinyBlockSize) + arr := bufArr(mb.cache.buf) + // Drop the strong references so only the weak pointer remains. + mb.finishedWithCache() + require_True(t, mb.cache == nil) + mb.ecache.Weaken() + require_NotNil(t, mb.ecache.Value()) + return arr + }() + + // GC collects the weakly referenced cache and recycles its buffer. + expectRecycled(t, arr) + mb.mu.Lock() + defer mb.mu.Unlock() + require_True(t, mb.ecache.Value() == nil) + }) + + t.Run("weakLoadedCacheGC", func(t *testing.T) { + mb := newTestStore(t) + arr := func() *[defaultTinyBlockSize]byte { + mb.mu.Lock() + defer mb.mu.Unlock() + // Reload the block from disk so indexCacheBuf registers the cleanup. + mb.clearCache() + require_True(t, mb.cache == nil) + require_NoError(t, mb.loadMsgsWithLock()) + require_NotNil(t, mb.cache) + require_True(t, mb.cache.clean != (runtime.Cleanup{})) + require_Equal(t, cap(mb.cache.buf), defaultTinyBlockSize) + arr := bufArr(mb.cache.buf) + mb.finishedWithCache() + require_True(t, mb.cache == nil) + mb.ecache.Weaken() + require_NotNil(t, mb.ecache.Value()) + return arr + }() + + expectRecycled(t, arr) + mb.mu.Lock() + defer mb.mu.Unlock() + require_True(t, mb.ecache.Value() == nil) + }) + + t.Run("clearCacheNoDoubleRecycle", func(t *testing.T) { + mb := newTestStore(t) + arr := func() *[defaultTinyBlockSize]byte { + mb.mu.Lock() + defer mb.mu.Unlock() + require_NotNil(t, mb.cache) + require_Equal(t, cap(mb.cache.buf), defaultTinyBlockSize) + arr := bufArr(mb.cache.buf) + mb.clearCache() + require_True(t, mb.cache == nil) + require_True(t, mb.ecache.Value() == nil) + return arr + }() + + // clearCache recycled the buffer into the pool explicitly once... + expectRecycled(t, arr) + // ...and GC of the disused cache must not recycle it a second time. + expectNotRecycled(t, arr) + }) + + t.Run("writeCacheHandoffNoRecycle", func(t *testing.T) { + mb := newTestStore(t) + arr, buf := func() (*[defaultTinyBlockSize]byte, []byte) { + mb.mu.Lock() + defer mb.mu.Unlock() + require_NotNil(t, mb.cache) + require_Equal(t, cap(mb.cache.buf), defaultTinyBlockSize) + arr := bufArr(mb.cache.buf) + // Pretend the block was never loaded for reads, so the write cache + // buffer is handed off for reuse by the next write block. + mb.llts = 0 + buf := mb.tryExpireWriteCache() + require_True(t, buf != nil) + require_True(t, bufArr(buf) == arr) + // Drop any remaining references to the old cache object. + mb.cache = nil + mb.ecache.Set(nil) + return arr, buf + }() + + // The handed-off buffer is still in use, so GC of the old cache + // object must not recycle it into the pool. + expectNotRecycled(t, arr) + runtime.KeepAlive(buf) + }) + + t.Run("expiredWriteCacheNoHandoffRecycles", func(t *testing.T) { + mb := newTestStore(t) + arr := func() *[defaultTinyBlockSize]byte { + mb.mu.Lock() + defer mb.mu.Unlock() + require_NotNil(t, mb.cache) + require_Equal(t, cap(mb.cache.buf), defaultTinyBlockSize) + arr := bufArr(mb.cache.buf) + // Pretend the block was read, but long ago. The cache will + // expire, but the stale read blocks the buffer handoff. + mb.llts = 1 + buf := mb.tryExpireWriteCache() + require_True(t, buf == nil) + require_True(t, mb.cache == nil) + return arr + }() + + // The buffer was recycled into the pool explicitly once... + expectRecycled(t, arr) + // ...and GC of the disused cache must not recycle it a second time. + expectNotRecycled(t, arr) + }) +} diff --git a/server/ocsp.go b/server/ocsp.go new file mode 100644 index 0000000..05a082c --- /dev/null +++ b/server/ocsp.go @@ -0,0 +1,997 @@ +// Copyright 2021-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/asn1" + "encoding/base64" + "encoding/pem" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "golang.org/x/crypto/ocsp" + + "github.com/nats-io/nats-server/v2/server/certidp" + "github.com/nats-io/nats-server/v2/server/certstore" +) + +const ( + defaultOCSPStoreDir = "ocsp" + defaultOCSPCheckInterval = 24 * time.Hour + minOCSPCheckInterval = 2 * time.Minute +) + +type OCSPMode uint8 + +const ( + // OCSPModeAuto staples a status, only if "status_request" is set in cert. + OCSPModeAuto OCSPMode = iota + + // OCSPModeAlways enforces OCSP stapling for certs and shuts down the server in + // case a server is revoked or cannot get OCSP staples. + OCSPModeAlways + + // OCSPModeNever disables OCSP stapling even if cert has Must-Staple flag. + OCSPModeNever + + // OCSPModeMust honors the Must-Staple flag from a certificate but also causing shutdown + // in case the certificate has been revoked. + OCSPModeMust +) + +// OCSPMonitor monitors the state of a staple per certificate. +type OCSPMonitor struct { + kind string + mu sync.Mutex + raw []byte + srv *Server + certFile string + resp *ocsp.Response + hc *http.Client + stopCh chan struct{} + Leaf *x509.Certificate + Issuer *x509.Certificate + + shutdownOnRevoke bool +} + +func (oc *OCSPMonitor) getNextRun() time.Duration { + oc.mu.Lock() + nextUpdate := oc.resp.NextUpdate + oc.mu.Unlock() + + now := time.Now() + if nextUpdate.IsZero() { + // If response is missing NextUpdate, we check the day after. + // Technically, if NextUpdate is missing, we can try whenever. + // https://tools.ietf.org/html/rfc6960#section-4.2.2.1 + return defaultOCSPCheckInterval + } + dur := nextUpdate.Sub(now) / 2 + + // If negative, then wait a couple of minutes before getting another staple. + if dur < 0 { + return minOCSPCheckInterval + } + + return dur +} + +func (oc *OCSPMonitor) getStatus() ([]byte, *ocsp.Response, error) { + raw, resp := oc.getCacheStatus() + if len(raw) > 0 && resp != nil { + // Check if the OCSP is still valid. + if err := validOCSPResponse(resp); err == nil { + return raw, resp, nil + } + } + var err error + raw, resp, err = oc.getLocalStatus() + if err == nil { + return raw, resp, nil + } + + return oc.getRemoteStatus() +} + +func (oc *OCSPMonitor) getCacheStatus() ([]byte, *ocsp.Response) { + oc.mu.Lock() + defer oc.mu.Unlock() + return oc.raw, oc.resp +} + +func (oc *OCSPMonitor) getLocalStatus() ([]byte, *ocsp.Response, error) { + opts := oc.srv.getOpts() + storeDir := opts.StoreDir + if storeDir == _EMPTY_ { + return nil, nil, fmt.Errorf("store_dir not set") + } + + // This key must be based upon the current full certificate, not the public key, + // so MUST be on the full raw certificate and not an SPKI or other reduced form. + key := fmt.Sprintf("%x", sha256.Sum256(oc.Leaf.Raw)) + + oc.mu.Lock() + raw, err := os.ReadFile(filepath.Join(storeDir, defaultOCSPStoreDir, key)) + oc.mu.Unlock() + if err != nil { + return nil, nil, err + } + + resp, err := ocsp.ParseResponse(raw, oc.Issuer) + if err != nil { + return nil, nil, fmt.Errorf("failed to get local status: %w", err) + } + if err := validOCSPResponse(resp); err != nil { + return nil, nil, err + } + + // Cache the response. + oc.mu.Lock() + oc.raw = raw + oc.resp = resp + oc.mu.Unlock() + + return raw, resp, nil +} + +func (oc *OCSPMonitor) getRemoteStatus() ([]byte, *ocsp.Response, error) { + opts := oc.srv.getOpts() + var overrideURLs []string + if config := opts.OCSPConfig; config != nil { + overrideURLs = config.OverrideURLs + } + getRequestBytes := func(u string, reqDER []byte, hc *http.Client) ([]byte, error) { + reqEnc := base64.StdEncoding.EncodeToString(reqDER) + u = fmt.Sprintf("%s/%s", u, reqEnc) + start := time.Now() + resp, err := hc.Get(u) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + oc.srv.Debugf("Received OCSP response (method=GET, status=%v, url=%s, duration=%.3fs)", + resp.StatusCode, u, time.Since(start).Seconds()) + if resp.StatusCode > 299 { + return nil, fmt.Errorf("non-ok http status on GET request (reqlen=%d): %d", len(reqEnc), resp.StatusCode) + } + return io.ReadAll(resp.Body) + } + postRequestBytes := func(u string, body []byte, hc *http.Client) ([]byte, error) { + hreq, err := http.NewRequest("POST", u, bytes.NewReader(body)) + if err != nil { + return nil, err + } + hreq.Header.Add("Content-Type", "application/ocsp-request") + hreq.Header.Add("Accept", "application/ocsp-response") + + start := time.Now() + resp, err := hc.Do(hreq) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + oc.srv.Debugf("Received OCSP response (method=POST, status=%v, url=%s, duration=%.3fs)", + resp.StatusCode, u, time.Since(start).Seconds()) + if resp.StatusCode > 299 { + return nil, fmt.Errorf("non-ok http status on POST request (reqlen=%d): %d", len(body), resp.StatusCode) + } + return io.ReadAll(resp.Body) + } + + // Request documentation: + // https://tools.ietf.org/html/rfc6960#appendix-A.1 + + reqDER, err := ocsp.CreateRequest(oc.Leaf, oc.Issuer, nil) + if err != nil { + return nil, nil, err + } + + responders := oc.Leaf.OCSPServer + if len(overrideURLs) > 0 { + responders = overrideURLs + } + if len(responders) == 0 { + return nil, nil, fmt.Errorf("no available ocsp servers") + } + + oc.mu.Lock() + hc := oc.hc + oc.mu.Unlock() + + var raw []byte + for _, u := range responders { + var postErr, getErr error + u = strings.TrimSuffix(u, "/") + // Prefer to make POST requests first. + raw, postErr = postRequestBytes(u, reqDER, hc) + if postErr == nil { + err = nil + break + } else { + // Fallback to use a GET request. + raw, getErr = getRequestBytes(u, reqDER, hc) + if getErr == nil { + err = nil + break + } else { + err = errors.Join(postErr, getErr) + } + } + } + if err != nil { + return nil, nil, fmt.Errorf("exhausted ocsp servers: %w", err) + } + resp, err := ocsp.ParseResponse(raw, oc.Issuer) + if err != nil { + return nil, nil, fmt.Errorf("failed to get remote status: %w", err) + } + if err := validOCSPResponse(resp); err != nil { + return nil, nil, err + } + + if storeDir := opts.StoreDir; storeDir != _EMPTY_ { + key := fmt.Sprintf("%x", sha256.Sum256(oc.Leaf.Raw)) + if err := oc.writeOCSPStatus(storeDir, key, raw); err != nil { + return nil, nil, fmt.Errorf("failed to write ocsp status: %w", err) + } + } + + oc.mu.Lock() + oc.raw = raw + oc.resp = resp + oc.mu.Unlock() + + return raw, resp, nil +} + +func (oc *OCSPMonitor) run() { + s := oc.srv + s.mu.Lock() + quitCh := s.quitCh + s.mu.Unlock() + + var doShutdown bool + defer func() { + // Need to decrement before shuting down, otherwise shutdown + // would be stuck waiting on grWG to go down to 0. + s.grWG.Done() + if doShutdown { + s.Shutdown() + } + }() + + oc.mu.Lock() + shutdownOnRevoke := oc.shutdownOnRevoke + certFile := oc.certFile + stopCh := oc.stopCh + kind := oc.kind + oc.mu.Unlock() + + var nextRun time.Duration + _, resp, err := oc.getStatus() + if err == nil && resp.Status == ocsp.Good { + nextRun = oc.getNextRun() + t := resp.NextUpdate.Format(time.RFC3339Nano) + s.Noticef( + "Found OCSP status for %s certificate at '%s': good, next update %s, checking again in %s", + kind, certFile, t, nextRun, + ) + } else if err == nil && shutdownOnRevoke { + // If resp.Status is ocsp.Revoked, ocsp.Unknown, or any other value. + s.Errorf("Found OCSP status for %s certificate at '%s': %s", kind, certFile, ocspStatusString(resp.Status)) + doShutdown = true + return + } + + nextRunTimer := time.NewTimer(nextRun) + defer stopAndClearTimer(&nextRunTimer) + + for { + // On reload, if the certificate changes then need to stop this monitor. + select { + case <-nextRunTimer.C: + case <-stopCh: + // In case of reload and have to restart the OCSP stapling monitoring. + return + case <-quitCh: + // Server quit channel. + return + } + _, resp, err := oc.getRemoteStatus() + if err != nil { + nextRun = oc.getNextRun() + s.Errorf("Bad OCSP status update for certificate '%s': %s, trying again in %v", certFile, err, nextRun) + nextRunTimer.Reset(nextRun) + continue + } + + switch n := resp.Status; n { + case ocsp.Good: + nextRun = oc.getNextRun() + t := resp.NextUpdate.Format(time.RFC3339Nano) + s.Noticef( + "Received OCSP status for %s certificate '%s': good, next update %s, checking again in %s", + kind, certFile, t, nextRun, + ) + nextRunTimer.Reset(nextRun) + continue + default: + s.Errorf("Received OCSP status for %s certificate '%s': %s", kind, certFile, ocspStatusString(n)) + if shutdownOnRevoke { + doShutdown = true + } + return + } + } +} + +func (oc *OCSPMonitor) stop() { + oc.mu.Lock() + stopCh := oc.stopCh + oc.mu.Unlock() + stopCh <- struct{}{} +} + +// NewOCSPMonitor takes a TLS configuration then wraps it with the callbacks set for OCSP verification +// along with a monitor that will periodically fetch OCSP staples. +func (srv *Server) NewOCSPMonitor(config *tlsConfigKind) (*tls.Config, *OCSPMonitor, error) { + kind := config.kind + tc := config.tlsConfig + tcOpts := config.tlsOpts + opts := srv.getOpts() + oc := opts.OCSPConfig + + // We need to track the CA certificate in case the CA is not present + // in the chain to be able to verify the signature of the OCSP staple. + var ( + certFile string + caFile string + ) + if kind == kindStringMap[CLIENT] { + tcOpts = opts.tlsConfigOpts + if opts.TLSCert != _EMPTY_ { + certFile = opts.TLSCert + } + if opts.TLSCaCert != _EMPTY_ { + caFile = opts.TLSCaCert + } + } + if tcOpts != nil { + certFile = tcOpts.CertFile + caFile = tcOpts.CaFile + } + + // NOTE: Currently OCSP Stapling is enabled only for the first certificate found. + var mon *OCSPMonitor + for _, currentCert := range tc.Certificates { + // Create local copy since this will be used in the GetCertificate callback. + cert := currentCert + + // This is normally non-nil, but can still be nil here when in tests + // or in some embedded scenarios. + if cert.Leaf == nil { + if len(cert.Certificate) <= 0 { + return nil, nil, fmt.Errorf("no certificate found") + } + var err error + cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return nil, nil, fmt.Errorf("error parsing certificate: %v", err) + } + } + var shutdownOnRevoke bool + mustStaple := hasOCSPStatusRequest(cert.Leaf) + if oc != nil { + switch { + case oc.Mode == OCSPModeNever: + if mustStaple { + srv.Warnf("Certificate at '%s' has MustStaple but OCSP is disabled", certFile) + } + return tc, nil, nil + case oc.Mode == OCSPModeAlways: + // Start the monitor for this cert even if it does not have + // the MustStaple flag and shutdown the server in case the + // staple ever gets revoked. + mustStaple = true + shutdownOnRevoke = true + case oc.Mode == OCSPModeMust && mustStaple: + shutdownOnRevoke = true + case oc.Mode == OCSPModeAuto && !mustStaple: + // "status_request" MustStaple flag not set in certificate. No need to do anything. + return tc, nil, nil + } + } + if !mustStaple { + // No explicit OCSP config and cert does not have MustStaple flag either. + return tc, nil, nil + } + + if err := srv.setupOCSPStapleStoreDir(); err != nil { + return nil, nil, err + } + + // TODO: Add OCSP 'responder_cert' option in case CA cert not available. + issuer, err := getOCSPIssuer(caFile, cert.Certificate) + if err != nil { + return nil, nil, err + } + + mon = &OCSPMonitor{ + kind: kind, + srv: srv, + hc: &http.Client{Timeout: 30 * time.Second}, + shutdownOnRevoke: shutdownOnRevoke, + certFile: certFile, + stopCh: make(chan struct{}, 1), + Leaf: cert.Leaf, + Issuer: issuer, + } + + // Get the certificate status from the memory, then remote OCSP responder. + if _, resp, err := mon.getStatus(); err != nil { + return nil, nil, fmt.Errorf("bad OCSP status update for certificate at '%s': %s", certFile, err) + } else if resp != nil && resp.Status != ocsp.Good && shutdownOnRevoke { + return nil, nil, fmt.Errorf("found existing OCSP status for certificate at '%s': %s", certFile, ocspStatusString(resp.Status)) + } + + // Callbacks below will be in charge of returning the certificate instead, + // so this has to be nil. + tc.Certificates = nil + + // GetCertificate returns a certificate that's presented to a client. + tc.GetCertificate = func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { + ccert := cert + raw, _, err := mon.getStatus() + if err != nil { + return nil, err + } + return &tls.Certificate{ + OCSPStaple: raw, + Certificate: ccert.Certificate, + PrivateKey: ccert.PrivateKey, + SupportedSignatureAlgorithms: ccert.SupportedSignatureAlgorithms, + SignedCertificateTimestamps: ccert.SignedCertificateTimestamps, + Leaf: ccert.Leaf, + }, nil + } + + // Check whether need to verify staples from a peer router or gateway connection. + switch kind { + case kindStringMap[ROUTER], kindStringMap[GATEWAY]: + tc.VerifyConnection = func(s tls.ConnectionState) error { + oresp := s.OCSPResponse + if oresp == nil { + return fmt.Errorf("%s peer missing OCSP Staple", kind) + } + + // Peer connections will verify the response of the staple. + if len(s.VerifiedChains) == 0 { + return fmt.Errorf("%s peer missing TLS verified chains", kind) + } + + chain := s.VerifiedChains[0] + peerLeaf := chain[0] + peerIssuer := certidp.GetLeafIssuerCert(chain, 0) + if peerIssuer == nil { + return fmt.Errorf("failed to get issuer certificate for %s peer", kind) + } + + // Response signature of issuer or issuer delegate is checked in the library parse + resp, err := ocsp.ParseResponseForCert(oresp, peerLeaf, peerIssuer) + if err != nil { + return fmt.Errorf("failed to parse OCSP response from %s peer: %w", kind, err) + } + + // If signer was issuer delegate double-check issuer delegate authorization + if resp.Certificate != nil { + ok := false + for _, eku := range resp.Certificate.ExtKeyUsage { + if eku == x509.ExtKeyUsageOCSPSigning { + ok = true + break + } + } + if !ok { + return fmt.Errorf("OCSP staple's signer missing authorization by CA to act as OCSP signer") + } + } + + // Check that the OCSP response is effective, take defaults for clockskew and default validity + peerOpts := certidp.OCSPPeerConfig{ClockSkew: -1, TTLUnsetNextUpdate: -1} + sLog := certidp.Log{Debugf: srv.Debugf} + if !certidp.OCSPResponseCurrent(resp, &peerOpts, &sLog) { + return fmt.Errorf("OCSP staple from %s peer not current", kind) + } + + if resp.Status != ocsp.Good { + return fmt.Errorf("bad status for OCSP Staple from %s peer: %s", kind, ocspStatusString(resp.Status)) + } + + return nil + } + + // When server makes a peer connection, need to also present an OCSP Staple. + tc.GetClientCertificate = func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) { + ccert := cert + raw, _, err := mon.getStatus() + if err != nil { + return nil, err + } + // NOTE: crypto/tls.sendClientCertificate internally also calls getClientCertificate + // so if for some reason these callbacks are triggered concurrently during a reconnect + // there can be a race. To avoid that, the OCSP monitor lock is used to serialize access + // to the staple which could also change inflight during an update. + mon.mu.Lock() + ccert.OCSPStaple = raw + mon.mu.Unlock() + + return &ccert, nil + } + default: + // GetClientCertificate returns a certificate that's presented to a server. + tc.GetClientCertificate = func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) { + return &cert, nil + } + } + } + return tc, mon, nil +} + +func (s *Server) setupOCSPStapleStoreDir() error { + opts := s.getOpts() + storeDir := opts.StoreDir + if storeDir == _EMPTY_ { + return nil + } + storeDir = filepath.Join(storeDir, defaultOCSPStoreDir) + if stat, err := os.Stat(storeDir); os.IsNotExist(err) { + if err := os.MkdirAll(storeDir, defaultDirPerms); err != nil { + return fmt.Errorf("could not create OCSP storage directory - %v", err) + } + } else if stat == nil || !stat.IsDir() { + return fmt.Errorf("OCSP storage directory is not a directory") + } + return nil +} + +type tlsConfigKind struct { + tlsConfig *tls.Config + tlsOpts *TLSConfigOpts + kind string + isLeafSpoke bool + apply func(*tls.Config) +} + +func (s *Server) configureOCSP() []*tlsConfigKind { + sopts := s.getOpts() + + configs := make([]*tlsConfigKind, 0) + + if config := sopts.TLSConfig; config != nil { + opts := sopts.tlsConfigOpts + o := &tlsConfigKind{ + kind: kindStringMap[CLIENT], + tlsConfig: config, + tlsOpts: opts, + apply: func(tc *tls.Config) { sopts.TLSConfig = tc }, + } + configs = append(configs, o) + } + if config := sopts.Websocket.TLSConfig; config != nil { + opts := sopts.Websocket.tlsConfigOpts + o := &tlsConfigKind{ + kind: kindStringMap[CLIENT], + tlsConfig: config, + tlsOpts: opts, + apply: func(tc *tls.Config) { sopts.Websocket.TLSConfig = tc }, + } + configs = append(configs, o) + } + if config := sopts.MQTT.TLSConfig; config != nil { + opts := sopts.tlsConfigOpts + o := &tlsConfigKind{ + kind: kindStringMap[CLIENT], + tlsConfig: config, + tlsOpts: opts, + apply: func(tc *tls.Config) { sopts.MQTT.TLSConfig = tc }, + } + configs = append(configs, o) + } + if config := sopts.Cluster.TLSConfig; config != nil { + opts := sopts.Cluster.tlsConfigOpts + o := &tlsConfigKind{ + kind: kindStringMap[ROUTER], + tlsConfig: config, + tlsOpts: opts, + apply: func(tc *tls.Config) { sopts.Cluster.TLSConfig = tc }, + } + configs = append(configs, o) + } + if config := sopts.LeafNode.TLSConfig; config != nil { + opts := sopts.LeafNode.tlsConfigOpts + o := &tlsConfigKind{ + kind: kindStringMap[LEAF], + tlsConfig: config, + tlsOpts: opts, + apply: func(tc *tls.Config) { sopts.LeafNode.TLSConfig = tc }, + } + configs = append(configs, o) + } + for _, remote := range sopts.LeafNode.Remotes { + if config := remote.TLSConfig; config != nil { + // Use a copy of the remote here since will be used + // in the apply func callback below. + r, opts := remote, remote.tlsConfigOpts + o := &tlsConfigKind{ + kind: kindStringMap[LEAF], + tlsConfig: config, + tlsOpts: opts, + isLeafSpoke: true, + apply: func(tc *tls.Config) { r.TLSConfig = tc }, + } + configs = append(configs, o) + } + } + if config := sopts.Gateway.TLSConfig; config != nil { + opts := sopts.Gateway.tlsConfigOpts + o := &tlsConfigKind{ + kind: kindStringMap[GATEWAY], + tlsConfig: config, + tlsOpts: opts, + apply: func(tc *tls.Config) { sopts.Gateway.TLSConfig = tc }, + } + configs = append(configs, o) + } + for _, remote := range sopts.Gateway.Gateways { + if config := remote.TLSConfig; config != nil { + gw, opts := remote, remote.tlsConfigOpts + o := &tlsConfigKind{ + kind: kindStringMap[GATEWAY], + tlsConfig: config, + tlsOpts: opts, + apply: func(tc *tls.Config) { gw.TLSConfig = tc }, + } + configs = append(configs, o) + } + } + return configs +} + +func (s *Server) enableOCSP() error { + configs := s.configureOCSP() + + for _, config := range configs { + + // We do not staple Leaf Hub and Leaf Spokes, use ocsp_peer + if config.kind != kindStringMap[LEAF] { + // OCSP Stapling feature, will also enable tls server peer check for gateway and route peers + tc, mon, err := s.NewOCSPMonitor(config) + if err != nil { + return err + } + // Check if an OCSP stapling monitor is required for this certificate. + if mon != nil { + s.ocsps = append(s.ocsps, mon) + + // Override the TLS config with one that follows OCSP stapling + config.apply(tc) + } + } + + // OCSP peer check (client mTLS, leaf mTLS, leaf remote TLS) + if config.kind == kindStringMap[CLIENT] || config.kind == kindStringMap[LEAF] { + tc, plugged, err := s.plugTLSOCSPPeer(config) + if err != nil { + return err + } + if plugged && tc != nil { + s.ocspPeerVerify = true + config.apply(tc) + } + } + } + + return nil +} + +func (s *Server) startOCSPMonitoring() { + s.mu.Lock() + ocsps := s.ocsps + s.mu.Unlock() + if ocsps == nil { + return + } + for _, mon := range ocsps { + m := mon + m.mu.Lock() + kind := m.kind + m.mu.Unlock() + s.Noticef("OCSP Stapling enabled for %s connections", kind) + s.startGoRoutine(func() { m.run() }) + } +} + +func (s *Server) reloadOCSP() error { + if err := s.setupOCSPStapleStoreDir(); err != nil { + return err + } + + s.mu.Lock() + ocsps := s.ocsps + s.mu.Unlock() + + // Stop all OCSP Stapling monitors in case there were any running. + for _, oc := range ocsps { + oc.stop() + } + + configs := s.configureOCSP() + + // Restart the monitors under the new configuration. + ocspm := make([]*OCSPMonitor, 0) + + // Reset server's ocspPeerVerify flag to re-detect at least one plugged OCSP peer + s.mu.Lock() + s.ocspPeerVerify = false + s.mu.Unlock() + s.stopOCSPResponseCache() + + for _, config := range configs { + // We do not staple Leaf Hub and Leaf Spokes, use ocsp_peer + if config.kind != kindStringMap[LEAF] { + tc, mon, err := s.NewOCSPMonitor(config) + if err != nil { + return err + } + // Check if an OCSP stapling monitor is required for this certificate. + if mon != nil { + ocspm = append(ocspm, mon) + + // Apply latest TLS configuration after OCSP monitors have started. + defer config.apply(tc) + } + } + + // OCSP peer check (client mTLS, leaf mTLS, leaf remote TLS) + if config.kind == kindStringMap[CLIENT] || config.kind == kindStringMap[LEAF] { + tc, plugged, err := s.plugTLSOCSPPeer(config) + if err != nil { + return err + } + if plugged && tc != nil { + s.ocspPeerVerify = true + defer config.apply(tc) + } + } + } + + // Replace stopped monitors with the new ones. + s.mu.Lock() + s.ocsps = ocspm + s.mu.Unlock() + + // Dispatch all goroutines once again. + s.startOCSPMonitoring() + + // Init and restart OCSP responder cache + s.stopOCSPResponseCache() + s.initOCSPResponseCache() + s.startOCSPResponseCache() + + return nil +} + +func hasOCSPStatusRequest(cert *x509.Certificate) bool { + // OID for id-pe-tlsfeature defined in RFC here: + // https://datatracker.ietf.org/doc/html/rfc7633 + tlsFeatures := asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 1, 24} + const statusRequestExt = 5 + + // Example values: + // * [48 3 2 1 5] - seen when creating own certs locally + // * [30 3 2 1 5] - seen in the wild + // Documentation: + // https://tools.ietf.org/html/rfc6066 + + for _, ext := range cert.Extensions { + if !ext.Id.Equal(tlsFeatures) { + continue + } + + var val []int + rest, err := asn1.Unmarshal(ext.Value, &val) + if err != nil || len(rest) > 0 { + return false + } + + for _, n := range val { + if n == statusRequestExt { + return true + } + } + break + } + + return false +} + +// writeOCSPStatus writes an OCSP status to a temporary file then moves it to a +// new path, in an attempt to avoid corrupting existing data. +func (oc *OCSPMonitor) writeOCSPStatus(storeDir, file string, data []byte) error { + storeDir = filepath.Join(storeDir, defaultOCSPStoreDir) + tmp, err := os.CreateTemp(storeDir, "tmp-cert-status") + if err != nil { + return err + } + + if _, err := tmp.Write(data); err != nil { + tmp.Close() + os.Remove(tmp.Name()) + return err + } + if err := tmp.Close(); err != nil { + return err + } + + oc.mu.Lock() + err = os.Rename(tmp.Name(), filepath.Join(storeDir, file)) + oc.mu.Unlock() + if err != nil { + os.Remove(tmp.Name()) + return err + } + + return nil +} + +func parseCertPEM(name string) ([]*x509.Certificate, error) { + data, err := os.ReadFile(name) + if err != nil { + return nil, err + } + + var pemBytes []byte + + var block *pem.Block + for len(data) != 0 { + block, data = pem.Decode(data) + if block == nil { + break + } + if block.Type != "CERTIFICATE" { + return nil, fmt.Errorf("unexpected PEM certificate type: %s", block.Type) + } + + pemBytes = append(pemBytes, block.Bytes...) + } + + return x509.ParseCertificates(pemBytes) +} + +// getOCSPIssuerLocally determines a leaf's issuer from locally configured certificates +func getOCSPIssuerLocally(trustedCAs []*x509.Certificate, certBundle []*x509.Certificate) (*x509.Certificate, error) { + var vOpts x509.VerifyOptions + var leaf *x509.Certificate + trustedCAPool := x509.NewCertPool() + + // Require Leaf as first cert in bundle + if len(certBundle) > 0 { + leaf = certBundle[0] + } else { + return nil, fmt.Errorf("invalid ocsp ca configuration") + } + + // Allow Issuer to be configured as second cert in bundle + if len(certBundle) > 1 { + // The operator may have misconfigured the cert bundle + issuerCandidate := certBundle[1] + err := issuerCandidate.CheckSignature(leaf.SignatureAlgorithm, leaf.RawTBSCertificate, leaf.Signature) + if err != nil { + return nil, fmt.Errorf("invalid issuer configuration: %w", err) + } else { + return issuerCandidate, nil + } + } + + // Operator did not provide the Leaf Issuer in cert bundle second position + // so we will attempt to create at least one ordered verified chain from the + // trusted CA pool. + + // Specify CA trust store to validator; if unset, system trust store used + if len(trustedCAs) > 0 { + for _, ca := range trustedCAs { + trustedCAPool.AddCert(ca) + } + vOpts.Roots = trustedCAPool + } + + return certstore.GetLeafIssuer(leaf, vOpts), nil +} + +// getOCSPIssuer determines an issuer certificate from the cert (bundle) or the file-based CA trust store +func getOCSPIssuer(caFile string, chain [][]byte) (*x509.Certificate, error) { + var issuer *x509.Certificate + var trustedCAs []*x509.Certificate + var certBundle []*x509.Certificate + var err error + + // FIXME(tgb): extend if pluggable CA store provider added to NATS (i.e. other than PEM file) + + // Non-system default CA trust store passed + if caFile != _EMPTY_ { + trustedCAs, err = parseCertPEM(caFile) + if err != nil { + return nil, fmt.Errorf("failed to parse ca_file: %v", err) + } + } + + // Specify bundled intermediate CA store + for _, certBytes := range chain { + cert, err := x509.ParseCertificate(certBytes) + if err != nil { + return nil, fmt.Errorf("failed to parse cert: %v", err) + } + certBundle = append(certBundle, cert) + } + + issuer, err = getOCSPIssuerLocally(trustedCAs, certBundle) + if err != nil || issuer == nil { + return nil, fmt.Errorf("no issuers found") + } + + if !issuer.IsCA { + return nil, fmt.Errorf("%s invalid ca basic constraints: is not ca", issuer.Subject) + } + return issuer, nil +} + +func ocspStatusString(n int) string { + switch n { + case ocsp.Good: + return "good" + case ocsp.Revoked: + return "revoked" + default: + return "unknown" + } +} + +func validOCSPResponse(r *ocsp.Response) error { + // Time validation not handled by ParseResponse. + // https://tools.ietf.org/html/rfc6960#section-4.2.2.1 + if !r.NextUpdate.IsZero() && r.NextUpdate.Before(time.Now()) { + t := r.NextUpdate.Format(time.RFC3339Nano) + return fmt.Errorf("invalid ocsp NextUpdate, is past time: %s", t) + } + if r.ThisUpdate.After(time.Now()) { + t := r.ThisUpdate.Format(time.RFC3339Nano) + return fmt.Errorf("invalid ocsp ThisUpdate, is future time: %s", t) + } + + return nil +} diff --git a/server/ocsp_peer.go b/server/ocsp_peer.go new file mode 100644 index 0000000..22d42ef --- /dev/null +++ b/server/ocsp_peer.go @@ -0,0 +1,405 @@ +// Copyright 2023-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "strings" + "time" + + "golang.org/x/crypto/ocsp" + + "github.com/nats-io/nats-server/v2/server/certidp" +) + +func parseOCSPPeer(v any) (pcfg *certidp.OCSPPeerConfig, retError error) { + var lt token + defer convertPanicToError(<, &retError) + tk, v := unwrapValue(v, <) + cm, ok := v.(map[string]any) + if !ok { + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrIllegalPeerOptsConfig, v)} + } + pcfg = certidp.NewOCSPPeerConfig() + retError = nil + for mk, mv := range cm { + tk, mv = unwrapValue(mv, <) + switch strings.ToLower(mk) { + case "verify": + verify, ok := mv.(bool) + if !ok { + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldGeneric, mk)} + } + pcfg.Verify = verify + case "allowed_clockskew": + at := float64(0) + switch mv := mv.(type) { + case int64: + at = float64(mv) + case float64: + at = mv + case string: + d, err := time.ParseDuration(mv) + if err != nil { + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, "unexpected type")} + } + at = d.Seconds() + default: + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, "unexpected type")} + } + if at >= 0 { + pcfg.ClockSkew = at + } + case "ca_timeout": + at := float64(0) + switch mv := mv.(type) { + case int64: + at = float64(mv) + case float64: + at = mv + case string: + d, err := time.ParseDuration(mv) + if err != nil { + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, err)} + } + at = d.Seconds() + default: + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, "unexpected type")} + } + if at >= 0 { + pcfg.Timeout = at + } + case "cache_ttl_when_next_update_unset": + at := float64(0) + switch mv := mv.(type) { + case int64: + at = float64(mv) + case float64: + at = mv + case string: + d, err := time.ParseDuration(mv) + if err != nil { + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, err)} + } + at = d.Seconds() + default: + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, "unexpected type")} + } + if at >= 0 { + pcfg.TTLUnsetNextUpdate = at + } + case "warn_only": + warnOnly, ok := mv.(bool) + if !ok { + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldGeneric, mk)} + } + pcfg.WarnOnly = warnOnly + case "unknown_is_good": + unknownIsGood, ok := mv.(bool) + if !ok { + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldGeneric, mk)} + } + pcfg.UnknownIsGood = unknownIsGood + case "allow_when_ca_unreachable": + allowWhenCAUnreachable, ok := mv.(bool) + if !ok { + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldGeneric, mk)} + } + pcfg.AllowWhenCAUnreachable = allowWhenCAUnreachable + default: + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldGeneric, mk)} + } + } + return pcfg, nil +} + +func peerFromVerifiedChains(chains [][]*x509.Certificate) *x509.Certificate { + if len(chains) == 0 || len(chains[0]) == 0 { + return nil + } + return chains[0][0] +} + +// plugTLSOCSPPeer will plug the TLS handshake lifecycle for client mTLS connections and Leaf connections +func (s *Server) plugTLSOCSPPeer(config *tlsConfigKind) (*tls.Config, bool, error) { + if config == nil || config.tlsConfig == nil { + return nil, false, errors.New(certidp.ErrUnableToPlugTLSEmptyConfig) + } + kind := config.kind + isSpoke := config.isLeafSpoke + tcOpts := config.tlsOpts + if tcOpts == nil || tcOpts.OCSPPeerConfig == nil || !tcOpts.OCSPPeerConfig.Verify { + return nil, false, nil + } + s.Debugf(certidp.DbgPlugTLSForKind, config.kind) + // peer is a tls client + if kind == kindStringMap[CLIENT] || (kind == kindStringMap[LEAF] && !isSpoke) { + if !tcOpts.Verify { + return nil, false, errors.New(certidp.ErrMTLSRequired) + } + return s.plugClientTLSOCSPPeer(config) + } + // peer is a tls server + if kind == kindStringMap[LEAF] && isSpoke { + return s.plugServerTLSOCSPPeer(config) + } + return nil, false, nil +} + +func (s *Server) plugClientTLSOCSPPeer(config *tlsConfigKind) (*tls.Config, bool, error) { + if config == nil || config.tlsConfig == nil || config.tlsOpts == nil { + return nil, false, errors.New(certidp.ErrUnableToPlugTLSClient) + } + tc := config.tlsConfig + tcOpts := config.tlsOpts + kind := config.kind + if tcOpts.OCSPPeerConfig == nil || !tcOpts.OCSPPeerConfig.Verify { + return tc, false, nil + } + tc.VerifyConnection = func(cs tls.ConnectionState) error { + if !s.tlsClientOCSPValid(cs.VerifiedChains, tcOpts.OCSPPeerConfig) { + s.sendOCSPPeerRejectEvent(kind, peerFromVerifiedChains(cs.VerifiedChains), certidp.MsgTLSClientRejectConnection) + return errors.New(certidp.MsgTLSClientRejectConnection) + } + return nil + } + return tc, true, nil +} + +func (s *Server) plugServerTLSOCSPPeer(config *tlsConfigKind) (*tls.Config, bool, error) { + if config == nil || config.tlsConfig == nil || config.tlsOpts == nil { + return nil, false, errors.New(certidp.ErrUnableToPlugTLSServer) + } + tc := config.tlsConfig + tcOpts := config.tlsOpts + kind := config.kind + if tcOpts.OCSPPeerConfig == nil || !tcOpts.OCSPPeerConfig.Verify { + return tc, false, nil + } + tc.VerifyConnection = func(cs tls.ConnectionState) error { + if !s.tlsServerOCSPValid(cs.VerifiedChains, tcOpts.OCSPPeerConfig) { + s.sendOCSPPeerRejectEvent(kind, peerFromVerifiedChains(cs.VerifiedChains), certidp.MsgTLSServerRejectConnection) + return errors.New(certidp.MsgTLSServerRejectConnection) + } + return nil + } + return tc, true, nil +} + +// tlsServerOCSPValid evaluates verified chains (post successful TLS handshake) against OCSP +// eligibility. A verified chain is considered OCSP Valid if either none of the links are +// OCSP eligible, or current "good" responses from the CA can be obtained for each eligible link. +// Upon first OCSP Valid chain found, the Server is deemed OCSP Valid. If none of the chains are +// OCSP Valid, the Server is deemed OCSP Invalid. A verified self-signed certificate (chain length 1) +// is also considered OCSP Valid. +func (s *Server) tlsServerOCSPValid(chains [][]*x509.Certificate, opts *certidp.OCSPPeerConfig) bool { + s.Debugf(certidp.DbgNumServerChains, len(chains)) + return s.peerOCSPValid(chains, opts) +} + +// tlsClientOCSPValid evaluates verified chains (post successful TLS handshake) against OCSP +// eligibility. A verified chain is considered OCSP Valid if either none of the links are +// OCSP eligible, or current "good" responses from the CA can be obtained for each eligible link. +// Upon first OCSP Valid chain found, the Client is deemed OCSP Valid. If none of the chains are +// OCSP Valid, the Client is deemed OCSP Invalid. A verified self-signed certificate (chain length 1) +// is also considered OCSP Valid. +func (s *Server) tlsClientOCSPValid(chains [][]*x509.Certificate, opts *certidp.OCSPPeerConfig) bool { + s.Debugf(certidp.DbgNumClientChains, len(chains)) + return s.peerOCSPValid(chains, opts) +} + +func (s *Server) peerOCSPValid(chains [][]*x509.Certificate, opts *certidp.OCSPPeerConfig) bool { + peer := peerFromVerifiedChains(chains) + if peer == nil { + s.Errorf(certidp.ErrPeerEmptyAutoReject) + return false + } + for ci, chain := range chains { + s.Debugf(certidp.DbgLinksInChain, ci, len(chain)) + // Self-signed certificate is Client OCSP Valid (no CA) + if len(chain) == 1 { + s.Debugf(certidp.DbgSelfSignedValid, ci) + return true + } + // Check if any of the links in the chain are OCSP eligible + chainEligible := false + var eligibleLinks []*certidp.ChainLink + // Iterate over links skipping the root cert which is not OCSP eligible (self == issuer) + for linkPos := 0; linkPos < len(chain)-1; linkPos++ { + cert := chain[linkPos] + link := &certidp.ChainLink{ + Leaf: cert, + } + if certidp.CertOCSPEligible(link) { + chainEligible = true + issuerCert := certidp.GetLeafIssuerCert(chain, linkPos) + if issuerCert == nil { + // unexpected chain condition, reject Client as OCSP Invalid + return false + } + link.Issuer = issuerCert + eligibleLinks = append(eligibleLinks, link) + } + } + // A trust-store verified chain that is not OCSP eligible is always OCSP Valid + if !chainEligible { + s.Debugf(certidp.DbgValidNonOCSPChain, ci) + return true + } + s.Debugf(certidp.DbgChainIsOCSPEligible, ci, len(eligibleLinks)) + // Chain has at least one OCSP eligible link, so check each eligible link; + // any link with a !good OCSP response chain OCSP Invalid + chainValid := true + for _, link := range eligibleLinks { + // if option selected, good could reflect either ocsp.Good or ocsp.Unknown + if badReason, good := s.certOCSPGood(link, opts); !good { + s.Debugf(badReason) + s.sendOCSPPeerChainlinkInvalidEvent(peer, link.Leaf, badReason) + chainValid = false + break + } + } + if chainValid { + s.Debugf(certidp.DbgChainIsOCSPValid, ci) + return true + } + } + // If we are here, all chains had OCSP eligible links, but none of the chains achieved OCSP valid + s.Debugf(certidp.DbgNoOCSPValidChains) + return false +} + +func (s *Server) certOCSPGood(link *certidp.ChainLink, opts *certidp.OCSPPeerConfig) (string, bool) { + if link == nil || link.Leaf == nil || link.Issuer == nil || link.OCSPWebEndpoints == nil || len(*link.OCSPWebEndpoints) < 1 { + return "Empty chainlink found", false + } + var err error + sLogs := &certidp.Log{ + Debugf: s.Debugf, + Noticef: s.Noticef, + Warnf: s.Warnf, + Errorf: s.Errorf, + Tracef: s.Tracef, + } + fingerprint := certidp.GenerateFingerprint(link.Leaf) + // Used for debug/operator only, not match + subj := certidp.GetSubjectDNForm(link.Leaf) + var rawResp []byte + var ocspr *ocsp.Response + var useCachedResp bool + var rc = s.ocsprc + var cachedRevocation bool + // Check our cache before calling out to the CA OCSP responder + s.Debugf(certidp.DbgCheckingCacheForCert, subj, fingerprint) + if rawResp = rc.Get(fingerprint, sLogs); len(rawResp) > 0 { + // Signature validation of CA's OCSP response occurs in ParseResponse + ocspr, err = ocsp.ParseResponse(rawResp, link.Issuer) + if err == nil && ocspr != nil { + // Check if OCSP Response delegation present and if so is valid + if !certidp.ValidDelegationCheck(link.Issuer, ocspr) { + // Invalid delegation was already in cache, purge it and don't use it + s.Debugf(certidp.MsgCachedOCSPResponseInvalid, subj) + rc.Delete(fingerprint, true, sLogs) + goto AFTERCACHE + } + if certidp.OCSPResponseCurrent(ocspr, opts, sLogs) { + s.Debugf(certidp.DbgCurrentResponseCached, certidp.GetStatusAssertionStr(ocspr.Status)) + useCachedResp = true + } else { + // Cached response is not current, delete it and tidy runtime stats to reflect a miss; + // if preserve_revoked is enabled, the cache will not delete the cached response + s.Debugf(certidp.DbgExpiredResponseCached, certidp.GetStatusAssertionStr(ocspr.Status)) + rc.Delete(fingerprint, true, sLogs) + } + // Regardless of currency, record a cached revocation found in case AllowWhenCAUnreachable is set + if ocspr.Status == ocsp.Revoked { + cachedRevocation = true + } + } else { + // Bogus cached assertion, purge it and don't use it + s.Debugf(certidp.MsgCachedOCSPResponseInvalid, subj, fingerprint) + rc.Delete(fingerprint, true, sLogs) + goto AFTERCACHE + } + } +AFTERCACHE: + if !useCachedResp { + // CA OCSP responder callout needed + rawResp, err = certidp.FetchOCSPResponse(link, opts, sLogs) + if err != nil || rawResp == nil || len(rawResp) == 0 { + s.Warnf(certidp.ErrCAResponderCalloutFail, subj, err) + if opts.WarnOnly { + s.Warnf(certidp.MsgAllowWarnOnlyOccurred, subj) + return _EMPTY_, true + } + if opts.AllowWhenCAUnreachable && !cachedRevocation { + // Link has no cached history of revocation, so allow it to pass + s.Warnf(certidp.MsgAllowWhenCAUnreachableOccurred, subj) + return _EMPTY_, true + } else if opts.AllowWhenCAUnreachable { + // Link has cached but expired revocation so reject when CA is unreachable + s.Warnf(certidp.MsgAllowWhenCAUnreachableOccurredCachedRevoke, subj) + } + return certidp.MsgFailedOCSPResponseFetch, false + } + // Signature validation of CA's OCSP response occurs in ParseResponse + ocspr, err = ocsp.ParseResponse(rawResp, link.Issuer) + if err == nil && ocspr != nil { + // Check if OCSP Response delegation present and if so is valid + if !certidp.ValidDelegationCheck(link.Issuer, ocspr) { + s.Warnf(certidp.MsgOCSPResponseDelegationInvalid, subj) + if opts.WarnOnly { + // Can't use bogus assertion, but warn-only set so allow link to pass + s.Warnf(certidp.MsgAllowWarnOnlyOccurred, subj) + return _EMPTY_, true + } + return fmt.Sprintf(certidp.MsgOCSPResponseDelegationInvalid, subj), false + } + if !certidp.OCSPResponseCurrent(ocspr, opts, sLogs) { + s.Warnf(certidp.ErrNewCAResponseNotCurrent, subj) + if opts.WarnOnly { + // Can't use non-effective assertion, but warn-only set so allow link to pass + s.Warnf(certidp.MsgAllowWarnOnlyOccurred, subj) + return _EMPTY_, true + } + return certidp.MsgOCSPResponseNotEffective, false + } + } else { + s.Errorf(certidp.ErrCAResponseParseFailed, subj, err) + if opts.WarnOnly { + // Can't use bogus assertion, but warn-only set so allow link to pass + s.Warnf(certidp.MsgAllowWarnOnlyOccurred, subj) + return _EMPTY_, true + } + return certidp.MsgFailedOCSPResponseParse, false + } + // cache the valid fetched CA OCSP Response + rc.Put(fingerprint, ocspr, subj, sLogs) + } + + // Whether through valid cache response available or newly fetched valid response, now check the status + if ocspr.Status == ocsp.Revoked || (ocspr.Status == ocsp.Unknown && !opts.UnknownIsGood) { + s.Warnf(certidp.ErrOCSPInvalidPeerLink, subj, certidp.GetStatusAssertionStr(ocspr.Status)) + if opts.WarnOnly { + s.Warnf(certidp.MsgAllowWarnOnlyOccurred, subj) + return _EMPTY_, true + } + return fmt.Sprintf(certidp.MsgOCSPResponseInvalidStatus, certidp.GetStatusAssertionStr(ocspr.Status)), false + } + s.Debugf(certidp.DbgOCSPValidPeerLink, subj) + return _EMPTY_, true +} diff --git a/server/ocsp_responsecache.go b/server/ocsp_responsecache.go new file mode 100644 index 0000000..c384812 --- /dev/null +++ b/server/ocsp_responsecache.go @@ -0,0 +1,636 @@ +// Copyright 2023-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/klauspost/compress/s2" + "golang.org/x/crypto/ocsp" + + "github.com/nats-io/nats-server/v2/server/certidp" +) + +const ( + OCSPResponseCacheDefaultDir = "_rc_" + OCSPResponseCacheDefaultFilename = "cache.json" + OCSPResponseCacheDefaultTempFilePrefix = "ocsprc-*" + OCSPResponseCacheMinimumSaveInterval = 1 * time.Second + OCSPResponseCacheDefaultSaveInterval = 5 * time.Minute +) + +type OCSPResponseCacheType int + +const ( + NONE OCSPResponseCacheType = iota + 1 + LOCAL +) + +var OCSPResponseCacheTypeMap = map[string]OCSPResponseCacheType{ + "none": NONE, + "local": LOCAL, +} + +type OCSPResponseCacheConfig struct { + Type OCSPResponseCacheType + LocalStore string + PreserveRevoked bool + SaveInterval float64 +} + +func NewOCSPResponseCacheConfig() *OCSPResponseCacheConfig { + return &OCSPResponseCacheConfig{ + Type: LOCAL, + LocalStore: OCSPResponseCacheDefaultDir, + PreserveRevoked: false, + SaveInterval: OCSPResponseCacheDefaultSaveInterval.Seconds(), + } +} + +type OCSPResponseCacheStats struct { + Responses int64 `json:"size"` + Hits int64 `json:"hits"` + Misses int64 `json:"misses"` + Revokes int64 `json:"revokes"` + Goods int64 `json:"goods"` + Unknowns int64 `json:"unknowns"` +} + +type OCSPResponseCacheItem struct { + Subject string `json:"subject,omitempty"` + CachedAt time.Time `json:"cached_at"` + RespStatus certidp.StatusAssertion `json:"resp_status"` + RespExpires time.Time `json:"resp_expires,omitempty"` + Resp []byte `json:"resp"` +} + +type OCSPResponseCache interface { + Put(key string, resp *ocsp.Response, subj string, log *certidp.Log) + Get(key string, log *certidp.Log) []byte + Delete(key string, miss bool, log *certidp.Log) + Type() string + Start(s *Server) + Stop(s *Server) + Online() bool + Config() *OCSPResponseCacheConfig + Stats() *OCSPResponseCacheStats +} + +// NoOpCache is a no-op implementation of OCSPResponseCache +type NoOpCache struct { + config *OCSPResponseCacheConfig + stats *OCSPResponseCacheStats + online bool + mu *sync.RWMutex +} + +func (c *NoOpCache) Put(_ string, _ *ocsp.Response, _ string, _ *certidp.Log) {} + +func (c *NoOpCache) Get(_ string, _ *certidp.Log) []byte { + return nil +} + +func (c *NoOpCache) Delete(_ string, _ bool, _ *certidp.Log) {} + +func (c *NoOpCache) Start(_ *Server) { + c.mu.Lock() + defer c.mu.Unlock() + c.stats = &OCSPResponseCacheStats{} + c.online = true +} + +func (c *NoOpCache) Stop(_ *Server) { + c.mu.Lock() + defer c.mu.Unlock() + c.online = false +} + +func (c *NoOpCache) Online() bool { + c.mu.RLock() + defer c.mu.RUnlock() + return c.online +} + +func (c *NoOpCache) Type() string { + c.mu.RLock() + defer c.mu.RUnlock() + return "none" +} + +func (c *NoOpCache) Config() *OCSPResponseCacheConfig { + c.mu.RLock() + defer c.mu.RUnlock() + return c.config +} + +func (c *NoOpCache) Stats() *OCSPResponseCacheStats { + c.mu.RLock() + defer c.mu.RUnlock() + return c.stats +} + +// LocalCache is a local file implementation of OCSPResponseCache +type LocalCache struct { + config *OCSPResponseCacheConfig + stats *OCSPResponseCacheStats + online bool + cache map[string]OCSPResponseCacheItem + mu *sync.RWMutex + saveInterval time.Duration + dirty bool + timer *time.Timer +} + +// Put captures a CA OCSP response to the OCSP peer cache indexed by response fingerprint (a hash) +func (c *LocalCache) Put(key string, caResp *ocsp.Response, subj string, log *certidp.Log) { + c.mu.RLock() + if !c.online || caResp == nil || key == "" { + c.mu.RUnlock() + return + } + c.mu.RUnlock() + log.Debugf(certidp.DbgCachingResponse, subj, key) + rawC, err := c.Compress(caResp.Raw) + if err != nil { + log.Errorf(certidp.ErrResponseCompressFail, key, err) + return + } + log.Debugf(certidp.DbgAchievedCompression, float64(len(rawC))/float64(len(caResp.Raw))) + c.mu.Lock() + defer c.mu.Unlock() + // check if we are replacing and do stats + item, ok := c.cache[key] + if ok { + c.adjustStats(-1, item.RespStatus) + } + item = OCSPResponseCacheItem{ + Subject: subj, + CachedAt: time.Now().UTC().Round(time.Second), + RespStatus: certidp.StatusAssertionIntToVal[caResp.Status], + RespExpires: caResp.NextUpdate, + Resp: rawC, + } + c.cache[key] = item + c.adjustStats(1, item.RespStatus) + c.dirty = true +} + +// Get returns a CA OCSP response from the OCSP peer cache matching the response fingerprint (a hash) +func (c *LocalCache) Get(key string, log *certidp.Log) []byte { + c.mu.RLock() + defer c.mu.RUnlock() + if !c.online || key == "" { + return nil + } + val, ok := c.cache[key] + if ok { + atomic.AddInt64(&c.stats.Hits, 1) + log.Debugf(certidp.DbgCacheHit, key) + } else { + atomic.AddInt64(&c.stats.Misses, 1) + log.Debugf(certidp.DbgCacheMiss, key) + return nil + } + resp, err := c.Decompress(val.Resp) + if err != nil { + log.Errorf(certidp.ErrResponseDecompressFail, key, err) + return nil + } + return resp +} + +func (c *LocalCache) adjustStatsHitToMiss() { + atomic.AddInt64(&c.stats.Misses, 1) + atomic.AddInt64(&c.stats.Hits, -1) +} + +func (c *LocalCache) adjustStats(delta int64, rs certidp.StatusAssertion) { + if delta == 0 { + return + } + atomic.AddInt64(&c.stats.Responses, delta) + switch rs { + case ocsp.Good: + atomic.AddInt64(&c.stats.Goods, delta) + case ocsp.Revoked: + atomic.AddInt64(&c.stats.Revokes, delta) + case ocsp.Unknown: + atomic.AddInt64(&c.stats.Unknowns, delta) + } +} + +// Delete removes a CA OCSP response from the OCSP peer cache matching the response fingerprint (a hash) +func (c *LocalCache) Delete(key string, wasMiss bool, log *certidp.Log) { + c.mu.Lock() + defer c.mu.Unlock() + if !c.online || key == "" || c.config == nil { + return + } + item, ok := c.cache[key] + if !ok { + return + } + if item.RespStatus == ocsp.Revoked && c.config.PreserveRevoked { + log.Debugf(certidp.DbgPreservedRevocation, key) + if wasMiss { + c.adjustStatsHitToMiss() + } + return + } + log.Debugf(certidp.DbgDeletingCacheResponse, key) + delete(c.cache, key) + c.adjustStats(-1, item.RespStatus) + if wasMiss { + c.adjustStatsHitToMiss() + } + c.dirty = true +} + +// Start initializes the configured OCSP peer cache, loads a saved cache from disk (if present), and initializes runtime statistics +func (c *LocalCache) Start(s *Server) { + s.Debugf(certidp.DbgStartingCache) + c.loadCache(s) + c.initStats() + c.mu.Lock() + c.online = true + c.mu.Unlock() +} + +func (c *LocalCache) Stop(s *Server) { + c.mu.Lock() + s.Debugf(certidp.DbgStoppingCache) + c.online = false + c.timer.Stop() + c.mu.Unlock() + c.saveCache(s) +} + +func (c *LocalCache) Online() bool { + c.mu.RLock() + defer c.mu.RUnlock() + return c.online +} + +func (c *LocalCache) Type() string { + c.mu.RLock() + defer c.mu.RUnlock() + return "local" +} + +func (c *LocalCache) Config() *OCSPResponseCacheConfig { + c.mu.RLock() + defer c.mu.RUnlock() + return c.config +} + +func (c *LocalCache) Stats() *OCSPResponseCacheStats { + c.mu.RLock() + defer c.mu.RUnlock() + if c.stats == nil { + return nil + } + stats := OCSPResponseCacheStats{ + Responses: c.stats.Responses, + Hits: c.stats.Hits, + Misses: c.stats.Misses, + Revokes: c.stats.Revokes, + Goods: c.stats.Goods, + Unknowns: c.stats.Unknowns, + } + return &stats +} + +func (c *LocalCache) initStats() { + c.mu.Lock() + defer c.mu.Unlock() + c.stats = &OCSPResponseCacheStats{} + c.stats.Hits = 0 + c.stats.Misses = 0 + c.stats.Responses = int64(len(c.cache)) + for _, resp := range c.cache { + switch resp.RespStatus { + case ocsp.Good: + c.stats.Goods++ + case ocsp.Revoked: + c.stats.Revokes++ + case ocsp.Unknown: + c.stats.Unknowns++ + } + } +} + +func (c *LocalCache) Compress(buf []byte) ([]byte, error) { + bodyLen := int64(len(buf)) + var output bytes.Buffer + writer := s2.NewWriter(&output) + input := bytes.NewReader(buf[:bodyLen]) + if n, err := io.CopyN(writer, input, bodyLen); err != nil { + return nil, fmt.Errorf(certidp.ErrCannotWriteCompressed, err) + } else if n != bodyLen { + return nil, fmt.Errorf(certidp.ErrTruncatedWrite, n, bodyLen) + } + if err := writer.Close(); err != nil { + return nil, fmt.Errorf(certidp.ErrCannotCloseWriter, err) + } + return output.Bytes(), nil +} + +func (c *LocalCache) Decompress(buf []byte) ([]byte, error) { + bodyLen := int64(len(buf)) + input := bytes.NewReader(buf[:bodyLen]) + reader := io.NopCloser(s2.NewReader(input)) + output, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf(certidp.ErrCannotReadCompressed, err) + } + return output, reader.Close() +} + +func (c *LocalCache) loadCache(s *Server) { + d := s.opts.OCSPCacheConfig.LocalStore + if d == _EMPTY_ { + d = OCSPResponseCacheDefaultDir + } + f := OCSPResponseCacheDefaultFilename + store, err := filepath.Abs(path.Join(d, f)) + if err != nil { + s.Errorf(certidp.ErrLoadCacheFail, err) + return + } + s.Debugf(certidp.DbgLoadingCache, store) + c.mu.Lock() + defer c.mu.Unlock() + c.cache = make(map[string]OCSPResponseCacheItem) + dat, err := os.ReadFile(store) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + s.Debugf(certidp.DbgNoCacheFound) + } else { + s.Warnf(certidp.ErrLoadCacheFail, err) + } + return + } + err = json.Unmarshal(dat, &c.cache) + if err != nil { + // make sure clean cache + c.cache = make(map[string]OCSPResponseCacheItem) + s.Warnf(certidp.ErrLoadCacheFail, err) + c.dirty = true + return + } + c.dirty = false +} + +func (c *LocalCache) saveCache(s *Server) { + c.mu.RLock() + dirty := c.dirty + c.mu.RUnlock() + if !dirty { + return + } + s.Debugf(certidp.DbgCacheDirtySave) + var d string + if c.config.LocalStore != _EMPTY_ { + d = c.config.LocalStore + } else { + d = OCSPResponseCacheDefaultDir + } + f := OCSPResponseCacheDefaultFilename + store, err := filepath.Abs(path.Join(d, f)) + if err != nil { + s.Errorf(certidp.ErrSaveCacheFail, err) + return + } + s.Debugf(certidp.DbgSavingCache, store) + if _, err := os.Stat(d); os.IsNotExist(err) { + err = os.Mkdir(d, defaultDirPerms) + if err != nil { + s.Errorf(certidp.ErrSaveCacheFail, err) + return + } + } + tmp, err := os.CreateTemp(d, OCSPResponseCacheDefaultTempFilePrefix) + if err != nil { + s.Errorf(certidp.ErrSaveCacheFail, err) + return + } + defer func() { + tmp.Close() + os.Remove(tmp.Name()) + }() // clean up any temp files + + // RW lock here because we're going to snapshot the cache to disk and mark as clean if successful + c.mu.Lock() + defer c.mu.Unlock() + dat, err := json.MarshalIndent(c.cache, "", " ") + if err != nil { + s.Errorf(certidp.ErrSaveCacheFail, err) + return + } + cacheSize, err := tmp.Write(dat) + if err != nil { + s.Errorf(certidp.ErrSaveCacheFail, err) + return + } + err = tmp.Sync() + if err != nil { + s.Errorf(certidp.ErrSaveCacheFail, err) + return + } + err = tmp.Close() + if err != nil { + s.Errorf(certidp.ErrSaveCacheFail, err) + return + } + // do the final swap and overwrite any old saved peer cache + err = os.Rename(tmp.Name(), store) + if err != nil { + s.Errorf(certidp.ErrSaveCacheFail, err) + return + } + c.dirty = false + s.Debugf(certidp.DbgCacheSaved, cacheSize) +} + +var OCSPResponseCacheUsage = ` +You may enable OCSP peer response cacheing at server configuration root level: + +(If no TLS blocks are configured with OCSP peer verification, ocsp_cache is ignored.) + + ... + # short form enables with defaults + ocsp_cache: true + + # if false or undefined and one or more TLS blocks are configured with OCSP peer verification, "none" is implied + + # long form includes settable options + ocsp_cache { + + # Cache type (default local) + type: local + + # Cache file directory for local-type cache (default _rc_ in current working directory) + local_store: "_rc_" + + # Ignore cache deletes if cached OCSP response is Revoked status (default false) + preserve_revoked: false + + # For local store, interval to save in-memory cache to disk in seconds (default 300 seconds, minimum 1 second) + save_interval: 300 + } + ... + +Note: Cache of server's own OCSP response (staple) is enabled using the 'ocsp' configuration option. +` + +func (s *Server) initOCSPResponseCache() { + // No mTLS OCSP or Leaf OCSP enablements, so no need to init cache + s.mu.RLock() + if !s.ocspPeerVerify { + s.mu.RUnlock() + return + } + s.mu.RUnlock() + so := s.getOpts() + if so.OCSPCacheConfig == nil { + so.OCSPCacheConfig = NewOCSPResponseCacheConfig() + } + var cc = so.OCSPCacheConfig + s.mu.Lock() + defer s.mu.Unlock() + switch cc.Type { + case NONE: + s.ocsprc = &NoOpCache{config: cc, online: true, mu: &sync.RWMutex{}} + case LOCAL: + c := &LocalCache{ + config: cc, + online: false, + cache: make(map[string]OCSPResponseCacheItem), + mu: &sync.RWMutex{}, + dirty: false, + } + c.saveInterval = time.Duration(cc.SaveInterval) * time.Second + c.timer = time.AfterFunc(c.saveInterval, func() { + s.Debugf(certidp.DbgCacheSaveTimerExpired) + c.saveCache(s) + c.timer.Reset(c.saveInterval) + }) + s.ocsprc = c + default: + s.Fatalf(certidp.ErrBadCacheTypeConfig, cc.Type) + } +} + +func (s *Server) startOCSPResponseCache() { + // No mTLS OCSP or Leaf OCSP enablements, so no need to start cache + s.mu.RLock() + if !s.ocspPeerVerify || s.ocsprc == nil { + s.mu.RUnlock() + return + } + s.mu.RUnlock() + + // Could be heavier operation depending on cache implementation + s.ocsprc.Start(s) + if s.ocsprc.Online() { + s.Noticef(certidp.MsgCacheOnline, s.ocsprc.Type()) + } else { + s.Noticef(certidp.MsgCacheOffline, s.ocsprc.Type()) + } +} + +func (s *Server) stopOCSPResponseCache() { + s.mu.RLock() + if s.ocsprc == nil { + s.mu.RUnlock() + return + } + s.mu.RUnlock() + s.ocsprc.Stop(s) +} + +func parseOCSPResponseCache(v any) (pcfg *OCSPResponseCacheConfig, retError error) { + var lt token + defer convertPanicToError(<, &retError) + tk, v := unwrapValue(v, <) + cm, ok := v.(map[string]any) + if !ok { + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrIllegalCacheOptsConfig, v)} + } + pcfg = NewOCSPResponseCacheConfig() + retError = nil + for mk, mv := range cm { + // Again, unwrap token value if line check is required. + tk, mv = unwrapValue(mv, <) + switch strings.ToLower(mk) { + case "type": + cache, ok := mv.(string) + if !ok { + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingCacheOptFieldGeneric, mk)} + } + cacheType, exists := OCSPResponseCacheTypeMap[strings.ToLower(cache)] + if !exists { + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrUnknownCacheType, cache)} + } + pcfg.Type = cacheType + case "local_store": + store, ok := mv.(string) + if !ok { + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingCacheOptFieldGeneric, mk)} + } + pcfg.LocalStore = store + case "preserve_revoked": + preserve, ok := mv.(bool) + if !ok { + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingCacheOptFieldGeneric, mk)} + } + pcfg.PreserveRevoked = preserve + case "save_interval": + at := float64(0) + switch mv := mv.(type) { + case int64: + at = float64(mv) + case float64: + at = mv + case string: + d, err := time.ParseDuration(mv) + if err != nil { + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, err)} + } + at = d.Seconds() + default: + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingCacheOptFieldTypeConversion, "unexpected type")} + } + si := time.Duration(at) * time.Second + if si < OCSPResponseCacheMinimumSaveInterval { + si = OCSPResponseCacheMinimumSaveInterval + } + pcfg.SaveInterval = si.Seconds() + default: + return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingCacheOptFieldGeneric, mk)} + } + } + return pcfg, nil +} diff --git a/server/parser.go b/server/parser.go new file mode 100644 index 0000000..3cd39be --- /dev/null +++ b/server/parser.go @@ -0,0 +1,1339 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bufio" + "bytes" + "fmt" + "net/http" + "net/textproto" +) + +type parserState int +type parseState struct { + state parserState + op byte + as int + drop int + pa pubArg + argBuf []byte + msgBuf []byte + header http.Header // access via getHeader + scratch [MAX_CONTROL_LINE_SIZE]byte + argsa [MAX_HMSG_ARGS + 1][]byte // pre-allocated args array to avoid per-call heap escape +} + +type pubArg struct { + arg []byte + pacache []byte + origin []byte + account []byte + subject []byte + deliver []byte + mapped []byte + reply []byte + szb []byte + hdb []byte + queues [][]byte + size int + hdr int + psi []*serviceImport + trace *msgTrace + delivered bool // Only used for service imports +} + +// Parser constants +const ( + OP_START parserState = iota + OP_PLUS + OP_PLUS_O + OP_PLUS_OK + OP_MINUS + OP_MINUS_E + OP_MINUS_ER + OP_MINUS_ERR + OP_MINUS_ERR_SPC + MINUS_ERR_ARG + OP_C + OP_CO + OP_CON + OP_CONN + OP_CONNE + OP_CONNEC + OP_CONNECT + CONNECT_ARG + OP_H + OP_HP + OP_HPU + OP_HPUB + OP_HPUB_SPC + HPUB_ARG + OP_HM + OP_HMS + OP_HMSG + OP_HMSG_SPC + HMSG_ARG + OP_P + OP_PU + OP_PUB + OP_PUB_SPC + PUB_ARG + OP_PI + OP_PIN + OP_PING + OP_PO + OP_PON + OP_PONG + MSG_PAYLOAD + MSG_END_R + MSG_END_N + OP_S + OP_SU + OP_SUB + OP_SUB_SPC + SUB_ARG + OP_A + OP_ASUB + OP_ASUB_SPC + ASUB_ARG + OP_AUSUB + OP_AUSUB_SPC + AUSUB_ARG + OP_L + OP_LS + OP_R + OP_RS + OP_U + OP_UN + OP_UNS + OP_UNSU + OP_UNSUB + OP_UNSUB_SPC + UNSUB_ARG + OP_M + OP_MS + OP_MSG + OP_MSG_SPC + MSG_ARG + OP_I + OP_IN + OP_INF + OP_INFO + INFO_ARG +) + +func (c *client) parse(buf []byte) error { + // Branch out to mqtt clients. c.mqtt is immutable, but should it become + // an issue (say data race detection), we could branch outside in readLoop + if c.isMqtt() { + return c.mqttParse(buf) + } + var i int + var b byte + var lmsg bool + + // Snapshots + c.mu.Lock() + // Snapshot and then reset when we receive a + // proper CONNECT if needed. + authSet := c.awaitingAuth() + // Snapshot max control line as well. + s, mcl, trace := c.srv, c.mcl, c.trace + c.mu.Unlock() + + // Move to loop instead of range syntax to allow jumping of i + for i = 0; i < len(buf); i++ { + b = buf[i] + + switch c.state { + case OP_START: + c.op = b + if b != 'C' && b != 'c' { + if authSet { + if s == nil { + goto authErr + } + var ok bool + switch c.kind { + case CLIENT: + // Check here for NoAuthUser. If this is set allow non CONNECT protos as our first. + // E.g. telnet proto demos. + opts := s.getOpts() + noAuthUser := opts.NoAuthUser + if c.ws != nil { + if noAuthUserWS := opts.Websocket.NoAuthUser; noAuthUserWS != _EMPTY_ { + noAuthUser = noAuthUserWS + } + } + if noAuthUser != _EMPTY_ { + s.mu.Lock() + user, exists := s.users[noAuthUser] + s.mu.Unlock() + // Run the same authentication pipeline as CONNECT. In addition to + // the connection restrictions checked here, this delegates the + // decision to auth callouts or custom authenticators. + if exists && !user.ProxyRequired && c.connectionTypeAllowed(user.AllowedConnectionTypes) { + // Mirror processConnect: clear the auth-timeout timer and + // mark CONNECT received *before* authenticating. Auth may + // install a JWT/callout expiration timer into the same c.atmr + // slot, so clearing it afterwards would drop the expiration + // and leave the client connected past expiry. Setting + // connectReceived first also lets the expiration deadline be + // recorded on c.expires. + c.mu.Lock() + c.clearAuthTimer() + c.flags.set(connectReceived) + c.mu.Unlock() + if s.checkAuthentication(c) { + authSet, ok = false, true + } + } + } + case LEAF: + // Compressed inbound leaf-node negotiation may require INFO + // before CONNECT. Without compression, leaf connections must + // still start with CONNECT. + ok = (b == 'I' || b == 'i') && needsCompression(s.getOpts().LeafNode.Compression.Mode) + } + if !ok { + goto authErr + } + } + } + switch b { + case 'P', 'p': + c.state = OP_P + case 'H', 'h': + c.state = OP_H + case 'S', 's': + c.state = OP_S + case 'U', 'u': + c.state = OP_U + case 'R', 'r': + if c.kind == CLIENT { + goto parseErr + } else { + c.state = OP_R + } + case 'L', 'l': + if c.kind != LEAF && c.kind != ROUTER { + goto parseErr + } else { + c.state = OP_L + } + case 'A', 'a': + if c.kind == CLIENT { + goto parseErr + } else { + c.state = OP_A + } + case 'C', 'c': + c.state = OP_C + case 'I', 'i': + c.state = OP_I + case '+': + c.state = OP_PLUS + case '-': + c.state = OP_MINUS + default: + goto parseErr + } + case OP_H: + switch b { + case 'P', 'p': + c.state = OP_HP + case 'M', 'm': + c.state = OP_HM + default: + goto parseErr + } + case OP_HP: + switch b { + case 'U', 'u': + c.state = OP_HPU + default: + goto parseErr + } + case OP_HPU: + switch b { + case 'B', 'b': + c.state = OP_HPUB + default: + goto parseErr + } + case OP_HPUB: + switch b { + case ' ', '\t': + c.state = OP_HPUB_SPC + default: + goto parseErr + } + case OP_HPUB_SPC: + switch b { + case ' ', '\t': + continue + default: + c.pa.hdr = 0 + c.state = HPUB_ARG + c.as = i + } + case HPUB_ARG: + switch b { + case '\r': + c.drop = 1 + case '\n': + var arg []byte + if c.argBuf != nil { + arg = c.argBuf + c.argBuf = nil + } else { + arg = buf[c.as : i-c.drop] + } + if err := c.overMaxControlLineLimit(arg, mcl); err != nil { + return err + } + if trace { + c.traceInOp("HPUB", arg) + } + var remaining []byte + if i < len(buf) { + remaining = buf[i+1:] + } + if err := c.processHeaderPub(arg, remaining); err != nil { + return err + } + + c.drop, c.as, c.state = 0, i+1, MSG_PAYLOAD + // If we don't have a saved buffer then jump ahead with + // the index. If this overruns what is left we fall out + // and process split buffer. + if c.msgBuf == nil { + i = c.as + c.pa.size - LEN_CR_LF + } + default: + if c.argBuf != nil { + c.argBuf = append(c.argBuf, b) + } + } + case OP_HM: + switch b { + case 'S', 's': + c.state = OP_HMS + default: + goto parseErr + } + case OP_HMS: + switch b { + case 'G', 'g': + c.state = OP_HMSG + default: + goto parseErr + } + case OP_HMSG: + switch b { + case ' ', '\t': + c.state = OP_HMSG_SPC + default: + goto parseErr + } + case OP_HMSG_SPC: + switch b { + case ' ', '\t': + continue + default: + c.pa.hdr = 0 + c.state = HMSG_ARG + c.as = i + } + case HMSG_ARG: + switch b { + case '\r': + c.drop = 1 + case '\n': + var arg []byte + if c.argBuf != nil { + arg = c.argBuf + c.argBuf = nil + } else { + arg = buf[c.as : i-c.drop] + } + if err := c.overMaxControlLineLimit(arg, mcl); err != nil { + return err + } + var err error + if c.kind == ROUTER || c.kind == GATEWAY { + if trace { + c.traceInOp("HMSG", arg) + } + err = c.processRoutedHeaderMsgArgs(arg) + } else if c.kind == LEAF { + if trace { + c.traceInOp("HMSG", arg) + } + err = c.processLeafHeaderMsgArgs(arg) + } + if err != nil { + return err + } + c.drop, c.as, c.state = 0, i+1, MSG_PAYLOAD + + // jump ahead with the index. If this overruns + // what is left we fall out and process split + // buffer. + i = c.as + c.pa.size - LEN_CR_LF + default: + if c.argBuf != nil { + c.argBuf = append(c.argBuf, b) + } + } + case OP_P: + switch b { + case 'U', 'u': + c.state = OP_PU + case 'I', 'i': + c.state = OP_PI + case 'O', 'o': + c.state = OP_PO + default: + goto parseErr + } + case OP_PU: + switch b { + case 'B', 'b': + c.state = OP_PUB + default: + goto parseErr + } + case OP_PUB: + switch b { + case ' ', '\t': + c.state = OP_PUB_SPC + default: + goto parseErr + } + case OP_PUB_SPC: + switch b { + case ' ', '\t': + continue + default: + c.pa.hdr = -1 + c.state = PUB_ARG + c.as = i + } + case PUB_ARG: + switch b { + case '\r': + c.drop = 1 + case '\n': + var arg []byte + if c.argBuf != nil { + arg = c.argBuf + c.argBuf = nil + } else { + arg = buf[c.as : i-c.drop] + } + if err := c.overMaxControlLineLimit(arg, mcl); err != nil { + return err + } + if trace { + c.traceInOp("PUB", arg) + } + if err := c.processPub(arg); err != nil { + return err + } + + c.drop, c.as, c.state = 0, i+1, MSG_PAYLOAD + // If we don't have a saved buffer then jump ahead with + // the index. If this overruns what is left we fall out + // and process split buffer. + if c.msgBuf == nil { + i = c.as + c.pa.size - LEN_CR_LF + } + default: + if c.argBuf != nil { + c.argBuf = append(c.argBuf, b) + } + } + case MSG_PAYLOAD: + if c.msgBuf != nil { + // copy as much as we can to the buffer and skip ahead. + toCopy := c.pa.size - len(c.msgBuf) + avail := len(buf) - i + if avail < toCopy { + toCopy = avail + } + if toCopy > 0 { + start := len(c.msgBuf) + // This is needed for copy to work. + c.msgBuf = c.msgBuf[:start+toCopy] + copy(c.msgBuf[start:], buf[i:i+toCopy]) + // Update our index + i = (i + toCopy) - 1 + } else { + // Fall back to append if needed. + c.msgBuf = append(c.msgBuf, b) + } + if len(c.msgBuf) >= c.pa.size { + c.state = MSG_END_R + } + } else if i-c.as+1 >= c.pa.size { + c.state = MSG_END_R + } + case MSG_END_R: + if b != '\r' { + goto parseErr + } + if c.msgBuf != nil { + c.msgBuf = append(c.msgBuf, b) + } + c.state = MSG_END_N + case MSG_END_N: + if b != '\n' { + goto parseErr + } + if c.msgBuf != nil { + c.msgBuf = append(c.msgBuf, b) + } else { + c.msgBuf = buf[c.as : i+1] + } + + var mt *msgTrace + var skip bool + if c.pa.hdr > 0 { + skip, mt = c.initMsgTrace(c.msgBuf[:c.pa.hdr], nil) + } + if !skip { + // Check for mappings. + if (c.kind == CLIENT || c.kind == LEAF) && c.in.flags.isSet(hasMappings) { + changed := c.selectMappedSubject() + if changed { + if trace { + c.traceInOp("MAPPING", []byte(fmt.Sprintf("%s -> %s", c.pa.mapped, c.pa.subject))) + } + // c.pa.subject is the subject the original is now mapped to. + mt.addSubjectMappingEvent(c.pa.subject) + } + } + if trace { + c.traceMsg(c.msgBuf) + } + + c.processInboundMsg(c.msgBuf) + + mt.sendEvent() + } + c.argBuf, c.msgBuf, c.header = nil, nil, nil + c.drop, c.as, c.state = 0, i+1, OP_START + // Drop all pub args + c.pa.arg, c.pa.pacache, c.pa.origin, c.pa.account, c.pa.subject, c.pa.mapped = nil, nil, nil, nil, nil, nil + c.pa.reply, c.pa.hdr, c.pa.size, c.pa.szb, c.pa.hdb, c.pa.queues = nil, -1, 0, nil, nil, nil + c.pa.trace = nil + c.pa.delivered = false + lmsg = false + case OP_A: + switch b { + case '+': + c.state = OP_ASUB + case '-', 'u': + c.state = OP_AUSUB + default: + goto parseErr + } + case OP_ASUB: + switch b { + case ' ', '\t': + c.state = OP_ASUB_SPC + default: + goto parseErr + } + case OP_ASUB_SPC: + switch b { + case ' ', '\t': + continue + default: + c.state = ASUB_ARG + c.as = i + } + case ASUB_ARG: + switch b { + case '\r': + c.drop = 1 + case '\n': + var arg []byte + if c.argBuf != nil { + arg = c.argBuf + c.argBuf = nil + } else { + arg = buf[c.as : i-c.drop] + } + if err := c.overMaxControlLineLimit(arg, mcl); err != nil { + return err + } + if trace { + c.traceInOp("A+", arg) + } + if err := c.processAccountSub(arg); err != nil { + return err + } + c.drop, c.as, c.state = 0, i+1, OP_START + default: + if c.argBuf != nil { + c.argBuf = append(c.argBuf, b) + } + } + case OP_AUSUB: + switch b { + case ' ', '\t': + c.state = OP_AUSUB_SPC + default: + goto parseErr + } + case OP_AUSUB_SPC: + switch b { + case ' ', '\t': + continue + default: + c.state = AUSUB_ARG + c.as = i + } + case AUSUB_ARG: + switch b { + case '\r': + c.drop = 1 + case '\n': + var arg []byte + if c.argBuf != nil { + arg = c.argBuf + c.argBuf = nil + } else { + arg = buf[c.as : i-c.drop] + } + if err := c.overMaxControlLineLimit(arg, mcl); err != nil { + return err + } + if trace { + c.traceInOp("A-", arg) + } + c.processAccountUnsub(arg) + c.drop, c.as, c.state = 0, i+1, OP_START + default: + if c.argBuf != nil { + c.argBuf = append(c.argBuf, b) + } + } + case OP_S: + switch b { + case 'U', 'u': + c.state = OP_SU + default: + goto parseErr + } + case OP_SU: + switch b { + case 'B', 'b': + c.state = OP_SUB + default: + goto parseErr + } + case OP_SUB: + switch b { + case ' ', '\t': + c.state = OP_SUB_SPC + default: + goto parseErr + } + case OP_SUB_SPC: + switch b { + case ' ', '\t': + continue + default: + c.state = SUB_ARG + c.as = i + } + case SUB_ARG: + switch b { + case '\r': + c.drop = 1 + case '\n': + var arg []byte + if c.argBuf != nil { + arg = c.argBuf + c.argBuf = nil + } else { + arg = buf[c.as : i-c.drop] + } + if err := c.overMaxControlLineLimit(arg, mcl); err != nil { + return err + } + var err error + + switch c.kind { + case CLIENT: + if trace { + c.traceInOp("SUB", arg) + } + err = c.parseSub(arg, false) + case ROUTER: + switch c.op { + case 'R', 'r': + if trace { + c.traceInOp("RS+", arg) + } + err = c.processRemoteSub(arg, false, false) + case 'L', 'l': + if trace { + c.traceInOp("LS+", arg) + } + err = c.processRemoteSub(arg, true, true) + } + case GATEWAY: + if trace { + c.traceInOp("RS+", arg) + } + err = c.processGatewayRSub(arg) + case LEAF: + if trace { + c.traceInOp("LS+", arg) + } + err = c.processLeafSub(arg) + } + if err != nil { + return err + } + c.drop, c.as, c.state = 0, i+1, OP_START + default: + if c.argBuf != nil { + c.argBuf = append(c.argBuf, b) + } + } + case OP_L: + switch b { + case 'S', 's': + c.state = OP_LS + case 'M', 'm': + c.state = OP_M + default: + goto parseErr + } + case OP_LS: + switch b { + case '+': + c.state = OP_SUB + case '-': + c.state = OP_UNSUB + default: + goto parseErr + } + case OP_R: + switch b { + case 'S', 's': + c.state = OP_RS + case 'M', 'm': + c.state = OP_M + default: + goto parseErr + } + case OP_RS: + switch b { + case '+': + c.state = OP_SUB + case '-': + c.state = OP_UNSUB + default: + goto parseErr + } + case OP_U: + switch b { + case 'N', 'n': + c.state = OP_UN + default: + goto parseErr + } + case OP_UN: + switch b { + case 'S', 's': + c.state = OP_UNS + default: + goto parseErr + } + case OP_UNS: + switch b { + case 'U', 'u': + c.state = OP_UNSU + default: + goto parseErr + } + case OP_UNSU: + switch b { + case 'B', 'b': + c.state = OP_UNSUB + default: + goto parseErr + } + case OP_UNSUB: + switch b { + case ' ', '\t': + c.state = OP_UNSUB_SPC + default: + goto parseErr + } + case OP_UNSUB_SPC: + switch b { + case ' ', '\t': + continue + default: + c.state = UNSUB_ARG + c.as = i + } + case UNSUB_ARG: + switch b { + case '\r': + c.drop = 1 + case '\n': + var arg []byte + if c.argBuf != nil { + arg = c.argBuf + c.argBuf = nil + } else { + arg = buf[c.as : i-c.drop] + } + if err := c.overMaxControlLineLimit(arg, mcl); err != nil { + return err + } + var err error + + switch c.kind { + case CLIENT: + if trace { + c.traceInOp("UNSUB", arg) + } + err = c.processUnsub(arg) + case ROUTER: + if trace && c.srv != nil { + switch c.op { + case 'R', 'r': + c.traceInOp("RS-", arg) + case 'L', 'l': + c.traceInOp("LS-", arg) + } + } + leafUnsub := c.op == 'L' || c.op == 'l' + err = c.processRemoteUnsub(arg, leafUnsub) + case GATEWAY: + if trace { + c.traceInOp("RS-", arg) + } + err = c.processGatewayRUnsub(arg) + case LEAF: + if trace { + c.traceInOp("LS-", arg) + } + err = c.processLeafUnsub(arg) + } + if err != nil { + return err + } + c.drop, c.as, c.state = 0, i+1, OP_START + default: + if c.argBuf != nil { + c.argBuf = append(c.argBuf, b) + } + } + case OP_PI: + switch b { + case 'N', 'n': + c.state = OP_PIN + default: + goto parseErr + } + case OP_PIN: + switch b { + case 'G', 'g': + c.state = OP_PING + default: + goto parseErr + } + case OP_PING: + switch b { + case '\n': + if trace { + c.traceInOp("PING", nil) + } + c.processPing() + c.drop, c.state = 0, OP_START + } + case OP_PO: + switch b { + case 'N', 'n': + c.state = OP_PON + default: + goto parseErr + } + case OP_PON: + switch b { + case 'G', 'g': + c.state = OP_PONG + default: + goto parseErr + } + case OP_PONG: + switch b { + case '\n': + if trace { + c.traceInOp("PONG", nil) + } + c.processPong() + c.drop, c.state = 0, OP_START + } + case OP_C: + switch b { + case 'O', 'o': + c.state = OP_CO + default: + goto parseErr + } + case OP_CO: + switch b { + case 'N', 'n': + c.state = OP_CON + default: + goto parseErr + } + case OP_CON: + switch b { + case 'N', 'n': + c.state = OP_CONN + default: + goto parseErr + } + case OP_CONN: + switch b { + case 'E', 'e': + c.state = OP_CONNE + default: + goto parseErr + } + case OP_CONNE: + switch b { + case 'C', 'c': + c.state = OP_CONNEC + default: + goto parseErr + } + case OP_CONNEC: + switch b { + case 'T', 't': + c.state = OP_CONNECT + default: + goto parseErr + } + case OP_CONNECT: + switch b { + case ' ', '\t': + continue + default: + c.state = CONNECT_ARG + c.as = i + } + case CONNECT_ARG: + switch b { + case '\r': + c.drop = 1 + case '\n': + var arg []byte + if c.argBuf != nil { + arg = c.argBuf + c.argBuf = nil + } else { + arg = buf[c.as : i-c.drop] + } + if err := c.overMaxControlLineLimit(arg, mcl); err != nil { + return err + } + if trace { + c.traceInOp("CONNECT", removeSecretsFromTrace(arg)) + } + if err := c.processConnect(arg); err != nil { + return err + } + c.drop, c.state = 0, OP_START + // Reset notion on authSet + c.mu.Lock() + authSet = c.awaitingAuth() + c.mu.Unlock() + default: + if c.argBuf != nil { + c.argBuf = append(c.argBuf, b) + } + } + case OP_M: + switch b { + case 'S', 's': + c.state = OP_MS + default: + goto parseErr + } + case OP_MS: + switch b { + case 'G', 'g': + c.state = OP_MSG + default: + goto parseErr + } + case OP_MSG: + switch b { + case ' ', '\t': + c.state = OP_MSG_SPC + default: + goto parseErr + } + case OP_MSG_SPC: + switch b { + case ' ', '\t': + continue + default: + c.pa.hdr = -1 + c.state = MSG_ARG + c.as = i + } + case MSG_ARG: + switch b { + case '\r': + c.drop = 1 + case '\n': + var arg []byte + if c.argBuf != nil { + arg = c.argBuf + c.argBuf = nil + } else { + arg = buf[c.as : i-c.drop] + } + if err := c.overMaxControlLineLimit(arg, mcl); err != nil { + return err + } + var err error + if c.kind == ROUTER || c.kind == GATEWAY { + switch c.op { + case 'R', 'r': + if trace { + c.traceInOp("RMSG", arg) + } + err = c.processRoutedMsgArgs(arg) + case 'L', 'l': + if trace { + c.traceInOp("LMSG", arg) + } + lmsg = true + err = c.processRoutedOriginClusterMsgArgs(arg) + } + } else if c.kind == LEAF { + if trace { + c.traceInOp("LMSG", arg) + } + err = c.processLeafMsgArgs(arg) + } + if err != nil { + return err + } + c.drop, c.as, c.state = 0, i+1, MSG_PAYLOAD + + // jump ahead with the index. If this overruns + // what is left we fall out and process split + // buffer. + i = c.as + c.pa.size - LEN_CR_LF + default: + if c.argBuf != nil { + c.argBuf = append(c.argBuf, b) + } + } + case OP_I: + switch b { + case 'N', 'n': + c.state = OP_IN + default: + goto parseErr + } + case OP_IN: + switch b { + case 'F', 'f': + c.state = OP_INF + default: + goto parseErr + } + case OP_INF: + switch b { + case 'O', 'o': + c.state = OP_INFO + default: + goto parseErr + } + case OP_INFO: + switch b { + case ' ', '\t': + continue + default: + c.state = INFO_ARG + c.as = i + } + case INFO_ARG: + switch b { + case '\r': + c.drop = 1 + case '\n': + var arg []byte + if c.argBuf != nil { + arg = c.argBuf + c.argBuf = nil + } else { + arg = buf[c.as : i-c.drop] + } + if err := c.overMaxControlLineLimit(arg, mcl); err != nil { + return err + } + if err := c.processInfo(arg); err != nil { + return err + } + c.drop, c.as, c.state = 0, i+1, OP_START + default: + if c.argBuf != nil { + c.argBuf = append(c.argBuf, b) + } + } + case OP_PLUS: + switch b { + case 'O', 'o': + c.state = OP_PLUS_O + default: + goto parseErr + } + case OP_PLUS_O: + switch b { + case 'K', 'k': + c.state = OP_PLUS_OK + default: + goto parseErr + } + case OP_PLUS_OK: + switch b { + case '\n': + c.drop, c.state = 0, OP_START + } + case OP_MINUS: + switch b { + case 'E', 'e': + c.state = OP_MINUS_E + default: + goto parseErr + } + case OP_MINUS_E: + switch b { + case 'R', 'r': + c.state = OP_MINUS_ER + default: + goto parseErr + } + case OP_MINUS_ER: + switch b { + case 'R', 'r': + c.state = OP_MINUS_ERR + default: + goto parseErr + } + case OP_MINUS_ERR: + switch b { + case ' ', '\t': + c.state = OP_MINUS_ERR_SPC + default: + goto parseErr + } + case OP_MINUS_ERR_SPC: + switch b { + case ' ', '\t': + continue + default: + c.state = MINUS_ERR_ARG + c.as = i + } + case MINUS_ERR_ARG: + switch b { + case '\r': + c.drop = 1 + case '\n': + var arg []byte + if c.argBuf != nil { + arg = c.argBuf + c.argBuf = nil + } else { + arg = buf[c.as : i-c.drop] + } + if err := c.overMaxControlLineLimit(arg, mcl); err != nil { + return err + } + c.processErr(string(arg)) + c.drop, c.as, c.state = 0, i+1, OP_START + default: + if c.argBuf != nil { + c.argBuf = append(c.argBuf, b) + } + } + default: + goto parseErr + } + } + + // Check for split buffer scenarios for any ARG state. + if c.state == SUB_ARG || c.state == UNSUB_ARG || + c.state == PUB_ARG || c.state == HPUB_ARG || + c.state == ASUB_ARG || c.state == AUSUB_ARG || + c.state == MSG_ARG || c.state == HMSG_ARG || + c.state == MINUS_ERR_ARG || c.state == CONNECT_ARG || c.state == INFO_ARG { + + // Setup a holder buffer to deal with split buffer scenario. + if c.argBuf == nil { + c.argBuf = c.scratch[:0] + c.argBuf = append(c.argBuf, buf[c.as:i-c.drop]...) + } + // Check for violations of control line length here. Note that this is not + // exact at all but the performance hit is too great to be precise, and + // catching here should prevent memory exhaustion attacks. + if err := c.overMaxControlLineLimit(c.argBuf, mcl); err != nil { + return err + } + } + + // Check for split msg + if (c.state == MSG_PAYLOAD || c.state == MSG_END_R || c.state == MSG_END_N) && c.msgBuf == nil { + // We need to clone the pubArg if it is still referencing the + // read buffer and we are not able to process the msg. + + if c.argBuf == nil { + // Works also for MSG_ARG, when message comes from ROUTE or GATEWAY. + if err := c.clonePubArg(lmsg); err != nil { + goto parseErr + } + } + + // If we will overflow the scratch buffer, just create a + // new buffer to hold the split message. + if c.pa.size > cap(c.scratch)-len(c.argBuf) { + lrem := len(buf[c.as:]) + // Consider it a protocol error when the remaining payload + // is larger than the reported size for PUB. It can happen + // when processing incomplete messages from rogue clients. + if lrem > c.pa.size+LEN_CR_LF { + goto parseErr + } + c.msgBuf = make([]byte, lrem, c.pa.size+LEN_CR_LF) + copy(c.msgBuf, buf[c.as:]) + } else { + c.msgBuf = c.scratch[len(c.argBuf):len(c.argBuf)] + c.msgBuf = append(c.msgBuf, (buf[c.as:])...) + } + } + + return nil + +authErr: + c.authViolation() + return ErrAuthentication + +parseErr: + c.sendErr("Unknown Protocol Operation") + snip := protoSnippet(i, PROTO_SNIPPET_SIZE, buf) + err := fmt.Errorf("%s parser ERROR, state=%d, i=%d: proto='%s...'", c.kindString(), c.state, i, snip) + return err +} + +func protoSnippet(start, max int, buf []byte) string { + stop := start + max + bufSize := len(buf) + if start >= bufSize { + return `""` + } + if stop > bufSize { + stop = bufSize - 1 + } + return fmt.Sprintf("%q", buf[start:stop]) +} + +// Check if the length of buffer `arg` is over the max control line limit `mcl`. +// If so, an error is sent to the client and the connection is closed. +// The error ErrMaxControlLine is returned. +func (c *client) overMaxControlLineLimit(arg []byte, mcl int32) error { + // Widen to int64 so mcl*16 cannot overflow for large configured values. + effective := int64(mcl) + if c.kind != CLIENT { + // This is the upper bound on argBuf length for LEAF, ROUTER, and GATEWAY connections. + // These kinds need longer arg lines than CLIENT (which is capped at mcl=4096 by default) + // because cluster/leaf frames encode origin, account, reply, and queue groups. + // By default, this is 64 KB, which matches maxBufSize so a single oversized read + // is caught on the very next parse call. + effective *= 16 + } + if int64(len(arg)) > effective { + err := NewErrorCtx(ErrMaxControlLine, "State %d, max_control_line %d, Buffer len %d (snip: %s...)", + c.state, int(mcl), len(c.argBuf), protoSnippet(0, MAX_CONTROL_LINE_SNIPPET_SIZE, arg)) + c.sendErr(err.Error()) + c.closeConnection(MaxControlLineExceeded) + return err + } + return nil +} + +// clonePubArg is used when the split buffer scenario has the pubArg in the existing read buffer, but +// we need to hold onto it into the next read. +func (c *client) clonePubArg(lmsg bool) error { + // Just copy and re-process original arg buffer. + c.argBuf = c.scratch[:0] + c.argBuf = append(c.argBuf, c.pa.arg...) + + switch c.kind { + case ROUTER, GATEWAY: + if lmsg { + return c.processRoutedOriginClusterMsgArgs(c.argBuf) + } + if c.pa.hdr < 0 { + return c.processRoutedMsgArgs(c.argBuf) + } else { + return c.processRoutedHeaderMsgArgs(c.argBuf) + } + case LEAF: + if c.pa.hdr < 0 { + return c.processLeafMsgArgs(c.argBuf) + } else { + return c.processLeafHeaderMsgArgs(c.argBuf) + } + default: + if c.pa.hdr < 0 { + return c.processPub(c.argBuf) + } else { + return c.processHeaderPub(c.argBuf, nil) + } + } +} + +func (ps *parseState) getHeader() http.Header { + if ps.header == nil { + if hdr := ps.pa.hdr; hdr > 0 { + reader := bufio.NewReader(bytes.NewReader(ps.msgBuf[0:hdr])) + tp := textproto.NewReader(reader) + tp.ReadLine() // skip over first line, contains version + if mimeHeader, err := tp.ReadMIMEHeader(); err == nil { + ps.header = http.Header(mimeHeader) + } + } + } + return ps.header +} diff --git a/server/parser_fuzz_test.go b/server/parser_fuzz_test.go new file mode 100644 index 0000000..65422c5 --- /dev/null +++ b/server/parser_fuzz_test.go @@ -0,0 +1,98 @@ +// Copyright 2025-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "sync" + "testing" +) + +func dummyFuzzClient(kind int) *client { + var r *route + var gw *gateway + var lf *leaf + + switch kind { + case ROUTER: + r = &route{} + case GATEWAY: + gw = &gateway{outbound: false, connected: true, insim: make(map[string]*insie), outsim: &sync.Map{}} + case LEAF: + lf = &leaf{} + } + + return &client{ + srv: func() *Server { + opts := defaultServerOptions + return New(&opts) + }(), + kind: kind, + msubs: -1, + in: readCache{ + results: make(map[string]*SublistResult), + pacache: make(map[string]*perAccountCache), + }, + mpay: MAX_PAYLOAD_SIZE, + mcl: MAX_CONTROL_LINE_SIZE, + route: r, + gw: gw, + leaf: lf, + } +} + +// FuzzParser performs fuzz testing on the NATS protocol parser implementation. +// It tests the parser's ability to handle various NATS protocol messages, including +// partial (chunked) message delivery scenarios that may occur in real-world usage. +func FuzzParser(f *testing.F) { + msgs := []string{ + "PING\r\n", + "PONG\r\n", + "PUB foo 33333\r\n", + "HPUB foo INBOX.22 0 5\r\nHELLO\r", + "HMSG $foo foo 10 8\r\nXXXhello\r", + "MSG $foo foo 5\r\nhello\r", + "SUB foo 1\r\nSUB foo 2\r\n", + "UNSUB 1 5\r\n", + "RMSG $G foo.bar | baz 11\r\nhello world\r", + "CONNECT {\"verbose\":false,\"pedantic\":true,\"tls_required\":false}\r\n", + } + + clientKinds := []int{ + CLIENT, + ROUTER, + GATEWAY, + LEAF, + } + + for _, ck := range clientKinds { + for _, crp := range msgs { + f.Add(ck, crp) + } + } + + f.Fuzz(func(t *testing.T, kind int, orig string) { + c := dummyFuzzClient(kind) + + data := []byte(orig) + half := len(data) / 2 + + if err := c.parse(data[:half]); err != nil { + return + } + + if err := c.parse(data[half:]); err != nil { + return + } + }) +} diff --git a/server/parser_test.go b/server/parser_test.go new file mode 100644 index 0000000..5e9082f --- /dev/null +++ b/server/parser_test.go @@ -0,0 +1,1094 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "fmt" + "testing" + "time" + + "github.com/nats-io/jwt/v2" +) + +func dummyClient() *client { + opts := defaultServerOptions + return &client{srv: New(&opts), msubs: -1, mpay: -1, mcl: MAX_CONTROL_LINE_SIZE} +} + +func dummyRouteClient() *client { + opts := defaultServerOptions + return &client{srv: New(&opts), kind: ROUTER, mcl: MAX_CONTROL_LINE_SIZE} +} + +func TestParsePing(t *testing.T) { + c := dummyClient() + if c.state != OP_START { + t.Fatalf("Expected OP_START vs %d\n", c.state) + } + ping := []byte("PING\r\n") + err := c.parse(ping[:1]) + if err != nil || c.state != OP_P { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + err = c.parse(ping[1:2]) + if err != nil || c.state != OP_PI { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + err = c.parse(ping[2:3]) + if err != nil || c.state != OP_PIN { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + err = c.parse(ping[3:4]) + if err != nil || c.state != OP_PING { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + err = c.parse(ping[4:5]) + if err != nil || c.state != OP_PING { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + err = c.parse(ping[5:6]) + if err != nil || c.state != OP_START { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + err = c.parse(ping) + if err != nil || c.state != OP_START { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + // Should tolerate spaces + ping = []byte("PING \r") + err = c.parse(ping) + if err != nil || c.state != OP_PING { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + c.state = OP_START + ping = []byte("PING \r \n") + err = c.parse(ping) + if err != nil || c.state != OP_START { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } +} + +func TestParsePong(t *testing.T) { + c := dummyClient() + if c.state != OP_START { + t.Fatalf("Expected OP_START vs %d\n", c.state) + } + pong := []byte("PONG\r\n") + err := c.parse(pong[:1]) + if err != nil || c.state != OP_P { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + err = c.parse(pong[1:2]) + if err != nil || c.state != OP_PO { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + err = c.parse(pong[2:3]) + if err != nil || c.state != OP_PON { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + err = c.parse(pong[3:4]) + if err != nil || c.state != OP_PONG { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + err = c.parse(pong[4:5]) + if err != nil || c.state != OP_PONG { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + err = c.parse(pong[5:6]) + if err != nil || c.state != OP_START { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if c.ping.out != 0 { + t.Fatalf("Unexpected ping.out value: %d vs 0\n", c.ping.out) + } + err = c.parse(pong) + if err != nil || c.state != OP_START { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if c.ping.out != 0 { + t.Fatalf("Unexpected ping.out value: %d vs 0\n", c.ping.out) + } + // Should tolerate spaces + pong = []byte("PONG \r") + err = c.parse(pong) + if err != nil || c.state != OP_PONG { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + c.state = OP_START + pong = []byte("PONG \r \n") + err = c.parse(pong) + if err != nil || c.state != OP_START { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if c.ping.out != 0 { + t.Fatalf("Unexpected ping.out value: %d vs 0\n", c.ping.out) + } + + // Should be adjusting c.pout (Pings Outstanding): reset to 0 + c.state = OP_START + c.ping.out = 10 + pong = []byte("PONG\r\n") + err = c.parse(pong) + if err != nil || c.state != OP_START { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if c.ping.out != 0 { + t.Fatalf("Unexpected ping.out: %d vs 0\n", c.ping.out) + } +} + +func TestParseConnect(t *testing.T) { + c := dummyClient() + connect := []byte("CONNECT {\"verbose\":false,\"pedantic\":true,\"tls_required\":false}\r\n") + err := c.parse(connect) + if err != nil || c.state != OP_START { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + // Check saved state + if c.as != 8 { + t.Fatalf("ArgStart state incorrect: 8 vs %d\n", c.as) + } +} + +func TestParseSub(t *testing.T) { + c := dummyClient() + sub := []byte("SUB foo 1\r") + err := c.parse(sub) + if err != nil || c.state != SUB_ARG { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + // Check saved state + if c.as != 4 { + t.Fatalf("ArgStart state incorrect: 4 vs %d\n", c.as) + } + if c.drop != 1 { + t.Fatalf("Drop state incorrect: 1 vs %d\n", c.as) + } + if !bytes.Equal(sub[c.as:], []byte("foo 1\r")) { + t.Fatalf("Arg state incorrect: %s\n", sub[c.as:]) + } +} + +func TestParsePub(t *testing.T) { + c := dummyClient() + + pub := []byte("PUB foo 5\r\nhello\r") + err := c.parse(pub) + if err != nil || c.state != MSG_END_N { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if !bytes.Equal(c.pa.subject, []byte("foo")) { + t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", string(c.pa.subject)) + } + if c.pa.reply != nil { + t.Fatalf("Did not parse reply correctly: 'nil' vs '%s'\n", string(c.pa.reply)) + } + if c.pa.size != 5 { + t.Fatalf("Did not parse msg size correctly: 5 vs %d\n", c.pa.size) + } + + // Clear snapshots + c.argBuf, c.msgBuf, c.state = nil, nil, OP_START + + pub = []byte("PUB foo.bar INBOX.22 11\r\nhello world\r") + err = c.parse(pub) + if err != nil || c.state != MSG_END_N { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { + t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", string(c.pa.subject)) + } + if !bytes.Equal(c.pa.reply, []byte("INBOX.22")) { + t.Fatalf("Did not parse reply correctly: 'INBOX.22' vs '%s'\n", string(c.pa.reply)) + } + if c.pa.size != 11 { + t.Fatalf("Did not parse msg size correctly: 11 vs %d\n", c.pa.size) + } + + // Clear snapshots + c.argBuf, c.msgBuf, c.state = nil, nil, OP_START + + // This is the case when data has more bytes than expected by size. + pub = []byte("PUB foo.bar 11\r\nhello world hello world\r") + err = c.parse(pub) + if err == nil { + t.Fatalf("Expected an error parsing longer than expected message body") + } + if c.msgBuf != nil { + t.Fatalf("Did not expect a c.msgBuf to be non-nil") + } +} + +// https://www.twistlock.com/labs-blog/finding-dos-vulnerability-nats-go-fuzz-cve-2019-13126/ +func TestParsePubSizeOverflow(t *testing.T) { + c := dummyClient() + + pub := []byte("PUB foo 3333333333333333333333333333333333333333333333333333333333333333\r\n") + if err := c.parse(pub); err == nil { + t.Fatalf("Expected an error") + } +} + +func TestParsePubArg(t *testing.T) { + c := dummyClient() + + for _, test := range []struct { + arg string + subject string + reply string + size int + szb string + }{ + {arg: "a 2", subject: "a", reply: "", size: 2, szb: "2"}, + {arg: "a 222", subject: "a", reply: "", size: 222, szb: "222"}, + {arg: "foo 22", subject: "foo", reply: "", size: 22, szb: "22"}, + {arg: " foo 22", subject: "foo", reply: "", size: 22, szb: "22"}, + {arg: "foo 22 ", subject: "foo", reply: "", size: 22, szb: "22"}, + {arg: "foo 22", subject: "foo", reply: "", size: 22, szb: "22"}, + {arg: " foo 22 ", subject: "foo", reply: "", size: 22, szb: "22"}, + {arg: " foo 22 ", subject: "foo", reply: "", size: 22, szb: "22"}, + {arg: "foo bar 22", subject: "foo", reply: "bar", size: 22, szb: "22"}, + {arg: " foo bar 22", subject: "foo", reply: "bar", size: 22, szb: "22"}, + {arg: "foo bar 22 ", subject: "foo", reply: "bar", size: 22, szb: "22"}, + {arg: "foo bar 22", subject: "foo", reply: "bar", size: 22, szb: "22"}, + {arg: " foo bar 22 ", subject: "foo", reply: "bar", size: 22, szb: "22"}, + {arg: " foo bar 22 ", subject: "foo", reply: "bar", size: 22, szb: "22"}, + {arg: " foo bar 2222 ", subject: "foo", reply: "bar", size: 2222, szb: "2222"}, + {arg: " foo 2222 ", subject: "foo", reply: "", size: 2222, szb: "2222"}, + {arg: "a\t2", subject: "a", reply: "", size: 2, szb: "2"}, + {arg: "a\t222", subject: "a", reply: "", size: 222, szb: "222"}, + {arg: "foo\t22", subject: "foo", reply: "", size: 22, szb: "22"}, + {arg: "\tfoo\t22", subject: "foo", reply: "", size: 22, szb: "22"}, + {arg: "foo\t22\t", subject: "foo", reply: "", size: 22, szb: "22"}, + {arg: "foo\t\t\t22", subject: "foo", reply: "", size: 22, szb: "22"}, + {arg: "\tfoo\t22\t", subject: "foo", reply: "", size: 22, szb: "22"}, + {arg: "\tfoo\t\t\t22\t", subject: "foo", reply: "", size: 22, szb: "22"}, + {arg: "foo\tbar\t22", subject: "foo", reply: "bar", size: 22, szb: "22"}, + {arg: "\tfoo\tbar\t22", subject: "foo", reply: "bar", size: 22, szb: "22"}, + {arg: "foo\tbar\t22\t", subject: "foo", reply: "bar", size: 22, szb: "22"}, + {arg: "foo\t\tbar\t\t22", subject: "foo", reply: "bar", size: 22, szb: "22"}, + {arg: "\tfoo\tbar\t22\t", subject: "foo", reply: "bar", size: 22, szb: "22"}, + {arg: "\t \tfoo\t \t \tbar\t \t22\t \t", subject: "foo", reply: "bar", size: 22, szb: "22"}, + {arg: "\t\tfoo\t\t\tbar\t\t2222\t\t", subject: "foo", reply: "bar", size: 2222, szb: "2222"}, + {arg: "\t \tfoo\t \t \t\t\t2222\t \t", subject: "foo", reply: "", size: 2222, szb: "2222"}, + } { + t.Run(test.arg, func(t *testing.T) { + if err := c.processPub([]byte(test.arg)); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if !bytes.Equal(c.pa.subject, []byte(test.subject)) { + t.Fatalf("Mismatched subject: '%s'\n", c.pa.subject) + } + if !bytes.Equal(c.pa.reply, []byte(test.reply)) { + t.Fatalf("Mismatched reply subject: '%s'\n", c.pa.reply) + } + if !bytes.Equal(c.pa.szb, []byte(test.szb)) { + t.Fatalf("Bad size buf: '%s'\n", c.pa.szb) + } + if c.pa.size != test.size { + t.Fatalf("Bad size: %d\n", c.pa.size) + } + }) + } +} + +func TestParsePubBadSize(t *testing.T) { + c := dummyClient() + // Setup localized max payload + c.mpay = 32768 + if err := c.processPub([]byte("foo 2222222222222222")); err == nil { + t.Fatalf("Expected parse error for size too large") + } +} + +func TestParseLeafMsgBadSize(t *testing.T) { + c := dummyClient() + c.kind = LEAF + c.leaf = &leaf{} + c.mpay = 16 + require_Error(t, c.processLeafMsgArgs([]byte("foo 17")), ErrMaxPayload) +} + +func TestParseLeafHeaderMsgBadSize(t *testing.T) { + c := dummyClient() + c.kind = LEAF + c.leaf = &leaf{} + c.mpay = 16 + require_Error(t, c.processLeafHeaderMsgArgs([]byte("foo 4 17")), ErrMaxPayload) +} + +func TestParseHeaderPub(t *testing.T) { + c := dummyClient() + c.headers = true + + hpub := []byte("HPUB foo 12 17\r\nname:derek\r\nHELLO\r") + if err := c.parse(hpub); err != nil || c.state != MSG_END_N { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if !bytes.Equal(c.pa.subject, []byte("foo")) { + t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'", c.pa.subject) + } + if c.pa.reply != nil { + t.Fatalf("Did not parse reply correctly: 'nil' vs '%s'", c.pa.reply) + } + if c.pa.hdr != 12 { + t.Fatalf("Did not parse msg header size correctly: 12 vs %d", c.pa.hdr) + } + if !bytes.Equal(c.pa.hdb, []byte("12")) { + t.Fatalf("Did not parse or capture the header size as bytes correctly: %q", c.pa.hdb) + } + if c.pa.size != 17 { + t.Fatalf("Did not parse msg size correctly: 17 vs %d", c.pa.size) + } + + // Clear snapshots + c.argBuf, c.msgBuf, c.state = nil, nil, OP_START + + hpub = []byte("HPUB foo INBOX.22 12 17\r\nname:derek\r\nHELLO\r") + if err := c.parse(hpub); err != nil || c.state != MSG_END_N { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if !bytes.Equal(c.pa.subject, []byte("foo")) { + t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'", c.pa.subject) + } + if !bytes.Equal(c.pa.reply, []byte("INBOX.22")) { + t.Fatalf("Did not parse reply correctly: 'INBOX.22' vs '%s'", c.pa.reply) + } + if c.pa.hdr != 12 { + t.Fatalf("Did not parse msg header size correctly: 12 vs %d", c.pa.hdr) + } + if !bytes.Equal(c.pa.hdb, []byte("12")) { + t.Fatalf("Did not parse or capture the header size as bytes correctly: %q", c.pa.hdb) + } + if c.pa.size != 17 { + t.Fatalf("Did not parse msg size correctly: 17 vs %d", c.pa.size) + } + + // Clear snapshots + c.argBuf, c.msgBuf, c.state = nil, nil, OP_START + + hpub = []byte("HPUB foo INBOX.22 0 5\r\nHELLO\r") + if err := c.parse(hpub); err != nil || c.state != MSG_END_N { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if !bytes.Equal(c.pa.subject, []byte("foo")) { + t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) + } + if !bytes.Equal(c.pa.reply, []byte("INBOX.22")) { + t.Fatalf("Did not parse reply correctly: 'INBOX.22' vs '%s'\n", c.pa.reply) + } + if c.pa.hdr != 0 { + t.Fatalf("Did not parse msg header size correctly: 0 vs %d\n", c.pa.hdr) + } + if !bytes.Equal(c.pa.hdb, []byte("0")) { + t.Fatalf("Did not parse or capture the header size as bytes correctly: %q", c.pa.hdb) + } + if c.pa.size != 5 { + t.Fatalf("Did not parse msg size correctly: 5 vs %d\n", c.pa.size) + } +} + +func TestParseHeaderPubArg(t *testing.T) { + c := dummyClient() + c.headers = true + + for _, test := range []struct { + arg string + subject string + reply string + hdr int + size int + szb string + }{ + {arg: "a 2 4", subject: "a", reply: "", hdr: 2, size: 4, szb: "4"}, + {arg: "a 22 222", subject: "a", reply: "", hdr: 22, size: 222, szb: "222"}, + {arg: "foo 3 22", subject: "foo", reply: "", hdr: 3, size: 22, szb: "22"}, + {arg: " foo 1 22", subject: "foo", reply: "", hdr: 1, size: 22, szb: "22"}, + {arg: "foo 0 22 ", subject: "foo", reply: "", hdr: 0, size: 22, szb: "22"}, + {arg: "foo 0 22", subject: "foo", reply: "", hdr: 0, size: 22, szb: "22"}, + {arg: " foo 1 22 ", subject: "foo", reply: "", hdr: 1, size: 22, szb: "22"}, + {arg: " foo 3 22 ", subject: "foo", reply: "", hdr: 3, size: 22, szb: "22"}, + {arg: "foo bar 1 22", subject: "foo", reply: "bar", hdr: 1, size: 22, szb: "22"}, + {arg: " foo bar 11 22", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, + {arg: "foo bar 11 22 ", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, + {arg: "foo bar 11 22", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, + {arg: " foo bar 11 22 ", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, + {arg: " foo bar 11 22 ", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, + {arg: " foo bar 22 2222 ", subject: "foo", reply: "bar", hdr: 22, size: 2222, szb: "2222"}, + {arg: " foo 1 2222 ", subject: "foo", reply: "", hdr: 1, size: 2222, szb: "2222"}, + {arg: "a\t2\t22", subject: "a", reply: "", hdr: 2, size: 22, szb: "22"}, + {arg: "a\t2\t\t222", subject: "a", reply: "", hdr: 2, size: 222, szb: "222"}, + {arg: "foo\t2 22", subject: "foo", reply: "", hdr: 2, size: 22, szb: "22"}, + {arg: "\tfoo\t11\t 22", subject: "foo", reply: "", hdr: 11, size: 22, szb: "22"}, + {arg: "foo\t11\t22\t", subject: "foo", reply: "", hdr: 11, size: 22, szb: "22"}, + {arg: "foo\t\t\t11 22", subject: "foo", reply: "", hdr: 11, size: 22, szb: "22"}, + {arg: "\tfoo\t11\t \t 22\t", subject: "foo", reply: "", hdr: 11, size: 22, szb: "22"}, + {arg: "\tfoo\t\t\t11 22\t", subject: "foo", reply: "", hdr: 11, size: 22, szb: "22"}, + {arg: "foo\tbar\t2 22", subject: "foo", reply: "bar", hdr: 2, size: 22, szb: "22"}, + {arg: "\tfoo\tbar\t11\t22", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, + {arg: "foo\tbar\t11\t\t22\t ", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, + {arg: "foo\t\tbar\t\t11\t\t\t22", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, + {arg: "\tfoo\tbar\t11\t22\t", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, + {arg: "\t \tfoo\t \t \tbar\t \t11\t 22\t \t", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, + {arg: "\t\tfoo\t\t\tbar\t\t22\t\t\t2222\t\t", subject: "foo", reply: "bar", hdr: 22, size: 2222, szb: "2222"}, + {arg: "\t \tfoo\t \t \t\t\t11\t\t 2222\t \t", subject: "foo", reply: "", hdr: 11, size: 2222, szb: "2222"}, + } { + t.Run(test.arg, func(t *testing.T) { + if err := c.processHeaderPub([]byte(test.arg), nil); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if !bytes.Equal(c.pa.subject, []byte(test.subject)) { + t.Fatalf("Mismatched subject: '%s'\n", c.pa.subject) + } + if !bytes.Equal(c.pa.reply, []byte(test.reply)) { + t.Fatalf("Mismatched reply subject: '%s'\n", c.pa.reply) + } + if !bytes.Equal(c.pa.szb, []byte(test.szb)) { + t.Fatalf("Bad size buf: '%s'\n", c.pa.szb) + } + if c.pa.hdr != test.hdr { + t.Fatalf("Bad header size: %d\n", c.pa.hdr) + } + if c.pa.size != test.size { + t.Fatalf("Bad size: %d\n", c.pa.size) + } + }) + } +} + +func TestParseRoutedHeaderMsg(t *testing.T) { + c := dummyRouteClient() + c.route = &route{} + + pub := []byte("HMSG $foo foo 10 8\r\nXXXhello\r") + if err := c.parse(pub); err == nil { + t.Fatalf("Expected an error") + } + + // Clear snapshots + c.argBuf, c.msgBuf, c.state = nil, nil, OP_START + + pub = []byte("HMSG $foo foo 3 8\r\nXXXhello\r") + err := c.parse(pub) + if err != nil || c.state != MSG_END_N { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if !bytes.Equal(c.pa.account, []byte("$foo")) { + t.Fatalf("Did not parse account correctly: '$foo' vs '%s'\n", c.pa.account) + } + if !bytes.Equal(c.pa.subject, []byte("foo")) { + t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) + } + if c.pa.reply != nil { + t.Fatalf("Did not parse reply correctly: 'nil' vs '%s'\n", c.pa.reply) + } + if c.pa.hdr != 3 { + t.Fatalf("Did not parse header size correctly: 3 vs %d\n", c.pa.hdr) + } + if c.pa.size != 8 { + t.Fatalf("Did not parse msg size correctly: 8 vs %d\n", c.pa.size) + } + + // Clear snapshots + c.argBuf, c.msgBuf, c.state = nil, nil, OP_START + + pub = []byte("HMSG $G foo.bar INBOX.22 3 14\r\nOK:hello world\r") + err = c.parse(pub) + if err != nil || c.state != MSG_END_N { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if !bytes.Equal(c.pa.account, []byte("$G")) { + t.Fatalf("Did not parse account correctly: '$G' vs '%s'\n", c.pa.account) + } + if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { + t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) + } + if !bytes.Equal(c.pa.reply, []byte("INBOX.22")) { + t.Fatalf("Did not parse reply correctly: 'INBOX.22' vs '%s'\n", c.pa.reply) + } + if c.pa.hdr != 3 { + t.Fatalf("Did not parse header size correctly: 3 vs %d\n", c.pa.hdr) + } + if c.pa.size != 14 { + t.Fatalf("Did not parse msg size correctly: 14 vs %d\n", c.pa.size) + } + + // Clear snapshots + c.argBuf, c.msgBuf, c.state = nil, nil, OP_START + + pub = []byte("HMSG $G foo.bar + reply baz 3 14\r\nOK:hello world\r") + err = c.parse(pub) + if err != nil || c.state != MSG_END_N { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if !bytes.Equal(c.pa.account, []byte("$G")) { + t.Fatalf("Did not parse account correctly: '$G' vs '%s'\n", c.pa.account) + } + if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { + t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) + } + if !bytes.Equal(c.pa.reply, []byte("reply")) { + t.Fatalf("Did not parse reply correctly: 'reply' vs '%s'\n", c.pa.reply) + } + if len(c.pa.queues) != 1 { + t.Fatalf("Expected 1 queue, got %d", len(c.pa.queues)) + } + if !bytes.Equal(c.pa.queues[0], []byte("baz")) { + t.Fatalf("Did not parse queues correctly: 'baz' vs '%q'\n", c.pa.queues[0]) + } + if c.pa.hdr != 3 { + t.Fatalf("Did not parse header size correctly: 3 vs %d\n", c.pa.hdr) + } + if c.pa.size != 14 { + t.Fatalf("Did not parse msg size correctly: 14 vs %d\n", c.pa.size) + } + + // Clear snapshots + c.argBuf, c.msgBuf, c.state = nil, nil, OP_START + + pub = []byte("HMSG $G foo.bar | baz 3 14\r\nOK:hello world\r") + err = c.parse(pub) + if err != nil || c.state != MSG_END_N { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if !bytes.Equal(c.pa.account, []byte("$G")) { + t.Fatalf("Did not parse account correctly: '$G' vs '%s'\n", c.pa.account) + } + if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { + t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) + } + if !bytes.Equal(c.pa.reply, []byte("")) { + t.Fatalf("Did not parse reply correctly: '' vs '%s'\n", c.pa.reply) + } + if len(c.pa.queues) != 1 { + t.Fatalf("Expected 1 queue, got %d", len(c.pa.queues)) + } + if !bytes.Equal(c.pa.queues[0], []byte("baz")) { + t.Fatalf("Did not parse queues correctly: 'baz' vs '%q'\n", c.pa.queues[0]) + } + if c.pa.hdr != 3 { + t.Fatalf("Did not parse header size correctly: 3 vs %d\n", c.pa.hdr) + } + if c.pa.size != 14 { + t.Fatalf("Did not parse msg size correctly: 14 vs %d\n", c.pa.size) + } +} + +func TestParseRouteMsg(t *testing.T) { + c := dummyRouteClient() + c.route = &route{} + + pub := []byte("MSG $foo foo 5\r\nhello\r") + err := c.parse(pub) + if err == nil { + t.Fatalf("Expected an error, got none") + } + pub = []byte("RMSG $foo foo 5\r\nhello\r") + err = c.parse(pub) + if err != nil || c.state != MSG_END_N { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if !bytes.Equal(c.pa.account, []byte("$foo")) { + t.Fatalf("Did not parse account correctly: '$foo' vs '%s'\n", c.pa.account) + } + if !bytes.Equal(c.pa.subject, []byte("foo")) { + t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) + } + if c.pa.reply != nil { + t.Fatalf("Did not parse reply correctly: 'nil' vs '%s'\n", c.pa.reply) + } + if c.pa.size != 5 { + t.Fatalf("Did not parse msg size correctly: 5 vs %d\n", c.pa.size) + } + + // Clear snapshots + c.argBuf, c.msgBuf, c.state = nil, nil, OP_START + + pub = []byte("RMSG $G foo.bar INBOX.22 11\r\nhello world\r") + err = c.parse(pub) + if err != nil || c.state != MSG_END_N { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if !bytes.Equal(c.pa.account, []byte("$G")) { + t.Fatalf("Did not parse account correctly: '$G' vs '%s'\n", c.pa.account) + } + if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { + t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) + } + if !bytes.Equal(c.pa.reply, []byte("INBOX.22")) { + t.Fatalf("Did not parse reply correctly: 'INBOX.22' vs '%s'\n", c.pa.reply) + } + if c.pa.size != 11 { + t.Fatalf("Did not parse msg size correctly: 11 vs %d\n", c.pa.size) + } + + // Clear snapshots + c.argBuf, c.msgBuf, c.state = nil, nil, OP_START + + pub = []byte("RMSG $G foo.bar + reply baz 11\r\nhello world\r") + err = c.parse(pub) + if err != nil || c.state != MSG_END_N { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if !bytes.Equal(c.pa.account, []byte("$G")) { + t.Fatalf("Did not parse account correctly: '$G' vs '%s'\n", c.pa.account) + } + if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { + t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) + } + if !bytes.Equal(c.pa.reply, []byte("reply")) { + t.Fatalf("Did not parse reply correctly: 'reply' vs '%s'\n", c.pa.reply) + } + if len(c.pa.queues) != 1 { + t.Fatalf("Expected 1 queue, got %d", len(c.pa.queues)) + } + if !bytes.Equal(c.pa.queues[0], []byte("baz")) { + t.Fatalf("Did not parse queues correctly: 'baz' vs '%q'\n", c.pa.queues[0]) + } + + // Clear snapshots + c.argBuf, c.msgBuf, c.state = nil, nil, OP_START + + pub = []byte("RMSG $G foo.bar | baz 11\r\nhello world\r") + err = c.parse(pub) + if err != nil || c.state != MSG_END_N { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if !bytes.Equal(c.pa.account, []byte("$G")) { + t.Fatalf("Did not parse account correctly: '$G' vs '%s'\n", c.pa.account) + } + if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { + t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) + } + if !bytes.Equal(c.pa.reply, []byte("")) { + t.Fatalf("Did not parse reply correctly: '' vs '%s'\n", c.pa.reply) + } + if len(c.pa.queues) != 1 { + t.Fatalf("Expected 1 queue, got %d", len(c.pa.queues)) + } + if !bytes.Equal(c.pa.queues[0], []byte("baz")) { + t.Fatalf("Did not parse queues correctly: 'baz' vs '%q'\n", c.pa.queues[0]) + } +} + +func TestParseMsgSpace(t *testing.T) { + c := dummyRouteClient() + + // Ivan bug he found + if err := c.parse([]byte("MSG \r\n")); err == nil { + t.Fatalf("Expected parse error for MSG ") + } + + c = dummyClient() + + // Anything with an M from a client should parse error + if err := c.parse([]byte("M")); err == nil { + t.Fatalf("Expected parse error for M* from a client") + } +} + +func TestParsePingNoAuthUserExceptionsAndRestrictions(t *testing.T) { + opts := defaultServerOptions + s := New(&opts) + s.opts.NoAuthUser = "foo" + s.users = map[string]*User{ + "foo": {Username: "foo"}, + } + + newAuthPendingClient := func(kind int, gw *gateway, route *route, leaf *leaf, ws *websocket) *client { + c := &client{ + srv: s, kind: kind, + gw: gw, route: route, leaf: leaf, ws: ws, + atmr: time.AfterFunc(time.Minute, func() {}), + } + c.flags.set(expectConnect) + t.Cleanup(func() { + c.mu.Lock() + defer c.mu.Unlock() + c.clearAuthTimer() + }) + return c + } + + for _, test := range []struct { + name string + kind int + gw *gateway + route *route + leaf *leaf + }{ + {name: "Leaf", kind: LEAF, leaf: &leaf{}}, + {name: "Route", kind: ROUTER, route: &route{}}, + {name: "Gateway", kind: GATEWAY, gw: &gateway{}}, + } { + t.Run(test.name+"NotAllowed", func(t *testing.T) { + c := newAuthPendingClient(test.kind, test.gw, test.route, test.leaf, nil) + require_Error(t, c.parse([]byte("PING\r\n")), ErrAuthentication) + require_False(t, c.flags.isSet(connectReceived)) + }) + } + + for _, test := range []struct { + name string + kind int + ws *websocket + }{ + {name: "Client", kind: CLIENT}, + {name: "WebSocket", kind: CLIENT, ws: &websocket{}}, + } { + t.Run(test.name+"Allowed", func(t *testing.T) { + c := newAuthPendingClient(test.kind, nil, nil, nil, test.ws) + require_NoError(t, c.parse([]byte("PING\r\n"))) + require_True(t, c.flags.isSet(connectReceived)) + }) + } +} + +func TestParsePingNoAuthUserEnforcesUserRestrictions(t *testing.T) { + newServer := func(user *User) *Server { + opts := defaultServerOptions + s := New(&opts) + s.opts.NoAuthUser = user.Username + s.users = map[string]*User{user.Username: user} + return s + } + + newAuthPendingClient := func(s *Server, ws *websocket) *client { + c := &client{ + srv: s, kind: CLIENT, ws: ws, + atmr: time.AfterFunc(time.Minute, func() {}), + } + c.flags.set(expectConnect) + t.Cleanup(func() { + c.mu.Lock() + defer c.mu.Unlock() + c.clearAuthTimer() + }) + return c + } + + t.Run("AllowedConnectionTypes", func(t *testing.T) { + // no_auth_user restricted to WebSocket connections only. + s := newServer(&User{ + Username: "foo", + AllowedConnectionTypes: map[string]struct{}{jwt.ConnectionTypeWebsocket: {}}, + }) + + // A standard TCP client must be rejected, just like the CONNECT path. + c := newAuthPendingClient(s, nil) + require_Error(t, c.parse([]byte("PING\r\n")), ErrAuthentication) + require_False(t, c.flags.isSet(connectReceived)) + + // A WebSocket client satisfies the restriction and is allowed. + c = newAuthPendingClient(s, &websocket{}) + require_NoError(t, c.parse([]byte("PING\r\n"))) + require_True(t, c.flags.isSet(connectReceived)) + }) + + t.Run("ProxyRequired", func(t *testing.T) { + // no_auth_user that may only connect through a trusted proxy. A + // connection reaching the pre-CONNECT fast path never carries a proxy + // signature, so it must always be rejected. + s := newServer(&User{Username: "foo", ProxyRequired: true}) + + for _, ws := range []*websocket{nil, {}} { + c := newAuthPendingClient(s, ws) + require_Error(t, c.parse([]byte("PING\r\n")), ErrAuthentication) + require_False(t, c.flags.isSet(connectReceived)) + } + }) +} + +func TestParsePingAllowedAfterConnectForNonClients(t *testing.T) { + for _, test := range []struct { + name string + kind int + gw *gateway + route *route + leaf *leaf + }{ + {name: "Leaf", kind: LEAF, leaf: &leaf{}}, + {name: "Route", kind: ROUTER, route: &route{}}, + {name: "Gateway", kind: GATEWAY, gw: &gateway{}}, + } { + t.Run(test.name, func(t *testing.T) { + c := &client{ + srv: func() *Server { + opts := defaultServerOptions + return New(&opts) + }(), + kind: test.kind, + gw: test.gw, + route: test.route, + leaf: test.leaf, + } + c.flags.set(connectReceived) + + require_NoError(t, c.parse([]byte("PING\r\n"))) + require_Equal(t, c.state, OP_START) + }) + } +} + +func TestShouldFail(t *testing.T) { + wrongProtos := []string{ + "xxx", + "Px", "PIx", "PINx", " PING", + "POx", "PONx", + "+x", "+Ox", + "-x", "-Ex", "-ERx", "-ERRx", + "Cx", "COx", "CONx", "CONNx", "CONNEx", "CONNECx", "CONNECx", "CONNECT \r\n", + "PUx", "PUB foo\r\n", "PUB \r\n", "PUB foo bar \r\n", + "PUB foo 2\r\nok \r\n", "PUB foo 2\r\nok\r \n", + "Sx", "SUx", "SUB\r\n", "SUB \r\n", "SUB foo\r\n", + "SUB foo bar baz 22\r\n", + "Ux", "UNx", "UNSx", "UNSUx", "UNSUBx", "UNSUBUNSUB 1\r\n", "UNSUB_2\r\n", + "UNSUB_UNSUB_UNSUB 2\r\n", "UNSUB_\t2\r\n", "UNSUB\r\n", "UNSUB \r\n", + "UNSUB \t \r\n", + "Ix", "INx", "INFx", "INFO \r\n", + } + for _, proto := range wrongProtos { + c := dummyClient() + if err := c.parse([]byte(proto)); err == nil { + t.Fatalf("Should have received a parse error for: %v", proto) + } + } + + // Special case for MSG, type needs to not be client. + wrongProtos = []string{"Mx", "MSx", "MSGx", "MSG \r\n"} + for _, proto := range wrongProtos { + c := dummyClient() + c.kind = ROUTER + if err := c.parse([]byte(proto)); err == nil { + t.Fatalf("Should have received a parse error for: %v", proto) + } + } +} + +func TestProtoSnippet(t *testing.T) { + sample := []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + + tests := []struct { + input int + expected string + }{ + {0, `"abcdefghijklmnopqrstuvwxyzABCDEF"`}, + {1, `"bcdefghijklmnopqrstuvwxyzABCDEFG"`}, + {2, `"cdefghijklmnopqrstuvwxyzABCDEFGH"`}, + {3, `"defghijklmnopqrstuvwxyzABCDEFGHI"`}, + {4, `"efghijklmnopqrstuvwxyzABCDEFGHIJ"`}, + {5, `"fghijklmnopqrstuvwxyzABCDEFGHIJK"`}, + {6, `"ghijklmnopqrstuvwxyzABCDEFGHIJKL"`}, + {7, `"hijklmnopqrstuvwxyzABCDEFGHIJKLM"`}, + {8, `"ijklmnopqrstuvwxyzABCDEFGHIJKLMN"`}, + {9, `"jklmnopqrstuvwxyzABCDEFGHIJKLMNO"`}, + {10, `"klmnopqrstuvwxyzABCDEFGHIJKLMNOP"`}, + {11, `"lmnopqrstuvwxyzABCDEFGHIJKLMNOPQ"`}, + {12, `"mnopqrstuvwxyzABCDEFGHIJKLMNOPQR"`}, + {13, `"nopqrstuvwxyzABCDEFGHIJKLMNOPQRS"`}, + {14, `"opqrstuvwxyzABCDEFGHIJKLMNOPQRST"`}, + {15, `"pqrstuvwxyzABCDEFGHIJKLMNOPQRSTU"`}, + {16, `"qrstuvwxyzABCDEFGHIJKLMNOPQRSTUV"`}, + {17, `"rstuvwxyzABCDEFGHIJKLMNOPQRSTUVW"`}, + {18, `"stuvwxyzABCDEFGHIJKLMNOPQRSTUVWX"`}, + {19, `"tuvwxyzABCDEFGHIJKLMNOPQRSTUVWXY"`}, + {20, `"uvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"`}, + {21, `"vwxyzABCDEFGHIJKLMNOPQRSTUVWXY"`}, + {22, `"wxyzABCDEFGHIJKLMNOPQRSTUVWXY"`}, + {23, `"xyzABCDEFGHIJKLMNOPQRSTUVWXY"`}, + {24, `"yzABCDEFGHIJKLMNOPQRSTUVWXY"`}, + {25, `"zABCDEFGHIJKLMNOPQRSTUVWXY"`}, + {26, `"ABCDEFGHIJKLMNOPQRSTUVWXY"`}, + {27, `"BCDEFGHIJKLMNOPQRSTUVWXY"`}, + {28, `"CDEFGHIJKLMNOPQRSTUVWXY"`}, + {29, `"DEFGHIJKLMNOPQRSTUVWXY"`}, + {30, `"EFGHIJKLMNOPQRSTUVWXY"`}, + {31, `"FGHIJKLMNOPQRSTUVWXY"`}, + {32, `"GHIJKLMNOPQRSTUVWXY"`}, + {33, `"HIJKLMNOPQRSTUVWXY"`}, + {34, `"IJKLMNOPQRSTUVWXY"`}, + {35, `"JKLMNOPQRSTUVWXY"`}, + {36, `"KLMNOPQRSTUVWXY"`}, + {37, `"LMNOPQRSTUVWXY"`}, + {38, `"MNOPQRSTUVWXY"`}, + {39, `"NOPQRSTUVWXY"`}, + {40, `"OPQRSTUVWXY"`}, + {41, `"PQRSTUVWXY"`}, + {42, `"QRSTUVWXY"`}, + {43, `"RSTUVWXY"`}, + {44, `"STUVWXY"`}, + {45, `"TUVWXY"`}, + {46, `"UVWXY"`}, + {47, `"VWXY"`}, + {48, `"WXY"`}, + {49, `"XY"`}, + {50, `"Y"`}, + {51, `""`}, + {52, `""`}, + {53, `""`}, + {54, `""`}, + } + + for _, tt := range tests { + got := protoSnippet(tt.input, PROTO_SNIPPET_SIZE, sample) + if tt.expected != got { + t.Errorf("Expected protocol snippet to be %s when start=%d but got %s\n", tt.expected, tt.input, got) + } + } +} + +func TestParseOK(t *testing.T) { + c := dummyClient() + if c.state != OP_START { + t.Fatalf("Expected OP_START vs %d\n", c.state) + } + okProto := []byte("+OK\r\n") + err := c.parse(okProto[:1]) + if err != nil || c.state != OP_PLUS { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + err = c.parse(okProto[1:2]) + if err != nil || c.state != OP_PLUS_O { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + err = c.parse(okProto[2:3]) + if err != nil || c.state != OP_PLUS_OK { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + err = c.parse(okProto[3:4]) + if err != nil || c.state != OP_PLUS_OK { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + err = c.parse(okProto[4:5]) + if err != nil || c.state != OP_START { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } +} + +func TestMaxControlLine(t *testing.T) { + for _, test := range []struct { + name string + kind int + shouldFail bool + }{ + {"client", CLIENT, true}, + {"leaf", LEAF, false}, + {"route", ROUTER, false}, + {"gateway", GATEWAY, false}, + } { + t.Run(test.name, func(t *testing.T) { + pub := []byte("PUB foo.bar.baz 2\r\nok\r\n") + + setupClient := func() *client { + c := dummyClient() + c.setNoReconnect() + c.flags.set(connectReceived) + c.kind = test.kind + switch test.kind { + case ROUTER: + c.route = &route{} + case GATEWAY: + c.gw = &gateway{outbound: false, connected: true, insim: make(map[string]*insie)} + } + c.mcl = 8 + return c + } + + c := setupClient() + // First try with a partial: + // PUB foo.bar.baz 2\r\nok\r\n + // .............^ + err := c.parse(pub[:14]) + switch test.shouldFail { + case true: + if !ErrorIs(err, ErrMaxControlLine) { + t.Fatalf("Expected an error parsing longer than expected control line") + } + case false: + if err != nil { + t.Fatalf("Should not have failed, got %v", err) + } + } + + // Now with full protocol (no split) and we should still enforce. + c = setupClient() + err = c.parse(pub) + switch test.shouldFail { + case true: + if !ErrorIs(err, ErrMaxControlLine) { + t.Fatalf("Expected an error parsing longer than expected control line") + } + case false: + if err != nil { + t.Fatalf("Should not have failed, got %v", err) + } + } + }) + } +} + +func TestMaxControlLineNonClientUpperBound(t *testing.T) { + for _, test := range []struct { + name string + kind int + }{ + {"leaf", LEAF}, + {"route", ROUTER}, + {"gateway", GATEWAY}, + } { + t.Run(test.name, func(t *testing.T) { + setupClient := func() *client { + c := dummyClient() + c.setNoReconnect() + c.flags.set(connectReceived) + c.kind = test.kind + switch test.kind { + case ROUTER: + c.route = &route{} + case GATEWAY: + c.gw = &gateway{outbound: false, connected: true, insim: make(map[string]*insie)} + } + // mcl is the CLIENT default; non-CLIENT uses 16 times that instead + c.mcl = MAX_CONTROL_LINE_SIZE + return c + } + + for _, off := range []int{-1, 0, 1} { + t.Run(fmt.Sprintf("%d", off), func(t *testing.T) { + // Build a PUB arg that exceeds the max control line. + // Feed it in two chunks so argBuf accumulates across parse calls + // (replicates the split-buffer trickle-feed exploit path). + defaultMaxControlLineSize := MAX_CONTROL_LINE_SIZE * 16 + subject := bytes.Repeat([]byte("x"), defaultMaxControlLineSize+off) + chunk1 := append([]byte("PUB "), subject[:defaultMaxControlLineSize/2]...) + chunk2 := subject[defaultMaxControlLineSize/2:] + + c := setupClient() + // First chunk — no error expected yet (argBuf < limit) + if err := c.parse(chunk1); err != nil { + t.Fatalf("unexpected error on first chunk: %v", err) + } + // Second chunk pushes argBuf over max control line + err := c.parse(chunk2) + if off <= 0 && err != nil { + t.Fatalf("unexpected error on second chunk: %v", err) + } else if off > 0 && !ErrorIs(err, ErrMaxControlLine) { + t.Fatalf("expected ErrMaxControlLine after oversized accumulation, got: %v", err) + } + }) + } + }) + } +} diff --git a/server/ping_test.go b/server/ping_test.go new file mode 100644 index 0000000..482c4d3 --- /dev/null +++ b/server/ping_test.go @@ -0,0 +1,68 @@ +// Copyright 2015-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bufio" + "fmt" + "net" + "testing" + "time" +) + +const PING_CLIENT_PORT = 11228 + +var DefaultPingOptions = Options{ + Host: "127.0.0.1", + Port: PING_CLIENT_PORT, + NoLog: true, + NoSigs: true, + PingInterval: 50 * time.Millisecond, +} + +func TestPing(t *testing.T) { + o := DefaultPingOptions + o.DisableShortFirstPing = true + s := RunServer(&o) + defer s.Shutdown() + + c, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", PING_CLIENT_PORT)) + if err != nil { + t.Fatalf("Error connecting: %v", err) + } + defer c.Close() + br := bufio.NewReader(c) + // Wait for INFO + br.ReadLine() + // Send CONNECT + c.Write([]byte("CONNECT {\"verbose\":false}\r\nPING\r\n")) + // Wait for first PONG + br.ReadLine() + // Wait for PING + start := time.Now() + for i := 0; i < 3; i++ { + l, _, err := br.ReadLine() + if err != nil { + t.Fatalf("Error: %v", err) + } + if string(l) != "PING" { + t.Fatalf("Expected PING, got %q", l) + } + if dur := time.Since(start); dur < 25*time.Millisecond || dur > 75*time.Millisecond { + t.Fatalf("Pings duration off: %v", dur) + } + c.Write([]byte(pongProto)) + start = time.Now() + } +} diff --git a/server/proto.go b/server/proto.go new file mode 100644 index 0000000..511d0f6 --- /dev/null +++ b/server/proto.go @@ -0,0 +1,275 @@ +// Copyright 2024-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Inspired by https://github.com/protocolbuffers/protobuf-go/blob/master/encoding/protowire/wire.go + +package server + +import ( + "errors" + "fmt" + "math" +) + +var errProtoInsufficient = errors.New("insufficient data to read a value") +var errProtoOverflow = errors.New("too much data for a value") +var errProtoInvalidFieldNumber = errors.New("invalid field number") + +func protoScanField(b []byte) (num, typ, size int, err error) { + num, typ, sizeTag, err := protoScanTag(b) + if err != nil { + return 0, 0, 0, err + } + b = b[sizeTag:] + + sizeValue, err := protoScanFieldValue(typ, b) + if err != nil { + return 0, 0, 0, err + } + return num, typ, sizeTag + sizeValue, nil +} + +func protoScanTag(b []byte) (num, typ, size int, err error) { + tagint, size, err := protoScanVarint(b) + if err != nil { + return 0, 0, 0, err + } + + // NOTE: MessageSet allows for larger field numbers than normal. + if (tagint >> 3) > uint64(math.MaxInt32) { + return 0, 0, 0, errProtoInvalidFieldNumber + } + num = int(tagint >> 3) + if num < 1 { + return 0, 0, 0, errProtoInvalidFieldNumber + } + typ = int(tagint & 7) + + return num, typ, size, nil +} + +func protoScanFieldValue(typ int, b []byte) (size int, err error) { + switch typ { + case 0: + _, size, err = protoScanVarint(b) + case 5: // fixed32 + if len(b) < 4 { + return 0, errProtoInsufficient + } + size = 4 + case 1: // fixed64 + if len(b) < 8 { + return 0, errProtoInsufficient + } + size = 8 + case 2: // length-delimited + size, err = protoScanBytes(b) + default: + return 0, fmt.Errorf("unsupported type: %d", typ) + } + return size, err +} + +func protoScanVarint(b []byte) (v uint64, size int, err error) { + var y uint64 + if len(b) <= 0 { + return 0, 0, errProtoInsufficient + } + v = uint64(b[0]) + if v < 0x80 { + return v, 1, nil + } + v -= 0x80 + + if len(b) <= 1 { + return 0, 0, errProtoInsufficient + } + y = uint64(b[1]) + v += y << 7 + if y < 0x80 { + return v, 2, nil + } + v -= 0x80 << 7 + + if len(b) <= 2 { + return 0, 0, errProtoInsufficient + } + y = uint64(b[2]) + v += y << 14 + if y < 0x80 { + return v, 3, nil + } + v -= 0x80 << 14 + + if len(b) <= 3 { + return 0, 0, errProtoInsufficient + } + y = uint64(b[3]) + v += y << 21 + if y < 0x80 { + return v, 4, nil + } + v -= 0x80 << 21 + + if len(b) <= 4 { + return 0, 0, errProtoInsufficient + } + y = uint64(b[4]) + v += y << 28 + if y < 0x80 { + return v, 5, nil + } + v -= 0x80 << 28 + + if len(b) <= 5 { + return 0, 0, errProtoInsufficient + } + y = uint64(b[5]) + v += y << 35 + if y < 0x80 { + return v, 6, nil + } + v -= 0x80 << 35 + + if len(b) <= 6 { + return 0, 0, errProtoInsufficient + } + y = uint64(b[6]) + v += y << 42 + if y < 0x80 { + return v, 7, nil + } + v -= 0x80 << 42 + + if len(b) <= 7 { + return 0, 0, errProtoInsufficient + } + y = uint64(b[7]) + v += y << 49 + if y < 0x80 { + return v, 8, nil + } + v -= 0x80 << 49 + + if len(b) <= 8 { + return 0, 0, errProtoInsufficient + } + y = uint64(b[8]) + v += y << 56 + if y < 0x80 { + return v, 9, nil + } + v -= 0x80 << 56 + + if len(b) <= 9 { + return 0, 0, errProtoInsufficient + } + y = uint64(b[9]) + v += y << 63 + if y < 2 { + return v, 10, nil + } + return 0, 0, errProtoOverflow +} + +func protoScanBytes(b []byte) (size int, err error) { + l, lenSize, err := protoScanVarint(b) + if err != nil { + return 0, err + } + if l > uint64(len(b[lenSize:])) { + return 0, errProtoInsufficient + } + return lenSize + int(l), nil +} + +func protoEncodeVarint(v uint64) []byte { + b := make([]byte, 0, 10) + switch { + case v < 1<<7: + b = append(b, byte(v)) + case v < 1<<14: + b = append(b, + byte((v>>0)&0x7f|0x80), + byte(v>>7)) + case v < 1<<21: + b = append(b, + byte((v>>0)&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte(v>>14)) + case v < 1<<28: + b = append(b, + byte((v>>0)&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte(v>>21)) + case v < 1<<35: + b = append(b, + byte((v>>0)&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte(v>>28)) + case v < 1<<42: + b = append(b, + byte((v>>0)&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte((v>>28)&0x7f|0x80), + byte(v>>35)) + case v < 1<<49: + b = append(b, + byte((v>>0)&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte((v>>28)&0x7f|0x80), + byte((v>>35)&0x7f|0x80), + byte(v>>42)) + case v < 1<<56: + b = append(b, + byte((v>>0)&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte((v>>28)&0x7f|0x80), + byte((v>>35)&0x7f|0x80), + byte((v>>42)&0x7f|0x80), + byte(v>>49)) + case v < 1<<63: + b = append(b, + byte((v>>0)&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte((v>>28)&0x7f|0x80), + byte((v>>35)&0x7f|0x80), + byte((v>>42)&0x7f|0x80), + byte((v>>49)&0x7f|0x80), + byte(v>>56)) + default: + b = append(b, + byte((v>>0)&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte((v>>28)&0x7f|0x80), + byte((v>>35)&0x7f|0x80), + byte((v>>42)&0x7f|0x80), + byte((v>>49)&0x7f|0x80), + byte((v>>56)&0x7f|0x80), + 1) + } + return b +} diff --git a/server/pse/freebsd.txt b/server/pse/freebsd.txt new file mode 100644 index 0000000..877068c --- /dev/null +++ b/server/pse/freebsd.txt @@ -0,0 +1,30 @@ +/* + * Compile and run this as a C program to get the kinfo_proc offsets + * for your architecture. + * While FreeBSD works hard at binary-compatibility within an ABI, various + * we can't say for sure that these are right for _all_ use on a hardware + * platform. The LP64 ifdef affects the offsets considerably. + * + * We use these offsets in hardware-specific files for FreeBSD, to avoid a cgo + * compilation-time dependency, allowing us to cross-compile for FreeBSD from + * other hardware platforms, letting us distribute binaries for FreeBSD. + */ + +#include +#include +#include +#include +#include + +#define SHOW_OFFSET(FIELD) printf(" KIP_OFF_%s = %zu\n", #FIELD, offsetof(struct kinfo_proc, ki_ ## FIELD)) + +int main(int argc, char *argv[]) { + /* Uncomment these if you want some extra debugging aids: + SHOW_OFFSET(pid); + SHOW_OFFSET(ppid); + SHOW_OFFSET(uid); + */ + SHOW_OFFSET(size); + SHOW_OFFSET(rssize); + SHOW_OFFSET(pctcpu); +} diff --git a/server/pse/pse_darwin.go b/server/pse/pse_darwin.go new file mode 100644 index 0000000..7fd26b1 --- /dev/null +++ b/server/pse/pse_darwin.go @@ -0,0 +1,86 @@ +// Copyright 2015-2021 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pse + +// On macs after some studying it seems that typical tools like ps and activity monitor report MaxRss and not +// current RSS. I wrote some C code to pull the real RSS and although it does not go down very often, when it does +// that is not reflected in the typical tooling one might compare us to, so we can skip cgo and just use rusage imo. +// We also do not use virtual memory in the upper layers at all, so ok to skip since rusage does not report vss. + +import ( + "math" + "sync" + "syscall" + "time" +) + +type lastUsage struct { + sync.Mutex + last time.Time + cpu time.Duration + rss int64 + pcpu float64 +} + +// To hold the last usage and call time. +var lu lastUsage + +func init() { + updateUsage() + periodic() +} + +// Get our usage. +func getUsage() (now time.Time, cpu time.Duration, rss int64) { + var ru syscall.Rusage + syscall.Getrusage(syscall.RUSAGE_SELF, &ru) + now = time.Now() + cpu = time.Duration(ru.Utime.Sec)*time.Second + time.Duration(ru.Utime.Usec)*time.Microsecond + cpu += time.Duration(ru.Stime.Sec)*time.Second + time.Duration(ru.Stime.Usec)*time.Microsecond + return now, cpu, ru.Maxrss +} + +// Update last usage. +// We need to have a prior sample to compute pcpu. +func updateUsage() (pcpu float64, rss int64) { + lu.Lock() + defer lu.Unlock() + + now, cpu, rss := getUsage() + // Don't skew pcpu by sampling too close to last sample. + if elapsed := now.Sub(lu.last); elapsed < 500*time.Millisecond { + // Always update rss. + lu.rss = rss + } else { + tcpu := float64(cpu - lu.cpu) + lu.last, lu.cpu, lu.rss = now, cpu, rss + // Want to make this one decimal place and not count on upper layers. + // Cores already taken into account via cpu time measurements. + lu.pcpu = math.Round(tcpu/float64(elapsed)*1000) / 10 + } + return lu.pcpu, lu.rss +} + +// Sampling function to keep pcpu relevant. +func periodic() { + updateUsage() + time.AfterFunc(time.Second, periodic) +} + +// ProcUsage returns CPU and memory usage. +// Note upper layers do not use virtual memory size, so ok that it is not filled in here. +func ProcUsage(pcpu *float64, rss, vss *int64) error { + *pcpu, *rss = updateUsage() + return nil +} diff --git a/server/pse/pse_dragonfly.go b/server/pse/pse_dragonfly.go new file mode 100644 index 0000000..55a917a --- /dev/null +++ b/server/pse/pse_dragonfly.go @@ -0,0 +1,36 @@ +// Copyright 2015-2023 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copied from pse_openbsd.go + +package pse + +import ( + "fmt" + "os" + "os/exec" +) + +// ProcUsage returns CPU usage +func ProcUsage(pcpu *float64, rss, vss *int64) error { + pidStr := fmt.Sprintf("%d", os.Getpid()) + out, err := exec.Command("ps", "o", "pcpu=,rss=,vsz=", "-p", pidStr).Output() + if err != nil { + *rss, *vss = -1, -1 + return fmt.Errorf("ps call failed:%v", err) + } + fmt.Sscanf(string(out), "%f %d %d", pcpu, rss, vss) + *rss *= 1024 // 1k blocks, want bytes. + *vss *= 1024 // 1k blocks, want bytes. + return nil +} diff --git a/server/pse/pse_freebsd_cgo.go b/server/pse/pse_freebsd_cgo.go new file mode 100644 index 0000000..e759a48 --- /dev/null +++ b/server/pse/pse_freebsd_cgo.go @@ -0,0 +1,85 @@ +// Copyright 2015-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build cgo && freebsd + +package pse + +/* +#include +#include +#include +#include +#include + +long pagetok(long size) +{ + int pageshift, pagesize; + + pagesize = getpagesize(); + pageshift = 0; + + while (pagesize > 1) { + pageshift++; + pagesize >>= 1; + } + + return (size << pageshift); +} + +int getusage(double *pcpu, unsigned int *rss, unsigned int *vss) +{ + int mib[4], ret; + size_t len; + struct kinfo_proc kp; + + len = 4; + sysctlnametomib("kern.proc.pid", mib, &len); + + mib[3] = getpid(); + len = sizeof(kp); + + ret = sysctl(mib, 4, &kp, &len, NULL, 0); + if (ret != 0) { + return (errno); + } + + *rss = pagetok(kp.ki_rssize); + *vss = kp.ki_size; + *pcpu = (double)kp.ki_pctcpu / FSCALE; + + return 0; +} + +*/ +import "C" + +import ( + "syscall" +) + +// This is a placeholder for now. +func ProcUsage(pcpu *float64, rss, vss *int64) error { + var r, v C.uint + var c C.double + + if ret := C.getusage(&c, &r, &v); ret != 0 { + return syscall.Errno(ret) + } + + *pcpu = float64(c) + *rss = int64(r) + *vss = int64(v) + + return nil +} diff --git a/server/pse/pse_freebsd_sysctl.go b/server/pse/pse_freebsd_sysctl.go new file mode 100644 index 0000000..f4ea199 --- /dev/null +++ b/server/pse/pse_freebsd_sysctl.go @@ -0,0 +1,125 @@ +// Copyright 2015-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// There are two FreeBSD implementations; one which uses cgo and should build +// locally on any FreeBSD, and this one which uses sysctl but needs us to know +// the offset constants for the fields we care about. +// +// The advantage of this one is that without cgo, it is much easier to +// cross-compile to a target. The official releases are all built with +// cross-compilation. +// +// We've switched the other implementation to include '_cgo' in the filename, +// to show that it's not the default. This isn't an os or arch build tag, +// so we have to use explicit build-tags within. +// If lacking CGO support and targeting an unsupported arch, then before the +// change you would have a compile failure for not being able to cross-compile. +// After the change, you have a compile failure for not having the symbols +// because no source file satisfies them. +// Thus we are no worse off, and it's now much easier to extend support for +// non-CGO to new architectures, just by editing this file. +// +// To generate for other architectures: +// 1. Copy `freebsd.txt` to have a .c filename on a box running the target +// architecture, compile and run it. +// 2. Update the init() function below to include a case for this architecture +// 3. Update the build-tags in this file. + +//go:build !cgo && freebsd && (amd64 || arm64) + +package pse + +import ( + "encoding/binary" + "runtime" + "syscall" + + "golang.org/x/sys/unix" +) + +// On FreeBSD, to get proc information we read it out of the kernel using a +// binary sysctl. The endianness of integers is thus explicitly "host", rather +// than little or big endian. +var nativeEndian = binary.LittleEndian + +var pageshift int // derived from getpagesize(3) in init() below + +var ( + // These are populated in the init function, based on the current architecture. + // (It's less file-count explosion than having one small file for each + // FreeBSD architecture). + KIP_OFF_size int + KIP_OFF_rssize int + KIP_OFF_pctcpu int +) + +func init() { + switch runtime.GOARCH { + // These are the values which come from compiling and running + // freebsd.txt as a C program. + // Most recently validated: 2025-04 with FreeBSD 14.2R in AWS. + case "amd64": + KIP_OFF_size = 256 + KIP_OFF_rssize = 264 + KIP_OFF_pctcpu = 308 + case "arm64": + KIP_OFF_size = 256 + KIP_OFF_rssize = 264 + KIP_OFF_pctcpu = 308 + default: + panic("code bug: server/pse FreeBSD support missing case for '" + runtime.GOARCH + "' but build-tags allowed us to build anyway?") + } + + // To get the physical page size, the C library checks two places: + // process ELF auxiliary info, AT_PAGESZ + // as a fallback, the hw.pagesize sysctl + // In looking closely, I found that the Go runtime support is handling + // this for us, and exposing that as syscall.Getpagesize, having checked + // both in the same ways, at process start, so a call to that should return + // a memory value without even a syscall bounce. + pagesize := syscall.Getpagesize() + pageshift = 0 + for pagesize > 1 { + pageshift += 1 + pagesize >>= 1 + } +} + +func ProcUsage(pcpu *float64, rss, vss *int64) error { + rawdata, err := unix.SysctlRaw("kern.proc.pid", unix.Getpid()) + if err != nil { + return err + } + + r_vss_bytes := nativeEndian.Uint32(rawdata[KIP_OFF_size:]) + r_rss_pages := nativeEndian.Uint32(rawdata[KIP_OFF_rssize:]) + rss_bytes := r_rss_pages << pageshift + + // In C: fixpt_t ki_pctcpu + // Doc: %cpu for process during ki_swtime + // fixpt_t is __uint32_t + // usr.bin/top uses pctdouble to convert to a double (float64) + // define pctdouble(p) ((double)(p) / FIXED_PCTCPU) + // FIXED_PCTCPU is _usually_ FSCALE (some architectures are special) + // has: + // #define FSHIFT 11 /* bits to right of fixed binary point */ + // #define FSCALE (1< 0 { + atomic.StoreInt64(&ipcpu, (total*1000/ticks)/seconds) + } + + time.AfterFunc(1*time.Second, periodic) +} + +// ProcUsage returns CPU usage +func ProcUsage(pcpu *float64, rss, vss *int64) error { + contents, err := os.ReadFile(procStatFile) + if err != nil { + return err + } + fields := bytes.Fields(contents) + + // Memory + *rss = (parseInt64(fields[rssPos])) * pageSize + *vss = parseInt64(fields[vssPos]) + + // PCPU + // We track this with periodic sampling, so just load and go. + *pcpu = float64(atomic.LoadInt64(&ipcpu)) / 10.0 + + return nil +} + +// Ascii numbers 0-9 +const ( + asciiZero = 48 + asciiNine = 57 +) + +// parseInt64 expects decimal positive numbers. We +// return -1 to signal error +func parseInt64(d []byte) (n int64) { + if len(d) == 0 { + return -1 + } + for _, dec := range d { + if dec < asciiZero || dec > asciiNine { + return -1 + } + n = n*10 + (int64(dec) - asciiZero) + } + return n +} diff --git a/server/pse/pse_netbsd.go b/server/pse/pse_netbsd.go new file mode 100644 index 0000000..d8393f6 --- /dev/null +++ b/server/pse/pse_netbsd.go @@ -0,0 +1,36 @@ +// Copyright 2022 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copied from pse_openbsd.go + +package pse + +import ( + "fmt" + "os" + "os/exec" +) + +// ProcUsage returns CPU usage +func ProcUsage(pcpu *float64, rss, vss *int64) error { + pidStr := fmt.Sprintf("%d", os.Getpid()) + out, err := exec.Command("ps", "o", "pcpu=,rss=,vsz=", "-p", pidStr).Output() + if err != nil { + *rss, *vss = -1, -1 + return fmt.Errorf("ps call failed:%v", err) + } + fmt.Sscanf(string(out), "%f %d %d", pcpu, rss, vss) + *rss *= 1024 // 1k blocks, want bytes. + *vss *= 1024 // 1k blocks, want bytes. + return nil +} diff --git a/server/pse/pse_openbsd.go b/server/pse/pse_openbsd.go new file mode 100644 index 0000000..260f1a7 --- /dev/null +++ b/server/pse/pse_openbsd.go @@ -0,0 +1,36 @@ +// Copyright 2015-2018 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copied from pse_darwin.go + +package pse + +import ( + "fmt" + "os" + "os/exec" +) + +// ProcUsage returns CPU usage +func ProcUsage(pcpu *float64, rss, vss *int64) error { + pidStr := fmt.Sprintf("%d", os.Getpid()) + out, err := exec.Command("ps", "o", "pcpu=,rss=,vsz=", "-p", pidStr).Output() + if err != nil { + *rss, *vss = -1, -1 + return fmt.Errorf("ps call failed:%v", err) + } + fmt.Sscanf(string(out), "%f %d %d", pcpu, rss, vss) + *rss *= 1024 // 1k blocks, want bytes. + *vss *= 1024 // 1k blocks, want bytes. + return nil +} diff --git a/server/pse/pse_rumprun.go b/server/pse/pse_rumprun.go new file mode 100644 index 0000000..94ddf8f --- /dev/null +++ b/server/pse/pse_rumprun.go @@ -0,0 +1,25 @@ +// Copyright 2015-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build rumprun + +package pse + +// This is a placeholder for now. +func ProcUsage(pcpu *float64, rss, vss *int64) error { + *pcpu = 0.0 + *rss = 0 + *vss = 0 + + return nil +} diff --git a/server/pse/pse_solaris.go b/server/pse/pse_solaris.go new file mode 100644 index 0000000..f812ba4 --- /dev/null +++ b/server/pse/pse_solaris.go @@ -0,0 +1,45 @@ +// Copyright 2015-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copied from pse_openbsd.go + +//go:build illumos || solaris + +package pse + +import ( + "fmt" + "os" + "os/exec" + "strings" +) + +// ProcUsage returns CPU usage +func ProcUsage(pcpu *float64, rss, vss *int64) error { + pidStr := fmt.Sprintf("%d", os.Getpid()) + out, err := exec.Command("ps", "-o", "pcpu,rss,vsz", "-p", pidStr).Output() + if err != nil { + *rss, *vss = -1, -1 + return fmt.Errorf("ps call failed:%v", err) + } + lines := strings.Split(string(out), "\n") + if len(lines) < 2 { + *rss, *vss = -1, -1 + return fmt.Errorf("no ps output") + } + output := lines[1] + fmt.Sscanf(output, "%f %d %d", pcpu, rss, vss) + *rss *= 1024 // 1k blocks, want bytes. + *vss *= 1024 // 1k blocks, want bytes. + return nil +} diff --git a/server/pse/pse_test.go b/server/pse/pse_test.go new file mode 100644 index 0000000..5924a2b --- /dev/null +++ b/server/pse/pse_test.go @@ -0,0 +1,91 @@ +// Copyright 2015-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pse + +import ( + "fmt" + "os" + "os/exec" + "runtime" + "runtime/debug" + "testing" +) + +func TestPSEmulationCPU(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("Skipping this test on Windows") + } + var rss, vss int64 + var pcpu, psPcpu float64 + + debug.FreeOSMemory() + + // PS version first + pidStr := fmt.Sprintf("%d", os.Getpid()) + out, err := exec.Command("ps", "o", "pcpu=", "-p", pidStr).Output() + if err != nil { + t.Fatalf("Failed to execute ps command: %v\n", err) + } + + // Our internal version + ProcUsage(&pcpu, &rss, &vss) + + fmt.Sscanf(string(out), "%f", &psPcpu) + + if pcpu != psPcpu { + delta := int64(pcpu - psPcpu) + if delta < 0 { + delta = -delta + } + if delta > 30 { // 30%? + t.Fatalf("CPUs did not match close enough: %f vs %f", pcpu, psPcpu) + } + } +} + +func TestPSEmulationMem(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("Skipping this test on Windows") + } + var rss, vss, psRss, psVss int64 + var pcpu float64 + + debug.FreeOSMemory() + + // PS version first + pidStr := fmt.Sprintf("%d", os.Getpid()) + out, err := exec.Command("ps", "o", "rss=,vsz=", "-p", pidStr).Output() + if err != nil { + t.Fatalf("Failed to execute ps command: %v\n", err) + } + + fmt.Sscanf(string(out), "%d %d", &psRss, &psVss) + psRss *= 1024 // 1k blocks, want bytes. + psVss *= 1024 // 1k blocks, want bytes. + + debug.FreeOSMemory() + + // Our internal version + ProcUsage(&pcpu, &rss, &vss) + + if rss != psRss { + delta := rss - psRss + if delta < 0 { + delta = -delta + } + if delta > 1024*1024 { // 1MB + t.Fatalf("RSSs did not match close enough: %d vs %d", rss, psRss) + } + } +} diff --git a/server/pse/pse_wasm.go b/server/pse/pse_wasm.go new file mode 100644 index 0000000..6fcfda7 --- /dev/null +++ b/server/pse/pse_wasm.go @@ -0,0 +1,25 @@ +// Copyright 2022-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build wasm + +package pse + +// This is a placeholder for now. +func ProcUsage(pcpu *float64, rss, vss *int64) error { + *pcpu = 0.0 + *rss = 0 + *vss = 0 + + return nil +} diff --git a/server/pse/pse_windows.go b/server/pse/pse_windows.go new file mode 100644 index 0000000..ab0adde --- /dev/null +++ b/server/pse/pse_windows.go @@ -0,0 +1,296 @@ +// Copyright 2015-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package pse + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + "unsafe" + + "golang.org/x/sys/windows" +) + +var ( + pdh = windows.NewLazySystemDLL("pdh.dll") + winPdhOpenQuery = pdh.NewProc("PdhOpenQuery") + winPdhAddCounter = pdh.NewProc("PdhAddCounterW") + winPdhCollectQueryData = pdh.NewProc("PdhCollectQueryData") + winPdhGetFormattedCounterValue = pdh.NewProc("PdhGetFormattedCounterValue") + winPdhGetFormattedCounterArray = pdh.NewProc("PdhGetFormattedCounterArrayW") +) + +func init() { + if err := pdh.Load(); err != nil { + panic(err) + } + for _, p := range []*windows.LazyProc{ + winPdhOpenQuery, winPdhAddCounter, winPdhCollectQueryData, + winPdhGetFormattedCounterValue, winPdhGetFormattedCounterArray, + } { + if err := p.Find(); err != nil { + panic(err) + } + } +} + +// global performance counter query handle and counters +var ( + pcHandle PDH_HQUERY + pidCounter, cpuCounter, rssCounter, vssCounter PDH_HCOUNTER + prevCPU float64 + prevRss int64 + prevVss int64 + lastSampleTime time.Time + processPid int + pcQueryLock sync.Mutex + initialSample = true +) + +// maxQuerySize is the number of values to return from a query. +// It represents the maximum # of servers that can be queried +// simultaneously running on a machine. +const maxQuerySize = 512 + +// Keep static memory around to reuse; this works best for passing +// into the pdh API. +var counterResults [maxQuerySize]PDH_FMT_COUNTERVALUE_ITEM_DOUBLE + +// PDH Types +type ( + PDH_HQUERY syscall.Handle + PDH_HCOUNTER syscall.Handle +) + +// PDH constants used here +const ( + PDH_FMT_DOUBLE = 0x00000200 + PDH_INVALID_DATA = 0xC0000BC6 + PDH_MORE_DATA = 0x800007D2 +) + +// PDH_FMT_COUNTERVALUE_DOUBLE - double value +type PDH_FMT_COUNTERVALUE_DOUBLE struct { + CStatus uint32 + DoubleValue float64 +} + +// PDH_FMT_COUNTERVALUE_ITEM_DOUBLE is an array +// element of a double value +type PDH_FMT_COUNTERVALUE_ITEM_DOUBLE struct { + SzName *uint16 // pointer to a string + FmtValue PDH_FMT_COUNTERVALUE_DOUBLE +} + +func pdhAddCounter(hQuery PDH_HQUERY, szFullCounterPath string, dwUserData uintptr, phCounter *PDH_HCOUNTER) error { + ptxt, _ := syscall.UTF16PtrFromString(szFullCounterPath) + r0, _, _ := winPdhAddCounter.Call( + uintptr(hQuery), + uintptr(unsafe.Pointer(ptxt)), + dwUserData, + uintptr(unsafe.Pointer(phCounter))) + + if r0 != 0 { + return fmt.Errorf("pdhAddCounter failed. %d", r0) + } + return nil +} + +func pdhOpenQuery(datasrc *uint16, userdata uint32, query *PDH_HQUERY) error { + r0, _, _ := syscall.Syscall(winPdhOpenQuery.Addr(), 3, 0, uintptr(userdata), uintptr(unsafe.Pointer(query))) + if r0 != 0 { + return fmt.Errorf("pdhOpenQuery failed - %d", r0) + } + return nil +} + +func pdhCollectQueryData(hQuery PDH_HQUERY) error { + r0, _, _ := winPdhCollectQueryData.Call(uintptr(hQuery)) + if r0 != 0 { + return fmt.Errorf("pdhCollectQueryData failed - %d", r0) + } + return nil +} + +// pdhGetFormattedCounterArrayDouble returns the value of return code +// rather than error, to easily check return codes +func pdhGetFormattedCounterArrayDouble(hCounter PDH_HCOUNTER, lpdwBufferSize *uint32, lpdwBufferCount *uint32, itemBuffer *PDH_FMT_COUNTERVALUE_ITEM_DOUBLE) uint32 { + ret, _, _ := winPdhGetFormattedCounterArray.Call( + uintptr(hCounter), + uintptr(PDH_FMT_DOUBLE), + uintptr(unsafe.Pointer(lpdwBufferSize)), + uintptr(unsafe.Pointer(lpdwBufferCount)), + uintptr(unsafe.Pointer(itemBuffer))) + + return uint32(ret) +} + +func getCounterArrayData(counter PDH_HCOUNTER) ([]float64, error) { + var bufSize uint32 + var bufCount uint32 + + // Retrieving array data requires two calls, the first which + // requires an addressable empty buffer, and sets size fields. + // The second call returns the data. + initialBuf := make([]PDH_FMT_COUNTERVALUE_ITEM_DOUBLE, 1) + ret := pdhGetFormattedCounterArrayDouble(counter, &bufSize, &bufCount, &initialBuf[0]) + if ret == PDH_MORE_DATA { + // we'll likely never get here, but be safe. + if bufCount > maxQuerySize { + bufCount = maxQuerySize + } + ret = pdhGetFormattedCounterArrayDouble(counter, &bufSize, &bufCount, &counterResults[0]) + if ret == 0 { + rv := make([]float64, bufCount) + for i := 0; i < int(bufCount); i++ { + rv[i] = counterResults[i].FmtValue.DoubleValue + } + return rv, nil + } + } + if ret != 0 { + return nil, fmt.Errorf("getCounterArrayData failed - %d", ret) + } + + return nil, nil +} + +// getProcessImageName returns the name of the process image, as expected by +// the performance counter API. +func getProcessImageName() (name string) { + name = filepath.Base(os.Args[0]) + name = strings.TrimSuffix(name, ".exe") + return +} + +// initialize our counters +func initCounters() (err error) { + + processPid = os.Getpid() + // require an addressible nil pointer + var source uint16 + if err := pdhOpenQuery(&source, 0, &pcHandle); err != nil { + return err + } + + // setup the performance counters, search for all server instances + name := fmt.Sprintf("%s*", getProcessImageName()) + pidQuery := fmt.Sprintf("\\Process(%s)\\ID Process", name) + cpuQuery := fmt.Sprintf("\\Process(%s)\\%% Processor Time", name) + rssQuery := fmt.Sprintf("\\Process(%s)\\Working Set - Private", name) + vssQuery := fmt.Sprintf("\\Process(%s)\\Virtual Bytes", name) + + if err = pdhAddCounter(pcHandle, pidQuery, 0, &pidCounter); err != nil { + return err + } + if err = pdhAddCounter(pcHandle, cpuQuery, 0, &cpuCounter); err != nil { + return err + } + if err = pdhAddCounter(pcHandle, rssQuery, 0, &rssCounter); err != nil { + return err + } + if err = pdhAddCounter(pcHandle, vssQuery, 0, &vssCounter); err != nil { + return err + } + + // prime the counters by collecting once, and sleep to get somewhat + // useful information the first request. Counters for the CPU require + // at least two collect calls. + if err = pdhCollectQueryData(pcHandle); err != nil { + return err + } + time.Sleep(50) + + return nil +} + +// ProcUsage returns process CPU and memory statistics +func ProcUsage(pcpu *float64, rss, vss *int64) error { + var err error + + // For simplicity, protect the entire call. + // Most simultaneous requests will immediately return + // with cached values. + pcQueryLock.Lock() + defer pcQueryLock.Unlock() + + // First time through, initialize counters. + if initialSample { + if err = initCounters(); err != nil { + return err + } + initialSample = false + } else if time.Since(lastSampleTime) < (2 * time.Second) { + // only refresh every two seconds as to minimize impact + // on the server. + *pcpu = prevCPU + *rss = prevRss + *vss = prevVss + return nil + } + + // always save the sample time, even on errors. + defer func() { + lastSampleTime = time.Now() + }() + + // refresh the performance counter data + if err = pdhCollectQueryData(pcHandle); err != nil { + return err + } + + // retrieve the data + var pidAry, cpuAry, rssAry, vssAry []float64 + if pidAry, err = getCounterArrayData(pidCounter); err != nil { + return err + } + if cpuAry, err = getCounterArrayData(cpuCounter); err != nil { + return err + } + if rssAry, err = getCounterArrayData(rssCounter); err != nil { + return err + } + if vssAry, err = getCounterArrayData(vssCounter); err != nil { + return err + } + // find the index of the entry for this process + idx := int(-1) + for i := range pidAry { + if int(pidAry[i]) == processPid { + idx = i + break + } + } + // no pid found... + if idx < 0 { + return fmt.Errorf("could not find pid in performance counter results") + } + // assign values from the performance counters + *pcpu = cpuAry[idx] + *rss = int64(rssAry[idx]) + *vss = int64(vssAry[idx]) + + // save off cache values + prevCPU = *pcpu + prevRss = *rss + prevVss = *vss + + return nil +} diff --git a/server/pse/pse_windows_test.go b/server/pse/pse_windows_test.go new file mode 100644 index 0000000..ba39535 --- /dev/null +++ b/server/pse/pse_windows_test.go @@ -0,0 +1,97 @@ +// Copyright 2015-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package pse + +import ( + "fmt" + "os/exec" + "runtime" + "strconv" + "strings" + "testing" +) + +func checkValues(t *testing.T, pcpu, tPcpu float64, rss, tRss int64) { + if pcpu != tPcpu { + delta := int64(pcpu - tPcpu) + if delta < 0 { + delta = -delta + } + if delta > 30 { // 30%? + t.Fatalf("CPUs did not match close enough: %f vs %f", pcpu, tPcpu) + } + } + if rss != tRss { + delta := rss - tRss + if delta < 0 { + delta = -delta + } + if delta > 1024*1024 { // 1MB + t.Fatalf("RSSs did not match close enough: %d vs %d", rss, tRss) + } + } +} + +func TestPSEmulationWin(t *testing.T) { + var pcpu, tPcpu float64 + var rss, vss, tRss int64 + + runtime.GC() + + if err := ProcUsage(&pcpu, &rss, &vss); err != nil { + t.Fatalf("Error: %v", err) + } + + runtime.GC() + + imageName := getProcessImageName() + // query the counters using typeperf + out, err := exec.Command("typeperf.exe", + fmt.Sprintf("\\Process(%s)\\%% Processor Time", imageName), + fmt.Sprintf("\\Process(%s)\\Working Set - Private", imageName), + fmt.Sprintf("\\Process(%s)\\Virtual Bytes", imageName), + "-sc", "1").Output() + if err != nil { + t.Fatal("unable to run command", err) + } + + // parse out results - refer to comments in procUsage for detail + results := strings.Split(string(out), "\r\n") + values := strings.Split(results[2], ",") + + // parse pcpu + tPcpu, err = strconv.ParseFloat(strings.Trim(values[1], "\""), 64) + if err != nil { + t.Fatalf("Unable to parse percent cpu: %s", values[1]) + } + + // parse private bytes (rss) + fval, err := strconv.ParseFloat(strings.Trim(values[2], "\""), 64) + if err != nil { + t.Fatalf("Unable to parse private bytes: %s", values[2]) + } + tRss = int64(fval) + + checkValues(t, pcpu, tPcpu, rss, tRss) + + runtime.GC() + + // Again to test caching + if err = ProcUsage(&pcpu, &rss, &vss); err != nil { + t.Fatalf("Error: %v", err) + } + checkValues(t, pcpu, tPcpu, rss, tRss) +} diff --git a/server/pse/pse_zos.go b/server/pse/pse_zos.go new file mode 100644 index 0000000..8596c49 --- /dev/null +++ b/server/pse/pse_zos.go @@ -0,0 +1,25 @@ +// Copyright 2023-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build zos + +package pse + +// This is a placeholder for now. +func ProcUsage(pcpu *float64, rss, vss *int64) error { + *pcpu = 0.0 + *rss = 0 + *vss = 0 + + return nil +} diff --git a/server/raft_benchmark_test.go b/server/raft_benchmark_test.go new file mode 100644 index 0000000..8154fc4 --- /dev/null +++ b/server/raft_benchmark_test.go @@ -0,0 +1,124 @@ +// Copyright 2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "encoding/binary" + "fmt" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" +) + +func benchmarkWriteTermVote(b *testing.B, workers int) { + root := b.TempDir() + dirs := make([]string, workers) + for i := range dirs { + dir := filepath.Join(root, fmt.Sprintf("raft-%d", i)) + if err := os.MkdirAll(dir, defaultDirPerms); err != nil { + b.Fatal(err) + } + dirs[i] = dir + } + + wtvs := make([][]byte, workers) + for i := range wtvs { + wtv := make([]byte, termVoteLen) + copy(wtv[termLen:], fmt.Sprintf("%08d", i)) + wtvs[i] = wtv + } + + dios := defaultDiskIOSemaphore() + var next atomic.Uint64 + var wg sync.WaitGroup + type latencyStats struct { + count uint64 + total time.Duration + min time.Duration + max time.Duration + } + stats := make([]latencyStats, workers) + wg.Add(workers) + + b.ResetTimer() + for i := range workers { + go func(worker int) { + defer wg.Done() + dir := dirs[worker] + wtv := wtvs[worker] + stat := &stats[worker] + for { + n := next.Add(1) + if n > uint64(b.N) { + return + } + binary.LittleEndian.PutUint64(wtv[:termLen], n) + start := time.Now() + if err := writeTermVote(dios, dir, wtv); err != nil { + b.Errorf("writeTermVote failed: %v", err) + return + } + elapsed := time.Since(start) + stat.count++ + stat.total += elapsed + if stat.min == 0 || elapsed < stat.min { + stat.min = elapsed + } + if elapsed > stat.max { + stat.max = elapsed + } + } + }(i) + } + wg.Wait() + b.StopTimer() + + var count uint64 + var total, minLatency, maxLatency time.Duration + for i := range stats { + stat := stats[i] + if stat.count == 0 { + continue + } + count += stat.count + total += stat.total + if minLatency == 0 || stat.min < minLatency { + minLatency = stat.min + } + if stat.max > maxLatency { + maxLatency = stat.max + } + } + if count > 0 { + avgLatency := time.Duration(total.Nanoseconds() / int64(count)) + b.ReportMetric(float64(minLatency.Microseconds()), "min-latency-us") + b.ReportMetric(float64(avgLatency.Microseconds()), "avg-latency-us") + b.ReportMetric(float64(maxLatency.Microseconds()), "max-latency-us") + b.ReportMetric(0, "ns/op") + } + if elapsed := b.Elapsed(); elapsed > 0 { + b.ReportMetric(float64(b.N)/elapsed.Seconds(), "writes/s") + } +} + +func BenchmarkWriteTermVote(b *testing.B) { + for _, workers := range []int{1, 16, 128, 512} { + b.Run(fmt.Sprintf("workers=%d", workers), func(b *testing.B) { + benchmarkWriteTermVote(b, workers) + }) + } +} diff --git a/server/raft_chain_of_blocks_helpers_test.go b/server/raft_chain_of_blocks_helpers_test.go new file mode 100644 index 0000000..5f2987a --- /dev/null +++ b/server/raft_chain_of_blocks_helpers_test.go @@ -0,0 +1,378 @@ +// Copyright 2024-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Raft Chain of Blocks (RCOB) is a replicated state machine on top of Raft used to test the latter. +// Each value ("block") proposed to raft is a simple array of bytes (stateless, can be small or large). +// Each replica state consists of a hash of all blocks delivered so far. +// This makes it easy to check consistency over time. +// If all replicas deliver the same blocks in the same order (the whole point of Raft!), then the N-th hash on each +// replica should be identical. This invariant is easy to verify in test settings, because if at any point a replica +// delivered (hashed) a different value, then all subsequent hashes will diverge from the rest. + +package server + +import ( + "encoding" + "encoding/binary" + "encoding/json" + "fmt" + "hash" + "hash/crc32" + "math/rand/v2" + "sync" +) + +// Static options (but may be useful to tweak when debugging) +var RCOBOptions = struct { + verbose bool + // Proposed block [data] size is random between 1 and maxBlockSize + maxBlockSize int + // If set to true, the state machine will not take snapshots while in recovery + // (corresponding to the 'ready' state machine member variable) + safeSnapshots bool +}{ + false, + 10240, + true, +} + +// Simple state machine on top of Raft. +// Each value delivered is hashed, N-th value hash should match on all replicas or something went wrong. +// If even a single value in the chain of hash differs on one replica, hashes will diverge. +type RCOBStateMachine struct { + sync.Mutex + s *Server + n RaftNode + cfg *RaftConfig + wg sync.WaitGroup + leader bool + proposalSequence uint64 + rng *rand.ChaCha8 + hash hash.Hash + blocksApplied uint64 + blocksAppliedSinceSnapshot uint64 + stopped bool + ready bool +} + +func (sm *RCOBStateMachine) waitGroup() *sync.WaitGroup { + sm.Lock() + defer sm.Unlock() + return &sm.wg +} + +// RCOBBlock is the structure of values replicated through Raft. +// Block data is a random array of bytes. +// Additional proposer metadata is present for debugging and for further ordering invariant checks (but not included +// in the hash). +type RCOBBlock struct { + Proposer string + ProposerSequence uint64 + Data []byte +} + +// logDebug is for fine-grained event logging, useful for debugging but default off +func (sm *RCOBStateMachine) logDebug(format string, args ...any) { + if RCOBOptions.verbose { + fmt.Printf("["+sm.s.Name()+" ("+sm.n.ID()+")] "+format+"\n", args...) + } +} + +func (sm *RCOBStateMachine) server() *Server { + sm.Lock() + defer sm.Unlock() + return sm.s +} + +func (sm *RCOBStateMachine) node() RaftNode { + sm.Lock() + defer sm.Unlock() + return sm.n +} + +func (sm *RCOBStateMachine) propose(data []byte) { + sm.Lock() + defer sm.Unlock() + if !sm.ready { + sm.logDebug("Refusing to propose during recovery") + } + err := sm.n.ForwardProposal(data) + if err != nil { + sm.logDebug("block proposal error: %s", err) + } +} + +func (sm *RCOBStateMachine) applyEntry(ce *CommittedEntry) { + sm.Lock() + defer sm.Unlock() + if ce == nil { + // A nil entry signals that the previous recovery backlog is over + sm.logDebug("Recovery complete") + sm.ready = true + return + } + sm.logDebug("Apply entries #%d (%d entries)", ce.Index, len(ce.Entries)) + for _, entry := range ce.Entries { + if entry.Type == EntryNormal { + sm.applyBlock(entry.Data) + } else if entry.Type == EntrySnapshot { + sm.loadSnapshot(entry.Data) + } else if entry.Type == EntryCatchup { + // Ignore. + } else { + panic(fmt.Sprintf("[%s] unknown entry type: %s", sm.s.Name(), entry.Type)) + } + } + // Signal to the node that entries were applied + sm.n.Applied(ce.Index) +} + +func (sm *RCOBStateMachine) leaderChange(isLeader bool) { + sm.Lock() + defer sm.Unlock() + if sm.leader && !isLeader { + sm.logDebug("Leader change: no longer leader") + } else if sm.leader && isLeader { + sm.logDebug("Elected leader while already leader") + } else if !sm.leader && isLeader { + sm.logDebug("Leader change: i am leader") + } else { + sm.logDebug("Leader change") + } + sm.leader = isLeader + if isLeader != sm.n.Leader() { + sm.logDebug("⚠️ Leader state out of sync with underlying node") + } +} + +func (sm *RCOBStateMachine) stop() { + n, wg := sm.node(), sm.waitGroup() + n.Stop() + n.WaitForStop() + wg.Wait() + + // Clear state, on restart it will be recovered from snapshot or peers + sm.Lock() + defer sm.Unlock() + + sm.stopped = true + sm.blocksApplied = 0 + sm.hash.Reset() + sm.leader = false + sm.logDebug("Stopped") +} + +func (sm *RCOBStateMachine) restart() { + sm.Lock() + defer sm.Unlock() + + sm.logDebug("Restarting") + + sm.stopped = false + sm.ready = false + if sm.n.State() != Closed { + return + } + + // The filestore is stopped as well, so need to extract the parts to recreate it. + rn := sm.n.(*raft) + fs := rn.wal.(*fileStore) + + var err error + sm.cfg.Log, err = newFileStore(fs.fcfg, fs.cfg.StreamConfig) + if err != nil { + panic(err) + } + sm.n, err = sm.s.startRaftNode(globalAccountName, sm.cfg, pprofLabels{}) + if err != nil { + panic(err) + } + // Finally restart the driver. + go smLoop(sm) +} + +func (sm *RCOBStateMachine) proposeBlock() { + // Keep track how many blocks this replica proposed + sm.proposalSequence += 1 + + // Create a block + blockSize := 1 + rand.New(sm.rng).IntN(RCOBOptions.maxBlockSize) + block := RCOBBlock{ + Proposer: sm.s.Name(), + ProposerSequence: sm.proposalSequence, + Data: make([]byte, blockSize), + } + + // Data is random bytes + sm.rng.Read(block.Data) + + // Serialize block to JSON + blockData, err := json.Marshal(block) + if err != nil { + panic(fmt.Sprintf("serialization error: %s", err)) + } + + sm.logDebug( + "Proposing block <%s, %d, [%dB]>", + block.Proposer, + block.ProposerSequence, + len(block.Data), + ) + + // Propose block (may fail, and that's ok) + sm.propose(blockData) +} + +func (sm *RCOBStateMachine) applyBlock(data []byte) { + // Deserialize block + var block RCOBBlock + err := json.Unmarshal(data, &block) + if err != nil { + panic(fmt.Sprintf("deserialization error: %s", err)) + } + + sm.logDebug("Applying block <%s, %d>", block.Proposer, block.ProposerSequence) + + // Hash the data on top of the existing running hash + n, err := sm.hash.Write(block.Data) + if n != len(block.Data) { + panic(fmt.Sprintf("unexpected checksum written %d data block size: %d", n, len(block.Data))) + } else if err != nil { + panic(fmt.Sprintf("checksum error: %s", err)) + } + + // Track numbers of blocks applied + sm.blocksApplied += 1 + sm.blocksAppliedSinceSnapshot += 1 + + sm.logDebug("Hash after %d blocks: %X ", sm.blocksApplied, sm.hash.Sum(nil)) +} + +func (sm *RCOBStateMachine) getCurrentHash() (bool, uint64, string) { + sm.Lock() + defer sm.Unlock() + + // Return running, the number of blocks applied and the current running hash + return !sm.stopped, sm.blocksApplied, fmt.Sprintf("%X", sm.hash.Sum(nil)) +} + +// RCOBSnapshot structure for RCOB snapshots +// Consists of the hash (32b) after BlocksCount blocks were hashed. +// TODO: Could start storing last N blocks to make snapshot larger and exercise more code paths. +// (a big chunk of random data would also do the trick) +type RCOBSnapshot struct { + SourceNode string + HashData []byte + BlocksCount uint64 +} + +// createSnapshot asks the state machine to create a snapshot. +// The request may be ignored if no blocks were applied since the last snapshot. +func (sm *RCOBStateMachine) createSnapshot() { + sm.Lock() + defer sm.Unlock() + + if sm.blocksAppliedSinceSnapshot == 0 { + sm.logDebug("Skip snapshot, no new entries") + return + } + + if RCOBOptions.safeSnapshots && !sm.ready { + sm.logDebug("Skip snapshot, still recovering") + return + } + + sm.logDebug( + "Snapshot (with %d blocks applied, %d since last snapshot)", + sm.blocksApplied, + sm.blocksAppliedSinceSnapshot, + ) + + // Serialize the internal state of the hash block + serializedHash, err := sm.hash.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + panic(fmt.Sprintf("failed to marshal hash: %s", err)) + } + + // Create snapshot + snapshot := RCOBSnapshot{ + SourceNode: fmt.Sprintf("%s (%s)", sm.s.Name(), sm.n.ID()), + HashData: serializedHash, + BlocksCount: sm.blocksApplied, + } + + // Serialize snapshot as JSON + snapshotData, err := json.Marshal(snapshot) + if err != nil { + panic(fmt.Sprintf("failed to marshal snapshot: %s", err)) + } + + // InstallSnapshot is actually "save the snapshot", which is an operation delegated to the node + err = sm.n.InstallSnapshot(snapshotData, false) + if err != nil { + sm.logDebug("failed to snapshot: %s", err) + return + } + + // Reset counter since last snapshot + sm.blocksAppliedSinceSnapshot = 0 +} + +func (sm *RCOBStateMachine) loadSnapshot(data []byte) { + // Deserialize snapshot from JSON + var snapshot RCOBSnapshot + err := json.Unmarshal(data, &snapshot) + if err != nil { + panic(fmt.Sprintf("failed to unmarshal snapshot: %s", err)) + } + + sm.logDebug( + "Applying snapshot (created by %s) taken after %d blocks", + snapshot.SourceNode, + snapshot.BlocksCount, + ) + + // Load internal hash state + err = sm.hash.(encoding.BinaryUnmarshaler).UnmarshalBinary(snapshot.HashData) + if err != nil { + panic(fmt.Sprintf("failed to unmarshal hash data: %s", err)) + } + + // Load block counter + sm.blocksApplied = snapshot.BlocksCount + + // Reset number of blocks applied since snapshot + sm.blocksAppliedSinceSnapshot = 0 + + sm.logDebug("Hash after snapshot with %d blocks: %X ", sm.blocksApplied, sm.hash.Sum(nil)) +} + +// Factory function to create state machine on top of the given server/node +func newRaftChainStateMachine(s *Server, cfg *RaftConfig, n RaftNode) stateMachine { + // Create RNG seed based on server name and node id + var seed int64 + for _, c := range []byte(s.Name()) { + seed += int64(c) + } + for _, c := range []byte(n.ID()) { + seed += int64(c) + } + var seedBytes [32]byte + binary.LittleEndian.PutUint64(seedBytes[:8], uint64(seed)) + rng := rand.NewChaCha8(seedBytes) + + // Initialize empty hash block + hashBlock := crc32.NewIEEE() + + return &RCOBStateMachine{s: s, n: n, cfg: cfg, rng: rng, hash: hashBlock} +} diff --git a/server/raft_helpers_test.go b/server/raft_helpers_test.go new file mode 100644 index 0000000..574f6c7 --- /dev/null +++ b/server/raft_helpers_test.go @@ -0,0 +1,473 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Do not exlude this file with the !skip_js_tests since those helpers +// are also used by MQTT. + +package server + +import ( + "encoding/binary" + "errors" + "fmt" + "math/rand/v2" + "sync" + "testing" + "time" +) + +type stateMachine interface { + server() *Server + node() RaftNode + waitGroup() *sync.WaitGroup + // This will call forward as needed so can be called on any node. + propose(data []byte) + // When entries have been committed and can be applied. + applyEntry(ce *CommittedEntry) + // When a leader change happens. + leaderChange(isLeader bool) + // Stop the raft group. + stop() + // Restart + restart() +} + +// Factory function needed for constructor. +type smFactory func(s *Server, cfg *RaftConfig, node RaftNode) stateMachine + +type smGroup []stateMachine + +// Leader of the group. +func (sg smGroup) leader() stateMachine { + for _, sm := range sg { + if sm.node().Leader() { + return sm + } + } + return nil +} + +func (sg smGroup) followers() smGroup { + var f []stateMachine + for _, sm := range sg { + if sm.node().Leader() { + continue + } + f = append(f, sm) + } + return f +} + +// Wait on a leader to be elected. +func (sg smGroup) waitOnLeader() stateMachine { + expires := time.Now().Add(10 * time.Second) + for time.Now().Before(expires) { + for _, sm := range sg { + if sm.node().Leader() { + return sm + } + } + time.Sleep(100 * time.Millisecond) + } + return nil +} + +// Pick a random member. +func (sg smGroup) randomMember() stateMachine { + return sg[rand.IntN(len(sg))] +} + +// Return a non-leader +func (sg smGroup) nonLeader() stateMachine { + for _, sm := range sg { + if !sm.node().Leader() { + return sm + } + } + return nil +} + +// Take out the lock on all nodes. +func (sg smGroup) lockAll() { + for _, sm := range sg { + sm.node().(*raft).Lock() + } +} + +// Release the lock on all nodes. +func (sg smGroup) unlockAll() { + for _, sm := range sg { + sm.node().(*raft).Unlock() + } +} + +// Acquire the lock on all follower nodes. +func (sg smGroup) lockFollowers() []stateMachine { + var locked []stateMachine + for _, sm := range sg { + if !sm.node().Leader() { + locked = append(locked, sm) + sm.node().(*raft).Lock() + } + } + return locked[:] +} + +// Create a raft group and place on numMembers servers at random. +// Filestore based. +func (c *cluster) createRaftGroup(name string, numMembers int, smf smFactory) smGroup { + return c.createRaftGroupEx(name, numMembers, smf, defaultRaftTransport, FileStorage) +} + +func (c *cluster) createMemRaftGroup(name string, numMembers int, smf smFactory) smGroup { + return c.createRaftGroupEx(name, numMembers, smf, defaultRaftTransport, MemoryStorage) +} + +func (c *cluster) createMockMemRaftGroup(name string, members int, smf smFactory) (*raftTransportHub, smGroup) { + hub := newRaftTransportHub() + return hub, c.createRaftGroupEx(name, members, smf, hub.newTransport, MemoryStorage) +} + +func (c *cluster) createRaftGroupEx(name string, numMembers int, smf smFactory, rtf newTransportFunc, st StorageType) smGroup { + c.t.Helper() + if numMembers > len(c.servers) { + c.t.Fatalf("Members > Peers: %d vs %d", numMembers, len(c.servers)) + } + servers := append([]*Server{}, c.servers...) + rand.Shuffle(len(servers), func(i, j int) { servers[i], servers[j] = servers[j], servers[i] }) + return c.createRaftGroupWithPeers(name, servers[:numMembers], smf, rtf, st) +} + +func (c *cluster) createWAL(name string, st StorageType) WAL { + c.t.Helper() + var err error + var store WAL + if st == FileStorage { + store, err = newFileStore( + FileStoreConfig{ + StoreDir: c.t.TempDir(), + BlockSize: defaultMediumBlockSize, + AsyncFlush: false, + SyncInterval: 5 * time.Minute}, + StreamConfig{ + Name: name, + Storage: FileStorage}) + } else { + store, err = newMemStore( + &StreamConfig{ + Name: name, + Storage: MemoryStorage}) + } + require_NoError(c.t, err) + return store +} + +func serverPeerNames(servers []*Server) []string { + var peers []string + + for _, s := range servers { + // generate peer names. + s.mu.RLock() + peers = append(peers, s.sys.shash) + s.mu.RUnlock() + } + + return peers +} + +func (c *cluster) createStateMachine(s *Server, cfg *RaftConfig, peers []string, smf smFactory) stateMachine { + s.bootstrapRaftNode(cfg, peers, true) + n, err := s.startRaftNode(globalAccountName, cfg, pprofLabels{}) + require_NoError(c.t, err) + sm := smf(s, cfg, n) + go smLoop(sm) + return sm +} + +func (c *cluster) createRaftGroupWithPeers(name string, servers []*Server, smf smFactory, rtf newTransportFunc, st StorageType) smGroup { + c.t.Helper() + + var sg smGroup + peers := serverPeerNames(servers) + + for _, s := range servers { + cfg := &RaftConfig{ + Name: name, + Store: c.t.TempDir(), + Log: c.createWAL(name, st), + NewTransport: rtf} + sg = append(sg, c.createStateMachine(s, cfg, peers, smf)) + } + + // Start campaigning early to speed up bootstrap leader election. + sg[0].node().CampaignImmediately() + return sg +} + +func (c *cluster) addNodeEx(name string, smf smFactory, rtf newTransportFunc, st StorageType) stateMachine { + c.t.Helper() + + server := c.addInNewServer() + + cfg := &RaftConfig{ + Name: name, + Store: c.t.TempDir(), + Log: c.createWAL(name, st), + NewTransport: rtf} + + peers := serverPeerNames(c.servers) + return c.createStateMachine(server, cfg, peers, smf) +} + +func (c *cluster) addRaftNode(name string, smf smFactory) stateMachine { + return c.addNodeEx(name, smf, defaultRaftTransport, FileStorage) +} + +func (c *cluster) addMemRaftNode(name string, smf smFactory) stateMachine { + return c.addNodeEx(name, smf, defaultRaftTransport, MemoryStorage) +} + +func (c *cluster) addMockMemRaftNode(name string, hub *raftTransportHub, smf smFactory) stateMachine { + return c.addNodeEx(name, smf, hub.newTransport, MemoryStorage) +} + +// Driver program for the state machine. +// Should be run in its own go routine. +func smLoop(sm stateMachine) { + s, n, wg := sm.server(), sm.node(), sm.waitGroup() + qch, lch, aq := n.QuitC(), n.LeadChangeC(), n.ApplyQ() + + // Wait group used to allow waiting until we exit from here. + wg.Add(1) + defer wg.Done() + + for { + select { + case <-s.quitCh: + return + case <-qch: + return + case <-aq.ch: + ces := aq.pop() + for _, ce := range ces { + sm.applyEntry(ce) + } + aq.recycle(&ces) + + case lc := <-lch: + sm.leaderChange(lc.isLeader) + } + } +} + +// Simple implementation of a replicated state. +// The adder state just sums up int64 values. +type stateAdder struct { + sync.Mutex + s *Server + n RaftNode + wg sync.WaitGroup + cfg *RaftConfig + sum int64 + lch chan bool +} + +// Simple getters for server and the raft node. +func (a *stateAdder) server() *Server { + a.Lock() + defer a.Unlock() + return a.s +} + +func (a *stateAdder) node() RaftNode { + a.Lock() + defer a.Unlock() + return a.n +} + +func (a *stateAdder) waitGroup() *sync.WaitGroup { + a.Lock() + defer a.Unlock() + return &a.wg +} + +func (a *stateAdder) propose(data []byte) { + // Don't hold state machine lock as we could deadlock if the node was locked as part of the test. + n := a.node() + n.ForwardProposal(data) +} + +func (a *stateAdder) applyEntry(ce *CommittedEntry) { + a.Lock() + if ce == nil { + // This means initial state is done/replayed. + a.Unlock() + return + } + for _, e := range ce.Entries { + if e.Type == EntryNormal { + delta, _ := binary.Varint(e.Data) + a.sum += delta + } else if e.Type == EntrySnapshot { + a.sum, _ = binary.Varint(e.Data) + } + } + // Update applied. + // But don't hold state machine lock as we could deadlock if the node was locked as part of the test. + n := a.n + a.Unlock() + n.Applied(ce.Index) +} + +func (a *stateAdder) leaderChange(isLeader bool) { + select { + case a.lch <- isLeader: + default: + } +} + +// Adder specific to change the total. +func (a *stateAdder) proposeDelta(delta int64) { + data := make([]byte, binary.MaxVarintLen64) + n := binary.PutVarint(data, int64(delta)) + a.propose(data[:n]) +} + +// Stop the group. +func (a *stateAdder) stop() { + n, wg := a.node(), a.waitGroup() + n.Stop() + n.WaitForStop() + wg.Wait() +} + +// Restart the group +func (a *stateAdder) restart() { + a.Lock() + defer a.Unlock() + + if a.n.State() != Closed { + return + } + + // The filestore is stopped as well, so need to extract the parts to recreate it. + rn := a.n.(*raft) + var err error + + switch rn.wal.(type) { + case *fileStore: + fs := rn.wal.(*fileStore) + a.cfg.Log, err = newFileStore(fs.fcfg, fs.cfg.StreamConfig) + case *memStore: + ms := rn.wal.(*memStore) + a.cfg.Log, err = newMemStore(&ms.cfg) + } + if err != nil { + panic(err) + } + + // Must reset in-memory state. + // A real restart would not preserve it, but more importantly we have no way to detect if we + // already applied an entry. So, the sum must only be updated based on append entries or snapshots. + a.sum = 0 + + a.n, err = a.s.startRaftNode(globalAccountName, a.cfg, pprofLabels{}) + if err != nil { + panic(err) + } + // Finally restart the driver. + go smLoop(a) +} + +// Total for the adder state machine. +func (a *stateAdder) total() int64 { + a.Lock() + defer a.Unlock() + return a.sum +} + +// Install a snapshot. +func (a *stateAdder) snapshot(t *testing.T) { + // Don't hold state machine lock as we could deadlock if the node was locked as part of the test. + a.Lock() + sum := a.sum + rn := a.n + a.Unlock() + + data := make([]byte, binary.MaxVarintLen64) + n := binary.PutVarint(data, sum) + snap := data[:n] + require_NoError(t, rn.InstallSnapshot(snap, false)) +} + +// Helper to wait for a certain state. +func (rg smGroup) waitOnTotal(t *testing.T, expected int64) { + t.Helper() + checkFor(t, 5*time.Second, 200*time.Millisecond, func() error { + var err error + for _, sm := range rg { + if sm.node().State() == Closed { + continue + } + asm := sm.(*stateAdder) + if total := asm.total(); total != expected { + err = errors.Join(err, fmt.Errorf("Adder on %v has wrong total: %d vs %d", + asm.server(), total, expected)) + } + } + return err + }) +} + +// Factory function. +func newStateAdder(s *Server, cfg *RaftConfig, n RaftNode) stateMachine { + return &stateAdder{s: s, n: n, cfg: cfg, lch: make(chan bool, 1)} +} + +func initSingleMemRaftNode(t *testing.T) (*raft, func()) { + t.Helper() + n, c := initSingleMemRaftNodeWithCluster(t) + cleanup := func() { + c.shutdown() + } + return n, cleanup +} + +func initSingleMemRaftNodeWithCluster(t *testing.T) (*raft, *cluster) { + t.Helper() + c := createJetStreamClusterExplicit(t, "R3S", 3) + s := c.servers[0] // RunBasicJetStreamServer not available + + ms, err := newMemStore(&StreamConfig{Name: "TEST", Storage: MemoryStorage}) + require_NoError(t, err) + cfg := &RaftConfig{Name: "TEST", Store: t.TempDir(), Log: ms} + + id := s.sys.shash[:idLen] + err = s.bootstrapRaftNode(cfg, []string{id}, true) + require_NoError(t, err) + n, err := s.initRaftNode(globalAccountName, cfg, pprofLabels{}) + require_NoError(t, err) + + return n, c +} + +// Encode an AppendEntry. +// An AppendEntry is encoded into a buffer and that's stored into the WAL. +// This is a helper function to generate that buffer. +func encode(t *testing.T, ae *appendEntry) *appendEntry { + t.Helper() + buf, err := ae.encode(nil) + require_NoError(t, err) + ae.buf = buf + return ae +} diff --git a/server/raft_transport.go b/server/raft_transport.go new file mode 100644 index 0000000..af49ea0 --- /dev/null +++ b/server/raft_transport.go @@ -0,0 +1,114 @@ +// Copyright 2026-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +// raftTransport is an interface that defines the communication +// mechanism for Raft nodes. +type raftTransport interface { + // Node returns the RaftNode associated with this transport. + Node() RaftNode + + // Account returns the NATS Account this transport operates within. + Account() *Account + + // Reset reconfigures the transport for a new account. + // This involves tearing down existing client resources and + // setting up new ones for the provided account. + Reset(acc *Account) + + // Close shuts down the transport, releasing any associated resources + // like internal clients and subscriptions. + Close() + + // Publish sends a message to the specified subject. + Publish(subject string, reply string, msg []byte) + + // Subscribe creates a subscription for to the specified subject. + Subscribe(subject string, cb msgHandler) (*subscription, error) + + // Unsubscribe removes a previously established subscription. + Unsubscribe(sub *subscription) +} + +type newTransportFunc func(*Server, RaftNode) raftTransport + +// defaultTransport is the default implementation of the raftTransport interface. +// It uses an internal NATS client to allow communication between Raft nodes. +type defaultTransport struct { + n RaftNode + s *Server + c *client + sq *sendq + acc *Account +} + +func defaultRaftTransport(server *Server, raft RaftNode) raftTransport { + return &defaultTransport{s: server, n: raft} +} + +func (t *defaultTransport) Node() RaftNode { + return t.n +} + +func (t *defaultTransport) Account() *Account { + return t.acc +} + +func (t *defaultTransport) Reset(acc *Account) { + t.Close() + + t.c = t.s.createInternalSystemClient() + t.c.registerWithAccount(acc) + if acc.sq == nil { + acc.sq = t.s.newSendQ(acc) + } + t.sq = acc.sq + t.acc = acc +} + +func (t *defaultTransport) Close() { + if c := t.c; c != nil { + c.mu.Lock() + subs := make([]*subscription, 0, len(c.subs)) + for _, sub := range c.subs { + subs = append(subs, sub) + } + c.mu.Unlock() + for _, sub := range subs { + t.Unsubscribe(sub) + } + c.closeConnection(InternalClient) + t.c = nil + } +} + +func (t *defaultTransport) Publish(subject string, reply string, msg []byte) { + if t.sq == nil { + return + } + t.sq.send(subject, reply, nil, msg) +} + +func (t *defaultTransport) Subscribe(subject string, cb msgHandler) (*subscription, error) { + if t.c == nil { + return nil, errNoInternalClient + } + return t.s.systemSubscribe(subject, _EMPTY_, false, t.c, cb) +} + +func (t *defaultTransport) Unsubscribe(sub *subscription) { + if t.c != nil && sub != nil { + t.c.processUnsub(sub.sid) + } +} diff --git a/server/raft_transport_helpers_test.go b/server/raft_transport_helpers_test.go new file mode 100644 index 0000000..3278df0 --- /dev/null +++ b/server/raft_transport_helpers_test.go @@ -0,0 +1,178 @@ +// Copyright 2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file contains mock raft transport implementations and helper functions +// for testing. The `raftTransportHub` manages multiple `mockTransport` instances, +// allowing for the simulation of network partitions (`partition`, `heal`) +// and message interception (`setAfterMsgHook`). This setup is used to test +// raft interactions without actual network communication, providing control +// over message delivery and network conditions. + +package server + +import ( + "sync" +) + +type msgHook func(subject, reply string, msg []byte) + +type raftTransportHub struct { + mu sync.Mutex + transports map[string]*mockTransport + partitions map[string]int + afterMsg msgHook +} + +func newRaftTransportHub() *raftTransportHub { + return &raftTransportHub{ + partitions: make(map[string]int), + transports: make(map[string]*mockTransport), + } +} + +func (h *raftTransportHub) newTransport(s *Server, n RaftNode) raftTransport { + return &mockTransport{ + server: s, + node: n, + hub: h, + } +} + +func (h *raftTransportHub) register(t *mockTransport) { + h.mu.Lock() + defer h.mu.Unlock() + h.transports[t.Node().ID()] = t +} + +func (h *raftTransportHub) unregister(t *mockTransport) { + h.mu.Lock() + defer h.mu.Unlock() + delete(h.transports, t.Node().ID()) +} + +// Simulate a network partition. Nodes assigned to different `partitionID`s +// will not be able to exchange messages, effectively isolating them. +// By default, all nodes are considered to be in partition ID 0. +// See `heal(nodeID)` and `healPartitions()` to heal a partition. +func (h *raftTransportHub) partition(nodeID string, partitionID int) { + h.mu.Lock() + defer h.mu.Unlock() + h.partitions[nodeID] = partitionID +} + +// Reassign the node with the given ID to the default partition 0. +func (h *raftTransportHub) heal(nodeID string) { + h.mu.Lock() + defer h.mu.Unlock() + delete(h.partitions, nodeID) +} + +// Reassign all nodes to the default partition 0. +func (h *raftTransportHub) healPartitions() { + h.mu.Lock() + defer h.mu.Unlock() + clear(h.partitions) +} + +// Set a hook that is called back after a message is published. The after +// message hook can expect the raftTransportHub to be unlocked. It is OK +// to interact with the raftTransportHub inside the message hook. On the +// other hand, the underlying RaftNode that has sent the message remains +// locked throughout the execution of the hook. The hook can only call +// methods that do not lock the RaftNode. +func (h *raftTransportHub) setAfterMsgHook(hook msgHook) { + h.mu.Lock() + defer h.mu.Unlock() + h.afterMsg = hook +} + +func (h *raftTransportHub) publish(t *mockTransport, subject, reply string, msg []byte) { + h.mu.Lock() + afterMsgHook := h.afterMsg + sender := t.Node().ID() + partition := h.partitions[sender] + var subscriptions []*subscription + + for id, transport := range h.transports { + if sender == id { + continue + } + + if partition != h.partitions[id] { + continue + } + + result := transport.sub.Match(subject) + subscriptions = append(subscriptions, result.psubs...) + } + h.mu.Unlock() + + // Deliver without holding the hub lock, to avoid a lock inversion between + // Raft and hub locks. + for _, sub := range subscriptions { + sub.icb(sub, nil, nil, subject, reply, msg) + } + + if afterMsgHook != nil { + afterMsgHook(subject, reply, msg) + } +} + +type mockTransport struct { + server *Server + node RaftNode + hub *raftTransportHub + sub *Sublist + acc *Account +} + +func (t *mockTransport) Reset(acc *Account) { + t.Close() + t.acc = acc + t.sub = NewSublist(false) + t.hub.register(t) +} + +func (t *mockTransport) Close() { + t.hub.unregister(t) + t.sub = nil +} + +func (t *mockTransport) Node() RaftNode { + return t.node +} + +func (t *mockTransport) Account() *Account { + return t.acc +} + +func (t *mockTransport) Publish(subject string, reply string, msg []byte) { + t.hub.publish(t, subject, reply, msg) +} + +func (t *mockTransport) Subscribe(subject string, cb msgHandler) (*subscription, error) { + if t.sub == nil { + return nil, errNoInternalClient + } + sub := &subscription{subject: []byte(subject), sid: nil, icb: cb} + if err := t.sub.Insert(sub); err != nil { + return nil, err + } + return sub, nil +} + +func (t *mockTransport) Unsubscribe(sub *subscription) { + if t.sub != nil { + t.sub.Remove(sub) + } +} diff --git a/server/rate_counter.go b/server/rate_counter.go new file mode 100644 index 0000000..0988b83 --- /dev/null +++ b/server/rate_counter.go @@ -0,0 +1,65 @@ +// Copyright 2021-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "sync" + "time" +) + +type rateCounter struct { + limit int64 + count int64 + blocked uint64 + end time.Time + interval time.Duration + mu sync.Mutex +} + +func newRateCounter(limit int64) *rateCounter { + return &rateCounter{ + limit: limit, + interval: time.Second, + } +} + +func (r *rateCounter) allow() bool { + now := time.Now() + + r.mu.Lock() + + if now.After(r.end) { + r.count = 0 + r.end = now.Add(r.interval) + } else { + r.count++ + } + allow := r.count < r.limit + if !allow { + r.blocked++ + } + + r.mu.Unlock() + + return allow +} + +func (r *rateCounter) countBlocked() uint64 { + r.mu.Lock() + blocked := r.blocked + r.blocked = 0 + r.mu.Unlock() + + return blocked +} diff --git a/server/rate_counter_test.go b/server/rate_counter_test.go new file mode 100644 index 0000000..8b609af --- /dev/null +++ b/server/rate_counter_test.go @@ -0,0 +1,52 @@ +// Copyright 2021-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "testing" + "time" +) + +func TestRateCounter(t *testing.T) { + counter := newRateCounter(10) + counter.interval = 100 * time.Millisecond + + var i int + for i = 0; i < 10; i++ { + if !counter.allow() { + t.Errorf("counter should allow (iteration %d)", i) + } + } + for i = 0; i < 5; i++ { + if counter.allow() { + t.Errorf("counter should not allow (iteration %d)", i) + } + } + + blocked := counter.countBlocked() + if blocked != 5 { + t.Errorf("Expected blocked = 5, got %d", blocked) + } + + blocked = counter.countBlocked() + if blocked != 0 { + t.Errorf("Expected blocked = 0, got %d", blocked) + } + + time.Sleep(150 * time.Millisecond) + + if !counter.allow() { + t.Errorf("Expected true after current time window expired") + } +} diff --git a/server/reload.go b/server/reload.go new file mode 100644 index 0000000..3138bcb --- /dev/null +++ b/server/reload.go @@ -0,0 +1,2900 @@ +// Copyright 2017-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "cmp" + "crypto/tls" + "errors" + "fmt" + "net/url" + "reflect" + "slices" + "strings" + "sync/atomic" + "time" + + "github.com/klauspost/compress/s2" + + "github.com/nats-io/jwt/v2" + "github.com/nats-io/nuid" +) + +// FlagSnapshot captures the server options as specified by CLI flags at +// startup. This should not be modified once the server has started. +var FlagSnapshot *Options + +type reloadContext struct { + oldClusterPerms *RoutePermissions +} + +// option is a hot-swappable configuration setting. +type option interface { + // Apply the server option. + Apply(server *Server) + + // IsLoggingChange indicates if this option requires reloading the logger. + IsLoggingChange() bool + + // IsTraceLevelChange indicates if this option requires reloading cached trace level. + // Clients store trace level separately. + IsTraceLevelChange() bool + + // IsAuthChange indicates if this option requires reloading authorization. + IsAuthChange() bool + + // IsTLSChange indicates if this option requires reloading TLS. + IsTLSChange() bool + + // IsClusterPermsChange indicates if this option requires reloading + // cluster permissions. + IsClusterPermsChange() bool + + // IsClusterPoolSizeOrAccountsChange indicates if this option requires + // special handling for changes in cluster's pool size or accounts list. + IsClusterPoolSizeOrAccountsChange() bool + + // IsJetStreamChange inidicates a change in the servers config for JetStream. + // Account changes will be handled separately in reloadAuthorization. + IsJetStreamChange() bool + + // Indicates a change in the server that requires publishing the server's statz + IsStatszChange() bool +} + +// noopOption is a base struct that provides default no-op behaviors. +type noopOption struct{} + +func (n noopOption) IsLoggingChange() bool { + return false +} + +func (n noopOption) IsTraceLevelChange() bool { + return false +} + +func (n noopOption) IsAuthChange() bool { + return false +} + +func (n noopOption) IsTLSChange() bool { + return false +} + +func (n noopOption) IsClusterPermsChange() bool { + return false +} + +func (n noopOption) IsClusterPoolSizeOrAccountsChange() bool { + return false +} + +func (n noopOption) IsJetStreamChange() bool { + return false +} + +func (n noopOption) IsStatszChange() bool { + return false +} + +// loggingOption is a base struct that provides default option behaviors for +// logging-related options. +type loggingOption struct { + noopOption +} + +func (l loggingOption) IsLoggingChange() bool { + return true +} + +// traceLevelOption is a base struct that provides default option behaviors for +// tracelevel-related options. +type traceLevelOption struct { + loggingOption +} + +func (l traceLevelOption) IsTraceLevelChange() bool { + return true +} + +// traceOption implements the option interface for the `trace` setting. +type traceOption struct { + traceLevelOption + newValue bool +} + +// Apply is a no-op because logging will be reloaded after options are applied. +func (t *traceOption) Apply(server *Server) { + server.Noticef("Reloaded: trace = %v", t.newValue) +} + +// traceVersboseOption implements the option interface for the `trace_verbose` setting. +type traceVerboseOption struct { + traceLevelOption + newValue bool +} + +// Apply is a no-op because logging will be reloaded after options are applied. +func (t *traceVerboseOption) Apply(server *Server) { + server.Noticef("Reloaded: trace_verbose = %v", t.newValue) +} + +// traceHeadersOption implements the option interface for the `trace_headers` setting. +type traceHeadersOption struct { + traceLevelOption + newValue bool +} + +// Apply is a no-op because logging will be reloaded after options are applied. +func (t *traceHeadersOption) Apply(server *Server) { + server.Noticef("Reloaded: trace_headers = %v", t.newValue) +} + +// debugOption implements the option interface for the `debug` setting. +type debugOption struct { + loggingOption + newValue bool +} + +// Apply is mostly a no-op because logging will be reloaded after options are applied. +// However we will kick the raft nodes if they exist to reload. +func (d *debugOption) Apply(server *Server) { + server.Noticef("Reloaded: debug = %v", d.newValue) + server.reloadDebugRaftNodes(d.newValue) +} + +// logtimeOption implements the option interface for the `logtime` setting. +type logtimeOption struct { + loggingOption + newValue bool +} + +// Apply is a no-op because logging will be reloaded after options are applied. +func (l *logtimeOption) Apply(server *Server) { + server.Noticef("Reloaded: logtime = %v", l.newValue) +} + +// logtimeUTCOption implements the option interface for the `logtime_utc` setting. +type logtimeUTCOption struct { + loggingOption + newValue bool +} + +// Apply is a no-op because logging will be reloaded after options are applied. +func (l *logtimeUTCOption) Apply(server *Server) { + server.Noticef("Reloaded: logtime_utc = %v", l.newValue) +} + +// logfileOption implements the option interface for the `log_file` setting. +type logfileOption struct { + loggingOption + newValue string +} + +// Apply is a no-op because logging will be reloaded after options are applied. +func (l *logfileOption) Apply(server *Server) { + server.Noticef("Reloaded: log_file = %v", l.newValue) +} + +// syslogOption implements the option interface for the `syslog` setting. +type syslogOption struct { + loggingOption + newValue bool +} + +// Apply is a no-op because logging will be reloaded after options are applied. +func (s *syslogOption) Apply(server *Server) { + server.Noticef("Reloaded: syslog = %v", s.newValue) +} + +// remoteSyslogOption implements the option interface for the `remote_syslog` +// setting. +type remoteSyslogOption struct { + loggingOption + newValue string +} + +// Apply is a no-op because logging will be reloaded after options are applied. +func (r *remoteSyslogOption) Apply(server *Server) { + server.Noticef("Reloaded: remote_syslog = %v", r.newValue) +} + +// tlsOption implements the option interface for the `tls` setting. +type tlsOption struct { + noopOption + newValue *tls.Config +} + +// Apply the tls change. +func (t *tlsOption) Apply(server *Server) { + server.mu.Lock() + tlsRequired := t.newValue != nil + server.info.TLSRequired = tlsRequired && !server.getOpts().AllowNonTLS + message := "disabled" + if tlsRequired { + server.info.TLSVerify = (t.newValue.ClientAuth == tls.RequireAndVerifyClientCert) + message = "enabled" + } + server.mu.Unlock() + server.Noticef("Reloaded: tls = %s", message) +} + +func (t *tlsOption) IsTLSChange() bool { + return true +} + +// tlsTimeoutOption implements the option interface for the tls `timeout` +// setting. +type tlsTimeoutOption struct { + noopOption + newValue float64 +} + +// Apply is a no-op because the timeout will be reloaded after options are +// applied. +func (t *tlsTimeoutOption) Apply(server *Server) { + server.Noticef("Reloaded: tls timeout = %v", t.newValue) +} + +// tlsPinnedCertOption implements the option interface for the tls `pinned_certs` setting. +type tlsPinnedCertOption struct { + noopOption + newValue PinnedCertSet +} + +// Apply is a no-op because the pinned certs will be reloaded after options are applied. +func (t *tlsPinnedCertOption) Apply(server *Server) { + server.Noticef("Reloaded: %d pinned_certs", len(t.newValue)) +} + +// tlsHandshakeFirst implements the option interface for the tls `handshake first` setting. +type tlsHandshakeFirst struct { + noopOption + newValue bool +} + +// Apply is a no-op because the timeout will be reloaded after options are applied. +func (t *tlsHandshakeFirst) Apply(server *Server) { + server.Noticef("Reloaded: Client TLS handshake first: %v", t.newValue) +} + +// tlsHandshakeFirstFallback implements the option interface for the tls `handshake first fallback delay` setting. +type tlsHandshakeFirstFallback struct { + noopOption + newValue time.Duration +} + +// Apply is a no-op because the timeout will be reloaded after options are applied. +func (t *tlsHandshakeFirstFallback) Apply(server *Server) { + server.Noticef("Reloaded: Client TLS handshake first fallback delay: %v", t.newValue) +} + +// authOption is a base struct that provides default option behaviors. +type authOption struct { + noopOption +} + +func (o authOption) IsAuthChange() bool { + return true +} + +// usernameOption implements the option interface for the `username` setting. +type usernameOption struct { + authOption +} + +// Apply is a no-op because authorization will be reloaded after options are +// applied. +func (u *usernameOption) Apply(server *Server) { + server.Noticef("Reloaded: authorization username") +} + +// passwordOption implements the option interface for the `password` setting. +type passwordOption struct { + authOption +} + +// Apply is a no-op because authorization will be reloaded after options are +// applied. +func (p *passwordOption) Apply(server *Server) { + server.Noticef("Reloaded: authorization password") +} + +// authorizationOption implements the option interface for the `token` +// authorization setting. +type authorizationOption struct { + authOption +} + +// Apply is a no-op because authorization will be reloaded after options are +// applied. +func (a *authorizationOption) Apply(server *Server) { + server.Noticef("Reloaded: authorization token") +} + +// authTimeoutOption implements the option interface for the authorization +// `timeout` setting. +type authTimeoutOption struct { + noopOption // Not authOption because this is a no-op; will be reloaded with options. + newValue float64 +} + +// Apply is a no-op because the timeout will be reloaded after options are +// applied. +func (a *authTimeoutOption) Apply(server *Server) { + server.Noticef("Reloaded: authorization timeout = %v", a.newValue) +} + +// tagsOption implements the option interface for the `tags` setting. +type tagsOption struct { + noopOption // Not authOption because this is a no-op; will be reloaded with options. +} + +func (u *tagsOption) Apply(server *Server) { + server.Noticef("Reloaded: tags") +} + +func (u *tagsOption) IsStatszChange() bool { + return true +} + +// metadataOption implements the option interface for the `metadata` setting. +type metadataOption struct { + noopOption // Not authOption because this is a no-op; will be reloaded with options. +} + +func (u *metadataOption) Apply(server *Server) { + server.Noticef("Reloaded: metadata") +} + +func (u *metadataOption) IsStatszChange() bool { + return true +} + +// usersOption implements the option interface for the authorization `users` +// setting. +type usersOption struct { + authOption +} + +func (u *usersOption) Apply(server *Server) { + server.Noticef("Reloaded: authorization users") +} + +// nkeysOption implements the option interface for the authorization `users` +// setting. +type nkeysOption struct { + authOption +} + +func (u *nkeysOption) Apply(server *Server) { + server.Noticef("Reloaded: authorization nkey users") +} + +// clusterOption implements the option interface for the `cluster` setting. +type clusterOption struct { + authOption + newValue ClusterOpts + permsChanged bool + accsAdded []string + accsRemoved []string + poolSizeChanged bool + compressChanged bool +} + +// Apply the cluster change. +func (c *clusterOption) Apply(s *Server) { + // TODO: support enabling/disabling clustering. + s.mu.Lock() + tlsRequired := c.newValue.TLSConfig != nil + s.routeInfo.TLSRequired = tlsRequired + s.routeInfo.TLSVerify = tlsRequired + s.routeInfo.AuthRequired = c.newValue.Username != "" + if c.newValue.NoAdvertise { + s.routeInfo.ClientConnectURLs = nil + s.routeInfo.WSConnectURLs = nil + } else { + s.routeInfo.ClientConnectURLs = s.clientConnectURLs + s.routeInfo.WSConnectURLs = s.websocket.connectURLs + } + s.setRouteInfoHostPortAndIP() + var routes []*client + if c.compressChanged { + co := &s.getOpts().Cluster.Compression + newMode := co.Mode + s.forEachRoute(func(r *client) { + r.mu.Lock() + // Skip routes that are "not supported" (because they will never do + // compression) or the routes that have already the new compression + // mode. + if r.route.compression == CompressionNotSupported || r.route.compression == newMode { + r.mu.Unlock() + return + } + // We need to close the route if it had compression "off" or the new + // mode is compression "off", or if the new mode is "accept", because + // these require negotiation. + if r.route.compression == CompressionOff || newMode == CompressionOff || newMode == CompressionAccept { + routes = append(routes, r) + } else if newMode == CompressionS2Auto { + // If the mode is "s2_auto", we need to check if there is really + // need to change, and at any rate, we want to save the actual + // compression level here, not s2_auto. + r.updateS2AutoCompressionLevel(co, &r.route.compression) + } else { + // Simply change the compression writer + r.out.cw = s2.NewWriter(nil, s2WriterOptions(newMode)...) + r.route.compression = newMode + } + r.mu.Unlock() + }) + } + s.mu.Unlock() + if c.newValue.Name != "" && c.newValue.Name != s.ClusterName() { + s.setClusterName(c.newValue.Name) + } + for _, r := range routes { + r.closeConnection(ClientClosed) + } + s.Noticef("Reloaded: cluster") + if tlsRequired && c.newValue.TLSConfig.InsecureSkipVerify { + s.Warnf(clusterTLSInsecureWarning) + } +} + +func (c *clusterOption) IsClusterPermsChange() bool { + return c.permsChanged +} + +func (c *clusterOption) IsClusterPoolSizeOrAccountsChange() bool { + return c.poolSizeChanged || len(c.accsAdded) > 0 || len(c.accsRemoved) > 0 +} + +func (c *clusterOption) diffPoolAndAccounts(old *ClusterOpts) { + c.poolSizeChanged = c.newValue.PoolSize != old.PoolSize +addLoop: + for _, na := range c.newValue.PinnedAccounts { + for _, oa := range old.PinnedAccounts { + if na == oa { + continue addLoop + } + } + c.accsAdded = append(c.accsAdded, na) + } +removeLoop: + for _, oa := range old.PinnedAccounts { + for _, na := range c.newValue.PinnedAccounts { + if oa == na { + continue removeLoop + } + } + c.accsRemoved = append(c.accsRemoved, oa) + } +} + +// routesOption implements the option interface for the cluster `routes` +// setting. +type routesOption struct { + noopOption + add []*url.URL + remove []*url.URL +} + +// Apply the route changes by adding and removing the necessary routes. +func (r *routesOption) Apply(server *Server) { + server.mu.Lock() + routes := make([]*client, server.numRoutes()) + i := 0 + server.forEachRoute(func(r *client) { + routes[i] = r + i++ + }) + // If there was a change, notify monitoring code that it should + // update the route URLs if /varz endpoint is inspected. + if len(r.add)+len(r.remove) > 0 { + server.varzUpdateRouteURLs = true + } + server.mu.Unlock() + + // Remove routes. + for _, remove := range r.remove { + for _, client := range routes { + var url *url.URL + client.mu.Lock() + if client.route != nil { + url = client.route.url + } + client.mu.Unlock() + if url != nil && urlsAreEqual(url, remove) { + // Do not attempt to reconnect when route is removed. + client.setNoReconnect() + client.closeConnection(RouteRemoved) + server.Noticef("Removed route %v", remove) + } + } + } + + // Add routes. + server.mu.Lock() + server.solicitRoutes(r.add, server.getOpts().Cluster.PinnedAccounts) + server.mu.Unlock() + + server.Noticef("Reloaded: cluster routes") +} + +// maxConnOption implements the option interface for the `max_connections` +// setting. +type maxConnOption struct { + noopOption + newValue int +} + +// Apply the max connections change by closing random connections til we are +// below the limit if necessary. +func (m *maxConnOption) Apply(server *Server) { + server.mu.Lock() + clients := make([]*client, 0, len(server.clients)) + // Map iteration is random, which allows us to close random connections. + for _, client := range server.clients { + if isInternalClient(client.kind) { + continue + } + clients = append(clients, client) + } + server.mu.Unlock() + + if newc := max(0, m.newValue); len(clients) > newc { + // Close connections til we are within the limit. + var ( + numClose = len(clients) - newc + closed = 0 + ) + for _, client := range clients { + client.maxConnExceeded() + closed++ + if closed >= numClose { + break + } + } + server.Noticef("Closed %d connections to fall within max_connections", closed) + } + server.Noticef("Reloaded: max_connections = %v", m.newValue) +} + +// pidFileOption implements the option interface for the `pid_file` setting. +type pidFileOption struct { + noopOption + newValue string +} + +// Apply the setting by logging the pid to the new file. +func (p *pidFileOption) Apply(server *Server) { + if p.newValue == "" { + return + } + if err := server.logPid(); err != nil { + server.Errorf("Failed to write pidfile: %v", err) + } + server.Noticef("Reloaded: pid_file = %v", p.newValue) +} + +// portsFileDirOption implements the option interface for the `portFileDir` setting. +type portsFileDirOption struct { + noopOption + oldValue string + newValue string +} + +func (p *portsFileDirOption) Apply(server *Server) { + server.deletePortsFile(p.oldValue) + server.logPorts() + server.Noticef("Reloaded: ports_file_dir = %v", p.newValue) +} + +// maxControlLineOption implements the option interface for the +// `max_control_line` setting. +type maxControlLineOption struct { + noopOption + newValue int32 +} + +// Apply the setting by updating each client. +func (m *maxControlLineOption) Apply(server *Server) { + mcl := int32(m.newValue) + server.mu.Lock() + for _, client := range server.clients { + atomic.StoreInt32(&client.mcl, mcl) + } + server.mu.Unlock() + server.Noticef("Reloaded: max_control_line = %d", mcl) +} + +// maxPayloadOption implements the option interface for the `max_payload` +// setting. +type maxPayloadOption struct { + noopOption + newValue int32 +} + +// Apply the setting by updating the server info and each client. +func (m *maxPayloadOption) Apply(server *Server) { + server.mu.Lock() + server.info.MaxPayload = m.newValue + for _, client := range server.clients { + atomic.StoreInt32(&client.mpay, int32(m.newValue)) + } + server.mu.Unlock() + server.Noticef("Reloaded: max_payload = %d", m.newValue) +} + +// pingIntervalOption implements the option interface for the `ping_interval` +// setting. +type pingIntervalOption struct { + noopOption + newValue time.Duration +} + +// Apply is a no-op because the ping interval will be reloaded after options +// are applied. +func (p *pingIntervalOption) Apply(server *Server) { + server.Noticef("Reloaded: ping_interval = %s", p.newValue) +} + +// maxPingsOutOption implements the option interface for the `ping_max` +// setting. +type maxPingsOutOption struct { + noopOption + newValue int +} + +// Apply is a no-op because the ping interval will be reloaded after options +// are applied. +func (m *maxPingsOutOption) Apply(server *Server) { + server.Noticef("Reloaded: ping_max = %d", m.newValue) +} + +// writeDeadlineOption implements the option interface for the `write_deadline` +// setting. +type writeDeadlineOption struct { + noopOption + newValue time.Duration +} + +// Apply is a no-op because the write deadline will be reloaded after options +// are applied. +func (w *writeDeadlineOption) Apply(server *Server) { + server.Noticef("Reloaded: write_deadline = %s", w.newValue) +} + +// clientAdvertiseOption implements the option interface for the `client_advertise` setting. +type clientAdvertiseOption struct { + noopOption + newValue string +} + +// Apply the setting by updating the server info and regenerate the infoJSON byte array. +func (c *clientAdvertiseOption) Apply(server *Server) { + server.mu.Lock() + server.setInfoHostPort() + server.mu.Unlock() + server.Noticef("Reload: client_advertise = %s", c.newValue) +} + +// accountsOption implements the option interface. +// Ensure that authorization code is executed if any change in accounts +type accountsOption struct { + authOption +} + +// Apply is a no-op. Changes will be applied in reloadAuthorization +func (a *accountsOption) Apply(s *Server) { + s.Noticef("Reloaded: accounts") +} + +// For changes to a server's config. +type jetStreamOption struct { + noopOption + newValue bool +} + +func (a *jetStreamOption) Apply(s *Server) { + s.Noticef("Reloaded: JetStream") +} + +func (jso jetStreamOption) IsJetStreamChange() bool { + return true +} + +func (jso jetStreamOption) IsStatszChange() bool { + return true +} + +type jetStreamLimitsOption struct { + noopOption + newMaxMemory int64 + newMaxStore int64 +} + +func (jso *jetStreamLimitsOption) Apply(s *Server) { + js := s.getJetStream() + if js == nil { + return + } + js.mu.Lock() + if jso.newMaxMemory > 0 { + js.config.MaxMemory = jso.newMaxMemory + atomic.StoreInt64(&js.memMax, js.config.MaxMemory) + s.Noticef("Reloaded: JetStream max_mem_store = %s", friendlyBytes(jso.newMaxMemory)) + } + if jso.newMaxStore > 0 { + js.config.MaxStore = jso.newMaxStore + atomic.StoreInt64(&js.storeMax, js.config.MaxStore) + s.Noticef("Reloaded: JetStream max_file_store = %s", friendlyBytes(jso.newMaxStore)) + } + js.mu.Unlock() +} + +func (jso *jetStreamLimitsOption) IsStatszChange() bool { + return true +} + +// For changes to the server-wide JetStream limits. +type jetStreamServerLimitsOption struct { + noopOption + newValue JSLimitOpts +} + +func (jso *jetStreamServerLimitsOption) Apply(s *Server) { + s.Noticef("Reloaded: JetStream limits") +} + +type defaultSentinelOption struct { + noopOption + newValue string +} + +func (so *defaultSentinelOption) Apply(s *Server) { + s.Noticef("Reloaded: default_sentinel = %s", so.newValue) +} + +type ocspOption struct { + tlsOption + newValue *OCSPConfig +} + +func (a *ocspOption) Apply(s *Server) { + s.Noticef("Reloaded: OCSP") +} + +type ocspResponseCacheOption struct { + tlsOption + newValue *OCSPResponseCacheConfig +} + +func (a *ocspResponseCacheOption) Apply(s *Server) { + s.Noticef("Reloaded OCSP peer cache") +} + +// connectErrorReports implements the option interface for the `connect_error_reports` +// setting. +type connectErrorReports struct { + noopOption + newValue int +} + +// Apply is a no-op because the value will be reloaded after options are applied. +func (c *connectErrorReports) Apply(s *Server) { + s.Noticef("Reloaded: connect_error_reports = %v", c.newValue) +} + +// connectErrorReports implements the option interface for the `connect_error_reports` +// setting. +type reconnectErrorReports struct { + noopOption + newValue int +} + +// Apply is a no-op because the value will be reloaded after options are applied. +func (r *reconnectErrorReports) Apply(s *Server) { + s.Noticef("Reloaded: reconnect_error_reports = %v", r.newValue) +} + +// maxTracedMsgLenOption implements the option interface for the `max_traced_msg_len` setting. +type maxTracedMsgLenOption struct { + noopOption + newValue int +} + +// Apply the setting by updating the maximum traced message length. +func (m *maxTracedMsgLenOption) Apply(server *Server) { + server.mu.Lock() + defer server.mu.Unlock() + server.opts.MaxTracedMsgLen = m.newValue + server.Noticef("Reloaded: max_traced_msg_len = %d", m.newValue) +} + +type mqttAckWaitReload struct { + noopOption + newValue time.Duration +} + +func (o *mqttAckWaitReload) Apply(s *Server) { + s.Noticef("Reloaded: MQTT ack_wait = %v", o.newValue) +} + +type mqttMaxAckPendingReload struct { + noopOption + newValue uint16 +} + +func (o *mqttMaxAckPendingReload) Apply(s *Server) { + s.mqttUpdateMaxAckPending(o.newValue) + s.Noticef("Reloaded: MQTT max_ack_pending = %v", o.newValue) +} + +type mqttStreamReplicasReload struct { + noopOption + newValue int +} + +func (o *mqttStreamReplicasReload) Apply(s *Server) { + s.Noticef("Reloaded: MQTT stream_replicas = %v", o.newValue) +} + +type mqttConsumerReplicasReload struct { + noopOption + newValue int +} + +func (o *mqttConsumerReplicasReload) Apply(s *Server) { + s.Noticef("Reloaded: MQTT consumer_replicas = %v", o.newValue) +} + +type mqttConsumerMemoryStorageReload struct { + noopOption + newValue bool +} + +func (o *mqttConsumerMemoryStorageReload) Apply(s *Server) { + s.Noticef("Reloaded: MQTT consumer_memory_storage = %v", o.newValue) +} + +type mqttInactiveThresholdReload struct { + noopOption + newValue time.Duration +} + +func (o *mqttInactiveThresholdReload) Apply(s *Server) { + s.Noticef("Reloaded: MQTT consumer_inactive_threshold = %v", o.newValue) +} + +type profBlockRateReload struct { + noopOption + newValue int +} + +func (o *profBlockRateReload) Apply(s *Server) { + s.setBlockProfileRate(o.newValue) + s.Noticef("Reloaded: prof_block_rate = %v", o.newValue) +} + +type leafNodeOption struct { + noopOption + tlsFirstChanged bool + compressionChanged bool + // These are for the remotes + added []*RemoteLeafOpts + changed map[*leafNodeCfg]*remoteLeafOption +} + +type remoteLeafOption struct { + tlsFirstChanged bool + compressionChanged bool + disabledChanged bool + opts *RemoteLeafOpts +} + +// Given `old` and `new` Leafnode options, this function will return the structure +// used for applying the configuration, or an error is there are changes that +// are not supported. +func getLeafNodeOptionsChanges(s *Server, old, new *LeafNodeOpts) (*leafNodeOption, error) { + + // We can't use DeepEqual for `Users` field, so do custom check. + if usersHaveChanged(old.Users, new.Users) { + return nil, fmt.Errorf("field \"Users\": old=%v, new=%v", old.Users, new.Users) + } + + // Check the main leafnodes{} block to see if there are any changes that are + // not supported. We provide a list of fields to ignore (we already checked, + // allow them to be modified or will check later). + if err := checkConfigsEqual(old, new, []string{ + "Compression", + "Remotes", + "TLSHandshakeFirst", + "TLSHandshakeFirstFallback", + "TLSConfig", + "Users", + }); err != nil { + return nil, err + } + + const ( + remoteErrFormat = "remote %s: %s" + maxAttempts = 20 + ) + var ( + nlo *leafNodeOption + // Track whether any existing remote was not found (i.e. removed). + removed bool + ) + +forLoop: + for failed := range maxAttempts { + removed = false + if failed > 0 { + // If we failed once, we will wait a bit before trying again the remotes. + // This could give enough time for connections that were in progress to complete. + select { + case <-time.After(50 * time.Millisecond): + case <-s.quitCh: + return nil, ErrServerNotRunning + } + } + nlo = &leafNodeOption{ + tlsFirstChanged: (old.TLSHandshakeFirst != new.TLSHandshakeFirst || old.TLSHandshakeFirstFallback != new.TLSHandshakeFirstFallback), + compressionChanged: !old.Compression.equals(&new.Compression), + // Start with all, will update when processing existing ones. + // Since the list will be modified, we need to clone it. + added: slices.Clone(new.Remotes), + } + + s.mu.RLock() + // Go through the list of existing remote configurations. + for lrc := range s.leafRemoteCfgs { + var rlo *RemoteLeafOpts + // Look for the corresponding `*RemoteLeafOpts` in the `nlo.added` + // list. If it is found, that function returns an updated list + // with the element removed from it. + lrc.RLock() + rlo, nlo.added = getRemoteLeafOpts(lrc.name(), nlo.added) + if rlo == nil { + // Not found, will be removed in leafNodeOption.Apply(). + removed = true + lrc.RUnlock() + continue + } + // Now we need to make sure that there are no changes that we don't + // support for a RemoteLeafOpts. + err := checkConfigsEqual(lrc.RemoteLeafOpts, rlo, []string{ + "Compression", + "Disabled", + "TLS", + "TLSHandshakeFirst", + "TLSConfig", + }) + if err != nil { + lrc.RUnlock() + s.mu.RUnlock() + return nil, fmt.Errorf(remoteErrFormat, rlo.safeName(), err) + } + disabledChanged := lrc.Disabled != rlo.Disabled + // If this remote was disabled and is now enabled, we need to make sure + // that there is no connect in progress. If that is the case, either + // try again (if it is the first failure) or return an error. + if disabledChanged && lrc.Disabled && lrc.connInProgress { + lrc.RUnlock() + s.mu.RUnlock() + if failed < maxAttempts-1 { + continue forLoop + } + return nil, fmt.Errorf(remoteErrFormat, rlo.safeName(), + "cannot be enabled at the moment, try again") + } + // Since we will use the new `rlo.TLSConfig` later on, consider all + // existing remote configs as "changed" and store them in the + // `nlo.changed` map. + if nlo.changed == nil { + nlo.changed = make(map[*leafNodeCfg]*remoteLeafOption) + } + lnro := &remoteLeafOption{ + tlsFirstChanged: lrc.TLSHandshakeFirst != rlo.TLSHandshakeFirst, + compressionChanged: !lrc.Compression.equals(&rlo.Compression), + disabledChanged: disabledChanged, + opts: rlo, + } + lrc.RUnlock() + nlo.changed[lrc] = lnro + } + if len(nlo.added) > 0 { + // Go through the added list and check if an added was recently removed and, + // if that is the case, is it still in the `s.rmLeafRemoteCfgs` map, which + // may mean that there was a connect-in-progress that did not complete yet. + // Either try again (if it is the first failure) or return an error. + for _, rlo := range nlo.added { + if _, cip := s.rmLeafRemoteCfgs[rlo.name()]; cip { + s.mu.RUnlock() + if failed < maxAttempts-1 { + continue forLoop + } + return nil, fmt.Errorf(remoteErrFormat, rlo.safeName(), + "cannot be added at the moment, try again") + } + } + } + s.mu.RUnlock() + break + } + + // Now we want to make sure that there were actual changes, so that we don't + // cause a reload of leafnodes for nothing. However, if one has (or all have) + // been removed we still need to invoke leafNodeOption.Apply(). + if !nlo.tlsFirstChanged && !nlo.compressionChanged && !removed && len(nlo.added) == 0 && len(nlo.changed) == 0 { + return nil, nil + } + + return nlo, nil +} + +func usersHaveChanged(ousers, nusers []*User) bool { + if len(ousers) != len(nusers) { + return true + } + // We did not do a strict list order check in the past, so maintain this to + // avoid possible breaking changes. + oua := make(map[string]*User, len(ousers)) + nua := make(map[string]*User, len(nusers)) + for _, u := range ousers { + oua[u.Username] = u + } + for _, u := range nusers { + nua[u.Username] = u + } + for uname, u := range oua { + // If we can not find new one with same name, consider that they have changed. + nu, ok := nua[uname] + if !ok { + return true + } + // Same if password or account has changed. + if u.Password != nu.Password || u.Account.GetName() != nu.Account.GetName() { + return true + } + } + return false +} + +// Given the `search` remote leafnode options name, searches for a match in the `list`. +// If found, returns the `*RemoteLeafOpts` from the list, and the updated list +// without the element in it. If not found, returns `nil` and the unmodified list. +func getRemoteLeafOpts(search string, list []*RemoteLeafOpts) (*RemoteLeafOpts, []*RemoteLeafOpts) { + for i, rlo := range list { + if search == rlo.name() { + lastIdx := len(list) - 1 + if lastIdx == 0 { + return rlo, nil + } + if i < lastIdx { + list[i] = list[lastIdx] + } + list = list[:lastIdx] + return rlo, list + } + } + return nil, list +} + +func (l *leafNodeOption) Apply(s *Server) { + opts := s.getOpts() + if l.tlsFirstChanged { + s.Noticef("Reloaded: LeafNode TLS HandshakeFirst value is: %v", opts.LeafNode.TLSHandshakeFirst) + s.Noticef("Reloaded: LeafNode TLS HandshakeFirstFallback value is: %v", opts.LeafNode.TLSHandshakeFirstFallback) + } + if l.compressionChanged { + s.Noticef("Reloaded: LeafNode Compression value is: %v", opts.LeafNode.Compression) + } + + var close []*client + var enable []*leafNodeCfg + var removed bool + + s.mu.Lock() + acceptSideCompOpts := &opts.LeafNode.Compression + // First go over existing leafnode remote configurations and + // either remove if no longer present, or update the config. + for lrc := range s.leafRemoteCfgs { + rlo := l.changed[lrc] + if rlo == nil { + delete(s.leafRemoteCfgs, lrc) + removed = true + if s.rmLeafRemoteCfgs == nil { + s.rmLeafRemoteCfgs = make(map[string]*leafNodeCfg) + } + s.rmLeafRemoteCfgs[lrc.name()] = lrc + lrc.markAsRemoved() + s.Noticef("Reloaded: LeafNode Remote %s removed", lrc.RemoteLeafOpts.safeName()) + // We will close the existing connection in the next for-loop. + continue + } + lrc.Lock() + // TLSConfig is always applied. + lrc.TLSConfig = rlo.opts.TLSConfig.Clone() + // Now update what has been detected has changed. + if rlo.tlsFirstChanged { + lrc.TLSHandshakeFirst = rlo.opts.TLSHandshakeFirst + s.Noticef("Reloaded: LeafNode Remote %s TLS HandshakeFirst value is: %v", + lrc.RemoteLeafOpts.safeName(), rlo.opts.TLSHandshakeFirst) + } + if rlo.compressionChanged { + lrc.Compression = rlo.opts.Compression + s.Noticef("Reloaded: LeafNode Remote %s Compression value is: %v", + lrc.RemoteLeafOpts.safeName(), rlo.opts.Compression) + } + if rlo.disabledChanged { + // Change to new value. + lrc.Disabled = rlo.opts.Disabled + if lrc.Disabled { + lrc.notifyQuitChannel() + } else { + enable = append(enable, lrc) + } + s.Noticef("Reloaded: LeafNode Remote %s Disabled value is: %v", + lrc.RemoteLeafOpts.safeName(), rlo.opts.Disabled) + } + lrc.Unlock() + } + // Second, go over existing leaf connections and apply compression + // changes (if applicable) and collect connections that need to be + // closed and/or disabled. + for _, c := range s.leafs { + var co *CompressionOpts + + c.mu.Lock() + if r := c.leaf.remote; r != nil { + rlo := l.changed[r] + // If the config is not in the `changed` map, or the new config says that + // the connection is disabled, collect so we can close it after the server + // lock is released. + if rlo == nil || (rlo.disabledChanged && rlo.opts.Disabled) { + c.flags.set(noReconnect) + close = append(close, c) + c.mu.Unlock() + continue + } + if rlo.compressionChanged { + co = &r.Compression + } + } else if l.compressionChanged { + co = acceptSideCompOpts + } + if co != nil && applyCompressionChanges(c, co) { + close = append(close, c) + } + c.mu.Unlock() + } + s.mu.Unlock() + + // Close the connections for which negotiation is required, have been disabled + // or simply removed. + for _, c := range close { + c.closeConnection(ClientClosed) + } + // Start the ones that have been enabled. + for _, r := range enable { + s.connectToRemoteLeafNodeAsynchronously(r, true) + } + // Finally, deal with the ones that have been added. + if len(l.added) > 0 { + s.solicitLeafNodeRemotes(l.added) + } + // Deal with removed configs. Make sure there are no connect-in-progress. + // If there are still some, have a go routine to check in the background. + if removed { + if checkAgain := checkRemovedLeafNodeCfgs(s); checkAgain { + checkRemovedLeafNodeCfgsAsync(s) + } + } +} + +// Go through the removed remote leafnode configs map to check if the +// connect-in-progress flag is set. If not, remove from the map. +// Returns `true` if there are still some that are in progress. +func checkRemovedLeafNodeCfgs(s *Server) bool { + var inProgress int + s.mu.Lock() + for rn, r := range s.rmLeafRemoteCfgs { + if r.isConnectInProgress() { + inProgress++ + } else { + delete(s.rmLeafRemoteCfgs, rn) + } + } + s.mu.Unlock() + // Needs to be called again if inProgress > 0 + return inProgress > 0 +} + +// Will start a go routine that will periodically call `checkRemovedLeafNodeCfgs`. +// When the removed map has been emptied, the go routine will end. It is ok for +// this function to be invoked multiple times and have multiple instances running +// concurrently. +func checkRemovedLeafNodeCfgsAsync(s *Server) { + s.startGoRoutine(func() { + defer s.grWG.Done() + tick := time.NewTicker(50 * time.Millisecond) + defer tick.Stop() + for { + select { + case <-tick.C: + if checkAgain := checkRemovedLeafNodeCfgs(s); !checkAgain { + return + } + case <-s.quitCh: + return + } + } + }) +} + +// The `co` compression options are applied to the given leaf connection `c`. +// If a "restart" of the connection is needed, will return true, false otherwise. +func applyCompressionChanges(c *client, co *CompressionOpts) bool { + newMode := co.Mode + // Skip leaf connections that are "not supported" (because they + // will never do compression) or the ones that have already the + // new compression mode. + if c.leaf.compression == CompressionNotSupported || c.leaf.compression == newMode { + return false + } + // We need to close the connections if it had compression "off" or the new + // mode is compression "off", or if the new mode is "accept", because + // these require negotiation. + if c.leaf.compression == CompressionOff || newMode == CompressionOff || newMode == CompressionAccept { + return true + } else if newMode == CompressionS2Auto { + // If the mode is "s2_auto", we need to check if there is really + // need to change, and at any rate, we want to save the actual + // compression level here, not s2_auto. + c.updateS2AutoCompressionLevel(co, &c.leaf.compression) + } else { + // Simply change the compression writer + c.out.cw = s2.NewWriter(nil, s2WriterOptions(newMode)...) + c.leaf.compression = newMode + } + return false +} + +type noFastProdStallReload struct { + noopOption + noStall bool +} + +func (l *noFastProdStallReload) Apply(s *Server) { + var not string + if l.noStall { + not = "not " + } + s.Noticef("Reloaded: fast producers will %sbe stalled", not) +} + +// Compares options and disconnects clients that are no longer listed in pinned certs. Lock must not be held. +func (s *Server) recheckPinnedCerts(curOpts *Options, newOpts *Options) { + s.mu.Lock() + disconnectClients := []*client{} + protoToPinned := map[int]PinnedCertSet{} + if !reflect.DeepEqual(newOpts.TLSPinnedCerts, curOpts.TLSPinnedCerts) { + protoToPinned[NATS] = curOpts.TLSPinnedCerts + } + if !reflect.DeepEqual(newOpts.MQTT.TLSPinnedCerts, curOpts.MQTT.TLSPinnedCerts) { + protoToPinned[MQTT] = curOpts.MQTT.TLSPinnedCerts + } + if !reflect.DeepEqual(newOpts.Websocket.TLSPinnedCerts, curOpts.Websocket.TLSPinnedCerts) { + protoToPinned[WS] = curOpts.Websocket.TLSPinnedCerts + } + for _, c := range s.clients { + if c.kind != CLIENT { + continue + } + if pinned, ok := protoToPinned[c.clientType()]; ok { + if !c.matchesPinnedCert(pinned) { + disconnectClients = append(disconnectClients, c) + } + } + } + checkClients := func(kind int, clients map[uint64]*client, set PinnedCertSet) { + for _, c := range clients { + if c.kind == kind && !c.matchesPinnedCert(set) { + disconnectClients = append(disconnectClients, c) + } + } + } + if !reflect.DeepEqual(newOpts.LeafNode.TLSPinnedCerts, curOpts.LeafNode.TLSPinnedCerts) { + checkClients(LEAF, s.leafs, newOpts.LeafNode.TLSPinnedCerts) + } + if !reflect.DeepEqual(newOpts.Cluster.TLSPinnedCerts, curOpts.Cluster.TLSPinnedCerts) { + s.forEachRoute(func(c *client) { + if !c.matchesPinnedCert(newOpts.Cluster.TLSPinnedCerts) { + disconnectClients = append(disconnectClients, c) + } + }) + } + if s.gateway.enabled && !reflect.DeepEqual(newOpts.Gateway.TLSPinnedCerts, curOpts.Gateway.TLSPinnedCerts) { + gw := s.gateway + gw.RLock() + for _, c := range gw.out { + if !c.matchesPinnedCert(newOpts.Gateway.TLSPinnedCerts) { + disconnectClients = append(disconnectClients, c) + } + } + checkClients(GATEWAY, gw.in, newOpts.Gateway.TLSPinnedCerts) + gw.RUnlock() + } + s.mu.Unlock() + if len(disconnectClients) > 0 { + s.Noticef("Disconnect %d clients due to pinned certs reload", len(disconnectClients)) + for _, c := range disconnectClients { + c.closeConnection(TLSHandshakeError) + } + } +} + +type proxiesReload struct { + noopOption + add []string + del []string +} + +func (p *proxiesReload) Apply(s *Server) { + var clients []*client + s.mu.Lock() + for _, k := range p.del { + cc := s.proxiedConns[k] + delete(s.proxiedConns, k) + if len(cc) > 0 { + for _, c := range cc { + clients = append(clients, c) + } + } + } + s.processProxiesTrustedKeys() + s.mu.Unlock() + if len(p.del) > 0 { + for _, c := range clients { + c.setAuthError(ErrAuthProxyNotTrusted) + c.authViolation() + } + s.Noticef("Reloaded: proxies trusted keys %q were removed", p.del) + } + if len(p.add) > 0 { + s.Noticef("Reloaded: proxies trusted keys %q were added", p.add) + } +} + +// Reload reads the current configuration file and calls out to ReloadOptions +// to apply the changes. This returns an error if the server was not started +// with a config file or an option which doesn't support hot-swapping was changed. +func (s *Server) Reload() error { + s.mu.Lock() + configFile := s.configFile + s.mu.Unlock() + if configFile == "" { + return errors.New("can only reload config when a file is provided using -c or --config") + } + + newOpts, err := ProcessConfigFile(configFile) + if err != nil { + // TODO: Dump previous good config to a .bak file? + return err + } + return s.ReloadOptions(newOpts) +} + +// ReloadOptions applies any supported options from the provided Options +// type. This returns an error if an option which doesn't support +// hot-swapping was changed. +// The provided Options type should not be re-used afterwards. +// Either use Options.Clone() to pass a copy, or make a new one. +func (s *Server) ReloadOptions(newOpts *Options) error { + s.reloadMu.Lock() + defer s.reloadMu.Unlock() + + s.mu.Lock() + + curOpts := s.getOpts() + + clientOrgPort := curOpts.Port + clusterOrgPort := curOpts.Cluster.Port + gatewayOrgPort := curOpts.Gateway.Port + leafnodesOrgPort := curOpts.LeafNode.Port + websocketOrgPort := curOpts.Websocket.Port + mqttOrgPort := curOpts.MQTT.Port + + s.mu.Unlock() + + // In case "-cluster ..." was provided through the command line, this will + // properly set the Cluster.Host/Port etc... + if l := curOpts.Cluster.ListenStr; l != _EMPTY_ { + newOpts.Cluster.ListenStr = l + overrideCluster(newOpts) + } + + // Apply flags over config file settings. + newOpts = MergeOptions(newOpts, FlagSnapshot) + + // Need more processing for boolean flags... + if FlagSnapshot != nil { + applyBoolFlags(newOpts, FlagSnapshot) + } + + setBaselineOptions(newOpts) + + // setBaselineOptions sets Port to 0 if set to -1 (RANDOM port) + // If that's the case, set it to the saved value when the accept loop was + // created. + if newOpts.Port == 0 { + newOpts.Port = clientOrgPort + } + // We don't do that for cluster, so check against -1. + if newOpts.Cluster.Port == -1 { + newOpts.Cluster.Port = clusterOrgPort + } + if newOpts.Gateway.Port == -1 { + newOpts.Gateway.Port = gatewayOrgPort + } + if newOpts.LeafNode.Port == -1 { + newOpts.LeafNode.Port = leafnodesOrgPort + } + if newOpts.Websocket.Port == -1 { + newOpts.Websocket.Port = websocketOrgPort + } + if newOpts.MQTT.Port == -1 { + newOpts.MQTT.Port = mqttOrgPort + } + + if err := s.reloadOptions(curOpts, newOpts); err != nil { + return err + } + + s.recheckPinnedCerts(curOpts, newOpts) + + s.varzMu.Lock() + s.mu.Lock() + s.configTime = time.Now().UTC() + s.updateVarzConfigReloadableFields(s.varz) + s.mu.Unlock() + s.varzMu.Unlock() + return nil +} +func applyBoolFlags(newOpts, flagOpts *Options) { + // Reset fields that may have been set to `true` in + // MergeOptions() when some of the flags default to `true` + // but have not been explicitly set and therefore value + // from config file should take precedence. + for name, val := range newOpts.inConfig { + f := reflect.ValueOf(newOpts).Elem() + names := strings.Split(name, ".") + for _, name := range names { + f = f.FieldByName(name) + } + f.SetBool(val) + } + // Now apply value (true or false) from flags that have + // been explicitly set in command line + for name, val := range flagOpts.inCmdLine { + f := reflect.ValueOf(newOpts).Elem() + names := strings.Split(name, ".") + for _, name := range names { + f = f.FieldByName(name) + } + f.SetBool(val) + } +} + +// reloadOptions reloads the server config with the provided options. If an +// option that doesn't support hot-swapping is changed, this returns an error. +func (s *Server) reloadOptions(curOpts, newOpts *Options) error { + // Apply to the new options some of the options that may have been set + // that can't be configured in the config file (this can happen in + // applications starting NATS Server programmatically). + newOpts.CustomClientAuthentication = curOpts.CustomClientAuthentication + newOpts.CustomRouterAuthentication = curOpts.CustomRouterAuthentication + + // Do the validation before checking for differences. We need to ensure + // that the new options are valid. Note that there are possible side + // effects of calling validateOptions(), in that some default values + // may be set, etc... but that should be ok since the current options + // went through the same process on startup/previous reload. + if err := validateOptions(newOpts); err != nil { + return err + } + + changed, err := s.diffOptions(newOpts) + if err != nil { + return err + } + + // Create a context that is used to pass special info that we may need + // while applying the new options. + ctx := reloadContext{oldClusterPerms: curOpts.Cluster.Permissions} + s.setOpts(newOpts) + s.applyOptions(&ctx, changed) + return nil +} + +// For the purpose of comparing, impose a order on slice data types where order does not matter +func imposeOrder(value any) error { + switch value := value.(type) { + case []*Account: + slices.SortFunc(value, func(i, j *Account) int { return cmp.Compare(i.Name, j.Name) }) + for _, a := range value { + slices.SortFunc(a.imports.streams, func(i, j *streamImport) int { return cmp.Compare(i.acc.Name, j.acc.Name) }) + } + case []*User: + slices.SortFunc(value, func(i, j *User) int { return cmp.Compare(i.Username, j.Username) }) + case []*NkeyUser: + slices.SortFunc(value, func(i, j *NkeyUser) int { return cmp.Compare(i.Nkey, j.Nkey) }) + case []*url.URL: + slices.SortFunc(value, func(i, j *url.URL) int { return cmp.Compare(i.String(), j.String()) }) + case []string: + slices.Sort(value) + case []*jwt.OperatorClaims: + slices.SortFunc(value, func(i, j *jwt.OperatorClaims) int { return cmp.Compare(i.Issuer, j.Issuer) }) + case GatewayOpts: + slices.SortFunc(value.Gateways, func(i, j *RemoteGatewayOpts) int { return cmp.Compare(i.Name, j.Name) }) + case WebsocketOpts: + slices.Sort(value.AllowedOrigins) + case string, bool, uint8, uint16, uint64, int, int32, int64, time.Duration, float64, nil, LeafNodeOpts, ClusterOpts, *tls.Config, PinnedCertSet, + *URLAccResolver, *MemAccResolver, *DirAccResolver, *CacheDirAccResolver, Authentication, MQTTOpts, jwt.TagList, + *OCSPConfig, map[string]string, map[string]bool, JSLimitOpts, StoreCipher, *OCSPResponseCacheConfig, *ProxiesConfig, WriteTimeoutPolicy: + // explicitly skipped types + case *AuthCallout: + case JSTpmOpts: + default: + // this will fail during unit tests + return fmt.Errorf("OnReload, sort or explicitly skip type: %s", + reflect.TypeOf(value)) + } + return nil +} + +// diffOptions returns a slice containing options which have been changed. If +// an option that doesn't support hot-swapping is changed, this returns an +// error. +func (s *Server) diffOptions(newOpts *Options) ([]option, error) { + var ( + oldOpts = s.getOpts() + oldConfig = reflect.ValueOf(oldOpts).Elem() + newConfig = reflect.ValueOf(newOpts).Elem() + diffOpts = []option{} + skipTKeys = len(oldOpts.TrustedOperators) > 0 && len(oldOpts.TrustedKeys) > 0 + + // Need to keep track of whether JS is being disabled + // to prevent changing limits at runtime. + jsEnabled = s.JetStreamEnabled() + disableJS bool + jsMemLimitsChanged bool + jsFileLimitsChanged bool + jsStoreDirChanged bool + jsLimitsUpdate *jetStreamLimitsOption + ) + for i := 0; i < oldConfig.NumField(); i++ { + field := oldConfig.Type().Field(i) + // field.PkgPath is empty for exported fields, and is not for unexported ones. + // We skip the unexported fields. + if field.PkgPath != _EMPTY_ { + continue + } + optName := strings.ToLower(field.Name) + if skipTKeys && optName == "trustedkeys" { + // TrustedOperators and TrustedKeys change is not supported. During options + // validation, if they are both specified, a conflict error is returned. + // If only TrustedOperators is specified, the TrustedKeys is filled with + // the operators' signing keys. So here, if we detect that the current + // options have operators, we don't do the trusted keys comparison, so + // we can fail with the "not supported for TrustedOperators" config reload + // error instead of TrustedKeys (that the user would not have set). + continue + } + var ( + oldValue = oldConfig.Field(i).Interface() + newValue = newConfig.Field(i).Interface() + ) + if err := imposeOrder(oldValue); err != nil { + return nil, err + } + if err := imposeOrder(newValue); err != nil { + return nil, err + } + + // accounts and users (referencing accounts) will always differ as accounts + // contain internal state, say locks etc..., so we don't bother here. + // This also avoids races with atomic stats counters + if optName != "accounts" && optName != "users" { + if changed := !reflect.DeepEqual(oldValue, newValue); !changed { + // Check to make sure we are running JetStream if we think we should be. + if optName == "jetstream" && newValue.(bool) { + if !jsEnabled { + diffOpts = append(diffOpts, &jetStreamOption{newValue: true}) + } + } + continue + } + } + switch optName { + case "traceverbose": + diffOpts = append(diffOpts, &traceVerboseOption{newValue: newValue.(bool)}) + case "traceheaders": + diffOpts = append(diffOpts, &traceHeadersOption{newValue: newValue.(bool)}) + case "trace": + diffOpts = append(diffOpts, &traceOption{newValue: newValue.(bool)}) + case "debug": + diffOpts = append(diffOpts, &debugOption{newValue: newValue.(bool)}) + case "logtime": + diffOpts = append(diffOpts, &logtimeOption{newValue: newValue.(bool)}) + case "logtimeutc": + diffOpts = append(diffOpts, &logtimeUTCOption{newValue: newValue.(bool)}) + case "logfile": + diffOpts = append(diffOpts, &logfileOption{newValue: newValue.(string)}) + case "syslog": + diffOpts = append(diffOpts, &syslogOption{newValue: newValue.(bool)}) + case "remotesyslog": + diffOpts = append(diffOpts, &remoteSyslogOption{newValue: newValue.(string)}) + case "tlsconfig": + diffOpts = append(diffOpts, &tlsOption{newValue: newValue.(*tls.Config)}) + case "tlstimeout": + diffOpts = append(diffOpts, &tlsTimeoutOption{newValue: newValue.(float64)}) + case "tlspinnedcerts": + diffOpts = append(diffOpts, &tlsPinnedCertOption{newValue: newValue.(PinnedCertSet)}) + case "tlshandshakefirst": + diffOpts = append(diffOpts, &tlsHandshakeFirst{newValue: newValue.(bool)}) + case "tlshandshakefirstfallback": + diffOpts = append(diffOpts, &tlsHandshakeFirstFallback{newValue: newValue.(time.Duration)}) + case "username": + diffOpts = append(diffOpts, &usernameOption{}) + case "password": + diffOpts = append(diffOpts, &passwordOption{}) + case "tags": + diffOpts = append(diffOpts, &tagsOption{}) + case "metadata": + diffOpts = append(diffOpts, &metadataOption{}) + case "authorization": + diffOpts = append(diffOpts, &authorizationOption{}) + case "authtimeout": + diffOpts = append(diffOpts, &authTimeoutOption{newValue: newValue.(float64)}) + case "users": + diffOpts = append(diffOpts, &usersOption{}) + case "nkeys": + diffOpts = append(diffOpts, &nkeysOption{}) + case "cluster": + newClusterOpts := newValue.(ClusterOpts) + oldClusterOpts := oldValue.(ClusterOpts) + if err := validateClusterOpts(oldClusterOpts, newClusterOpts); err != nil { + return nil, err + } + co := &clusterOption{ + newValue: newClusterOpts, + permsChanged: !reflect.DeepEqual(newClusterOpts.Permissions, oldClusterOpts.Permissions), + compressChanged: !oldClusterOpts.Compression.equals(&newClusterOpts.Compression), + } + co.diffPoolAndAccounts(&oldClusterOpts) + // If there are added accounts, first make sure that we can look them up. + // If we can't let's fail the reload. + for _, acc := range co.accsAdded { + if _, err := s.LookupAccount(acc); err != nil { + return nil, fmt.Errorf("unable to add account %q to the list of dedicated routes: %v", acc, err) + } + } + // If pool_size has been set to negative (but was not before), then let's + // add the system account to the list of removed accounts (we don't have + // to check if already there, duplicates are ok in that case). + if newClusterOpts.PoolSize < 0 && oldClusterOpts.PoolSize >= 0 { + if sys := s.SystemAccount(); sys != nil { + co.accsRemoved = append(co.accsRemoved, sys.GetName()) + } + } + diffOpts = append(diffOpts, co) + case "routes": + add, remove := diffRoutes(oldValue.([]*url.URL), newValue.([]*url.URL)) + diffOpts = append(diffOpts, &routesOption{add: add, remove: remove}) + case "maxconn": + diffOpts = append(diffOpts, &maxConnOption{newValue: newValue.(int)}) + case "pidfile": + diffOpts = append(diffOpts, &pidFileOption{newValue: newValue.(string)}) + case "portsfiledir": + diffOpts = append(diffOpts, &portsFileDirOption{newValue: newValue.(string), oldValue: oldValue.(string)}) + case "maxcontrolline": + diffOpts = append(diffOpts, &maxControlLineOption{newValue: newValue.(int32)}) + case "maxpayload": + diffOpts = append(diffOpts, &maxPayloadOption{newValue: newValue.(int32)}) + case "pinginterval": + diffOpts = append(diffOpts, &pingIntervalOption{newValue: newValue.(time.Duration)}) + case "maxpingsout": + diffOpts = append(diffOpts, &maxPingsOutOption{newValue: newValue.(int)}) + case "writedeadline": + diffOpts = append(diffOpts, &writeDeadlineOption{newValue: newValue.(time.Duration)}) + case "clientadvertise": + cliAdv := newValue.(string) + if cliAdv != "" { + // Validate ClientAdvertise syntax + if _, _, err := parseHostPort(cliAdv, 0); err != nil { + return nil, fmt.Errorf("invalid ClientAdvertise value of %s, err=%v", cliAdv, err) + } + } + diffOpts = append(diffOpts, &clientAdvertiseOption{newValue: cliAdv}) + case "accounts": + diffOpts = append(diffOpts, &accountsOption{}) + case "resolver", "accountresolver", "accountsresolver": + // We can't move from no resolver to one. So check for that. + if (oldValue == nil && newValue != nil) || + (oldValue != nil && newValue == nil) { + return nil, fmt.Errorf("config reload does not support moving to or from an account resolver") + } + diffOpts = append(diffOpts, &accountsOption{}) + case "accountresolvertlsconfig": + diffOpts = append(diffOpts, &accountsOption{}) + case "gateway": + // Not supported for now, but report warning if configuration of gateway + // is actually changed so that user knows that it won't take effect. + + // Any deep-equal is likely to fail for when there is a TLSConfig. so + // remove for the test. + tmpOld := oldValue.(GatewayOpts) + tmpNew := newValue.(GatewayOpts) + tmpOld.TLSConfig = nil + tmpNew.TLSConfig = nil + tmpOld.tlsConfigOpts = nil + tmpNew.tlsConfigOpts = nil + + // Allow TLSPinnedCerts through reload, existing connections + // are checked in recheckPinnedCerts + tmpOld.TLSPinnedCerts = nil + tmpNew.TLSPinnedCerts = nil + + // Need to do the same for remote gateways' TLS configs. + // But we can't just set remotes' TLSConfig to nil otherwise this + // would lose the real TLS configuration. + tmpOld.Gateways = copyRemoteGWConfigsWithoutTLSConfig(tmpOld.Gateways) + tmpNew.Gateways = copyRemoteGWConfigsWithoutTLSConfig(tmpNew.Gateways) + + // If there is really a change prevents reload. + if !reflect.DeepEqual(tmpOld, tmpNew) { + // See TODO(ik) note below about printing old/new values. + return nil, fmt.Errorf("config reload not supported for %s: old=%v, new=%v", + field.Name, oldValue, newValue) + } + case "leafnode": + tmpOld := oldValue.(LeafNodeOpts) + tmpNew := newValue.(LeafNodeOpts) + + lno, err := getLeafNodeOptionsChanges(s, &tmpOld, &tmpNew) + // If there was an unsupported change, we will get an error with the name + // of the (first) field and its old and new value. + if err != nil { + return nil, fmt.Errorf("config reload not supported for %s: %v", field.Name, err) + } + // If there was an actual change... + if lno != nil { + diffOpts = append(diffOpts, lno) + } + case "jetstream": + new := newValue.(bool) + old := oldValue.(bool) + if new != old { + diffOpts = append(diffOpts, &jetStreamOption{newValue: new}) + } + + // Mark whether JS will be disabled. + disableJS = !new + case "storedir": + new := newValue.(string) + old := oldValue.(string) + modified := new != old + + // Check whether JS is being disabled and/or storage dir attempted to change. + if jsEnabled && modified { + if new == _EMPTY_ { + // This means that either JS is being disabled or it is using an temp dir. + // Allow the change but error in case JS was not disabled. + jsStoreDirChanged = true + } else { + return nil, fmt.Errorf("config reload not supported for jetstream storage directory") + } + } + case "jetstreammaxmemory", "jetstreammaxstore": + old := oldValue.(int64) + new := newValue.(int64) + + // Check whether JS is being disabled and/or limits are being changed. + var ( + modified = new != old + fromUnset = old == -1 + fromSet = !fromUnset + toUnset = new == -1 + toSet = !toUnset + increased = fromSet && toSet && new > old + ) + if jsEnabled && modified { + // Cannot change limits from dynamic storage at runtime. + switch { + case increased: + // Allowed to increase, but not decrease. + if jsLimitsUpdate == nil { + jsLimitsUpdate = &jetStreamLimitsOption{} + diffOpts = append(diffOpts, jsLimitsUpdate) + } + if optName == "jetstreammaxmemory" { + jsLimitsUpdate.newMaxMemory = new + } else { + jsLimitsUpdate.newMaxStore = new + } + case fromSet && toUnset: + // Limits changed but it may mean that JS is being disabled, + // keep track of the change and error in case it is not. + if optName == "jetstreammaxmemory" { + jsMemLimitsChanged = true + } else { + jsFileLimitsChanged = true + } + case fromUnset && toSet: + // Prevent changing from dynamic max memory / file at runtime. + return nil, fmt.Errorf("config reload not supported for jetstream dynamic max memory and store") + default: + return nil, fmt.Errorf("config reload not supported for decreasing jetstream max memory and store") + } + } + case "jetstreamlimits": + // Server-wide JetStream limits are enforced when assets are created or updated. + // Some limits, like MaxRequestBatch, MaxAckPending, Duplicates, are materialized + // into the asset's config, so existing assets keep the value they had originally. + new := newValue.(JSLimitOpts) + old := oldValue.(JSLimitOpts) + if new != old { + diffOpts = append(diffOpts, &jetStreamServerLimitsOption{newValue: new}) + } + case "jetstreammetacompact", "jetstreammetacompactsize", "jetstreammetacompactsync": + // Allowed at runtime but monitorCluster looks at s.opts directly, so no further work needed here. + case "jetstreamconcurrentios": + // Not reloadable at runtime; preserve the current value while JetStream is disabled, + // e.g. the entire jetstream{} block was deleted. + if newOpts.JetStream { + return nil, fmt.Errorf("config reload not supported for %s: old=%v, new=%v", + field.Name, oldValue, newValue) + } else { + newOpts.JetStreamConcurrentIOs = oldValue.(int) + } + case "websocket": + // Similar to gateways + tmpOld := oldValue.(WebsocketOpts) + tmpNew := newValue.(WebsocketOpts) + tmpOld.TLSConfig, tmpOld.tlsConfigOpts = nil, nil + tmpNew.TLSConfig, tmpNew.tlsConfigOpts = nil, nil + // If there is really a change prevents reload. + if !reflect.DeepEqual(tmpOld, tmpNew) { + // See TODO(ik) note below about printing old/new values. + return nil, fmt.Errorf("config reload not supported for %s: old=%v, new=%v", + field.Name, oldValue, newValue) + } + case "mqtt": + diffOpts = append(diffOpts, &mqttAckWaitReload{newValue: newValue.(MQTTOpts).AckWait}) + diffOpts = append(diffOpts, &mqttMaxAckPendingReload{newValue: newValue.(MQTTOpts).MaxAckPending}) + diffOpts = append(diffOpts, &mqttStreamReplicasReload{newValue: newValue.(MQTTOpts).StreamReplicas}) + diffOpts = append(diffOpts, &mqttConsumerReplicasReload{newValue: newValue.(MQTTOpts).ConsumerReplicas}) + diffOpts = append(diffOpts, &mqttConsumerMemoryStorageReload{newValue: newValue.(MQTTOpts).ConsumerMemoryStorage}) + diffOpts = append(diffOpts, &mqttInactiveThresholdReload{newValue: newValue.(MQTTOpts).ConsumerInactiveThreshold}) + + // Nil out/set to 0 the options that we allow to be reloaded so that + // we only fail reload if some that we don't support are changed. + tmpOld := oldValue.(MQTTOpts) + tmpNew := newValue.(MQTTOpts) + tmpOld.TLSConfig, tmpOld.tlsConfigOpts, tmpOld.AckWait, tmpOld.MaxAckPending, tmpOld.StreamReplicas, tmpOld.ConsumerReplicas, tmpOld.ConsumerMemoryStorage = nil, nil, 0, 0, 0, 0, false + tmpOld.ConsumerInactiveThreshold = 0 + tmpNew.TLSConfig, tmpNew.tlsConfigOpts, tmpNew.AckWait, tmpNew.MaxAckPending, tmpNew.StreamReplicas, tmpNew.ConsumerReplicas, tmpNew.ConsumerMemoryStorage = nil, nil, 0, 0, 0, 0, false + tmpNew.ConsumerInactiveThreshold = 0 + + if !reflect.DeepEqual(tmpOld, tmpNew) { + // See TODO(ik) note below about printing old/new values. + return nil, fmt.Errorf("config reload not supported for %s: old=%v, new=%v", + field.Name, oldValue, newValue) + } + tmpNew.AckWait = newValue.(MQTTOpts).AckWait + tmpNew.MaxAckPending = newValue.(MQTTOpts).MaxAckPending + tmpNew.StreamReplicas = newValue.(MQTTOpts).StreamReplicas + tmpNew.ConsumerReplicas = newValue.(MQTTOpts).ConsumerReplicas + tmpNew.ConsumerMemoryStorage = newValue.(MQTTOpts).ConsumerMemoryStorage + tmpNew.ConsumerInactiveThreshold = newValue.(MQTTOpts).ConsumerInactiveThreshold + case "connecterrorreports": + diffOpts = append(diffOpts, &connectErrorReports{newValue: newValue.(int)}) + case "reconnecterrorreports": + diffOpts = append(diffOpts, &reconnectErrorReports{newValue: newValue.(int)}) + case "nolog", "nosigs": + // Ignore NoLog and NoSigs options since they are not parsed and only used in + // testing. + continue + case "disableshortfirstping": + newOpts.DisableShortFirstPing = oldValue.(bool) + continue + case "maxtracedmsglen": + diffOpts = append(diffOpts, &maxTracedMsgLenOption{newValue: newValue.(int)}) + case "port": + // check to see if newValue == 0 and continue if so. + if newValue == 0 { + // ignore RANDOM_PORT + continue + } + fallthrough + case "noauthuser": + if oldValue != _EMPTY_ && newValue == _EMPTY_ { + for _, user := range newOpts.Users { + if user.Username == oldValue { + return nil, fmt.Errorf("config reload not supported for %s: old=%v, new=%v", + field.Name, oldValue, newValue) + } + } + } else { + return nil, fmt.Errorf("config reload not supported for %s: old=%v, new=%v", + field.Name, oldValue, newValue) + } + case "defaultsentinel": + diffOpts = append(diffOpts, &defaultSentinelOption{newValue: newValue.(string)}) + case "systemaccount": + if oldValue != DEFAULT_SYSTEM_ACCOUNT || newValue != _EMPTY_ { + return nil, fmt.Errorf("config reload not supported for %s: old=%v, new=%v", + field.Name, oldValue, newValue) + } + case "ocspconfig": + diffOpts = append(diffOpts, &ocspOption{newValue: newValue.(*OCSPConfig)}) + case "ocspcacheconfig": + diffOpts = append(diffOpts, &ocspResponseCacheOption{newValue: newValue.(*OCSPResponseCacheConfig)}) + case "profblockrate": + new := newValue.(int) + old := oldValue.(int) + if new != old { + diffOpts = append(diffOpts, &profBlockRateReload{newValue: new}) + } + case "configdigest": + // skip changes in config digest, this is handled already while + // processing the config. + continue + case "nofastproducerstall": + diffOpts = append(diffOpts, &noFastProdStallReload{noStall: newValue.(bool)}) + case "proxies": + new := newValue.(*ProxiesConfig) + old := oldValue.(*ProxiesConfig) + if add, del := diffProxiesTrustedKeys(old.Trusted, new.Trusted); len(add) > 0 || len(del) > 0 { + diffOpts = append(diffOpts, &proxiesReload{add: add, del: del}) + } + default: + // TODO(ik): Implement String() on those options to have a nice print. + // %v is difficult to figure what's what, %+v print private fields and + // would print passwords. Tried json.Marshal but it is too verbose for + // the URL array. + + // Bail out if attempting to reload any unsupported options. + return nil, fmt.Errorf("config reload not supported for %s: old=%v, new=%v", + field.Name, oldValue, newValue) + } + } + + // If not disabling JS but limits have changed then it is an error. + if !disableJS { + if jsMemLimitsChanged || jsFileLimitsChanged { + return nil, fmt.Errorf("config reload not supported for jetstream max memory and max store") + } + if jsStoreDirChanged { + return nil, fmt.Errorf("config reload not supported for jetstream storage dir") + } + } + + return diffOpts, nil +} + +func copyRemoteGWConfigsWithoutTLSConfig(current []*RemoteGatewayOpts) []*RemoteGatewayOpts { + l := len(current) + if l == 0 { + return nil + } + rgws := make([]*RemoteGatewayOpts, 0, l) + for _, rcfg := range current { + cp := *rcfg + cp.TLSConfig = nil + cp.tlsConfigOpts = nil + rgws = append(rgws, &cp) + } + return rgws +} + +func (s *Server) applyOptions(ctx *reloadContext, opts []option) { + var ( + reloadLogging = false + reloadAuth = false + reloadClusterPerms = false + reloadClientTrcLvl = false + reloadJetstream = false + jsEnabled = false + isStatszChange = false + co *clusterOption + ) + for _, opt := range opts { + opt.Apply(s) + if opt.IsLoggingChange() { + reloadLogging = true + } + if opt.IsTraceLevelChange() { + reloadClientTrcLvl = true + } + if opt.IsAuthChange() { + reloadAuth = true + } + if opt.IsClusterPoolSizeOrAccountsChange() { + co = opt.(*clusterOption) + } + if opt.IsClusterPermsChange() { + reloadClusterPerms = true + } + if opt.IsJetStreamChange() { + reloadJetstream = true + jsEnabled = opt.(*jetStreamOption).newValue + } + if opt.IsStatszChange() { + isStatszChange = true + } + } + + if reloadLogging { + s.ConfigureLogger() + } + if reloadClientTrcLvl { + s.reloadClientTraceLevel() + } + if reloadAuth { + s.reloadAuthorization() + } + if reloadClusterPerms { + s.reloadClusterPermissions(ctx.oldClusterPerms) + } + newOpts := s.getOpts() + // If we need to reload cluster pool/per-account, then co will be not nil + if co != nil { + s.reloadClusterPoolAndAccounts(co, newOpts) + } + if reloadJetstream { + if !jsEnabled { + s.DisableJetStream() + } else if !s.JetStreamEnabled() { + if err := s.restartJetStream(); err != nil { + s.Warnf("Can't start JetStream: %v", err) + } + } + // Make sure to reset the internal loop's version of JS. + s.resetInternalLoopInfo() + } + if isStatszChange { + s.sendStatszUpdate() + } + + // For remote gateways, make sure that their TLS configuration + // is updated (since the config is "captured" early and changes would otherwise + // not be visible). + if s.gateway.enabled { + s.gateway.updateRemotesTLSConfig(newOpts) + } + + // Always restart OCSP monitoring on reload. + if err := s.reloadOCSP(); err != nil { + s.Warnf("Can't restart OCSP features: %v", err) + } + var cd string + if newOpts.configDigest != "" { + cd = fmt.Sprintf("(%s)", newOpts.configDigest) + } + s.Noticef("Reloaded server configuration %s", cd) +} + +// This will send a reset to the internal send loop. +func (s *Server) resetInternalLoopInfo() { + var resetCh chan struct{} + s.mu.Lock() + if s.sys != nil { + // can't hold the lock as go routine reading it may be waiting for lock as well + resetCh = s.sys.resetCh + } + s.mu.Unlock() + + if resetCh != nil { + select { + case resetCh <- struct{}{}: + case <-s.quitCh: + } + } +} + +// Update all cached debug and trace settings for every client +func (s *Server) reloadClientTraceLevel() { + opts := s.getOpts() + + if opts.NoLog { + return + } + + // Create a list of all clients. + // Update their trace level when not holding server or gateway lock + + s.mu.Lock() + clientCnt := 1 + len(s.clients) + len(s.grTmpClients) + s.numRoutes() + len(s.leafs) + s.mu.Unlock() + + s.gateway.RLock() + clientCnt += len(s.gateway.in) + len(s.gateway.outo) + s.gateway.RUnlock() + + clients := make([]*client, 0, clientCnt) + + s.mu.Lock() + if s.eventsEnabled() { + clients = append(clients, s.sys.client) + } + + cMaps := []map[uint64]*client{s.clients, s.grTmpClients, s.leafs} + for _, m := range cMaps { + for _, c := range m { + clients = append(clients, c) + } + } + s.forEachRoute(func(c *client) { + clients = append(clients, c) + }) + s.mu.Unlock() + + s.gateway.RLock() + for _, c := range s.gateway.in { + clients = append(clients, c) + } + clients = append(clients, s.gateway.outo...) + s.gateway.RUnlock() + + for _, c := range clients { + // client.trace is commonly read while holding the lock + c.mu.Lock() + c.setTraceLevel() + c.mu.Unlock() + } +} + +// reloadAuthorization reconfigures the server authorization settings, +// disconnects any clients who are no longer authorized, and removes any +// unauthorized subscriptions. +func (s *Server) reloadAuthorization() { + // This map will contain the names of accounts that have their streams + // import configuration changed. + var awcsti map[string]struct{} + checkJetStream := false + opts := s.getOpts() + s.mu.Lock() + + deletedAccounts := make(map[string]*Account) + + // This can not be changed for now so ok to check server's trustedKeys unlocked. + // If plain configured accounts, process here. + if s.trustedKeys == nil { + // Make a map of the configured account names so we figure out the accounts + // that should be removed later on. + configAccs := make(map[string]struct{}, len(opts.Accounts)) + for _, acc := range opts.Accounts { + configAccs[acc.GetName()] = struct{}{} + } + // Now range over existing accounts and keep track of the ones deleted + // so some cleanup can be made after releasing the server lock. + s.accounts.Range(func(k, v any) bool { + an, acc := k.(string), v.(*Account) + // Exclude default and system account from this test since those + // may not actually be in opts.Accounts. + if an == DEFAULT_GLOBAL_ACCOUNT || an == DEFAULT_SYSTEM_ACCOUNT { + return true + } + // Check check if existing account is still in opts.Accounts. + if _, ok := configAccs[an]; !ok { + deletedAccounts[an] = acc + s.accounts.Delete(k) + } + return true + }) + // This will update existing and add new ones. + awcsti, _ = s.configureAccounts(true) + s.configureAuthorization() + // Double check any JetStream configs. + checkJetStream = s.getJetStream() != nil + } else if opts.AccountResolver != nil { + s.configureResolver() + if _, ok := s.accResolver.(*MemAccResolver); ok { + // Check preloads so we can issue warnings etc if needed. + s.checkResolvePreloads() + // With a memory resolver we want to do something similar to configured accounts. + // We will walk the accounts and delete them if they are no longer present via fetch. + // If they are present we will force a claim update to process changes. + s.accounts.Range(func(k, v any) bool { + acc := v.(*Account) + // Skip global account. + if acc == s.gacc { + return true + } + accName := acc.GetName() + // Release server lock for following actions + s.mu.Unlock() + accClaims, claimJWT, _ := s.fetchAccountClaims(accName) + if accClaims != nil { + if err := s.updateAccountWithClaimJWT(acc, claimJWT); err != nil { + s.Noticef("Reloaded: deleting account [bad claims]: %q", accName) + s.accounts.Delete(k) + } + } else { + s.Noticef("Reloaded: deleting account [removed]: %q", accName) + s.accounts.Delete(k) + } + // Regrab server lock. + s.mu.Lock() + return true + }) + } + } + + var ( + cclientsa [64]*client + cclients = cclientsa[:0] + clientsa [64]*client + clients = clientsa[:0] + routesa [64]*client + routes = routesa[:0] + ) + + // Gather clients that changed accounts. We will close them and they + // will reconnect, doing the right thing. + for _, client := range s.clients { + if s.clientHasMovedToDifferentAccount(client) { + cclients = append(cclients, client) + } else { + clients = append(clients, client) + } + } + s.forEachRoute(func(route *client) { + routes = append(routes, route) + }) + // Check here for any system/internal clients which will not be in the servers map of normal clients. + if s.sys != nil && s.sys.account != nil && !opts.NoSystemAccount { + s.accounts.Store(s.sys.account.Name, s.sys.account) + } + + s.accounts.Range(func(k, v any) bool { + acc := v.(*Account) + acc.mu.RLock() + // Check for sysclients accounting, ignore the system account. + if acc.sysclients > 0 && (s.sys == nil || s.sys.account != acc) { + for c := range acc.clients { + if c.kind != CLIENT && c.kind != LEAF { + clients = append(clients, c) + } + } + } + acc.mu.RUnlock() + return true + }) + + var resetCh chan struct{} + if s.sys != nil { + // can't hold the lock as go routine reading it may be waiting for lock as well + resetCh = s.sys.resetCh + } + s.mu.Unlock() + + // Clear some timers and remove service import subs for deleted accounts. + for _, acc := range deletedAccounts { + acc.mu.Lock() + clearTimer(&acc.etmr) + clearTimer(&acc.ctmr) + for _, se := range acc.exports.services { + se.clearResponseThresholdTimer() + } + acc.mu.Unlock() + acc.removeAllServiceImportSubs() + } + + if resetCh != nil { + select { + case resetCh <- struct{}{}: + case <-s.quitCh: + } + } + + // Close clients that have moved accounts + for _, client := range cclients { + client.closeConnection(ClientClosed) + } + + for _, c := range clients { + // Disconnect any unauthorized clients. + // Ignore internal clients. + if (c.kind == CLIENT || c.kind == LEAF) && !s.isClientAuthorized(c) { + c.authViolation() + continue + } + // Check to make sure account is correct. + c.swapAccountAfterReload() + // Remove any unauthorized subscriptions and check for account imports. + c.processSubsOnConfigReload(awcsti) + } + + for _, route := range routes { + // Disconnect any unauthorized routes. + // Do this only for routes that were accepted, not initiated + // because in the later case, we don't have the user name/password + // of the remote server. + if !route.isSolicitedRoute() && !s.isRouterAuthorized(route) { + route.setNoReconnect() + route.authViolation() + } + } + + if res := s.AccountResolver(); res != nil { + res.Reload() + } + + // We will double check all JetStream configs on a reload. + if checkJetStream { + if err := s.enableJetStreamAccounts(); err != nil { + s.Errorf(err.Error()) + } + } + + // Check that publish retained messages sources are still allowed to publish. + // Do this after dealing with JetStream. + s.mqttCheckPubRetainedPerms() +} + +// Returns true if given client current account has changed (or user +// no longer exist) in the new config, false if the user did not +// change accounts. +// Server lock is held on entry. +func (s *Server) clientHasMovedToDifferentAccount(c *client) bool { + var ( + nu *NkeyUser + u *User + ) + c.mu.Lock() + defer c.mu.Unlock() + if c.opts.Nkey != _EMPTY_ { + if s.nkeys != nil { + nu = s.nkeys[c.opts.Nkey] + } + } else if c.opts.Username != _EMPTY_ { + if s.users != nil { + u = s.users[c.opts.Username] + } + } else { + return false + } + // Get the current account name + var curAccName string + if c.acc != nil { + curAccName = c.acc.Name + } + if nu != nil && nu.Account != nil { + return curAccName != nu.Account.Name + } else if u != nil && u.Account != nil { + return curAccName != u.Account.Name + } + // user/nkey no longer exists. + return true +} + +// reloadClusterPermissions reconfigures the cluster's permssions +// and set the permissions to all existing routes, sending an +// update INFO protocol so that remote can resend their local +// subs if needed, and sending local subs matching cluster's +// import subjects. +func (s *Server) reloadClusterPermissions(oldPerms *RoutePermissions) { + s.mu.Lock() + newPerms := s.getOpts().Cluster.Permissions + routes := make(map[uint64]*client, s.numRoutes()) + // Get all connected routes + s.forEachRoute(func(route *client) { + route.mu.Lock() + routes[route.cid] = route + route.mu.Unlock() + }) + // If new permissions is nil, then clear routeInfo import/export + if newPerms == nil { + s.routeInfo.Import = nil + s.routeInfo.Export = nil + } else { + s.routeInfo.Import = newPerms.Import + s.routeInfo.Export = newPerms.Export + } + infoJSON := generateInfoJSON(&s.routeInfo) + s.mu.Unlock() + + // Close connections for routes that don't understand async INFO. + for _, route := range routes { + route.mu.Lock() + close := route.opts.Protocol < RouteProtoInfo + cid := route.cid + route.mu.Unlock() + if close { + route.closeConnection(RouteRemoved) + delete(routes, cid) + } + } + + // If there are no route left, we are done + if len(routes) == 0 { + return + } + + // Fake clients to test cluster permissions + oldPermsTester := &client{} + oldPermsTester.setRoutePermissions(oldPerms) + newPermsTester := &client{} + newPermsTester.setRoutePermissions(newPerms) + + var ( + _localSubs [4096]*subscription + subsNeedSUB = map[*client][]*subscription{} + subsNeedUNSUB = map[*client][]*subscription{} + deleteRoutedSubs []*subscription + ) + + getRouteForAccount := func(accName string, poolIdx int) *client { + for _, r := range routes { + r.mu.Lock() + ok := (poolIdx >= 0 && poolIdx == r.route.poolIdx) || (string(r.route.accName) == accName) || r.route.noPool + r.mu.Unlock() + if ok { + return r + } + } + return nil + } + + // First set the new permissions on all routes. + for _, route := range routes { + route.mu.Lock() + route.setRoutePermissions(newPerms) + route.mu.Unlock() + } + + // Then, go over all accounts and gather local subscriptions that need to be + // sent over as SUB or removed as UNSUB, and routed subscriptions that need + // to be dropped due to export permissions. + s.accounts.Range(func(_, v any) bool { + acc := v.(*Account) + acc.mu.RLock() + accName, sl, poolIdx := acc.Name, acc.sl, acc.routePoolIdx + acc.mu.RUnlock() + // Get the route handling this account. If no route or sublist, bail out. + route := getRouteForAccount(accName, poolIdx) + if route == nil || sl == nil { + return true + } + localSubs := _localSubs[:0] + sl.localSubs(&localSubs, false) + + // Go through all local subscriptions + for _, sub := range localSubs { + // Get all subs that can now be imported + subj := string(sub.subject) + couldImportThen := oldPermsTester.canImport(subj) + canImportNow := newPermsTester.canImport(subj) + if canImportNow { + // If we could not before, then will need to send a SUB protocol. + if !couldImportThen { + subsNeedSUB[route] = append(subsNeedSUB[route], sub) + } + } else if couldImportThen { + // We were previously able to import this sub, but now + // we can't so we need to send an UNSUB protocol + subsNeedUNSUB[route] = append(subsNeedUNSUB[route], sub) + } + } + deleteRoutedSubs = deleteRoutedSubs[:0] + route.mu.Lock() + pa, _, hasSubType := route.getRoutedSubKeyInfo() + for key, sub := range route.subs { + // If this is not a pinned-account route, we need to get the + // account name from the key to see if we collect this sub. + if !pa { + if an := getAccNameFromRoutedSubKey(sub, key, hasSubType); an != accName { + continue + } + } + // If we can't export, we need to drop the subscriptions that + // we have on behalf of this route. + // Need to make a string cast here since canExport call sl.Match() + subj := string(sub.subject) + if !route.canExport(subj) { + // We can use bytesToString() here. + delete(route.subs, bytesToString(sub.sid)) + deleteRoutedSubs = append(deleteRoutedSubs, sub) + } + } + route.mu.Unlock() + // Remove as a batch all the subs that we have removed from each route. + sl.RemoveBatch(deleteRoutedSubs) + return true + }) + + // Send an update INFO, which will allow remote server to show + // our current route config in monitoring and resend subscriptions + // that we now possibly allow with a change of Export permissions. + for _, route := range routes { + route.mu.Lock() + route.enqueueProto(infoJSON) + // Now send SUB and UNSUB protocols as needed. + if subs, ok := subsNeedSUB[route]; ok && len(subs) > 0 { + route.sendRouteSubProtos(subs, false, nil) + } + if unsubs, ok := subsNeedUNSUB[route]; ok && len(unsubs) > 0 { + route.sendRouteUnSubProtos(unsubs, false, nil) + } + route.mu.Unlock() + } +} + +func (s *Server) reloadClusterPoolAndAccounts(co *clusterOption, opts *Options) { + s.mu.Lock() + // Prevent adding new routes until we are ready to do so. + s.routesReject = true + var ch chan struct{} + // Number of per-account route INFO protocols sent that we expect a + // confirmation for. Declared here so the wait below can block on it. + protosSent := 0 + // For accounts that have been added to the list of dedicated routes, + // send a protocol to their current assigned routes to allow the + // other side to prepare for the changes. + if len(co.accsAdded) > 0 { + s.accAddedReqID = nuid.Next() + for _, an := range co.accsAdded { + if s.accRoutes == nil { + s.accRoutes = make(map[string]map[string]*client) + } + // In case a config reload was first done on another server, + // we may have already switched this account to a dedicated route. + // But we still want to send the protocol over the routes that + // would have otherwise handled it. + if _, ok := s.accRoutes[an]; !ok { + s.accRoutes[an] = make(map[string]*client) + } + if a, ok := s.accounts.Load(an); ok { + acc := a.(*Account) + acc.mu.Lock() + sl := acc.sl + // Get the current route pool index before calling setRouteInfo. + rpi := acc.routePoolIdx + // Switch to per-account route if not already done. + if rpi >= 0 { + s.setRouteInfo(acc) + } else { + // If it was transitioning, make sure we set it to the state + // that indicates that it has a dedicated route + if rpi == accTransitioningToDedicatedRoute { + acc.routePoolIdx = accDedicatedRoute + } + // Otherwise get the route pool index it would have been before + // the move so we can send the protocol to those routes. + rpi = computeRoutePoolIdx(s.routesPoolSize, acc.Name) + } + acc.mu.Unlock() + // Generate the INFO protocol to send indicating that this account + // is being moved to a dedicated route. + ri := Info{ + RoutePoolSize: s.routesPoolSize, + RouteAccount: an, + RouteAccReqID: s.accAddedReqID, + } + proto := generateInfoJSON(&ri) + // Since v2.11.0, we support remotes with a different pool size + // (for rolling upgrades), so we need to use the remote route + // pool index (based on the remote configured pool size) since + // the remote subscriptions will be attached to the route at + // that index, not at our account's route pool index. However, + // we are going to send the protocol through the route that + // handles this account from our pool size perspective (that + // would be the route at index `rpi`). + removeSubsAndSendProto := func(r *client, doSubs, doProto bool) { + r.mu.Lock() + defer r.mu.Unlock() + // Exclude routes to servers that don't support pooling. + if r.route.noPool { + return + } + if doSubs { + if subs := r.removeRemoteSubsForAcc(an); len(subs) > 0 { + sl.RemoveBatch(subs) + } + } + if doProto { + r.enqueueProto(proto) + protosSent++ + } + } + for remote, conns := range s.routes { + r := conns[rpi] + // The route connection at this index is currently not up, + // so we won't be able to send the protocol, so move to the + // next remote. + if r == nil { + continue + } + doSubs := true + // Check the remote's route pool size and if different than + // ours, remove the subs on that other route. + remotePoolSize, ok := s.remoteRoutePoolSize[remote] + if ok && remotePoolSize != s.routesPoolSize { + // This is the remote's route pool index for this account + rrpi := computeRoutePoolIdx(remotePoolSize, an) + if rr := conns[rrpi]; rr != nil { + removeSubsAndSendProto(rr, true, false) + // Indicate that we have already remove the subs. + doSubs = false + } + } + // Now send the protocol from the route that handles the + // account from this server perspective. + removeSubsAndSendProto(r, doSubs, true) + } + } + } + if protosSent > 0 { + s.accAddedCh = make(chan struct{}, protosSent) + ch = s.accAddedCh + } + } + // Collect routes that need to be closed. + routes := make(map[*client]struct{}) + // Collect the per-account routes that need to be closed. + if len(co.accsRemoved) > 0 { + for _, an := range co.accsRemoved { + if remotes, ok := s.accRoutes[an]; ok && remotes != nil { + for _, r := range remotes { + if r != nil { + r.setNoReconnect() + routes[r] = struct{}{} + } + } + } + } + } + // If the pool size has changed, we need to close all pooled routes. + if co.poolSizeChanged { + s.forEachNonPerAccountRoute(func(r *client) { + routes[r] = struct{}{} + }) + } + // If there are routes to close, we need to release the server lock. + // Same if we need to wait on responses from the remotes when + // processing new per-account routes. + if len(routes) > 0 || protosSent > 0 { + s.mu.Unlock() + + // Wait for the expected number of confirmations from the remotes. + for received := 0; received < protosSent; { + select { + case <-ch: + received++ + case <-time.After(2 * time.Second): + s.Warnf("Timed out waiting for confirmation from all routes regarding per-account routes changes") + received = protosSent + } + } + + for r := range routes { + r.closeConnection(RouteRemoved) + } + + s.mu.Lock() + } + // Clear the accAddedCh/ReqID fields in case they were set. + s.accAddedReqID, s.accAddedCh = _EMPTY_, nil + // Now that per-account routes that needed to be closed are closed, + // remove them from s.accRoutes. Doing so before would prevent + // removeRoute() to do proper cleanup because the route would not + // be found in s.accRoutes. + for _, an := range co.accsRemoved { + delete(s.accRoutes, an) + // Do not lookup and call setRouteInfo() on the accounts here. + // We need first to set the new s.routesPoolSize value and + // anyway, there is no need to do here if the pool size has + // changed (since it will be called for all accounts). + } + // We have already added the accounts to s.accRoutes that needed to + // be added. + + // We should always have at least the system account with a dedicated route, + // but in case we have a configuration that disables pooling and without + // a system account, possibly set the accRoutes to nil. + if len(opts.Cluster.PinnedAccounts) == 0 { + s.accRoutes = nil + } + // Now deal with pool size updates. + if ps := opts.Cluster.PoolSize; ps > 0 { + s.routesPoolSize = ps + s.routeInfo.RoutePoolSize = ps + } else { + s.routesPoolSize = 1 + s.routeInfo.RoutePoolSize = 0 + } + // If the pool size has changed, we need to recompute all accounts' route + // pool index. Note that the added/removed accounts will be reset there + // too, but that's ok (we could use a map to exclude them, but not worth it). + if co.poolSizeChanged { + s.accounts.Range(func(_, v any) bool { + acc := v.(*Account) + acc.mu.Lock() + s.setRouteInfo(acc) + acc.mu.Unlock() + return true + }) + } else if len(co.accsRemoved) > 0 { + // For accounts that no longer have a dedicated route, we need to send + // the subsriptions on the existing pooled routes for those accounts. + for _, an := range co.accsRemoved { + if a, ok := s.accounts.Load(an); ok { + acc := a.(*Account) + acc.mu.Lock() + // First call this which will assign a new route pool index. + s.setRouteInfo(acc) + // Get the value so we can send the subscriptions interest + // on all routes with this pool index. + rpi := acc.routePoolIdx + acc.mu.Unlock() + s.forEachRouteIdx(rpi, func(r *client) bool { + // We have the guarantee that if the route exists, it + // is not a new one that would have been created when + // we released the server lock if some routes needed + // to be closed, because we have set s.routesReject + // to `true` at the top of this function. + s.sendSubsToRoute(r, rpi, an) + return true + }) + } + } + } + // Allow routes to be accepted now. + s.routesReject = false + // If there is a pool size change or added accounts, solicit routes now. + if co.poolSizeChanged || len(co.accsAdded) > 0 { + s.solicitRoutes(opts.Routes, co.accsAdded) + } + s.mu.Unlock() +} + +// validateClusterOpts ensures the new ClusterOpts does not change some of the +// fields that do not support reload. +func validateClusterOpts(old, new ClusterOpts) error { + if old.Host != new.Host { + return fmt.Errorf("config reload not supported for cluster host: old=%s, new=%s", + old.Host, new.Host) + } + if old.Port != new.Port { + return fmt.Errorf("config reload not supported for cluster port: old=%d, new=%d", + old.Port, new.Port) + } + // Validate Cluster.Advertise syntax + if new.Advertise != "" { + if _, _, err := parseHostPort(new.Advertise, 0); err != nil { + return fmt.Errorf("invalid Cluster.Advertise value of %s, err=%v", new.Advertise, err) + } + } + return nil +} + +// diffRoutes diffs the old routes and the new routes and returns the ones that +// should be added and removed from the server. +func diffRoutes(old, new []*url.URL) (add, remove []*url.URL) { + // Find routes to remove. +removeLoop: + for _, oldRoute := range old { + for _, newRoute := range new { + if urlsAreEqual(oldRoute, newRoute) { + continue removeLoop + } + } + remove = append(remove, oldRoute) + } + + // Find routes to add. +addLoop: + for _, newRoute := range new { + for _, oldRoute := range old { + if urlsAreEqual(oldRoute, newRoute) { + continue addLoop + } + } + add = append(add, newRoute) + } + + return add, remove +} + +func diffProxiesTrustedKeys(old, new []*ProxyConfig) ([]string, []string) { + var add []string + var del []string + // Both "old" and "new" lists should be small... + for _, op := range old { + if !slices.ContainsFunc(new, func(pc *ProxyConfig) bool { + return pc.Key == op.Key + }) { + del = append(del, op.Key) + } + } + for _, np := range new { + if !slices.ContainsFunc(old, func(pc *ProxyConfig) bool { + return pc.Key == np.Key + }) { + add = append(add, np.Key) + } + } + return add, del +} + +// This function calls `reflect.DeepEqual` on all public fields that are +// not part of the `ignoreFields` list. If they are all equal, returns nil, +// otherwise returns an error that will contain the name of the first field +// that fails the comparison, along with its old and new values. +func checkConfigsEqual(c1, c2 any, ignoreFields []string) error { + oldConfig := reflect.ValueOf(c1).Elem() + newConfig := reflect.ValueOf(c2).Elem() + for i := 0; i < oldConfig.NumField(); i++ { + field := oldConfig.Type().Field(i) + // field.PkgPath is empty for exported fields, and is not for unexported ones. + // We skip the unexported fields. + if field.PkgPath != _EMPTY_ { + continue + } + // If it is in the set of fields to ignore, move to the next. + // We expect the number of ignore fields to be small. + var ignored bool + for _, f := range ignoreFields { + if f == field.Name { + ignored = true + break + } + } + if ignored { + continue + } + oldValue := oldConfig.Field(i).Interface() + newValue := newConfig.Field(i).Interface() + if !reflect.DeepEqual(oldValue, newValue) { + return fmt.Errorf("field %q: old=%v, new=%v", field.Name, oldValue, newValue) + } + } + return nil +} diff --git a/server/ring.go b/server/ring.go new file mode 100644 index 0000000..e7b64bc --- /dev/null +++ b/server/ring.go @@ -0,0 +1,93 @@ +// Copyright 2018-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import "sync" + +// We wrap to hold onto optional items for /connz. +type closedClient struct { + ConnInfo + subs []SubDetail + user string + acc string +} + +// Fixed sized ringbuffer for closed connections. +type closedRingBuffer struct { + mu sync.Mutex + total uint64 + conns []*closedClient +} + +// Create a new ring buffer with at most max items. +func newClosedRingBuffer(max int) *closedRingBuffer { + rb := &closedRingBuffer{} + rb.conns = make([]*closedClient, max) + return rb +} + +// Adds in a new closed connection. If there is no more room, +// remove the oldest. +func (rb *closedRingBuffer) append(cc *closedClient) { + rb.mu.Lock() + defer rb.mu.Unlock() + rb.conns[rb.nextLocked()] = cc + rb.total++ +} + +func (rb *closedRingBuffer) nextLocked() int { + return int(rb.total % uint64(cap(rb.conns))) +} + +func (rb *closedRingBuffer) len() int { + rb.mu.Lock() + defer rb.mu.Unlock() + return rb.lenLocked() +} + +func (rb *closedRingBuffer) lenLocked() int { + if rb.total > uint64(cap(rb.conns)) { + return cap(rb.conns) + } + return int(rb.total) +} + +func (rb *closedRingBuffer) totalConns() uint64 { + rb.mu.Lock() + defer rb.mu.Unlock() + return rb.total +} + +// This will return a sorted copy of the list which recipient can +// modify. If the contents of the client itself need to be modified, +// meaning swapping in any optional items, a copy should be made. We +// could introduce a new lock and hold that but since we return this +// list inside monitor which allows programatic access, we do not +// know when it would be done. +func (rb *closedRingBuffer) closedClients() []*closedClient { + rb.mu.Lock() + defer rb.mu.Unlock() + n := rb.lenLocked() + dup := make([]*closedClient, n) + head := rb.nextLocked() + if rb.total <= uint64(cap(rb.conns)) || head == 0 { + copy(dup, rb.conns[:n]) + } else { + fp := rb.conns[head:] + sp := rb.conns[:head] + copy(dup, fp) + copy(dup[len(fp):], sp) + } + return dup +} diff --git a/server/ring_test.go b/server/ring_test.go new file mode 100644 index 0000000..ff29e28 --- /dev/null +++ b/server/ring_test.go @@ -0,0 +1,76 @@ +// Copyright 2018 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "fmt" + "reflect" + "testing" +) + +func TestRBAppendAndLenAndTotal(t *testing.T) { + rb := newClosedRingBuffer(10) + for i := 0; i < 5; i++ { + rb.append(&closedClient{}) + } + if rbl := rb.len(); rbl != 5 { + t.Fatalf("Expected len of 5, got %d", rbl) + } + if rbt := rb.totalConns(); rbt != 5 { + t.Fatalf("Expected total of 5, got %d", rbt) + } + for i := 0; i < 25; i++ { + rb.append(&closedClient{}) + } + if rbl := rb.len(); rbl != 10 { + t.Fatalf("Expected len of 10, got %d", rbl) + } + if rbt := rb.totalConns(); rbt != 30 { + t.Fatalf("Expected total of 30, got %d", rbt) + } +} + +func (cc *closedClient) String() string { + return cc.user +} + +func TestRBclosedClients(t *testing.T) { + rb := newClosedRingBuffer(10) + + var ui int + addConn := func() { + ui++ + rb.append(&closedClient{user: fmt.Sprintf("%d", ui)}) + } + + max := 100 + master := make([]*closedClient, 0, max) + for i := 1; i <= max; i++ { + master = append(master, &closedClient{user: fmt.Sprintf("%d", i)}) + } + + testList := func(i int) { + ccs := rb.closedClients() + start := int(rb.totalConns()) - len(ccs) + ms := master[start : start+len(ccs)] + if !reflect.DeepEqual(ccs, ms) { + t.Fatalf("test %d: List result did not match master: %+v vs %+v", i, ccs, ms) + } + } + + for i := 0; i < max; i++ { + addConn() + testList(i) + } +} diff --git a/server/route.go b/server/route.go new file mode 100644 index 0000000..fffd322 --- /dev/null +++ b/server/route.go @@ -0,0 +1,3362 @@ +// Copyright 2013-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "hash/fnv" + "math/rand/v2" + "net" + "net/url" + "reflect" + "runtime" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/klauspost/compress/s2" +) + +// RouteType designates the router type +type RouteType int + +// Type of Route +const ( + // This route we learned from speaking to other routes. + Implicit RouteType = iota + // This route was explicitly configured. + Explicit +) + +// Include the space for the proto +var ( + aSubBytes = []byte{'A', '+', ' '} + aUnsubBytes = []byte{'A', '-', ' '} + rSubBytes = []byte{'R', 'S', '+', ' '} + rUnsubBytes = []byte{'R', 'S', '-', ' '} + lSubBytes = []byte{'L', 'S', '+', ' '} + lUnsubBytes = []byte{'L', 'S', '-', ' '} +) + +const leafNoOriginCluster = "_" + +type route struct { + remoteID string + remoteName string + didSolicit bool + retry bool + lnoc bool + lnocu bool + ln bool + routeType RouteType + url *url.URL + authRequired bool + tlsRequired bool + jetstream bool + connectURLs []string + wsConnURLs []string + gatewayURL string + leafnodeURL string + hash string + idHash string + // Location of the route in the slice: s.routes[remoteID][]*client. + // Initialized to -1 on creation, as to indicate that it is not + // added to the list. + poolIdx int + // If this is set, it means that the route is dedicated for this + // account and the account name will not be included in protocols. + accName []byte + // This is set to true if this is a route connection to an old + // server or a server that has pooling completely disabled. + noPool bool + // Selected compression mode, which may be different from the + // server configured mode. + compression string + // Transient value used to set the Info.GossipMode when initiating + // an implicit route and sending to the remote. + gossipMode byte + // This will be set in case of pooling so that a route can trigger + // the creation of the next after receiving a PONG, ensuring + // that authentication did not fail. + startNewRoute *routeInfo +} + +// This contains the information required to create a new route. +type routeInfo struct { + url *url.URL + rtype RouteType + gossipMode byte +} + +// Do not change the values/order since they are exchanged between servers. +const ( + gossipDefault = byte(iota) + gossipDisabled + gossipOverride +) + +type connectInfo struct { + Echo bool `json:"echo"` + Verbose bool `json:"verbose"` + Pedantic bool `json:"pedantic"` + User string `json:"user,omitempty"` + Pass string `json:"pass,omitempty"` + TLS bool `json:"tls_required"` + Headers bool `json:"headers"` + Name string `json:"name"` + Cluster string `json:"cluster"` + Dynamic bool `json:"cluster_dynamic,omitempty"` + LNOC bool `json:"lnoc,omitempty"` + LNOCU bool `json:"lnocu,omitempty"` // Support for LS- with origin cluster name + LN bool `json:"ln,omitempty"` // Support for LS+/LS- leaf interest using a sentinel origin cluster + Gateway string `json:"gateway,omitempty"` +} + +// Route protocol constants +const ( + ConProto = "CONNECT %s" + _CRLF_ + InfoProto = "INFO %s" + _CRLF_ +) + +const ( + // Warning when user configures cluster TLS insecure + clusterTLSInsecureWarning = "TLS certificate chain and hostname of solicited routes will not be verified. DO NOT USE IN PRODUCTION!" + + // The default ping interval is set to 2 minutes, which is fine for client + // connections, etc.. but for route compression, the CompressionS2Auto + // mode uses RTT measurements (ping/pong) to decide which compression level + // to use, we want the interval to not be that high. + defaultRouteMaxPingInterval = 30 * time.Second +) + +// Can be changed for tests +var ( + routeConnectDelay = DEFAULT_ROUTE_CONNECT + routeConnectMaxDelay = DEFAULT_ROUTE_CONNECT_MAX + routeReconnectDelay = DEFAULT_ROUTE_RECONNECT + routeMaxPingInterval = defaultRouteMaxPingInterval +) + +// removeReplySub is called when we trip the max on remoteReply subs. +func (c *client) removeReplySub(sub *subscription) { + if sub == nil { + return + } + // Lookup the account based on sub.sid. + if i := bytes.Index(sub.sid, []byte(" ")); i > 0 { + // First part of SID for route is account name. + if v, ok := c.srv.accounts.Load(bytesToString(sub.sid[:i])); ok { + (v.(*Account)).sl.Remove(sub) + } + c.mu.Lock() + delete(c.subs, bytesToString(sub.sid)) + c.mu.Unlock() + } +} + +func (c *client) processAccountSub(arg []byte) error { + if c.kind == GATEWAY { + return c.processGatewayAccountSub(string(arg)) + } + return nil +} + +func (c *client) processAccountUnsub(arg []byte) { + if c.kind == GATEWAY { + c.processGatewayAccountUnsub(string(arg)) + } +} + +// Process an inbound LMSG specification from the remote route. This means +// we have an origin cluster and we force header semantics. +func (c *client) processRoutedOriginClusterMsgArgs(arg []byte) error { + // Unroll splitArgs to avoid runtime/heap issues + args := c.argsa[:0] + start := -1 + for i, b := range arg { + switch b { + case ' ', '\t', '\r', '\n': + if start >= 0 { + args = append(args, arg[start:i]) + start = -1 + } + default: + if start < 0 { + start = i + } + } + } + if start >= 0 { + args = append(args, arg[start:]) + } + + var an []byte + if c.kind == ROUTER { + if an = c.route.accName; len(an) > 0 && len(args) > 2 { + args = append(args[:2], args[1:]...) + args[1] = an + } + } + c.pa.arg = arg + switch len(args) { + case 0, 1, 2, 3, 4: + return fmt.Errorf("processRoutedOriginClusterMsgArgs Parse Error: '%s'", args) + case 5: + c.pa.reply = nil + c.pa.queues = nil + c.pa.hdb = args[3] + c.pa.hdr = parseSize(args[3]) + c.pa.szb = args[4] + c.pa.size = parseSize(args[4]) + case 6: + c.pa.reply = args[3] + c.pa.queues = nil + c.pa.hdb = args[4] + c.pa.hdr = parseSize(args[4]) + c.pa.szb = args[5] + c.pa.size = parseSize(args[5]) + default: + // args[2] is our reply indicator. Should be + or | normally. + if len(args[3]) != 1 { + return fmt.Errorf("processRoutedOriginClusterMsgArgs Bad or Missing Reply Indicator: '%s'", args[3]) + } + switch args[3][0] { + case '+': + c.pa.reply = args[4] + case '|': + c.pa.reply = nil + default: + return fmt.Errorf("processRoutedOriginClusterMsgArgs Bad or Missing Reply Indicator: '%s'", args[3]) + } + + // Grab header size. + c.pa.hdb = args[len(args)-2] + c.pa.hdr = parseSize(c.pa.hdb) + + // Grab size. + c.pa.szb = args[len(args)-1] + c.pa.size = parseSize(c.pa.szb) + + // Grab queue names. + if c.pa.reply != nil { + c.pa.queues = args[5 : len(args)-2] + } else { + c.pa.queues = args[4 : len(args)-2] + } + } + if c.pa.hdr < 0 { + return fmt.Errorf("processRoutedOriginClusterMsgArgs Bad or Missing Header Size: '%s'", arg) + } + if c.pa.size < 0 { + return fmt.Errorf("processRoutedOriginClusterMsgArgs Bad or Missing Size: '%s'", args) + } + if c.pa.hdr > c.pa.size { + return fmt.Errorf("processRoutedOriginClusterMsgArgs Header Size larger then TotalSize: '%s'", arg) + } + + // Common ones processed after check for arg length + c.pa.origin = args[0] + c.pa.account = args[1] + c.pa.subject = args[2] + if len(an) > 0 { + c.pa.pacache = c.pa.subject + } else { + c.pa.pacache = arg[len(args[0])+1 : len(args[0])+len(args[1])+len(args[2])+2] + } + return nil +} + +// Process an inbound HMSG specification from the remote route. +func (c *client) processRoutedHeaderMsgArgs(arg []byte) error { + // Unroll splitArgs to avoid runtime/heap issues + args := c.argsa[:0] + var an []byte + if c.kind == ROUTER { + if an = c.route.accName; len(an) > 0 { + args = append(args, an) + } + } + start := -1 + for i, b := range arg { + switch b { + case ' ', '\t', '\r', '\n': + if start >= 0 { + args = append(args, arg[start:i]) + start = -1 + } + default: + if start < 0 { + start = i + } + } + } + if start >= 0 { + args = append(args, arg[start:]) + } + + c.pa.arg = arg + switch len(args) { + case 0, 1, 2, 3: + return fmt.Errorf("processRoutedHeaderMsgArgs Parse Error: '%s'", args) + case 4: + c.pa.reply = nil + c.pa.queues = nil + c.pa.hdb = args[2] + c.pa.hdr = parseSize(args[2]) + c.pa.szb = args[3] + c.pa.size = parseSize(args[3]) + case 5: + c.pa.reply = args[2] + c.pa.queues = nil + c.pa.hdb = args[3] + c.pa.hdr = parseSize(args[3]) + c.pa.szb = args[4] + c.pa.size = parseSize(args[4]) + default: + // args[2] is our reply indicator. Should be + or | normally. + if len(args[2]) != 1 { + return fmt.Errorf("processRoutedHeaderMsgArgs Bad or Missing Reply Indicator: '%s'", args[2]) + } + switch args[2][0] { + case '+': + c.pa.reply = args[3] + case '|': + c.pa.reply = nil + default: + return fmt.Errorf("processRoutedHeaderMsgArgs Bad or Missing Reply Indicator: '%s'", args[2]) + } + + // Grab header size. + c.pa.hdb = args[len(args)-2] + c.pa.hdr = parseSize(c.pa.hdb) + + // Grab size. + c.pa.szb = args[len(args)-1] + c.pa.size = parseSize(c.pa.szb) + + // Grab queue names. + if c.pa.reply != nil { + c.pa.queues = args[4 : len(args)-2] + } else { + c.pa.queues = args[3 : len(args)-2] + } + } + if c.pa.hdr < 0 { + return fmt.Errorf("processRoutedHeaderMsgArgs Bad or Missing Header Size: '%s'", arg) + } + if c.pa.size < 0 { + return fmt.Errorf("processRoutedHeaderMsgArgs Bad or Missing Size: '%s'", args) + } + if c.pa.hdr > c.pa.size { + return fmt.Errorf("processRoutedHeaderMsgArgs Header Size larger then TotalSize: '%s'", arg) + } + + // Common ones processed after check for arg length + c.pa.account = args[0] + c.pa.subject = args[1] + if len(an) > 0 { + c.pa.pacache = c.pa.subject + } else { + c.pa.pacache = arg[:len(args[0])+len(args[1])+1] + } + return nil +} + +// Process an inbound RMSG or LMSG specification from the remote route. +func (c *client) processRoutedMsgArgs(arg []byte) error { + // Unroll splitArgs to avoid runtime/heap issues + args := c.argsa[:0] + var an []byte + if c.kind == ROUTER { + if an = c.route.accName; len(an) > 0 { + args = append(args, an) + } + } + start := -1 + for i, b := range arg { + switch b { + case ' ', '\t', '\r', '\n': + if start >= 0 { + args = append(args, arg[start:i]) + start = -1 + } + default: + if start < 0 { + start = i + } + } + } + if start >= 0 { + args = append(args, arg[start:]) + } + + c.pa.arg = arg + switch len(args) { + case 0, 1, 2: + return fmt.Errorf("processRoutedMsgArgs Parse Error: '%s'", args) + case 3: + c.pa.reply = nil + c.pa.queues = nil + c.pa.szb = args[2] + c.pa.size = parseSize(args[2]) + case 4: + c.pa.reply = args[2] + c.pa.queues = nil + c.pa.szb = args[3] + c.pa.size = parseSize(args[3]) + default: + // args[2] is our reply indicator. Should be + or | normally. + if len(args[2]) != 1 { + return fmt.Errorf("processRoutedMsgArgs Bad or Missing Reply Indicator: '%s'", args[2]) + } + switch args[2][0] { + case '+': + c.pa.reply = args[3] + case '|': + c.pa.reply = nil + default: + return fmt.Errorf("processRoutedMsgArgs Bad or Missing Reply Indicator: '%s'", args[2]) + } + // Grab size. + c.pa.szb = args[len(args)-1] + c.pa.size = parseSize(c.pa.szb) + + // Grab queue names. + if c.pa.reply != nil { + c.pa.queues = args[4 : len(args)-1] + } else { + c.pa.queues = args[3 : len(args)-1] + } + } + if c.pa.size < 0 { + return fmt.Errorf("processRoutedMsgArgs Bad or Missing Size: '%s'", args) + } + + // Common ones processed after check for arg length + c.pa.account = args[0] + c.pa.subject = args[1] + if len(an) > 0 { + c.pa.pacache = c.pa.subject + } else { + c.pa.pacache = arg[:len(args[0])+len(args[1])+1] + } + return nil +} + +// processInboundRoutedMsg is called to process an inbound msg from a route. +func (c *client) processInboundRoutedMsg(msg []byte) { + // Update statistics + c.in.msgs++ + // The msg includes the CR_LF, so pull back out for accounting. + size := len(msg) - LEN_CR_LF + c.in.bytes += int32(size) + + if c.opts.Verbose { + c.sendOK() + } + + // Mostly under testing scenarios. + if c.srv == nil { + return + } + + // If the subject (c.pa.subject) has the gateway prefix, this function will handle it. + if c.handleGatewayReply(msg) { + // We are done here. + return + } + + acc, r := c.getAccAndResultFromCache() + if acc == nil { + c.Debugf("Unknown account %q for routed message on subject: %q", c.pa.account, c.pa.subject) + return + } + + acc.stats.Lock() + acc.stats.inMsgs++ + acc.stats.inBytes += int64(size) + acc.stats.rt.inMsgs++ + acc.stats.rt.inBytes += int64(size) + acc.stats.Unlock() + + // Check for no interest, short circuit if so. + // This is the fanout scale. + if len(r.psubs)+len(r.qsubs) > 0 { + c.processMsgResults(acc, r, msg, nil, c.pa.subject, c.pa.reply, pmrNoFlag) + } +} + +// Lock should be held entering here. +func (c *client) sendRouteConnect(clusterName string, tlsRequired bool) error { + var user, pass string + if userInfo := c.route.url.User; userInfo != nil { + user = userInfo.Username() + pass, _ = userInfo.Password() + } + s := c.srv + cinfo := connectInfo{ + Echo: true, + Verbose: false, + Pedantic: false, + User: user, + Pass: pass, + TLS: tlsRequired, + Name: s.info.ID, + Headers: s.supportsHeaders(), + Cluster: clusterName, + Dynamic: s.isClusterNameDynamic(), + LNOC: true, + LN: true, + } + + b, err := json.Marshal(cinfo) + if err != nil { + c.Errorf("Error marshaling CONNECT to route: %v\n", err) + return err + } + c.enqueueProto([]byte(fmt.Sprintf(ConProto, b))) + return nil +} + +// Returns a route pool index for this account based on the given pool size. +// If `poolSize` is smaller or equal to 1, the returned value will always +// be 0, regardless of the account name. If not, the returned value will +// be in the range [0..poolSize-1]. The value for a given account name +// is constant and same on all servers (given the same `poolSize` value). +func computeRoutePoolIdx(poolSize int, an string) int { + if poolSize <= 1 { + return 0 + } + h := fnv.New32a() + h.Write([]byte(an)) + sum32 := h.Sum32() + return int((sum32 % uint32(poolSize))) +} + +// Process the info message if we are a route. +func (c *client) processRouteInfo(info *Info) { + + supportsHeaders := c.srv.supportsHeaders() + clusterName := c.srv.ClusterName() + srvName := c.srv.Name() + + c.mu.Lock() + // Connection can be closed at any time (by auth timeout, etc). + // Does not make sense to continue here if connection is gone. + if c.route == nil || c.isClosed() { + c.mu.Unlock() + return + } + + s := c.srv + + // Detect route to self. + if info.ID == s.info.ID { + // Need to set this so that the close does the right thing + c.route.remoteID = info.ID + c.mu.Unlock() + c.closeConnection(DuplicateRoute) + return + } + if info.Cluster == leafNoOriginCluster { + c.mu.Unlock() + c.closeConnection(ClusterNameConflict) + return + } + + // Detect if we have a mismatch of cluster names. + if info.Cluster != "" && info.Cluster != clusterName { + c.mu.Unlock() + // If we are dynamic we may update our cluster name. + // Use other if remote is non dynamic or their name is "bigger" + if s.isClusterNameDynamic() && (!info.Dynamic || (strings.Compare(clusterName, info.Cluster) < 0)) { + s.setClusterName(info.Cluster) + s.removeAllRoutesExcept(info.ID) + c.mu.Lock() + } else { + c.closeConnection(ClusterNameConflict) + return + } + } + + opts := s.getOpts() + + didSolicit := c.route.didSolicit + + // If this is an async INFO from an existing route... + if c.flags.isSet(infoReceived) { + remoteID := c.route.remoteID + + // Check if this is an INFO about adding a per-account route during + // a configuration reload. + if info.RouteAccReqID != _EMPTY_ { + c.mu.Unlock() + + // If there is an account name, then the remote server is telling + // us that this account will now have its dedicated route. + if an := info.RouteAccount; an != _EMPTY_ { + acc, err := s.LookupAccount(an) + if err != nil { + s.Errorf("Error looking up account %q: %v", an, err) + return + } + s.mu.Lock() + // If running without system account and adding a dedicated + // route for an account for the first time, it could be that + // the map is nil. If so, create it. + if s.accRoutes == nil { + s.accRoutes = make(map[string]map[string]*client) + } + if _, ok := s.accRoutes[an]; !ok { + s.accRoutes[an] = make(map[string]*client) + } + acc.mu.Lock() + sl := acc.sl + rpi := acc.routePoolIdx + // Make sure that the account was not already switched. + if rpi >= 0 { + s.setRouteInfo(acc) + // Change the route pool index to indicate that this + // account is actually transitioning. This will be used + // to suppress possible remote subscription interest coming + // in while the transition is happening. + acc.routePoolIdx = accTransitioningToDedicatedRoute + } + acc.mu.Unlock() + // Since v2.11.0, we support remotes with a different pool size + // (for rolling upgrades), so we need to use the remote route + // pool index (based on the remote configured pool size) since + // the remote subscriptions will be attached to the route at + // that index, not at our account's route pool index. But we + // need to compute only if rpi is negative or the pool sizes + // are different. + if rpi <= 0 || info.RoutePoolSize != s.routesPoolSize { + rpi = computeRoutePoolIdx(info.RoutePoolSize, an) + } + // Go over each remote's route at pool index `rpi` and remove + // remote subs for this account. + s.forEachRouteIdx(rpi, func(r *client) bool { + r.mu.Lock() + // Exclude routes to servers that don't support pooling. + if !r.route.noPool { + if subs := r.removeRemoteSubsForAcc(an); len(subs) > 0 { + sl.RemoveBatch(subs) + } + } + r.mu.Unlock() + return true + }) + // Respond to the remote by clearing the RouteAccount field. + info.RouteAccount = _EMPTY_ + proto := generateInfoJSON(info) + c.mu.Lock() + c.enqueueProto(proto) + c.mu.Unlock() + s.mu.Unlock() + } else { + // If no account name is specified, this is a response from the + // remote. Simply send to the communication channel, if the + // request ID matches the current one. + s.mu.Lock() + if info.RouteAccReqID == s.accAddedReqID && s.accAddedCh != nil { + select { + case s.accAddedCh <- struct{}{}: + default: + } + } + s.mu.Unlock() + } + // In both cases, we are done here. + return + } + + // Check if this is an INFO for gateways... + if info.Gateway != _EMPTY_ { + c.mu.Unlock() + // If this server has no gateway configured, report error and return. + if !s.gateway.enabled { + // FIXME: Should this be a Fatalf()? + s.Errorf("Received information about gateway %q from %s, but gateway is not configured", + info.Gateway, remoteID) + return + } + s.processGatewayInfoFromRoute(info, remoteID) + return + } + + // We receive an INFO from a server that informs us about another server, + // so the info.ID in the INFO protocol does not match the ID of this route. + if remoteID != _EMPTY_ && remoteID != info.ID { + // We want to know if the existing route supports pooling/pinned-account + // or not when processing the implicit route. + noPool := c.route.noPool + c.mu.Unlock() + + // Process this implicit route. We will check that it is not an explicit + // route and/or that it has not been connected already. + s.processImplicitRoute(info, noPool) + return + } + + var connectURLs []string + var wsConnectURLs []string + var updateRoutePerms bool + + // If we are notified that the remote is going into LDM mode, capture route's connectURLs. + if info.LameDuckMode { + connectURLs = c.route.connectURLs + wsConnectURLs = c.route.wsConnURLs + } else { + // Update only if we detect a difference + updateRoutePerms = !reflect.DeepEqual(c.opts.Import, info.Import) || !reflect.DeepEqual(c.opts.Export, info.Export) + } + c.mu.Unlock() + + if updateRoutePerms { + s.updateRemoteRoutePerms(c, info) + } + + // If the remote is going into LDM and there are client connect URLs + // associated with this route and we are allowed to advertise, remove + // those URLs and update our clients. + if (len(connectURLs) > 0 || len(wsConnectURLs) > 0) && !opts.Cluster.NoAdvertise { + s.mu.Lock() + s.removeConnectURLsAndSendINFOToClients(connectURLs, wsConnectURLs) + s.mu.Unlock() + } + return + } + + // Check if remote has same server name than this server. + if !didSolicit && info.Name == srvName { + c.mu.Unlock() + // This is now an error and we close the connection. We need unique names for JetStream clustering. + c.Errorf("Remote server has a duplicate name: %q", info.Name) + c.closeConnection(DuplicateServerName) + return + } + + var sendDelayedInfo bool + + // First INFO, check if this server is configured for compression because + // if that is the case, we need to negotiate it with the remote server. + if needsCompression(opts.Cluster.Compression.Mode) { + accName := bytesToString(c.route.accName) + // If we did not yet negotiate... + compNeg := c.flags.isSet(compressionNegotiated) + if !compNeg { + // Prevent from getting back here. + c.flags.set(compressionNegotiated) + // Release client lock since following function will need server lock. + c.mu.Unlock() + compress, err := s.negotiateRouteCompression(c, didSolicit, accName, info.Compression, opts) + if err != nil { + c.sendErrAndErr(err.Error()) + c.closeConnection(ProtocolViolation) + return + } + if compress { + // Done for now, will get back another INFO protocol... + return + } + // No compression because one side does not want/can't, so proceed. + c.mu.Lock() + // Check that the connection did not close if the lock was released. + if c.isClosed() { + c.mu.Unlock() + return + } + } + // We can set the ping timer after we just negotiated compression above, + // or for solicited routes if we already negotiated. + if !compNeg || didSolicit { + c.setFirstPingTimer() + } + // When compression is configured, we delay the initial INFO for any + // solicited route. So we need to send the delayed INFO simply based + // on the didSolicit boolean. + sendDelayedInfo = didSolicit + } else { + // Coming from an old server, the Compression field would be the empty + // string. For servers that are configured with CompressionNotSupported, + // this makes them behave as old servers. + if info.Compression == _EMPTY_ || opts.Cluster.Compression.Mode == CompressionNotSupported { + c.route.compression = CompressionNotSupported + } else { + c.route.compression = CompressionOff + } + // When compression is not configured, we delay the initial INFO only + // for solicited pooled routes, so use the same check that we did when + // we decided to delay in createRoute(). + sendDelayedInfo = didSolicit && routeShouldDelayInfo(bytesToString(c.route.accName), opts) + } + + // Mark that the INFO protocol has been received, so we can detect updates. + c.flags.set(infoReceived) + + // Get the route's proto version. It will be used to check if the connection + // supports certain features, such as message tracing. + c.opts.Protocol = info.Proto + + // Headers + c.headers = supportsHeaders && info.Headers + + // Copy over important information. + c.route.remoteID = info.ID + c.route.authRequired = info.AuthRequired + c.route.tlsRequired = info.TLSRequired + c.route.gatewayURL = info.GatewayURL + c.route.remoteName = info.Name + c.route.ln = info.LN + c.route.lnoc = info.LNOC || c.route.ln + c.route.lnocu = info.LNOCU || c.route.ln + c.route.jetstream = info.JetStream + + // When sent through route INFO, if the field is set, it should be of size 1. + if len(info.LeafNodeURLs) == 1 { + c.route.leafnodeURL = info.LeafNodeURLs[0] + } + // Compute the hash of this route based on remote server name + c.route.hash = getHash(info.Name) + // Same with remote server ID (used for GW mapped replies routing). + // Use getGWHash since we don't use the same hash len for that + // for backward compatibility. + c.route.idHash = string(getGWHash(info.ID)) + + // Copy over permissions as well. + c.opts.Import = info.Import + c.opts.Export = info.Export + + // If we do not know this route's URL, construct one on the fly + // from the information provided. + if c.route.url == nil { + // Add in the URL from host and port + hp := net.JoinHostPort(info.Host, strconv.Itoa(info.Port)) + url, err := url.Parse(fmt.Sprintf("nats-route://%s/", hp)) + if err != nil { + c.Errorf("Error parsing URL from INFO: %v\n", err) + c.mu.Unlock() + c.closeConnection(ParseError) + return + } + c.route.url = url + } + // The incoming INFO from the route will have IP set + // if it has Cluster.Advertise. In that case, use that + // otherwise construct it from the remote TCP address. + if info.IP == _EMPTY_ { + // Need to get the remote IP address. + switch conn := c.nc.(type) { + case *net.TCPConn, *tls.Conn: + addr := conn.RemoteAddr().(*net.TCPAddr) + info.IP = fmt.Sprintf("nats-route://%s/", net.JoinHostPort(addr.IP.String(), + strconv.Itoa(info.Port))) + default: + info.IP = c.route.url.String() + } + } + // For accounts that are configured to have their own route: + // If this is a solicited route, we already have c.route.accName set in createRoute. + // For non solicited route (the accept side), we will set the account name that + // is present in the INFO protocol. + if didSolicit && len(c.route.accName) > 0 { + // Set it in the info.RouteAccount so that addRoute can use that + // and we properly gossip that this is a route for an account. + info.RouteAccount = string(c.route.accName) + } else if !didSolicit && info.RouteAccount != _EMPTY_ { + c.route.accName = []byte(info.RouteAccount) + } + accName := string(c.route.accName) + + // Capture the noGossip value and reset it here. + gossipMode := c.route.gossipMode + c.route.gossipMode = 0 + + // Check to see if we have this remote already registered. + // This can happen when both servers have routes to each other. + c.mu.Unlock() + + if added := s.addRoute(c, didSolicit, sendDelayedInfo, gossipMode, info, accName); added { + if accName != _EMPTY_ { + c.Debugf("Registering remote route %q for account %q", info.ID, accName) + } else { + c.Debugf("Registering remote route %q", info.ID) + } + } else { + c.Debugf("Detected duplicate remote route %q", info.ID) + c.closeConnection(DuplicateRoute) + } +} + +func (s *Server) negotiateRouteCompression(c *client, didSolicit bool, accName, infoCompression string, opts *Options) (bool, error) { + // Negotiate the appropriate compression mode (or no compression) + cm, err := selectCompressionMode(opts.Cluster.Compression.Mode, infoCompression) + if err != nil { + return false, err + } + c.mu.Lock() + // For "auto" mode, set the initial compression mode based on RTT + if cm == CompressionS2Auto { + if c.rttStart.IsZero() { + c.rtt = computeRTT(c.start) + } + cm = selectS2AutoModeBasedOnRTT(c.rtt, opts.Cluster.Compression.RTTThresholds) + } + // Keep track of the negotiated compression mode. + c.route.compression = cm + c.mu.Unlock() + + // If we end-up doing compression... + if needsCompression(cm) { + // Generate an INFO with the chosen compression mode. + s.mu.Lock() + infoProto := s.generateRouteInitialInfoJSON(accName, cm, 0, gossipDefault) + s.mu.Unlock() + + // If we solicited, then send this INFO protocol BEFORE switching + // to compression writer. However, if we did not, we send it after. + c.mu.Lock() + if didSolicit { + c.enqueueProto(infoProto) + // Make sure it is completely flushed (the pending bytes goes to + // 0) before proceeding. + for c.out.pb > 0 && !c.isClosed() { + c.flushOutbound() + } + } + // This is to notify the readLoop that it should switch to a + // (de)compression reader. + c.in.flags.set(switchToCompression) + // Create the compress writer before queueing the INFO protocol for + // a route that did not solicit. It will make sure that that proto + // is sent with compression on. + c.out.cw = s2.NewWriter(nil, s2WriterOptions(cm)...) + if !didSolicit { + c.enqueueProto(infoProto) + } + // We can now set the ping timer. + c.setFirstPingTimer() + c.mu.Unlock() + return true, nil + } + return false, nil +} + +// Possibly sends local subscriptions interest to this route +// based on changes in the remote's Export permissions. +func (s *Server) updateRemoteRoutePerms(c *client, info *Info) { + c.mu.Lock() + // Interested only on Export permissions for the remote server. + // Create "fake" clients that we will use to check permissions + // using the old permissions... + oldPerms := &RoutePermissions{Export: c.opts.Export} + oldPermsTester := &client{} + oldPermsTester.setRoutePermissions(oldPerms) + // and the new ones. + newPerms := &RoutePermissions{Export: info.Export} + newPermsTester := &client{} + newPermsTester.setRoutePermissions(newPerms) + + c.opts.Import = info.Import + c.opts.Export = info.Export + + routeAcc, poolIdx, noPool := bytesToString(c.route.accName), c.route.poolIdx, c.route.noPool + c.mu.Unlock() + + var ( + _localSubs [4096]*subscription + _allSubs [4096]*subscription + allSubs = _allSubs[:0] + ) + + s.accounts.Range(func(_, v any) bool { + acc := v.(*Account) + acc.mu.RLock() + accName, sl, accPoolIdx := acc.Name, acc.sl, acc.routePoolIdx + acc.mu.RUnlock() + + // Do this only for accounts handled by this route + if (accPoolIdx >= 0 && accPoolIdx == poolIdx) || (routeAcc == accName) || noPool { + localSubs := _localSubs[:0] + sl.localSubs(&localSubs, false) + if len(localSubs) > 0 { + allSubs = append(allSubs, localSubs...) + } + } + return true + }) + + if len(allSubs) == 0 { + return + } + + c.mu.Lock() + c.sendRouteSubProtos(allSubs, false, func(sub *subscription) bool { + subj := string(sub.subject) + // If the remote can now export but could not before, and this server can import this + // subject, then send SUB protocol. + if newPermsTester.canExport(subj) && !oldPermsTester.canExport(subj) && c.canImport(subj) { + return true + } + return false + }) + c.mu.Unlock() +} + +// sendAsyncInfoToClients sends an INFO protocol to all +// connected clients that accept async INFO updates. +// The server lock is held on entry. +func (s *Server) sendAsyncInfoToClients(regCli, wsCli bool) { + // If there are no clients supporting async INFO protocols, we are done. + // Also don't send if we are shutting down... + if s.cproto == 0 || s.isShuttingDown() { + return + } + info := s.copyInfo() + + for _, c := range s.clients { + c.mu.Lock() + // Here, we are going to send only to the clients that are fully + // registered (server has received CONNECT and first PING). For + // clients that are not at this stage, this will happen in the + // processing of the first PING (see client.processPing) + if ((regCli && !c.isWebsocket()) || (wsCli && c.isWebsocket())) && + c.opts.Protocol >= ClientProtoInfo && + c.flags.isSet(firstPongSent) { + // sendInfo takes care of checking if the connection is still + // valid or not, so don't duplicate tests here. + c.enqueueProto(c.generateClientInfoJSON(info, true)) + } + c.mu.Unlock() + } +} + +// This will process implicit route information received from another server. +// We will check to see if we have configured or are already connected, +// and if so we will ignore. Otherwise we will attempt to connect. +func (s *Server) processImplicitRoute(info *Info, routeNoPool bool) { + remoteID := info.ID + + s.mu.Lock() + defer s.mu.Unlock() + + // Don't connect to ourself + if remoteID == s.info.ID { + return + } + + // Snapshot server options. + opts := s.getOpts() + + // Check if this route already exists + if accName := info.RouteAccount; accName != _EMPTY_ { + // If we don't support pooling/pinned account, bail. + if opts.Cluster.PoolSize <= 0 { + return + } + if remotes, ok := s.accRoutes[accName]; ok { + if r := remotes[remoteID]; r != nil { + return + } + } + } else if _, exists := s.routes[remoteID]; exists { + return + } + // Check if we have this route as a configured route + if s.hasThisRouteConfigured(info) { + return + } + + // Initiate the connection, using info.IP instead of info.URL here... + r, err := url.Parse(info.IP) + if err != nil { + s.Errorf("Error parsing URL from INFO: %v\n", err) + return + } + + if info.AuthRequired { + r.User = url.UserPassword(opts.Cluster.Username, opts.Cluster.Password) + } + s.startGoRoutine(func() { s.connectToRoute(r, Implicit, true, info.GossipMode, info.RouteAccount) }) + // If we are processing an implicit route from a route that does not + // support pooling/pinned-accounts, we won't receive an INFO for each of + // the pinned-accounts that we would normally receive. In that case, just + // initiate routes for all our configured pinned accounts. + if routeNoPool && info.RouteAccount == _EMPTY_ && len(opts.Cluster.PinnedAccounts) > 0 { + // Copy since we are going to pass as closure to a go routine. + rURL := r + for _, an := range opts.Cluster.PinnedAccounts { + accName := an + s.startGoRoutine(func() { s.connectToRoute(rURL, Implicit, true, info.GossipMode, accName) }) + } + } +} + +// hasThisRouteConfigured returns true if info.Host:info.Port is present +// in the server's opts.Routes, false otherwise. +// Server lock is assumed to be held by caller. +func (s *Server) hasThisRouteConfigured(info *Info) bool { + routes := s.getOpts().Routes + if len(routes) == 0 { + return false + } + // This could possibly be a 0.0.0.0 host so we will also construct a second + // url with the host section of the `info.IP` (if present). + sPort := strconv.Itoa(info.Port) + urlOne := strings.ToLower(net.JoinHostPort(info.Host, sPort)) + var urlTwo string + if info.IP != _EMPTY_ { + if u, _ := url.Parse(info.IP); u != nil { + urlTwo = strings.ToLower(net.JoinHostPort(u.Hostname(), sPort)) + // Ignore if same than the first + if urlTwo == urlOne { + urlTwo = _EMPTY_ + } + } + } + for _, ri := range routes { + rHost := strings.ToLower(ri.Host) + if rHost == urlOne { + return true + } + if urlTwo != _EMPTY_ && rHost == urlTwo { + return true + } + } + return false +} + +// forwardNewRouteInfoToKnownServers possibly sends the INFO protocol of the +// new route to all routes known by this server. In turn, each server will +// contact this new route. +// Server lock held on entry. +func (s *Server) forwardNewRouteInfoToKnownServers(info *Info, rtype RouteType, didSolicit bool, localGossipMode byte) { + // Determine if this connection is resulting from a gossip notification. + fromGossip := didSolicit && rtype == Implicit + // If from gossip (but we are not overriding it) or if the remote disabled gossip, bail out. + if (fromGossip && localGossipMode != gossipOverride) || info.GossipMode == gossipDisabled { + return + } + + // Note: nonce is not used in routes. + // That being said, the info we get is the initial INFO which + // contains a nonce, but we now forward this to existing routes, + // so clear it now. + info.Nonce = _EMPTY_ + + var ( + infoGMDefault []byte + infoGMDisabled []byte + infoGMOverride []byte + ) + + generateJSON := func(gm byte) []byte { + info.GossipMode = gm + b, _ := json.Marshal(info) + return []byte(fmt.Sprintf(InfoProto, b)) + } + + getJSON := func(r *client) []byte { + if (!didSolicit && r.route.routeType == Explicit) || (didSolicit && rtype == Explicit) { + if infoGMOverride == nil { + infoGMOverride = generateJSON(gossipOverride) + } + return infoGMOverride + } else if !didSolicit { + if infoGMDisabled == nil { + infoGMDisabled = generateJSON(gossipDisabled) + } + return infoGMDisabled + } + if infoGMDefault == nil { + infoGMDefault = generateJSON(0) + } + return infoGMDefault + } + + var accRemotes map[string]*client + pinnedAccount := info.RouteAccount != _EMPTY_ + // If this is for a pinned account, we will try to send the gossip + // through our pinned account routes, but fall back to the other + // routes in case we don't have one for a given remote. + if pinnedAccount { + var ok bool + if accRemotes, ok = s.accRoutes[info.RouteAccount]; ok { + for remoteID, r := range accRemotes { + if r == nil { + continue + } + r.mu.Lock() + // Do not send to a remote that does not support pooling/pinned-accounts. + if remoteID != info.ID && !r.route.noPool { + r.enqueueProto(getJSON(r)) + } + r.mu.Unlock() + } + } + } + + s.forEachRemote(func(r *client) { + r.mu.Lock() + remoteID := r.route.remoteID + if pinnedAccount { + if _, processed := accRemotes[remoteID]; processed { + r.mu.Unlock() + return + } + } + // If this is a new route for a given account, do not send to a server + // that does not support pooling/pinned-accounts. + if remoteID != info.ID && (!pinnedAccount || !r.route.noPool) { + r.enqueueProto(getJSON(r)) + } + r.mu.Unlock() + }) +} + +// canImport is whether or not we will send a SUB for interest to the other side. +// This is for ROUTER connections only. +// Lock is held on entry. +func (c *client) canImport(subject string) bool { + // Use pubAllowed() since this checks Publish permissions which + // is what Import maps to. + return c.pubAllowedFullCheck(subject, false, true) +} + +// canExport is whether or not we will accept a SUB from the remote for a given subject. +// This is for ROUTER connections only. +// Lock is held on entry +func (c *client) canExport(subject string) bool { + // Use canSubscribe() since this checks Subscribe permissions which + // is what Export maps to. + return c.canSubscribe(subject) +} + +// Initialize or reset cluster's permissions. +// This is for ROUTER connections only. +// Client lock is held on entry +func (c *client) setRoutePermissions(perms *RoutePermissions) { + // Reset if some were set + if perms == nil { + c.perms = nil + c.mperms = nil + return + } + // Convert route permissions to user permissions. + // The Import permission is mapped to Publish + // and Export permission is mapped to Subscribe. + // For meaning of Import/Export, see canImport and canExport. + p := &Permissions{ + Publish: perms.Import, + Subscribe: perms.Export, + } + c.setPermissions(p) +} + +// Type used to hold a list of subs on a per account basis. +type asubs struct { + acc *Account + subs []*subscription +} + +// Returns the account name from the subscription's key. +// This is invoked knowing that the key contains an account name, so for a sub +// that is not from a pinned-account route. +// The `keyHasSubType` boolean indicates that the key starts with the indicator +// for leaf or regular routed subscriptions. No-origin leaf subscriptions always have the +// indicator so they remain distinct from RS+ even without LNOCU. +func getAccNameFromRoutedSubKey(sub *subscription, key string, keyHasSubType bool) string { + fields := strings.Fields(key) + var accIdx int + if keyHasSubType || (sub.leaf && len(sub.origin) == 0) { + // Start after the sub type indicator. + accIdx = 1 + // But if there is an origin, bump its index. + if len(sub.origin) > 0 || (sub.leaf && len(fields) > 1 && fields[1] == leafNoOriginCluster) { + accIdx = 2 + } + } + return fields[accIdx] +} + +// Returns if the route is dedicated to an account, its name, and a boolean +// that indicates if this route uses the routed subscription indicator at +// the beginning of the subscription key. +// Lock held on entry. +func (c *client) getRoutedSubKeyInfo() (bool, string, bool) { + var accName string + if an := c.route.accName; len(an) > 0 { + accName = string(an) + } + return accName != _EMPTY_, accName, c.route.lnocu +} + +// removeRemoteSubs will walk the subs and remove them from the appropriate account. +func (c *client) removeRemoteSubs() { + // We need to gather these on a per account basis. + // FIXME(dlc) - We should be smarter about this.. + as := map[string]*asubs{} + c.mu.Lock() + srv := c.srv + subs := c.subs + c.subs = nil + pa, accountName, hasSubType := c.getRoutedSubKeyInfo() + c.mu.Unlock() + + for key, sub := range subs { + c.mu.Lock() + sub.max = 0 + c.mu.Unlock() + // If not a pinned-account route, we need to find the account + // name from the sub's key. + if !pa { + accountName = getAccNameFromRoutedSubKey(sub, key, hasSubType) + } + ase := as[accountName] + if ase == nil { + if v, ok := srv.accounts.Load(accountName); ok { + ase = &asubs{acc: v.(*Account), subs: []*subscription{sub}} + as[accountName] = ase + } else { + continue + } + } else { + ase.subs = append(ase.subs, sub) + } + delta := int32(1) + if len(sub.queue) > 0 { + delta = sub.qw + } + if srv.gateway.enabled { + srv.gatewayUpdateSubInterest(accountName, sub, -delta) + } + ase.acc.updateLeafNodes(sub, -delta) + } + + // Now remove the subs by batch for each account sublist. + for _, ase := range as { + c.Debugf("Removing %d subscriptions for account %q", len(ase.subs), ase.acc.Name) + ase.acc.mu.Lock() + ase.acc.sl.RemoveBatch(ase.subs) + ase.acc.mu.Unlock() + } +} + +// Removes (and returns) the subscriptions from this route's subscriptions map +// that belong to the given account. +// Lock is held on entry +func (c *client) removeRemoteSubsForAcc(name string) []*subscription { + var subs []*subscription + _, _, hasSubType := c.getRoutedSubKeyInfo() + for key, sub := range c.subs { + an := getAccNameFromRoutedSubKey(sub, key, hasSubType) + if an == name { + sub.max = 0 + subs = append(subs, sub) + delete(c.subs, key) + } + } + return subs +} + +func (c *client) parseUnsubProto(arg []byte, accInProto, hasOrigin bool) ([]byte, string, []byte, []byte, error) { + // Indicate any activity, so pub and sub or unsubs. + c.in.subs++ + + args := splitArg(arg) + + var ( + origin []byte + accountName string + queue []byte + subjIdx int + ) + // If `hasOrigin` is true, then it means this is a LS- with origin in proto. + if hasOrigin { + // We would not be here if there was not at least 1 field. + origin = args[0] + subjIdx = 1 + } + // If there is an account in the protocol, bump the subject index. + if accInProto { + subjIdx++ + } + + switch len(args) { + case subjIdx + 1: + case subjIdx + 2: + queue = args[subjIdx+1] + default: + return nil, _EMPTY_, nil, nil, fmt.Errorf("parse error: '%s'", arg) + } + if accInProto { + // If there is an account in the protocol, it is before the subject. + accountName = string(args[subjIdx-1]) + } + return origin, accountName, args[subjIdx], queue, nil +} + +// Indicates no more interest in the given account/subject for the remote side. +func (c *client) processRemoteUnsub(arg []byte, leafUnsub bool) (err error) { + srv := c.srv + if srv == nil { + return nil + } + + var accountName string + // Assume the account will be in the protocol. + accInProto := true + + c.mu.Lock() + originSupport := c.route.lnocu + lnSupport := c.route.ln + if c.route != nil && len(c.route.accName) > 0 { + accountName, accInProto = string(c.route.accName), false + } + c.mu.Unlock() + + hasOrigin := leafUnsub && originSupport + origin, accNameFromProto, subject, _, err := c.parseUnsubProto(arg, accInProto, hasOrigin) + if err != nil { + return fmt.Errorf("processRemoteUnsub %s", err.Error()) + } + noOrigin := lnSupport && bytesToString(origin) == leafNoOriginCluster + if accInProto { + accountName = accNameFromProto + } + // Lookup the account + var acc *Account + if v, ok := srv.accounts.Load(accountName); ok { + acc = v.(*Account) + } else { + c.Debugf("Unknown account %q for subject %q", accountName, subject) + return nil + } + + c.mu.Lock() + if c.isClosed() { + c.mu.Unlock() + return nil + } + + _keya := [128]byte{} + _key := _keya[:0] + + var key string + if !originSupport && !noOrigin { + // If it is an LS- or RS-, we use the protocol as-is as the key. + key = bytesToString(arg) + } else { + // We need to prefix with the sub type. + if leafUnsub { + _key = append(_key, keyRoutedLeafSubByte) + } else { + _key = append(_key, keyRoutedSubByte) + } + _key = append(_key, ' ') + _key = append(_key, arg...) + key = bytesToString(_key) + } + delta := int32(1) + sub, ok := c.subs[key] + if ok { + delete(c.subs, key) + acc.sl.Remove(sub) + if len(sub.queue) > 0 { + delta = sub.qw + } + } + c.mu.Unlock() + + // Update gateways and leaf nodes only if the subscription was found. + if ok { + if srv.gateway.enabled { + srv.gatewayUpdateSubInterest(accountName, sub, -delta) + } + + // Now check on leafnode updates. + acc.updateLeafNodes(sub, -delta) + } + + if c.opts.Verbose { + c.sendOK() + } + return nil +} + +func (c *client) processRemoteSub(argo []byte, leafSub, hasOrigin bool) (err error) { + // Indicate activity. + c.in.subs++ + + srv := c.srv + if srv == nil { + return nil + } + + // We copy `argo` to not reference the read buffer. However, we will + // prefix with a code that says if the remote sub is for a leaf + // (hasOrigin == true) or not to prevent key collisions. Imagine: + // "RS+ foo bar baz 1\r\n" => "foo bar baz" (a routed queue sub) + // "LS+ foo bar baz\r\n" => "foo bar baz" (a route leaf sub on "baz", + // for account "bar" with origin "foo"). + // + // The sub.sid/key will be set respectively to "R foo bar baz" and + // "L foo bar baz". + // + // We also no longer add the account if it was not present (due to + // pinned-account route) since there is no need really. + // + // For routes to older server, we will still create the "arg" with + // the above layout, but we will create the sub.sid/key as before, + // that is, not including the origin for LS+ because older server + // only send LS- without origin, so we would not be able to find + // the sub in the map. + c.mu.Lock() + accountName := string(c.route.accName) + lnSupport := c.route.ln + c.mu.Unlock() + oldStyle := !c.route.lnocu && !(leafSub && !hasOrigin) + + // Indicate if the account name should be in the protocol. It would be the + // case if accountName is empty. + accInProto := accountName == _EMPTY_ + + // Copy so we do not reference a potentially large buffer. + // Add 2 more bytes for the routed sub type. + arg := make([]byte, 0, 2+len(argo)) + if leafSub { + arg = append(arg, keyRoutedLeafSubByte) + } else { + arg = append(arg, keyRoutedSubByte) + } + arg = append(arg, ' ') + arg = append(arg, argo...) + + // Now split to get all fields. Unroll splitArgs to avoid runtime/heap issues. + a := [MAX_RSUB_ARGS][]byte{} + args := a[:0] + start := -1 + for i, b := range arg { + switch b { + case ' ', '\t', '\r', '\n': + if start >= 0 { + args = append(args, arg[start:i]) + start = -1 + } + default: + if start < 0 { + start = i + } + } + } + if start >= 0 { + args = append(args, arg[start:]) + } + + delta := int32(1) + sub := &subscription{client: c, leaf: leafSub} + + // There will always be at least a subject, but its location will depend + // on if there is an origin, an account name, etc.. Since we know that + // we have added the sub type indicator as the first field, the subject + // position will be at minimum at index 1. + subjIdx := 1 + if hasOrigin { + subjIdx++ + } + if accInProto { + subjIdx++ + } + switch len(args) { + case subjIdx + 1: + sub.queue = nil + case subjIdx + 3: + sub.queue = args[subjIdx+1] + sub.qw = int32(parseSize(args[subjIdx+2])) + // TODO: (ik) We should have a non empty queue name and a queue + // weight >= 1. For 2.11, we may want to return an error if that + // is not the case, but for now just overwrite `delta` if queue + // weight is greater than 1 (it is possible after a reconnect/ + // server restart to receive a queue weight > 1 for a new sub). + if sub.qw > 1 { + delta = sub.qw + } + default: + return fmt.Errorf("processRemoteSub Parse Error: '%s'", arg) + } + // We know that the number of fields is correct. So we can access args[] based + // on where we expect the fields to be. + + // If there is an origin, it will be at index 1. The negotiated sentinel + // denotes leaf interest without an actual origin cluster. + noOrigin := lnSupport && hasOrigin && bytesToString(args[1]) == leafNoOriginCluster + if hasOrigin && !noOrigin { + sub.origin = args[1] + } + // For subject, use subjIdx. + sub.subject = args[subjIdx] + // If the account name is in the protocol, it will be before the subject. + if accInProto { + accountName = bytesToString(args[subjIdx-1]) + } + // Now set the sub.sid from the arg slice. However, we will have a different + // one if we use the origin or not. + start = 0 + end := len(arg) + if sub.queue != nil { + // Remove the ' ' from the arg length. + end -= 1 + len(args[subjIdx+2]) + } + if oldStyle { + // We will start at the account (if present) or at the subject. + // We first skip the "R " or "L " + start = 2 + // And if there is an origin skip that. + if hasOrigin { + start += len(sub.origin) + 1 + } + // Here we are pointing at the account (if present), or at the subject. + } + sub.sid = arg[start:end] + + // Lookup account while avoiding fetch. + // A slow fetch delays subsequent remote messages. It also avoids the expired check (see below). + // With all but memory resolver lookup can be delayed or fail. + // It is also possible that the account can't be resolved yet. + // This does not apply to the memory resolver. + // When used, perform the fetch. + staticResolver := true + if res := srv.AccountResolver(); res != nil { + if _, ok := res.(*MemAccResolver); !ok { + staticResolver = false + } + } + var acc *Account + if staticResolver { + acc, _ = srv.LookupAccount(accountName) + } else if v, ok := srv.accounts.Load(accountName); ok { + acc = v.(*Account) + } + if acc == nil { + // if the option of retrieving accounts later exists, create an expired one. + // When a client comes along, expiration will prevent it from being used, + // cause a fetch and update the account to what is should be. + if staticResolver { + c.Errorf("Unknown account %q for remote subject %q", accountName, sub.subject) + return + } + c.Debugf("Unknown account %q for remote subject %q", accountName, sub.subject) + + var isNew bool + if acc, isNew = srv.LookupOrRegisterAccount(accountName); isNew { + acc.mu.Lock() + acc.expired.Store(true) + acc.incomplete = true + acc.mu.Unlock() + } + } + + c.mu.Lock() + if c.isClosed() { + c.mu.Unlock() + return nil + } + + // Check permissions if applicable. + if c.perms != nil && !c.canExport(string(sub.subject)) { + c.mu.Unlock() + c.Debugf("Can not export %q, ignoring remote subscription request", sub.subject) + return nil + } + + // Check if we have a maximum on the number of subscriptions. + if c.subsAtLimit() { + c.mu.Unlock() + c.maxSubsExceeded() + return nil + } + + acc.mu.RLock() + // For routes (this can be called by leafnodes), check if the account is + // transitioning (from pool to dedicated route) and this route is not a + // per-account route (route.poolIdx >= 0). If so, ignore this subscription. + // Exclude "no pool" routes from this check. + if c.kind == ROUTER && !c.route.noPool && + acc.routePoolIdx == accTransitioningToDedicatedRoute && c.route.poolIdx >= 0 { + acc.mu.RUnlock() + c.mu.Unlock() + // Do not return an error, which would cause the connection to be closed. + return nil + } + sl := acc.sl + acc.mu.RUnlock() + + // We use the sub.sid for the key of the c.subs map. + key := bytesToString(sub.sid) + osub := c.subs[key] + if osub == nil { + c.subs[key] = sub + // Now place into the account sl. + if err = sl.Insert(sub); err != nil { + delete(c.subs, key) + c.mu.Unlock() + c.Errorf("Could not insert subscription: %v", err) + c.sendErr("Invalid Subscription") + return nil + } + } else if sub.queue != nil { + // For a queue we need to update the weight. + delta = sub.qw - atomic.LoadInt32(&osub.qw) + atomic.StoreInt32(&osub.qw, sub.qw) + sl.UpdateRemoteQSub(osub) + } + c.mu.Unlock() + + if srv.gateway.enabled { + srv.gatewayUpdateSubInterest(acc.Name, sub, delta) + } + + // Now check on leafnode updates. + acc.updateLeafNodes(sub, delta) + + if c.opts.Verbose { + c.sendOK() + } + + return nil +} + +// Lock is held on entry +func (c *client) addRouteSubOrUnsubProtoToBuf(buf []byte, accName string, sub *subscription, isSubProto bool) []byte { + // If we have an origin cluster and the other side supports leafnode origin clusters + // send an LS+/LS- version instead. + if len(sub.origin) > 0 && c.route.lnoc { + if isSubProto { + buf = append(buf, lSubBytes...) + buf = append(buf, sub.origin...) + buf = append(buf, ' ') + } else { + buf = append(buf, lUnsubBytes...) + if c.route.lnocu { + buf = append(buf, sub.origin...) + buf = append(buf, ' ') + } + } + } else if sub.leaf && len(sub.origin) == 0 && c.route.ln { + if isSubProto { + buf = append(buf, lSubBytes...) + } else { + buf = append(buf, lUnsubBytes...) + } + buf = append(buf, leafNoOriginCluster...) + buf = append(buf, ' ') + } else { + if isSubProto { + buf = append(buf, rSubBytes...) + } else { + buf = append(buf, rUnsubBytes...) + } + } + if len(c.route.accName) == 0 { + buf = append(buf, accName...) + buf = append(buf, ' ') + } + buf = append(buf, sub.subject...) + if len(sub.queue) > 0 { + buf = append(buf, ' ') + buf = append(buf, sub.queue...) + // Send our weight if we are a sub proto + if isSubProto { + buf = append(buf, ' ') + var b [12]byte + var i = len(b) + for l := sub.qw; l > 0; l /= 10 { + i-- + b[i] = digits[l%10] + } + buf = append(buf, b[i:]...) + } + } + buf = append(buf, CR_LF...) + return buf +} + +// sendSubsToRoute will send over our subject interest to +// the remote side. For each account we will send the +// complete interest for all subjects, both normal as a binary +// and queue group weights. +// +// Server lock held on entry. +func (s *Server) sendSubsToRoute(route *client, idx int, account string) { + var noPool bool + if idx >= 0 { + // We need to check if this route is "no_pool" in which case we + // need to select all accounts. + route.mu.Lock() + noPool = route.route.noPool + route.mu.Unlock() + } + // Estimated size of all protocols. It does not have to be accurate at all. + var eSize int + estimateProtosSize := func(a *Account, addAccountName bool) { + if ns := len(a.rm); ns > 0 { + var accSize int + if addAccountName { + accSize = len(a.Name) + 1 + } + // Proto looks like: "RS+ [ ][ ]\r\n" + eSize += ns * (len(rSubBytes) + 1 + accSize) + for key := range a.rm { + // Key contains "[ ]" + eSize += len(key) + // In case this is a queue, just add some bytes for the queue weight. + // If we want to be accurate, would have to check if "key" has a space, + // if so, then figure out how many bytes we need to represent the weight. + eSize += 5 + } + } + } + // Send over our account subscriptions. + accs := make([]*Account, 0, 1024) + if idx < 0 || account != _EMPTY_ { + if ai, ok := s.accounts.Load(account); ok { + a := ai.(*Account) + a.mu.RLock() + // Estimate size and add account name in protocol if idx is not -1 + estimateProtosSize(a, idx >= 0) + accs = append(accs, a) + a.mu.RUnlock() + } + } else { + s.accounts.Range(func(k, v any) bool { + a := v.(*Account) + a.mu.RLock() + // We are here for regular or pooled routes (not per-account). + // So we collect all accounts whose routePoolIdx matches the + // one for this route, or only the account provided, or all + // accounts if dealing with a "no pool" route. + if a.routePoolIdx == idx || noPool { + estimateProtosSize(a, true) + accs = append(accs, a) + } + a.mu.RUnlock() + return true + }) + } + + buf := make([]byte, 0, eSize) + + route.mu.Lock() + for _, a := range accs { + a.mu.RLock() + for key, n := range a.rm { + var origin, qn []byte + s := strings.Fields(key) + // Subject will always be the second field (index 1). + subj := stringToBytes(s[1]) + // Check if the key is for a leaf (will be field 0). + leafWithOrigin := s[0] == keyRoutedLeafSub + leafWithoutOrigin := s[0] == keyRoutedLeafNoOriginSub + forLeaf := leafWithOrigin || leafWithoutOrigin + // For queue, regular and no-origin leaf keys have 3 fields, + // while a leaf key with an origin has 4. + if l := len(s); (!leafWithOrigin && l == 3) || (leafWithOrigin && l == 4) { + qn = stringToBytes(s[2]) + } + if leafWithOrigin { + // The leaf origin will be the last field. + origin = stringToBytes(s[len(s)-1]) + } + // s[1] is the subject and already as a string, so use that + // instead of converting back `subj` to a string. + if !route.canImport(s[1]) { + continue + } + sub := subscription{leaf: forLeaf, origin: origin, subject: subj, queue: qn, qw: n} + buf = route.addRouteSubOrUnsubProtoToBuf(buf, a.Name, &sub, true) + } + a.mu.RUnlock() + } + if len(buf) > 0 { + route.enqueueProto(buf) + route.Debugf("Sent local subscriptions to route") + } + route.mu.Unlock() +} + +// Sends SUBs protocols for the given subscriptions. If a filter is specified, it is +// invoked for each subscription. If the filter returns false, the subscription is skipped. +// This function may release the route's lock due to flushing of outbound data. A boolean +// is returned to indicate if the connection has been closed during this call. +// Lock is held on entry. +func (c *client) sendRouteSubProtos(subs []*subscription, trace bool, filter func(sub *subscription) bool) { + c.sendRouteSubOrUnSubProtos(subs, true, trace, filter) +} + +// Sends UNSUBs protocols for the given subscriptions. If a filter is specified, it is +// invoked for each subscription. If the filter returns false, the subscription is skipped. +// This function may release the route's lock due to flushing of outbound data. A boolean +// is returned to indicate if the connection has been closed during this call. +// Lock is held on entry. +func (c *client) sendRouteUnSubProtos(subs []*subscription, trace bool, filter func(sub *subscription) bool) { + c.sendRouteSubOrUnSubProtos(subs, false, trace, filter) +} + +// Low-level function that sends RS+ or RS- protocols for the given subscriptions. +// This can now also send LS+ and LS- for origin cluster based leafnode subscriptions for cluster no-echo. +// Use sendRouteSubProtos or sendRouteUnSubProtos instead for clarity. +// Lock is held on entry. +func (c *client) sendRouteSubOrUnSubProtos(subs []*subscription, isSubProto, trace bool, filter func(sub *subscription) bool) { + var ( + _buf [1024]byte + buf = _buf[:0] + ) + + for _, sub := range subs { + if filter != nil && !filter(sub) { + continue + } + // Determine the account. If sub has an ImportMap entry, use that, otherwise scoped to + // client. Default to global if all else fails. + var accName string + if sub.client != nil && sub.client != c { + sub.client.mu.Lock() + } + if sub.im != nil { + accName = sub.im.acc.Name + } else if sub.client != nil && sub.client.acc != nil { + accName = sub.client.acc.Name + } else { + c.Debugf("Falling back to default account for sending subs") + accName = globalAccountName + } + if sub.client != nil && sub.client != c { + sub.client.mu.Unlock() + } + + as := len(buf) + buf = c.addRouteSubOrUnsubProtoToBuf(buf, accName, sub, isSubProto) + if trace { + c.traceOutOp("", buf[as:len(buf)-LEN_CR_LF]) + } + } + c.enqueueProto(buf) +} + +func (s *Server) createRoute(conn net.Conn, rURL *url.URL, rtype RouteType, gossipMode byte, accName string) *client { + // Snapshot server options. + opts := s.getOpts() + + didSolicit := rURL != nil + r := &route{routeType: rtype, didSolicit: didSolicit, poolIdx: -1, gossipMode: gossipMode} + + c := &client{srv: s, nc: conn, opts: ClientOpts{}, kind: ROUTER, msubs: -1, mpay: -1, route: r, start: time.Now()} + + // Is the server configured for compression? + compressionConfigured := needsCompression(opts.Cluster.Compression.Mode) + + var infoJSON []byte + // Grab server variables and generates route INFO Json. Note that we set + // and reset some of s.routeInfo fields when that happens, so we need + // the server write lock. + s.mu.Lock() + // If we are creating a pooled connection and this is the server soliciting + // the connection, we will delay sending the INFO after we have processed + // the incoming INFO from the remote. Also delay if configured for compression. + delayInfo := didSolicit && (compressionConfigured || routeShouldDelayInfo(accName, opts)) + if !delayInfo { + infoJSON = s.generateRouteInitialInfoJSON(accName, opts.Cluster.Compression.Mode, 0, gossipMode) + } + authRequired := s.routeInfo.AuthRequired + tlsRequired := s.routeInfo.TLSRequired + clusterName := s.info.Cluster + tlsName := s.routeTLSName + s.mu.Unlock() + + // Grab lock + c.mu.Lock() + + // Initialize + c.initClient() + + if didSolicit { + // Do this before the TLS code, otherwise, in case of failure + // and if route is explicit, it would try to reconnect to 'nil'... + r.url = rURL + r.accName = []byte(accName) + } else { + c.flags.set(expectConnect) + } + + // Check for TLS + if tlsRequired { + tlsConfig := opts.Cluster.TLSConfig + if didSolicit { + // Copy off the config to add in ServerName if we need to. + tlsConfig = tlsConfig.Clone() + } + // Perform (server or client side) TLS handshake. + if resetTLSName, err := c.doTLSHandshake("route", didSolicit, rURL, tlsConfig, tlsName, opts.Cluster.TLSTimeout, opts.Cluster.TLSPinnedCerts); err != nil { + c.mu.Unlock() + if resetTLSName { + s.mu.Lock() + s.routeTLSName = _EMPTY_ + s.mu.Unlock() + } + return nil + } + } + + // Do final client initialization + + // Initialize the per-account cache. + c.in.pacache = make(map[string]*perAccountCache) + if didSolicit { + // Set permissions associated with the route user (if applicable). + // No lock needed since we are already under client lock. + c.setRoutePermissions(opts.Cluster.Permissions) + } + + // We can't safely send the pings until we have negotiated compression + // with the remote, but we want to protect against a connection that + // does not perform the handshake. We will start a timer that will close + // the connection as stale based on the ping interval and max out values, + // but without actually sending pings. + if compressionConfigured { + pingInterval := opts.PingInterval + pingMax := opts.MaxPingsOut + if opts.Cluster.PingInterval > 0 { + pingInterval = opts.Cluster.PingInterval + } + if opts.Cluster.MaxPingsOut > 0 { + pingMax = opts.Cluster.MaxPingsOut + } + c.watchForStaleConnection(adjustPingInterval(ROUTER, pingInterval), pingMax) + } else { + // Set the Ping timer + c.setFirstPingTimer() + } + + // For routes, the "client" is added to s.routes only when processing + // the INFO protocol, that is much later. + // In the meantime, if the server shutsdown, there would be no reference + // to the client (connection) to be closed, leaving this readLoop + // uinterrupted, causing the Shutdown() to wait indefinitively. + // We need to store the client in a special map, under a special lock. + if !s.addToTempClients(c.cid, c) { + c.mu.Unlock() + c.setNoReconnect() + c.closeConnection(ServerShutdown) + return nil + } + + // Check for Auth required state for incoming connections. + // Make sure to do this before spinning up readLoop. + if authRequired && !didSolicit { + ttl := secondsToDuration(opts.Cluster.AuthTimeout) + c.setAuthTimer(ttl) + } + + // Spin up the read loop. + s.startGoRoutine(func() { c.readLoop(nil) }) + + // Spin up the write loop. + s.startGoRoutine(func() { c.writeLoop() }) + + if tlsRequired { + c.Debugf("TLS handshake complete") + cs := c.nc.(*tls.Conn).ConnectionState() + c.Debugf("TLS version %s, cipher suite %s", tlsVersion(cs.Version), tls.CipherSuiteName(cs.CipherSuite)) + } + + // Queue Connect proto if we solicited the connection. + if didSolicit { + c.Debugf("Route connect msg sent") + if err := c.sendRouteConnect(clusterName, tlsRequired); err != nil { + c.mu.Unlock() + c.closeConnection(ProtocolViolation) + return nil + } + } + + if !delayInfo { + // Send our info to the other side. + // Our new version requires dynamic information for accounts and a nonce. + c.enqueueProto(infoJSON) + } + c.mu.Unlock() + + c.Noticef("Route connection created") + return c +} + +func routeShouldDelayInfo(accName string, opts *Options) bool { + return accName == _EMPTY_ && opts.Cluster.PoolSize >= 1 +} + +// Generates a nonce and set some route info's fields before marshal'ing into JSON. +// To be used only when a route is created (to send the initial INFO protocol). +// +// Server lock held on entry. +func (s *Server) generateRouteInitialInfoJSON(accName, compression string, poolIdx int, gossipMode byte) []byte { + // New proto wants a nonce (although not used in routes, that is, not signed in CONNECT) + var raw [nonceLen]byte + nonce := raw[:] + s.generateNonce(nonce) + ri := &s.routeInfo + // Override compression with s2_auto instead of actual compression level. + if s.getOpts().Cluster.Compression.Mode == CompressionS2Auto { + compression = CompressionS2Auto + } + ri.Nonce, ri.RouteAccount, ri.RoutePoolIdx, ri.Compression, ri.GossipMode = string(nonce), accName, poolIdx, compression, gossipMode + infoJSON := generateInfoJSON(&s.routeInfo) + // Clear now that it has been serialized. Will prevent nonce to be included in async INFO that we may send. + // Same for some other fields. + ri.Nonce, ri.RouteAccount, ri.RoutePoolIdx, ri.Compression, ri.GossipMode = _EMPTY_, _EMPTY_, 0, _EMPTY_, 0 + return infoJSON +} + +const ( + _CRLF_ = "\r\n" + _EMPTY_ = "" +) + +func (s *Server) addRoute(c *client, didSolicit, sendDelayedInfo bool, gossipMode byte, info *Info, accName string) bool { + id := info.ID + + var acc *Account + if accName != _EMPTY_ { + var err error + acc, err = s.LookupAccount(accName) + if err != nil { + c.sendErrAndErr(fmt.Sprintf("Unable to lookup account %q: %v", accName, err)) + c.closeConnection(MissingAccount) + return false + } + } + + s.mu.Lock() + if !s.isRunning() || s.routesReject { + s.mu.Unlock() + return false + } + var invProtoErr string + + opts := s.getOpts() + + // Assume we are in pool mode if info.RoutePoolSize is set. We may disable + // in some cases. + pool := info.RoutePoolSize > 0 + // This is used to prevent a server with pooling to constantly trying + // to connect to a server with no pooling (for instance old server) after + // the first connection is established. + var noReconnectForOldServer bool + + // To allow rolling updates, we now allow servers with different pool sizes + // so we will use as the effective pool size here, the max between our + // configured size and the size we receive in the info protocol. + effectivePoolSize := max(s.routesPoolSize, info.RoutePoolSize) + + // If the remote is an old server, info.RoutePoolSize will be 0, or if + // this server's Cluster.PoolSize is negative, we will behave as an old + // server and need to handle things differently. + if info.RoutePoolSize <= 0 || opts.Cluster.PoolSize < 0 { + if accName != _EMPTY_ { + invProtoErr = fmt.Sprintf("Not possible to have a dedicated route for account %q between those servers", accName) + // In this case, make sure this route does not attempt to reconnect + c.setNoReconnect() + } else { + // We will accept, but treat this remote has "no pool" + pool, noReconnectForOldServer = false, true + c.mu.Lock() + c.route.poolIdx = 0 + c.route.noPool = true + c.mu.Unlock() + // Keep track of number of routes like that. We will use that when + // sending subscriptions over routes. + s.routesNoPool++ + } + } else if didSolicit { + // For solicited route, the incoming's RoutePoolIdx should not be set. + if info.RoutePoolIdx != 0 { + invProtoErr = fmt.Sprintf("Route pool index should not be set but is set to %v", info.RoutePoolIdx) + } + } else if info.RoutePoolIdx < 0 || info.RoutePoolIdx >= effectivePoolSize { + // For non solicited routes, if the remote sends a RoutePoolIdx, make + // sure it is a valid one (in range of the pool size). + invProtoErr = fmt.Sprintf("Invalid route pool index: %v - pool size is %v", info.RoutePoolIdx, info.RoutePoolSize) + } + if invProtoErr != _EMPTY_ { + s.mu.Unlock() + c.sendErrAndErr(invProtoErr) + c.closeConnection(ProtocolViolation) + return false + } + // If accName is set, we are dealing with a per-account connection. + if accName != _EMPTY_ { + // When an account has its own route, it will be an error if the given + // account name is not found in s.accRoutes map. + conns, exists := s.accRoutes[accName] + if !exists { + s.mu.Unlock() + c.sendErrAndErr(fmt.Sprintf("No route for account %q", accName)) + c.closeConnection(ProtocolViolation) + return false + } + remote, exists := conns[id] + if !exists { + conns[id] = c + c.mu.Lock() + idHash := c.route.idHash + cid := c.cid + rtype := c.route.routeType + if sendDelayedInfo { + cm := compressionModeForInfoProtocol(&opts.Cluster.Compression, c.route.compression) + c.enqueueProto(s.generateRouteInitialInfoJSON(accName, cm, 0, gossipMode)) + } + if c.last.IsZero() { + c.last = time.Now() + } + if acc != nil { + c.acc = acc + } + c.mu.Unlock() + + // Store this route with key being the route id hash + account name + s.storeRouteByHash(idHash+accName, c) + + // Now that we have registered the route, we can remove from the temp map. + s.removeFromTempClients(cid) + + // We don't need to send if the only route is the one we just accepted. + if len(conns) > 1 { + s.forwardNewRouteInfoToKnownServers(info, rtype, didSolicit, gossipMode) + } + + // Send subscription interest + s.sendSubsToRoute(c, -1, accName) + } else { + handleDuplicateRoute(remote, c, true) + } + s.mu.Unlock() + return !exists + } + var remote *client + // That will be the position of the connection in the slice, we initialize + // to -1 to indicate that no space was found. + idx := -1 + // This will be the size (or number of connections) in a given slice. + sz := 0 + // Check if we know about the remote server + conns, exists := s.routes[id] + if !exists { + // Now, create a slice for route connections of the size of the pool + // or 1 when not in pool mode. + conns = make([]*client, effectivePoolSize) + // Track this slice for this remote server. + s.routes[id] = conns + // Set the index to info.RoutePoolIdx because if this is a solicited + // route, this value will be 0, which is what we want, otherwise, we + // will use whatever index the remote has chosen. + idx = info.RoutePoolIdx + } else if pool { + // The remote could have done a config reload and increased the pool size. + // It will close the connections before soliciting again, however, if + // on this side, one of the route is not yet fully removed, but the + // first one is, it would accept the new connection (with a greater pool + // size) and we would not go through the phase of `!exists` above creating + // the slice with the right size. So we need to check here and add new empty + // entries to complete the effective pool size. + if n := effectivePoolSize - len(conns); n > 0 { + for range n { + conns = append(conns, nil) + } + s.routes[id] = conns + } + // The remote was found. If this is a non solicited route, we will place + // the connection in the pool at the index given by info.RoutePoolIdx. + // But if there is already one, close this incoming connection as a + // duplicate. + if !didSolicit { + idx = info.RoutePoolIdx + if remote = conns[idx]; remote != nil { + handleDuplicateRoute(remote, c, false) + s.mu.Unlock() + return false + } + // Look if there is a solicited route in the pool. If there is one, + // they should all be, so stop at the first. + if url, rtype, hasSolicited := hasSolicitedRoute(conns); hasSolicited { + upgradeRouteToSolicited(c, url, rtype) + } + } else { + // If we solicit, upgrade to solicited all non-solicited routes that + // we may have registered. + c.mu.Lock() + url := c.route.url + rtype := c.route.routeType + c.mu.Unlock() + for _, r := range conns { + upgradeRouteToSolicited(r, url, rtype) + } + } + // For all cases (solicited and not) we need to count how many connections + // we already have, and for solicited route, we will find a free spot in + // the slice. + for i, r := range conns { + if idx == -1 && r == nil { + idx = i + } else if r != nil { + sz++ + } + } + } else { + remote = conns[0] + } + // If there is a spot, idx will be greater or equal to 0. + if idx >= 0 { + c.mu.Lock() + c.route.connectURLs = info.ClientConnectURLs + c.route.wsConnURLs = info.WSConnectURLs + c.route.poolIdx = idx + rtype := c.route.routeType + cid := c.cid + idHash := c.route.idHash + rHash := c.route.hash + rn := c.route.remoteName + url := c.route.url + if sendDelayedInfo { + cm := compressionModeForInfoProtocol(&opts.Cluster.Compression, c.route.compression) + c.enqueueProto(s.generateRouteInitialInfoJSON(_EMPTY_, cm, idx, gossipMode)) + } + if c.last.IsZero() { + c.last = time.Now() + } + c.mu.Unlock() + + // With pooling, we keep track of the remote's configured route pool size. + // We do so when adding the connection in the first slot, not when `sz == 1` + // because there could be situations where we have old connections that have + // not yet been removed and so we would not have `sz == `. However, we will + // always have the condition where we are adding the new connection at `idx==0` + // so use that as the condition to store the remote pool size. + if pool && idx == 0 { + if s.remoteRoutePoolSize == nil { + s.remoteRoutePoolSize = make(map[string]int) + } + s.remoteRoutePoolSize[id] = info.RoutePoolSize + } + + // Add to the slice and bump the count of connections for this remote + conns[idx] = c + sz++ + // This boolean will indicate that we are registering the only + // connection in non pooled situation or we stored the very first + // connection for a given remote server. + doOnce := !pool || sz == 1 + if doOnce { + // check to be consistent and future proof. but will be same domain + if s.sameDomain(info.Domain) { + s.nodeToInfo.Store(rHash, nodeInfo{ + name: rn, + version: s.info.Version, + cluster: s.info.Cluster, + domain: info.Domain, + id: id, + tags: nil, + cfg: nil, + stats: nil, + offline: false, + js: info.JetStream, + binarySnapshots: true, // Updated default to true. Versions 2.10.0+ support it. + accountNRG: false, + }) + } + } + + // Store this route using the hash as the key + if pool { + idHash += strconv.Itoa(idx) + } + s.storeRouteByHash(idHash, c) + + // Now that we have registered the route, we can remove from the temp map. + s.removeFromTempClients(cid) + + if doOnce { + // If the INFO contains a Gateway URL, add it to the list for our cluster. + if info.GatewayURL != _EMPTY_ && s.addGatewayURL(info.GatewayURL) { + s.sendAsyncGatewayInfo() + } + + // We don't need to send if the only route is the one we just accepted. + if len(s.routes) > 1 { + s.forwardNewRouteInfoToKnownServers(info, rtype, didSolicit, gossipMode) + } + + // Send info about the known gateways to this route. + s.sendGatewayConfigsToRoute(c) + + // Unless disabled, possibly update the server's INFO protocol + // and send to clients that know how to handle async INFOs. + if !opts.Cluster.NoAdvertise { + s.addConnectURLsAndSendINFOToClients(info.ClientConnectURLs, info.WSConnectURLs) + } + + // Add the remote's leafnodeURL to our list of URLs and send the update + // to all LN connections. (Note that when coming from a route, LeafNodeURLs + // is an array of size 1 max). + if len(info.LeafNodeURLs) == 1 && s.addLeafNodeURL(info.LeafNodeURLs[0]) { + s.sendAsyncLeafNodeInfo() + } + } + + // Send the subscriptions interest. + s.sendSubsToRoute(c, idx, _EMPTY_) + + // In pool mode, if we did not yet reach the cap, try to connect a new connection, + // but do so only after receiving the first PONG to our PING, which will ensure + // that we have proper authentication. + if pool && didSolicit && sz != effectivePoolSize { + c.mu.Lock() + c.route.startNewRoute = &routeInfo{ + url: url, + rtype: rtype, + gossipMode: gossipMode, + } + c.sendPing() + c.mu.Unlock() + } + } + s.mu.Unlock() + if pool { + if idx == -1 { + // Was full, so need to close connection + c.Debugf("Route pool size reached, closing extra connection to %q", id) + handleDuplicateRoute(nil, c, true) + return false + } + return true + } + // This is for non-pool mode at this point. + if exists { + handleDuplicateRoute(remote, c, noReconnectForOldServer) + } + + return !exists +} + +func hasSolicitedRoute(conns []*client) (*url.URL, RouteType, bool) { + var url *url.URL + var rtype RouteType + for _, r := range conns { + if r == nil { + continue + } + r.mu.Lock() + if r.route.didSolicit { + url = r.route.url + rtype = r.route.routeType + } + r.mu.Unlock() + if url != nil { + return url, rtype, true + } + } + return nil, 0, false +} + +func upgradeRouteToSolicited(r *client, url *url.URL, rtype RouteType) { + if r == nil { + return + } + r.mu.Lock() + // On disconnect, an explicit route only reconnects if the connection's + // URL matches a configured route. So adopt the given URL rather than + // keeping a gossiped one, which may not match any configured route. + if !r.route.didSolicit || (rtype == Explicit && r.route.routeType != Explicit) { + r.route.url = url + } + r.route.didSolicit = true + if rtype == Explicit { + r.route.routeType = Explicit + } + r.mu.Unlock() +} + +func handleDuplicateRoute(remote, c *client, setNoReconnect bool) { + // We used to clear some fields when closing a duplicate connection + // to prevent sending INFO protocols for the remotes to update + // their leafnode/gateway URLs. This is no longer needed since + // removeRoute() now does the right thing of doing that only when + // the closed connection was an added route connection. + c.mu.Lock() + didSolicit := c.route.didSolicit + url := c.route.url + rtype := c.route.routeType + if setNoReconnect { + c.flags.set(noReconnect) + } + c.mu.Unlock() + + if remote == nil { + return + } + + remote.mu.Lock() + if didSolicit { + // On disconnect, an explicit route only reconnects if the connection's + // URL matches a configured route. So adopt the given URL rather than + // keeping a gossiped one, which may not match any configured route. + if !remote.route.didSolicit || (rtype == Explicit && remote.route.routeType != Explicit) { + remote.route.url = url + } + remote.route.didSolicit = true + } + // The extra route might be an configured explicit route + // so keep the state that the remote was configured. + if rtype == Explicit { + remote.route.routeType = rtype + } + // This is to mitigate the issue where both sides add the route + // on the opposite connection, and therefore end-up with both + // connections being dropped. + remote.route.retry = true + remote.mu.Unlock() +} + +// Import filter check. +func (c *client) importFilter(sub *subscription) bool { + if c.perms == nil { + return true + } + return c.canImport(string(sub.subject)) +} + +// updateRouteSubscriptionMap will make sure to update the route map for the subscription. Will +// also forward to all routes if needed. +func (s *Server) updateRouteSubscriptionMap(acc *Account, sub *subscription, delta int32) { + if acc == nil || sub == nil { + return + } + + // We only store state on local subs for transmission across all other routes. + if sub.client == nil || sub.client.kind == ROUTER || sub.client.kind == GATEWAY { + return + } + + if sub.si { + return + } + + // Copy to hold outside acc lock. + var n int32 + var ok bool + + isq := len(sub.queue) > 0 + + accLock := func() { + // Not required for code correctness, but helps reduce the number of + // updates sent to the routes when processing high number of concurrent + // queue subscriptions updates (sub/unsub). + // See https://github.com/nats-io/nats-server/pull/1126 for more details. + if isq { + acc.smu.Lock() + } + acc.mu.Lock() + } + accUnlock := func() { + acc.mu.Unlock() + if isq { + acc.smu.Unlock() + } + } + + accLock() + + // This is non-nil when we know we are in cluster mode. + rm, lws := acc.rm, acc.lws + if rm == nil { + accUnlock() + return + } + + // Create the subscription key which will prevent collisions between regular + // and leaf routed subscriptions. See keyFromSubWithOrigin() for details. + key := keyFromSubWithOrigin(sub) + + // Decide whether we need to send an update out to all the routes. + update := isq + + // This is where we do update to account. For queues we need to take + // special care that this order of updates is same as what is sent out + // over routes. + if n, ok = rm[key]; ok { + n += delta + if n <= 0 { + delete(rm, key) + delete(lws, key) + update = true // Update for deleting (N->0) + } else { + rm[key] = n + } + } else if delta > 0 { + n = delta + rm[key] = delta + update = true // Adding a new entry for normal sub means update (0->1) + } + + accUnlock() + + if !update { + return + } + + // If we are sending a queue sub, make a copy and place in the queue weight. + // FIXME(dlc) - We can be smarter here and avoid copying and acquiring the lock. + if isq { + sub.client.mu.Lock() + nsub := *sub + sub.client.mu.Unlock() + nsub.qw = n + sub = &nsub + } + + // We need to send out this update. Gather routes + var _routes [32]*client + routes := _routes[:0] + + s.mu.RLock() + // The account's routePoolIdx field is set/updated under the server lock + // (but also the account's lock). So we don't need to acquire the account's + // lock here to get the value. + if poolIdx := acc.routePoolIdx; poolIdx < 0 { + if conns, ok := s.accRoutes[acc.Name]; ok { + for _, r := range conns { + routes = append(routes, r) + } + } + if s.routesNoPool > 0 { + // We also need to look for "no pool" remotes (that is, routes to older + // servers or servers that have explicitly disabled pooling). + s.forEachRemote(func(r *client) { + r.mu.Lock() + if r.route.noPool { + routes = append(routes, r) + } + r.mu.Unlock() + }) + } + } else { + // We can't use s.forEachRouteIdx here since we want to check/get the + // "no pool" route ONLY if we don't find a route at the given `poolIdx`. + for _, conns := range s.routes { + if r := conns[poolIdx]; r != nil { + routes = append(routes, r) + } else if s.routesNoPool > 0 { + // Check if we have a "no pool" route at index 0, and if so, it + // means that for this remote, we have a single connection because + // that server does not have pooling. + if r := conns[0]; r != nil { + r.mu.Lock() + if r.route.noPool { + routes = append(routes, r) + } + r.mu.Unlock() + } + } + } + } + trace := atomic.LoadInt32(&s.logging.trace) == 1 + s.mu.RUnlock() + + // We need to make sure our updates are serialized from potential multiple connections. We want + // to make sure that the order above is preserved here but not necessarily all updates need to + // be sent. We need to block and recheck the n count with the lock held through sending here. + // + // However, we can't hold the acc.mu lock since we allow client.mu.Lock -> acc.mu.Lock + // but not the opposite. So use a dedicated lock while holding the route's lock. + acc.smu.Lock() + defer acc.smu.Unlock() + + acc.mu.Lock() + n = rm[key] + if isq { + sub.qw = n + } + // Check the last sent value here. If same, then someone beat us to it and + // we can just return here. Otherwise update. + if ls, ok := lws[key]; ok && ls == n { + acc.mu.Unlock() + return + } else if n > 0 { + lws[key] = n + } + acc.mu.Unlock() + + // Snapshot into array + subs := []*subscription{sub} + + // Deliver to all routes. + for _, route := range routes { + route.mu.Lock() + // Note that queue unsubs where n > 0 are still + // subscribes with a smaller weight. + route.sendRouteSubOrUnSubProtos(subs, n > 0, trace, route.importFilter) + route.mu.Unlock() + } +} + +// This starts the route accept loop in a go routine, unless it +// is detected that the server has already been shutdown. +// It will also start soliciting explicit routes. +func (s *Server) startRouteAcceptLoop() { + if s.isShuttingDown() { + return + } + + // Snapshot server options. + opts := s.getOpts() + + // Snapshot server options. + port := opts.Cluster.Port + + if port == -1 { + port = 0 + } + + // This requires lock, so do this outside of may block. + clusterName := s.ClusterName() + + s.mu.Lock() + s.Noticef("Cluster name is %s", clusterName) + if s.isClusterNameDynamic() { + s.Warnf("Cluster name was dynamically generated, consider setting one") + } + + hp := net.JoinHostPort(opts.Cluster.Host, strconv.Itoa(port)) + l, e := natsListen("tcp", hp) + s.routeListenerErr = e + if e != nil { + s.mu.Unlock() + s.Fatalf("Error listening on router port: %d - %v", opts.Cluster.Port, e) + return + } + s.Noticef("Listening for route connections on %s", + net.JoinHostPort(opts.Cluster.Host, strconv.Itoa(l.Addr().(*net.TCPAddr).Port))) + + // Check for TLSConfig + tlsReq := opts.Cluster.TLSConfig != nil + info := Info{ + ID: s.info.ID, + Name: s.info.Name, + Version: s.info.Version, + GoVersion: runtime.Version(), + AuthRequired: false, + TLSRequired: tlsReq, + TLSVerify: tlsReq, + MaxPayload: s.info.MaxPayload, + JetStream: s.info.JetStream, + Proto: s.getServerProto(), + GatewayURL: s.getGatewayURL(), + Headers: s.supportsHeaders(), + Cluster: s.info.Cluster, + Domain: s.info.Domain, + Dynamic: s.isClusterNameDynamic(), + LNOC: true, + LNOCU: true, + LN: true, + } + // For tests that want to simulate old servers, do not set the compression + // on the INFO protocol if configured with CompressionNotSupported. + if cm := opts.Cluster.Compression.Mode; cm != CompressionNotSupported { + info.Compression = cm + } + if ps := opts.Cluster.PoolSize; ps > 0 { + info.RoutePoolSize = ps + } + // Set this if only if advertise is not disabled + if !opts.Cluster.NoAdvertise { + info.ClientConnectURLs = s.clientConnectURLs + info.WSConnectURLs = s.websocket.connectURLs + } + // If we have selected a random port... + if port == 0 { + // Write resolved port back to options. + opts.Cluster.Port = l.Addr().(*net.TCPAddr).Port + } + // Check for Auth items + if opts.Cluster.Username != "" { + info.AuthRequired = true + } + // Check for permissions. + if opts.Cluster.Permissions != nil { + info.Import = opts.Cluster.Permissions.Import + info.Export = opts.Cluster.Permissions.Export + } + // If this server has a LeafNode accept loop, s.leafNodeInfo.IP is, + // at this point, set to the host:port for the leafnode accept URL, + // taking into account possible advertise setting. Use the LeafNodeURLs + // and set this server's leafnode accept URL. This will be sent to + // routed servers. + if !opts.LeafNode.NoAdvertise && s.leafNodeInfo.IP != _EMPTY_ { + info.LeafNodeURLs = []string{s.leafNodeInfo.IP} + } + s.routeInfo = info + // Possibly override Host/Port and set IP based on Cluster.Advertise + if err := s.setRouteInfoHostPortAndIP(); err != nil { + s.Fatalf("Error setting route INFO with Cluster.Advertise value of %s, err=%v", opts.Cluster.Advertise, err) + l.Close() + s.mu.Unlock() + return + } + // Setup state that can enable shutdown + s.routeListener = l + // Warn if using Cluster.Insecure + if tlsReq && opts.Cluster.TLSConfig.InsecureSkipVerify { + s.Warnf(clusterTLSInsecureWarning) + } + + // Now that we have the port, keep track of all ip:port that resolve to this server. + if interfaceAddr, err := net.InterfaceAddrs(); err == nil { + var localIPs []string + for i := 0; i < len(interfaceAddr); i++ { + interfaceIP, _, _ := net.ParseCIDR(interfaceAddr[i].String()) + ipStr := interfaceIP.String() + if net.ParseIP(ipStr) != nil { + localIPs = append(localIPs, ipStr) + } + } + var portStr = strconv.FormatInt(int64(s.routeInfo.Port), 10) + for _, ip := range localIPs { + ipPort := net.JoinHostPort(ip, portStr) + s.routesToSelf[ipPort] = struct{}{} + } + } + + // Start the accept loop in a different go routine. + go s.acceptConnections(l, "Route", func(conn net.Conn) { s.createRoute(conn, nil, Implicit, gossipDefault, _EMPTY_) }, nil) + + // Solicit Routes if applicable. This will not block. + s.solicitRoutes(opts.Routes, opts.Cluster.PinnedAccounts) + + s.mu.Unlock() +} + +// Similar to setInfoHostPortAndGenerateJSON, but for routeInfo. +func (s *Server) setRouteInfoHostPortAndIP() error { + opts := s.getOpts() + if opts.Cluster.Advertise != _EMPTY_ { + advHost, advPort, err := parseHostPort(opts.Cluster.Advertise, opts.Cluster.Port) + if err != nil { + return err + } + s.routeInfo.Host = advHost + s.routeInfo.Port = advPort + s.routeInfo.IP = fmt.Sprintf("nats-route://%s/", net.JoinHostPort(advHost, strconv.Itoa(advPort))) + } else { + s.routeInfo.Host = opts.Cluster.Host + s.routeInfo.Port = opts.Cluster.Port + s.routeInfo.IP = "" + } + return nil +} + +// StartRouting will start the accept loop on the cluster host:port +// and will actively try to connect to listed routes. +func (s *Server) StartRouting(clientListenReady chan struct{}) { + defer s.grWG.Done() + + // Wait for the client and leafnode listen ports to be opened, + // and the possible ephemeral ports to be selected. + <-clientListenReady + + // Start the accept loop and solicitation of explicit routes (if applicable) + s.startRouteAcceptLoop() + +} + +func (s *Server) reConnectToRoute(rURL *url.URL, rtype RouteType, accName string) { + // If A connects to B, and B to A (regardless if explicit or + // implicit - due to auto-discovery), and if each server first + // registers the route on the opposite TCP connection, the + // two connections will end-up being closed. + // Add some random delay to reduce risk of repeated failures. + delay := time.Duration(rand.IntN(100)) * time.Millisecond + if rtype == Explicit { + delay += routeReconnectDelay + } + select { + case <-time.After(delay): + case <-s.quitCh: + s.grWG.Done() + return + } + s.connectToRoute(rURL, rtype, false, gossipDefault, accName) +} + +// Checks to make sure the route is still valid. +func (s *Server) routeStillValid(rURL *url.URL) bool { + for _, ri := range s.getOpts().Routes { + if urlsAreEqual(ri, rURL) { + return true + } + } + return false +} + +func (s *Server) connectToRoute(rURL *url.URL, rtype RouteType, firstConnect bool, gossipMode byte, accName string) { + defer s.grWG.Done() + if rURL == nil { + return + } + // For explicit routes, we will try to connect until we succeed. For implicit + // we will try only based on the number of ConnectRetries optin. + tryForEver := rtype == Explicit + + // Snapshot server options. + opts := s.getOpts() + + const connErrFmt = "Error trying to connect to route (attempt %v): %v" + + s.mu.RLock() + resolver := s.routeResolver + excludedAddresses := s.routesToSelf + s.mu.RUnlock() + + attemptDelay := routeConnectDelay + reconnectTimer := time.NewTimer(attemptDelay) + reconnectTimer.Stop() + defer stopAndClearTimer(&reconnectTimer) + + for attempts := 0; s.isRunning(); { + if tryForEver { + if !s.routeStillValid(rURL) { + s.Debugf("Not attempting to connect to explicit route %q, no longer part of configured routes", rURL.Redacted()) + return + } + if accName != _EMPTY_ { + s.mu.RLock() + _, valid := s.accRoutes[accName] + s.mu.RUnlock() + if !valid { + s.Debugf("Not attempting to connect to route %q for account %q, no longer a per-account route", rURL.Redacted(), accName) + return + } + } + } + var conn net.Conn + address, err := s.getRandomIP(resolver, rURL.Host, excludedAddresses) + if err == errNoIPAvail { + // This is ok, we are done. + return + } + if err == nil { + s.Debugf("Trying to connect to route on %s (%s)", rURL.Host, address) + conn, err = natsDialTimeout("tcp", address, DEFAULT_ROUTE_DIAL) + } + if err != nil { + attempts++ + if s.shouldReportConnectErr(firstConnect, attempts) { + s.Errorf(connErrFmt, attempts, err) + } else { + s.Debugf(connErrFmt, attempts, err) + } + if !tryForEver { + if opts.Cluster.ConnectRetries <= 0 { + return + } + if attempts > opts.Cluster.ConnectRetries { + return + } + } + reconnectTimer.Reset(attemptDelay) + select { + case <-s.quitCh: + return + case <-reconnectTimer.C: + if opts.Cluster.ConnectBackoff { + // Use exponential backoff for connection attempts. + attemptDelay *= 2 + if attemptDelay > routeConnectMaxDelay { + attemptDelay = routeConnectMaxDelay + } + } + continue + } + } + + if tryForEver && !s.routeStillValid(rURL) { + conn.Close() + return + } + + // We have a route connection here. + // Go ahead and create it and exit this func. + s.createRoute(conn, rURL, rtype, gossipMode, accName) + return + } +} + +func (c *client) isSolicitedRoute() bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.kind == ROUTER && c.route != nil && c.route.didSolicit +} + +// Save the first hostname found in route URLs. This will be used in gossip mode +// when trying to create a TLS connection by setting the tlsConfig.ServerName. +// Lock is held on entry +func (s *Server) saveRouteTLSName(routes []*url.URL) { + for _, u := range routes { + if s.routeTLSName == _EMPTY_ && net.ParseIP(u.Hostname()) == nil { + s.routeTLSName = u.Hostname() + } + } +} + +// Start connection process to provided routes. Each route connection will +// be started in a dedicated go routine. +// Lock is held on entry +func (s *Server) solicitRoutes(routes []*url.URL, accounts []string) { + s.saveRouteTLSName(routes) + for _, r := range routes { + route := r + s.startGoRoutine(func() { s.connectToRoute(route, Explicit, true, gossipDefault, _EMPTY_) }) + } + // Now go over possible per-account routes and create them. + for _, an := range accounts { + for _, r := range routes { + route, accName := r, an + s.startGoRoutine(func() { s.connectToRoute(route, Explicit, true, gossipDefault, accName) }) + } + } +} + +func (c *client) processRouteConnect(srv *Server, arg []byte, lang string) error { + // Way to detect clients that incorrectly connect to the route listen + // port. Client provide Lang in the CONNECT protocol while ROUTEs don't. + if lang != "" { + c.sendErrAndErr(ErrClientConnectedToRoutePort.Error()) + c.closeConnection(WrongPort) + return ErrClientConnectedToRoutePort + } + // Unmarshal as a route connect protocol + proto := &connectInfo{} + + if err := json.Unmarshal(arg, proto); err != nil { + return err + } + // Reject if this has Gateway which means that it would be from a gateway + // connection that incorrectly connects to the Route port. + if proto.Gateway != "" { + errTxt := fmt.Sprintf("Rejecting connection from gateway %q on the Route port", proto.Gateway) + c.Errorf(errTxt) + c.sendErr(errTxt) + c.closeConnection(WrongGateway) + return ErrWrongGateway + } + + if srv == nil { + return ErrServerNotRunning + } + if proto.Cluster == leafNoOriginCluster { + errTxt := fmt.Sprintf("Rejecting connection, cluster name %q is reserved", proto.Cluster) + c.Errorf(errTxt) + c.sendErr(errTxt) + c.closeConnection(ClusterNameConflict) + return ErrClusterNameReserved + } + + perms := srv.getOpts().Cluster.Permissions + clusterName := srv.ClusterName() + + // If we have a cluster name set, make sure it matches ours. + if proto.Cluster != clusterName { + shouldReject := true + // If we have a dynamic name we will do additional checks. + if srv.isClusterNameDynamic() { + if !proto.Dynamic || strings.Compare(clusterName, proto.Cluster) < 0 { + // We will take on their name since theirs is configured or higher then ours. + srv.setClusterName(proto.Cluster) + if !proto.Dynamic { + srv.optsMu.Lock() + srv.opts.Cluster.Name = proto.Cluster + srv.optsMu.Unlock() + } + c.mu.Lock() + remoteID := c.opts.Name + c.mu.Unlock() + srv.removeAllRoutesExcept(remoteID) + shouldReject = false + } + } + if shouldReject { + errTxt := fmt.Sprintf("Rejecting connection, cluster name %q does not match %q", proto.Cluster, clusterName) + c.Errorf(errTxt) + c.sendErr(errTxt) + c.closeConnection(ClusterNameConflict) + return ErrClusterNameRemoteConflict + } + } + + supportsHeaders := c.srv.supportsHeaders() + + // Grab connection name of remote route. + c.mu.Lock() + c.route.remoteID = c.opts.Name + c.route.ln = proto.LN // ... also implies LNOC+LNOCU + c.route.lnoc = proto.LNOC || c.route.ln + c.route.lnocu = proto.LNOCU || c.route.ln + c.setRoutePermissions(perms) + c.headers = supportsHeaders && proto.Headers + c.mu.Unlock() + return nil +} + +// Called when we update our cluster name during negotiations with remotes. +func (s *Server) removeAllRoutesExcept(remoteID string) { + s.mu.Lock() + routes := make([]*client, 0, s.numRoutes()) + for rID, conns := range s.routes { + if rID == remoteID { + continue + } + for _, r := range conns { + if r != nil { + routes = append(routes, r) + } + } + } + for _, conns := range s.accRoutes { + for rID, r := range conns { + if rID == remoteID { + continue + } + routes = append(routes, r) + } + } + s.mu.Unlock() + + for _, r := range routes { + r.closeConnection(ClusterNameConflict) + } +} + +func (s *Server) removeRoute(c *client) { + s.mu.Lock() + defer s.mu.Unlock() + + var ( + rID string + lnURL string + gwURL string + idHash string + accName string + poolIdx = -1 + connectURLs []string + wsConnectURLs []string + opts = s.getOpts() + rURL *url.URL + noPool bool + rtype RouteType + ) + c.mu.Lock() + cid := c.cid + r := c.route + if r != nil { + rID = r.remoteID + lnURL = r.leafnodeURL + idHash = r.idHash + gwURL = r.gatewayURL + poolIdx = r.poolIdx + accName = bytesToString(r.accName) + if r.noPool { + s.routesNoPool-- + noPool = true + } + connectURLs = r.connectURLs + wsConnectURLs = r.wsConnURLs + rURL = r.url + rtype = r.routeType + } + c.mu.Unlock() + if accName != _EMPTY_ { + if conns, ok := s.accRoutes[accName]; ok { + if r := conns[rID]; r == c { + s.removeRouteByHash(idHash + accName) + delete(conns, rID) + // Do not remove or set to nil when all remotes have been + // removed from the map. The configured accounts must always + // be in the accRoutes map and addRoute expects "conns" map + // to be created. + } + } + } + // If this is still -1, it means that it was not added to the routes + // so simply remove from temp clients and we are done. + if poolIdx == -1 || accName != _EMPTY_ { + s.removeFromTempClients(cid) + return + } + if conns, ok := s.routes[rID]; ok { + // If this route was not the one stored, simply remove from the + // temporary map and be done. + if conns[poolIdx] != c { + s.removeFromTempClients(cid) + return + } + conns[poolIdx] = nil + // Now check if this was the last connection to be removed. + empty := true + for _, c := range conns { + if c != nil { + empty = false + break + } + } + // This was the last route for this remote. Remove the remote entry + // and possibly send some async INFO protocols regarding gateway + // and leafnode URLs. + if empty { + delete(s.routes, rID) + + // Since this is the last route for this remote, possibly update + // the client connect URLs and send an update to connected + // clients. + if (len(connectURLs) > 0 || len(wsConnectURLs) > 0) && !opts.Cluster.NoAdvertise { + s.removeConnectURLsAndSendINFOToClients(connectURLs, wsConnectURLs) + } + // Remove the remote's gateway URL from our list and + // send update to inbound Gateway connections. + if gwURL != _EMPTY_ && s.removeGatewayURL(gwURL) { + s.sendAsyncGatewayInfo() + } + // Remove the remote's leafNode URL from + // our list and send update to LN connections. + if lnURL != _EMPTY_ && s.removeLeafNodeURL(lnURL) { + s.sendAsyncLeafNodeInfo() + } + // We can remove the configured route pool size of this remote. + delete(s.remoteRoutePoolSize, rID) + // If this server has pooling/pinned accounts and the route for + // this remote was a "no pool" route, attempt to reconnect. + if noPool { + if s.routesPoolSize > 1 { + s.startGoRoutine(func() { s.connectToRoute(rURL, rtype, true, gossipDefault, _EMPTY_) }) + } + if len(opts.Cluster.PinnedAccounts) > 0 { + for _, an := range opts.Cluster.PinnedAccounts { + accName := an + s.startGoRoutine(func() { s.connectToRoute(rURL, rtype, true, gossipDefault, accName) }) + } + } + } + } + // This is for gateway code. Remove this route from a map that uses + // the route hash in combination with the pool index as the key. + if s.routesPoolSize > 1 { + idHash += strconv.Itoa(poolIdx) + } + s.removeRouteByHash(idHash) + } + s.removeFromTempClients(cid) +} + +func (s *Server) isDuplicateServerName(name string) bool { + if name == _EMPTY_ { + return false + } + s.mu.RLock() + defer s.mu.RUnlock() + + if s.info.Name == name { + return true + } + for _, conns := range s.routes { + for _, r := range conns { + if r != nil { + r.mu.Lock() + duplicate := r.route.remoteName == name + r.mu.Unlock() + if duplicate { + return true + } + break + } + } + } + return false +} + +// Goes over each non-nil route connection for all remote servers +// and invokes the function `f`. It does not go over per-account +// routes. +// Server lock is held on entry. +func (s *Server) forEachNonPerAccountRoute(f func(r *client)) { + for _, conns := range s.routes { + for _, r := range conns { + if r != nil { + f(r) + } + } + } +} + +// Goes over each non-nil route connection for all remote servers +// and invokes the function `f`. This also includes the per-account +// routes. +// Server lock is held on entry. +func (s *Server) forEachRoute(f func(r *client)) { + s.forEachNonPerAccountRoute(f) + for _, conns := range s.accRoutes { + for _, r := range conns { + f(r) + } + } +} + +// Goes over each non-nil route connection at the given pool index +// location in the slice and invokes the function `f`. If the +// callback returns `true`, this function moves to the next remote. +// Otherwise, the iteration over removes stops. +// This does not include per-account routes. +// Server lock is held on entry. +func (s *Server) forEachRouteIdx(idx int, f func(r *client) bool) { + for _, conns := range s.routes { + if r := conns[idx]; r != nil { + if !f(r) { + return + } + } + } +} + +// Goes over each remote and for the first non nil route connection, +// invokes the function `f`. +// Server lock is held on entry. +func (s *Server) forEachRemote(f func(r *client)) { + for _, conns := range s.routes { + for _, r := range conns { + if r != nil { + f(r) + break + } + } + } +} diff --git a/server/scheduler.go b/server/scheduler.go new file mode 100644 index 0000000..67ad723 --- /dev/null +++ b/server/scheduler.go @@ -0,0 +1,383 @@ +// Copyright 2025-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "encoding/binary" + "errors" + "io" + "math" + "slices" + "strings" + "time" + + "github.com/nats-io/nats-server/v2/server/thw" +) + +// Error for when we try to decode a binary-encoded message schedule with an unknown version number. +var ErrMsgScheduleInvalidVersion = errors.New("msg scheduling: encoded version not known") + +const ( + headerLen = 17 // 1 byte magic + 2x uint64s +) + +type MsgScheduling struct { + run func() + ttls *thw.HashWheel + timer *time.Timer + running bool + paused bool + deadline int64 + schedules map[string]*MsgSchedule + seqToSubj map[uint64]string + inflight map[string]struct{} +} + +type MsgSchedule struct { + seq uint64 + ts int64 +} + +func newMsgScheduling(run func()) *MsgScheduling { + return &MsgScheduling{ + run: run, + ttls: thw.NewHashWheel(), + schedules: make(map[string]*MsgSchedule), + seqToSubj: make(map[uint64]string), + inflight: make(map[string]struct{}), + } +} + +func (ms *MsgScheduling) add(seq uint64, subj string, ts int64) { + ms.init(seq, subj, ts) + ms.resetTimer() +} + +func (ms *MsgScheduling) init(seq uint64, subj string, ts int64) { + if sched, ok := ms.schedules[subj]; ok { + delete(ms.seqToSubj, sched.seq) + // Remove and add separately, since they'll have different sequences. + ms.ttls.Remove(sched.seq, sched.ts) + ms.ttls.Add(seq, ts) + sched.ts, sched.seq = ts, seq + } else { + ms.ttls.Add(seq, ts) + ms.schedules[subj] = &MsgSchedule{seq: seq, ts: ts} + } + ms.seqToSubj[seq] = subj + delete(ms.inflight, subj) +} + +func (ms *MsgScheduling) update(subj string, ts int64) { + if sched, ok := ms.schedules[subj]; ok { + // Remove and add separately, it's for the same sequence, but if replicated + // this server could not know the previous timestamp yet. + ms.ttls.Remove(sched.seq, sched.ts) + ms.ttls.Add(sched.seq, ts) + sched.ts = ts + delete(ms.inflight, subj) + ms.resetTimer() + } +} + +func (ms *MsgScheduling) markInflight(subj string) { + if _, ok := ms.schedules[subj]; ok { + ms.inflight[subj] = struct{}{} + } +} + +func (ms *MsgScheduling) isInflight(subj string) bool { + _, ok := ms.inflight[subj] + return ok +} + +func (ms *MsgScheduling) remove(seq uint64) { + if subj, ok := ms.seqToSubj[seq]; ok { + ms.removeSubject(subj) + } +} + +func (ms *MsgScheduling) removeSubject(subj string) { + if sched, ok := ms.schedules[subj]; ok { + ms.ttls.Remove(sched.seq, sched.ts) + delete(ms.schedules, subj) + delete(ms.seqToSubj, sched.seq) + delete(ms.inflight, subj) + } +} + +func (ms *MsgScheduling) clearInflight() { + ms.inflight = make(map[string]struct{}) +} + +func (ms *MsgScheduling) resetTimer() { + if ms.paused { + return + } + // If we're already scheduling messages, it will make sure to reset. + // Don't trigger again, as that could result in many expire goroutines. + if ms.running { + return + } + + next := ms.ttls.GetNextExpiration(math.MaxInt64) + if next == math.MaxInt64 { + clearTimer(&ms.timer) + return + } + fireIn := time.Until(time.Unix(0, next)) + + // Make sure we aren't firing too often either way, otherwise we can + // negatively impact stream ingest performance. + if fireIn < 250*time.Millisecond { + fireIn = 250 * time.Millisecond + } + + // If we want to kick the timer to run later than what was assigned before, don't reset it. + // Otherwise, we could get in a situation where the timer is continuously reset, and it never runs. + deadline := time.Now().UnixNano() + fireIn.Nanoseconds() + if ms.deadline > 0 && deadline > ms.deadline { + return + } + + ms.deadline = deadline + if ms.timer != nil { + ms.timer.Reset(fireIn) + } else { + ms.timer = time.AfterFunc(fireIn, ms.run) + } +} + +func (ms *MsgScheduling) getScheduledMessages(loadMsg func(seq uint64, smv *StoreMsg) *StoreMsg, loadLast func(subj string, smv *StoreMsg) *StoreMsg) []*inMsg { + var ( + smv StoreMsg + srcSmv StoreMsg + sm *StoreMsg + msgs []*inMsg + ) + ms.ttls.ExpireTasks(func(seq uint64, ts int64) bool { + // Need to grab the message for the specified sequence, and check + // if it hasn't been removed in the meantime. + sm = loadMsg(seq, &smv) + if sm != nil { + // If already inflight, don't duplicate a scheduled message. The stream could + // be replicated and the scheduled message could take some time to propagate. + subj := sm.subj + if ms.isInflight(subj) { + return false + } + // Validate the contents are correct if not, we just remove it from THW. + pattern := bytesToString(sliceHeader(JSSchedulePattern, sm.hdr)) + if pattern == _EMPTY_ { + ms.remove(seq) + return true + } + loc, apiErr := loadMessageScheduleLocation(sm.hdr) + if apiErr != nil { + ms.remove(seq) + return true + } + next, repeat, ok := parseMsgSchedule(pattern, loc, ts) + if !ok { + ms.remove(seq) + return true + } + ttl, ok := getMessageScheduleTTL(sm.hdr) + if !ok { + ms.remove(seq) + return true + } + target := getMessageScheduleTarget(sm.hdr) + if target == _EMPTY_ { + ms.remove(seq) + return true + } + rollup := getMessageScheduleRollup(sm.hdr) + source := getMessageScheduleSource(sm.hdr) + if source != _EMPTY_ { + // Fall back to the scheduled message's own content if the source has no last message. + if srcSm := loadLast(source, &srcSmv); srcSm != nil { + sm = srcSm + } + } + + // Copy, as this is retrieved directly from storage, and we'll need to keep hold of this for some time. + // And in the case of headers, we'll copy all of them, but make changes. + hdr, msg := copyBytes(sm.hdr), copyBytes(sm.msg) + + // Strip headers specific to message scheduling. + // Covers Nats-Schedule, Nats-Schedule-*, and Nats-Scheduler. + hdr = removeHeaderIfPrefixPresent(hdr, "Nats-Schedule") + // Strip headers that could prevent persisting this scheduled message. + hdr = removeHeaderIfPrefixPresent(hdr, "Nats-Expected-") + hdr = removeHeaderIfPresent(hdr, JSMsgId) + hdr = removeHeaderIfPresent(hdr, JSMessageTTL) + hdr = removeHeaderIfPresent(hdr, JSMsgRollup) + + // Add headers for the scheduled message. + hdr = genHeader(hdr, JSScheduler, subj) + if !repeat { + hdr = genHeader(hdr, JSScheduleNext, JSScheduleNextPurge) // Purge the schedule message itself. + } else { + hdr = genHeader(hdr, JSScheduleNext, next.Format(time.RFC3339Nano)) // Next time the schedule fires. + } + if ttl != _EMPTY_ { + hdr = genHeader(hdr, JSMessageTTL, ttl) + } + if rollup != _EMPTY_ { + hdr = genHeader(hdr, JSMsgRollup, rollup) + } + msgs = append(msgs, &inMsg{seq: seq, subj: target, hdr: hdr, msg: msg}) + ms.markInflight(subj) + return false + } + ms.remove(seq) + return true + }) + // THW is unordered, so must sort by sequence. + slices.SortFunc(msgs, func(a, b *inMsg) int { + if a.seq == b.seq { + return 0 + } else if a.seq < b.seq { + return -1 + } else { + return 1 + } + }) + return msgs +} + +// encode writes out the contents of the schedule into a binary snapshot +// and returns it. The high seq number is included in the snapshot and will +// be returned on decode. +func (ms *MsgScheduling) encode(highSeq uint64) []byte { + count := uint64(len(ms.schedules)) + b := make([]byte, 0, headerLen+(count*(2*binary.MaxVarintLen64))) + b = append(b, 1) // Magic version + b = binary.LittleEndian.AppendUint64(b, count) // Entry count + b = binary.LittleEndian.AppendUint64(b, highSeq) // Stamp + for subj, sched := range ms.schedules { + slen := min(uint64(len(subj)), math.MaxUint16) + b = binary.LittleEndian.AppendUint16(b, uint16(slen)) + b = append(b, subj[:slen]...) + b = binary.AppendVarint(b, sched.ts) + b = binary.AppendUvarint(b, sched.seq) + } + return b +} + +// decode snapshots a binary-encoded schedule and replaces the contents of this +// schedule with them. Returns the high seq number from the snapshot. +func (ms *MsgScheduling) decode(b []byte) (uint64, error) { + if len(b) < headerLen { + return 0, io.ErrShortBuffer + } + if b[0] != 1 { + return 0, ErrMsgScheduleInvalidVersion + } + + count := binary.LittleEndian.Uint64(b[1:]) + stamp := binary.LittleEndian.Uint64(b[9:]) + b = b[headerLen:] + for i := uint64(0); i < count; i++ { + if len(b) < 2 { + return 0, io.ErrUnexpectedEOF + } + sl := int(binary.LittleEndian.Uint16(b)) + b = b[2:] + if len(b) < sl { + return 0, io.ErrUnexpectedEOF + } + subj := string(b[:sl]) + b = b[sl:] + ts, tn := binary.Varint(b) + if tn <= 0 { + return 0, io.ErrUnexpectedEOF + } + seq, vn := binary.Uvarint(b[tn:]) + if vn <= 0 { + return 0, io.ErrUnexpectedEOF + } + ms.init(seq, subj, ts) + b = b[tn+vn:] + } + return stamp, nil +} + +// parseMsgSchedule parses a message schedule pattern and returns the time +// to fire, whether it is a repeating schedule, and whether the pattern was valid. +func parseMsgSchedule(pattern string, loc *time.Location, ts int64) (time.Time, bool, bool) { + if pattern == _EMPTY_ { + return time.Time{}, false, true + } + // Exact time. + if strings.HasPrefix(pattern, "@at ") { + // Time zone is not supported for @at. + if loc != nil { + return time.Time{}, false, false + } + t, err := time.Parse(time.RFC3339, pattern[4:]) + return t, false, err == nil + } + // Repeating on a simple interval. + if strings.HasPrefix(pattern, "@every ") { + // Time zone is not supported for @every. + if loc != nil { + return time.Time{}, false, false + } + dur, err := time.ParseDuration(pattern[7:]) + if err != nil { + return time.Time{}, false, false + } + // Only allow intervals of at least a second. + if dur.Seconds() < 1 { + return time.Time{}, false, false + } + // If this schedule would trigger multiple times, for example after a restart, skip ahead and only fire once. + next := time.Unix(0, ts).UTC().Round(time.Second).Add(dur) + if now := time.Now().UTC(); next.Before(now) { + next = now.Round(time.Second).Add(dur) + } + return next, true, true + } + + // Predefined schedules for cron. + switch pattern { + case "@yearly", "@annually": + pattern = "0 0 0 1 1 *" + case "@monthly": + pattern = "0 0 0 1 * *" + case "@weekly": + pattern = "0 0 0 * * 0" + case "@daily", "@midnight": + pattern = "0 0 0 * * *" + case "@hourly": + pattern = "0 0 * * * *" + } + + // Parse the cron pattern. + next, err := parseCron(pattern, loc, ts) + if err != nil { + return time.Time{}, false, false + } + // If this schedule would trigger multiple times, for example after a restart, skip ahead and only fire once. + if now := time.Now().UTC(); next.Before(now) { + ts = now.Round(time.Second).UnixNano() + next, err = parseCron(pattern, loc, ts) + if err != nil { + return time.Time{}, false, false + } + } + return next, true, true +} diff --git a/server/sdm.go b/server/sdm.go new file mode 100644 index 0000000..dfb2a75 --- /dev/null +++ b/server/sdm.go @@ -0,0 +1,80 @@ +// Copyright 2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "time" +) + +// SDMMeta holds pending/proposed data for subject delete markers or message removals. +type SDMMeta struct { + totals map[string]uint64 + pending map[uint64]SDMBySeq +} + +// SDMBySeq holds data for a message with a specific sequence. +type SDMBySeq struct { + last bool // Whether the message for this sequence was the last for this subject. + ts int64 // Last timestamp we proposed a removal/sdm. +} + +func newSDMMeta() *SDMMeta { + return &SDMMeta{ + totals: make(map[string]uint64, 1), + pending: make(map[uint64]SDMBySeq, 1), + } +} + +// isSubjectDeleteMarker returns whether the headers indicate this message is a subject delete marker. +// Either it's a usual marker with JSMarkerReason, or it's a KV Purge marker as the KVOperation. +func isSubjectDeleteMarker(hdr []byte) bool { + return len(sliceHeader(JSMarkerReason, hdr)) != 0 || bytes.Equal(sliceHeader(KVOperation, hdr), KVOperationValuePurge) +} + +// empty clears all data. +func (sdm *SDMMeta) empty() { + if sdm == nil { + return + } + clear(sdm.totals) + clear(sdm.pending) +} + +// trackPending caches the given seq and subj and whether it's the last message for that subject. +func (sdm *SDMMeta) trackPending(seq uint64, subj string, last bool) bool { + if p, ok := sdm.pending[seq]; ok { + return p.last + } + sdm.pending[seq] = SDMBySeq{last, time.Now().UnixNano()} + sdm.totals[subj]++ + return last +} + +// removeSeqAndSubject clears the seq and subj from the cache. +func (sdm *SDMMeta) removeSeqAndSubject(seq uint64, subj string) { + if sdm == nil { + return + } + if _, ok := sdm.pending[seq]; ok { + delete(sdm.pending, seq) + if msgs, ok := sdm.totals[subj]; ok { + if msgs <= 1 { + delete(sdm.totals, subj) + } else { + sdm.totals[subj] = msgs - 1 + } + } + } +} diff --git a/server/sendq.go b/server/sendq.go new file mode 100644 index 0000000..9cde157 --- /dev/null +++ b/server/sendq.go @@ -0,0 +1,118 @@ +// Copyright 2020-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "strconv" + "sync" +) + +type outMsg struct { + subj string + rply string + hdr []byte + msg []byte +} + +type sendq struct { + mu sync.Mutex + q *ipQueue[*outMsg] + s *Server + a *Account +} + +func (s *Server) newSendQ(acc *Account) *sendq { + sq := &sendq{s: s, q: newIPQueue[*outMsg](s, "SendQ"), a: acc} + s.startGoRoutine(sq.internalLoop) + return sq +} + +func (sq *sendq) internalLoop() { + sq.mu.Lock() + s, q := sq.s, sq.q + sq.mu.Unlock() + + defer s.grWG.Done() + + c := s.createInternalSystemClient() + c.registerWithAccount(sq.a) + c.noIcb = true + + defer c.closeConnection(ClientClosed) + + // To optimize for not converting a string to a []byte slice. + var ( + subj [256]byte + rply [256]byte + szb [10]byte + hdb [10]byte + _msg [4096]byte + msg = _msg[:0] + ) + + for s.isRunning() { + select { + case <-s.quitCh: + return + case <-q.ch: + pms := q.pop() + for _, pm := range pms { + c.pa.subject = append(subj[:0], pm.subj...) + c.pa.size = len(pm.msg) + len(pm.hdr) + c.pa.szb = append(szb[:0], strconv.Itoa(c.pa.size)...) + if len(pm.rply) > 0 { + c.pa.reply = append(rply[:0], pm.rply...) + } else { + c.pa.reply = nil + } + msg = msg[:0] + if len(pm.hdr) > 0 { + c.pa.hdr = len(pm.hdr) + c.pa.hdb = append(hdb[:0], strconv.Itoa(c.pa.hdr)...) + msg = append(msg, pm.hdr...) + msg = append(msg, pm.msg...) + msg = append(msg, _CRLF_...) + } else { + c.pa.hdr = -1 + c.pa.hdb = nil + msg = append(msg, pm.msg...) + msg = append(msg, _CRLF_...) + } + c.processInboundClientMsg(msg) + c.pa.szb = nil + outMsgPool.Put(pm) + } + // TODO: should this be in the for-loop instead? + c.flushClients(0) + q.recycle(&pms) + } + } +} + +var outMsgPool = sync.Pool{ + New: func() any { + return &outMsg{} + }, +} + +func (sq *sendq) send(subj, rply string, hdr, msg []byte) { + if sq == nil { + return + } + out := outMsgPool.Get().(*outMsg) + out.subj, out.rply = subj, rply + out.hdr = append(out.hdr[:0], hdr...) + out.msg = append(out.msg[:0], msg...) + sq.q.push(out) +} diff --git a/server/server_fuzz_test.go b/server/server_fuzz_test.go new file mode 100644 index 0000000..ea282e4 --- /dev/null +++ b/server/server_fuzz_test.go @@ -0,0 +1,410 @@ +// Copyright 2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bufio" + "crypto/tls" + "fmt" + "io" + "math" + "net" + "testing" + "time" +) + +const ( + tlsHandshake byte = 22 + tlsClientHello byte = 1 +) + +type ClientHelloInjector struct { + sock io.ReadWriteCloser + tlsVersion uint16 + buf []byte +} + +func NewClientHelloInjector(s io.ReadWriteCloser, tlsVer uint16, b []byte) *ClientHelloInjector { + return &ClientHelloInjector{ + sock: s, + tlsVersion: tlsVer, + buf: b, + } +} + +func (i *ClientHelloInjector) inject(b []byte) []byte { + if !(b != nil && b[0] == tlsHandshake) { + return b + } + + hsLen := (uint16(b[3]) << 8) + uint16(b[4]) + + if !(hsLen > 0 && b[5] == tlsClientHello) { + return b + } + + // fuzz tls version in client hello + b[9] = uint8(i.tlsVersion >> 8) + b[10] = uint8(i.tlsVersion & 0xFF) + + // Go to begin of random opaque + offset := 11 + + randomOpaque := b[offset : offset+32] + + copy(randomOpaque, i.buf) + + offset += 32 + + sessionIDLen := b[offset] + + // Skip session id len + opaque + offset += 1 + int(sessionIDLen) + + cypherSuiteLen := (uint16(b[offset]) << 8) + uint16(b[offset+1]) + + // Skip cypherSuiteLen + offset += 2 + + cupherSuites := b[offset : offset+int(cypherSuiteLen)] + + // Leave unchanged if i.cypherSuites empty + copy(cupherSuites, i.buf) + + // Skip cypherSuites + offset += int(cypherSuiteLen) + + // Skip CompressionMethod + offset += 2 + + extensionsLen := (uint16(b[offset]) << 8) + uint16(b[offset+1]) + + // Skip extensions length + offset += 2 + + // Extensions slice. Stub for future use + _ = b[offset : offset+int(extensionsLen)] + + return b +} + +func (i *ClientHelloInjector) Write(b []byte) (int, error) { + return i.sock.Write(i.inject(b)) +} + +func (i *ClientHelloInjector) Read(b []byte) (int, error) { + return i.sock.Read(b) +} + +func (i *ClientHelloInjector) Close() error { + return i.sock.Close() +} + +type FakeSocket struct { + sockName string + buf []byte + data chan []byte + done chan struct{} +} + +func NewFakeSocket(name string, capacity int) *FakeSocket { + return &FakeSocket{ + sockName: name, + data: make(chan []byte, capacity), + done: make(chan struct{}), + } +} + +func (s *FakeSocket) Write(b []byte) (int, error) { + select { + case s.data <- b: + return len(b), nil + case <-s.done: + return 0, net.ErrClosed + } +} + +func (s *FakeSocket) readChunk(b []byte) (int, error) { + n := copy(b, s.buf) + s.buf = s.buf[n:] + return n, nil +} + +func (s *FakeSocket) Read(b []byte) (int, error) { + if len(s.buf) > 0 { + return s.readChunk(b) + } + + select { + case buf, ok := <-s.data: + if !ok { + return 0, nil + } + s.buf = buf + return s.readChunk(b) + case <-s.done: + return 0, nil + } +} + +func (s *FakeSocket) Close() error { + close(s.done) + return nil +} + +type FakeConn struct { + local io.ReadWriteCloser + remote io.ReadWriteCloser +} + +func NewFakeConn(loc io.ReadWriteCloser, rem io.ReadWriteCloser) *FakeConn { + return &FakeConn{ + local: loc, + remote: rem, + } +} + +func (c *FakeConn) Read(b []byte) (int, error) { + return c.local.Read(b) +} + +func (c *FakeConn) Write(b []byte) (int, error) { + return c.remote.Write(b) +} + +func (c *FakeConn) Close() error { + return c.local.Close() +} + +func (c *FakeConn) LocalAddr() net.Addr { + return &net.TCPAddr{IP: net.IP{127, 0, 0, 1}, Port: 4222, Zone: ""} +} + +func (c *FakeConn) RemoteAddr() net.Addr { + return &net.TCPAddr{IP: net.IP{127, 0, 0, 1}, Port: 4222, Zone: ""} +} + +func (c *FakeConn) SetDeadline(t time.Time) error { + return nil +} + +func (c *FakeConn) SetReadDeadline(t time.Time) error { + return nil +} + +func (c *FakeConn) SetWriteDeadline(t time.Time) error { + return nil +} + +type FakeListener struct { + ch chan *FakeConn + acceptErr error +} + +func NewFakeListener() *FakeListener { + return &FakeListener{ + ch: make(chan *FakeConn), + acceptErr: nil, + } +} + +func (ln *FakeListener) Accept() (c net.Conn, err error) { + return <-ln.ch, ln.acceptErr +} + +func (ln *FakeListener) Close() error { + ln.acceptErr = io.EOF + close(ln.ch) + return nil +} + +func (ln *FakeListener) Addr() net.Addr { + return &net.TCPAddr{IP: net.IP{127, 0, 0, 1}, Port: 4222, Zone: ""} +} + +func getTlsVersion(useTls13 bool) uint16 { + if useTls13 { + return tls.VersionTLS13 + } + + return tls.VersionTLS12 +} + +func corruptCert(crt []byte, i uint16) []byte { + crt[int(i)%len(crt)] ^= 0xFF + return crt +} + +func runServerWithListener(ln net.Listener, opts *Options) *Server { + if opts == nil { + opts = DefaultOptions() + } + s, err := NewServer(opts) + if err != nil || s == nil { + panic(fmt.Sprintf("No NATS Server object returned: %v", err)) + } + + if !opts.NoLog { + s.ConfigureLogger() + } + + s.listener = ln + s.listenerErr = nil + + // Run server in Go routine. + s.Start() + + // Wait for accept loop(s) to be started + if err := s.readyForConnections(10 * time.Second); err != nil { + panic(err) + } + return s +} + +type MathRandReader byte + +func (m MathRandReader) Read(buf []byte) (int, error) { + for i := range buf { + buf[i] = byte(m) + } + return len(buf), nil +} + +// FuzzServerTLS performs fuzz testing of the NATS server's TLS handshake implementation. +// It verifies the server's ability to handle various TLS connection scenarios, including: +// - Different TLS versions (1.2 and 1.3) +// - Malformed/mutated client certificates +// - Corrupted TLS handshake data +// - Edge cases in the TLS negotiation process +// +// Test Setup: +// - Configures a server with mutual TLS authentication using test certificates +// - Creates a client with configurable TLS parameters +// - Uses fake network connections to inject test cases +// +// Fuzzing Parameters: +// - useTls13: Boolean flag to test TLS 1.3 (true) or TLS 1.2 (false) +// - tlsVer: TLS version number to use in ClientHello +// - buf: Additional bytes to inject into ClientHello message +// - corruptCertOffset: Position to corrupt in client certificate (MaxUint16 = no corruption) +// +// Expectations +// The server should either: +// - Successfully complete the TLS handshake and protocol exchange, or +// - Cleanly reject invalid connections without crashing +func FuzzServerTLS(f *testing.F) { + srvTc := &TLSConfigOpts{ + CertFile: "../test/configs/certs/tlsauth/server.pem", + KeyFile: "../test/configs/certs/tlsauth/server-key.pem", + CaFile: "../test/configs/certs/tlsauth/ca.pem", + Insecure: false, + Verify: true, + } + + srvTlsCfg, err := GenTLSConfig(srvTc) + if err != nil { + f.Fatalf("Error generating server tls config: %v", err) + } + + opts := &Options{ + Host: "127.0.0.1", + Port: 4222, + NoLog: true, + NoSigs: true, + Debug: true, + Trace: true, + TLSHandshakeFirst: true, + AllowNonTLS: false, + JetStream: false, + TLSConfig: srvTlsCfg, + CheckConfig: false, + } + + clientCerts, err := tls.LoadX509KeyPair("../test/configs/certs/tlsauth/client.pem", "../test/configs/certs/tlsauth/client-key.pem") + if err != nil { + f.Fatalf("client1 certificate load error: %s", err) + } + + clientTlsCfg := &tls.Config{ + InsecureSkipVerify: true, + Rand: MathRandReader(0), + Time: func() time.Time { return time.Date(2025, 1, 1, 1, 1, 1, 1, nil) }, + } + + tlsVer := uint16(0x0303) + + corpuses := []struct { + useTls13 bool + clientHelloTlsVer uint16 + buf []byte + corruptCertOffset uint16 + }{ + {useTls13: false, clientHelloTlsVer: tlsVer, buf: []byte{}, corruptCertOffset: math.MaxUint16}, + {useTls13: true, clientHelloTlsVer: tlsVer, buf: []byte{}, corruptCertOffset: math.MaxUint16}, + } + + for _, crp := range corpuses { + f.Add(crp.useTls13, crp.clientHelloTlsVer, crp.buf, crp.corruptCertOffset) + } + + f.Fuzz(func(t *testing.T, useTls13 bool, tlsVer uint16, buf []byte, corruptCertOffset uint16) { + ln := NewFakeListener() + s := runServerWithListener(ln, opts.Clone()) + defer s.Shutdown() + + clientSocket := NewFakeSocket("CLIENT", 8) + serverSocket := NewClientHelloInjector(NewFakeSocket("SERVER", 8), tlsVer, buf) + + clientConn := NewFakeConn(clientSocket, serverSocket) + serverConn := NewFakeConn(serverSocket, clientSocket) + + // Connect to server + ln.ch <- serverConn + + tlsVersion := getTlsVersion(useTls13) + + tlsCfg := clientTlsCfg.Clone() + tlsCfg.GetClientCertificate = func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) { + if corruptCertOffset == math.MaxUint16 { + t.Log("Leave certificate unchanged") + return &clientCerts, nil + } + + origCert := clientCerts.Certificate[0] + newCert := make([]byte, len(origCert)) + copy(newCert, origCert) + + newTlsCerts := clientCerts + newTlsCerts.Certificate[0] = corruptCert(newCert, corruptCertOffset) + + return &newTlsCerts, nil + } + tlsCfg.MaxVersion = tlsVersion + tlsCfg.MinVersion = tlsVersion + + tlsClientConn := tls.Client(clientConn, tlsCfg) + defer tlsClientConn.Close() + + if err := tlsClientConn.Handshake(); err != nil { + t.Logf("Handshake error: %v", err) + return + } + + br := bufio.NewReaderSize(tlsClientConn, 128) + if _, err := br.ReadString('\n'); err != nil { + t.Logf("Unexpected error reading INFO message: %v", err) + return + } + }) +} diff --git a/server/server_test.go b/server/server_test.go new file mode 100644 index 0000000..31fed51 --- /dev/null +++ b/server/server_test.go @@ -0,0 +1,2528 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bufio" + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net" + "net/url" + "os" + "reflect" + "runtime" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats.go" + + "github.com/nats-io/nats-server/v2/internal/antithesis" + srvlog "github.com/nats-io/nats-server/v2/logger" +) + +func checkForErr(totalWait, sleepDur time.Duration, f func() error) error { + timeout := time.Now().Add(totalWait) + var err error + for time.Now().Before(timeout) { + err = f() + if err == nil { + return nil + } + time.Sleep(sleepDur) + } + return err +} + +func checkFor(t testing.TB, totalWait, sleepDur time.Duration, f func() error) { + t.Helper() + err := checkForErr(totalWait, sleepDur, f) + if err != nil { + antithesis.AssertUnreachable(t, "Timeout in checkFor", nil) + t.Fatal(err.Error()) + } +} + +func DefaultOptions() *Options { + return &Options{ + Host: "127.0.0.1", + Port: -1, + HTTPPort: -1, + Cluster: ClusterOpts{Port: -1, Name: "abc"}, + NoLog: true, + NoSigs: true, + Debug: true, + Trace: true, + } +} + +// New Go Routine based server +func RunServer(opts *Options) *Server { + if opts == nil { + opts = DefaultOptions() + } + s, err := NewServer(opts) + if err != nil || s == nil { + panic(fmt.Sprintf("No NATS Server object returned: %v", err)) + } + + if !opts.NoLog { + s.ConfigureLogger() + } + + if ll := os.Getenv("NATS_LOGGING"); ll != "" { + log := srvlog.NewTestLogger(fmt.Sprintf("[%s] | ", s), true) + debug := ll == "debug" || ll == "trace" + trace := ll == "trace" + s.SetLoggerV2(log, debug, trace, false) + } + + // Run server in Go routine. + s.Start() + + // Wait for accept loop(s) to be started + if err := s.readyForConnections(10 * time.Second); err != nil { + panic(err) + } + return s +} + +// LoadConfig loads a configuration from a filename +func LoadConfig(configFile string) (opts *Options) { + opts, err := ProcessConfigFile(configFile) + if err != nil { + panic(fmt.Sprintf("Error processing configuration file: %v", err)) + } + opts.NoSigs, opts.NoLog = true, opts.LogFile == _EMPTY_ + return +} + +// RunServerWithConfig starts a new Go routine based server with a configuration file. +func RunServerWithConfig(configFile string) (srv *Server, opts *Options) { + opts = LoadConfig(configFile) + srv = RunServer(opts) + return +} + +func TestSemanticVersion(t *testing.T) { + if !semVerRe.MatchString(VERSION) { + t.Fatalf("Version (%s) is not a valid SemVer string", VERSION) + } +} + +func TestVersionMatchesTag(t *testing.T) { + tag := os.Getenv("TRAVIS_TAG") + // Travis started to return '' when no tag is set. Support both now. + if tag == "" || tag == "''" { + t.SkipNow() + } + // We expect a tag of the form vX.Y.Z. If that's not the case, + // we need someone to have a look. So fail if first letter is not + // a `v` + if tag[0] != 'v' { + t.Fatalf("Expect tag to start with `v`, tag is: %s", tag) + } + // Strip the `v` from the tag for the version comparison. + if VERSION != tag[1:] { + t.Fatalf("Version (%s) does not match tag (%s)", VERSION, tag[1:]) + } + // Check that the version dynamically set via ldflags matches the version + // from the server previous to releasing. + if serverVersion == _EMPTY_ { + t.Fatal("Version missing in ldflags") + } + // Unlike VERSION constant, serverVersion is prefixed with a 'v' + // since it should be the same as the git tag. + expected := "v" + VERSION + if serverVersion != _EMPTY_ && expected != serverVersion { + t.Fatalf("Version (%s) does not match ldflags version (%s)", expected, serverVersion) + } +} + +func TestStartProfiler(t *testing.T) { + s := New(DefaultOptions()) + s.StartProfiler() + s.mu.Lock() + s.profiler.Close() + s.mu.Unlock() +} + +func TestStartupAndShutdown(t *testing.T) { + opts := DefaultOptions() + opts.NoSystemAccount = true + + s := RunServer(opts) + defer s.Shutdown() + + if !s.isRunning() { + t.Fatal("Could not run server") + } + + // Debug stuff. + numRoutes := s.NumRoutes() + if numRoutes != 0 { + t.Fatalf("Expected numRoutes to be 0 vs %d\n", numRoutes) + } + + numRemotes := s.NumRemotes() + if numRemotes != 0 { + t.Fatalf("Expected numRemotes to be 0 vs %d\n", numRemotes) + } + + numClients := s.NumClients() + if numClients != 0 && numClients != 1 { + t.Fatalf("Expected numClients to be 1 or 0 vs %d\n", numClients) + } + + numSubscriptions := s.NumSubscriptions() + if numSubscriptions != 0 { + t.Fatalf("Expected numSubscriptions to be 0 vs %d\n", numSubscriptions) + } +} + +func TestConcurrentShutdown(t *testing.T) { + opts := DefaultOptions() + opts.DisableShortFirstPing = true + opts.Accounts = []*Account{NewAccount("$SYS")} + opts.SystemAccount = "$SYS" + + s := RunServer(opts) + if !s.EventsEnabled() { + t.Fatal("Expected events to be enabled") + } + + const callers = 32 + start := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(callers) + for i := 0; i < callers; i++ { + go func() { + defer wg.Done() + <-start + s.Shutdown() + }() + } + close(start) + + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Timed out waiting for concurrent shutdown calls") + } + s.WaitForShutdown() +} + +func TestTLSVersions(t *testing.T) { + for _, test := range []struct { + name string + value uint16 + expected string + }{ + {"1.0", tls.VersionTLS10, "1.0"}, + {"1.1", tls.VersionTLS11, "1.1"}, + {"1.2", tls.VersionTLS12, "1.2"}, + {"1.3", tls.VersionTLS13, "1.3"}, + {"unknown", 0x999, "Unknown [0x999]"}, + } { + t.Run(test.name, func(t *testing.T) { + if v := tlsVersion(test.value); v != test.expected { + t.Fatalf("Expected value 0x%x to be %q, got %q", test.value, test.expected, v) + } + }) + } +} + +func TestTLSMinVersionConfig(t *testing.T) { + tmpl := ` + listen: "127.0.0.1:-1" + tls { + cert_file: "../test/configs/certs/server-cert.pem" + key_file: "../test/configs/certs/server-key.pem" + timeout: 1 + min_version: %s + } + ` + conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, `"1.3"`))) + s, o := RunServerWithConfig(conf) + defer s.Shutdown() + + connect := func(t *testing.T, tlsConf *tls.Config, expectedErr error) { + t.Helper() + opts := []nats.Option{} + if tlsConf != nil { + opts = append(opts, nats.Secure(tlsConf)) + } + opts = append(opts, nats.RootCAs("../test/configs/certs/ca.pem")) + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", o.Port), opts...) + if err == nil { + defer nc.Close() + } + if expectedErr == nil { + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + } else if err == nil || err.Error() != expectedErr.Error() { + nc.Close() + t.Fatalf("Expected error %v, got: %v", expectedErr, err) + } + } + + // Cannot connect with client requiring a lower minimum TLS Version. + connect(t, &tls.Config{ + MaxVersion: tls.VersionTLS12, + }, errors.New(`nats: tls error: remote error: tls: protocol version not supported`)) + + // Should connect since matching minimum TLS version. + connect(t, &tls.Config{ + MinVersion: tls.VersionTLS13, + }, nil) + + // Reloading with invalid values should fail. + if err := os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, `"1.0"`)), 0666); err != nil { + t.Fatalf("Error creating config file: %v", err) + } + if err := s.Reload(); err == nil { + t.Fatalf("Expected reload to fail: %v", err) + } + + // Reloading with original values and no changes should be ok. + if err := os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, `"1.3"`)), 0666); err != nil { + t.Fatalf("Error creating config file: %v", err) + } + if err := s.Reload(); err != nil { + t.Fatalf("Unexpected error reloading TLS version: %v", err) + } + + // Reloading with a new minimum lower version. + if err := os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, `"1.2"`)), 0666); err != nil { + t.Fatalf("Error creating config file: %v", err) + } + if err := s.Reload(); err != nil { + t.Fatalf("Unexpected error reloading: %v", err) + } + + // Should connect since now matching minimum TLS version. + connect(t, &tls.Config{ + MaxVersion: tls.VersionTLS12, + }, nil) + connect(t, &tls.Config{ + MinVersion: tls.VersionTLS13, + }, nil) + + // Setting unsupported TLS versions + if err := os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, `"1.4"`)), 0666); err != nil { + t.Fatalf("Error creating config file: %v", err) + } + if err := s.Reload(); err == nil || !strings.Contains(err.Error(), `unknown version: 1.4`) { + t.Fatalf("Unexpected error reloading: %v", err) + } + + tc := &TLSConfigOpts{ + CertFile: "../test/configs/certs/server-cert.pem", + KeyFile: "../test/configs/certs/server-key.pem", + CaFile: "../test/configs/certs/ca.pem", + Timeout: 4.0, + MinVersion: tls.VersionTLS11, + } + _, err := GenTLSConfig(tc) + if err == nil || err.Error() != `unsupported minimum TLS version: TLS 1.1` { + t.Fatalf("Expected error generating TLS config: %v", err) + } +} + +func TestTLSCipher(t *testing.T) { + require_Equal(t, tls.CipherSuiteName(0x0005), "TLS_RSA_WITH_RC4_128_SHA") + require_Equal(t, tls.CipherSuiteName(0x000a), "TLS_RSA_WITH_3DES_EDE_CBC_SHA") + require_Equal(t, tls.CipherSuiteName(0x002f), "TLS_RSA_WITH_AES_128_CBC_SHA") + require_Equal(t, tls.CipherSuiteName(0x0035), "TLS_RSA_WITH_AES_256_CBC_SHA") + require_Equal(t, tls.CipherSuiteName(0xc007), "TLS_ECDHE_ECDSA_WITH_RC4_128_SHA") + require_Equal(t, tls.CipherSuiteName(0xc009), "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA") + require_Equal(t, tls.CipherSuiteName(0xc00a), "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA") + require_Equal(t, tls.CipherSuiteName(0xc011), "TLS_ECDHE_RSA_WITH_RC4_128_SHA") + require_Equal(t, tls.CipherSuiteName(0xc012), "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA") + require_Equal(t, tls.CipherSuiteName(0xc013), "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA") + require_Equal(t, tls.CipherSuiteName(0xc014), "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA") + require_Equal(t, tls.CipherSuiteName(0xc02f), "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256") + require_Equal(t, tls.CipherSuiteName(0xc02b), "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256") + require_Equal(t, tls.CipherSuiteName(0xc030), "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384") + require_Equal(t, tls.CipherSuiteName(0xc02c), "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384") + require_Equal(t, tls.CipherSuiteName(0x1301), "TLS_AES_128_GCM_SHA256") + require_Equal(t, tls.CipherSuiteName(0x1302), "TLS_AES_256_GCM_SHA384") + require_Equal(t, tls.CipherSuiteName(0x1303), "TLS_CHACHA20_POLY1305_SHA256") + require_Equal(t, tls.CipherSuiteName(0x9999), "0x9999") +} + +func TestTLSDefaultCipherAndCurvePreferences(t *testing.T) { + // The module still targets Go 1.25 in go.mod, so Go 1.26 keeps the SecP ML-KEM + // groups disabled by default unless the test opts in explicitly. + t.Setenv("GODEBUG", "tlssecpmlkem=1") + + runTLSServer := func(t *testing.T, cipherSuites []uint16, curvePreferences []tls.CurveID, minVersion, maxVersion uint16) *Server { + t.Helper() + tc := &TLSConfigOpts{ + Certificates: []*TLSCertPairOpt{ + { + CertFile: "../test/configs/certs/server-cert.pem", + KeyFile: "../test/configs/certs/server-key.pem", + }, + { + CertFile: "../test/configs/certs/tlsauth/certstore/ecdsa_server.pem", + KeyFile: "../test/configs/certs/tlsauth/certstore/ecdsa_server.key", + }, + }, + Ciphers: cipherSuites, + CurvePreferences: curvePreferences, + MinVersion: minVersion, + } + config, err := GenTLSConfig(tc) + require_NoError(t, err) + config.MaxVersion = maxVersion + + opts := DefaultOptions() + opts.TLSConfig = config + opts.TLSHandshakeFirst = true + return RunServer(opts) + } + + dial := func(s *Server, config *tls.Config) (*tls.Conn, error) { + return tls.Dial("tcp", s.Addr().String(), config) + } + + t.Run("cipher suites", func(t *testing.T) { + for _, id := range defaultCipherSuites() { + cs := cipherMapByID[id] + if !slices.Contains(cs.SupportedVersions, tls.VersionTLS12) { + continue // t.Skip("tls.Config.CipherSuites does not configure TLS 1.3 cipher suites") + } + + t.Run(cs.Name, func(t *testing.T) { + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + ServerName: "localhost", + MinVersion: tls.VersionTLS12, + MaxVersion: tls.VersionTLS12, + CipherSuites: []uint16{cs.ID}, + CurvePreferences: defaultCurvePreferences(), + } + + s := runTLSServer(t, []uint16{cs.ID}, tlsConfig.CurvePreferences, tlsConfig.MinVersion, tlsConfig.MaxVersion) + defer s.Shutdown() + + // Prove that we can connect with the right curve preference. + conn, err := dial(s, tlsConfig) + require_NoError(t, err) + defer conn.Close() + require_Equal(t, conn.ConnectionState().CipherSuite, cs.ID) + + // Now configure other ciphers and check that we fail to connect. + tlsConfig.CipherSuites = slices.DeleteFunc(slices.Clone(defaultCipherSuites()), func(id uint16) bool { + return id == cs.ID || !slices.Contains(cipherMapByID[id].SupportedVersions, tls.VersionTLS12) + }) + if conn, err = dial(s, tlsConfig); conn != nil { + defer conn.Close() + } + require_Error(t, err) + }) + } + }) + + t.Run("curve preferences", func(t *testing.T) { + for name, curve := range curvePreferenceMap { + t.Run(name, func(t *testing.T) { + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + MinVersion: tls.VersionTLS13, + MaxVersion: tls.VersionTLS13, + CurvePreferences: []tls.CurveID{curve}, + } + + s := runTLSServer(t, nil, []tls.CurveID{curve}, tlsConfig.MinVersion, tlsConfig.MaxVersion) + defer s.Shutdown() + + // Prove that we can connect with the right cipher suites. + conn, err := dial(s, tlsConfig) + require_NoError(t, err) + defer conn.Close() + require_Equal(t, conn.ConnectionState().CurveID, curve) + + // Now configure other curves and check that we fail to connect. + tlsConfig.CurvePreferences = slices.DeleteFunc(slices.Clone(defaultCurvePreferences()), func(id tls.CurveID) bool { + return id == curve + }) + if conn, err = dial(s, tlsConfig); conn != nil { + defer conn.Close() + } + require_Error(t, err) + }) + } + }) +} + +func TestGetConnectURLs(t *testing.T) { + opts := DefaultOptions() + opts.Port = 4222 + + var globalIP net.IP + + checkGlobalConnectURLs := func() { + s := New(opts) + defer s.Shutdown() + + s.mu.Lock() + urls := s.getClientConnectURLs() + s.mu.Unlock() + if len(urls) == 0 { + t.Fatalf("Expected to get a list of urls, got none for listen addr: %v", opts.Host) + } + for _, u := range urls { + tcpaddr, err := net.ResolveTCPAddr("tcp", u) + if err != nil { + t.Fatalf("Error resolving: %v", err) + } + ip := tcpaddr.IP + if !ip.IsGlobalUnicast() { + t.Fatalf("IP %v is not global", ip.String()) + } + if ip.IsUnspecified() { + t.Fatalf("IP %v is unspecified", ip.String()) + } + addr := strings.TrimSuffix(u, ":4222") + if addr == opts.Host { + t.Fatalf("Returned url is not right: %v", u) + } + if globalIP == nil { + globalIP = ip + } + } + } + + listenAddrs := []string{"0.0.0.0", "::"} + for _, listenAddr := range listenAddrs { + opts.Host = listenAddr + checkGlobalConnectURLs() + } + + checkConnectURLsHasOnlyOne := func() { + s := New(opts) + defer s.Shutdown() + + s.mu.Lock() + urls := s.getClientConnectURLs() + s.mu.Unlock() + if len(urls) != 1 { + t.Fatalf("Expected one URL, got %v", urls) + } + tcpaddr, err := net.ResolveTCPAddr("tcp", urls[0]) + if err != nil { + t.Fatalf("Error resolving: %v", err) + } + ip := tcpaddr.IP + if ip.String() != opts.Host { + t.Fatalf("Expected connect URL to be %v, got %v", opts.Host, ip.String()) + } + } + + singleConnectReturned := []string{"127.0.0.1", "::1"} + if globalIP != nil { + singleConnectReturned = append(singleConnectReturned, globalIP.String()) + } + for _, listenAddr := range singleConnectReturned { + opts.Host = listenAddr + checkConnectURLsHasOnlyOne() + } +} + +func TestInfoServerNameDefaultsToPK(t *testing.T) { + opts := DefaultOptions() + opts.Port = 4222 + opts.ClientAdvertise = "nats.example.com" + s := New(opts) + defer s.Shutdown() + + if s.info.Name != s.info.ID { + t.Fatalf("server info hostname is incorrect, got: '%v' expected: '%v'", s.info.Name, s.info.ID) + } +} + +func TestInfoServerNameIsSettable(t *testing.T) { + opts := DefaultOptions() + opts.Port = 4222 + opts.ClientAdvertise = "nats.example.com" + opts.ServerName = "test_server_name" + s := New(opts) + defer s.Shutdown() + + if s.info.Name != "test_server_name" { + t.Fatalf("server info hostname is incorrect, got: '%v' expected: 'test_server_name'", s.info.Name) + } +} + +func TestClientAdvertiseConnectURL(t *testing.T) { + opts := DefaultOptions() + opts.Port = 4222 + opts.ClientAdvertise = "nats.example.com" + s := New(opts) + defer s.Shutdown() + + s.mu.Lock() + urls := s.getClientConnectURLs() + s.mu.Unlock() + if len(urls) != 1 { + t.Fatalf("Expected to get one url, got none: %v with ClientAdvertise %v", + opts.Host, opts.ClientAdvertise) + } + if urls[0] != "nats.example.com:4222" { + t.Fatalf("Expected to get '%s', got: '%v'", "nats.example.com:4222", urls[0]) + } + s.Shutdown() + + opts.ClientAdvertise = "nats.example.com:7777" + s = New(opts) + s.mu.Lock() + urls = s.getClientConnectURLs() + s.mu.Unlock() + if len(urls) != 1 { + t.Fatalf("Expected to get one url, got none: %v with ClientAdvertise %v", + opts.Host, opts.ClientAdvertise) + } + if urls[0] != "nats.example.com:7777" { + t.Fatalf("Expected 'nats.example.com:7777', got: '%v'", urls[0]) + } + if s.info.Host != "nats.example.com" { + t.Fatalf("Expected host to be set to nats.example.com") + } + if s.info.Port != 7777 { + t.Fatalf("Expected port to be set to 7777") + } + s.Shutdown() + + opts = DefaultOptions() + opts.Port = 0 + opts.ClientAdvertise = "nats.example.com:7777" + s = New(opts) + if s.info.Host != "nats.example.com" && s.info.Port != 7777 { + t.Fatalf("Expected Client Advertise Host:Port to be nats.example.com:7777, got: %s:%d", + s.info.Host, s.info.Port) + } + s.Shutdown() +} + +func TestClientAdvertiseInCluster(t *testing.T) { + optsA := DefaultOptions() + optsA.ClientAdvertise = "srvA:4222" + srvA := RunServer(optsA) + defer srvA.Shutdown() + + nc := natsConnect(t, srvA.ClientURL()) + defer nc.Close() + + optsB := DefaultOptions() + optsB.ClientAdvertise = "srvBC:4222" + optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", optsA.Cluster.Port)) + srvB := RunServer(optsB) + defer srvB.Shutdown() + + checkClusterFormed(t, srvA, srvB) + + checkURLs := func(expected string) { + t.Helper() + checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { + srvs := nc.DiscoveredServers() + for _, u := range srvs { + if u == expected { + return nil + } + } + return fmt.Errorf("Url %q not found in %q", expected, srvs) + }) + } + checkURLs("nats://srvBC:4222") + + optsC := DefaultOptions() + optsC.ClientAdvertise = "srvBC:4222" + optsC.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", optsA.Cluster.Port)) + srvC := RunServer(optsC) + defer srvC.Shutdown() + + checkClusterFormed(t, srvA, srvB, srvC) + checkURLs("nats://srvBC:4222") + + srvB.Shutdown() + checkNumRoutes(t, srvA, DEFAULT_ROUTE_POOL_SIZE+1) + checkURLs("nats://srvBC:4222") +} + +func TestClientAdvertiseErrorOnStartup(t *testing.T) { + opts := DefaultOptions() + // Set invalid address + opts.ClientAdvertise = "addr:::123" + testFatalErrorOnStart(t, opts, "ClientAdvertise") +} + +func TestNoDeadlockOnStartFailure(t *testing.T) { + opts := DefaultOptions() + opts.Host = "x.x.x.x" // bad host + opts.Port = 4222 + opts.HTTPHost = opts.Host + opts.Cluster.Host = "127.0.0.1" + opts.Cluster.Port = -1 + opts.ProfPort = -1 + s := New(opts) + + // This should return since it should fail to start a listener + // on x.x.x.x:4222 + ch := make(chan struct{}) + go func() { + s.Start() + close(ch) + }() + select { + case <-ch: + case <-time.After(time.Second): + t.Fatalf("Start() should have returned due to failure to start listener") + } + + // We should be able to shutdown + s.Shutdown() +} + +func TestMaxConnections(t *testing.T) { + opts := DefaultOptions() + opts.MaxConn = 1 + s := RunServer(opts) + defer s.Shutdown() + + addr := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(addr) + if err != nil { + t.Fatalf("Error creating client: %v\n", err) + } + defer nc.Close() + + nc2, err := nats.Connect(addr) + if err == nil { + nc2.Close() + t.Fatal("Expected connection to fail") + } +} + +func TestMaxConnectionsPreventsAll(t *testing.T) { + opts := DefaultOptions() + opts.MaxConn = -1 + s := RunServer(opts) + defer s.Shutdown() + + addr := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + _, err := nats.Connect(addr) + require_Error(t, err) +} + +func TestMaxSubscriptions(t *testing.T) { + opts := DefaultOptions() + opts.MaxSubs = 10 + s := RunServer(opts) + defer s.Shutdown() + + addr := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(addr) + if err != nil { + t.Fatalf("Error creating client: %v\n", err) + } + defer nc.Close() + + for i := 0; i < 10; i++ { + _, err := nc.Subscribe(fmt.Sprintf("foo.%d", i), func(*nats.Msg) {}) + if err != nil { + t.Fatalf("Error subscribing: %v\n", err) + } + } + // This should cause the error. + nc.Subscribe("foo.22", func(*nats.Msg) {}) + nc.Flush() + if err := nc.LastError(); err == nil { + t.Fatal("Expected an error but got none\n") + } +} + +func TestProcessCommandLineArgs(t *testing.T) { + var host string + var port int + cmd := flag.NewFlagSet("nats-server", flag.ExitOnError) + cmd.StringVar(&host, "a", "0.0.0.0", "Host.") + cmd.IntVar(&port, "p", 4222, "Port.") + + cmd.Parse([]string{"-a", "127.0.0.1", "-p", "9090"}) + showVersion, showHelp, err := ProcessCommandLineArgs(cmd) + if err != nil { + t.Errorf("Expected no errors, got: %s", err) + } + if showVersion || showHelp { + t.Errorf("Expected not having to handle subcommands") + } + + cmd.Parse([]string{"version"}) + showVersion, showHelp, err = ProcessCommandLineArgs(cmd) + if err != nil { + t.Errorf("Expected no errors, got: %s", err) + } + if !showVersion { + t.Errorf("Expected having to handle version command") + } + if showHelp { + t.Errorf("Expected not having to handle help command") + } + + cmd.Parse([]string{"help"}) + showVersion, showHelp, err = ProcessCommandLineArgs(cmd) + if err != nil { + t.Errorf("Expected no errors, got: %s", err) + } + if showVersion { + t.Errorf("Expected not having to handle version command") + } + if !showHelp { + t.Errorf("Expected having to handle help command") + } + + cmd.Parse([]string{"foo", "-p", "9090"}) + _, _, err = ProcessCommandLineArgs(cmd) + if err == nil { + t.Errorf("Expected an error handling the command arguments") + } +} + +func TestRandomPorts(t *testing.T) { + opts := DefaultOptions() + opts.HTTPPort = -1 + opts.Port = -1 + s := RunServer(opts) + + defer s.Shutdown() + + if s.Addr() == nil || s.Addr().(*net.TCPAddr).Port <= 0 { + t.Fatal("Should have dynamically assigned server port.") + } + + if s.Addr() == nil || s.Addr().(*net.TCPAddr).Port == 4222 { + t.Fatal("Should not have dynamically assigned default port: 4222.") + } + + if s.MonitorAddr() == nil || s.MonitorAddr().Port <= 0 { + t.Fatal("Should have dynamically assigned monitoring port.") + } + +} + +func TestNilMonitoringPort(t *testing.T) { + opts := DefaultOptions() + opts.HTTPPort = 0 + opts.HTTPSPort = 0 + s := RunServer(opts) + + defer s.Shutdown() + + if s.MonitorAddr() != nil { + t.Fatal("HttpAddr should be nil.") + } +} + +type DummyAuth struct { + t *testing.T + needNonce bool + deadline time.Time + register bool +} + +func (d *DummyAuth) Check(c ClientAuthentication) bool { + if d.needNonce && len(c.GetNonce()) == 0 { + d.t.Fatalf("Expected a nonce but received none") + } else if !d.needNonce && len(c.GetNonce()) > 0 { + d.t.Fatalf("Received a nonce when none was expected") + } + + if c.GetOpts().Username != "valid" { + return false + } + + if !d.register { + return true + } + + u := &User{ + Username: c.GetOpts().Username, + ConnectionDeadline: d.deadline, + } + c.RegisterUser(u) + + return true +} + +func TestCustomClientAuthentication(t *testing.T) { + testAuth := func(t *testing.T, nonce bool) { + clientAuth := &DummyAuth{t: t, needNonce: nonce} + + opts := DefaultOptions() + opts.CustomClientAuthentication = clientAuth + opts.AlwaysEnableNonce = nonce + + s := RunServer(opts) + defer s.Shutdown() + + addr := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + + nc, err := nats.Connect(addr, nats.UserInfo("valid", "")) + if err != nil { + t.Fatalf("Expected client to connect, got: %s", err) + } + nc.Close() + if _, err := nats.Connect(addr, nats.UserInfo("invalid", "")); err == nil { + t.Fatal("Expected client to fail to connect") + } + } + + t.Run("with nonce", func(t *testing.T) { testAuth(t, true) }) + t.Run("without nonce", func(t *testing.T) { testAuth(t, false) }) +} + +func TestCustomRouterAuthentication(t *testing.T) { + opts := DefaultOptions() + opts.CustomRouterAuthentication = &DummyAuth{} + opts.Cluster.Host = "127.0.0.1" + s := RunServer(opts) + defer s.Shutdown() + clusterPort := s.ClusterAddr().Port + + opts2 := DefaultOptions() + opts2.Cluster.Host = "127.0.0.1" + opts2.Routes = RoutesFromStr(fmt.Sprintf("nats://invalid@127.0.0.1:%d", clusterPort)) + s2 := RunServer(opts2) + defer s2.Shutdown() + + // s2 will attempt to connect to s, which should reject. + // Keep in mind that s2 will try again... + time.Sleep(50 * time.Millisecond) + checkNumRoutes(t, s2, 0) + + opts3 := DefaultOptions() + opts3.Cluster.Host = "127.0.0.1" + opts3.Routes = RoutesFromStr(fmt.Sprintf("nats://valid@127.0.0.1:%d", clusterPort)) + s3 := RunServer(opts3) + defer s3.Shutdown() + checkClusterFormed(t, s, s3) + // Default pool size + 1 for system account + checkNumRoutes(t, s3, DEFAULT_ROUTE_POOL_SIZE+1) +} + +func TestMonitoringNoTimeout(t *testing.T) { + s := runMonitorServer() + defer s.Shutdown() + + s.mu.Lock() + srv := s.monitoringServer + s.mu.Unlock() + + if srv == nil { + t.Fatalf("Monitoring server not set") + } + if srv.ReadTimeout != 0 { + t.Fatalf("ReadTimeout should not be set, was set to %v", srv.ReadTimeout) + } + if srv.WriteTimeout != 0 { + t.Fatalf("WriteTimeout should not be set, was set to %v", srv.WriteTimeout) + } +} + +func TestProfilingNoTimeout(t *testing.T) { + opts := DefaultOptions() + opts.ProfPort = -1 + s := RunServer(opts) + defer s.Shutdown() + + paddr := s.ProfilerAddr() + if paddr == nil { + t.Fatalf("Profiler not started") + } + pport := paddr.Port + if pport <= 0 { + t.Fatalf("Expected profiler port to be set, got %v", pport) + } + s.mu.Lock() + srv := s.profilingServer + s.mu.Unlock() + + if srv == nil { + t.Fatalf("Profiling server not set") + } + if srv.ReadTimeout != time.Second*5 { + t.Fatalf("ReadTimeout should not be set, was set to %v", srv.ReadTimeout) + } + if srv.WriteTimeout != 0 { + t.Fatalf("WriteTimeout should not be set, was set to %v", srv.WriteTimeout) + } +} + +func TestLameDuckOptionsValidation(t *testing.T) { + o := DefaultOptions() + o.LameDuckDuration = 5 * time.Second + o.LameDuckGracePeriod = 10 * time.Second + s, err := NewServer(o) + if s != nil { + s.Shutdown() + } + if err == nil || !strings.Contains(err.Error(), "should be strictly lower") { + t.Fatalf("Expected error saying that ldm grace period should be lower than ldm duration, got %v", err) + } +} + +func testSetLDMGracePeriod(o *Options, val time.Duration) { + // For tests, we set the grace period as a negative value + // so we can have a grace period bigger than the total duration. + // When validating options, we would not be able to run the + // server without this trick. + o.LameDuckGracePeriod = val * -1 +} + +func TestLameDuckMode(t *testing.T) { + optsA := DefaultOptions() + testSetLDMGracePeriod(optsA, time.Nanosecond) + optsA.Cluster.Host = "127.0.0.1" + srvA := RunServer(optsA) + defer srvA.Shutdown() + + // Check that if there is no client, server is shutdown + srvA.lameDuckMode() + if !srvA.isShuttingDown() { + t.Fatalf("Server should have shutdown") + } + + optsA.LameDuckDuration = 10 * time.Nanosecond + srvA = RunServer(optsA) + defer srvA.Shutdown() + + optsB := DefaultOptions() + optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srvA.ClusterAddr().Port)) + srvB := RunServer(optsB) + defer srvB.Shutdown() + + checkClusterFormed(t, srvA, srvB) + + total := 50 + connectClients := func() []*nats.Conn { + ncs := make([]*nats.Conn, 0, total) + for i := 0; i < total; i++ { + nc, err := nats.Connect(fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port), + nats.ReconnectWait(50*time.Millisecond)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + ncs = append(ncs, nc) + } + return ncs + } + stopClientsAndSrvB := func(ncs []*nats.Conn) { + for _, nc := range ncs { + nc.Close() + } + srvB.Shutdown() + } + + ncs := connectClients() + + checkClientsCount(t, srvA, total) + checkClientsCount(t, srvB, 0) + + start := time.Now() + srvA.lameDuckMode() + // Make sure that nothing bad happens if called twice + srvA.lameDuckMode() + // Wait that shutdown completes + elapsed := time.Since(start) + // It should have taken more than the allotted time of 10ms since we had 50 clients. + if elapsed <= optsA.LameDuckDuration { + t.Fatalf("Expected to take more than %v, got %v", optsA.LameDuckDuration, elapsed) + } + + checkClientsCount(t, srvA, 0) + checkClientsCount(t, srvB, total) + + // Check closed status on server A + // Connections are saved in go routines, so although we have evaluated the number + // of connections in the server A to be 0, the polling of connection closed may + // need a bit more time. + checkFor(t, time.Second, 15*time.Millisecond, func() error { + cz := pollConnz(t, srvA, 1, "", &ConnzOptions{State: ConnClosed}) + if n := len(cz.Conns); n != total { + return fmt.Errorf("expected %v closed connections, got %v", total, n) + } + return nil + }) + cz := pollConnz(t, srvA, 1, "", &ConnzOptions{State: ConnClosed}) + if n := len(cz.Conns); n != total { + t.Fatalf("Expected %v closed connections, got %v", total, n) + } + for _, c := range cz.Conns { + checkReason(t, c.Reason, ServerShutdown) + } + + stopClientsAndSrvB(ncs) + + optsA.LameDuckDuration = time.Second + srvA = RunServer(optsA) + defer srvA.Shutdown() + + optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srvA.ClusterAddr().Port)) + srvB = RunServer(optsB) + defer srvB.Shutdown() + + checkClusterFormed(t, srvA, srvB) + + ncs = connectClients() + + checkClientsCount(t, srvA, total) + checkClientsCount(t, srvB, 0) + + start = time.Now() + go srvA.lameDuckMode() + // Check that while in lameDuckMode, it is not possible to connect + // to the server. Wait to be in LD mode first + checkFor(t, 500*time.Millisecond, 15*time.Millisecond, func() error { + srvA.mu.Lock() + ldm := srvA.ldm + srvA.mu.Unlock() + if !ldm { + return fmt.Errorf("Did not reach lame duck mode") + } + return nil + }) + if _, err := nats.Connect(fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port)); err != nats.ErrNoServers { + t.Fatalf("Expected %v, got %v", nats.ErrNoServers, err) + } + srvA.grWG.Wait() + elapsed = time.Since(start) + + checkClientsCount(t, srvA, 0) + checkClientsCount(t, srvB, total) + + if elapsed > time.Duration(float64(optsA.LameDuckDuration)*1.1) { + t.Fatalf("Expected to not take more than %v, got %v", optsA.LameDuckDuration, elapsed) + } + + stopClientsAndSrvB(ncs) + + // Now check that we can shutdown server while in LD mode. + optsA.LameDuckDuration = 60 * time.Second + srvA = RunServer(optsA) + defer srvA.Shutdown() + + optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srvA.ClusterAddr().Port)) + srvB = RunServer(optsB) + defer srvB.Shutdown() + + checkClusterFormed(t, srvA, srvB) + + ncs = connectClients() + + checkClientsCount(t, srvA, total) + checkClientsCount(t, srvB, 0) + + start = time.Now() + go srvA.lameDuckMode() + time.Sleep(100 * time.Millisecond) + srvA.Shutdown() + elapsed = time.Since(start) + // Make sure that it did not take that long + if elapsed > time.Second { + t.Fatalf("Took too long: %v", elapsed) + } + checkClientsCount(t, srvA, 0) + checkClientsCount(t, srvB, total) + + stopClientsAndSrvB(ncs) + + // Now test that we introduce delay before starting closing client connections. + // This allow to "signal" multiple servers and avoid their clients to reconnect + // to a server that is going to be going in LD mode. + testSetLDMGracePeriod(optsA, 100*time.Millisecond) + optsA.LameDuckDuration = 10 * time.Millisecond + srvA = RunServer(optsA) + defer srvA.Shutdown() + + optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srvA.ClusterAddr().Port)) + testSetLDMGracePeriod(optsB, 100*time.Millisecond) + optsB.LameDuckDuration = 10 * time.Millisecond + srvB = RunServer(optsB) + defer srvB.Shutdown() + + optsC := DefaultOptions() + optsC.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srvA.ClusterAddr().Port)) + testSetLDMGracePeriod(optsC, 100*time.Millisecond) + optsC.LameDuckGracePeriod = -100 * time.Millisecond + optsC.LameDuckDuration = 10 * time.Millisecond + srvC := RunServer(optsC) + defer srvC.Shutdown() + + checkClusterFormed(t, srvA, srvB, srvC) + + rt := int32(0) + nc, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsA.Port), + nats.ReconnectWait(15*time.Millisecond), + nats.ReconnectHandler(func(*nats.Conn) { + atomic.AddInt32(&rt, 1) + })) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + go srvA.lameDuckMode() + // Wait a bit, but less than lameDuckModeInitialDelay that we set in this + // test to 100ms. + time.Sleep(30 * time.Millisecond) + go srvB.lameDuckMode() + + srvA.grWG.Wait() + srvB.grWG.Wait() + checkClientsCount(t, srvC, 1) + checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { + if n := atomic.LoadInt32(&rt); n != 1 { + return fmt.Errorf("Expected client to reconnect only once, got %v", n) + } + return nil + }) +} + +func TestLameDuckModeInfo(t *testing.T) { + optsA := testWSOptions() + optsA.Cluster.Name = "abc" + optsA.Cluster.Host = "127.0.0.1" + optsA.Cluster.Port = -1 + // Ensure that initial delay is set very high so that we can + // check that some events occur as expected before the client + // is disconnected. + testSetLDMGracePeriod(optsA, 5*time.Second) + optsA.LameDuckDuration = 50 * time.Millisecond + optsA.DisableShortFirstPing = true + srvA := RunServer(optsA) + defer srvA.Shutdown() + + curla := fmt.Sprintf("127.0.0.1:%d", optsA.Port) + wscurla := fmt.Sprintf("127.0.0.1:%d", optsA.Websocket.Port) + c, err := net.Dial("tcp", curla) + if err != nil { + t.Fatalf("Error connecting: %v", err) + } + defer c.Close() + client := bufio.NewReaderSize(c, maxBufSize) + + wsconn, wsclient := testWSCreateClient(t, false, false, optsA.Websocket.Host, optsA.Websocket.Port) + defer wsconn.Close() + + getInfo := func(ws bool) *serverInfo { + t.Helper() + var l string + var err error + if ws { + l = string(testWSReadFrame(t, wsclient)) + } else { + l, err = client.ReadString('\n') + if err != nil { + t.Fatalf("Error receiving info from server: %v\n", err) + } + } + var info serverInfo + if err = json.Unmarshal([]byte(l[5:]), &info); err != nil { + t.Fatalf("Could not parse INFO json: %v\n", err) + } + return &info + } + + getInfo(false) + c.Write([]byte("CONNECT {\"protocol\":1,\"verbose\":false}\r\nPING\r\n")) + // Consume both the first PONG and INFO in response to the Connect. + client.ReadString('\n') + client.ReadString('\n') + + optsB := testWSOptions() + optsB.Cluster.Name = "abc" + optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srvA.ClusterAddr().Port)) + srvB := RunServer(optsB) + defer srvB.Shutdown() + + checkClusterFormed(t, srvA, srvB) + + checkConnectURLs := func(expected [][]string) *serverInfo { + t.Helper() + var si *serverInfo + for i, ws := range []bool{false, true} { + slices.Sort(expected[i]) + si = getInfo(ws) + slices.Sort(si.ConnectURLs) + if !reflect.DeepEqual(expected[i], si.ConnectURLs) { + t.Fatalf("Expected %q, got %q", expected, si.ConnectURLs) + } + } + return si + } + + curlb := fmt.Sprintf("127.0.0.1:%d", optsB.Port) + wscurlb := fmt.Sprintf("127.0.0.1:%d", optsB.Websocket.Port) + expected := [][]string{{curla, curlb}, {wscurla, wscurlb}} + checkConnectURLs(expected) + + optsC := testWSOptions() + testSetLDMGracePeriod(optsA, 5*time.Second) + optsC.Cluster.Name = "abc" + optsC.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srvA.ClusterAddr().Port)) + srvC := RunServer(optsC) + defer srvC.Shutdown() + + checkClusterFormed(t, srvA, srvB, srvC) + + curlc := fmt.Sprintf("127.0.0.1:%d", optsC.Port) + wscurlc := fmt.Sprintf("127.0.0.1:%d", optsC.Websocket.Port) + expected = [][]string{{curla, curlb, curlc}, {wscurla, wscurlb, wscurlc}} + checkConnectURLs(expected) + + optsD := testWSOptions() + optsD.Cluster.Name = "abc" + optsD.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srvA.ClusterAddr().Port)) + srvD := RunServer(optsD) + defer srvD.Shutdown() + + checkClusterFormed(t, srvA, srvB, srvC, srvD) + + curld := fmt.Sprintf("127.0.0.1:%d", optsD.Port) + wscurld := fmt.Sprintf("127.0.0.1:%d", optsD.Websocket.Port) + expected = [][]string{{curla, curlb, curlc, curld}, {wscurla, wscurlb, wscurlc, wscurld}} + checkConnectURLs(expected) + + // Now lame duck server A and C. We should have client connected to A + // receive info that A is in LDM without A's URL, but also receive + // an update with C's URL gone. + // But first we need to create a client to C because otherwise the + // LDM signal will just shut it down because it would have no client. + nc, err := nats.Connect(srvC.ClientURL()) + if err != nil { + t.Fatalf("Failed to connect: %v", err) + } + defer nc.Close() + nc.Flush() + + start := time.Now() + wg := sync.WaitGroup{} + wg.Add(2) + go func() { + defer wg.Done() + srvA.lameDuckMode() + }() + + expected = [][]string{{curlb, curlc, curld}, {wscurlb, wscurlc, wscurld}} + si := checkConnectURLs(expected) + if !si.LameDuckMode { + t.Fatal("Expected LameDuckMode to be true, it was not") + } + + // Start LDM for server C. This should send an update to A + // which in turn should remove C from the list of URLs and + // update its client. + go func() { + defer wg.Done() + srvC.lameDuckMode() + }() + + expected = [][]string{{curlb, curld}, {wscurlb, wscurld}} + si = checkConnectURLs(expected) + // This update should not say that it is LDM. + if si.LameDuckMode { + t.Fatal("Expected LameDuckMode to be false, it was true") + } + + // Now shutdown D, and we also should get an update. + srvD.Shutdown() + + expected = [][]string{{curlb}, {wscurlb}} + si = checkConnectURLs(expected) + // This update should not say that it is LDM. + if si.LameDuckMode { + t.Fatal("Expected LameDuckMode to be false, it was true") + } + if time.Since(start) > 2*time.Second { + t.Fatalf("Did not get the expected events prior of server A and C shutting down") + } + + // Now explicitly shutdown srvA. When a server shutdown, it closes all its + // connections. For routes, it means that it is going to remove the remote's + // URL from its map. We want to make sure that in that case, server does not + // actually send an updated INFO to its clients. + srvA.Shutdown() + + // Expect nothing to be received on the client connection. + if l, err := client.ReadString('\n'); err == nil { + t.Fatalf("Expected connection to fail, instead got %q", l) + } + + c.Close() + nc.Close() + // Don't need to wait for actual disconnect of clients. + srvC.Shutdown() + wg.Wait() +} + +func TestServerValidateGatewaysOptions(t *testing.T) { + baseOpt := testDefaultOptionsForGateway("A") + u, _ := url.Parse("host:5222") + g := &RemoteGatewayOpts{ + URLs: []*url.URL{u}, + } + baseOpt.Gateway.Gateways = append(baseOpt.Gateway.Gateways, g) + + for _, test := range []struct { + name string + opts func() *Options + expectedErr string + }{ + { + name: "gateway_has_no_name", + opts: func() *Options { + o := baseOpt.Clone() + o.Gateway.Name = "" + return o + }, + expectedErr: "has no name", + }, + { + name: "gateway_has_no_port", + opts: func() *Options { + o := baseOpt.Clone() + o.Gateway.Port = 0 + return o + }, + expectedErr: "no port specified", + }, + { + name: "gateway_dst_has_no_name", + opts: func() *Options { + o := baseOpt.Clone() + return o + }, + expectedErr: "has no name", + }, + { + name: "gateway_dst_urls_is_nil", + opts: func() *Options { + o := baseOpt.Clone() + o.Gateway.Gateways[0].Name = "B" + o.Gateway.Gateways[0].URLs = nil + return o + }, + expectedErr: "has no URL", + }, + { + name: "gateway_dst_urls_is_empty", + opts: func() *Options { + o := baseOpt.Clone() + o.Gateway.Gateways[0].Name = "B" + o.Gateway.Gateways[0].URLs = []*url.URL{} + return o + }, + expectedErr: "has no URL", + }, + } { + t.Run(test.name, func(t *testing.T) { + if err := validateOptions(test.opts()); err == nil || !strings.Contains(err.Error(), test.expectedErr) { + t.Fatalf("Expected error about %q, got %v", test.expectedErr, err) + } + }) + } +} + +func TestAcceptError(t *testing.T) { + o := DefaultOptions() + s := New(o) + s.running.Store(true) + defer s.Shutdown() + orgDelay := time.Hour + delay := s.acceptError("Test", fmt.Errorf("any error"), orgDelay) + if delay != orgDelay { + t.Fatalf("With this type of error, delay should have stayed same, got %v", delay) + } + + // Create any net.Error and make it a temporary + ne := &net.DNSError{IsTemporary: true} + orgDelay = 10 * time.Millisecond + delay = s.acceptError("Test", ne, orgDelay) + if delay != 2*orgDelay { + t.Fatalf("Expected delay to double, got %v", delay) + } + // Now check the max + orgDelay = 60 * ACCEPT_MAX_SLEEP / 100 + delay = s.acceptError("Test", ne, orgDelay) + if delay != ACCEPT_MAX_SLEEP { + t.Fatalf("Expected delay to double, got %v", delay) + } + wg := sync.WaitGroup{} + wg.Add(1) + start := time.Now() + go func() { + s.acceptError("Test", ne, orgDelay) + wg.Done() + }() + time.Sleep(100 * time.Millisecond) + // This should kick out the sleep in acceptError + s.Shutdown() + if dur := time.Since(start); dur >= ACCEPT_MAX_SLEEP { + t.Fatalf("Shutdown took too long: %v", dur) + } + wg.Wait() + if d := s.acceptError("Test", ne, orgDelay); d >= 0 { + t.Fatalf("Expected delay to be negative, got %v", d) + } +} + +func TestServerShutdownDuringStart(t *testing.T) { + o := DefaultOptions() + o.ServerName = "server" + o.DisableShortFirstPing = true + o.Accounts = []*Account{NewAccount("$SYS")} + o.SystemAccount = "$SYS" + o.Cluster.Name = "abc" + o.Cluster.Host = "127.0.0.1" + o.Cluster.Port = -1 + o.Gateway.Name = "abc" + o.Gateway.Host = "127.0.0.1" + o.Gateway.Port = -1 + o.LeafNode.Host = "127.0.0.1" + o.LeafNode.Port = -1 + o.Websocket.Host = "127.0.0.1" + o.Websocket.Port = -1 + o.Websocket.HandshakeTimeout = 1 + o.Websocket.NoTLS = true + o.MQTT.Host = "127.0.0.1" + o.MQTT.Port = -1 + + // We are going to test that if the server is shutdown + // while Start() runs (in this case, before), we don't + // start the listeners and therefore leave accept loops + // hanging. + s, err := NewServer(o) + if err != nil { + t.Fatalf("Error creating server: %v", err) + } + s.Shutdown() + + // Start() should not block, but just in case, start in + // different go routine. + ch := make(chan struct{}, 1) + go func() { + s.Start() + close(ch) + }() + select { + case <-ch: + case <-time.After(time.Second): + t.Fatal("Start appear to have blocked after server was shutdown") + } + // Now make sure that none of the listeners have been created + listeners := []string{} + s.mu.Lock() + if s.listener != nil { + listeners = append(listeners, "client") + } + if s.routeListener != nil { + listeners = append(listeners, "route") + } + if s.gatewayListener != nil { + listeners = append(listeners, "gateway") + } + if s.leafNodeListener != nil { + listeners = append(listeners, "leafnode") + } + if s.websocket.listener != nil { + listeners = append(listeners, "websocket") + } + if s.mqtt.listener != nil { + listeners = append(listeners, "mqtt") + } + s.mu.Unlock() + if len(listeners) > 0 { + lst := "" + for i, l := range listeners { + if i > 0 { + lst += ", " + } + lst += l + } + t.Fatalf("Following listeners have been created: %s", lst) + } +} + +type myDummyDNSResolver struct { + ips []string + err error +} + +func (r *myDummyDNSResolver) LookupHost(ctx context.Context, host string) ([]string, error) { + if r.err != nil { + return nil, r.err + } + return r.ips, nil +} + +func TestGetRandomIP(t *testing.T) { + s := &Server{} + resolver := &myDummyDNSResolver{} + // no port... + if _, err := s.getRandomIP(resolver, "noport", nil); err == nil || !strings.Contains(err.Error(), "port") { + t.Fatalf("Expected error about port missing, got %v", err) + } + resolver.err = fmt.Errorf("on purpose") + if _, err := s.getRandomIP(resolver, "localhost:4222", nil); err == nil || !strings.Contains(err.Error(), "on purpose") { + t.Fatalf("Expected error about no port, got %v", err) + } + resolver.err = nil + a, err := s.getRandomIP(resolver, "localhost:4222", nil) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if a != "localhost:4222" { + t.Fatalf("Expected address to be %q, got %q", "localhost:4222", a) + } + resolver.ips = []string{"1.2.3.4"} + a, err = s.getRandomIP(resolver, "localhost:4222", nil) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if a != "1.2.3.4:4222" { + t.Fatalf("Expected address to be %q, got %q", "1.2.3.4:4222", a) + } + // Check for randomness + resolver.ips = []string{"1.2.3.4", "2.2.3.4", "3.2.3.4"} + dist := [3]int{} + for i := 0; i < 100; i++ { + ip, err := s.getRandomIP(resolver, "localhost:4222", nil) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + v := int(ip[0]-'0') - 1 + dist[v]++ + } + low := 20 + high := 47 + for i, d := range dist { + if d == 0 || d == 100 { + t.Fatalf("Unexpected distribution for ip %v, got %v", i, d) + } else if d < low || d > high { + t.Logf("Warning: out of expected range [%v,%v] for ip %v, got %v", low, high, i, d) + } + } + + // Check IP exclusions + excludedIPs := map[string]struct{}{"1.2.3.4:4222": {}} + for i := 0; i < 100; i++ { + ip, err := s.getRandomIP(resolver, "localhost:4222", excludedIPs) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if ip[0] == '1' { + t.Fatalf("Should not have returned this ip: %q", ip) + } + } + excludedIPs["2.2.3.4:4222"] = struct{}{} + for i := 0; i < 100; i++ { + ip, err := s.getRandomIP(resolver, "localhost:4222", excludedIPs) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if ip[0] != '3' { + t.Fatalf("Should only have returned '3.2.3.4', got returned %q", ip) + } + } + excludedIPs["3.2.3.4:4222"] = struct{}{} + for i := 0; i < 100; i++ { + if _, err := s.getRandomIP(resolver, "localhost:4222", excludedIPs); err != errNoIPAvail { + t.Fatalf("Unexpected error: %v", err) + } + } + + // Now check that exclusion takes into account the port number. + resolver.ips = []string{"127.0.0.1"} + excludedIPs = map[string]struct{}{"127.0.0.1:4222": {}} + for i := 0; i < 100; i++ { + if _, err := s.getRandomIP(resolver, "localhost:4223", excludedIPs); err == errNoIPAvail { + t.Fatal("Should not have failed") + } + } +} + +type shortWriteConn struct { + net.Conn +} + +func (swc *shortWriteConn) Write(b []byte) (int, error) { + // Limit the write to 10 bytes at a time. + short := false + max := len(b) + if max > 10 { + max = 10 + short = true + } + n, err := swc.Conn.Write(b[:max]) + if err == nil && short { + return n, io.ErrShortWrite + } + return n, err +} + +func TestClientWriteLoopStall(t *testing.T) { + opts := DefaultOptions() + s := RunServer(opts) + defer s.Shutdown() + + errCh := make(chan error, 1) + + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(url, + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, e error) { + select { + case errCh <- e: + default: + } + })) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + sub, err := nc.SubscribeSync("foo") + if err != nil { + t.Fatalf("Error on subscribe: %v", err) + } + nc.Flush() + cid, _ := nc.GetClientID() + + sender, err := nats.Connect(url) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer sender.Close() + + c := s.getClient(cid) + c.mu.Lock() + c.nc = &shortWriteConn{Conn: c.nc} + c.mu.Unlock() + + sender.Publish("foo", make([]byte, 100)) + + if _, err := sub.NextMsg(3 * time.Second); err != nil { + t.Fatalf("WriteLoop has stalled!") + } + + // Make sure that we did not get any async error + select { + case e := <-errCh: + t.Fatalf("Got error: %v", e) + case <-time.After(250 * time.Millisecond): + } +} + +func TestInsecureSkipVerifyWarning(t *testing.T) { + checkWarnReported := func(t *testing.T, o *Options, expectedWarn string) { + t.Helper() + s, err := NewServer(o) + if err != nil { + t.Fatalf("Error on new server: %v", err) + } + l := &captureWarnLogger{warn: make(chan string, 1)} + s.SetLogger(l, false, false) + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + s.Start() + wg.Done() + }() + if err := s.readyForConnections(time.Second); err != nil { + t.Fatal(err) + } + select { + case w := <-l.warn: + if !strings.Contains(w, expectedWarn) { + t.Fatalf("Expected warning %q, got %q", expectedWarn, w) + } + case <-time.After(2 * time.Second): + t.Fatalf("Did not get warning %q", expectedWarn) + } + s.Shutdown() + wg.Wait() + } + + tc := &TLSConfigOpts{} + tc.CertFile = "../test/configs/certs/server-cert.pem" + tc.KeyFile = "../test/configs/certs/server-key.pem" + tc.CaFile = "../test/configs/certs/ca.pem" + tc.Insecure = true + config, err := GenTLSConfig(tc) + if err != nil { + t.Fatalf("Error generating tls config: %v", err) + } + + o := DefaultOptions() + o.Cluster.Name = "A" + o.Cluster.Port = -1 + o.Cluster.TLSConfig = config.Clone() + checkWarnReported(t, o, clusterTLSInsecureWarning) + + // Remove the route setting + o.Cluster.Port = 0 + o.Cluster.TLSConfig = nil + + // Configure LeafNode with no TLS in the main block first, but only with remotes. + o.LeafNode.Port = -1 + rurl, _ := url.Parse("nats://127.0.0.1:1234") + o.LeafNode.Remotes = []*RemoteLeafOpts{ + { + URLs: []*url.URL{rurl}, + TLSConfig: config.Clone(), + }, + } + checkWarnReported(t, o, leafnodeTLSInsecureWarning) + + // Now add to main block. + o.LeafNode.TLSConfig = config.Clone() + checkWarnReported(t, o, leafnodeTLSInsecureWarning) + + // Now remove remote and check warning still reported + o.LeafNode.Remotes = nil + checkWarnReported(t, o, leafnodeTLSInsecureWarning) + + // Remove the LN setting + o.LeafNode.Port = 0 + o.LeafNode.TLSConfig = nil + + // Configure GW with no TLS in main block first, but only with remotes + o.Gateway.Name = "A" + o.Gateway.Host = "127.0.0.1" + o.Gateway.Port = -1 + o.Gateway.Gateways = []*RemoteGatewayOpts{ + { + Name: "B", + URLs: []*url.URL{rurl}, + TLSConfig: config.Clone(), + }, + } + checkWarnReported(t, o, gatewayTLSInsecureWarning) + + // Now add to main block. + o.Gateway.TLSConfig = config.Clone() + checkWarnReported(t, o, gatewayTLSInsecureWarning) + + // Now remove remote and check warning still reported + o.Gateway.Gateways = nil + checkWarnReported(t, o, gatewayTLSInsecureWarning) +} + +func TestConnectErrorReports(t *testing.T) { + // On Windows, an attempt to connect to a port that has no listener will + // take whatever timeout specified in DialTimeout() before failing. + // So skip for now. + if runtime.GOOS == "windows" { + t.Skip() + } + // Check that default report attempts is as expected + opts := DefaultOptions() + s := RunServer(opts) + defer s.Shutdown() + + if ra := s.getOpts().ConnectErrorReports; ra != DEFAULT_CONNECT_ERROR_REPORTS { + t.Fatalf("Expected default value to be %v, got %v", DEFAULT_CONNECT_ERROR_REPORTS, ra) + } + + tmpFile := createTempFile(t, "") + log := tmpFile.Name() + tmpFile.Close() + + remoteURLs := RoutesFromStr("nats://127.0.0.1:1234") + + opts = DefaultOptions() + opts.ConnectErrorReports = 3 + opts.Cluster.Port = -1 + opts.Routes = remoteURLs + opts.NoLog = false + opts.LogFile = log + opts.Logtime = true + opts.Debug = true + s = RunServer(opts) + defer s.Shutdown() + + checkContent := func(t *testing.T, txt string, attempt int, shouldBeThere bool) { + t.Helper() + checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { + content, err := os.ReadFile(log) + if err != nil { + return fmt.Errorf("Error reading log file: %v", err) + } + present := bytes.Contains(content, []byte(fmt.Sprintf("%s (attempt %d)", txt, attempt))) + if shouldBeThere && !present { + return fmt.Errorf("Did not find expected log statement (%s) for attempt %d: %s", txt, attempt, content) + } else if !shouldBeThere && present { + return fmt.Errorf("Log statement (%s) for attempt %d should not be present: %s", txt, attempt, content) + } + return nil + }) + } + + type testConnect struct { + name string + attempt int + errExpected bool + } + for _, test := range []testConnect{ + {"route_attempt_1", 1, true}, + {"route_attempt_2", 2, false}, + {"route_attempt_3", 3, true}, + {"route_attempt_4", 4, false}, + {"route_attempt_6", 6, true}, + {"route_attempt_7", 7, false}, + } { + t.Run(test.name, func(t *testing.T) { + debugExpected := !test.errExpected + checkContent(t, "[DBG] Error trying to connect to route", test.attempt, debugExpected) + checkContent(t, "[ERR] Error trying to connect to route", test.attempt, test.errExpected) + }) + } + + s.Shutdown() + removeFile(t, log) + + // Now try with leaf nodes + opts.Cluster.Port = 0 + opts.Routes = nil + opts.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{remoteURLs[0]}}} + opts.LeafNode.ReconnectInterval = 15 * time.Millisecond + s = RunServer(opts) + defer s.Shutdown() + + checkLeafContent := func(t *testing.T, txt, host string, attempt int, shouldBeThere bool) { + t.Helper() + checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { + content, err := os.ReadFile(log) + if err != nil { + return fmt.Errorf("Error reading log file: %v", err) + } + present := bytes.Contains(content, []byte(fmt.Sprintf("%s %q (attempt %d)", txt, host, attempt))) + if shouldBeThere && !present { + return fmt.Errorf("Did not find expected log statement (%s %q) for attempt %d: %s", txt, host, attempt, content) + } else if !shouldBeThere && present { + return fmt.Errorf("Log statement (%s %q) for attempt %d should not be present: %s", txt, host, attempt, content) + } + return nil + }) + } + + for _, test := range []testConnect{ + {"leafnode_attempt_1", 1, true}, + {"leafnode_attempt_2", 2, false}, + {"leafnode_attempt_3", 3, true}, + {"leafnode_attempt_4", 4, false}, + {"leafnode_attempt_6", 6, true}, + {"leafnode_attempt_7", 7, false}, + } { + t.Run(test.name, func(t *testing.T) { + debugExpected := !test.errExpected + checkLeafContent(t, "[DBG] Error trying to connect as leafnode to remote server", remoteURLs[0].Host, test.attempt, debugExpected) + checkLeafContent(t, "[ERR] Error trying to connect as leafnode to remote server", remoteURLs[0].Host, test.attempt, test.errExpected) + }) + } + + s.Shutdown() + removeFile(t, log) + + // Now try with gateways + opts.LeafNode.Remotes = nil + opts.Cluster.Name = "A" + opts.Gateway.Name = "A" + opts.Gateway.Port = -1 + opts.Gateway.Gateways = []*RemoteGatewayOpts{ + { + Name: "B", + URLs: remoteURLs, + }, + } + opts.gatewaysSolicitDelay = 15 * time.Millisecond + s = RunServer(opts) + defer s.Shutdown() + + for _, test := range []testConnect{ + {"gateway_attempt_1", 1, true}, + {"gateway_attempt_2", 2, false}, + {"gateway_attempt_3", 3, true}, + {"gateway_attempt_4", 4, false}, + {"gateway_attempt_6", 6, true}, + {"gateway_attempt_7", 7, false}, + } { + t.Run(test.name, func(t *testing.T) { + debugExpected := !test.errExpected + infoExpected := test.errExpected + // For gateways, we also check our notice that we attempt to connect + checkContent(t, "[DBG] Connecting to explicit gateway \"B\" (127.0.0.1:1234) at 127.0.0.1:1234", test.attempt, debugExpected) + checkContent(t, "[INF] Connecting to explicit gateway \"B\" (127.0.0.1:1234) at 127.0.0.1:1234", test.attempt, infoExpected) + checkContent(t, "[DBG] Error connecting to explicit gateway \"B\" (127.0.0.1:1234) at 127.0.0.1:1234", test.attempt, debugExpected) + checkContent(t, "[ERR] Error connecting to explicit gateway \"B\" (127.0.0.1:1234) at 127.0.0.1:1234", test.attempt, test.errExpected) + }) + } +} + +func TestReconnectErrorReports(t *testing.T) { + // On Windows, an attempt to connect to a port that has no listener will + // take whatever timeout specified in DialTimeout() before failing. + // So skip for now. + if runtime.GOOS == "windows" { + t.Skip() + } + // Check that default report attempts is as expected + opts := DefaultOptions() + s := RunServer(opts) + defer s.Shutdown() + + if ra := s.getOpts().ReconnectErrorReports; ra != DEFAULT_RECONNECT_ERROR_REPORTS { + t.Fatalf("Expected default value to be %v, got %v", DEFAULT_RECONNECT_ERROR_REPORTS, ra) + } + + tmpFile := createTempFile(t, "") + log := tmpFile.Name() + tmpFile.Close() + + csOpts := DefaultOptions() + csOpts.Cluster.Port = -1 + cs := RunServer(csOpts) + defer cs.Shutdown() + + opts = DefaultOptions() + opts.ReconnectErrorReports = 3 + opts.Cluster.Port = -1 + opts.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", cs.ClusterAddr().Port)) + opts.NoLog = false + opts.LogFile = log + opts.Logtime = true + opts.Debug = true + s = RunServer(opts) + defer s.Shutdown() + + // Wait for cluster to be formed + checkClusterFormed(t, s, cs) + + // Now shutdown the server s connected to. + cs.Shutdown() + + // Specifically for route test, wait at least reconnect interval before checking logs + time.Sleep(routeReconnectDelay) + + checkContent := func(t *testing.T, txt string, attempt int, shouldBeThere bool) { + t.Helper() + checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { + content, err := os.ReadFile(log) + if err != nil { + return fmt.Errorf("Error reading log file: %v", err) + } + present := bytes.Contains(content, []byte(fmt.Sprintf("%s (attempt %d)", txt, attempt))) + if shouldBeThere && !present { + return fmt.Errorf("Did not find expected log statement (%s) for attempt %d: %s", txt, attempt, content) + } else if !shouldBeThere && present { + return fmt.Errorf("Log statement (%s) for attempt %d should not be present: %s", txt, attempt, content) + } + return nil + }) + } + + type testConnect struct { + name string + attempt int + errExpected bool + } + for _, test := range []testConnect{ + {"route_attempt_1", 1, true}, + {"route_attempt_2", 2, false}, + {"route_attempt_3", 3, true}, + {"route_attempt_4", 4, false}, + {"route_attempt_6", 6, true}, + {"route_attempt_7", 7, false}, + } { + t.Run(test.name, func(t *testing.T) { + debugExpected := !test.errExpected + checkContent(t, "[DBG] Error trying to connect to route", test.attempt, debugExpected) + checkContent(t, "[ERR] Error trying to connect to route", test.attempt, test.errExpected) + }) + } + + s.Shutdown() + removeFile(t, log) + + // Now try with leaf nodes + csOpts.Cluster.Port = 0 + csOpts.Cluster.Name = _EMPTY_ + csOpts.LeafNode.Host = "127.0.0.1" + csOpts.LeafNode.Port = -1 + + cs = RunServer(csOpts) + defer cs.Shutdown() + + opts.Cluster.Port = 0 + opts.Cluster.Name = _EMPTY_ + opts.Routes = nil + u, _ := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", csOpts.LeafNode.Port)) + opts.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}} + opts.LeafNode.ReconnectInterval = 15 * time.Millisecond + s = RunServer(opts) + defer s.Shutdown() + + checkLeafNodeConnected(t, s) + + // Now shutdown the server s is connected to + cs.Shutdown() + + checkLeafContent := func(t *testing.T, txt, host string, attempt int, shouldBeThere bool) { + t.Helper() + checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { + content, err := os.ReadFile(log) + if err != nil { + return fmt.Errorf("Error reading log file: %v", err) + } + present := bytes.Contains(content, []byte(fmt.Sprintf("%s %q (attempt %d)", txt, host, attempt))) + if shouldBeThere && !present { + return fmt.Errorf("Did not find expected log statement (%s %q) for attempt %d: %s", txt, host, attempt, content) + } else if !shouldBeThere && present { + return fmt.Errorf("Log statement (%s %q) for attempt %d should not be present: %s", txt, host, attempt, content) + } + return nil + }) + } + + for _, test := range []testConnect{ + {"leafnode_attempt_1", 1, true}, + {"leafnode_attempt_2", 2, false}, + {"leafnode_attempt_3", 3, true}, + {"leafnode_attempt_4", 4, false}, + {"leafnode_attempt_6", 6, true}, + {"leafnode_attempt_7", 7, false}, + } { + t.Run(test.name, func(t *testing.T) { + debugExpected := !test.errExpected + checkLeafContent(t, "[DBG] Error trying to connect as leafnode to remote server", u.Host, test.attempt, debugExpected) + checkLeafContent(t, "[ERR] Error trying to connect as leafnode to remote server", u.Host, test.attempt, test.errExpected) + }) + } + + s.Shutdown() + removeFile(t, log) + + // Now try with gateways + csOpts.LeafNode.Port = 0 + csOpts.Cluster.Name = "B" + csOpts.Gateway.Name = "B" + csOpts.Gateway.Port = -1 + cs = RunServer(csOpts) + + opts.LeafNode.Remotes = nil + opts.Cluster.Name = "A" + opts.Gateway.Name = "A" + opts.Gateway.Port = -1 + remoteGWPort := cs.GatewayAddr().Port + u, _ = url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", remoteGWPort)) + opts.Gateway.Gateways = []*RemoteGatewayOpts{ + { + Name: "B", + URLs: []*url.URL{u}, + }, + } + opts.gatewaysSolicitDelay = 15 * time.Millisecond + s = RunServer(opts) + defer s.Shutdown() + + waitForOutboundGateways(t, s, 1, 2*time.Second) + waitForInboundGateways(t, s, 1, 2*time.Second) + + // Now stop server s is connecting to + cs.Shutdown() + + connTxt := fmt.Sprintf("Connecting to explicit gateway \"B\" (127.0.0.1:%d) at 127.0.0.1:%d", remoteGWPort, remoteGWPort) + dbgConnTxt := fmt.Sprintf("[DBG] %s", connTxt) + infConnTxt := fmt.Sprintf("[INF] %s", connTxt) + + errTxt := fmt.Sprintf("Error connecting to explicit gateway \"B\" (127.0.0.1:%d) at 127.0.0.1:%d", remoteGWPort, remoteGWPort) + dbgErrTxt := fmt.Sprintf("[DBG] %s", errTxt) + errErrTxt := fmt.Sprintf("[ERR] %s", errTxt) + + for _, test := range []testConnect{ + {"gateway_attempt_1", 1, true}, + {"gateway_attempt_2", 2, false}, + {"gateway_attempt_3", 3, true}, + {"gateway_attempt_4", 4, false}, + {"gateway_attempt_6", 6, true}, + {"gateway_attempt_7", 7, false}, + } { + t.Run(test.name, func(t *testing.T) { + debugExpected := !test.errExpected + infoExpected := test.errExpected + // For gateways, we also check our notice that we attempt to connect + checkContent(t, dbgConnTxt, test.attempt, debugExpected) + checkContent(t, infConnTxt, test.attempt, infoExpected) + checkContent(t, dbgErrTxt, test.attempt, debugExpected) + checkContent(t, errErrTxt, test.attempt, test.errExpected) + }) + } +} + +func TestServerLogsConfigurationFile(t *testing.T) { + file := createTempFile(t, "nats_server_log_") + file.Close() + + conf := createConfFile(t, []byte(fmt.Sprintf(` + port: -1 + logfile: '%s' + `, file.Name()))) + + o := LoadConfig(conf) + o.ConfigFile = file.Name() + o.NoLog = false + s := RunServer(o) + s.Shutdown() + + log, err := os.ReadFile(file.Name()) + if err != nil { + t.Fatalf("Error reading log file: %v", err) + } + if !bytes.Contains(log, []byte(fmt.Sprintf("Using configuration file: %s", file.Name()))) { + t.Fatalf("Config file location was not reported in log: %s", log) + } +} + +func TestServerRateLimitLogging(t *testing.T) { + s := RunServer(DefaultOptions()) + defer s.Shutdown() + + s.changeRateLimitLogInterval(100 * time.Millisecond) + + l := &captureWarnLogger{warn: make(chan string, 100)} + s.SetLogger(l, false, false) + + s.RateLimitWarnf("Warning number 1") + s.RateLimitWarnf("Warning number 2") + s.rateLimitFormatWarnf("warning value %d", 1) + s.RateLimitWarnf("Warning number 1") + s.RateLimitWarnf("Warning number 2") + s.rateLimitFormatWarnf("warning value %d", 2) + + checkLog := func(c1, c2 *client) { + t.Helper() + + nb1 := "Warning number 1" + nb2 := "Warning number 2" + nbv := "warning value" + gotOne := 0 + gotTwo := 0 + gotFormat := 0 + for done := false; !done; { + select { + case w := <-l.warn: + if strings.Contains(w, nb1) { + gotOne++ + } else if strings.Contains(w, nb2) { + gotTwo++ + } else if strings.Contains(w, nbv) { + gotFormat++ + } + case <-time.After(150 * time.Millisecond): + done = true + } + } + if gotOne != 1 { + t.Fatalf("Should have had only 1 warning for nb1, got %v", gotOne) + } + if gotTwo != 1 { + t.Fatalf("Should have had only 1 warning for nb2, got %v", gotTwo) + } + if gotFormat != 1 { + t.Fatalf("Should have had only 1 warning for format, got %v", gotFormat) + } + + // Wait for more than the expiration interval + time.Sleep(200 * time.Millisecond) + if c1 == nil { + s.RateLimitWarnf("%s", nb1) + s.rateLimitFormatWarnf("warning value %d", 1) + } else { + c1.RateLimitWarnf("%s", nb1) + c2.RateLimitWarnf("%s", nb1) + c1.rateLimitFormatWarnf("warning value %d", 1) + } + gotOne = 0 + gotFormat = 0 + for { + select { + case w := <-l.warn: + if strings.Contains(w, nb1) { + gotOne++ + } else if strings.Contains(w, nbv) { + gotFormat++ + } + case <-time.After(200 * time.Millisecond): + if gotOne == 0 { + t.Fatalf("Warning was still suppressed") + } else if gotOne > 1 { + t.Fatalf("Should have had only 1 warning for nb1, got %v", gotOne) + } else if gotFormat == 0 { + t.Fatalf("Warning was still suppressed") + } else if gotFormat > 1 { + t.Fatalf("Should have had only 1 warning for format, got %v", gotFormat) + } else { + // OK! we are done + return + } + } + } + } + + checkLog(nil, nil) + + nc1 := natsConnect(t, s.ClientURL(), nats.Name("c1")) + defer nc1.Close() + nc2 := natsConnect(t, s.ClientURL(), nats.Name("c2")) + defer nc2.Close() + + var c1 *client + var c2 *client + s.mu.Lock() + for _, cli := range s.clients { + cli.mu.Lock() + switch cli.opts.Name { + case "c1": + c1 = cli + case "c2": + c2 = cli + } + cli.mu.Unlock() + if c1 != nil && c2 != nil { + break + } + } + s.mu.Unlock() + if c1 == nil || c2 == nil { + t.Fatal("Did not find the clients") + } + + // Wait for more than the expiration interval + time.Sleep(200 * time.Millisecond) + + c1.RateLimitWarnf("Warning number 1") + c1.RateLimitWarnf("Warning number 2") + c1.rateLimitFormatWarnf("warning value %d", 1) + c2.RateLimitWarnf("Warning number 1") + c2.RateLimitWarnf("Warning number 2") + c2.rateLimitFormatWarnf("warning value %d", 2) + + checkLog(c1, c2) +} + +// https://github.com/nats-io/nats-server/discussions/4535 +func TestServerAuthBlockAndSysAccounts(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + server_name: s-test + authorization { + users = [ { user: "u", password: "pass"} ] + } + accounts { + $SYS: { users: [ { user: admin, password: pwd } ] } + } + `)) + + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + // This should work of course. + nc, err := nats.Connect(s.ClientURL(), nats.UserInfo("u", "pass")) + require_NoError(t, err) + defer nc.Close() + + // This should not. + _, err = nats.Connect(s.ClientURL()) + require_Error(t, err, nats.ErrAuthorization, errors.New("nats: Authorization Violation")) +} + +// https://github.com/nats-io/nats-server/issues/5396 +func TestServerConfigLastLineComments(t *testing.T) { + conf := createConfFile(t, []byte(` + { + "listen": "0.0.0.0:4222" + } + # wibble + `)) + + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + // This should work of course. + nc, err := nats.Connect(s.ClientURL()) + require_NoError(t, err) + defer nc.Close() +} + +func TestServerClusterAndGatewayNameNoSpace(t *testing.T) { + conf := createConfFile(t, []byte(` + port: -1 + server_name: "my server" + `)) + _, err := ProcessConfigFile(conf) + require_Error(t, err, ErrServerNameHasSpaces) + + o := DefaultOptions() + o.ServerName = "my server" + _, err = NewServer(o) + require_Error(t, err, ErrServerNameHasSpaces) + + conf = createConfFile(t, []byte(` + port: -1 + server_name: "myserver" + cluster { + port: -1 + name: "my cluster" + } + `)) + _, err = ProcessConfigFile(conf) + require_Error(t, err, ErrClusterNameHasSpaces) + + o = DefaultOptions() + o.Cluster.Name = "my cluster" + o.Cluster.Port = -1 + _, err = NewServer(o) + require_Error(t, err, ErrClusterNameHasSpaces) + + conf = createConfFile(t, []byte(` + port: -1 + server_name: "myserver" + gateway { + port: -1 + name: "my gateway" + } + `)) + _, err = ProcessConfigFile(conf) + require_Error(t, err, ErrGatewayNameHasSpaces) + + o = DefaultOptions() + o.Cluster.Name = _EMPTY_ + o.Cluster.Port = 0 + o.Gateway.Name = "my gateway" + o.Gateway.Port = -1 + _, err = NewServer(o) + require_Error(t, err, ErrGatewayNameHasSpaces) +} + +func TestServerClientURL(t *testing.T) { + for host, expected := range map[string]string{ + "host.com": "nats://host.com:12345", + "1.2.3.4": "nats://1.2.3.4:12345", + "2000::1": "nats://[2000::1]:12345", + } { + o := DefaultOptions() + o.Host = host + o.Port = 12345 + s, err := NewServer(o) + require_NoError(t, err) + require_Equal(t, s.ClientURL(), expected) + } +} + +// This is a test that guards against using goccy/go-json. +// At least until it's fully compatible with std encoding/json, and we've thoroughly tested it. +// This is just one bug (at the time of writing) that results in a panic. +// https://github.com/goccy/go-json/issues/519 +func TestServerJsonMarshalNestedStructsPanic(t *testing.T) { + type Item struct { + A string `json:"a"` + B string `json:"b,omitempty"` + } + + type Detail struct { + I Item `json:"i"` + } + + type Body struct { + Payload *Detail `json:"p,omitempty"` + } + + b, err := json.Marshal(Body{Payload: &Detail{I: Item{A: "a", B: "b"}}}) + require_NoError(t, err) + require_Equal(t, string(b), "{\"p\":{\"i\":{\"a\":\"a\",\"b\":\"b\"}}}") +} + +func TestBuildinfoFormatRevision(t *testing.T) { + tests := []struct { + name string + revision string + expected string + }{ + { + name: "Git-like longer version", + revision: "abc123def456789", + expected: "abc123d", + }, + { + name: "Git-like exactly 7 chars", + revision: "abc123d", + expected: "abc123d", + }, + { + name: "SVN shorter revision", + revision: "1234", + expected: "1234", + }, + { + name: "SVN single digit", + revision: "5", + expected: "5", + }, + { + name: "Empty revision", + revision: "", + expected: "", + }, + { + name: "6 character revision", + revision: "abc123", + expected: "abc123", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := formatRevision(tt.revision) + if result != tt.expected { + t.Errorf("formatRevision(%q) = %q, expected %q", tt.revision, result, tt.expected) + } + }) + } +} diff --git a/server/service.go b/server/service.go new file mode 100644 index 0000000..fe39419 --- /dev/null +++ b/server/service.go @@ -0,0 +1,28 @@ +// Copyright 2012-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !windows + +package server + +// Run starts the NATS server. This wrapper function allows Windows to add a +// hook for running NATS as a service. +func Run(server *Server) error { + server.Start() + return nil +} + +// isWindowsService indicates if NATS is running as a Windows service. +func isWindowsService() bool { + return false +} diff --git a/server/service_test.go b/server/service_test.go new file mode 100644 index 0000000..6a204a0 --- /dev/null +++ b/server/service_test.go @@ -0,0 +1,52 @@ +// Copyright 2012-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !windows + +package server + +import ( + "testing" + "time" +) + +func TestRun(t *testing.T) { + var ( + s = New(DefaultOptions()) + started = make(chan error, 1) + errC = make(chan error, 1) + ) + go func() { + errC <- Run(s) + }() + go func() { + if err := s.readyForConnections(time.Second); err != nil { + started <- err + return + } + s.Shutdown() + close(started) + }() + + select { + case err := <-errC: + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Timed out") + } + if err := <-started; err != nil { + t.Fatalf("Unexpected error: %v", err) + } +} diff --git a/server/service_windows.go b/server/service_windows.go new file mode 100644 index 0000000..2a6a581 --- /dev/null +++ b/server/service_windows.go @@ -0,0 +1,146 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "os" + "time" + + "golang.org/x/sys/windows/svc" +) + +const ( + reopenLogCode = 128 + reopenLogCmd = svc.Cmd(reopenLogCode) + ldmCode = 129 + ldmCmd = svc.Cmd(ldmCode) + acceptReopenLog = svc.Accepted(reopenLogCode) +) + +var serviceName = "nats-server" + +// SetServiceName allows setting a different service name +func SetServiceName(name string) { + serviceName = name +} + +// winServiceWrapper implements the svc.Handler interface for implementing +// nats-server as a Windows service. +type winServiceWrapper struct { + server *Server +} + +var dockerized = false + +func init() { + if v, exists := os.LookupEnv("NATS_DOCKERIZED"); exists && v == "1" { + dockerized = true + } +} + +// Execute will be called by the package code at the start of +// the service, and the service will exit once Execute completes. +// Inside Execute you must read service change requests from r and +// act accordingly. You must keep service control manager up to date +// about state of your service by writing into s as required. +// args contains service name followed by argument strings passed +// to the service. +// You can provide service exit code in exitCode return parameter, +// with 0 being "no error". You can also indicate if exit code, +// if any, is service specific or not by using svcSpecificEC +// parameter. +func (w *winServiceWrapper) Execute(args []string, changes <-chan svc.ChangeRequest, + status chan<- svc.Status) (bool, uint32) { + + status <- svc.Status{State: svc.StartPending} + go w.server.Start() + + var startupDelay = 10 * time.Second + if v, exists := os.LookupEnv("NATS_STARTUP_DELAY"); exists { + if delay, err := time.ParseDuration(v); err == nil { + startupDelay = delay + } else { + w.server.Errorf("Failed to parse \"%v\" as a duration for startup: %s", v, err) + } + } + // Wait for accept loop(s) to be started + if !w.server.ReadyForConnections(startupDelay) { + // Failed to start. + return false, 1 + } + + status <- svc.Status{ + State: svc.Running, + Accepts: svc.AcceptStop | svc.AcceptShutdown | svc.AcceptParamChange | acceptReopenLog, + } + +loop: + for { + select { + case change, ok := <-changes: + if !ok { + break loop + } + switch change.Cmd { + case svc.Interrogate: + status <- change.CurrentStatus + case svc.Stop, svc.Shutdown: + w.server.Shutdown() + break loop + case reopenLogCmd: + // File log re-open for rotating file logs. + w.server.ReOpenLogFile() + case ldmCmd: + go w.server.lameDuckMode() + case svc.ParamChange: + if err := w.server.Reload(); err != nil { + w.server.Errorf("Failed to reload server configuration: %s", err) + } + default: + w.server.Debugf("Unexpected control request: %v", change.Cmd) + } + case <-w.server.quitCh: + break loop + } + } + + status <- svc.Status{State: svc.StopPending} + return false, 0 +} + +// Run starts the NATS server as a Windows service. +func Run(server *Server) error { + if dockerized { + server.Start() + return nil + } + isWindowsService, err := svc.IsWindowsService() + if err != nil { + return err + } + if !isWindowsService { + server.Start() + return nil + } + return svc.Run(serviceName, &winServiceWrapper{server}) +} + +// isWindowsService indicates if NATS is running as a Windows service. +func isWindowsService() bool { + if dockerized { + return false + } + isWindowsService, _ := svc.IsWindowsService() + return isWindowsService +} diff --git a/server/service_windows_test.go b/server/service_windows_test.go new file mode 100644 index 0000000..567a042 --- /dev/null +++ b/server/service_windows_test.go @@ -0,0 +1,179 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package server + +import ( + "errors" + "fmt" + "testing" + "time" + + "golang.org/x/sys/windows/svc" +) + +// TestWinServiceWrapper reproduces the tests for service, +// with extra complication for windows API +func TestWinServiceWrapper(t *testing.T) { + /* + Since the windows API can't be tested through just the Run func, + a basic mock checks the intractions of the server, as happens in svc.Run. + This test checks : + - that the service fails to start within an unreasonable timeframe (serverC) + - that the service signals its state correctly to windows with svc.StartPending (mockC) + - that no other signal is sent to the windows service API(mockC) + */ + var ( + wsw = &winServiceWrapper{New(DefaultOptions())} + args = make([]string, 0) + // Size of 1 should be enough but don't want to block if the test fails. + serverC = make(chan error, 10) + mockC = make(chan error, 10) + changes = make(chan svc.ChangeRequest) + status = make(chan svc.Status) + ) + t.Setenv("NATS_STARTUP_DELAY", "1ns") // purposely small + // prepare mock expectations + wsm := &winSvcMock{status: status} + wsm.Expect(svc.StartPending) + + go func() { + mockC <- wsm.Listen(250 * time.Millisecond) + }() + + go func() { + var err error + _, exitCode := wsw.Execute(args, changes, status) + // We expect an error... + if exitCode == 0 { + err = errors.New("Should have exitCode != 0") + } + serverC <- err + }() + + checkErr := func(c chan error, txt string) { + select { + case err := <-c: + if err != nil { + t.Fatalf("%s: %v", txt, err) + } + case <-time.After(2 * time.Second): + t.Fatal("Test timed out") + } + } + checkErr(mockC, "windows.svc mock") + checkErr(serverC, "server behavior") + + wsw.server.Shutdown() +} + +func TestWinServiceLDMExit(t *testing.T) { + var ( + wsw = &winServiceWrapper{New(DefaultOptions())} + args = make([]string, 0) + // Size of 1 should be enough but don't want to block if the test fails. + serverC = make(chan error, 10) + mockC = make(chan error, 10) + changes = make(chan svc.ChangeRequest, 1) + status = make(chan svc.Status) + ) + // prepare mock expectations + wsm := &winSvcMock{status: status} + wsm.Expect(svc.StartPending) + // Expect that we will be running + wsm.Expect(svc.Running) + // Sending the LDM signal + changes <- svc.ChangeRequest{Cmd: svc.Cmd(ldmCode)} + // Expect to be stopping + wsm.Expect(svc.StopPending) + + go func() { + // Duration long enough for the test to complete, but not too long + // to wait for nothing. + mockC <- wsm.Listen(time.Second) + }() + + go func() { + var err error + if _, exitCode := wsw.Execute(args, changes, status); exitCode != 0 { + err = fmt.Errorf("exited with %v", exitCode) + } + serverC <- err + }() + + checkErr := func(c chan error, txt string) { + select { + case err := <-c: + if err != nil { + t.Fatalf("%s: %v", txt, err) + } + case <-time.After(2 * time.Second): + t.Fatal("Test timed out") + } + } + checkErr(mockC, "windows.svc mock") + checkErr(serverC, "server behavior") + + // At this point, we have proven already that the LDM signal did + // end the NATS server's Execute() loop because we received the + // svc.StopPending status. Still, we will verify that we get + // notified that the server has shutdown. + ch := make(chan struct{}, 1) + go func() { + wsw.server.WaitForShutdown() + ch <- struct{}{} + }() + select { + case <-ch: + // OK + case <-time.After(time.Second): + t.Fatal("Did not finish shutdown") + } +} + +// winSvcMock mocks part of the golang.org/x/sys/windows/svc +// execution stack, listening to svc.Status on its chan. +type winSvcMock struct { + status chan svc.Status + expectedSt []svc.State +} + +// Expect allows to prepare a winSvcMock to receive a specific type of StatusMessage +func (w *winSvcMock) Expect(st svc.State) { + w.expectedSt = append(w.expectedSt, st) +} + +// Listen is the mock's mainloop, expects messages to comply with previous Expect(). +func (w *winSvcMock) Listen(dur time.Duration) error { + timeout := time.NewTimer(dur) + defer timeout.Stop() + for _, state := range w.expectedSt { + select { + case status := <-w.status: + if status.State != state { + return fmt.Errorf("message to winsock: expected %v, got %v", state, status.State) + } + case <-timeout.C: + return errors.New("Mock timed out") + } + } + select { + case <-timeout.C: + return nil + case st := <-w.status: + return fmt.Errorf("extra message to winsock: got %v", st) + + } +} diff --git a/server/signal.go b/server/signal.go new file mode 100644 index 0000000..54e167d --- /dev/null +++ b/server/signal.go @@ -0,0 +1,211 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !windows && !wasm + +package server + +import ( + "errors" + "fmt" + "os" + "os/exec" + "os/signal" + "strconv" + "strings" + "syscall" +) + +var processName = "nats-server" + +// SetProcessName allows to change the expected name of the process. +func SetProcessName(name string) { + processName = name +} + +// Signal Handling +func (s *Server) handleSignals() { + if s.getOpts().NoSigs { + return + } + c := make(chan os.Signal, 1) + + signal.Notify(c, syscall.SIGINT, syscall.SIGTERM, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP) + + go func() { + for { + select { + case sig := <-c: + s.handleSignal(sig) + case <-s.quitCh: + return + } + } + }() +} + +func (s *Server) handleSignal(sig os.Signal) { + s.Noticef("Trapped %q signal", sig) + switch sig { + case syscall.SIGINT: + s.Shutdown() + s.WaitForShutdown() + os.Exit(0) + case syscall.SIGTERM: + // Shutdown unless graceful shutdown already in progress. + s.mu.Lock() + ldm := s.ldm + s.mu.Unlock() + + if !ldm { + s.Shutdown() + s.WaitForShutdown() + os.Exit(0) + } + case syscall.SIGUSR1: + // File log re-open for rotating file logs. + s.ReOpenLogFile() + case syscall.SIGUSR2: + go s.lameDuckMode() + case syscall.SIGHUP: + // Config reload. + if err := s.Reload(); err != nil { + s.Errorf("Failed to reload server configuration: %s", err) + } + } +} + +// ProcessSignal sends the given signal command to the given process. If pidStr +// is empty, this will send the signal to the single running instance of +// nats-server. If multiple instances are running, pidStr can be a globular +// expression ending with '*'. This returns an error if the given process is +// not running or the command is invalid. +func ProcessSignal(command Command, pidExpr string) error { + var ( + err error + errStr string + pids = make([]int, 1) + pidStr = strings.TrimSuffix(pidExpr, "*") + isGlob = strings.HasSuffix(pidExpr, "*") + ) + + // Validate input if given + if pidStr != "" { + if pids[0], err = strconv.Atoi(pidStr); err != nil { + return fmt.Errorf("invalid pid: %s", pidStr) + } + } + // Gather all PIDs unless the input is specific + if pidStr == "" || isGlob { + if pids, err = resolvePids(); err != nil { + return err + } + } + // Multiple instances are running and the input is not an expression + if len(pids) > 1 && !isGlob { + errStr = fmt.Sprintf("multiple %s processes running:", processName) + for _, p := range pids { + errStr += fmt.Sprintf("\n%d", p) + } + return errors.New(errStr) + } + // No instances are running + if len(pids) == 0 { + return fmt.Errorf("no %s processes running", processName) + } + + var signum syscall.Signal + if signum, err = CommandToSignal(command); err != nil { + return err + } + + for _, pid := range pids { + if _pidStr := strconv.Itoa(pid); _pidStr != pidStr && pidStr != "" { + if !isGlob || !strings.HasPrefix(_pidStr, pidStr) { + continue + } + } + if err = kill(pid, signum); err != nil { + errStr += fmt.Sprintf("\nsignal %q %d: %s", command, pid, err) + } + } + if errStr != "" { + return errors.New(errStr) + } + return nil +} + +// Translates a command to a signal number +func CommandToSignal(command Command) (syscall.Signal, error) { + switch command { + case CommandStop: + return syscall.SIGKILL, nil + case CommandQuit: + return syscall.SIGINT, nil + case CommandReopen: + return syscall.SIGUSR1, nil + case CommandReload: + return syscall.SIGHUP, nil + case commandLDMode: + return syscall.SIGUSR2, nil + case commandTerm: + return syscall.SIGTERM, nil + default: + return 0, fmt.Errorf("unknown signal %q", command) + } +} + +// resolvePids returns the pids for all running nats-server processes. +func resolvePids() ([]int, error) { + // If pgrep isn't available, this will just bail out and the user will be + // required to specify a pid. + output, err := pgrep() + if err != nil { + switch err.(type) { + case *exec.ExitError: + // ExitError indicates non-zero exit code, meaning no processes + // found. + break + default: + return nil, errors.New("unable to resolve pid, try providing one") + } + } + var ( + myPid = os.Getpid() + pidStrs = strings.Split(string(output), "\n") + pids = make([]int, 0, len(pidStrs)) + ) + for _, pidStr := range pidStrs { + if pidStr == "" { + continue + } + pid, err := strconv.Atoi(pidStr) + if err != nil { + return nil, errors.New("unable to resolve pid, try providing one") + } + // Ignore the current process. + if pid == myPid { + continue + } + pids = append(pids, pid) + } + return pids, nil +} + +var kill = func(pid int, signal syscall.Signal) error { + return syscall.Kill(pid, signal) +} + +var pgrep = func() ([]byte, error) { + return exec.Command("pgrep", processName).Output() +} diff --git a/server/signal_test.go b/server/signal_test.go new file mode 100644 index 0000000..04ead0e --- /dev/null +++ b/server/signal_test.go @@ -0,0 +1,515 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !windows + +package server + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/logger" + "github.com/nats-io/nats.go" +) + +func TestSignalToReOpenLogFile(t *testing.T) { + logFile := filepath.Join(t.TempDir(), "test.log") + opts := &Options{ + Host: "127.0.0.1", + Port: -1, + NoSigs: false, + LogFile: logFile, + } + s := RunServer(opts) + defer s.SetLogger(nil, false, false) + defer s.Shutdown() + + // Set the file log + fileLog := logger.NewFileLogger(s.opts.LogFile, s.opts.Logtime, s.opts.Debug, s.opts.Trace, true, logger.LogUTC(s.opts.LogtimeUTC)) + s.SetLogger(fileLog, false, false) + + // Add a trace + expectedStr := "This is a Notice" + s.Noticef(expectedStr) + buf, err := os.ReadFile(logFile) + if err != nil { + t.Fatalf("Error reading file: %v", err) + } + if !strings.Contains(string(buf), expectedStr) { + t.Fatalf("Expected log to contain %q, got %q", expectedStr, string(buf)) + } + // Rename the file + if err := os.Rename(logFile, logFile+".bak"); err != nil { + t.Fatalf("Unable to rename file: %v", err) + } + // This should cause file to be reopened. + syscall.Kill(syscall.Getpid(), syscall.SIGUSR1) + // Wait a bit for action to be performed + time.Sleep(500 * time.Millisecond) + buf, err = os.ReadFile(logFile) + if err != nil { + t.Fatalf("Error reading file: %v", err) + } + expectedStr = "File log re-opened" + if !strings.Contains(string(buf), expectedStr) { + t.Fatalf("Expected log to contain %q, got %q", expectedStr, string(buf)) + } +} + +func TestSignalToReloadConfig(t *testing.T) { + tmpl := ` + listen: 127.0.0.1:-1 + accounts: { + A: { users: [ { user: %s, password: foo } ] } + } + ` + conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, "foo"))) + opts, err := ProcessConfigFile(conf) + if err != nil { + t.Fatalf("Error processing config file: %v", err) + } + opts.NoLog = true + s := RunServer(opts) + defer s.Shutdown() + + // Check that the reload time does not change when there are no changes. + loaded := s.ConfigTime() + time.Sleep(250 * time.Millisecond) + syscall.Kill(syscall.Getpid(), syscall.SIGHUP) + time.Sleep(250 * time.Millisecond) + if reloaded := s.ConfigTime(); reloaded.Equal(loaded) { + t.Fatalf("ConfigTime is incorrect.\nexpected reload time to change: %s\ngot: %s", loaded, reloaded) + } + + // Repeat test to make sure that server services signals more than once... + for i := 0; i < 2; i++ { + loaded := s.ConfigTime() + user := fmt.Sprintf("foo:%d", i) + if err := os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, user)), 0666); err != nil { + t.Fatalf("Error creating config file: %v", err) + } + // Wait a bit to ensure ConfigTime changes. + time.Sleep(5 * time.Millisecond) + + // This should cause config to be reloaded. + syscall.Kill(syscall.Getpid(), syscall.SIGHUP) + // Wait a bit for action to be performed + time.Sleep(500 * time.Millisecond) + + if reloaded := s.ConfigTime(); !reloaded.After(loaded) { + t.Fatalf("ConfigTime is incorrect.\nexpected greater than: %s\ngot: %s", loaded, reloaded) + } + } +} + +func TestProcessSignalNoProcesses(t *testing.T) { + pgrepBefore := pgrep + pgrep = func() ([]byte, error) { + return nil, &exec.ExitError{} + } + defer func() { + pgrep = pgrepBefore + }() + + err := ProcessSignal(CommandStop, "") + if err == nil { + t.Fatal("Expected error") + } + expectedStr := "no nats-server processes running" + if err.Error() != expectedStr { + t.Fatalf("Error is incorrect.\nexpected: %s\ngot: %s", expectedStr, err.Error()) + } +} + +func TestProcessSignalMultipleProcesses(t *testing.T) { + pid := os.Getpid() + pgrepBefore := pgrep + pgrep = func() ([]byte, error) { + return []byte(fmt.Sprintf("123\n456\n%d\n", pid)), nil + } + defer func() { + pgrep = pgrepBefore + }() + + err := ProcessSignal(CommandStop, "") + if err == nil { + t.Fatal("Expected error") + } + expectedStr := "multiple nats-server processes running:\n123\n456" + if err.Error() != expectedStr { + t.Fatalf("Error is incorrect.\nexpected: %s\ngot: %s", expectedStr, err.Error()) + } +} + +func TestProcessSignalMultipleProcessesGlob(t *testing.T) { + pid := os.Getpid() + pgrepBefore := pgrep + pgrep = func() ([]byte, error) { + return []byte(fmt.Sprintf("123\n456\n%d\n", pid)), nil + } + defer func() { + pgrep = pgrepBefore + }() + + err := ProcessSignal(CommandStop, "*") + if err == nil { + t.Fatal("Expected error") + } + lines := strings.Split(err.Error(), "\n") + require_Len(t, len(lines), 3) + require_Equal(t, lines[0], "") // Empty line comes first + require_True(t, strings.HasPrefix(lines[1], "signal \"stop\" 123:")) + require_True(t, strings.HasPrefix(lines[2], "signal \"stop\" 456:")) +} + +func TestProcessSignalMultipleProcessesGlobPartial(t *testing.T) { + pid := os.Getpid() + pgrepBefore := pgrep + pgrep = func() ([]byte, error) { + return []byte(fmt.Sprintf("123\n124\n456\n%d\n", pid)), nil + } + defer func() { + pgrep = pgrepBefore + }() + + err := ProcessSignal(CommandStop, "12*") + if err == nil { + t.Fatal("Expected error") + } + lines := strings.Split(err.Error(), "\n") + require_Len(t, len(lines), 3) + require_Equal(t, lines[0], "") // Empty line comes first + require_True(t, strings.HasPrefix(lines[1], "signal \"stop\" 123:")) + require_True(t, strings.HasPrefix(lines[2], "signal \"stop\" 124:")) +} + +func TestProcessSignalPgrepError(t *testing.T) { + pgrepBefore := pgrep + pgrep = func() ([]byte, error) { + return nil, errors.New("error") + } + defer func() { + pgrep = pgrepBefore + }() + + err := ProcessSignal(CommandStop, "") + if err == nil { + t.Fatal("Expected error") + } + expectedStr := "unable to resolve pid, try providing one" + if err.Error() != expectedStr { + t.Fatalf("Error is incorrect.\nexpected: %s\ngot: %s", expectedStr, err.Error()) + } +} + +func TestProcessSignalPgrepMangled(t *testing.T) { + pgrepBefore := pgrep + pgrep = func() ([]byte, error) { + return []byte("12x"), nil + } + defer func() { + pgrep = pgrepBefore + }() + + err := ProcessSignal(CommandStop, "") + if err == nil { + t.Fatal("Expected error") + } + expectedStr := "unable to resolve pid, try providing one" + if err.Error() != expectedStr { + t.Fatalf("Error is incorrect.\nexpected: %s\ngot: %s", expectedStr, err.Error()) + } +} + +func TestProcessSignalResolveSingleProcess(t *testing.T) { + pid := os.Getpid() + pgrepBefore := pgrep + pgrep = func() ([]byte, error) { + return []byte(fmt.Sprintf("123\n%d\n", pid)), nil + } + defer func() { + pgrep = pgrepBefore + }() + killBefore := kill + called := false + kill = func(pid int, signal syscall.Signal) error { + called = true + if pid != 123 { + t.Fatalf("pid is incorrect.\nexpected: 123\ngot: %d", pid) + } + if signal != syscall.SIGKILL { + t.Fatalf("signal is incorrect.\nexpected: killed\ngot: %v", signal) + } + return nil + } + defer func() { + kill = killBefore + }() + + if err := ProcessSignal(CommandStop, ""); err != nil { + t.Fatalf("ProcessSignal failed: %v", err) + } + + if !called { + t.Fatal("Expected kill to be called") + } +} + +func TestProcessSignalInvalidCommand(t *testing.T) { + err := ProcessSignal(Command("invalid"), "123") + if err == nil { + t.Fatal("Expected error") + } + expectedStr := "unknown signal \"invalid\"" + if err.Error() != expectedStr { + t.Fatalf("Error is incorrect.\nexpected: %s\ngot: %s", expectedStr, err.Error()) + } +} + +func TestProcessSignalInvalidPid(t *testing.T) { + err := ProcessSignal(CommandStop, "abc") + if err == nil { + t.Fatal("Expected error") + } + expectedStr := "invalid pid: abc" + if err.Error() != expectedStr { + t.Fatalf("Error is incorrect.\nexpected: %s\ngot: %s", expectedStr, err.Error()) + } +} + +func TestProcessSignalQuitProcess(t *testing.T) { + killBefore := kill + called := false + kill = func(pid int, signal syscall.Signal) error { + called = true + if pid != 123 { + t.Fatalf("pid is incorrect.\nexpected: 123\ngot: %d", pid) + } + if signal != syscall.SIGINT { + t.Fatalf("signal is incorrect.\nexpected: interrupt\ngot: %v", signal) + } + return nil + } + defer func() { + kill = killBefore + }() + + if err := ProcessSignal(CommandQuit, "123"); err != nil { + t.Fatalf("ProcessSignal failed: %v", err) + } + + if !called { + t.Fatal("Expected kill to be called") + } +} + +func TestProcessSignalTermProcess(t *testing.T) { + killBefore := kill + called := false + kill = func(pid int, signal syscall.Signal) error { + called = true + if pid != 123 { + t.Fatalf("pid is incorrect.\nexpected: 123\ngot: %d", pid) + } + if signal != syscall.SIGTERM { + t.Fatalf("signal is incorrect.\nexpected: interrupt\ngot: %v", signal) + } + return nil + } + defer func() { + kill = killBefore + }() + + if err := ProcessSignal(commandTerm, "123"); err != nil { + t.Fatalf("ProcessSignal failed: %v", err) + } + + if !called { + t.Fatal("Expected kill to be called") + } +} + +func TestProcessSignalReopenProcess(t *testing.T) { + killBefore := kill + called := false + kill = func(pid int, signal syscall.Signal) error { + called = true + if pid != 123 { + t.Fatalf("pid is incorrect.\nexpected: 123\ngot: %d", pid) + } + if signal != syscall.SIGUSR1 { + t.Fatalf("signal is incorrect.\nexpected: user defined signal 1\ngot: %v", signal) + } + return nil + } + defer func() { + kill = killBefore + }() + + if err := ProcessSignal(CommandReopen, "123"); err != nil { + t.Fatalf("ProcessSignal failed: %v", err) + } + + if !called { + t.Fatal("Expected kill to be called") + } +} + +func TestProcessSignalReloadProcess(t *testing.T) { + killBefore := kill + called := false + kill = func(pid int, signal syscall.Signal) error { + called = true + if pid != 123 { + t.Fatalf("pid is incorrect.\nexpected: 123\ngot: %d", pid) + } + if signal != syscall.SIGHUP { + t.Fatalf("signal is incorrect.\nexpected: hangup\ngot: %v", signal) + } + return nil + } + defer func() { + kill = killBefore + }() + + if err := ProcessSignal(CommandReload, "123"); err != nil { + t.Fatalf("ProcessSignal failed: %v", err) + } + + if !called { + t.Fatal("Expected kill to be called") + } +} + +func TestProcessSignalLameDuckMode(t *testing.T) { + killBefore := kill + called := false + kill = func(pid int, signal syscall.Signal) error { + called = true + if pid != 123 { + t.Fatalf("pid is incorrect.\nexpected: 123\ngot: %d", pid) + } + if signal != syscall.SIGUSR2 { + t.Fatalf("signal is incorrect.\nexpected: sigusr2\ngot: %v", signal) + } + return nil + } + defer func() { + kill = killBefore + }() + + if err := ProcessSignal(commandLDMode, "123"); err != nil { + t.Fatalf("ProcessSignal failed: %v", err) + } + + if !called { + t.Fatal("Expected kill to be called") + } +} + +func TestProcessSignalTermDuringLameDuckMode(t *testing.T) { + opts := &Options{ + Host: "127.0.0.1", + Port: -1, + NoSigs: true, + NoLog: true, + LameDuckDuration: 2 * time.Second, + LameDuckGracePeriod: 1 * time.Second, + } + s := RunServer(opts) + defer s.Shutdown() + + // Create single NATS Connection which will cause the server + // to delay the shutdown. + doneCh := make(chan struct{}) + nc, err := nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port), + nats.DisconnectHandler(func(*nats.Conn) { + close(doneCh) + }), + ) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // Trigger lame duck based shutdown. + go s.lameDuckMode() + + // Wait for client to be disconnected. + select { + case <-doneCh: + break + case <-time.After(3 * time.Second): + t.Fatalf("Timed out waiting for client to disconnect") + } + + // Termination signal should not cause server to shutdown + // while in lame duck mode already. + s.handleSignal(syscall.SIGTERM) + + // Wait for server shutdown due to lame duck shutdown. + timeoutCh := make(chan error) + timer := time.AfterFunc(3*time.Second, func() { + timeoutCh <- errors.New("Timed out waiting for server shutdown") + }) + for range time.NewTicker(1 * time.Millisecond).C { + select { + case err := <-timeoutCh: + t.Fatal(err) + default: + } + + if !s.isRunning() { + timer.Stop() + break + } + } +} + +func TestSignalInterruptHasSuccessfulExit(t *testing.T) { + if os.Getenv("IN_TEST") == "1" { + s := RunServer(&Options{}) + defer s.Shutdown() + require_NoError(t, syscall.Kill(syscall.Getpid(), syscall.SIGINT)) + s.WaitForShutdown() + return + } + // To check for successful/0 exit code, need execute as separate process. + cmd := exec.Command(os.Args[0], "-test.run=TestSignalInterruptHasSuccessfulExit") + cmd.Env = append(os.Environ(), "IN_TEST=1") + err := cmd.Run() + require_NoError(t, err) +} + +func TestSignalTermHasSuccessfulExit(t *testing.T) { + if os.Getenv("IN_TEST") == "1" { + s := RunServer(&Options{}) + defer s.Shutdown() + require_NoError(t, syscall.Kill(syscall.Getpid(), syscall.SIGTERM)) + s.WaitForShutdown() + return + } + // To check for successful/0 exit code, need execute as separate process. + cmd := exec.Command(os.Args[0], "-test.run=TestSignalTermHasSuccessfulExit") + cmd.Env = append(os.Environ(), "IN_TEST=1") + err := cmd.Run() + require_NoError(t, err) +} diff --git a/server/signal_wasm.go b/server/signal_wasm.go new file mode 100644 index 0000000..38e072a --- /dev/null +++ b/server/signal_wasm.go @@ -0,0 +1,30 @@ +// Copyright 2022-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build wasm + +package server + +import "os" + +func (s *Server) handleSignals() { + +} + +func (s *Server) handleSignal(sig os.Signal) { + +} + +func ProcessSignal(command Command, service string) error { + return nil +} diff --git a/server/signal_windows.go b/server/signal_windows.go new file mode 100644 index 0000000..1ecabed --- /dev/null +++ b/server/signal_windows.go @@ -0,0 +1,110 @@ +// Copyright 2012-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "golang.org/x/sys/windows/svc" + "golang.org/x/sys/windows/svc/mgr" +) + +// Signal Handling +func (s *Server) handleSignals() { + if s.getOpts().NoSigs { + return + } + c := make(chan os.Signal, 1) + + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + + go func() { + for { + select { + case sig := <-c: + s.Debugf("Trapped %q signal", sig) + s.Shutdown() + os.Exit(0) + case <-s.quitCh: + return + } + } + }() +} + +// ProcessSignal sends the given signal command to the running nats-server service. +// If service is empty, this signals the "nats-server" service. This returns an +// error is the given service is not running or the command is invalid. +func ProcessSignal(command Command, service string) error { + if service == "" { + service = serviceName + } + + m, err := mgr.Connect() + if err != nil { + return err + } + defer m.Disconnect() + + s, err := m.OpenService(service) + if err != nil { + return fmt.Errorf("could not access service: %v", err) + } + defer s.Close() + + var ( + cmd svc.Cmd + to svc.State + ) + + switch command { + case CommandStop, CommandQuit: + cmd = svc.Stop + to = svc.Stopped + case CommandReopen: + cmd = reopenLogCmd + to = svc.Running + case CommandReload: + cmd = svc.ParamChange + to = svc.Running + case commandLDMode: + cmd = ldmCmd + to = svc.Running + default: + return fmt.Errorf("unknown signal %q", command) + } + + status, err := s.Control(cmd) + if err != nil { + return fmt.Errorf("could not send control=%d: %v", cmd, err) + } + + timeout := time.Now().Add(10 * time.Second) + for status.State != to { + if timeout.Before(time.Now()) { + return fmt.Errorf("timeout waiting for service to go to state=%d", to) + } + time.Sleep(300 * time.Millisecond) + status, err = s.Query() + if err != nil { + return fmt.Errorf("could not retrieve service status: %v", err) + } + } + + return nil +} diff --git a/server/split_test.go b/server/split_test.go new file mode 100644 index 0000000..ac91da6 --- /dev/null +++ b/server/split_test.go @@ -0,0 +1,570 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "net" + "testing" +) + +func TestSplitBufferSubOp(t *testing.T) { + cli, trash := net.Pipe() + defer cli.Close() + defer trash.Close() + + s := &Server{gacc: NewAccount(globalAccountName)} + if err := s.newGateway(DefaultOptions()); err != nil { + t.Fatalf("Error creating gateways: %v", err) + } + s.registerAccount(s.gacc) + c := &client{srv: s, acc: s.gacc, msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription), nc: cli} + + subop := []byte("SUB foo 1\r\n") + subop1 := subop[:6] + subop2 := subop[6:] + + if err := c.parse(subop1); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != SUB_ARG { + t.Fatalf("Expected SUB_ARG state vs %d\n", c.state) + } + if err := c.parse(subop2); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != OP_START { + t.Fatalf("Expected OP_START state vs %d\n", c.state) + } + r := s.gacc.sl.Match("foo") + if r == nil || len(r.psubs) != 1 { + t.Fatalf("Did not match subscription properly: %+v\n", r) + } + sub := r.psubs[0] + if !bytes.Equal(sub.subject, []byte("foo")) { + t.Fatalf("Subject did not match expected 'foo' : '%s'\n", sub.subject) + } + if !bytes.Equal(sub.sid, []byte("1")) { + t.Fatalf("Sid did not match expected '1' : '%s'\n", sub.sid) + } + if sub.queue != nil { + t.Fatalf("Received a non-nil queue: '%s'\n", sub.queue) + } +} + +func TestSplitBufferUnsubOp(t *testing.T) { + s := &Server{gacc: NewAccount(globalAccountName), gateway: &srvGateway{}} + s.registerAccount(s.gacc) + c := &client{srv: s, acc: s.gacc, msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)} + + subop := []byte("SUB foo 1024\r\n") + if err := c.parse(subop); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != OP_START { + t.Fatalf("Expected OP_START state vs %d\n", c.state) + } + + unsubop := []byte("UNSUB 1024\r\n") + unsubop1 := unsubop[:8] + unsubop2 := unsubop[8:] + + if err := c.parse(unsubop1); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != UNSUB_ARG { + t.Fatalf("Expected UNSUB_ARG state vs %d\n", c.state) + } + if err := c.parse(unsubop2); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != OP_START { + t.Fatalf("Expected OP_START state vs %d\n", c.state) + } + r := s.gacc.sl.Match("foo") + if r != nil && len(r.psubs) != 0 { + t.Fatalf("Should be no subscriptions in results: %+v\n", r) + } +} + +func TestSplitBufferPubOp(t *testing.T) { + c := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)} + pub := []byte("PUB foo.bar INBOX.22 11\r\nhello world\r") + pub1 := pub[:2] + pub2 := pub[2:9] + pub3 := pub[9:15] + pub4 := pub[15:22] + pub5 := pub[22:25] + pub6 := pub[25:33] + pub7 := pub[33:] + + if err := c.parse(pub1); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != OP_PU { + t.Fatalf("Expected OP_PU state vs %d\n", c.state) + } + if err := c.parse(pub2); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != PUB_ARG { + t.Fatalf("Expected OP_PU state vs %d\n", c.state) + } + if err := c.parse(pub3); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != PUB_ARG { + t.Fatalf("Expected OP_PU state vs %d\n", c.state) + } + if err := c.parse(pub4); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != PUB_ARG { + t.Fatalf("Expected PUB_ARG state vs %d\n", c.state) + } + if err := c.parse(pub5); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != MSG_PAYLOAD { + t.Fatalf("Expected MSG_PAYLOAD state vs %d\n", c.state) + } + + // Check c.pa + if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { + t.Fatalf("PUB arg subject incorrect: '%s'\n", c.pa.subject) + } + if !bytes.Equal(c.pa.reply, []byte("INBOX.22")) { + t.Fatalf("PUB arg reply subject incorrect: '%s'\n", c.pa.reply) + } + if c.pa.size != 11 { + t.Fatalf("PUB arg msg size incorrect: %d\n", c.pa.size) + } + if err := c.parse(pub6); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != MSG_PAYLOAD { + t.Fatalf("Expected MSG_PAYLOAD state vs %d\n", c.state) + } + if err := c.parse(pub7); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != MSG_END_N { + t.Fatalf("Expected MSG_END_N state vs %d\n", c.state) + } +} + +func TestSplitBufferPubOp2(t *testing.T) { + c := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)} + pub := []byte("PUB foo.bar INBOX.22 11\r\nhello world\r\n") + pub1 := pub[:30] + pub2 := pub[30:] + + if err := c.parse(pub1); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != MSG_PAYLOAD { + t.Fatalf("Expected MSG_PAYLOAD state vs %d\n", c.state) + } + if err := c.parse(pub2); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != OP_START { + t.Fatalf("Expected OP_START state vs %d\n", c.state) + } +} + +func TestSplitBufferPubOp3(t *testing.T) { + c := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)} + pubAll := []byte("PUB foo bar 11\r\nhello world\r\n") + pub := pubAll[:16] + + if err := c.parse(pub); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if !bytes.Equal(c.pa.subject, []byte("foo")) { + t.Fatalf("Unexpected subject: '%s' vs '%s'\n", c.pa.subject, "foo") + } + + // Simulate next read of network, make sure pub state is saved + // until msg payload has cleared. + copy(pubAll, "XXXXXXXXXXXXXXXX") + if !bytes.Equal(c.pa.subject, []byte("foo")) { + t.Fatalf("Unexpected subject: '%s' vs '%s'\n", c.pa.subject, "foo") + } + if !bytes.Equal(c.pa.reply, []byte("bar")) { + t.Fatalf("Unexpected reply: '%s' vs '%s'\n", c.pa.reply, "bar") + } + if !bytes.Equal(c.pa.szb, []byte("11")) { + t.Fatalf("Unexpected size bytes: '%s' vs '%s'\n", c.pa.szb, "11") + } +} + +func TestSplitBufferPubOp4(t *testing.T) { + c := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)} + pubAll := []byte("PUB foo 11\r\nhello world\r\n") + pub := pubAll[:12] + + if err := c.parse(pub); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if !bytes.Equal(c.pa.subject, []byte("foo")) { + t.Fatalf("Unexpected subject: '%s' vs '%s'\n", c.pa.subject, "foo") + } + + // Simulate next read of network, make sure pub state is saved + // until msg payload has cleared. + copy(pubAll, "XXXXXXXXXXXX") + if !bytes.Equal(c.pa.subject, []byte("foo")) { + t.Fatalf("Unexpected subject: '%s' vs '%s'\n", c.pa.subject, "foo") + } + if !bytes.Equal(c.pa.reply, []byte("")) { + t.Fatalf("Unexpected reply: '%s' vs '%s'\n", c.pa.reply, "") + } + if !bytes.Equal(c.pa.szb, []byte("11")) { + t.Fatalf("Unexpected size bytes: '%s' vs '%s'\n", c.pa.szb, "11") + } +} + +func TestSplitBufferPubOp5(t *testing.T) { + c := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)} + pubAll := []byte("PUB foo 11\r\nhello world\r\n") + + // Splits need to be on MSG_END_R now too, so make sure we check that. + // Split between \r and \n + pub := pubAll[:len(pubAll)-1] + + if err := c.parse(pub); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.msgBuf == nil { + t.Fatalf("msgBuf should not be nil!\n") + } + if !bytes.Equal(c.msgBuf, []byte("hello world\r")) { + t.Fatalf("c.msgBuf did not snaphot the msg") + } +} + +func TestSplitConnectArg(t *testing.T) { + c := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)} + connectAll := []byte("CONNECT {\"verbose\":false,\"tls_required\":false," + + "\"user\":\"test\",\"pedantic\":true,\"pass\":\"pass\"}\r\n") + + argJSON := connectAll[8:] + + c1 := connectAll[:5] + c2 := connectAll[5:22] + c3 := connectAll[22 : len(connectAll)-2] + c4 := connectAll[len(connectAll)-2:] + + if err := c.parse(c1); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.argBuf != nil { + t.Fatalf("Unexpected argBug placeholder.\n") + } + + if err := c.parse(c2); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.argBuf == nil { + t.Fatalf("Expected argBug to not be nil.\n") + } + if !bytes.Equal(c.argBuf, argJSON[:14]) { + t.Fatalf("argBuf not correct, received %q, wanted %q\n", argJSON[:14], c.argBuf) + } + + if err := c.parse(c3); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.argBuf == nil { + t.Fatalf("Expected argBug to not be nil.\n") + } + if !bytes.Equal(c.argBuf, argJSON[:len(argJSON)-2]) { + t.Fatalf("argBuf not correct, received %q, wanted %q\n", + argJSON[:len(argJSON)-2], c.argBuf) + } + + if err := c.parse(c4); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.argBuf != nil { + t.Fatalf("Unexpected argBuf placeholder.\n") + } +} + +func TestSplitDanglingArgBuf(t *testing.T) { + opts := defaultServerOptions + s := New(&opts) + c := &client{srv: s, acc: s.gacc, msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)} + + // We test to make sure we do not dangle any argBufs after processing + // since that could lead to performance issues. + + // SUB + subop := []byte("SUB foo 1\r\n") + c.parse(subop[:6]) + c.parse(subop[6:]) + if c.argBuf != nil { + t.Fatalf("Expected c.argBuf to be nil: %q\n", c.argBuf) + } + + // UNSUB + unsubop := []byte("UNSUB 1024\r\n") + c.parse(unsubop[:8]) + c.parse(unsubop[8:]) + if c.argBuf != nil { + t.Fatalf("Expected c.argBuf to be nil: %q\n", c.argBuf) + } + + // PUB + pubop := []byte("PUB foo.bar INBOX.22 11\r\nhello world\r\n") + c.parse(pubop[:22]) + c.parse(pubop[22:25]) + if c.argBuf == nil { + t.Fatal("Expected a non-nil argBuf!") + } + c.parse(pubop[25:]) + if c.argBuf != nil { + t.Fatalf("Expected c.argBuf to be nil: %q\n", c.argBuf) + } + + // MINUS_ERR + errop := []byte("-ERR Too Long\r\n") + c.parse(errop[:8]) + c.parse(errop[8:]) + if c.argBuf != nil { + t.Fatalf("Expected c.argBuf to be nil: %q\n", c.argBuf) + } + + // CONNECT_ARG + connop := []byte("CONNECT {\"verbose\":false,\"tls_required\":false," + + "\"user\":\"test\",\"pedantic\":true,\"pass\":\"pass\"}\r\n") + c.parse(connop[:22]) + c.parse(connop[22:]) + if c.argBuf != nil { + t.Fatalf("Expected c.argBuf to be nil: %q\n", c.argBuf) + } + + // INFO_ARG + infoop := []byte("INFO {\"server_id\":\"id\"}\r\n") + c.parse(infoop[:8]) + c.parse(infoop[8:]) + if c.argBuf != nil { + t.Fatalf("Expected c.argBuf to be nil: %q\n", c.argBuf) + } + + // MSG (the client has to be a ROUTE) + c = &client{msubs: -1, mpay: -1, subs: make(map[string]*subscription), kind: ROUTER, route: &route{}, mcl: MAX_CONTROL_LINE_SIZE} + msgop := []byte("RMSG $foo foo 5\r\nhello\r\n") + c.parse(msgop[:5]) + c.parse(msgop[5:10]) + if c.argBuf == nil { + t.Fatal("Expected a non-nil argBuf") + } + if string(c.argBuf) != "$foo " { + t.Fatalf("Expected argBuf to be \"$foo \", got %q", string(c.argBuf)) + } + c.parse(msgop[10:]) + if c.argBuf != nil { + t.Fatalf("Expected argBuf to be nil: %q", c.argBuf) + } + if c.msgBuf != nil { + t.Fatalf("Expected msgBuf to be nil: %q", c.msgBuf) + } + + c.state = OP_START + // Parse up-to somewhere in the middle of the payload. + // Verify that we have saved the MSG_ARG info + c.parse(msgop[:23]) + if c.argBuf == nil { + t.Fatal("Expected a non-nil argBuf") + } + if string(c.pa.account) != "$foo" { + t.Fatalf("Expected account to be \"$foo\", got %q", c.pa.account) + } + if string(c.pa.subject) != "foo" { + t.Fatalf("Expected subject to be \"foo\", got %q", c.pa.subject) + } + if string(c.pa.reply) != "" { + t.Fatalf("Expected reply to be \"\", got %q", c.pa.reply) + } + if c.pa.size != 5 { + t.Fatalf("Expected sid to 5, got %v", c.pa.size) + } + // msg buffer should be + if c.msgBuf == nil || string(c.msgBuf) != "hello\r" { + t.Fatalf("Expected msgBuf to be \"hello\r\", got %q", c.msgBuf) + } + c.parse(msgop[23:]) + // At the end, we should have cleaned-up both arg and msg buffers. + if c.argBuf != nil { + t.Fatalf("Expected argBuf to be nil: %q", c.argBuf) + } + if c.msgBuf != nil { + t.Fatalf("Expected msgBuf to be nil: %q", c.msgBuf) + } +} + +func TestSplitRoutedMsgArg(t *testing.T) { + _, c, _ := setupClient() + defer c.close() + // Allow parser to process RMSG + c.kind = ROUTER + c.route = &route{} + + b := make([]byte, 1024) + + copy(b, []byte("RMSG $G hello.world 6040\r\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")) + c.parse(b) + + copy(b, []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB\r\n")) + c.parse(b) + + wantAccount := "$G" + wantSubject := "hello.world" + wantSzb := "6040" + + if string(c.pa.account) != wantAccount { + t.Fatalf("Incorrect account: want %q, got %q", wantAccount, c.pa.account) + } + if string(c.pa.subject) != wantSubject { + t.Fatalf("Incorrect subject: want %q, got %q", wantSubject, c.pa.subject) + } + if string(c.pa.szb) != wantSzb { + t.Fatalf("Incorrect szb: want %q, got %q", wantSzb, c.pa.szb) + } +} + +func TestSplitBufferMsgOp(t *testing.T) { + c := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription), kind: ROUTER, route: &route{}} + msg := []byte("RMSG $G foo.bar _INBOX.22 11\r\nhello world\r") + msg1 := msg[:2] + msg2 := msg[2:9] + msg3 := msg[9:15] + msg4 := msg[15:22] + msg5 := msg[22:25] + msg6 := msg[25:37] + msg7 := msg[37:40] + msg8 := msg[40:] + + if err := c.parse(msg1); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != OP_M { + t.Fatalf("Expected OP_M state vs %d\n", c.state) + } + if err := c.parse(msg2); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != MSG_ARG { + t.Fatalf("Expected MSG_ARG state vs %d\n", c.state) + } + if err := c.parse(msg3); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != MSG_ARG { + t.Fatalf("Expected MSG_ARG state vs %d\n", c.state) + } + if err := c.parse(msg4); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != MSG_ARG { + t.Fatalf("Expected MSG_ARG state vs %d\n", c.state) + } + if err := c.parse(msg5); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != MSG_ARG { + t.Fatalf("Expected MSG_ARG state vs %d\n", c.state) + } + if err := c.parse(msg6); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != MSG_PAYLOAD { + t.Fatalf("Expected MSG_PAYLOAD state vs %d\n", c.state) + } + + // Check c.pa + if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { + t.Fatalf("MSG arg subject incorrect: '%s'\n", c.pa.subject) + } + if !bytes.Equal(c.pa.reply, []byte("_INBOX.22")) { + t.Fatalf("MSG arg reply subject incorrect: '%s'\n", c.pa.reply) + } + if c.pa.size != 11 { + t.Fatalf("MSG arg msg size incorrect: %d\n", c.pa.size) + } + if err := c.parse(msg7); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != MSG_PAYLOAD { + t.Fatalf("Expected MSG_PAYLOAD state vs %d\n", c.state) + } + if err := c.parse(msg8); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != MSG_END_N { + t.Fatalf("Expected MSG_END_N state vs %d\n", c.state) + } +} + +func TestSplitBufferLeafMsgArg(t *testing.T) { + c := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription), kind: LEAF} + msg := []byte("LMSG foo + bar baz 11\r\n") + if err := c.parse(msg); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != MSG_PAYLOAD { + t.Fatalf("Expected MSG_PAYLOAD state vs %d\n", c.state) + } + checkPA := func(t *testing.T) { + t.Helper() + if !bytes.Equal(c.pa.subject, []byte("foo")) { + t.Fatalf("Expected subject to be %q, got %q", "foo", c.pa.subject) + } + if !bytes.Equal(c.pa.reply, []byte("bar")) { + t.Fatalf("Expected reply to be %q, got %q", "bar", c.pa.reply) + } + if n := len(c.pa.queues); n != 1 { + t.Fatalf("Expected 1 queue, got %v", n) + } + if !bytes.Equal(c.pa.queues[0], []byte("baz")) { + t.Fatalf("Expected queues to be %q, got %q", "baz", c.pa.queues) + } + } + checkPA(t) + + // overwrite msg with payload + n := copy(msg, []byte("fffffffffff")) + if err := c.parse(msg[:n]); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != MSG_END_R { + t.Fatalf("Expected MSG_END_R state vs %d\n", c.state) + } + checkPA(t) + + // Finish processing + copy(msg, []byte("\r\n")) + if err := c.parse(msg[:2]); err != nil { + t.Fatalf("Unexpected parse error: %v\n", err) + } + if c.state != OP_START { + t.Fatalf("Expected OP_START state vs %d\n", c.state) + } + if c.pa.subject != nil || c.pa.reply != nil || c.pa.queues != nil || c.pa.size != 0 || + c.pa.szb != nil || c.pa.arg != nil { + t.Fatalf("parser state not cleaned-up properly: %+v", c.pa) + } +} diff --git a/server/store.go b/server/store.go new file mode 100644 index 0000000..03ab237 --- /dev/null +++ b/server/store.go @@ -0,0 +1,878 @@ +// Copyright 2019-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "iter" + "math/bits" + "os" + "strings" + "time" + "unsafe" + + "github.com/nats-io/nats-server/v2/server/avl" + "github.com/nats-io/nats-server/v2/server/gsl" +) + +// StorageType determines how messages are stored for retention. +type StorageType int + +const ( + // File specifies on disk, designated by the JetStream config StoreDir. + FileStorage = StorageType(22) + // MemoryStorage specifies in memory only. + MemoryStorage = StorageType(33) +) + +var ( + // ErrStoreClosed is returned when the store has been closed + ErrStoreClosed = errors.New("store is closed") + // ErrStoreMsgNotFound when message was not found but was expected to be. + ErrStoreMsgNotFound = errors.New("no message found") + // ErrStoreEOF is returned when message seq is greater than the last sequence. + ErrStoreEOF = errors.New("stream store EOF") + // ErrMaxMsgs is returned when we have discard new as a policy and we reached the message limit. + ErrMaxMsgs = errors.New("maximum messages exceeded") + // ErrMaxBytes is returned when we have discard new as a policy and we reached the bytes limit. + ErrMaxBytes = errors.New("maximum bytes exceeded") + // ErrMaxMsgsPerSubject is returned when we have discard new as a policy and we reached the message limit per subject. + ErrMaxMsgsPerSubject = errors.New("maximum messages per subject exceeded") + // ErrStoreSnapshotInProgress is returned when RemoveMsg or EraseMsg is called + // while a snapshot is in progress. + ErrStoreSnapshotInProgress = errors.New("snapshot in progress") + // ErrMsgTooLarge is returned when a message is considered too large. + ErrMsgTooLarge = errors.New("message too large") + // ErrStoreWrongType is for when you access the wrong storage type. + ErrStoreWrongType = errors.New("wrong storage type") + // ErrNoAckPolicy is returned when trying to update a consumer's acks with no ack policy. + ErrNoAckPolicy = errors.New("ack policy is none") + // ErrSequenceMismatch is returned when storing a raw message and the expected sequence is wrong. + ErrSequenceMismatch = errors.New("expected sequence does not match store") + // ErrCorruptStreamState + ErrCorruptStreamState = errors.New("stream state snapshot is corrupt") + // ErrTooManyResults + ErrTooManyResults = errors.New("too many matching results for request") + // ErrStoreOldUpdate is returned when a consumer update is older than the current state. + ErrStoreOldUpdate = errors.New("old update ignored") +) + +// StoreMsg is the stored message format for messages that are retained by the Store layer. +type StoreMsg struct { + subj string + hdr []byte + msg []byte + buf []byte + seq uint64 + ts int64 +} + +// Used to call back into the upper layers to report on changes in storage resources. +// For the cases where its a single message we will also supply sequence number and subject. +type StorageUpdateHandler func(msgs, bytes int64, seq uint64, subj string) + +// Used to call back into the upper layers to remove a message. +type StorageRemoveMsgHandler func(seq uint64) + +// Used to call back into the upper layers to process a JetStream message. +// Will propose the message if the stream is replicated. +type ProcessJetStreamMsgHandler func(*inMsg) + +type StreamStore interface { + StoreMsg(subject string, hdr, msg []byte, ttl int64) (uint64, int64, error) + StoreRawMsg(subject string, hdr, msg []byte, seq uint64, ts int64, ttl int64, discardNewCheck bool) error + SkipMsg(seq uint64) (uint64, error) + SkipMsgNoInterest(seq uint64) (uint64, error) + SkipMsgs(seq uint64, num uint64) error + FlushAllPending() error + LoadMsg(seq uint64, sm *StoreMsg) (*StoreMsg, error) + LoadNextMsg(filter string, wc bool, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error) + LoadNextMsgMulti(sl *gsl.SimpleSublist, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error) + LoadLastMsg(subject string, sm *StoreMsg) (*StoreMsg, error) + LoadPrevMsg(filter string, wc bool, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error) + LoadPrevMsgMulti(sl *gsl.SimpleSublist, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error) + RemoveMsg(seq uint64) (bool, error) + EraseMsg(seq uint64) (bool, error) + Purge() (uint64, error) + PurgeEx(subject string, seq, keep uint64) (uint64, error) + Compact(seq uint64) (uint64, error) + Truncate(seq uint64) error + GetSeqFromTime(t time.Time) uint64 + FilteredState(seq uint64, subject string) (SimpleState, error) + SubjectsState(filterSubject string) map[string]SimpleState + SubjectsTotals(filterSubject string) map[string]uint64 + SourcesState() map[string]StreamSourceState + AllLastSeqs() ([]uint64, error) + MultiLastSeqs(filters []string, maxSeq uint64, maxAllowed int) ([]uint64, error) + MultiLastMsgs(filters []string, minSeq, maxSeq uint64, maxAllowed int, cb func(sm *StoreMsg, np uint64) bool) (uint64, uint64, error) + SubjectForSeq(seq uint64) (string, error) + NumPending(sseq uint64, filter string, lastPerSubject bool) (total, validThrough uint64, err error) + NumPendingMulti(sseq uint64, sl *gsl.SimpleSublist, lastPerSubject bool) (total, validThrough uint64, err error) + State() StreamState + FastState(*StreamState) + EncodedStreamState(failed uint64, withSources bool) (enc []byte, err error) + ApplySourcesState(sources map[string]StreamSourceState) + SyncDeleted(dbs DeleteBlocks) error + Type() StorageType + RegisterStorageUpdates(StorageUpdateHandler) + RegisterStorageRemoveMsg(StorageRemoveMsgHandler) + RegisterProcessJetStreamMsg(ProcessJetStreamMsgHandler) + UpdateConfig(cfg *StreamConfig) error + Ready() // Only needed if store started in recovering mode. + Delete(inline bool) error + Stop() error + ConsumerStore(name string, created time.Time, cfg *ConsumerConfig) (ConsumerStore, error) + AddConsumer(o ConsumerStore) error + RemoveConsumer(o ConsumerStore) error + Consumers() iter.Seq[ConsumerStore] + Snapshot(deadline time.Duration, includeConsumers, checkMsgs bool) (*SnapshotResult, error) + Utilization() (total, reported uint64, err error) + ResetState() +} + +// RetentionPolicy determines how messages in a set are retained. +type RetentionPolicy int + +const ( + // LimitsPolicy (default) means that messages are retained until any given limit is reached. + // This could be one of MaxMsgs, MaxBytes, or MaxAge. + LimitsPolicy RetentionPolicy = iota + // InterestPolicy specifies that when all known consumers have acknowledged a message it can be removed. + InterestPolicy + // WorkQueuePolicy specifies that when the first worker or subscriber acknowledges the message it can be removed. + WorkQueuePolicy +) + +// Discard Policy determines how we proceed when limits of messages or bytes are hit. The default, DicscardOld will +// remove older messages. DiscardNew will fail to store the new message. +type DiscardPolicy int + +const ( + // DiscardOld will remove older messages to return to the limits. + DiscardOld = iota + // DiscardNew will error on a StoreMsg call + DiscardNew +) + +// StreamState is information about the given stream. +type StreamState struct { + Msgs uint64 `json:"messages"` + Bytes uint64 `json:"bytes"` + FirstSeq uint64 `json:"first_seq"` + FirstTime time.Time `json:"first_ts"` + LastSeq uint64 `json:"last_seq"` + LastTime time.Time `json:"last_ts"` + NumSubjects int `json:"num_subjects,omitempty"` + Subjects map[string]uint64 `json:"subjects,omitempty"` + NumDeleted int `json:"num_deleted,omitempty"` + Deleted []uint64 `json:"deleted,omitempty"` + Lost *LostStreamData `json:"lost,omitempty"` + Consumers int `json:"consumer_count"` +} + +// SimpleState for filtered subject specific state. +type SimpleState struct { + Msgs uint64 `json:"messages"` + First uint64 `json:"first_seq"` + Last uint64 `json:"last_seq"` + + // Internal usage for when the first needs to be updated before use. + firstNeedsUpdate bool + // Internal usage for when the last needs to be updated before use. + lastNeedsUpdate bool +} + +// LostStreamData indicates msgs that have been lost. +type LostStreamData struct { + Msgs []uint64 `json:"msgs"` + Bytes uint64 `json:"bytes"` +} + +// SnapshotResult contains information about the snapshot. +type SnapshotResult struct { + Reader io.ReadCloser + State StreamState + errCh chan error +} + +const ( + // Magic is used to identify stream state encodings. + streamStateMagic = uint8(42) + // Version + streamStateVersion = uint8(1) + // Version that additionally carries the stream's sourcing state. Only emitted + // when explicitly enabled, since a server that only understands the previous + // version rejects the encoding outright. + streamStateVersionSources = uint8(2) + // Magic / Identifier for run length encodings. + runLengthMagic = uint8(33) + // Magic / Identifier for AVL seqsets. + seqSetMagic = uint8(22) +) + +// Interface for DeleteBlock. +// These will be of three types: +// 1. AVL seqsets. +// 2. Run length encoding of a deleted range. +// 3. Legacy []uint64 +type DeleteBlock interface { + State() (first, last, num uint64) + Range(f func(uint64) bool) +} + +type DeleteBlocks []DeleteBlock + +// StreamReplicatedState represents what is encoded in a binary stream snapshot used +// for stream replication in an NRG. +type StreamReplicatedState struct { + Msgs uint64 + Bytes uint64 + FirstSeq uint64 + LastSeq uint64 + Failed uint64 + Deleted DeleteBlocks + // Sources is the stream's sourcing state, only present in encodings of + // streamStateVersionSources and above. It can't be derived from the messages + // alone, since it outlives the messages it was collected from. + Sources map[string]StreamSourceState +} + +// Determine if this is an encoded stream state. +func IsEncodedStreamState(buf []byte) bool { + return len(buf) >= hdrLen && buf[0] == streamStateMagic && + (buf[1] == streamStateVersion || buf[1] == streamStateVersionSources) +} + +var ErrBadStreamStateEncoding = errors.New("bad stream state encoding") + +func DecodeStreamState(buf []byte) (*StreamReplicatedState, error) { + ss := &StreamReplicatedState{} + if len(buf) < hdrLen || buf[0] != streamStateMagic || + (buf[1] != streamStateVersion && buf[1] != streamStateVersionSources) { + return nil, ErrBadStreamStateEncoding + } + withSources := buf[1] == streamStateVersionSources + var bi = hdrLen + + readU64 := func() uint64 { + if bi < 0 || bi >= len(buf) { + bi = -1 + return 0 + } + num, n := binary.Uvarint(buf[bi:]) + if n <= 0 { + bi = -1 + return 0 + } + bi += n + return num + } + + parserFailed := func() bool { + return bi < 0 + } + + ss.Msgs = readU64() + ss.Bytes = readU64() + ss.FirstSeq = readU64() + ss.LastSeq = readU64() + ss.Failed = readU64() + + if parserFailed() { + return nil, ErrCorruptStreamState + } + + // Sources come before the deleted blocks, since those are read up to the end + // of the buffer and can't be followed by anything else. + if withSources { + numSources := readU64() + if parserFailed() { + return nil, ErrCorruptStreamState + } + // Same entry layout as the on-disk sources index, all lengths as varints + // since both strings are short. + readStr := func() (string, bool) { + n := readU64() + if parserFailed() || uint64(len(buf)-bi) < n { + bi = -1 + return _EMPTY_, false + } + s := string(buf[bi : bi+int(n)]) + bi += int(n) + return s, true + } + for i := uint64(0); i < numSources; i++ { + name, ok := readStr() + if !ok { + return nil, ErrCorruptStreamState + } + seq := readU64() + ident, ok := readStr() + if !ok || parserFailed() { + return nil, ErrCorruptStreamState + } + if ss.Sources == nil { + ss.Sources = make(map[string]StreamSourceState, numSources) + } + ss.Sources[name] = StreamSourceState{Seq: seq, Ident: ident} + } + } + + if numDeleted := readU64(); numDeleted > 0 { + // If we have some deleted blocks. + for l := len(buf); l > bi; { + switch buf[bi] { + case seqSetMagic: + dmap, n, err := avl.Decode(buf[bi:]) + if err != nil { + return nil, ErrCorruptStreamState + } + bi += n + ss.Deleted = append(ss.Deleted, dmap) + case runLengthMagic: + bi++ + var rl DeleteRange + rl.First = readU64() + rl.Num = readU64() + if parserFailed() { + return nil, ErrCorruptStreamState + } + ss.Deleted = append(ss.Deleted, &rl) + default: + return nil, ErrCorruptStreamState + } + } + } + + return ss, nil +} + +// uvarintLen returns the number of bytes binary.PutUvarint/binary.AppendUvarint +// write for v: ceil(bits/7), with v=0 taking one byte. +func uvarintLen(v uint64) int { + return (bits.Len64(v|1) + 6) / 7 +} + +// runLengthEncodeLen returns the encoded size of a run-length delete record, +// exactly matching what appendRunLength writes. +func runLengthEncodeLen(first, num uint64) int { + return 1 + uvarintLen(first) + uvarintLen(num) +} + +// appendRunLength appends a run-length encoded delete record for num +// deleted sequences starting at first. +func appendRunLength(b []byte, first, num uint64) []byte { + b = append(b, runLengthMagic) + b = binary.AppendUvarint(b, first) + b = binary.AppendUvarint(b, num) + return b +} + +// DeleteRange is a run length encoded delete range. +type DeleteRange struct { + First uint64 + Num uint64 +} + +func (dr *DeleteRange) State() (first, last, num uint64) { + deletesAfterFirst := dr.Num + if deletesAfterFirst > 0 { + deletesAfterFirst-- + } + return dr.First, dr.First + deletesAfterFirst, dr.Num +} + +// Range will range over all the deleted sequences represented by this block. +func (dr *DeleteRange) Range(f func(uint64) bool) { + for seq := dr.First; seq < dr.First+dr.Num; seq++ { + if !f(seq) { + return + } + } +} + +// Legacy []uint64 +type DeleteSlice []uint64 + +func (ds DeleteSlice) State() (first, last, num uint64) { + if len(ds) == 0 { + return 0, 0, 0 + } + return ds[0], ds[len(ds)-1], uint64(len(ds)) +} + +// Range will range over all the deleted sequences represented by this []uint64. +func (ds DeleteSlice) Range(f func(uint64) bool) { + for _, seq := range ds { + if !f(seq) { + return + } + } +} + +func (dbs DeleteBlocks) NumDeleted() (total uint64) { + for _, db := range dbs { + _, _, num := db.State() + total += num + } + return total +} + +// ConsumerStore stores state on consumers for streams. +type ConsumerStore interface { + SetStarting(sseq uint64) error + UpdateStarting(sseq uint64) + Reset(sseq uint64) error + HasState() bool + UpdateDelivered(dseq, sseq, dc uint64, ts int64) error + UpdateAcks(dseq, sseq uint64) error + RemoveRedeliveredBelow(seq uint64) + GetConfig() *ConsumerConfig + UpdateConfig(cfg *ConsumerConfig) error + Update(*ConsumerState) error + ForceUpdate(*ConsumerState) error + State() (*ConsumerState, error) + BorrowState() (*ConsumerState, error) + EncodedState() ([]byte, error) + Type() StorageType + Stop() error + Delete() error + StreamDelete() error +} + +// SequencePair has both the consumer and the stream sequence. They point to same message. +type SequencePair struct { + Consumer uint64 `json:"consumer_seq"` + Stream uint64 `json:"stream_seq"` +} + +// ConsumerState represents a stored state for a consumer. +type ConsumerState struct { + // Delivered keeps track of last delivered sequence numbers for both the stream and the consumer. + Delivered SequencePair `json:"delivered"` + // AckFloor keeps track of the ack floors for both the stream and the consumer. + AckFloor SequencePair `json:"ack_floor"` + // These are both in stream sequence context. + // Pending is for all messages pending and the timestamp for the delivered time. + // This will only be present when the AckPolicy is ExplicitAck. + Pending map[uint64]*Pending `json:"pending,omitempty"` + // This is for messages that have been redelivered, so count > 1. + Redelivered map[uint64]uint64 `json:"redelivered,omitempty"` +} + +// Encode consumer state. +func encodeConsumerState(state *ConsumerState) []byte { + var hdr [seqsHdrSize]byte + var buf []byte + + maxSize := seqsHdrSize + if lp := len(state.Pending); lp > 0 { + maxSize += lp*(3*binary.MaxVarintLen64) + binary.MaxVarintLen64 + } + if lr := len(state.Redelivered); lr > 0 { + maxSize += lr*(2*binary.MaxVarintLen64) + binary.MaxVarintLen64 + } + if maxSize == seqsHdrSize { + buf = hdr[:seqsHdrSize] + } else { + buf = make([]byte, maxSize) + } + + // Write header + buf[0] = magic + buf[1] = 2 + + n := hdrLen + n += binary.PutUvarint(buf[n:], state.AckFloor.Consumer) + n += binary.PutUvarint(buf[n:], state.AckFloor.Stream) + n += binary.PutUvarint(buf[n:], state.Delivered.Consumer) + n += binary.PutUvarint(buf[n:], state.Delivered.Stream) + n += binary.PutUvarint(buf[n:], uint64(len(state.Pending))) + + asflr := state.AckFloor.Stream + adflr := state.AckFloor.Consumer + + // These are optional, but always write len. This is to avoid a truncate inline. + if len(state.Pending) > 0 { + // To save space we will use now rounded to seconds to be our base timestamp. + mints := time.Now().Round(time.Second).Unix() + // Write minimum timestamp we found from above. + n += binary.PutVarint(buf[n:], mints) + + for k, v := range state.Pending { + n += binary.PutUvarint(buf[n:], k-asflr) + n += binary.PutUvarint(buf[n:], v.Sequence-adflr) + // Downsample to seconds to save on space. + // Subsecond resolution not needed for recovery etc. + ts := v.Timestamp / int64(time.Second) + n += binary.PutVarint(buf[n:], mints-ts) + } + } + + // We always write the redelivered len. + n += binary.PutUvarint(buf[n:], uint64(len(state.Redelivered))) + + // We expect these to be small. + if len(state.Redelivered) > 0 { + for k, v := range state.Redelivered { + n += binary.PutUvarint(buf[n:], k-asflr) + n += binary.PutUvarint(buf[n:], v) + } + } + + return buf[:n] +} + +// Represents a pending message for explicit ack or ack all. +// Sequence is the original consumer sequence. +type Pending struct { + Sequence uint64 + Timestamp int64 +} + +const ( + limitsPolicyJSONString = `"limits"` + interestPolicyJSONString = `"interest"` + workQueuePolicyJSONString = `"workqueue"` +) + +var ( + limitsPolicyJSONBytes = []byte(limitsPolicyJSONString) + interestPolicyJSONBytes = []byte(interestPolicyJSONString) + workQueuePolicyJSONBytes = []byte(workQueuePolicyJSONString) +) + +func (rp RetentionPolicy) String() string { + switch rp { + case LimitsPolicy: + return "Limits" + case InterestPolicy: + return "Interest" + case WorkQueuePolicy: + return "WorkQueue" + default: + return "Unknown Retention Policy" + } +} + +func (rp RetentionPolicy) MarshalJSON() ([]byte, error) { + switch rp { + case LimitsPolicy: + return limitsPolicyJSONBytes, nil + case InterestPolicy: + return interestPolicyJSONBytes, nil + case WorkQueuePolicy: + return workQueuePolicyJSONBytes, nil + default: + return nil, fmt.Errorf("can not marshal %v", rp) + } +} + +func (rp *RetentionPolicy) UnmarshalJSON(data []byte) error { + switch string(data) { + case limitsPolicyJSONString: + *rp = LimitsPolicy + case interestPolicyJSONString: + *rp = InterestPolicy + case workQueuePolicyJSONString: + *rp = WorkQueuePolicy + default: + return fmt.Errorf("can not unmarshal %q", data) + } + return nil +} + +func (dp DiscardPolicy) String() string { + switch dp { + case DiscardOld: + return "DiscardOld" + case DiscardNew: + return "DiscardNew" + default: + return "Unknown Discard Policy" + } +} + +func (dp DiscardPolicy) MarshalJSON() ([]byte, error) { + switch dp { + case DiscardOld: + return []byte(`"old"`), nil + case DiscardNew: + return []byte(`"new"`), nil + default: + return nil, fmt.Errorf("can not marshal %v", dp) + } +} + +func (dp *DiscardPolicy) UnmarshalJSON(data []byte) error { + switch strings.ToLower(string(data)) { + case `"old"`: + *dp = DiscardOld + case `"new"`: + *dp = DiscardNew + default: + return fmt.Errorf("can not unmarshal %q", data) + } + return nil +} + +const ( + memoryStorageJSONString = `"memory"` + fileStorageJSONString = `"file"` +) + +var ( + memoryStorageJSONBytes = []byte(memoryStorageJSONString) + fileStorageJSONBytes = []byte(fileStorageJSONString) +) + +func (st StorageType) String() string { + switch st { + case MemoryStorage: + return "Memory" + case FileStorage: + return "File" + default: + return "Unknown Storage Type" + } +} + +func (st StorageType) MarshalJSON() ([]byte, error) { + switch st { + case MemoryStorage: + return memoryStorageJSONBytes, nil + case FileStorage: + return fileStorageJSONBytes, nil + default: + return nil, fmt.Errorf("can not marshal %v", st) + } +} + +func (st *StorageType) UnmarshalJSON(data []byte) error { + switch string(data) { + case memoryStorageJSONString: + *st = MemoryStorage + case fileStorageJSONString: + *st = FileStorage + default: + return fmt.Errorf("can not unmarshal %q", data) + } + return nil +} + +const ( + ackNonePolicyJSONString = `"none"` + ackAllPolicyJSONString = `"all"` + ackExplicitPolicyJSONString = `"explicit"` + ackFlowControlPolicyJSONString = `"flow_control"` +) + +var ( + ackNonePolicyJSONBytes = []byte(ackNonePolicyJSONString) + ackAllPolicyJSONBytes = []byte(ackAllPolicyJSONString) + ackExplicitPolicyJSONBytes = []byte(ackExplicitPolicyJSONString) + ackFlowControlPolicyJSONBytes = []byte(ackFlowControlPolicyJSONString) +) + +func (ap AckPolicy) MarshalJSON() ([]byte, error) { + switch ap { + case AckNone: + return ackNonePolicyJSONBytes, nil + case AckAll: + return ackAllPolicyJSONBytes, nil + case AckExplicit: + return ackExplicitPolicyJSONBytes, nil + case AckFlowControl: + return ackFlowControlPolicyJSONBytes, nil + default: + return nil, fmt.Errorf("can not marshal %v", ap) + } +} + +func (ap *AckPolicy) UnmarshalJSON(data []byte) error { + switch string(data) { + case ackNonePolicyJSONString: + *ap = AckNone + case ackAllPolicyJSONString: + *ap = AckAll + case ackExplicitPolicyJSONString: + *ap = AckExplicit + case ackFlowControlPolicyJSONString: + *ap = AckFlowControl + default: + return fmt.Errorf("can not unmarshal %q", data) + } + return nil +} + +const ( + replayInstantPolicyJSONString = `"instant"` + replayOriginalPolicyJSONString = `"original"` +) + +var ( + replayInstantPolicyJSONBytes = []byte(replayInstantPolicyJSONString) + replayOriginalPolicyJSONBytes = []byte(replayOriginalPolicyJSONString) +) + +func (rp ReplayPolicy) MarshalJSON() ([]byte, error) { + switch rp { + case ReplayInstant: + return replayInstantPolicyJSONBytes, nil + case ReplayOriginal: + return replayOriginalPolicyJSONBytes, nil + default: + return nil, fmt.Errorf("can not marshal %v", rp) + } +} + +func (rp *ReplayPolicy) UnmarshalJSON(data []byte) error { + switch string(data) { + case replayInstantPolicyJSONString: + *rp = ReplayInstant + case replayOriginalPolicyJSONString: + *rp = ReplayOriginal + default: + return fmt.Errorf("can not unmarshal %q", data) + } + return nil +} + +const ( + deliverAllPolicyJSONString = `"all"` + deliverLastPolicyJSONString = `"last"` + deliverNewPolicyJSONString = `"new"` + deliverByStartSequenceJSONString = `"by_start_sequence"` + deliverByStartTimeJSONString = `"by_start_time"` + deliverLastPerPolicyJSONString = `"last_per_subject"` + deliverUndefinedJSONString = `"undefined"` +) + +var ( + deliverAllPolicyJSONBytes = []byte(deliverAllPolicyJSONString) + deliverLastPolicyJSONBytes = []byte(deliverLastPolicyJSONString) + deliverNewPolicyJSONBytes = []byte(deliverNewPolicyJSONString) + deliverByStartSequenceJSONBytes = []byte(deliverByStartSequenceJSONString) + deliverByStartTimeJSONBytes = []byte(deliverByStartTimeJSONString) + deliverLastPerPolicyJSONBytes = []byte(deliverLastPerPolicyJSONString) + deliverUndefinedJSONBytes = []byte(deliverUndefinedJSONString) +) + +func (p *DeliverPolicy) UnmarshalJSON(data []byte) error { + switch string(data) { + case deliverAllPolicyJSONString, deliverUndefinedJSONString: + *p = DeliverAll + case deliverLastPolicyJSONString: + *p = DeliverLast + case deliverLastPerPolicyJSONString: + *p = DeliverLastPerSubject + case deliverNewPolicyJSONString: + *p = DeliverNew + case deliverByStartSequenceJSONString: + *p = DeliverByStartSequence + case deliverByStartTimeJSONString: + *p = DeliverByStartTime + default: + return fmt.Errorf("can not unmarshal %q", data) + } + + return nil +} + +func (p DeliverPolicy) MarshalJSON() ([]byte, error) { + switch p { + case DeliverAll: + return deliverAllPolicyJSONBytes, nil + case DeliverLast: + return deliverLastPolicyJSONBytes, nil + case DeliverLastPerSubject: + return deliverLastPerPolicyJSONBytes, nil + case DeliverNew: + return deliverNewPolicyJSONBytes, nil + case DeliverByStartSequence: + return deliverByStartSequenceJSONBytes, nil + case DeliverByStartTime: + return deliverByStartTimeJSONBytes, nil + default: + return deliverUndefinedJSONBytes, nil + } +} + +func isOutOfSpaceErr(err error) bool { + return err != nil && (strings.Contains(err.Error(), "no space left")) +} + +// For when our upper layer catchup detects its missing messages from the beginning of the stream. +var errFirstSequenceMismatch = errors.New("first sequence mismatch") + +func isClusterResetErr(err error) bool { + return err == errLastSeqMismatch || err == ErrStoreEOF || err == errFirstSequenceMismatch || errors.Is(err, errCatchupAbortedNoLeader) || err == errCatchupTooManyRetries || err == errAlreadyLeader +} + +// Copy all fields. +func (smo *StoreMsg) copy(sm *StoreMsg) { + if sm.buf != nil { + sm.buf = sm.buf[:0] + } + sm.buf = append(sm.buf, smo.buf...) + // We set cap on header in case someone wants to expand it. + sm.hdr, sm.msg = sm.buf[:len(smo.hdr):len(smo.hdr)], sm.buf[len(smo.hdr):] + sm.subj, sm.seq, sm.ts = smo.subj, smo.seq, smo.ts +} + +// Clear all fields except underlying buffer but reset that if present to [:0]. +func (sm *StoreMsg) clear() { + if sm == nil { + return + } + *sm = StoreMsg{_EMPTY_, nil, nil, sm.buf, 0, 0} + if len(sm.buf) > 0 { + sm.buf = sm.buf[:0] + } +} + +// Note this will avoid a copy of the data used for the string, but it will also reference the existing slice's data pointer. +// So this should be used sparingly when we know the encompassing byte slice's lifetime is the same. +func bytesToString(b []byte) string { + if len(b) == 0 { + return _EMPTY_ + } + p := unsafe.SliceData(b) + return unsafe.String(p, len(b)) +} + +// Same in reverse. Used less often. +func stringToBytes(s string) []byte { + if len(s) == 0 { + return nil + } + p := unsafe.StringData(s) + b := unsafe.Slice(p, len(s)) + return b +} + +// Forces a copy of a string, for use in the case that you might have been passed a value when bytesToString was used, +// but now you need a separate copy of it to store for longer-term use. +func copyString(s string) string { + b := make([]byte, len(s)) + copy(b, s) + return bytesToString(b) +} + +func isPermissionError(err error) bool { + return err != nil && os.IsPermission(err) +} diff --git a/server/store_test.go b/server/store_test.go new file mode 100644 index 0000000..f73e7ed --- /dev/null +++ b/server/store_test.go @@ -0,0 +1,1605 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !skip_store_tests + +package server + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "slices" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server/ats" + "github.com/nats-io/nats-server/v2/server/gsl" +) + +func testAllStoreAllPermutations(t *testing.T, compressionAndEncryption bool, cfg StreamConfig, fn func(t *testing.T, fs StreamStore)) { + t.Run("Memory", func(t *testing.T) { + cfg.Storage = MemoryStorage + fs, err := newMemStore(&cfg) + require_NoError(t, err) + defer fs.Stop() + fn(t, fs) + }) + t.Run("File", func(t *testing.T) { + cfg.Storage = FileStorage + if compressionAndEncryption { + testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { + fs, err := newFileStore(fcfg, cfg) + require_NoError(t, err) + defer fs.Stop() + fn(t, fs) + }) + } else { + fs, err := newFileStore(FileStoreConfig{ + StoreDir: t.TempDir(), + }, cfg) + require_NoError(t, err) + defer fs.Stop() + fn(t, fs) + } + }) +} + +func TestStoreMsgLoadNextMsgMulti(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}}, + func(t *testing.T, fs StreamStore) { + // Put 1k msgs in + for i := 0; i < 1000; i++ { + subj := fmt.Sprintf("foo.%d", i) + fs.StoreMsg(subj, nil, []byte("ZZZ"), 0) + } + + var smv StoreMsg + // Do multi load next with 1 wc entry. + sl := gsl.NewSublist[struct{}]() + sl.Insert("foo.>", struct{}{}) + for i, seq := 0, uint64(1); i < 1000; i++ { + sm, nseq, err := fs.LoadNextMsgMulti(sl, seq, &smv) + require_NoError(t, err) + require_Equal(t, sm.subj, fmt.Sprintf("foo.%d", i)) + require_Equal(t, nseq, seq) + seq++ + } + + // Now do multi load next with 1000 literal subjects. + sl = gsl.NewSublist[struct{}]() + for i := 0; i < 1000; i++ { + subj := fmt.Sprintf("foo.%d", i) + sl.Insert(subj, struct{}{}) + } + for i, seq := 0, uint64(1); i < 1000; i++ { + sm, nseq, err := fs.LoadNextMsgMulti(sl, seq, &smv) + require_NoError(t, err) + require_Equal(t, sm.subj, fmt.Sprintf("foo.%d", i)) + require_Equal(t, nseq, seq) + seq++ + } + + // Check that we can pull out 3 individuals. + sl = gsl.NewSublist[struct{}]() + sl.Insert("foo.2", struct{}{}) + sl.Insert("foo.222", struct{}{}) + sl.Insert("foo.999", struct{}{}) + sm, seq, err := fs.LoadNextMsgMulti(sl, 1, &smv) + require_NoError(t, err) + require_Equal(t, sm.subj, "foo.2") + require_Equal(t, seq, 3) + sm, seq, err = fs.LoadNextMsgMulti(sl, seq+1, &smv) + require_NoError(t, err) + require_Equal(t, sm.subj, "foo.222") + require_Equal(t, seq, 223) + sm, seq, err = fs.LoadNextMsgMulti(sl, seq+1, &smv) + require_NoError(t, err) + require_Equal(t, sm.subj, "foo.999") + require_Equal(t, seq, 1000) + _, seq, err = fs.LoadNextMsgMulti(sl, seq+1, &smv) + require_Error(t, err) + require_Equal(t, seq, 1000) + }, + ) +} + +func TestStoreLoadNextMsgWildcardStartBeforeFirstMatch(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "zzz", Subjects: []string{"bar.*", "foo.*"}}, + func(t *testing.T, fs StreamStore) { + // Fill non-matching subjects first so the first wildcard match starts + // strictly after the requested start sequence. + for i := 0; i < 100; i++ { + subj := fmt.Sprintf("bar.%d", i) + _, _, err := fs.StoreMsg(subj, nil, nil, 0) + require_NoError(t, err) + } + seq, _, err := fs.StoreMsg("foo.1", nil, nil, 0) + require_NoError(t, err) + require_Equal(t, seq, uint64(101)) + + var smv StoreMsg + sm, nseq, err := fs.LoadNextMsg("foo.*", true, 1, &smv) + require_NoError(t, err) + require_Equal(t, sm.subj, "foo.1") + require_Equal(t, nseq, uint64(101)) + + _, nseq, err = fs.LoadNextMsg("foo.*", true, nseq+1, &smv) + require_Error(t, err) + require_Equal(t, nseq, uint64(101)) + }, + ) +} + +func TestStoreDeleteSlice(t *testing.T) { + ds := DeleteSlice{2} + var deletes []uint64 + ds.Range(func(seq uint64) bool { + deletes = append(deletes, seq) + return true + }) + require_Len(t, len(deletes), 1) + require_Equal(t, deletes[0], 2) + + first, last, num := ds.State() + require_Equal(t, first, 2) + require_Equal(t, last, 2) + require_Equal(t, num, 1) +} + +func TestStoreDeleteRange(t *testing.T) { + dr := DeleteRange{First: 2, Num: 1} + var deletes []uint64 + dr.Range(func(seq uint64) bool { + deletes = append(deletes, seq) + return true + }) + require_Len(t, len(deletes), 1) + require_Equal(t, deletes[0], 2) + + first, last, num := dr.State() + require_Equal(t, first, 2) + require_Equal(t, last, 2) + require_Equal(t, num, 1) +} + +func TestStoreSubjectStateConsistency(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "TEST", Subjects: []string{"foo"}}, + func(t *testing.T, fs StreamStore) { + getSubjectState := func() SimpleState { + t.Helper() + ss := fs.SubjectsState("foo") + return ss["foo"] + } + var smp StoreMsg + expectFirstSeq := func(eseq uint64) { + t.Helper() + sm, _, err := fs.LoadNextMsg("foo", false, 0, &smp) + require_NoError(t, err) + require_Equal(t, sm.seq, eseq) + } + expectLastSeq := func(eseq uint64) { + t.Helper() + sm, err := fs.LoadLastMsg("foo", &smp) + require_NoError(t, err) + require_Equal(t, sm.seq, eseq) + } + + // Publish an initial batch of messages. + for i := 0; i < 4; i++ { + _, _, err := fs.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + } + + // Expect 4 msgs, with first=1, last=4. + ss := getSubjectState() + require_Equal(t, ss.Msgs, 4) + require_Equal(t, ss.First, 1) + expectFirstSeq(1) + require_Equal(t, ss.Last, 4) + expectLastSeq(4) + + // Remove first message, ss.First is lazy so will only mark ss.firstNeedsUpdate. + removed, err := fs.RemoveMsg(1) + require_NoError(t, err) + require_True(t, removed) + + // Will update first, so corrects to seq 2. + ss = getSubjectState() + require_Equal(t, ss.Msgs, 3) + require_Equal(t, ss.First, 2) + expectFirstSeq(2) + require_Equal(t, ss.Last, 4) + expectLastSeq(4) + + // Remove last message, ss.Last is lazy so will only mark ss.lastNeedsUpdate. + removed, err = fs.RemoveMsg(4) + require_NoError(t, err) + require_True(t, removed) + + // Will update last, so corrects to 3. + ss = getSubjectState() + require_Equal(t, ss.Msgs, 2) + require_Equal(t, ss.First, 2) + expectFirstSeq(2) + require_Equal(t, ss.Last, 3) + expectLastSeq(3) + + // Remove first message again. + removed, err = fs.RemoveMsg(2) + require_NoError(t, err) + require_True(t, removed) + + // Since we only have one message left, must update ss.First and ensure ss.Last equals. + ss = getSubjectState() + require_Equal(t, ss.Msgs, 1) + require_Equal(t, ss.First, 3) + expectFirstSeq(3) + require_Equal(t, ss.Last, 3) + expectLastSeq(3) + + // Publish some more messages so we can test another scenario. + for i := 0; i < 3; i++ { + _, _, err := fs.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + } + + // Just check the state is complete again. + ss = getSubjectState() + require_Equal(t, ss.Msgs, 4) + require_Equal(t, ss.First, 3) + expectFirstSeq(3) + require_Equal(t, ss.Last, 7) + expectLastSeq(7) + + // Remove last sequence, ss.Last is lazy so doesn't get updated. + removed, err = fs.RemoveMsg(7) + require_NoError(t, err) + require_True(t, removed) + + // Remove first sequence, ss.First is lazy so doesn't get updated. + removed, err = fs.RemoveMsg(3) + require_NoError(t, err) + require_True(t, removed) + + // Remove (now) first sequence. Both ss.First and ss.Last are lazy and both need to be recalculated later. + removed, err = fs.RemoveMsg(5) + require_NoError(t, err) + require_True(t, removed) + + // ss.First and ss.Last should both be recalculated and equal each other. + ss = getSubjectState() + require_Equal(t, ss.Msgs, 1) + require_Equal(t, ss.First, 6) + expectFirstSeq(6) + require_Equal(t, ss.Last, 6) + expectLastSeq(6) + + // We store a new message for ss.Last and remove it after, which marks it to be recalculated. + _, _, err = fs.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + removed, err = fs.RemoveMsg(8) + require_NoError(t, err) + require_True(t, removed) + // This will be the new ss.Last message, so reset ss.lastNeedsUpdate + _, _, err = fs.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + + // ss.First should remain the same, but ss.Last should equal the last message. + ss = getSubjectState() + require_Equal(t, ss.Msgs, 2) + require_Equal(t, ss.First, 6) + expectFirstSeq(6) + require_Equal(t, ss.Last, 9) + expectLastSeq(9) + }, + ) +} + +func TestStoreSubjectStateConsistencyOptimization(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "TEST", Subjects: []string{"foo"}}, + func(t *testing.T, fs StreamStore) { + fillMsgs := func(c int) { + t.Helper() + for i := 0; i < c; i++ { + _, _, err := fs.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + } + } + removeMsgs := func(seqs ...uint64) { + t.Helper() + for _, seq := range seqs { + removed, err := fs.RemoveMsg(seq) + require_NoError(t, err) + require_True(t, removed) + } + } + getSubjectState := func() (ss *SimpleState) { + t.Helper() + if f, ok := fs.(*fileStore); ok { + ss, ok = f.lmb.fss.Find([]byte("foo")) + require_True(t, ok) + } else if ms, ok := fs.(*memStore); ok { + ss, ok = ms.fss.Find([]byte("foo")) + require_True(t, ok) + } else { + t.Fatal("Store not supported") + } + return ss + } + var smp StoreMsg + expectSeq := func(seq uint64) { + t.Helper() + sm, _, err := fs.LoadNextMsg("foo", false, 0, &smp) + require_NoError(t, err) + require_Equal(t, sm.seq, seq) + sm, err = fs.LoadLastMsg("foo", &smp) + require_NoError(t, err) + require_Equal(t, sm.seq, seq) + } + + // results in ss.Last, ss.First is marked lazy (when we hit ss.Msgs-1==1). + fillMsgs(3) + removeMsgs(2, 1) + ss := getSubjectState() + require_Equal(t, ss.Msgs, 1) + require_Equal(t, ss.First, 3) + require_Equal(t, ss.Last, 3) + require_False(t, ss.firstNeedsUpdate) + require_False(t, ss.lastNeedsUpdate) + expectSeq(3) + + // ss.First is marked lazy first, then ss.Last is marked lazy (when we hit ss.Msgs-1==1). + fillMsgs(2) + removeMsgs(3, 5) + ss = getSubjectState() + require_Equal(t, ss.Msgs, 1) + require_Equal(t, ss.First, 3) + require_Equal(t, ss.Last, 5) + require_True(t, ss.firstNeedsUpdate) + require_True(t, ss.lastNeedsUpdate) + expectSeq(4) + + // ss.Last is marked lazy first, then ss.First is marked lazy (when we hit ss.Msgs-1==1). + fillMsgs(2) + removeMsgs(7, 4) + ss = getSubjectState() + require_Equal(t, ss.Msgs, 1) + require_Equal(t, ss.First, 4) + require_Equal(t, ss.Last, 7) + require_True(t, ss.firstNeedsUpdate) + require_True(t, ss.lastNeedsUpdate) + expectSeq(6) + + // ss.Msgs=1, results in ss.First, ss.Last is marked lazy (when we hit ss.Msgs-1==1). + fillMsgs(2) + removeMsgs(9, 8) + ss = getSubjectState() + require_Equal(t, ss.Msgs, 1) + require_Equal(t, ss.First, 6) + require_Equal(t, ss.Last, 6) + require_False(t, ss.firstNeedsUpdate) + require_False(t, ss.lastNeedsUpdate) + expectSeq(6) + }, + ) +} + +func TestStoreMaxMsgsPerUpdateBug(t *testing.T) { + config := func() StreamConfig { + return StreamConfig{Name: "TEST", Subjects: []string{"foo"}, MaxMsgsPer: 0} + } + testAllStoreAllPermutations( + t, false, config(), + func(t *testing.T, fs StreamStore) { + for i := 0; i < 5; i++ { + _, _, err := fs.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + } + + ss := fs.State() + require_Equal(t, ss.Msgs, 5) + require_Equal(t, ss.FirstSeq, 1) + require_Equal(t, ss.LastSeq, 5) + + // Update max messages per-subject from 0 (infinite) to 1. + // Since the per-subject limit was not specified before, messages should be removed upon config update. + cfg := config() + if _, ok := fs.(*fileStore); ok { + cfg.Storage = FileStorage + } else { + cfg.Storage = MemoryStorage + } + cfg.MaxMsgsPer = 1 + err := fs.UpdateConfig(&cfg) + require_NoError(t, err) + + // Only one message should remain. + ss = fs.State() + require_Equal(t, ss.Msgs, 1) + require_Equal(t, ss.FirstSeq, 5) + require_Equal(t, ss.LastSeq, 5) + + // Update max messages per-subject from 0 (infinite) to an invalid value (< -1). + cfg.MaxMsgsPer = -2 + err = fs.UpdateConfig(&cfg) + require_NoError(t, err) + require_Equal(t, cfg.MaxMsgsPer, -1) + }, + ) +} + +func TestStoreMaxMsgsPerUpdateToOneRemoveNewest(t *testing.T) { + config := func() StreamConfig { + return StreamConfig{Name: "TEST", Subjects: []string{"foo.*"}, MaxMsgsPer: -1} + } + testAllStoreAllPermutations( + t, false, config(), + func(t *testing.T, fs StreamStore) { + // Store the first copy of "foo.0". + _, _, err := fs.StoreMsg("foo.0", nil, nil, 0) + require_NoError(t, err) + + // Store filler subjects with enough data that the filestore rolls over into a + // second message block, so both copies of "foo.0" live in different blocks. + msg := make([]byte, 1024*1024) + fillBlock := func(expectedBlocks int) { + for i := 1; i <= 9; i++ { + _, _, err := fs.StoreMsg(fmt.Sprintf("foo.%d", i), nil, msg, 0) + require_NoError(t, err) + } + if f, ok := fs.(*fileStore); ok { + require_True(t, f.numMsgBlocks() >= expectedBlocks) + } + } + fillBlock(2) + + // Store a second copy of "foo.0". + nseq, _, err := fs.StoreMsg("foo.0", nil, nil, 0) + require_NoError(t, err) + + // And a third, in its own block, which we'll remove as well. + fillBlock(3) + lseq, _, err := fs.StoreMsg("foo.0", nil, nil, 0) + require_NoError(t, err) + removed, err := fs.RemoveMsg(lseq) + require_NoError(t, err) + require_True(t, removed) + + // Update max messages per-subject from -1 (unlimited) to 1. + // This transition does not run per-subject limit enforcement, so both copies remain. + cfg := config() + if _, ok := fs.(*fileStore); ok { + cfg.Storage = FileStorage + } else { + cfg.Storage = MemoryStorage + } + cfg.MaxMsgsPer = 1 + require_NoError(t, fs.UpdateConfig(&cfg)) + + ss := fs.SubjectsState("foo.0")["foo.0"] + require_Equal(t, ss.Msgs, 1) + require_Equal(t, ss.First, nseq) + require_Equal(t, ss.Last, nseq) + + var smv StoreMsg + sm, err := fs.LoadLastMsg("foo.0", &smv) + require_NoError(t, err) + require_Equal(t, sm.seq, nseq) + + sm, _, err = fs.LoadNextMsg("foo.0", false, 0, &smv) + require_NoError(t, err) + require_Equal(t, sm.seq, nseq) + + // The older copy must also still be found after a restart. + if f, ok := fs.(*fileStore); ok { + fcfg := f.fcfg + require_NoError(t, f.Stop()) + f, err = newFileStore(fcfg, cfg) + require_NoError(t, err) + defer f.Stop() + + sm, err = f.LoadLastMsg("foo.0", &smv) + require_NoError(t, err) + require_Equal(t, sm.seq, nseq) + } + }, + ) +} + +func TestStoreCompactCleansUpDmap(t *testing.T) { + config := func() StreamConfig { + return StreamConfig{Name: "TEST", Subjects: []string{"foo"}, MaxMsgsPer: 0} + } + for cseq := uint64(2); cseq <= 4; cseq++ { + t.Run(fmt.Sprintf("Compact(%d)", cseq), func(t *testing.T) { + testAllStoreAllPermutations( + t, false, config(), + func(t *testing.T, fs StreamStore) { + dmapEntries := func() int { + if fss, ok := fs.(*fileStore); ok { + return fss.dmapEntries() + } else if mss, ok := fs.(*memStore); ok { + mss.mu.RLock() + defer mss.mu.RUnlock() + return mss.dmap.Size() + } else { + return 0 + } + } + + // Publish messages, should have no interior deletes. + for i := 0; i < 3; i++ { + _, _, err := fs.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + } + require_Len(t, dmapEntries(), 0) + + // Removing one message in the middle should be an interior delete. + _, err := fs.RemoveMsg(2) + require_NoError(t, err) + require_Len(t, dmapEntries(), 1) + + // Compacting must always clean up the interior delete. + _, err = fs.Compact(cseq) + require_NoError(t, err) + require_Len(t, dmapEntries(), 0) + + // Validate first/last sequence. + state := fs.State() + fseq := uint64(3) + if fseq < cseq { + fseq = cseq + } + require_Equal(t, state.FirstSeq, fseq) + require_Equal(t, state.LastSeq, 3) + }) + }) + } +} + +func TestStoreCompactFullyDeletedRange(t *testing.T) { + config := func() StreamConfig { + return StreamConfig{Name: "TEST", Subjects: []string{"foo"}, MaxMsgsPer: 0} + } + testAllStoreAllPermutations( + t, false, config(), + func(t *testing.T, fs StreamStore) { + // Publish some messages. + for range 3 { + _, _, err := fs.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + } + + // Remove all messages in the range we're about to compact, + // keeping the first message so FirstSeq stays at 1. + _, err := fs.RemoveMsg(2) + require_NoError(t, err) + _, err = fs.RemoveMsg(3) + require_NoError(t, err) + + state := fs.State() + require_Equal(t, state.FirstSeq, 1) + require_Equal(t, state.LastSeq, 3) + + // Compact to a deleted sequence, and the stream is left empty. + _, err = fs.Compact(2) + require_NoError(t, err) + state = fs.State() + require_Equal(t, state.Msgs, 0) + require_Equal(t, state.LastSeq, 3) + require_Equal(t, state.FirstSeq, state.LastSeq+1) + require_True(t, state.FirstTime.IsZero()) + }) +} + +func TestStoreTruncateCleansUpDmap(t *testing.T) { + config := func() StreamConfig { + return StreamConfig{Name: "TEST", Subjects: []string{"foo"}, MaxMsgsPer: 0} + } + for tseq := uint64(0); tseq <= 1; tseq++ { + t.Run(fmt.Sprintf("Truncate(%d)", tseq), func(t *testing.T) { + testAllStoreAllPermutations( + t, false, config(), + func(t *testing.T, fs StreamStore) { + dmapEntries := func() int { + if fss, ok := fs.(*fileStore); ok { + return fss.dmapEntries() + } else if mss, ok := fs.(*memStore); ok { + mss.mu.RLock() + defer mss.mu.RUnlock() + return mss.dmap.Size() + } else { + return 0 + } + } + + // Publish messages, should have no interior deletes. + for i := 0; i < 3; i++ { + _, _, err := fs.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + } + require_Len(t, dmapEntries(), 0) + + // Removing one message in the middle should be an interior delete. + _, err := fs.RemoveMsg(2) + require_NoError(t, err) + require_Len(t, dmapEntries(), 1) + + // Truncating must always clean up the interior delete. + err = fs.Truncate(tseq) + require_NoError(t, err) + require_Len(t, dmapEntries(), 0) + + // Validate first/last sequence. + state := fs.State() + fseq := uint64(1) + if fseq > tseq { + fseq = tseq + } + require_Equal(t, state.FirstSeq, fseq) + require_Equal(t, state.LastSeq, tseq) + }) + }) + } +} + +// https://github.com/nats-io/nats-server/issues/6709 +func TestStorePurgeExZero(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "TEST", Subjects: []string{"foo"}}, + func(t *testing.T, fs StreamStore) { + // Simple purge all. + _, err := fs.Purge() + require_NoError(t, err) + ss := fs.State() + require_Equal(t, ss.FirstSeq, 1) + require_Equal(t, ss.LastSeq, 0) + + // PurgeEx(seq=0) must be equal. + _, err = fs.PurgeEx(_EMPTY_, 0, 0) + require_NoError(t, err) + ss = fs.State() + require_Equal(t, ss.FirstSeq, 1) + require_Equal(t, ss.LastSeq, 0) + }, + ) +} + +func TestStorePurgeExSequenceOne(t *testing.T) { + testAllStoreAllPermutations( + t, true, + StreamConfig{Name: "TEST", Subjects: []string{"foo", "bar"}}, + func(t *testing.T, fs StreamStore) { + for range 5 { + _, _, err := fs.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + _, _, err = fs.StoreMsg("bar", nil, nil, 0) + require_NoError(t, err) + } + before := fs.State() + check := func(t *testing.T, subject string, keep uint64) { + n, err := fs.PurgeEx(subject, 1, keep) + require_NoError(t, err) + require_Equal(t, n, 0) + after := fs.State() + require_Equal(t, after.Msgs, before.Msgs) + require_Equal(t, after.FirstSeq, before.FirstSeq) + require_Equal(t, after.LastSeq, before.LastSeq) + } + t.Run("empty-subject", func(t *testing.T) { check(t, _EMPTY_, 0) }) + t.Run("wildcard-subject", func(t *testing.T) { check(t, fwcs, 0) }) + t.Run("specific-subject", func(t *testing.T) { check(t, "foo", 0) }) + t.Run("empty-subject-with-keep", func(t *testing.T) { check(t, _EMPTY_, 3) }) + t.Run("specific-subject-with-keep", func(t *testing.T) { check(t, "foo", 3) }) + }, + ) +} + +func TestStorePurgeExKeepWithInteriorDeletes(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "TEST", Subjects: []string{"foo"}}, + func(t *testing.T, fs StreamStore) { + for range 50 { + _, _, err := fs.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + } + // Remove every other message to create interior gaps. + for seq := uint64(2); seq <= 50; seq += 2 { + _, err := fs.RemoveMsg(seq) + require_NoError(t, err) + } + ss := fs.State() + require_Equal(t, ss.Msgs, 25) + require_Equal(t, ss.FirstSeq, 1) + require_Equal(t, ss.LastSeq, 50) + + // Keep the 5 newest. Newest 5 existing seqs are 41, 43, 45, 47, 49. + n, err := fs.PurgeEx(_EMPTY_, 0, 5) + require_NoError(t, err) + require_Equal(t, n, 20) + + ss = fs.State() + require_Equal(t, ss.Msgs, 5) + require_Equal(t, ss.FirstSeq, 41) + require_Equal(t, ss.LastSeq, 50) + }, + ) +} + +func TestStoreUpdateConfigTTLState(t *testing.T) { + config := func() StreamConfig { + return StreamConfig{Name: "TEST", Subjects: []string{"foo"}} + } + testAllStoreAllPermutations( + t, false, config(), + func(t *testing.T, fs StreamStore) { + cfg := config() + switch fs.(type) { + case *fileStore: + cfg.Storage = FileStorage + case *memStore: + cfg.Storage = MemoryStorage + } + + // TTLs disabled at this point so this message should survive. + seq, _, err := fs.StoreMsg("foo", nil, nil, 1) + require_NoError(t, err) + time.Sleep(2 * time.Second) + _, err = fs.LoadMsg(seq, nil) + require_NoError(t, err) + + // Now enable TTLs. + cfg.AllowMsgTTL = true + require_NoError(t, fs.UpdateConfig(&cfg)) + + // TTLs enabled at this point so this message should be cleaned up. + seq, _, err = fs.StoreMsg("foo", nil, nil, 1) + require_NoError(t, err) + time.Sleep(2 * time.Second) + _, err = fs.LoadMsg(seq, nil) + require_Error(t, err) + + // Now disable TTLs again. + cfg.AllowMsgTTL = false + require_NoError(t, fs.UpdateConfig(&cfg)) + + // TTLs disabled again so this message should survive. + seq, _, err = fs.StoreMsg("foo", nil, nil, 1) + require_NoError(t, err) + time.Sleep(2 * time.Second) + _, err = fs.LoadMsg(seq, nil) + require_NoError(t, err) + }, + ) +} + +func TestStoreStreamInteriorDeleteAccounting(t *testing.T) { + tests := []struct { + title string + action func(s StreamStore, lseq uint64) + }{ + { + title: "TruncateWithRemove", + action: func(s StreamStore, lseq uint64) { + seq, _, err := s.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + require_Equal(t, seq, lseq) + removed, err := s.RemoveMsg(lseq) + require_NoError(t, err) + require_True(t, removed) + require_NoError(t, s.Truncate(lseq)) + }, + }, + { + title: "TruncateWithErase", + action: func(s StreamStore, lseq uint64) { + seq, _, err := s.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + require_Equal(t, seq, lseq) + removed, err := s.EraseMsg(lseq) + require_NoError(t, err) + require_True(t, removed) + require_NoError(t, s.Truncate(lseq)) + }, + }, + { + title: "TruncateWithTombstone", + action: func(s StreamStore, lseq uint64) { + seq, _, err := s.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + require_Equal(t, seq, lseq) + if fs, ok := s.(*fileStore); ok { + removed, err := fs.removeMsg(lseq, false, false, true) + require_NoError(t, err) + require_True(t, removed) + } else { + removed, err := s.RemoveMsg(lseq) + require_NoError(t, err) + require_True(t, removed) + } + require_NoError(t, s.Truncate(lseq)) + }, + }, + { + title: "SkipMsg", + action: func(s StreamStore, lseq uint64) { + s.SkipMsg(0) + }, + }, + { + title: "SkipMsgs", + action: func(s StreamStore, lseq uint64) { + require_NoError(t, s.SkipMsgs(lseq, 1)) + }, + }, + } + for _, empty := range []bool{false, true} { + for _, test := range tests { + t.Run(fmt.Sprintf("Empty=%v/%s", empty, test.title), func(t *testing.T) { + cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo"}} + testAllStoreAllPermutations(t, true, cfg, func(t *testing.T, s StreamStore) { + var err error + var lseq uint64 + if !empty { + lseq, _, err = s.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + require_Equal(t, lseq, 1) + } + lseq++ + + test.action(s, lseq) + + // Confirm state as baseline. + before := s.State() + if empty { + require_Equal(t, before.Msgs, 0) + require_Equal(t, before.FirstSeq, 2) + require_Equal(t, before.LastSeq, 1) + } else { + require_Equal(t, before.Msgs, 1) + require_Equal(t, before.FirstSeq, 1) + require_Equal(t, before.LastSeq, 2) + } + + var fs *fileStore + var ok bool + if fs, ok = s.(*fileStore); !ok { + return + } + cfg.Storage = FileStorage + fcfg := fs.fcfg + created := time.Time{} + + // Restart should equal state. + require_NoError(t, fs.Stop()) + fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) + require_NoError(t, err) + defer fs.Stop() + + if state := fs.State(); !reflect.DeepEqual(state, before) { + t.Fatalf("Expected state of:\n%+v, got:\n%+v", before, state) + } + + // Stop and remove stream state file. + require_NoError(t, fs.Stop()) + require_NoError(t, os.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile))) + + // Recovering based on blocks should result in the same state. + fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) + require_NoError(t, err) + defer fs.Stop() + + if state := fs.State(); !reflect.DeepEqual(state, before) { + t.Fatalf("Expected state of:\n%+v, got:\n%+v", before, state) + } + + // Rebuilding state must also result in the same state. + fs.rebuildState(nil) + if state := fs.State(); !reflect.DeepEqual(state, before) { + t.Fatalf("Expected state of:\n%+v, got:\n%+v", before, state) + } + }) + }) + } + } +} + +func TestStoreMsgLoadPrevMsgMulti(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}}, + func(t *testing.T, fs StreamStore) { + // Put 1k msgs in + for i := range 1000 { + subj := fmt.Sprintf("foo.%d", i+1) + fs.StoreMsg(subj, nil, []byte("ZZZ"), 0) + } + + var sm StoreMsg + var count int + var state StreamState + fs.FastState(&state) + + sl := gsl.NewSimpleSublist() + sl.Insert("foo.5", struct{}{}) + sl.Insert("foo.15", struct{}{}) + sl.Insert("foo.105", struct{}{}) + + for seq := state.LastSeq; seq > 5; seq-- { + var err error + _, seq, err = fs.LoadPrevMsgMulti(sl, seq, &sm) + require_NoError(t, err) + require_Equal(t, sm.subj, fmt.Sprintf("foo.%d", sm.seq)) + count++ + } + + _, _, err := fs.LoadPrevMsgMulti(sl, 4, &sm) + require_Error(t, err, ErrStoreEOF) + require_Equal(t, count, 3) + }, + ) +} + +func TestStoreMsgLoadPrevMsg(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "zzz", Subjects: []string{"foo.*", "bar.*"}}, + func(t *testing.T, fs StreamStore) { + for _, subj := range []string{"foo.1", "bar.1", "foo.2", "bar.2", "foo.3"} { + _, _, err := fs.StoreMsg(subj, nil, []byte("ZZZ"), 0) + require_NoError(t, err) + } + + var sm StoreMsg + + smp, seq, err := fs.LoadPrevMsg(_EMPTY_, false, 5, &sm) + require_NoError(t, err) + require_Equal(t, smp.subj, "foo.3") + require_Equal(t, seq, uint64(5)) + + smp, seq, err = fs.LoadPrevMsg("foo.2", false, 5, &sm) + require_NoError(t, err) + require_Equal(t, smp.subj, "foo.2") + require_Equal(t, seq, uint64(3)) + + smp, seq, err = fs.LoadPrevMsg("foo.*", true, 5, &sm) + require_NoError(t, err) + require_Equal(t, smp.subj, "foo.3") + require_Equal(t, seq, uint64(5)) + + _, seq, err = fs.LoadPrevMsg("baz.*", true, 5, &sm) + require_Error(t, err, ErrStoreEOF) + require_Equal(t, seq, uint64(1)) + }, + ) +} + +func TestStoreMsgLoadPrevMsgMultiFullWildcardSkip(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}}, + func(t *testing.T, fs StreamStore) { + for i := range 10 { + subj := fmt.Sprintf("foo.%d", i+1) + _, _, err := fs.StoreMsg(subj, nil, []byte("ZZZ"), 0) + require_NoError(t, err) + } + + var sm StoreMsg + var state StreamState + fs.FastState(&state) + + sl := gsl.NewSimpleSublist() + require_NoError(t, sl.Insert(">", struct{}{})) + + var got []uint64 + for seq := state.LastSeq; ; { + smp, nseq, err := fs.LoadPrevMsgMulti(sl, seq, &sm) + if err == ErrStoreEOF { + require_Equal(t, nseq, state.FirstSeq) + break + } + require_NoError(t, err) + require_Equal(t, smp.seq, nseq) + got = append(got, nseq) + if nseq == state.FirstSeq { + _, nseq, err = fs.LoadPrevMsgMulti(sl, nseq-1, &sm) + require_Error(t, err, ErrStoreEOF) + require_Equal(t, nseq, state.FirstSeq) + break + } + seq = nseq - 1 + } + + require_True(t, slices.Equal(got, []uint64{10, 9, 8, 7, 6, 5, 4, 3, 2, 1})) + }, + ) +} + +func TestStoreDiscardNew(t *testing.T) { + test := func(t *testing.T, updateConfig func(cfg *StreamConfig), expectedErr error) { + cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo"}, Discard: DiscardNew} + updateConfig(&cfg) + testAllStoreAllPermutations(t, false, cfg, func(t *testing.T, fs StreamStore) { + ts := time.Now().UnixNano() + expectedSeq := uint64(1) + requireState := func() { + t.Helper() + state := fs.State() + require_Equal(t, state.Msgs, 1) + require_Equal(t, state.FirstSeq, expectedSeq) + require_Equal(t, state.LastSeq, expectedSeq) + } + + _, _, err := fs.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + + err = fs.StoreRawMsg("foo", nil, nil, 0, ts, 0, true) + if expectedErr == nil { + require_NoError(t, err) + expectedSeq++ + } else { + require_Equal(t, err, expectedErr) + } + requireState() + + // For a clustered stream DiscardNew should only be enforced by the stream leader. + // Followers MUST always accept data that they've received from the leader, + // otherwise we risk stream desync if some servers decide to reject. + err = fs.StoreRawMsg("foo", nil, nil, 0, ts, 0, false) + require_NoError(t, err) + expectedSeq++ + + // Since DiscardNew we must only add to the stream, and not act like + // DiscardOld based on MaxMsgs/MaxBytes limits, unless MaxMsgsPer is set. + if cfg.MaxMsgsPer > 0 { + requireState() + } else { + state := fs.State() + require_Equal(t, state.Msgs, 2) + require_Equal(t, state.FirstSeq, expectedSeq-1) + require_Equal(t, state.LastSeq, expectedSeq) + } + }) + } + + t.Run("MaxMsgs", func(t *testing.T) { test(t, func(cfg *StreamConfig) { cfg.MaxMsgs = 1 }, ErrMaxMsgs) }) + t.Run("MaxBytes", func(t *testing.T) { test(t, func(cfg *StreamConfig) { cfg.MaxBytes = 33 }, ErrMaxBytes) }) + t.Run("MaxMsgsPer", func(t *testing.T) { test(t, func(cfg *StreamConfig) { cfg.MaxMsgsPer = 1 }, nil) }) + t.Run("MaxMsgsPer_DiscardNewPer", func(t *testing.T) { + test(t, func(cfg *StreamConfig) { + cfg.DiscardNewPer = true + cfg.MaxMsgsPer = 1 + }, ErrMaxMsgsPerSubject) + }) + t.Run("MaxMsgsPer_MaxMsgs", func(t *testing.T) { + test(t, func(cfg *StreamConfig) { + // Without DiscardNewPerSubject we can replace the message with the new one if it fits (it will for this test). + cfg.MaxMsgs = 1 + cfg.MaxMsgsPer = 1 + }, nil) + }) + t.Run("MaxMsgsPer_MaxBytes", func(t *testing.T) { + test(t, func(cfg *StreamConfig) { + // Without DiscardNewPerSubject we can replace the message with the new one if it fits (it will for this test). + cfg.MaxBytes = 33 + cfg.MaxMsgsPer = 1 + }, nil) + }) + t.Run("MaxMsgsPer_MaxMsgs_DiscardNewPer", func(t *testing.T) { + test(t, func(cfg *StreamConfig) { + cfg.DiscardNewPer = true + cfg.MaxMsgs = 1 + cfg.MaxMsgsPer = 1 + }, ErrMaxMsgsPerSubject) + }) + t.Run("MaxMsgsPer_MaxBytes_DiscardNewPer", func(t *testing.T) { + test(t, func(cfg *StreamConfig) { + cfg.DiscardNewPer = true + cfg.MaxBytes = 33 + cfg.MaxMsgsPer = 1 + }, ErrMaxMsgsPerSubject) + }) +} + +func TestStoreGetSeqFromTimeWithInteriorDeletesGap(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "zzz", Subjects: []string{"foo"}}, + func(t *testing.T, fs StreamStore) { + var start int64 + for i := range 10 { + _, ts, err := fs.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + if i == 1 { + start = ts + } + } + // Create a delete gap to prove a simple binary search between sequences + // does not work, and deletes need to be accounted for. A simple binary search + // will hit the deleted sequences and then return the last sequence. + for seq := uint64(4); seq <= 7; seq++ { + _, err := fs.RemoveMsg(seq) + require_NoError(t, err) + } + ts := time.Unix(0, start).UTC() + require_Equal(t, fs.GetSeqFromTime(ts), 2) + }, + ) +} + +func TestStoreGetSeqFromTimeWithTrailingDeletes(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "zzz", Subjects: []string{"foo"}}, + func(t *testing.T, fs StreamStore) { + var start int64 + for i := range 3 { + _, ts, err := fs.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + if i == 1 { + start = ts + } + } + _, err := fs.RemoveMsg(3) + require_NoError(t, err) + ts := time.Unix(0, start).UTC() + require_Equal(t, fs.GetSeqFromTime(ts), 2) + }, + ) +} + +func TestStoreSkipMsgsCarryForwardLastTime(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "zzz", Subjects: []string{"foo"}}, + func(t *testing.T, fs StreamStore) { + // Store a message with an old timestamp. + old := time.Now().Add(-30 * 24 * time.Hour).UTC().Truncate(time.Second) + err := fs.StoreRawMsg("foo", nil, nil, 1, old.UnixNano(), 0, false) + require_NoError(t, err) + + var ss StreamState + fs.FastState(&ss) + require_Equal(t, ss.LastSeq, 1) + require_Equal(t, ss.LastTime.UnixNano(), old.UnixNano()) + + // SkipMsgs should not move the last time forward. + require_NoError(t, fs.SkipMsgs(2, 10)) + fs.FastState(&ss) + require_Equal(t, ss.LastSeq, 11) + require_Equal(t, ss.LastTime.UnixNano(), old.UnixNano()) + + // SkipMsg should not move the last time forward. + seq, err := fs.SkipMsg(12) + require_NoError(t, err) + require_Equal(t, seq, 12) + fs.FastState(&ss) + require_Equal(t, ss.LastSeq, 12) + require_Equal(t, ss.LastTime.UnixNano(), old.UnixNano()) + + // Another chained skip should not move the last time forward. + require_NoError(t, fs.SkipMsgs(13, 5)) + fs.FastState(&ss) + require_Equal(t, ss.LastSeq, 17) + require_Equal(t, ss.LastTime.UnixNano(), old.UnixNano()) + + // Time-based lookups must resolve across the skip gaps. + newer := old.Add(15 * 24 * time.Hour) + err = fs.StoreRawMsg("foo", nil, nil, 18, newer.UnixNano(), 0, false) + require_NoError(t, err) + require_Equal(t, fs.GetSeqFromTime(old), 1) + require_Equal(t, fs.GetSeqFromTime(old.Add(5*24*time.Hour)), 18) + require_Equal(t, fs.GetSeqFromTime(newer), 18) + }, + ) +} + +func TestStoreSkipMsgsCarryForwardLastTimeFromEmpty(t *testing.T) { + t.Run("SkipMsg", func(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "zzz", Subjects: []string{"foo"}}, + func(t *testing.T, fs StreamStore) { + var ss StreamState + fs.FastState(&ss) + require_Equal(t, ss.LastSeq, uint64(0)) + require_True(t, ss.LastTime.IsZero()) + + // Zero state, so falls back to ats.AccessTime(). + before := ats.AccessTime() + seq, err := fs.SkipMsg(0) + require_NoError(t, err) + require_Equal(t, seq, uint64(1)) + + fs.FastState(&ss) + require_Equal(t, ss.LastSeq, uint64(1)) + require_False(t, ss.LastTime.IsZero()) + require_True(t, ss.LastTime.UnixNano() >= before) + require_True(t, ss.LastTime.UnixNano() <= ats.AccessTime()) + first := ss.LastTime + + // Advance the clock; a subsequent skip must preserve the time. + time.Sleep(2 * ats.TickInterval) + + seq, err = fs.SkipMsg(0) + require_NoError(t, err) + require_Equal(t, seq, uint64(2)) + + fs.FastState(&ss) + require_Equal(t, ss.LastSeq, uint64(2)) + require_Equal(t, ss.LastTime.UnixNano(), first.UnixNano()) + }, + ) + }) + + t.Run("SkipMsgs", func(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "zzz", Subjects: []string{"foo"}}, + func(t *testing.T, fs StreamStore) { + var ss StreamState + fs.FastState(&ss) + require_Equal(t, ss.LastSeq, uint64(0)) + require_True(t, ss.LastTime.IsZero()) + + // Zero state, so falls back to ats.AccessTime(). + before := ats.AccessTime() + require_NoError(t, fs.SkipMsgs(1, 5)) + + fs.FastState(&ss) + require_Equal(t, ss.LastSeq, uint64(5)) + require_False(t, ss.LastTime.IsZero()) + require_True(t, ss.LastTime.UnixNano() >= before) + require_True(t, ss.LastTime.UnixNano() <= ats.AccessTime()) + first := ss.LastTime + + // Advance the clock; a subsequent skip must preserve the time. + time.Sleep(2 * ats.TickInterval) + + seq, err := fs.SkipMsg(0) + require_NoError(t, err) + require_Equal(t, seq, uint64(6)) + + fs.FastState(&ss) + require_Equal(t, ss.LastSeq, uint64(6)) + require_Equal(t, ss.LastTime.UnixNano(), first.UnixNano()) + }, + ) + }) +} + +func TestStoreSkipMsgNoInterestAdvancesLastTime(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "zzz", Subjects: []string{"foo"}}, + func(t *testing.T, fs StreamStore) { + // A skip without interest still increases the sequence. + seq, err := fs.SkipMsgNoInterest(0) + require_NoError(t, err) + require_Equal(t, seq, uint64(1)) + + var ss StreamState + fs.FastState(&ss) + require_Equal(t, ss.Msgs, uint64(0)) + require_Equal(t, ss.LastSeq, uint64(1)) + first := ss.LastTime + + // Let the clock advance past the access time service granularity. + time.Sleep(2 * ats.TickInterval) + + // The second skip still has no interest, but should update both the sequence and time. + seq, err = fs.SkipMsgNoInterest(0) + require_NoError(t, err) + require_Equal(t, seq, uint64(2)) + + fs.FastState(&ss) + require_Equal(t, ss.Msgs, uint64(0)) + require_Equal(t, ss.LastSeq, uint64(2)) + require_True(t, ss.LastTime.After(first)) + }, + ) +} + +func TestFileStoreMultiLastSeqsAndLoadLastMsgWithLazySubjectState(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "zzz", Subjects: []string{"foo"}}, + func(t *testing.T, fs StreamStore) { + for range 3 { + _, _, err := fs.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + } + seqs, err := fs.MultiLastSeqs([]string{"foo"}, 0, 0) + require_NoError(t, err) + require_Equal(t, len(seqs), 1) + require_Equal(t, seqs[0], 3) + + _, err = fs.RemoveMsg(3) + require_NoError(t, err) + seqs, err = fs.MultiLastSeqs([]string{"foo"}, 0, 0) + require_NoError(t, err) + require_Equal(t, len(seqs), 1) + require_Equal(t, seqs[0], 2) + + _, _, err = fs.StoreMsg("foo", nil, nil, 0) + require_NoError(t, err) + sm, err := fs.LoadLastMsg("foo", nil) + require_NoError(t, err) + require_Equal(t, sm.seq, 4) + + _, err = fs.RemoveMsg(4) + require_NoError(t, err) + sm, err = fs.LoadLastMsg("foo", nil) + require_NoError(t, err) + require_Equal(t, sm.seq, 2) + }, + ) +} + +func TestStoreMultiLastMsgs(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}}, + func(t *testing.T, fs StreamStore) { + // Three rounds over ten subjects, so every subject has multiple + // revisions and the last per subject sits in the final round. + // Seqs 1-30, last for foo. is 21+i. + for range 3 { + for i := range 10 { + _, _, err := fs.StoreMsg(fmt.Sprintf("foo.%d", i), nil, nil, 0) + require_NoError(t, err) + } + } + + type delivery struct { + seq uint64 + np uint64 + } + load := func(filters []string, minSeq, maxSeq uint64, maxAllowed, stopAfter int) ([]delivery, uint64, uint64, error) { + var msgs []delivery + total, np, err := fs.MultiLastMsgs(filters, minSeq, maxSeq, maxAllowed, func(sm *StoreMsg, np uint64) bool { + msgs = append(msgs, delivery{sm.seq, np}) + return stopAfter == 0 || len(msgs) < stopAfter + }) + return msgs, total, np, err + } + + // Whole stream delivers the last message per subject in ascending + // sequence order, with np counting down the remaining messages. + msgs, total, np, err := load([]string{"foo.*"}, 0, 0, -1, 0) + require_NoError(t, err) + require_Equal(t, total, 10) + require_Equal(t, np, 0) + require_Len(t, len(msgs), 10) + for i, d := range msgs { + require_Equal(t, d.seq, uint64(21+i)) + require_Equal(t, d.np, uint64(9-i)) + } + + // minSeq skips lower sequences but still accounts for them in np. + msgs, total, np, err = load([]string{"foo.*"}, 26, 0, -1, 0) + require_NoError(t, err) + require_Equal(t, total, 10) + require_Equal(t, np, 0) + require_Len(t, len(msgs), 5) + for i, d := range msgs { + require_Equal(t, d.seq, uint64(26+i)) + require_Equal(t, d.np, uint64(4-i)) + } + + // maxSeq resolves the last message per subject at or below it, + // stepping back to an earlier round where needed. + msgs, total, np, err = load([]string{"foo.*"}, 0, 25, -1, 0) + require_NoError(t, err) + require_Equal(t, total, 10) + require_Equal(t, np, 0) + require_Len(t, len(msgs), 10) + for i, d := range msgs { + require_Equal(t, d.seq, uint64(16+i)) + } + + // Stopping the callback early keeps np at the remaining count. + msgs, total, np, err = load([]string{"foo.*"}, 0, 0, -1, 3) + require_NoError(t, err) + require_Equal(t, total, 10) + require_Equal(t, np, 7) + require_Len(t, len(msgs), 3) + + // Exceeding maxAllowed errors without delivering anything. + msgs, _, _, err = load([]string{"foo.*"}, 0, 0, 5, 0) + require_Error(t, err, ErrTooManyResults) + require_Len(t, len(msgs), 0) + + // No matches. + msgs, total, _, err = load([]string{"bar.>"}, 0, 0, -1, 0) + require_NoError(t, err) + require_Equal(t, total, 0) + require_Len(t, len(msgs), 0) + }, + ) +} + +func TestStoreNumPendingLastPerSubjectExcludeOvercount(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}}, + func(t *testing.T, fs StreamStore) { + put := func(subj string) { + _, _, err := fs.StoreMsg(subj, nil, []byte("x"), 0) + require_NoError(t, err) + } + // foo.A occupies seqs 1-10 (entirely below our start seq). + for range 10 { + put("foo.A") + } + put("foo.B") // seq 11 + put("foo.C") // seq 12 + require_NoError(t, fs.SkipMsgs(13, 87)) // skip 13-99 + put("foo.B") // seq 100 + require_NoError(t, fs.SkipMsgs(101, 899)) // skip 101-999 + put("foo.C") // seq 1000 + + // lastPerSubject starting at seq 13: foo.A's last (10) is below 13 so it + // contributes 0; foo.B (last 100) and foo.C (last 1000) each contribute 1. + total, _, err := fs.NumPending(13, "foo.*", true) + require_NoError(t, err) + require_Equal(t, total, 2) + + filters := gsl.NewSublist[struct{}]() + filters.Insert("foo.*", struct{}{}) + total, _, err = fs.NumPendingMulti(13, filters, true) + require_NoError(t, err) + require_Equal(t, total, 2) + }, + ) +} + +// Resolving last sequences for many subjects whose last messages are spread +// over the whole stream. For the filestore a small block size spreads them +// across many blocks: WholeStream can resolve via each subject's last block, +// Bounded must walk the blocks backwards from maxSeq. +func Benchmark_StoreMultiLastSeqsManyBlocks(b *testing.B) { + const ( + numSubjects = 1_000 + msgsPerSubj = 100 + ) + + run := func(b *testing.B, fs StreamStore) { + // Store each subject's messages contiguously so the per-subject last + // messages land spread out over the whole stream. + msg := []byte("ok") + for i := range numSubjects { + subj := fmt.Sprintf("foo.%d", i) + for range msgsPerSubj { + _, _, err := fs.StoreMsg(subj, nil, msg, 0) + require_NoError(b, err) + } + } + + // A sparse subset of subjects spread over the whole stream. Resolving + // via each subject's last block only visits these few blocks, while + // the backwards walk must scan nearly every block to find them all. + var sparse []string + for i := 0; i < numSubjects; i += numSubjects / 10 { + sparse = append(sparse, fmt.Sprintf("foo.%d", i)) + } + + for _, bc := range []struct { + name string + filters []string + matches int + maxSeq uint64 + }{ + {"AllSubjects/WholeStream", []string{">"}, numSubjects, 0}, + {"AllSubjects/Bounded", []string{">"}, numSubjects, numSubjects*msgsPerSubj - 1}, + {"SparseSubjects/WholeStream", sparse, len(sparse), 0}, + {"SparseSubjects/Bounded", sparse, len(sparse), numSubjects*msgsPerSubj - 1}, + } { + b.Run(bc.name, func(b *testing.B) { + for range b.N { + seqs, err := fs.MultiLastSeqs(bc.filters, bc.maxSeq, -1) + require_NoError(b, err) + require_Len(b, len(seqs), bc.matches) + } + }) + } + } + + cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo.*", "bar.*"}} + + b.Run("Memory", func(b *testing.B) { + cfg := cfg + cfg.Storage = MemoryStorage + ms, err := newMemStore(&cfg) + require_NoError(b, err) + defer ms.Stop() + run(b, ms) + }) + b.Run("File", func(b *testing.B) { + cfg := cfg + cfg.Storage = FileStorage + fs, err := newFileStore( + FileStoreConfig{StoreDir: b.TempDir(), BlockSize: 8192}, cfg) + require_NoError(b, err) + defer fs.Stop() + run(b, fs) + }) +} + +func TestStoreSourcesOnlyTracksConfiguredSources(t *testing.T) { + testAllStoreAllPermutations( + t, false, + StreamConfig{ + Name: "SOURCE", + Subjects: []string{">"}, + Sources: []*StreamSource{{Name: "ORIGIN1"}}, + }, + func(t *testing.T, fs StreamStore) { + h1 := genHeader(nil, JSStreamSource, "ORIGIN1 5 > > foo.a IDENT1") + _, _, err := fs.StoreMsg("foo.a", h1, nil, 0) + require_NoError(t, err) + + // Only configured sources are seeded into the map, so a header naming one that + // isn't configured is not tracked. It is recovered by a scan if it is added. + h2 := genHeader(nil, JSStreamSource, "ORIGIN2 7 > > foo.b IDENT2") + _, _, err = fs.StoreMsg("foo.b", h2, nil, 0) + require_NoError(t, err) + + // Nor is a pre-2.10 header, which carries no index name at all. + h3 := genHeader(nil, JSStreamSource, "ORIGIN3 9") + _, _, err = fs.StoreMsg("foo.c", h3, nil, 0) + require_NoError(t, err) + + state := fs.SourcesState() + require_Len(t, len(state), 1) + require_Equal(t, state["ORIGIN1 > >"].Seq, 5) + require_Equal(t, state["ORIGIN1 > >"].Ident, "IDENT1") + + var tracked int + if fss, ok := fs.(*fileStore); ok { + fss.mu.RLock() + tracked = len(fss.sources) + fss.mu.RUnlock() + } else if mss, ok := fs.(*memStore); ok { + mss.mu.RLock() + tracked = len(mss.sources) + mss.mu.RUnlock() + } else { + t.Fatal("unknown store") + } + require_Equal(t, tracked, 1) + + // A new message should update both the sequence and identity. + h4 := genHeader(nil, JSStreamSource, "ORIGIN1 2 > > foo.a IDENT3") + _, _, err = fs.StoreMsg("foo.a", h4, nil, 0) + require_NoError(t, err) + state = fs.SourcesState() + require_Len(t, len(state), 1) + require_Equal(t, state["ORIGIN1 > >"].Seq, 2) + require_Equal(t, state["ORIGIN1 > >"].Ident, "IDENT3") + }, + ) +} diff --git a/server/stream_backup.go b/server/stream_backup.go new file mode 100644 index 0000000..646ac91 --- /dev/null +++ b/server/stream_backup.go @@ -0,0 +1,574 @@ +// Copyright 2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net" + "path" + "slices" + "strings" + "time" + + "github.com/klauspost/compress/s2" + "github.com/nats-io/nats-server/v2/server/archive" +) + +type SnapshotConsumerState struct { + *ConsumerConfig `json:"config"` + *ConsumerState `json:"state"` +} + +// Create a snapshot of this stream and its consumer's state along with messages. +// sa is passed in when the stream is clustered, so we can find child consumer assignments. +func (js *jetStream) CreateStreamSnapshotV2(store StreamStore, deadline time.Duration, includeConsumers bool, sa *streamAssignment) (*SnapshotResult, error) { + pr, pw := net.Pipe() + + // Set a write deadline here to protect ourselves. + if deadline > 0 { + pw.SetWriteDeadline(time.Now().Add(deadline)) + } + + // We can add to our stream while snapshotting but not "user" delete anything. + var state StreamState + store.FastState(&state) + + // Stream in separate Go routine. + errCh := make(chan error, 1) + go js.streamSnapshotV2(store, &state, pw, includeConsumers, sa, errCh) + + return &SnapshotResult{pr, state, errCh}, nil +} + +// Stream our snapshot through S2 compression and the custom archive format. +func (js *jetStream) streamSnapshotV2(store StreamStore, state *StreamState, w io.WriteCloser, includeConsumers bool, sa *streamAssignment, errCh chan error) { + defer close(errCh) + defer w.Close() + + enc := s2.NewWriter(w) + defer enc.Close() + + tw := archive.NewWriter(enc) + defer tw.Close() + + now := time.Now() + clustered := js.isClustered() + + writeGeneric := func(name string, mod int64, seq uint64, headerSize, payloadSize int64, buf []byte) error { + hdr := &archive.Header{ + Name: name, + Timestamp: mod, + Sequence: seq, + HeaderSize: headerSize, + PayloadSize: payloadSize, + } + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if _, err := tw.Write(buf); err != nil { + return err + } + // Need to wait for flush here as the tar/s2 writer is handing off to a + // flow-controlled publisher, it's important that we handle backpressure. + return tw.Flush() + } + + writeStoreMsg := func(msg *StoreMsg) error { + return writeGeneric( + msg.subj, + msg.ts, + msg.seq, + int64(len(msg.hdr)), + int64(len(msg.msg)), + msg.buf, + ) + } + + writeConsumerMsg := func(scs SnapshotConsumerState) error { + // Bound the consumer state to the stream snapshot. Consumer sequence + // numbers are left intact since filtered consumers do not have a + // one-to-one mapping between consumer and stream sequences. + scs.Delivered.Stream = min(scs.Delivered.Stream, state.LastSeq) + scs.AckFloor.Stream = min(scs.AckFloor.Stream, state.LastSeq) + for seq := range scs.Pending { + if seq > state.LastSeq { + delete(scs.Pending, seq) + } + } + for seq := range scs.Redelivered { + if seq > state.LastSeq { + delete(scs.Redelivered, seq) + } + } + ssj, err := json.Marshal(scs) + if err != nil { + return err + } + return writeGeneric( + path.Join("consumers", scs.Name), + now.UnixNano(), + 0, + 0, + int64(len(ssj)), + ssj, + ) + } + + // If we aren't including consumers here then make sure the consumer count + // is set accordingly, this helps on the restore path. + var consumerAssignments map[string]*consumerAssignment + var consumerStores []ConsumerStore + var streamState = *state + if !includeConsumers { + streamState.Consumers = 0 + } else if clustered { + if sa == nil { + errCh <- errors.New("stream assignment not present in clustered mode") + return + } + js.mu.RLock() + consumerAssignments = make(map[string]*consumerAssignment, len(sa.consumers)) + for name, ca := range sa.consumers { + consumerAssignments[name] = ca.copyGroup() + } + streamState.Consumers = len(consumerAssignments) + js.mu.RUnlock() + } else { + consumerStores = slices.Collect(store.Consumers()) + streamState.Consumers = len(consumerStores) + } + + ssj, err := json.Marshal(streamState) + if err != nil { + errCh <- err + return + } + if err := writeGeneric("state.json", now.UnixNano(), 0, 0, int64(len(ssj)), ssj); err != nil { + errCh <- err + return + } + + // Do consumers first, if the stream is interest/WQ then this may be + // important for message retention. + if includeConsumers { + consumerStateFromInfo := func(ci *ConsumerInfo) *ConsumerState { + state := &ConsumerState{ + Delivered: SequencePair{ + Consumer: ci.Delivered.Consumer, + Stream: ci.Delivered.Stream, + }, + AckFloor: SequencePair{ + Consumer: ci.AckFloor.Consumer, + Stream: ci.AckFloor.Stream, + }, + } + if ci.NumAckPending > 0 { + // Cluster consumer info does not include the sparse pending state. + // Roll back to the ack floor so all possibly unacknowledged messages + // are delivered again instead of guessing which ones are pending. + state.Delivered = state.AckFloor + } + return state + } + + if clustered { + for _, ca := range consumerAssignments { + ci, err := sysRequest[ConsumerInfo](js.srv, clusterConsumerInfoT, sa.Client.serviceAccount(), sa.Config.Name, ca.Name) + if err != nil || ci == nil { + errCh <- fmt.Errorf("failed to get consumer state for '%s > %s'", sa.Config.Name, ca.Name) + return + } + if err := writeConsumerMsg(SnapshotConsumerState{ + ConsumerConfig: ca.Config, + ConsumerState: consumerStateFromInfo(ci), + }); err != nil { + errCh <- err + return + } + } + } else { + for _, o := range consumerStores { + config := o.GetConfig() + state, err := o.State() + if err != nil { + errCh <- fmt.Errorf("couldn't load consumer '%s' state: %s", config.Name, err) + return + } + if err := writeConsumerMsg(SnapshotConsumerState{ + ConsumerConfig: config, + ConsumerState: state, + }); err != nil { + errCh <- err + return + } + } + } + } + + var sm StoreMsg + for seq := state.FirstSeq - 1; seq < state.LastSeq; { + if _, seq, err = store.LoadNextMsg(fwcs, true, seq+1, &sm); err != nil { + if err == ErrStoreEOF { + break + } + errCh <- fmt.Errorf("couldn't load next message after seq %d: %s", seq+1, err) + return + } + // Concurrent deletions and publishes can move the next available + // message beyond the snapshot's original sequence range. + if seq > state.LastSeq { + break + } + if err = writeStoreMsg(&sm); err != nil { + errCh <- err + return + } + } + + // End of backup sentinel. A clear marker makes it obvious when + // a backup has been truncated or not without having to count + // messages or from first/last sequence, which may not be possible + // during the rewrite of a large stream backup. + if err = writeGeneric(_EMPTY_, 0, 0, 0, 0, nil); err != nil { + errCh <- err + } +} + +// RestoreStreamSnapshotV2 will restore a stream from a snapshot. +func (a *Account) RestoreStreamV2(ncfg *StreamConfig, r io.Reader) (retMset *stream, retErr error) { + dec := s2.NewReader(r) + tr := archive.NewReader(dec) + + var nstate StreamState + + // Load the stream state. + hdr, err := tr.Next() + if err != nil { + return nil, err + } + if hdr.Name != "state.json" { + return nil, fmt.Errorf("expected state.json first") + } + if err := json.NewDecoder(tr).Decode(&nstate); err != nil { + return nil, fmt.Errorf("error in state.json: %w", err) + } + if nstate.Consumers < 0 { + return nil, fmt.Errorf("invalid stream state: negative consumer count") + } + if nstate.FirstSeq == 0 { + if nstate.LastSeq != 0 || nstate.Msgs != 0 { + return nil, fmt.Errorf("invalid stream state: inconsistent message count or sequence range") + } + } else if nstate.FirstSeq > nstate.LastSeq { + // An empty, advanced stream has FirstSeq exactly one past LastSeq. + // Express this by subtracting from FirstSeq so a corrupt LastSeq of + // MaxUint64 can not overflow here. + if nstate.FirstSeq-1 != nstate.LastSeq || nstate.Msgs != 0 { + return nil, fmt.Errorf("invalid stream state: inconsistent message count or sequence range") + } + } else if nstate.Msgs > nstate.LastSeq-nstate.FirstSeq+1 { + return nil, fmt.Errorf("invalid stream state: message count exceeds sequence range") + } + + s, jsa, err := a.checkForJetStream() + if err != nil { + return nil, err + } + js := jsa.js + if js == nil { + return nil, NewJSNotEnabledForAccountError() + } + if _, err := a.lookupStream(ncfg.Name); err == nil { + return nil, NewJSStreamNameExistRestoreFailedError() + } + + cfg, apiErr := s.checkStreamCfg(ncfg, a, false) + if apiErr != nil { + return nil, apiErr + } + + _, isClustered := jsa.jetStreamAndClustered() + jsa.usageMu.RLock() + selected, tier, hasTier := jsa.selectLimits(cfg.Replicas) + jsa.usageMu.RUnlock() + reserved := int64(0) + if hasTier { + if isClustered { + js.mu.RLock() + _, reserved = js.tieredStreamAndReservationCount(a.Name, tier, &cfg) + js.mu.RUnlock() + } else { + reserved = jsa.tieredReservation(tier, &cfg) + } + } + + // Hold the full restore footprint as usage before creating the stream. As + // messages are stored, this temporary reservation is converted into actual + // usage so other streams cannot consume the capacity needed by the restore. + restoreBytes := nstate.Bytes + if restoreBytes > math.MaxInt64 { + if cfg.Storage == MemoryStorage { + return nil, NewJSMemoryResourcesExceededError() + } + return nil, NewJSStorageResourcesExceededError() + } + restoreRemaining := int64(restoreBytes) + jsa.updateUsage(tier, cfg.Storage, restoreRemaining) + defer func() { + // Inside a defer func() so that restoreRemaining is captured later. + jsa.updateUsage(tier, cfg.Storage, -restoreRemaining) + }() + releaseRestoreBytes := func(bytes int64) { + bytes = min(bytes, restoreRemaining) + jsa.updateUsage(tier, cfg.Storage, -bytes) + restoreRemaining -= bytes + } + checkUsageLimits := func() error { + lr := int64(max(1, cfg.Replicas)) + if tier == _EMPTY_ { + lr = 1 + } + jsa.usageMu.RLock() + usage := jsa.usage[tier] + var used, limit int64 + if cfg.Storage == MemoryStorage { + limit = selected.MaxMemory + if usage != nil { + used = usage.total.mem + } + } else { + limit = selected.MaxStore + if usage != nil { + used = usage.total.store + } + } + jsa.usageMu.RUnlock() + if limit >= 0 && used > mulSaturate(limit, lr) { + if cfg.Storage == MemoryStorage { + return NewJSMemoryResourcesExceededError() + } + return NewJSStorageResourcesExceededError() + } + if js.limitsExceeded(cfg.Storage) { + if cfg.Storage == MemoryStorage { + return NewJSMemoryResourcesExceededError() + } + return NewJSStorageResourcesExceededError() + } + return nil + } + + reserveCfg := cfg + reserveCfg.MaxBytes = max(reserveCfg.MaxBytes, restoreRemaining) + js.mu.RLock() + err = js.checkAllLimits(&selected, tier, &reserveCfg, reserved, 0) + js.mu.RUnlock() + if err != nil { + return nil, err + } + if err := checkUsageLimits(); err != nil { + return nil, err + } + + mset, err := a.addStreamForRestore(&cfg) + if err != nil { + return nil, fmt.Errorf("error adding stream: %w", err) + } + defer func() { + var state StreamState + mset.store.FastState(&state) + mset.mu.Lock() + mset.lseq = state.LastSeq + mset.mu.Unlock() + if err := mset.completeRestore(); err != nil { + if err = fmt.Errorf("failed to activate stream %q: %w", cfg.Name, err); retErr == nil { + retErr = err + } + s.Warnf("JetStream stream restore for '%s > %s' failed to activate stream: %v", a.Name, cfg.Name, err) + } + }() + + // Start off at the right sequence number. This is important in particular + // when the backup contains no messages or would restore to no interest. + if _, err = mset.store.Compact(nstate.FirstSeq); err != nil { + return nil, fmt.Errorf("error purging stream: %w", err) + } + + var restoredConsumers, ephemerals []*consumer + defer func() { + // Consumers must be unconditionally converted and completed, otherwise + // a partial restore that fails midway through can leave assets that are + // unusable. + for _, o := range ephemerals { + o.switchToEphemeral() + } + for _, o := range restoredConsumers { + if err := o.completeRestore(); err != nil { + if err = fmt.Errorf("failed to activate consumer %q: %w", o.name, err); retErr == nil { + retErr = err + } + s.Warnf("JetStream stream restore for '%s > %s' failed to activate consumers: %v", a.Name, cfg.Name, err) + } + } + }() + for range nstate.Consumers { + hdr, err := tr.Next() + if err != nil { + return nil, err + } + name, found := strings.CutPrefix(hdr.Name, "consumers/") + if !found { + return nil, fmt.Errorf("expected consumer, found %q", hdr.Name) + } + var consumer SnapshotConsumerState + if err := json.NewDecoder(tr).Decode(&consumer); err != nil { + return nil, fmt.Errorf("failed to decode consumer %q state: %w", name, err) + } + if consumer.ConsumerConfig == nil { + return nil, fmt.Errorf("consumer %q is missing config", name) + } + if consumer.ConsumerState == nil { + return nil, fmt.Errorf("consumer %q is missing state", name) + } + isEphemeral := !isDurableConsumer(consumer.ConsumerConfig) + if isEphemeral { + // Keep ephemerals alive and interested until all messages have + // been restored, then start their normal inactivity lifecycle. + consumer.Durable = name + } + o, err := mset.addConsumerForRestore(consumer.ConsumerConfig) + if err != nil { + return nil, fmt.Errorf("failed to add consumer %q: %w", name, err) + } + if isEphemeral { + ephemerals = append(ephemerals, o) + } + restoredConsumers = append(restoredConsumers, o) + o.mu.Lock() + err = o.setStoreState(consumer.ConsumerState) + o.mu.Unlock() + if err != nil { + return nil, fmt.Errorf("failed to set consumer %q state: %w", name, err) + } + } + + store := mset.store + lseq := nstate.FirstSeq - 1 + eob := false + mp := int64(s.getOpts().MaxPayload) + for { + hdr, err := tr.Next() + if err != nil { + return nil, err + } + seq := hdr.Sequence + if seq == 0 { + // Sentinel "end of backup" if all fields are zero. + if hdr.Name == _EMPTY_ && hdr.Timestamp == 0 && hdr.HeaderSize == 0 && hdr.PayloadSize == 0 { + eob = true + break + } + return nil, fmt.Errorf("expected message sequence") + } + if hdr.HeaderSize < 0 || hdr.PayloadSize < 0 || hdr.PayloadSize > math.MaxInt64-hdr.HeaderSize { + return nil, fmt.Errorf("invalid message lengths for sequence %d", seq) + } + declaredSize := hdr.HeaderSize + hdr.PayloadSize + if hdr.HeaderSize > mp || hdr.PayloadSize > mp-hdr.HeaderSize { + return nil, fmt.Errorf("message sequence %d exceeds maximum payload size", seq) + } + if mms := int64(cfg.MaxMsgSize); mms >= 0 && (hdr.HeaderSize > mms || hdr.PayloadSize > mms-hdr.HeaderSize) { + return nil, fmt.Errorf("message sequence %d exceeds maximum message size", seq) + } + var storedSizeRaw uint64 + switch cfg.Storage { + case MemoryStorage: + if storedSizeRaw = memStoreMsgSizeRaw(len(hdr.Name), int(hdr.HeaderSize), int(hdr.PayloadSize)); storedSizeRaw > math.MaxInt64 { + return nil, fmt.Errorf("snapshot message bytes exceed maximum store message size") + } + default: + if storedSizeRaw = fileStoreMsgSizeRaw(len(hdr.Name), int(hdr.HeaderSize), int(hdr.PayloadSize)); storedSizeRaw > rlBadThresh { + return nil, fmt.Errorf("snapshot message bytes exceed maximum store message size") + } + } + storedSize := int64(storedSizeRaw) + if additional := storedSize - restoreRemaining; additional > 0 { + jsa.updateUsage(tier, cfg.Storage, additional) + restoreRemaining += additional + if err := checkUsageLimits(); err != nil { + return nil, err + } + } + buf, err := io.ReadAll(io.LimitReader(tr, declaredSize)) + if err != nil { + return nil, fmt.Errorf("failed to read message sequence %d: %w", seq, err) + } + if hdr.HeaderSize > int64(len(buf)) { + return nil, fmt.Errorf("failed to parse message sequence %d: invalid header length", seq) + } + if int64(len(buf)) != declaredSize { + return nil, fmt.Errorf("failed to read message sequence %d: unexpected payload size", seq) + } + subj := hdr.Name + mhdr := buf[:hdr.HeaderSize] + msg := buf[hdr.HeaderSize : hdr.HeaderSize+hdr.PayloadSize] + if seq <= lseq { + return nil, fmt.Errorf("message sequence %d out of order", seq) + } + // We could have deleted messages since the last message we stored, if so + // we should work out what the gap is and skip those sequences. + if gap := seq - lseq - 1; gap > 0 { + if err := store.SkipMsgs(lseq+1, gap); err != nil { + return nil, fmt.Errorf("failed to process gap: %w", err) + } + } + lseq = seq + ttl, err := getMessageTTL(mhdr) + if err != nil { + return nil, fmt.Errorf("failed to parse message TTL: %w", err) + } + hdrTime := time.Unix(0, hdr.Timestamp) + if ttl > 0 && time.Now().After(hdrTime.Add(time.Duration(ttl)*time.Second)) { + // If the TTL has exceeded then there isn't much point in storing the message, + // but we still need to preserve the sequence. + if err := store.SkipMsgs(seq, 1); err != nil { + return nil, fmt.Errorf("failed to process expired message sequence %d: %w", seq, err) + } + releaseRestoreBytes(storedSize) + continue + } + if err = store.StoreRawMsg(subj, mhdr, msg, seq, hdr.Timestamp, ttl, false); err != nil { + return nil, fmt.Errorf("failed to store message sequence %d: %w", seq, err) + } + releaseRestoreBytes(storedSize) + } + + if !eob { + return mset, fmt.Errorf("backup was truncated") + } + + // Need to make sure that we pad out with skip msgs to preserve the last + // sequence, otherwise trailing deleted messages could reuse sequence numbers. + if lseq < nstate.LastSeq { + gap := nstate.LastSeq - lseq + if err := store.SkipMsgs(lseq+1, gap); err != nil { + return nil, fmt.Errorf("failed to process trailing gap: %w", err) + } + } + return mset, nil +} diff --git a/server/stree/dump.go b/server/stree/dump.go new file mode 100644 index 0000000..12c62f3 --- /dev/null +++ b/server/stree/dump.go @@ -0,0 +1,70 @@ +// Copyright 2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stree + +import ( + "fmt" + "io" + "strings" +) + +// For dumping out a text representation of a tree. +func (t *SubjectTree[T]) Dump(w io.Writer) { + t.dump(w, t.root, 0) + fmt.Fprintln(w) +} + +// Will dump out a node. +func (t *SubjectTree[T]) dump(w io.Writer, n node, depth int) { + if n == nil { + fmt.Fprintf(w, "EMPTY\n") + return + } + if n.isLeaf() { + leaf := n.(*leaf[T]) + fmt.Fprintf(w, "%s LEAF: Suffix: %q Value: %+v\n", dumpPre(depth), leaf.suffix, leaf.value) + n = nil + } else { + // We are a node type here, grab meta portion. + bn := n.base() + fmt.Fprintf(w, "%s %s Prefix: %q\n", dumpPre(depth), n.kind(), bn.prefix) + depth++ + n.iter(func(n node) bool { + t.dump(w, n, depth) + return true + }) + } +} + +// For individual node/leaf dumps. +func (n *leaf[T]) kind() string { return "LEAF" } +func (n *node4) kind() string { return "NODE4" } +func (n *node10) kind() string { return "NODE10" } +func (n *node16) kind() string { return "NODE16" } +func (n *node48) kind() string { return "NODE48" } +func (n *node256) kind() string { return "NODE256" } + +// Calculates the indendation, etc. +func dumpPre(depth int) string { + if depth == 0 { + return "-- " + } else { + var b strings.Builder + for i := 0; i < depth; i++ { + b.WriteString(" ") + } + b.WriteString("|__ ") + return b.String() + } +} diff --git a/server/stree/helper_test.go b/server/stree/helper_test.go new file mode 100644 index 0000000..ab343ab --- /dev/null +++ b/server/stree/helper_test.go @@ -0,0 +1,62 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stree + +import ( + "testing" + + "github.com/nats-io/nats-server/v2/internal/antithesis" +) + +func require_True(t testing.TB, b bool) { + t.Helper() + if !b { + antithesis.AssertUnreachable(t, "Failed require_True check", nil) + t.Fatalf("require true, but got false") + } +} + +func require_False(t testing.TB, b bool) { + t.Helper() + if b { + antithesis.AssertUnreachable(t, "Failed require_False check", nil) + t.Fatalf("require false, but got true") + } +} + +func require_NoError(t testing.TB, err error) { + t.Helper() + if err != nil { + antithesis.AssertUnreachable(t, "Failed require_NoError check", map[string]any{ + "error": err.Error(), + }) + t.Fatalf("require no error, but got: %v", err) + } +} + +func require_Equal[T comparable](t testing.TB, a, b T) { + t.Helper() + if a != b { + antithesis.AssertUnreachable(t, "Failed require_Equal check", nil) + t.Fatalf("require %T equal, but got: %v != %v", a, a, b) + } +} + +func require_Len(t testing.TB, a, b int) { + t.Helper() + if a != b { + antithesis.AssertUnreachable(t, "Failed require_Len check", nil) + t.Fatalf("require len, but got: %v != %v", a, b) + } +} diff --git a/server/stree/leaf.go b/server/stree/leaf.go new file mode 100644 index 0000000..08f5e9f --- /dev/null +++ b/server/stree/leaf.go @@ -0,0 +1,46 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stree + +// Leaf node +// Order of struct fields for best memory alignment (as per govet/fieldalignment) +type leaf[T any] struct { + value T + // This could be the whole subject, but most likely just the suffix portion. + // We will only store the suffix here and assume all prior prefix paths have + // been checked once we arrive at this leafnode. + suffix string +} + +func newLeaf[T any](suffix []byte, value T) *leaf[T] { + return &leaf[T]{value, string(suffix)} +} + +func (n *leaf[T]) isLeaf() bool { return true } +func (n *leaf[T]) base() *meta { return nil } +func (n *leaf[T]) match(subject []byte) bool { return string(subject) == n.suffix } +func (n *leaf[T]) setSuffix(suffix []byte) { n.suffix = string(suffix) } +func (n *leaf[T]) isFull() bool { return true } +func (n *leaf[T]) iter(f func(node) bool) {} +func (n *leaf[T]) children() []node { return nil } +func (n *leaf[T]) numChildren() uint16 { return 0 } +func (n *leaf[T]) path() string { return n.suffix } + +// Not applicable to leafs and should not be called, so panic if we do. +func (n *leaf[T]) setPrefix(pre []byte) { panic("setPrefix called on leaf") } +func (n *leaf[T]) addChild(_ byte, _ node) { panic("addChild called on leaf") } +func (n *leaf[T]) findChild(_ byte) *node { panic("findChild called on leaf") } +func (n *leaf[T]) grow() node { panic("grow called on leaf") } +func (n *leaf[T]) deleteChild(_ byte) { panic("deleteChild called on leaf") } +func (n *leaf[T]) shrink() node { panic("shrink called on leaf") } diff --git a/server/stree/node.go b/server/stree/node.go new file mode 100644 index 0000000..5260768 --- /dev/null +++ b/server/stree/node.go @@ -0,0 +1,47 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stree + +// Internal node interface. +type node interface { + isLeaf() bool + base() *meta + setPrefix(pre []byte) + addChild(c byte, n node) + findChild(c byte) *node + deleteChild(c byte) + isFull() bool + grow() node + shrink() node + kind() string + iter(f func(node) bool) + children() []node + numChildren() uint16 + path() string +} + +type meta struct { + prefix string + size uint16 +} + +func (n *meta) isLeaf() bool { return false } +func (n *meta) base() *meta { return n } + +func (n *meta) setPrefix(pre []byte) { + n.prefix = string(pre) +} + +func (n *meta) numChildren() uint16 { return n.size } +func (n *meta) path() string { return n.prefix } diff --git a/server/stree/node10.go b/server/stree/node10.go new file mode 100644 index 0000000..c02e797 --- /dev/null +++ b/server/stree/node10.go @@ -0,0 +1,107 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stree + +// Node with 10 children +// This node size is for the particular case that a part of the subject is numeric +// in nature, i.e. it only needs to satisfy the range 0-9 without wasting bytes +// Order of struct fields for best memory alignment (as per govet/fieldalignment) +type node10 struct { + child [10]node + meta + key [10]byte +} + +func newNode10(prefix []byte) *node10 { + nn := &node10{} + nn.setPrefix(prefix) + return nn +} + +// Currently we do not keep node10 sorted or use bitfields for traversal so just add to the end. +// TODO(dlc) - We should revisit here with more detailed benchmarks. +func (n *node10) addChild(c byte, nn node) { + if n.size >= 10 { + panic("node10 full!") + } + n.key[n.size] = c + n.child[n.size] = nn + n.size++ +} + +func (n *node10) findChild(c byte) *node { + for i := uint16(0); i < n.size; i++ { + if n.key[i] == c { + return &n.child[i] + } + } + return nil +} + +func (n *node10) isFull() bool { return n.size >= 10 } + +func (n *node10) grow() node { + nn := &node16{} + nn.prefix = n.prefix + for i := 0; i < 10; i++ { + nn.addChild(n.key[i], n.child[i]) + } + return nn +} + +// Deletes a child from the node. +func (n *node10) deleteChild(c byte) { + for i, last := uint16(0), n.size-1; i < n.size; i++ { + if n.key[i] == c { + // Unsorted so just swap in last one here, else nil if last. + if i < last { + n.key[i] = n.key[last] + n.child[i] = n.child[last] + n.key[last] = 0 + n.child[last] = nil + } else { + n.key[i] = 0 + n.child[i] = nil + } + n.size-- + return + } + } +} + +// Shrink if needed and return new node, otherwise return nil. +func (n *node10) shrink() node { + if n.size > 4 { + return nil + } + nn := newNode4(nil) + for i := uint16(0); i < n.size; i++ { + nn.addChild(n.key[i], n.child[i]) + } + return nn +} + +// Iterate over all children calling func f. +func (n *node10) iter(f func(node) bool) { + for i := uint16(0); i < n.size; i++ { + if !f(n.child[i]) { + return + } + } +} + +// Return our children as a slice. +func (n *node10) children() []node { + return n.child[:n.size] +} diff --git a/server/stree/node16.go b/server/stree/node16.go new file mode 100644 index 0000000..ba22c9a --- /dev/null +++ b/server/stree/node16.go @@ -0,0 +1,105 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stree + +// Node with 16 children +// Order of struct fields for best memory alignment (as per govet/fieldalignment) +type node16 struct { + child [16]node + meta + key [16]byte +} + +func newNode16(prefix []byte) *node16 { + nn := &node16{} + nn.setPrefix(prefix) + return nn +} + +// Currently we do not keep node16 sorted or use bitfields for traversal so just add to the end. +// TODO(dlc) - We should revisit here with more detailed benchmarks. +func (n *node16) addChild(c byte, nn node) { + if n.size >= 16 { + panic("node16 full!") + } + n.key[n.size] = c + n.child[n.size] = nn + n.size++ +} + +func (n *node16) findChild(c byte) *node { + for i := uint16(0); i < n.size; i++ { + if n.key[i] == c { + return &n.child[i] + } + } + return nil +} + +func (n *node16) isFull() bool { return n.size >= 16 } + +func (n *node16) grow() node { + nn := &node48{} + nn.prefix = n.prefix + for i := 0; i < 16; i++ { + nn.addChild(n.key[i], n.child[i]) + } + return nn +} + +// Deletes a child from the node. +func (n *node16) deleteChild(c byte) { + for i, last := uint16(0), n.size-1; i < n.size; i++ { + if n.key[i] == c { + // Unsorted so just swap in last one here, else nil if last. + if i < last { + n.key[i] = n.key[last] + n.child[i] = n.child[last] + n.key[last] = 0 + n.child[last] = nil + } else { + n.key[i] = 0 + n.child[i] = nil + } + n.size-- + return + } + } +} + +// Shrink if needed and return new node, otherwise return nil. +func (n *node16) shrink() node { + if n.size > 10 { + return nil + } + nn := newNode10(nil) + for i := uint16(0); i < n.size; i++ { + nn.addChild(n.key[i], n.child[i]) + } + return nn +} + +// Iterate over all children calling func f. +func (n *node16) iter(f func(node) bool) { + for i := uint16(0); i < n.size; i++ { + if !f(n.child[i]) { + return + } + } +} + +// Return our children as a slice. +func (n *node16) children() []node { + return n.child[:n.size] +} diff --git a/server/stree/node256.go b/server/stree/node256.go new file mode 100644 index 0000000..5d08b14 --- /dev/null +++ b/server/stree/node256.go @@ -0,0 +1,80 @@ +// Copyright 2023-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stree + +// Node with 256 children +// Order of struct fields for best memory alignment (as per govet/fieldalignment) +type node256 struct { + child [256]node + meta +} + +func newNode256(prefix []byte) *node256 { + nn := &node256{} + nn.setPrefix(prefix) + return nn +} + +func (n *node256) addChild(c byte, nn node) { + n.child[c] = nn + n.size++ +} + +func (n *node256) findChild(c byte) *node { + if n.child[c] != nil { + return &n.child[c] + } + return nil +} + +func (n *node256) isFull() bool { return false } +func (n *node256) grow() node { panic("grow can not be called on node256") } + +// Deletes a child from the node. +func (n *node256) deleteChild(c byte) { + if n.child[c] != nil { + n.child[c] = nil + n.size-- + } +} + +// Shrink if needed and return new node, otherwise return nil. +func (n *node256) shrink() node { + if n.size > 48 { + return nil + } + nn := newNode48(nil) + for c, child := range n.child { + if child != nil { + nn.addChild(byte(c), n.child[c]) + } + } + return nn +} + +// Iterate over all children calling func f. +func (n *node256) iter(f func(node) bool) { + for i := 0; i < 256; i++ { + if n.child[i] != nil { + if !f(n.child[i]) { + return + } + } + } +} + +// Return our children as a slice. +func (n *node256) children() []node { + return n.child[:256] +} diff --git a/server/stree/node4.go b/server/stree/node4.go new file mode 100644 index 0000000..90fa28e --- /dev/null +++ b/server/stree/node4.go @@ -0,0 +1,100 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stree + +// Node with 4 children +// Order of struct fields for best memory alignment (as per govet/fieldalignment) +type node4 struct { + child [4]node + meta + key [4]byte +} + +func newNode4(prefix []byte) *node4 { + nn := &node4{} + nn.setPrefix(prefix) + return nn +} + +// Currently we do not need to keep sorted for traversal so just add to the end. +func (n *node4) addChild(c byte, nn node) { + if n.size >= 4 { + panic("node4 full!") + } + n.key[n.size] = c + n.child[n.size] = nn + n.size++ +} + +func (n *node4) findChild(c byte) *node { + for i := uint16(0); i < n.size; i++ { + if n.key[i] == c { + return &n.child[i] + } + } + return nil +} + +func (n *node4) isFull() bool { return n.size >= 4 } + +func (n *node4) grow() node { + nn := &node10{} + nn.prefix = n.prefix + for i := 0; i < 4; i++ { + nn.addChild(n.key[i], n.child[i]) + } + return nn +} + +// Deletes a child from the node. +func (n *node4) deleteChild(c byte) { + for i, last := uint16(0), n.size-1; i < n.size; i++ { + if n.key[i] == c { + // Unsorted so just swap in last one here, else nil if last. + if i < last { + n.key[i] = n.key[last] + n.child[i] = n.child[last] + n.key[last] = 0 + n.child[last] = nil + } else { + n.key[i] = 0 + n.child[i] = nil + } + n.size-- + return + } + } +} + +// Shrink if needed and return new node, otherwise return nil. +func (n *node4) shrink() node { + if n.size == 1 { + return n.child[0] + } + return nil +} + +// Iterate over all children calling func f. +func (n *node4) iter(f func(node) bool) { + for i := uint16(0); i < n.size; i++ { + if !f(n.child[i]) { + return + } + } +} + +// Return our children as a slice. +func (n *node4) children() []node { + return n.child[:n.size] +} diff --git a/server/stree/node48.go b/server/stree/node48.go new file mode 100644 index 0000000..9c7b210 --- /dev/null +++ b/server/stree/node48.go @@ -0,0 +1,111 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stree + +// Node with 48 children +// Memory saving vs node256 comes from the fact that the child array is 16 bytes +// per `node` entry, so node256's 256*16=4096 vs node48's 256+(48*16)=1024 +// Note that `key` is effectively 1-indexed, as 0 means no entry, so offset by 1 +// Order of struct fields for best memory alignment (as per govet/fieldalignment) +type node48 struct { + child [48]node + meta + key [256]byte +} + +func newNode48(prefix []byte) *node48 { + nn := &node48{} + nn.setPrefix(prefix) + return nn +} + +func (n *node48) addChild(c byte, nn node) { + if n.size >= 48 { + panic("node48 full!") + } + n.child[n.size] = nn + n.key[c] = byte(n.size + 1) // 1-indexed + n.size++ +} + +func (n *node48) findChild(c byte) *node { + i := n.key[c] + if i == 0 { + return nil + } + return &n.child[i-1] +} + +func (n *node48) isFull() bool { return n.size >= 48 } + +func (n *node48) grow() node { + nn := &node256{} + nn.prefix = n.prefix + for c := 0; c < len(n.key); c++ { + if i := n.key[byte(c)]; i > 0 { + nn.addChild(byte(c), n.child[i-1]) + } + } + return nn +} + +// Deletes a child from the node. +func (n *node48) deleteChild(c byte) { + i := n.key[c] + if i == 0 { + return + } + i-- // Adjust for 1-indexing + last := byte(n.size - 1) + if i < last { + n.child[i] = n.child[last] + for ic := 0; ic < len(n.key); ic++ { + if n.key[byte(ic)] == last+1 { + n.key[byte(ic)] = i + 1 + break + } + } + } + n.child[last] = nil + n.key[c] = 0 + n.size-- +} + +// Shrink if needed and return new node, otherwise return nil. +func (n *node48) shrink() node { + if n.size > 16 { + return nil + } + nn := newNode16(nil) + for c := 0; c < len(n.key); c++ { + if i := n.key[byte(c)]; i > 0 { + nn.addChild(byte(c), n.child[i-1]) + } + } + return nn +} + +// Iterate over all children calling func f. +func (n *node48) iter(f func(node) bool) { + for _, c := range n.child { + if c != nil && !f(c) { + return + } + } +} + +// Return our children as a slice. +func (n *node48) children() []node { + return n.child[:n.size] +} diff --git a/server/stree/parts.go b/server/stree/parts.go new file mode 100644 index 0000000..e38e5b6 --- /dev/null +++ b/server/stree/parts.go @@ -0,0 +1,148 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stree + +import ( + "strings" +) + +// genParts will break a filter subject up into parts. +// We need to break this up into chunks based on wildcards, either pwc '*' or fwc '>'. +// We do not care about other tokens per se, just parts that are separated by wildcards with an optional end fwc. +func genParts(filter []byte, parts [][]byte) [][]byte { + var start int + for i, e := 0, len(filter)-1; i < len(filter); i++ { + if filter[i] == tsep { + // See if next token is pwc. Either internal or end pwc. + if i < e && filter[i+1] == pwc && (i+2 <= e && filter[i+2] == tsep || i+1 == e) { + if i > start { + parts = append(parts, filter[start:i+1]) + } + parts = append(parts, filter[i+1:i+2]) + i++ // Skip pwc + if i+2 <= e { + i++ // Skip next tsep from next part too. + } + start = i + 1 + } else if i < e && filter[i+1] == fwc && i+1 == e { + if i > start { + parts = append(parts, filter[start:i+1]) + } + parts = append(parts, filter[i+1:i+2]) + i++ // Skip fwc + start = i + 1 + } + } else if filter[i] == pwc || filter[i] == fwc { + // Wildcard must be at the start or preceded by tsep. + if prev := i - 1; prev >= 0 && filter[prev] != tsep { + continue + } + // Wildcard must be at the end or followed by tsep. + if next := i + 1; next == e || next < e && filter[next] != tsep { + continue + } + // Full wildcard must be terminal. + if filter[i] == fwc && i < e { + break + } + // We start with a pwc or fwc. + parts = append(parts, filter[i:i+1]) + if i+1 <= e { + i++ // Skip next tsep from next part too. + } + start = i + 1 + } + } + if start < len(filter) { + // Check to see if we need to eat a leading tsep. + if filter[start] == tsep { + start++ + } + parts = append(parts, filter[start:]) + } + return parts +} + +// Match our parts against a stored fragment, which could be a prefix for nodes +// or a suffix for leaves. +func matchParts(parts [][]byte, frag string) ([][]byte, bool) { + lf := len(frag) + if lf == 0 { + return parts, true + } + + var si int + lpi := len(parts) - 1 + + for i, part := range parts { + if si >= lf { + return parts[i:], true + } + lp := len(part) + // Check for pwc or fwc place holders. + if lp == 1 { + if part[0] == pwc { + index := strings.IndexByte(frag[si:], tsep) + // We are trying to match pwc and did not find our tsep. + // Will need to move to next node from caller. + if index < 0 { + if i == lpi { + return nil, true + } + return parts[i:], true + } + si += index + 1 + continue + } else if part[0] == fwc { + // If we are here we should be good. + return nil, true + } + } + end := min(si+lp, lf) + // If part is bigger then the remaining fragment, adjust to a portion on the part. + if si+lp > end { + // Frag is smaller then part itself. + part = part[:end-si] + } + if string(part) != frag[si:end] { + return parts, false + } + // If we still have a portion of the fragment left, update and continue. + if end < lf { + si = end + continue + } + // If we matched a partial, do not move past current part + // but update the part to what was consumed. This allows upper layers to continue. + if end < si+lp { + if end >= lf { + // Create a copy before modifying. Reuse slice capacity available at the + // end of the parts slice, since this saves us additional allocations. + lp := len(parts) + parts = append(parts[lp:], parts[:lp]...) + parts[i] = parts[i][lf-si:] + } else { + i++ + } + return parts[i:], true + } + if i == lpi { + return nil, true + } + // If we are here we are not the last part which means we have a wildcard + // gap, so we need to match anything up to next tsep. + si += len(part) + } + return parts, false +} diff --git a/server/stree/stree.go b/server/stree/stree.go new file mode 100644 index 0000000..c503adf --- /dev/null +++ b/server/stree/stree.go @@ -0,0 +1,552 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stree + +import ( + "bytes" + "slices" + "strings" + "unsafe" + + "github.com/nats-io/nats-server/v2/server/gsl" +) + +// SubjectTree is an adaptive radix trie (ART) for storing subject information on literal subjects. +// Will use dynamic nodes, path compression and lazy expansion. +// The reason this exists is to not only save some memory in our filestore but to greatly optimize matching +// a wildcard subject to certain members, e.g. consumer NumPending calculations. +type SubjectTree[T any] struct { + root node + size int +} + +// NewSubjectTree creates a new SubjectTree with values T. +func NewSubjectTree[T any]() *SubjectTree[T] { + return &SubjectTree[T]{} +} + +// Size returns the number of elements stored. +func (t *SubjectTree[T]) Size() int { + if t == nil { + return 0 + } + return t.size +} + +// Will empty out the tree, or if tree is nil create a new one. +func (t *SubjectTree[T]) Empty() *SubjectTree[T] { + if t == nil { + return NewSubjectTree[T]() + } + t.root, t.size = nil, 0 + return t +} + +// Insert a value into the tree. Will return if the value was updated and if so the old value. +func (t *SubjectTree[T]) Insert(subject []byte, value T) (*T, bool) { + if t == nil { + return nil, false + } + + // Make sure we never insert anything with a noPivot byte. + if bytes.IndexByte(subject, noPivot) >= 0 { + return nil, false + } + + old, updated := t.insert(&t.root, subject, value, 0) + if !updated { + t.size++ + } + return old, updated +} + +// Find will find the value and return it or false if it was not found. +func (t *SubjectTree[T]) Find(subject []byte) (*T, bool) { + if t == nil { + return nil, false + } + + var si int + for n := t.root; n != nil; { + if n.isLeaf() { + if ln := n.(*leaf[T]); ln.match(subject[si:]) { + return &ln.value, true + } + return nil, false + } + // We are a node type here, grab meta portion. + if bn := n.base(); len(bn.prefix) > 0 { + end := min(si+len(bn.prefix), len(subject)) + if string(subject[si:end]) != bn.prefix { + return nil, false + } + // Increment our subject index. + si += len(bn.prefix) + } + if an := n.findChild(pivot(subject, si)); an != nil { + n = *an + } else { + return nil, false + } + } + return nil, false +} + +// Delete will delete the item and return its value, or not found if it did not exist. +func (t *SubjectTree[T]) Delete(subject []byte) (*T, bool) { + if t == nil { + return nil, false + } + + val, deleted := t.delete(&t.root, subject, 0) + if deleted { + t.size-- + } + return val, deleted +} + +// Match will match against a subject that can have wildcards and invoke the callback func for each matched value. +func (t *SubjectTree[T]) Match(filter []byte, cb func(subject []byte, val *T)) { + if t == nil || t.root == nil || len(filter) == 0 || cb == nil { + return + } + // We need to break this up into chunks based on wildcards, either pwc '*' or fwc '>'. + var raw [32][]byte + parts := genParts(filter, raw[:0]) + var _pre [256]byte + t.match(t.root, parts, _pre[:0], func(subject []byte, val *T) bool { + cb(subject, val) + return true + }) +} + +// MatchUntil will match against a subject that can have wildcards and invoke +// the callback func for each matched value. +// Returning false from the callback will stop matching immediately. +// Returns true if matching ran to completion, false if callback stopped it early. +func (t *SubjectTree[T]) MatchUntil(filter []byte, cb func(subject []byte, val *T) bool) bool { + if t == nil || t.root == nil || len(filter) == 0 || cb == nil { + return true + } + // We need to break this up into chunks based on wildcards, either pwc '*' or fwc '>'. + var raw [32][]byte + parts := genParts(filter, raw[:0]) + var _pre [256]byte + return t.match(t.root, parts, _pre[:0], cb) +} + +// IterOrdered will walk all entries in the SubjectTree lexicographically. The callback can return false to terminate the walk. +func (t *SubjectTree[T]) IterOrdered(cb func(subject []byte, val *T) bool) { + if t == nil || t.root == nil { + return + } + var _pre [256]byte + t.iter(t.root, _pre[:0], true, cb) +} + +// IterFast will walk all entries in the SubjectTree with no guarantees of ordering. The callback can return false to terminate the walk. +func (t *SubjectTree[T]) IterFast(cb func(subject []byte, val *T) bool) { + if t == nil || t.root == nil { + return + } + var _pre [256]byte + t.iter(t.root, _pre[:0], false, cb) +} + +// Internal methods + +// Internal call to insert that can be recursive. +func (t *SubjectTree[T]) insert(np *node, subject []byte, value T, si int) (*T, bool) { + n := *np + if n == nil { + *np = newLeaf(subject, value) + return nil, false + } + if n.isLeaf() { + ln := n.(*leaf[T]) + if ln.match(subject[si:]) { + // Replace with new value. + old := ln.value + ln.value = value + return &old, true + } + // Here we need to split this leaf. + cpi := commonPrefixLen(stringToBytes(ln.suffix), subject[si:]) + nn := newNode4(subject[si : si+cpi]) + ln.setSuffix(stringToBytes(ln.suffix[cpi:])) + si += cpi + // Make sure we have different pivot, normally this will be the case unless we have overflowing prefixes. + if p := pivot(stringToBytes(ln.suffix), 0); cpi > 0 && si < len(subject) && p == subject[si] { + // We need to split the original leaf. Recursively call into insert. + t.insert(np, subject, value, si) + // Now add the update version of *np as a child to the new node4. + nn.addChild(p, *np) + } else { + // Can just add this new leaf as a sibling. + nl := newLeaf(subject[si:], value) + nn.addChild(pivot(stringToBytes(nl.suffix), 0), nl) + // Add back original. + nn.addChild(pivot(stringToBytes(ln.suffix), 0), ln) + } + *np = nn + return nil, false + } + + // Non-leaf nodes. + bn := n.base() + if len(bn.prefix) > 0 { + cpi := commonPrefixLen(stringToBytes(bn.prefix), subject[si:]) + if pli := len(bn.prefix); cpi >= pli { + // Move past this node. We look for an existing child node to recurse into. + // If one does not exist we can create a new leaf node. + si += pli + if nn := n.findChild(pivot(subject, si)); nn != nil { + return t.insert(nn, subject, value, si) + } + if n.isFull() { + n = n.grow() + *np = n + } + n.addChild(pivot(subject, si), newLeaf(subject[si:], value)) + return nil, false + } else { + // We did not match the prefix completely here. + // Calculate new prefix for this node. + prefix := subject[si : si+cpi] + si += len(prefix) + // We will insert a new node4 and attach our current node below after adjusting prefix. + nn := newNode4(prefix) + // Shift the prefix for our original node. + n.setPrefix(stringToBytes(bn.prefix[cpi:])) + nn.addChild(pivot(stringToBytes(bn.prefix), 0), n) + // Add in our new leaf. + nn.addChild(pivot(subject[si:], 0), newLeaf(subject[si:], value)) + // Update our node reference. + *np = nn + } + } else { + if nn := n.findChild(pivot(subject, si)); nn != nil { + return t.insert(nn, subject, value, si) + } + // No prefix and no matched child, so add in new leafnode as needed. + if n.isFull() { + n = n.grow() + *np = n + } + n.addChild(pivot(subject, si), newLeaf(subject[si:], value)) + } + + return nil, false +} + +// internal function to recursively find the leaf to delete. Will do compaction if the item is found and removed. +func (t *SubjectTree[T]) delete(np *node, subject []byte, si int) (*T, bool) { + if t == nil || np == nil || *np == nil || len(subject) == 0 { + return nil, false + } + n := *np + if n.isLeaf() { + ln := n.(*leaf[T]) + if ln.match(subject[si:]) { + *np = nil + return &ln.value, true + } + return nil, false + } + // Not a leaf node. + if bn := n.base(); len(bn.prefix) > 0 { + // subject could be shorter and would panic on bad index into subject slice. + if len(subject) < si+len(bn.prefix) { + return nil, false + } + if string(subject[si:si+len(bn.prefix)]) != bn.prefix { + return nil, false + } + // Increment our subject index. + si += len(bn.prefix) + } + p := pivot(subject, si) + nna := n.findChild(p) + if nna == nil { + return nil, false + } + nn := *nna + if nn.isLeaf() { + ln := nn.(*leaf[T]) + if ln.match(subject[si:]) { + n.deleteChild(p) + + if sn := n.shrink(); sn != nil { + bn := n.base() + pre := bn.prefix + // Need to fix up prefixes/suffixes. + if sn.isLeaf() { + ln := sn.(*leaf[T]) + ln.suffix = pre + ln.suffix + } else { + // We are a node here, we need to add in the old prefix. + if len(pre) > 0 { + bsn := sn.base() + bsn.prefix = pre + bsn.prefix + } + } + *np = sn + } + + return &ln.value, true + } + return nil, false + } + return t.delete(nna, subject, si) +} + +// Internal function which can be called recursively to match all leaf nodes to a given filter subject which +// once here has been decomposed to parts. These parts only care about wildcards, both pwc and fwc. +// Returns false if the callback requested to stop matching. +func (t *SubjectTree[T]) match(n node, parts [][]byte, pre []byte, cb func(subject []byte, val *T) bool) bool { + // Capture if we are sitting on a terminal fwc. + var hasFWC bool + if lp := len(parts); lp > 0 && len(parts[lp-1]) > 0 && parts[lp-1][0] == fwc { + hasFWC = true + } + + for n != nil { + nparts, matched := matchParts(parts, n.path()) + // Check if we did not match. + if !matched { + return true + } + // We have matched here. If we are a leaf and have exhausted all parts or he have a FWC fire callback. + if n.isLeaf() { + if len(nparts) == 0 || (hasFWC && len(nparts) == 1) { + ln := n.(*leaf[T]) + if !cb(append(pre, ln.suffix...), &ln.value) { + return false + } + } + return true + } + // We have normal nodes here. + // We need to append our prefix + bn := n.base() + if len(bn.prefix) > 0 { + // Note that this append may reallocate, but it doesn't modify "pre" at the "match" callsite. + pre = append(pre, bn.prefix...) + } + + // Check our remaining parts. + if len(nparts) == 0 && !hasFWC { + // We are a node with no parts left and we are not looking at a fwc. + // We could have a leafnode with no suffix which would be a match. + // We could also have a terminal pwc. Check for those here. + var hasTermPWC bool + if lp := len(parts); lp > 0 && len(parts[lp-1]) == 1 && parts[lp-1][0] == pwc { + // If we are sitting on a terminal pwc, put the pwc back and continue. + nparts = parts[len(parts)-1:] + hasTermPWC = true + } + for _, cn := range n.children() { + if cn == nil { + continue + } + if cn.isLeaf() { + ln := cn.(*leaf[T]) + if len(ln.suffix) == 0 { + if !cb(append(pre, ln.suffix...), &ln.value) { + return false + } + } else if hasTermPWC && strings.IndexByte(ln.suffix, tsep) < 0 { + if !cb(append(pre, ln.suffix...), &ln.value) { + return false + } + } + } else if hasTermPWC { + // We have terminal pwc so call into match again with the child node. + if !t.match(cn, nparts, pre, cb) { + return false + } + } + } + // Return regardless. + return true + } + // If we are sitting on a terminal fwc, put back and continue. + if hasFWC && len(nparts) == 0 { + nparts = parts[len(parts)-1:] + } + + // Here we are a node type with a partial match. + // Check if the first part is a wildcard. + fp := nparts[0] + p := pivot(fp, 0) + // Check if we have a pwc/fwc part here. This will cause us to iterate. + if len(fp) == 1 && (p == pwc || p == fwc) { + // We need to iterate over all children here for the current node + // to see if we match further down. + for _, cn := range n.children() { + if cn != nil { + if !t.match(cn, nparts, pre, cb) { + return false + } + } + } + return true + } + // Here we have normal traversal, so find the next child. + nn := n.findChild(p) + if nn == nil { + return true + } + n, parts = *nn, nparts + } + return true +} + +// Internal iter function to walk nodes in lexicographical order. +func (t *SubjectTree[T]) iter(n node, pre []byte, ordered bool, cb func(subject []byte, val *T) bool) bool { + if n.isLeaf() { + ln := n.(*leaf[T]) + return cb(append(pre, ln.suffix...), &ln.value) + } + // We are normal node here. + bn := n.base() + // Note that this append may reallocate, but it doesn't modify "pre" at the "iter" callsite. + pre = append(pre, bn.prefix...) + // Not everything requires lexicographical sorting, so support a fast path for iterating in + // whatever order the stree has things stored instead. + if !ordered { + for _, cn := range n.children() { + if cn == nil { + continue + } + if !t.iter(cn, pre, false, cb) { + return false + } + } + return true + } + // Collect nodes since unsorted. + var _nodes [256]node + nodes := _nodes[:0] + for _, cn := range n.children() { + if cn != nil { + nodes = append(nodes, cn) + } + } + // Now sort. + slices.SortStableFunc(nodes, func(a, b node) int { return strings.Compare(a.path(), b.path()) }) + // Now walk the nodes in order and call into next iter. + for i := range nodes { + if !t.iter(nodes[i], pre, true, cb) { + return false + } + } + return true +} + +// LazyIntersect iterates the smaller of the two provided subject trees and +// looks for matching entries in the other. It is lazy in that it does not +// aggressively optimize against repeated walks, but is considerably faster +// in most cases than intersecting against a potentially large sublist. +func LazyIntersect[TL, TR any](tl *SubjectTree[TL], tr *SubjectTree[TR], cb func([]byte, *TL, *TR)) { + if tl == nil || tr == nil || tl.root == nil || tr.root == nil { + return + } + // Iterate over the smaller tree to reduce the number of rounds. + if tl.Size() <= tr.Size() { + tl.IterFast(func(key []byte, v1 *TL) bool { + if v2, ok := tr.Find(key); ok { + cb(key, v1, v2) + } + return true + }) + } else { + tr.IterFast(func(key []byte, v2 *TR) bool { + if v1, ok := tl.Find(key); ok { + cb(key, v1, v2) + } + return true + }) + } +} + +// IntersectGSL will match all items in the given subject tree that +// have interest expressed in the given sublist. The callback will only be called +// once for each subject, regardless of overlapping subscriptions in the sublist. +func IntersectGSL[T any, SL comparable](t *SubjectTree[T], sl *gsl.GenericSublist[SL], cb func(subject []byte, val *T) bool) { + if t == nil || t.root == nil || sl == nil { + return + } + var _pre [256]byte + _intersectGSL(t.root, _pre[:0], sl, cb) +} + +func _intersectGSL[T any, SL comparable](n node, pre []byte, sl *gsl.GenericSublist[SL], cb func(subject []byte, val *T) bool) bool { + if n.isLeaf() { + ln := n.(*leaf[T]) + subj := append(pre, ln.suffix...) + if sl.HasInterest(bytesToString(subj)) { + return cb(subj, &ln.value) + } + return true + } + bn := n.base() + pre = append(pre, bn.prefix...) + for _, cn := range n.children() { + if cn == nil { + continue + } + subj := append(pre, cn.path()...) + if !hasInterestForTokens(sl, subj, len(pre)) { + continue + } + if !_intersectGSL(cn, pre, sl, cb) { + return false + } + } + return true +} + +// The subject tree can return partial tokens so we need to check starting interest +// only from whole tokens when we encounter a tsep. +func hasInterestForTokens[SL comparable](sl *gsl.GenericSublist[SL], subj []byte, since int) bool { + for i := since; i < len(subj); i++ { + if subj[i] == tsep { + if !sl.HasInterestStartingIn(bytesToString(subj[:i])) { + return false + } + } + } + return true +} + +// Note this will avoid a copy of the data used for the string, but it will also reference the existing slice's data pointer. +// So this should be used sparingly when we know the encompassing byte slice's lifetime is the same. +func bytesToString(b []byte) string { + if len(b) == 0 { + return "" + } + p := unsafe.SliceData(b) + return unsafe.String(p, len(b)) +} + +// Note this will avoid a copy of the string data, but the returned slice must +// only be used for reading since strings are immutable. +func stringToBytes(s string) []byte { + if len(s) == 0 { + return nil + } + p := unsafe.StringData(s) + return unsafe.Slice(p, len(s)) +} diff --git a/server/stree/stree_test.go b/server/stree/stree_test.go new file mode 100644 index 0000000..79832b2 --- /dev/null +++ b/server/stree/stree_test.go @@ -0,0 +1,2252 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stree + +import ( + crand "crypto/rand" + "encoding/hex" + "flag" + "fmt" + "math/rand/v2" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server/gsl" +) + +// Print Results: go test -v --args --results +// For some benchmarks. +var runResults = flag.Bool("results", false, "Enable Results Tests") + +func TestSubjectTreeBasics(t *testing.T) { + st := NewSubjectTree[int]() + require_Equal(t, st.Size(), 0) + // Single leaf + old, updated := st.Insert(b("foo.bar.baz"), 22) + require_True(t, old == nil) + require_False(t, updated) + require_Equal(t, st.Size(), 1) + // Find shouldn't work with a wildcard. + _, found := st.Find(b("foo.bar.*")) + require_False(t, found) + // But it should with a literal. Find with single leaf. + v, found := st.Find(b("foo.bar.baz")) + require_True(t, found) + require_Equal(t, *v, 22) + // Update single leaf + old, updated = st.Insert(b("foo.bar.baz"), 33) + require_True(t, old != nil) + require_Equal(t, *old, 22) + require_True(t, updated) + require_Equal(t, st.Size(), 1) + // Split the tree + old, updated = st.Insert(b("foo.bar"), 22) + require_True(t, old == nil) + require_False(t, updated) + require_Equal(t, st.Size(), 2) + // Now we have node4 -> leaf*2 + v, found = st.Find(b("foo.bar")) + require_True(t, found) + require_Equal(t, *v, 22) + // Make sure we can still retrieve the original after the split. + v, found = st.Find(b("foo.bar.baz")) + require_True(t, found) + require_Equal(t, *v, 33) +} + +func TestSubjectTreeNodeGrow(t *testing.T) { + st := NewSubjectTree[int]() + for i := 0; i < 4; i++ { + subj := b(fmt.Sprintf("foo.bar.%c", 'A'+i)) + old, updated := st.Insert(subj, 22) + require_True(t, old == nil) + require_False(t, updated) + } + // We have filled a node4. + _, ok := st.root.(*node4) + require_True(t, ok) + // This one will trigger us to grow. + old, updated := st.Insert(b("foo.bar.E"), 22) + require_True(t, old == nil) + require_False(t, updated) + _, ok = st.root.(*node10) + require_True(t, ok) + for i := 5; i < 10; i++ { + subj := b(fmt.Sprintf("foo.bar.%c", 'A'+i)) + old, updated := st.Insert(subj, 22) + require_True(t, old == nil) + require_False(t, updated) + } + // This one will trigger us to grow. + old, updated = st.Insert(b("foo.bar.K"), 22) + require_True(t, old == nil) + require_False(t, updated) + // We have filled a node10. + _, ok = st.root.(*node16) + require_True(t, ok) + for i := 11; i < 16; i++ { + subj := b(fmt.Sprintf("foo.bar.%c", 'A'+i)) + old, updated := st.Insert(subj, 22) + require_True(t, old == nil) + require_False(t, updated) + } + // This one will trigger us to grow. + old, updated = st.Insert(b("foo.bar.Q"), 22) + require_True(t, old == nil) + require_False(t, updated) + _, ok = st.root.(*node48) + require_True(t, ok) + // Fill the node48. + for i := 17; i < 48; i++ { + subj := b(fmt.Sprintf("foo.bar.%c", 'A'+i)) + old, updated := st.Insert(subj, 22) + require_True(t, old == nil) + require_False(t, updated) + } + // This one will trigger us to grow. + subj := b(fmt.Sprintf("foo.bar.%c", 'A'+49)) + old, updated = st.Insert(subj, 22) + require_True(t, old == nil) + require_False(t, updated) + _, ok = st.root.(*node256) + require_True(t, ok) +} + +func TestSubjectTreeNodePrefixMismatch(t *testing.T) { + st := NewSubjectTree[int]() + st.Insert(b("foo.bar.A"), 11) + st.Insert(b("foo.bar.B"), 22) + st.Insert(b("foo.bar.C"), 33) + // Grab current root. Split below will cause update. + or := st.root + // This one will force a split of the node + st.Insert(b("foo.foo.A"), 44) + require_True(t, or != st.root) + // Now make sure we can retrieve correctly. + v, found := st.Find(b("foo.bar.A")) + require_True(t, found) + require_Equal(t, *v, 11) + v, found = st.Find(b("foo.bar.B")) + require_True(t, found) + require_Equal(t, *v, 22) + v, found = st.Find(b("foo.bar.C")) + require_True(t, found) + require_Equal(t, *v, 33) + v, found = st.Find(b("foo.foo.A")) + require_True(t, found) + require_Equal(t, *v, 44) +} + +func TestSubjectTreeNodeDelete(t *testing.T) { + st := NewSubjectTree[int]() + st.Insert(b("foo.bar.A"), 22) + v, found := st.Delete(b("foo.bar.A")) + require_True(t, found) + require_Equal(t, *v, 22) + require_Equal(t, st.root, nil) + v, found = st.Delete(b("foo.bar.A")) + require_False(t, found) + require_Equal(t, v, nil) + v, found = st.Find(b("foo.foo.A")) + require_False(t, found) + require_Equal(t, v, nil) + // Kick to a node4. + st.Insert(b("foo.bar.A"), 11) + st.Insert(b("foo.bar.B"), 22) + st.Insert(b("foo.bar.C"), 33) + // Make sure we can delete and that we shrink back to leaf. + v, found = st.Delete(b("foo.bar.C")) + require_True(t, found) + require_Equal(t, *v, 33) + v, found = st.Delete(b("foo.bar.B")) + require_True(t, found) + require_Equal(t, *v, 22) + // We should have shrunk here. + require_True(t, st.root.isLeaf()) + v, found = st.Delete(b("foo.bar.A")) + require_True(t, found) + require_Equal(t, *v, 11) + require_Equal(t, st.root, nil) + // Now pop up to a node10 and make sure we can shrink back down. + for i := 0; i < 5; i++ { + subj := fmt.Sprintf("foo.bar.%c", 'A'+i) + st.Insert(b(subj), 22) + } + _, ok := st.root.(*node10) + require_True(t, ok) + v, found = st.Delete(b("foo.bar.A")) + require_True(t, found) + require_Equal(t, *v, 22) + _, ok = st.root.(*node4) + require_True(t, ok) + // Now pop up to node16 + for i := 0; i < 11; i++ { + subj := fmt.Sprintf("foo.bar.%c", 'A'+i) + st.Insert(b(subj), 22) + } + _, ok = st.root.(*node16) + require_True(t, ok) + v, found = st.Delete(b("foo.bar.A")) + require_True(t, found) + require_Equal(t, *v, 22) + _, ok = st.root.(*node10) + require_True(t, ok) + v, found = st.Find(b("foo.bar.B")) + require_True(t, found) + require_Equal(t, *v, 22) + // Now pop up to node48 + st = NewSubjectTree[int]() + for i := 0; i < 17; i++ { + subj := fmt.Sprintf("foo.bar.%c", 'A'+i) + st.Insert(b(subj), 22) + } + _, ok = st.root.(*node48) + require_True(t, ok) + v, found = st.Delete(b("foo.bar.A")) + require_True(t, found) + require_Equal(t, *v, 22) + _, ok = st.root.(*node16) + require_True(t, ok) + v, found = st.Find(b("foo.bar.B")) + require_True(t, found) + require_Equal(t, *v, 22) + // Now pop up to node256 + st = NewSubjectTree[int]() + for i := 0; i < 49; i++ { + subj := fmt.Sprintf("foo.bar.%c", 'A'+i) + st.Insert(b(subj), 22) + } + _, ok = st.root.(*node256) + require_True(t, ok) + v, found = st.Delete(b("foo.bar.A")) + require_True(t, found) + require_Equal(t, *v, 22) + _, ok = st.root.(*node48) + require_True(t, ok) + v, found = st.Find(b("foo.bar.B")) + require_True(t, found) + require_Equal(t, *v, 22) +} + +func TestSubjectTreeNodesAndPaths(t *testing.T) { + st := NewSubjectTree[int]() + check := func(subj string) { + t.Helper() + v, found := st.Find(b(subj)) + require_True(t, found) + require_Equal(t, *v, 22) + } + st.Insert(b("foo.bar.A"), 22) + st.Insert(b("foo.bar.B"), 22) + st.Insert(b("foo.bar.C"), 22) + st.Insert(b("foo.bar"), 22) + check("foo.bar.A") + check("foo.bar.B") + check("foo.bar.C") + check("foo.bar") + // This will do several things in terms of shrinking and pruning, + // want to make sure it gets prefix correct for new top node4. + st.Delete(b("foo.bar")) + check("foo.bar.A") + check("foo.bar.B") + check("foo.bar.C") +} + +// Check that we are constructing a proper tree with complex insert patterns. +func TestSubjectTreeConstruction(t *testing.T) { + st := NewSubjectTree[int]() + st.Insert(b("foo.bar.A"), 1) + st.Insert(b("foo.bar.B"), 2) + st.Insert(b("foo.bar.C"), 3) + st.Insert(b("foo.baz.A"), 11) + st.Insert(b("foo.baz.B"), 22) + st.Insert(b("foo.baz.C"), 33) + st.Insert(b("foo.bar"), 42) + + checkNode := func(an *node, kind string, pors string, numChildren uint16) { + t.Helper() + require_True(t, an != nil) + n := *an + require_True(t, n != nil) + require_Equal(t, n.kind(), kind) + require_Equal(t, pors, string(n.path())) + require_Equal(t, numChildren, n.numChildren()) + } + + checkNode(&st.root, "NODE4", "foo.ba", 2) + nn := st.root.findChild('r') + checkNode(nn, "NODE4", "r", 2) + checkNode((*nn).findChild(noPivot), "LEAF", "", 0) + rnn := (*nn).findChild('.') + checkNode(rnn, "NODE4", ".", 3) + checkNode((*rnn).findChild('A'), "LEAF", "A", 0) + checkNode((*rnn).findChild('B'), "LEAF", "B", 0) + checkNode((*rnn).findChild('C'), "LEAF", "C", 0) + znn := st.root.findChild('z') + checkNode(znn, "NODE4", "z.", 3) + checkNode((*znn).findChild('A'), "LEAF", "A", 0) + checkNode((*znn).findChild('B'), "LEAF", "B", 0) + checkNode((*znn).findChild('C'), "LEAF", "C", 0) + // Use st.Dump() if you want a tree print out. + + // Now delete "foo.bar" and make sure put ourselves back together properly. + v, found := st.Delete(b("foo.bar")) + require_True(t, found) + require_Equal(t, *v, 42) + + checkNode(&st.root, "NODE4", "foo.ba", 2) + nn = st.root.findChild('r') + checkNode(nn, "NODE4", "r.", 3) + checkNode((*nn).findChild('A'), "LEAF", "A", 0) + checkNode((*nn).findChild('B'), "LEAF", "B", 0) + checkNode((*nn).findChild('C'), "LEAF", "C", 0) + znn = st.root.findChild('z') + checkNode(znn, "NODE4", "z.", 3) + checkNode((*znn).findChild('A'), "LEAF", "A", 0) + checkNode((*znn).findChild('B'), "LEAF", "B", 0) + checkNode((*znn).findChild('C'), "LEAF", "C", 0) +} + +func match(t *testing.T, st *SubjectTree[int], filter string, expected int) { + t.Helper() + var matches []int + st.Match(b(filter), func(_ []byte, v *int) { + matches = append(matches, *v) + }) + require_Equal(t, expected, len(matches)) +} + +func TestSubjectTreeMatchLeafOnly(t *testing.T) { + st := NewSubjectTree[int]() + st.Insert(b("foo.bar.baz.A"), 1) + + // Check all placements of pwc in token space. + match(t, st, "foo.bar.*.A", 1) + match(t, st, "foo.*.baz.A", 1) + match(t, st, "foo.*.*.A", 1) + match(t, st, "foo.*.*.*", 1) + match(t, st, "*.*.*.*", 1) + // Now check fwc. + match(t, st, ">", 1) + match(t, st, "foo.>", 1) + match(t, st, "foo.*.>", 1) + match(t, st, "foo.bar.>", 1) + match(t, st, "foo.bar.*.>", 1) + + // Check partials so they do not trigger on leafs. + match(t, st, "foo.bar.baz", 0) +} + +func TestSubjectTreeMatchNodes(t *testing.T) { + st := NewSubjectTree[int]() + st.Insert(b("foo.bar.A"), 1) + st.Insert(b("foo.bar.B"), 2) + st.Insert(b("foo.bar.C"), 3) + st.Insert(b("foo.baz.A"), 11) + st.Insert(b("foo.baz.B"), 22) + st.Insert(b("foo.baz.C"), 33) + + // Test literals. + match(t, st, "foo.bar.A", 1) + match(t, st, "foo.baz.A", 1) + match(t, st, "foo.bar", 0) + // Test internal pwc + match(t, st, "foo.*.A", 2) + // Test terminal pwc + match(t, st, "foo.bar.*", 3) + match(t, st, "foo.baz.*", 3) + // Check fwc + match(t, st, ">", 6) + match(t, st, "foo.>", 6) + match(t, st, "foo.bar.>", 3) + match(t, st, "foo.baz.>", 3) + // Make sure we do not have false positives on prefix matches. + match(t, st, "foo.ba", 0) + + // Now add in "foo.bar" to make a more complex tree construction + // and re-test. + st.Insert(b("foo.bar"), 42) + + // Test literals. + match(t, st, "foo.bar.A", 1) + match(t, st, "foo.baz.A", 1) + match(t, st, "foo.bar", 1) + // Test internal pwc + match(t, st, "foo.*.A", 2) + // Test terminal pwc + match(t, st, "foo.bar.*", 3) + match(t, st, "foo.baz.*", 3) + // Check fwc + match(t, st, ">", 7) + match(t, st, "foo.>", 7) + match(t, st, "foo.bar.>", 3) + match(t, st, "foo.baz.>", 3) +} + +func matchUntilCount[T any](st *SubjectTree[T], filter string, stopAfter int) (int, bool) { + var n int + completed := st.MatchUntil(b(filter), func(_ []byte, _ *T) bool { + n++ + return n < stopAfter + }) + return n, completed +} + +func TestSubjectTreeMatchUntil(t *testing.T) { + st := NewSubjectTree[int]() + st.Insert(b("foo.bar.A"), 1) + st.Insert(b("foo.bar.B"), 2) + st.Insert(b("foo.bar.C"), 3) + st.Insert(b("foo.baz.A"), 11) + st.Insert(b("foo.baz.B"), 22) + st.Insert(b("foo.baz.C"), 33) + st.Insert(b("foo.bar"), 42) + + // Ensure early stop terminates traversal. + count, completed := matchUntilCount(st, "foo.>", 3) + require_Equal(t, count, 3) + require_False(t, completed) + + // Match completes + count, completed = matchUntilCount(st, "foo.bar", 3) + require_Equal(t, count, 1) + require_True(t, completed) + + count, completed = matchUntilCount(st, "foo.baz.*", 4) + require_Equal(t, count, 3) + require_True(t, completed) +} + +func TestSubjectTreeNoPrefix(t *testing.T) { + st := NewSubjectTree[int]() + for i := 0; i < 26; i++ { + subj := b(fmt.Sprintf("%c", 'A'+i)) + old, updated := st.Insert(subj, 22) + require_True(t, old == nil) + require_False(t, updated) + } + n, ok := st.root.(*node48) + require_True(t, ok) + require_Equal(t, n.numChildren(), 26) + v, found := st.Delete(b("B")) + require_True(t, found) + require_Equal(t, *v, 22) + require_Equal(t, n.numChildren(), 25) + v, found = st.Delete(b("Z")) + require_True(t, found) + require_Equal(t, *v, 22) + require_Equal(t, n.numChildren(), 24) +} + +func TestSubjectTreePartialTerminalWildcardBugMatch(t *testing.T) { + st := NewSubjectTree[int]() + st.Insert(b("STATE.GLOBAL.CELL1.7PDSGAALXNN000010.PROPERTY-A"), 5) + st.Insert(b("STATE.GLOBAL.CELL1.7PDSGAALXNN000010.PROPERTY-B"), 1) + st.Insert(b("STATE.GLOBAL.CELL1.7PDSGAALXNN000010.PROPERTY-C"), 2) + match(t, st, "STATE.GLOBAL.CELL1.7PDSGAALXNN000010.*", 3) +} + +func TestSubjectTreeMatchSubjectParam(t *testing.T) { + st := NewSubjectTree[int]() + st.Insert(b("foo.bar.A"), 1) + st.Insert(b("foo.bar.B"), 2) + st.Insert(b("foo.bar.C"), 3) + st.Insert(b("foo.baz.A"), 11) + st.Insert(b("foo.baz.B"), 22) + st.Insert(b("foo.baz.C"), 33) + st.Insert(b("foo.bar"), 42) + + checkValMap := map[string]int{ + "foo.bar.A": 1, + "foo.bar.B": 2, + "foo.bar.C": 3, + "foo.baz.A": 11, + "foo.baz.B": 22, + "foo.baz.C": 33, + "foo.bar": 42, + } + // Make sure we get a proper subject parameter and it matches our value properly. + st.Match([]byte(">"), func(subject []byte, v *int) { + if expected, ok := checkValMap[string(subject)]; !ok { + t.Fatalf("Unexpected subject parameter: %q", subject) + } else if expected != *v { + t.Fatalf("Expected %q to have value of %d, but got %d", subject, expected, *v) + } + }) +} + +func TestSubjectTreeMatchRandomDoublePWC(t *testing.T) { + st := NewSubjectTree[int]() + for i := 1; i <= 10_000; i++ { + subj := fmt.Sprintf("foo.%d.%d", rand.IntN(20)+1, i) + st.Insert(b(subj), 42) + } + match(t, st, "foo.*.*", 10_000) + + // Check with pwc and short interior token. + seen, verified := 0, 0 + st.Match(b("*.2.*"), func(_ []byte, _ *int) { + seen++ + }) + // Now check via walk to make sure we are right. + st.IterOrdered(func(subject []byte, v *int) bool { + tokens := strings.Split(string(subject), ".") + require_Equal(t, len(tokens), 3) + if tokens[1] == "2" { + verified++ + } + return true + }) + require_Equal(t, seen, verified) + + seen, verified = 0, 0 + st.Match(b("*.*.222"), func(_ []byte, _ *int) { + seen++ + }) + st.IterOrdered(func(subject []byte, v *int) bool { + tokens := strings.Split(string(subject), ".") + require_Equal(t, len(tokens), 3) + if tokens[2] == "222" { + verified++ + } + return true + }) + require_Equal(t, seen, verified) +} + +func TestSubjectTreeIterOrdered(t *testing.T) { + st := NewSubjectTree[int]() + st.Insert(b("foo.bar.A"), 1) + st.Insert(b("foo.bar.B"), 2) + st.Insert(b("foo.bar.C"), 3) + st.Insert(b("foo.baz.A"), 11) + st.Insert(b("foo.baz.B"), 22) + st.Insert(b("foo.baz.C"), 33) + st.Insert(b("foo.bar"), 42) + + checkValMap := map[string]int{ + "foo.bar.A": 1, + "foo.bar.B": 2, + "foo.bar.C": 3, + "foo.baz.A": 11, + "foo.baz.B": 22, + "foo.baz.C": 33, + "foo.bar": 42, + } + checkOrder := []string{ + "foo.bar", + "foo.bar.A", + "foo.bar.B", + "foo.bar.C", + "foo.baz.A", + "foo.baz.B", + "foo.baz.C", + } + var received int + walk := func(subject []byte, v *int) bool { + if expected := checkOrder[received]; expected != string(subject) { + t.Fatalf("Expected %q for %d item returned, got %q", expected, received, subject) + } + received++ + require_True(t, v != nil) + if expected := checkValMap[string(subject)]; expected != *v { + t.Fatalf("Expected %q to have value of %d, but got %d", subject, expected, *v) + } + return true + } + // Kick in the iter. + st.IterOrdered(walk) + require_Equal(t, received, len(checkOrder)) + + // Make sure we can terminate properly. + received = 0 + st.IterOrdered(func(subject []byte, v *int) bool { + received++ + return received != 4 + }) + require_Equal(t, received, 4) +} + +func TestSubjectTreeIterFast(t *testing.T) { + st := NewSubjectTree[int]() + st.Insert(b("foo.bar.A"), 1) + st.Insert(b("foo.bar.B"), 2) + st.Insert(b("foo.bar.C"), 3) + st.Insert(b("foo.baz.A"), 11) + st.Insert(b("foo.baz.B"), 22) + st.Insert(b("foo.baz.C"), 33) + st.Insert(b("foo.bar"), 42) + + checkValMap := map[string]int{ + "foo.bar.A": 1, + "foo.bar.B": 2, + "foo.bar.C": 3, + "foo.baz.A": 11, + "foo.baz.B": 22, + "foo.baz.C": 33, + "foo.bar": 42, + } + var received int + walk := func(subject []byte, v *int) bool { + received++ + require_True(t, v != nil) + if expected := checkValMap[string(subject)]; expected != *v { + t.Fatalf("Expected %q to have value of %d, but got %d", subject, expected, *v) + } + return true + } + // Kick in the iter. + st.IterFast(walk) + require_Equal(t, received, len(checkValMap)) + + // Make sure we can terminate properly. + received = 0 + st.IterFast(func(subject []byte, v *int) bool { + received++ + return received != 4 + }) + require_Equal(t, received, 4) +} + +func TestSubjectTreeInsertSamePivotBug(t *testing.T) { + testSubjects := [][]byte{ + []byte("0d00.2abbb82c1d.6e16.fa7f85470e.3e46"), + []byte("534b12.3486c17249.4dde0666"), + []byte("6f26aabd.920ee3.d4d3.5ffc69f6"), + []byte("8850.ade3b74c31.aa533f77.9f59.a4bd8415.b3ed7b4111"), + []byte("5a75047dcb.5548e845b6.76024a34.14d5b3.80c426.51db871c3a"), + []byte("825fa8acfc.5331.00caf8bbbd.107c4b.c291.126d1d010e"), + } + st := NewSubjectTree[int]() + for _, subj := range testSubjects { + old, updated := st.Insert(subj, 22) + require_True(t, old == nil) + require_False(t, updated) + if _, found := st.Find(subj); !found { + t.Fatalf("Could not find subject %q which should be findable", subj) + } + } +} + +func TestSubjectTreeMatchTsepSecondThenPartialPartBug(t *testing.T) { + st := NewSubjectTree[int]() + st.Insert(b("foo.xxxxx.foo1234.zz"), 22) + st.Insert(b("foo.yyy.foo123.zz"), 22) + st.Insert(b("foo.yyybar789.zz"), 22) + st.Insert(b("foo.yyy.foo12345.zz"), 22) + st.Insert(b("foo.yyy.foo12345.yy"), 22) + st.Insert(b("foo.yyy.foo123456789.zz"), 22) + match(t, st, "foo.*.foo123456789.*", 1) + match(t, st, "foo.*.*.zzz.foo.>", 0) +} + +func TestSubjectTreeMatchMultipleWildcardBasic(t *testing.T) { + st := NewSubjectTree[int]() + st.Insert(b("A.B.C.D.0.G.H.I.0"), 22) + st.Insert(b("A.B.C.D.1.G.H.I.0"), 22) + match(t, st, "A.B.*.D.1.*.*.I.0", 1) +} + +func TestSubjectTreeMatchInvalidWildcard(t *testing.T) { + st := NewSubjectTree[int]() + st.Insert(b("foo.123"), 22) + st.Insert(b("one.two.three.four.five"), 22) + st.Insert(b("'*.123"), 22) + st.Insert(b("bar"), 22) + match(t, st, "invalid.>", 0) + match(t, st, "foo.>.bar", 0) + match(t, st, ">", 4) + match(t, st, `'*.*`, 1) + match(t, st, `'*.*.*'`, 0) + // None of these should match. + match(t, st, "`>`", 0) + match(t, st, `">"`, 0) + match(t, st, `'>'`, 0) + match(t, st, `'*.>'`, 0) + match(t, st, `'*.>.`, 0) + match(t, st, "`invalid.>`", 0) + match(t, st, `'*.*'`, 0) +} + +func TestSubjectTreeRandomTrackEntries(t *testing.T) { + st := NewSubjectTree[int]() + smap := make(map[string]struct{}, 1000) + + // Make sure all added items can be found. + check := func() { + t.Helper() + for subj := range smap { + if _, found := st.Find(b(subj)); !found { + t.Fatalf("Could not find subject %q which should be findable", subj) + } + } + } + + buf := make([]byte, 10) + for i := 0; i < 1000; i++ { + var sb strings.Builder + // 1-6 tokens. + numTokens := rand.IntN(6) + 1 + for i := 0; i < numTokens; i++ { + tlen := rand.IntN(4) + 2 + tok := buf[:tlen] + crand.Read(tok) + sb.WriteString(hex.EncodeToString(tok)) + if i != numTokens-1 { + sb.WriteString(".") + } + } + subj := sb.String() + // Avoid dupes since will cause check to fail after we delete messages. + if _, ok := smap[subj]; ok { + continue + } + smap[subj] = struct{}{} + old, updated := st.Insert(b(subj), 22) + require_True(t, old == nil) + require_False(t, updated) + require_Equal(t, st.Size(), len(smap)) + check() + } +} + +// Needs to be longer then internal node prefix, which currently is 24. +func TestSubjectTreeLongTokens(t *testing.T) { + st := NewSubjectTree[int]() + st.Insert(b("a1.aaaaaaaaaaaaaaaaaaaaaa0"), 1) + st.Insert(b("a2.0"), 2) + st.Insert(b("a1.aaaaaaaaaaaaaaaaaaaaaa1"), 3) + st.Insert(b("a2.1"), 4) + // Simulate purge of a2.> + // This required to show bug. + st.Delete(b("a2.0")) + st.Delete(b("a2.1")) + require_Equal(t, st.Size(), 2) + v, found := st.Find(b("a1.aaaaaaaaaaaaaaaaaaaaaa0")) + require_True(t, found) + require_Equal(t, *v, 1) + v, found = st.Find(b("a1.aaaaaaaaaaaaaaaaaaaaaa1")) + require_True(t, found) + require_Equal(t, *v, 3) +} + +func b(s string) []byte { + return []byte(s) +} + +func TestSubjectTreeMatchAllPerf(t *testing.T) { + if !*runResults { + t.Skip() + } + st := NewSubjectTree[int]() + + for i := 0; i < 1_000_000; i++ { + subj := fmt.Sprintf("subj.%d.%d", rand.IntN(100)+1, i) + st.Insert(b(subj), 22) + } + + for _, f := range [][]byte{ + []byte(">"), + []byte("subj.>"), + []byte("subj.*.*"), + []byte("*.*.*"), + []byte("subj.1.*"), + []byte("subj.1.>"), + []byte("subj.*.1"), + []byte("*.*.1"), + } { + start := time.Now() + count := 0 + st.Match(f, func(_ []byte, _ *int) { + count++ + }) + t.Logf("Match %q took %s and matched %d entries", f, time.Since(start), count) + } +} + +func TestSubjectTreeIterPerf(t *testing.T) { + if !*runResults { + t.Skip() + } + st := NewSubjectTree[int]() + + for i := 0; i < 1_000_000; i++ { + subj := fmt.Sprintf("subj.%d.%d", rand.IntN(100)+1, i) + st.Insert(b(subj), 22) + } + + start := time.Now() + count := 0 + st.IterOrdered(func(_ []byte, _ *int) bool { + count++ + return true + }) + t.Logf("Iter took %s and matched %d entries", time.Since(start), count) +} + +func TestSubjectTreeMemoryPerf(t *testing.T) { + if !*runResults { + t.Skip() + } + + const entries = 250_000 + subjects := make([][]byte, entries) + for i := range subjects { + subjects[i] = []byte("account." + strconv.Itoa(i%1000) + ".stream." + strconv.Itoa(i)) + } + + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + + start := time.Now() + st := NewSubjectTree[int]() + for i, subject := range subjects { + st.Insert(subject, i) + } + buildTime := time.Since(start) + + runtime.GC() + runtime.KeepAlive(st) + runtime.ReadMemStats(&after) + + const lookupRounds = 8 + start = time.Now() + found := 0 + for range lookupRounds { + for _, subject := range subjects { + if _, ok := st.Find(subject); ok { + found++ + } + } + } + findTime := time.Since(start) + + const traversalRounds = 20 + start = time.Now() + iterated := 0 + for range traversalRounds { + st.IterFast(func(_ []byte, _ *int) bool { + iterated++ + return true + }) + } + iterTime := time.Since(start) + + filter := []byte("account.*.stream.>") + start = time.Now() + matched := 0 + for range traversalRounds { + st.Match(filter, func(_ []byte, _ *int) { + matched++ + }) + } + matchTime := time.Since(start) + + iterAllocs := testing.AllocsPerRun(20, func() { + st.IterFast(func(_ []byte, _ *int) bool { return true }) + }) + matchAllocs := testing.AllocsPerRun(20, func() { + st.Match(filter, func(_ []byte, _ *int) {}) + }) + + t.Logf("Build took %s for %d entries, allocated %d bytes total and retained %d heap bytes across %d allocations", + buildTime, st.Size(), after.TotalAlloc-before.TotalAlloc, + after.HeapAlloc-before.HeapAlloc, after.Mallocs-before.Mallocs) + t.Logf("Find took %s for %d lookups and found %d entries", + findTime, entries*lookupRounds, found) + t.Logf("IterFast took %s for %d entries with %.0f allocations per traversal", + iterTime, iterated, iterAllocs) + t.Logf("Match took %s for %d entries with %.0f allocations per traversal", + matchTime, matched, matchAllocs) + + runtime.KeepAlive(subjects) + runtime.KeepAlive(st) +} + +func TestSubjectTreeNode48(t *testing.T) { + var a, b, c leaf[int] + var n node48 + + n.addChild('A', &a) + require_Equal(t, n.key['A'], 1) + require_True(t, n.child[0] != nil) + require_Equal(t, n.child[0].(*leaf[int]), &a) + require_Equal(t, len(n.children()), 1) + + child := n.findChild('A') + require_True(t, child != nil) + require_Equal(t, (*child).(*leaf[int]), &a) + + n.addChild('B', &b) + require_Equal(t, n.key['B'], 2) + require_True(t, n.child[1] != nil) + require_Equal(t, n.child[1].(*leaf[int]), &b) + require_Equal(t, len(n.children()), 2) + + child = n.findChild('B') + require_True(t, child != nil) + require_Equal(t, (*child).(*leaf[int]), &b) + + n.addChild('C', &c) + require_Equal(t, n.key['C'], 3) + require_True(t, n.child[2] != nil) + require_Equal(t, n.child[2].(*leaf[int]), &c) + require_Equal(t, len(n.children()), 3) + + child = n.findChild('C') + require_True(t, child != nil) + require_Equal(t, (*child).(*leaf[int]), &c) + + n.deleteChild('A') + require_Equal(t, len(n.children()), 2) + require_Equal(t, n.key['A'], 0) // Now deleted + require_Equal(t, n.key['B'], 2) // Untouched + require_Equal(t, n.key['C'], 1) // Where A was + + child = n.findChild('A') + require_Equal(t, child, nil) + require_True(t, n.child[0] != nil) + require_Equal(t, n.child[0].(*leaf[int]), &c) + + child = n.findChild('B') + require_True(t, child != nil) + require_Equal(t, (*child).(*leaf[int]), &b) + require_True(t, n.child[1] != nil) + require_Equal(t, n.child[1].(*leaf[int]), &b) + + child = n.findChild('C') + require_True(t, child != nil) + require_Equal(t, (*child).(*leaf[int]), &c) + require_True(t, n.child[2] == nil) + + var gotB, gotC bool + var iterations int + n.iter(func(n node) bool { + iterations++ + if gb, ok := n.(*leaf[int]); ok && &b == gb { + gotB = true + } + if gc, ok := n.(*leaf[int]); ok && &c == gc { + gotC = true + } + return true + }) + require_Equal(t, iterations, 2) + require_True(t, gotB) + require_True(t, gotC) + + // Check for off-by-one on byte 255 as found by staticcheck, see + // https://github.com/nats-io/nats-server/pull/5826. + n.addChild(255, &c) + require_Equal(t, n.key[255], 3) + grown := n.grow().(*node256) + require_True(t, grown.findChild(255) != nil) + shrunk := n.shrink().(*node16) + require_True(t, shrunk.findChild(255) != nil) +} + +func TestSubjectTreeMatchNoCallbackDupe(t *testing.T) { + st := NewSubjectTree[int]() + st.Insert(b("foo.bar.A"), 1) + st.Insert(b("foo.bar.B"), 1) + st.Insert(b("foo.bar.C"), 1) + st.Insert(b("foo.bar.>"), 1) + + for _, f := range [][]byte{ + []byte(">"), + []byte("foo.>"), + []byte("foo.bar.>"), + } { + seen := map[string]struct{}{} + st.Match(f, func(bsubj []byte, _ *int) { + subj := string(bsubj) + if _, ok := seen[subj]; ok { + t.Logf("Match callback was called twice for %q", subj) + } + seen[subj] = struct{}{} + }) + } +} + +func TestSubjectTreeNilNoPanic(t *testing.T) { + var st *SubjectTree[int] + st.Match([]byte("foo"), func(_ []byte, _ *int) {}) + _, found := st.Find([]byte("foo")) + require_False(t, found) + _, found = st.Delete([]byte("foo")) + require_False(t, found) + _, found = st.Insert([]byte("foo"), 22) + require_False(t, found) +} + +// This bug requires the trailing suffix contain repeating nulls \x00 +// and the second subject be longer with more nulls. +func TestSubjectTreeInsertLongerLeafSuffixWithTrailingNulls(t *testing.T) { + st := NewSubjectTree[int]() + subj := []byte("foo.bar.baz_") + // add in 10 nulls. + for i := 0; i < 10; i++ { + subj = append(subj, 0) + } + + st.Insert(subj, 1) + // add in 10 more nulls. + subj2 := subj + for i := 0; i < 10; i++ { + subj2 = append(subj, 0) + } + st.Insert(subj2, 2) + + // Make sure we can look them up. + v, found := st.Find(subj) + require_True(t, found) + require_Equal(t, *v, 1) + v, found = st.Find(subj2) + require_True(t, found) + require_Equal(t, *v, 2) +} + +// Make sure the system does not insert any subject with the noPivot (DEL) in it. +func TestSubjectTreeInsertWithNoPivot(t *testing.T) { + st := NewSubjectTree[int]() + subj := []byte("foo.bar.baz.") + subj = append(subj, noPivot) + old, updated := st.Insert(subj, 22) + require_True(t, old == nil) + require_False(t, updated) + require_Equal(t, st.Size(), 0) +} + +// Make sure we don't panic when checking for fwc. +func TestSubjectTreeMatchHasFWCNoPanic(t *testing.T) { + defer func() { + p := recover() + require_True(t, p == nil) + }() + st := NewSubjectTree[int]() + subj := []byte("foo") + st.Insert(subj, 1) + st.Match([]byte("."), func(subject []byte, val *int) {}) +} + +func TestSubjectTreeLazyIntersect(t *testing.T) { + st1 := NewSubjectTree[int]() + st2 := NewSubjectTree[int]() + + // Should cause an intersection. + st1.Insert([]byte("foo.bar"), 1) + st2.Insert([]byte("foo.bar"), 1) + + // Should cause an intersection. + st1.Insert([]byte("foo.bar.baz.qux"), 1) + st2.Insert([]byte("foo.bar.baz.qux"), 1) + + // Should not cause any intersections. + st1.Insert([]byte("bar"), 1) + st2.Insert([]byte("baz"), 1) + st1.Insert([]byte("a.b.c"), 1) + st2.Insert([]byte("a.b.d"), 1) + st1.Insert([]byte("a.b.ee"), 1) + st2.Insert([]byte("a.b.e"), 1) + st1.Insert([]byte("bb.c.d"), 1) + st2.Insert([]byte("b.c.d"), 1) + st2.Insert([]byte("foo.bar.baz.qux.alice"), 1) + st2.Insert([]byte("foo.bar.baz.qux.bob"), 1) + + intersected := map[string]int{} + LazyIntersect(st1, st2, func(key []byte, val1, val2 *int) { + intersected[string(key)]++ + }) + require_Equal(t, len(intersected), 2) + require_Equal(t, intersected["foo.bar"], 1) + require_Equal(t, intersected["foo.bar.baz.qux"], 1) +} + +func TestSubjectTreeGSLIntersection(t *testing.T) { + st := NewSubjectTree[struct{}]() + st.Insert([]byte("one.two.three.four"), struct{}{}) + st.Insert([]byte("one.two.three.five"), struct{}{}) + st.Insert([]byte("one.two.six"), struct{}{}) + st.Insert([]byte("one.two.seven"), struct{}{}) + st.Insert([]byte("eight.nine"), struct{}{}) + st.Insert([]byte("stream.A"), struct{}{}) + st.Insert([]byte("stream.A.child"), struct{}{}) + + require_NoDuplicates := func(t *testing.T, got map[string]int) { + t.Helper() + for _, c := range got { + require_Equal(t, c, 1) + } + } + + t.Run("Literals", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("one.two.six", 11)) + require_NoError(t, sl.Insert("eight.nine", 22)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 2) + require_NoDuplicates(t, got) + }) + + t.Run("PWC", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("one.two.*.*", 11)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 2) + require_NoDuplicates(t, got) + }) + + t.Run("PWCOverlapping", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("one.two.*.four", 11)) + require_NoError(t, sl.Insert("one.two.*.*", 22)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 2) + require_NoDuplicates(t, got) + }) + + t.Run("PWCAll", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("*.*", 11)) + require_NoError(t, sl.Insert("*.*.*", 22)) + require_NoError(t, sl.Insert("*.*.*.*", 33)) + require_True(t, sl.HasInterest("foo.bar")) + require_True(t, sl.HasInterest("foo.bar.baz")) + require_True(t, sl.HasInterest("foo.bar.baz.qux")) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 7) + require_NoDuplicates(t, got) + }) + + t.Run("FWC", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("one.>", 11)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 4) + require_NoDuplicates(t, got) + }) + + t.Run("FWCOverlapping", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("one.two.three.four", 11)) + require_NoError(t, sl.Insert("one.>", 22)) + require_NoError(t, sl.Insert(">", 33)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 7) + require_NoDuplicates(t, got) + }) + + t.Run("FWCExtended", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("stream.A.>", 11)) + require_NoError(t, sl.Insert("stream.A", 22)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 2) + require_NoDuplicates(t, got) + }) + + t.Run("PWCExtended", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("stream.*.child", 11)) + require_NoError(t, sl.Insert("stream.A", 22)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 2) + require_NoDuplicates(t, got) + }) + + t.Run("PWCExtendedAggressive", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("stream.A.child", 11)) + require_NoError(t, sl.Insert("*.A.child", 22)) + require_NoError(t, sl.Insert("stream.*.child", 22)) + require_NoError(t, sl.Insert("stream.A.*", 22)) + require_NoError(t, sl.Insert("stream.*.*", 22)) + require_NoError(t, sl.Insert("*.A.*", 22)) + require_NoError(t, sl.Insert("*.*.child", 22)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 1) + require_NoDuplicates(t, got) + }) + + t.Run("PWCExtendedAggressive2", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("stream.A.child", 11)) + require_NoError(t, sl.Insert("*.A.child", 22)) + require_NoError(t, sl.Insert("stream.A.*", 22)) + require_NoError(t, sl.Insert("*.A.*", 22)) + require_NoError(t, sl.Insert("*.*.child", 22)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 1) + require_NoDuplicates(t, got) + }) + + t.Run("PWCExtendedAggressive3", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("stream.A.child", 11)) + require_NoError(t, sl.Insert("stream.*.*", 22)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 1) + require_NoDuplicates(t, got) + }) + + t.Run("PWCExtendedAggressive4", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("stream.A.child", 11)) + require_NoError(t, sl.Insert("stream.*.>", 22)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 1) + require_NoDuplicates(t, got) + }) + + t.Run("PWCExtendedAggressive5", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("stream.A.child", 11)) + require_NoError(t, sl.Insert("*.*", 22)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 3) + require_NoDuplicates(t, got) + }) + + t.Run("PWCExtendedAggressive6", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("stream.A.child", 11)) + require_NoError(t, sl.Insert("stream.*", 22)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 2) + require_NoDuplicates(t, got) + }) + + t.Run("PWCExtendedAggressive7", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("stream.A", 11)) + require_NoError(t, sl.Insert("*.*.*", 22)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 4) + require_NoDuplicates(t, got) + }) + + t.Run("FWCAll", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert(">", 11)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 7) + require_NoDuplicates(t, got) + }) + + t.Run("StopsWhenCallbackReturnsFalse", func(t *testing.T) { + var got int + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert(">", 11)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got++ + return false + }) + require_Equal(t, got, 1) + }) + + t.Run("NoMatch", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("one", 11)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 0) + }) + + t.Run("NoMatches", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("one", 11)) + require_NoError(t, sl.Insert("eight", 22)) + require_NoError(t, sl.Insert("ten", 33)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 0) + }) + + t.Run("NoMatchPartial", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("stream.A.not-child", 11)) + require_NoError(t, sl.Insert("stream.A.child.>", 22)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 0) + require_NoDuplicates(t, got) + }) + + t.Run("PWCDoesntHideLiteral", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("one.*.six", 11)) + require_NoError(t, sl.Insert("one.two.seven", 22)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 2) + require_NoDuplicates(t, got) + }) + + t.Run("PWCDoesntHideLiteral2", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("one.*.*.four", 11)) + require_NoError(t, sl.Insert("one.*.*.five", 22)) + require_NoError(t, sl.Insert("one.*.three", 33)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 2) + require_NoDuplicates(t, got) + }) + + t.Run("DisjointWildcardPositions", func(t *testing.T) { + // Test case documenting bitmask approach limitation: + // When patterns have wildcards at different positions (not strict subsets), + // the bitmask considers earlier-position wildcards as "less specific" + // which can cause some matches to be skipped. + // e.g., *.two.three.four (wildcard at pos 0) vs one.two.*.* (wildcards at pos 2,3) + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("one.two.*.*", 33)) + require_NoError(t, sl.Insert("*.two.three.four", 55)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + // Due to bitmask comparison, one.two.*.* may be skipped in favor of *.two.three.four + // This results in only one.two.three.four being matched, missing one.two.three.five + require_Len(t, len(got), 2) + require_NoDuplicates(t, got) + }) + + t.Run("LiteralMatchWithOverlappingPWCs", func(t *testing.T) { + got := map[string]int{} + sl := gsl.NewSublist[int]() + require_NoError(t, sl.Insert("one.*.four.five", 11)) + require_NoError(t, sl.Insert("one.two.three.four", 22)) + IntersectGSL(st, sl, func(subj []byte, entry *struct{}) bool { + got[string(subj)]++ + return true + }) + require_Len(t, len(got), 1) + require_NoDuplicates(t, got) + }) +} + +func TestSubjectTreeDeleteShortSubjectNoPanic(t *testing.T) { + defer func() { + p := recover() + require_True(t, p == nil) + }() + + st := NewSubjectTree[int]() + + st.Insert(b("foo.bar.baz"), 1) + st.Insert(b("foo.bar.qux"), 2) + + v, found := st.Delete(b("foo.bar")) + require_False(t, found) + require_Equal(t, v, nil) + v, found = st.Find(b("foo.bar.baz")) + require_True(t, found) + require_Equal(t, *v, 1) + v, found = st.Find(b("foo.bar.qux")) + require_True(t, found) + require_Equal(t, *v, 2) +} + +func TestSubjectTreeEmpty(t *testing.T) { + // Test Empty on nil tree + var st *SubjectTree[int] + st2 := st.Empty() + require_True(t, st2 != nil) + require_Equal(t, st2.Size(), 0) + + // Test Empty on new tree + st = NewSubjectTree[int]() + require_Equal(t, st.Size(), 0) + st2 = st.Empty() + require_True(t, st2 == st) // Should return same instance + require_Equal(t, st2.Size(), 0) + + // Test Empty on tree with data + st.Insert(b("foo.bar"), 1) + st.Insert(b("foo.baz"), 2) + st.Insert(b("bar.baz"), 3) + require_Equal(t, st.Size(), 3) + + // Empty should clear everything + st2 = st.Empty() + require_True(t, st2 == st) // Should return same instance + require_Equal(t, st.Size(), 0) + require_True(t, st.root == nil) + + // Verify we can't find old entries + _, found := st.Find(b("foo.bar")) + require_False(t, found) + _, found = st.Find(b("foo.baz")) + require_False(t, found) + _, found = st.Find(b("bar.baz")) + require_False(t, found) + + // Verify we can insert new entries after Empty + old, updated := st.Insert(b("new.entry"), 42) + require_True(t, old == nil) + require_False(t, updated) + require_Equal(t, st.Size(), 1) + + v, found := st.Find(b("new.entry")) + require_True(t, found) + require_Equal(t, *v, 42) +} + +func TestSubjectTreeLazyIntersectComprehensive(t *testing.T) { + // Test with nil trees + var st1 *SubjectTree[int] + var st2 *SubjectTree[string] + count := 0 + LazyIntersect(st1, st2, func(key []byte, v1 *int, v2 *string) { + count++ + }) + require_Equal(t, count, 0) + + // Test with one nil tree + st1 = NewSubjectTree[int]() + st1.Insert(b("foo"), 1) + LazyIntersect(st1, st2, func(key []byte, v1 *int, v2 *string) { + count++ + }) + require_Equal(t, count, 0) + + // Test with empty trees + st2 = NewSubjectTree[string]() + LazyIntersect(st1, st2, func(key []byte, v1 *int, v2 *string) { + count++ + }) + require_Equal(t, count, 0) + + // Test with different value types + st1 = NewSubjectTree[int]() + st2 = NewSubjectTree[string]() + + // Add some intersecting keys + st1.Insert(b("foo.bar"), 42) + st2.Insert(b("foo.bar"), "hello") + st1.Insert(b("baz.qux"), 100) + st2.Insert(b("baz.qux"), "world") + + // Add non-intersecting keys + st1.Insert(b("only.in.st1"), 1) + st2.Insert(b("only.in.st2"), "two") + + results := make(map[string]struct { + v1 int + v2 string + }) + + LazyIntersect(st1, st2, func(key []byte, v1 *int, v2 *string) { + results[string(key)] = struct { + v1 int + v2 string + }{*v1, *v2} + }) + + require_Equal(t, len(results), 2) + require_Equal(t, results["foo.bar"].v1, 42) + require_Equal(t, results["foo.bar"].v2, "hello") + require_Equal(t, results["baz.qux"].v1, 100) + require_Equal(t, results["baz.qux"].v2, "world") + + // Test that it iterates over smaller tree + // Create a large tree and a small tree + large := NewSubjectTree[int]() + small := NewSubjectTree[int]() + + // Large tree has many entries + for i := 0; i < 100; i++ { + large.Insert([]byte(fmt.Sprintf("large.%d", i)), i) + } + // Small tree has few entries with some overlap + small.Insert(b("large.5"), 500) + small.Insert(b("large.10"), 1000) + small.Insert(b("large.50"), 5000) + small.Insert(b("small.only"), 999) + + intersectCount := 0 + LazyIntersect(large, small, func(key []byte, v1 *int, v2 *int) { + intersectCount++ + // Verify we get the correct values + switch string(key) { + case "large.5": + require_Equal(t, *v1, 5) + require_Equal(t, *v2, 500) + case "large.10": + require_Equal(t, *v1, 10) + require_Equal(t, *v2, 1000) + case "large.50": + require_Equal(t, *v1, 50) + require_Equal(t, *v2, 5000) + default: + t.Fatalf("Unexpected key: %s", key) + } + }) + require_Equal(t, intersectCount, 3) + + // Test with complex subjects (multiple levels) + st3 := NewSubjectTree[int]() + st4 := NewSubjectTree[int]() + + // Deep nesting + st3.Insert(b("a.b.c.d.e.f.g"), 1) + st4.Insert(b("a.b.c.d.e.f.g"), 2) + + // Partial matches (should not intersect) + st3.Insert(b("a.b.c.d"), 3) + st4.Insert(b("a.b.c.d.e"), 4) + + // Same prefix different suffix + st3.Insert(b("prefix.suffix1"), 5) + st4.Insert(b("prefix.suffix2"), 6) + + intersections := 0 + LazyIntersect(st3, st4, func(key []byte, v1 *int, v2 *int) { + intersections++ + require_Equal(t, string(key), "a.b.c.d.e.f.g") + require_Equal(t, *v1, 1) + require_Equal(t, *v2, 2) + }) + require_Equal(t, intersections, 1) +} + +func TestNode256Operations(t *testing.T) { + // Test node256 creation and basic operations + n := newNode256(b("prefix")) + require_False(t, n.isFull()) // node256 is never full + + // Test findChild when child doesn't exist + child := n.findChild('a') + require_True(t, child == nil) + + // Add a child and find it + leaf := newLeaf(b("suffix"), 42) + n.addChild('a', leaf) + child = n.findChild('a') + require_True(t, child != nil) + require_Equal(t, n.size, uint16(1)) + + // Test iter function + iterCount := 0 + n.iter(func(node) bool { + iterCount++ + return true + }) + require_Equal(t, iterCount, 1) + + // Test iter with early termination + n.addChild('b', newLeaf(b("suffix2"), 43)) + n.addChild('c', newLeaf(b("suffix3"), 44)) + iterCount = 0 + n.iter(func(node) bool { + iterCount++ + return false // Stop after first + }) + require_Equal(t, iterCount, 1) + + // Test children() method + children := n.children() + require_Equal(t, len(children), 256) + + // Test that grow() panics + defer func() { + if r := recover(); r != nil { + require_Equal(t, r, "grow can not be called on node256") + } else { + t.Fatal("grow() should panic on node256") + } + }() + n.grow() +} + +func TestNode256Shrink(t *testing.T) { + // To get a node256, we need to go through the progression: + // node4 -> node10 -> node16 -> node48 -> node256 + // We need at least 49 children to get to node256 + + // Create nodes directly to test node256 shrinking + n256 := newNode256(b("prefix")) + + // Add 49 children + for i := 0; i < 49; i++ { + n256.addChild(byte(i), newLeaf([]byte{byte(i)}, i)) + } + require_Equal(t, n256.size, uint16(49)) + + // Shrink should not happen yet (> 48 children) + shrunk := n256.shrink() + require_True(t, shrunk == nil) + + // Delete one to get to 48 children + n256.deleteChild(0) + require_Equal(t, n256.size, uint16(48)) + + // Now shrink should return a node48 + shrunk = n256.shrink() + require_True(t, shrunk != nil) + _, isNode48 := shrunk.(*node48) + require_True(t, isNode48) + + // Verify the shrunk node has all remaining children + for i := 1; i < 49; i++ { + child := shrunk.findChild(byte(i)) + require_True(t, child != nil) + } +} + +func TestLeafPanicMethods(t *testing.T) { + leaf := newLeaf(b("test"), 42) + + // Test setPrefix panic + t.Run("setPrefix", func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + require_Equal(t, r, "setPrefix called on leaf") + } else { + t.Fatal("setPrefix should panic on leaf") + } + }() + leaf.setPrefix(b("prefix")) + }) + + // Test addChild panic + t.Run("addChild", func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + require_Equal(t, r, "addChild called on leaf") + } else { + t.Fatal("addChild should panic on leaf") + } + }() + leaf.addChild('a', nil) + }) + + // Test findChild panic + t.Run("findChild", func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + require_Equal(t, r, "findChild called on leaf") + } else { + t.Fatal("findChild should panic on leaf") + } + }() + leaf.findChild('a') + }) + + // Test grow panic + t.Run("grow", func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + require_Equal(t, r, "grow called on leaf") + } else { + t.Fatal("grow should panic on leaf") + } + }() + leaf.grow() + }) + + // Test deleteChild panic + t.Run("deleteChild", func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + require_Equal(t, r, "deleteChild called on leaf") + } else { + t.Fatal("deleteChild should panic on leaf") + } + }() + leaf.deleteChild('a') + }) + + // Test shrink panic + t.Run("shrink", func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + require_Equal(t, r, "shrink called on leaf") + } else { + t.Fatal("shrink should panic on leaf") + } + }() + leaf.shrink() + }) + + // Test other leaf methods that should work + require_True(t, leaf.isFull()) + require_True(t, leaf.base() == nil) + require_Equal(t, leaf.numChildren(), uint16(0)) + require_True(t, leaf.children() == nil) + + // Test iter (should do nothing) + called := false + leaf.iter(func(n node) bool { + called = true + return true + }) + require_False(t, called) +} + +func TestSizeOnNilTree(t *testing.T) { + var st *SubjectTree[int] + require_Equal(t, st.Size(), 0) +} + +func TestFindEdgeCases(t *testing.T) { + st := NewSubjectTree[int]() + + // Test Find with empty subject at root level + st.Insert(b("foo.bar.baz"), 1) + st.Insert(b("foo"), 2) + + // This should create a tree structure, now test finding with edge cases + v, found := st.Find(b("")) + require_False(t, found) + require_True(t, v == nil) +} + +func TestNodeIterMethods(t *testing.T) { + // Test node4 iter + n4 := newNode4(b("prefix")) + n4.addChild('a', newLeaf(b("1"), 1)) + n4.addChild('b', newLeaf(b("2"), 2)) + + count := 0 + n4.iter(func(n node) bool { + count++ + return true + }) + require_Equal(t, count, 2) + + // Test early termination + count = 0 + n4.iter(func(n node) bool { + count++ + return false + }) + require_Equal(t, count, 1) + + // Test node10 iter + n10 := newNode10(b("prefix")) + for i := 0; i < 5; i++ { + n10.addChild(byte('a'+i), newLeaf([]byte{byte('0' + i)}, i)) + } + + count = 0 + n10.iter(func(n node) bool { + count++ + return true + }) + require_Equal(t, count, 5) + + // Test node16 iter + n16 := newNode16(b("prefix")) + for i := 0; i < 8; i++ { + n16.addChild(byte('a'+i), newLeaf([]byte{byte('0' + i)}, i)) + } + + count = 0 + n16.iter(func(n node) bool { + count++ + return true + }) + require_Equal(t, count, 8) +} + +func TestIterOrderedAndIterFastNilRoot(t *testing.T) { + // Test IterOrdered with nil root + st := NewSubjectTree[int]() + count := 0 + st.IterOrdered(func(subject []byte, val *int) bool { + count++ + return true + }) + require_Equal(t, count, 0) + + // Test IterFast with nil root + count = 0 + st.IterFast(func(subject []byte, val *int) bool { + count++ + return true + }) + require_Equal(t, count, 0) +} + +func TestNodeAddChildPanic(t *testing.T) { + // Test node4 addChild panic when full + n4 := newNode4(b("prefix")) + n4.addChild('a', newLeaf(b("1"), 1)) + n4.addChild('b', newLeaf(b("2"), 2)) + n4.addChild('c', newLeaf(b("3"), 3)) + n4.addChild('d', newLeaf(b("4"), 4)) + + defer func() { + if r := recover(); r != nil { + require_Equal(t, r, "node4 full!") + } else { + t.Fatal("addChild should panic when node4 is full") + } + }() + n4.addChild('e', newLeaf(b("5"), 5)) +} + +func TestNodeAddChildPanicOthers(t *testing.T) { + // Test node10 addChild panic when full + t.Run("node10", func(t *testing.T) { + n10 := newNode10(b("prefix")) + for i := 0; i < 10; i++ { + n10.addChild(byte('a'+i), newLeaf([]byte{byte('0' + i)}, i)) + } + + defer func() { + if r := recover(); r != nil { + require_Equal(t, r, "node10 full!") + } else { + t.Fatal("addChild should panic when node10 is full") + } + }() + n10.addChild('k', newLeaf(b("11"), 11)) + }) + + // Test node16 addChild panic when full + t.Run("node16", func(t *testing.T) { + n16 := newNode16(b("prefix")) + for i := 0; i < 16; i++ { + n16.addChild(byte(i), newLeaf([]byte{byte(i)}, i)) + } + + defer func() { + if r := recover(); r != nil { + require_Equal(t, r, "node16 full!") + } else { + t.Fatal("addChild should panic when node16 is full") + } + }() + n16.addChild(16, newLeaf(b("16"), 16)) + }) + + // Test node48 addChild panic when full + t.Run("node48", func(t *testing.T) { + n48 := newNode48(b("prefix")) + for i := 0; i < 48; i++ { + n48.addChild(byte(i), newLeaf([]byte{byte(i)}, i)) + } + + defer func() { + if r := recover(); r != nil { + require_Equal(t, r, "node48 full!") + } else { + t.Fatal("addChild should panic when node48 is full") + } + }() + n48.addChild(48, newLeaf(b("48"), 48)) + }) +} + +func TestNodeDeleteChildNotFound(t *testing.T) { + // Test node10 deleteChild when child doesn't exist + n10 := newNode10(b("prefix")) + n10.addChild('a', newLeaf(b("1"), 1)) + n10.addChild('b', newLeaf(b("2"), 2)) + + // Try to delete non-existent child + n10.deleteChild('z') + require_Equal(t, n10.size, uint16(2)) // Size should remain unchanged + + // Test node16 deleteChild when child doesn't exist + n16 := newNode16(b("prefix")) + n16.addChild('a', newLeaf(b("1"), 1)) + n16.addChild('b', newLeaf(b("2"), 2)) + + n16.deleteChild('z') + require_Equal(t, n16.size, uint16(2)) + + // Test node48 deleteChild when child doesn't exist + n48 := newNode48(b("prefix")) + n48.addChild(0, newLeaf(b("1"), 1)) + n48.addChild(1, newLeaf(b("2"), 2)) + + n48.deleteChild(255) + require_Equal(t, n48.size, uint16(2)) +} + +func TestNodeShrinkNotNeeded(t *testing.T) { + // Test node10 shrink when not needed (has more than 4 children) + n10 := newNode10(b("prefix")) + for i := 0; i < 5; i++ { + n10.addChild(byte('a'+i), newLeaf([]byte{byte('0' + i)}, i)) + } + + shrunk := n10.shrink() + require_True(t, shrunk == nil) // Should not shrink + + // Test node16 shrink when not needed (has more than 10 children) + n16 := newNode16(b("prefix")) + for i := 0; i < 11; i++ { + n16.addChild(byte(i), newLeaf([]byte{byte(i)}, i)) + } + + shrunk = n16.shrink() + require_True(t, shrunk == nil) // Should not shrink +} + +func TestNode48IterEarlyTermination(t *testing.T) { + n48 := newNode48(b("prefix")) + for i := 0; i < 10; i++ { + n48.addChild(byte(i), newLeaf([]byte{byte(i)}, i)) + } + + count := 0 + n48.iter(func(n node) bool { + count++ + return false // Stop immediately + }) + require_Equal(t, count, 1) +} + +func TestNode10And16IterEarlyTermination(t *testing.T) { + // Test node10 early termination + n10 := newNode10(b("prefix")) + for i := 0; i < 5; i++ { + n10.addChild(byte('a'+i), newLeaf([]byte{byte('0' + i)}, i)) + } + + count := 0 + n10.iter(func(n node) bool { + count++ + return count < 2 // Stop after 2 + }) + require_Equal(t, count, 2) + + // Test node16 early termination + n16 := newNode16(b("prefix")) + for i := 0; i < 8; i++ { + n16.addChild(byte(i), newLeaf([]byte{byte(i)}, i)) + } + + count = 0 + n16.iter(func(n node) bool { + count++ + return count < 3 // Stop after 3 + }) + require_Equal(t, count, 3) +} + +func TestMatchPartsEdgeCases(t *testing.T) { + // Test the edge case in matchParts that's not covered + // This is the case where we have a part that needs to be copied and modified + + // Create a complex filter that will trigger the edge case + filter := b("foo.*.bar.>") + parts := genParts(filter, nil) + + // Test with a fragment that will cause partial matching + frag := b("foo.test") + remaining, matched := matchParts(parts, string(frag)) + require_True(t, matched) + require_True(t, len(remaining) > 0) +} + +func TestInsertEdgeCases(t *testing.T) { + st := NewSubjectTree[int]() + + // Test inserting with noPivot byte (should fail) + old, updated := st.Insert([]byte("foo\x7fbar"), 1) + require_True(t, old == nil) + require_False(t, updated) + require_Equal(t, st.Size(), 0) // Should not insert + + // Test the edge case where we need to split with same pivot + st = NewSubjectTree[int]() + // This case tests subjects that cause the same pivot after split + // Both subjects share prefix "a" and have same pivot "." after split + st.Insert(b("a.b"), 1) + // Now insert one that will cause the split with same pivot + st.Insert(b("a.c"), 2) + + require_Equal(t, st.Size(), 2) +} + +func TestDeleteEdgeCases(t *testing.T) { + st := NewSubjectTree[int]() + + // Test delete on empty tree + val, deleted := st.Delete(b("foo")) + require_False(t, deleted) + require_True(t, val == nil) + + // Test delete with empty subject + st.Insert(b("foo"), 1) + val, deleted = st.Delete(b("")) + require_False(t, deleted) + require_True(t, val == nil) + + // Test delete with subject shorter than prefix + st = NewSubjectTree[int]() + st.Insert(b("verylongprefix.suffix"), 1) + st.Insert(b("verylongprefix.suffix2"), 2) + val, deleted = st.Delete(b("very")) + require_False(t, deleted) + require_True(t, val == nil) +} + +func TestMatchEdgeCases(t *testing.T) { + st := NewSubjectTree[int]() + + // Test match with nil callback + st.Insert(b("foo.bar"), 1) + st.Match(b("foo.*"), nil) // Should not panic + + // Test match with empty filter + count := 0 + st.Match(b(""), func(subject []byte, val *int) { + count++ + }) + require_Equal(t, count, 0) +} + +func TestIterEdgeCases(t *testing.T) { + st := NewSubjectTree[int]() + + // Add multiple subjects to create a complex tree + st.Insert(b("a.b.c"), 1) + st.Insert(b("a.b.d"), 2) + st.Insert(b("a.c.d"), 3) + st.Insert(b("b.c.d"), 4) + + // Test iter with early termination at different points + count := 0 + st.iter(st.root, nil, false, func(subject []byte, val *int) bool { + count++ + return count < 2 + }) + require_Equal(t, count, 2) +} + +func TestLeafIter(t *testing.T) { + // Test that leaf iter does nothing (it's a no-op) + leaf := newLeaf(b("test"), 42) + called := false + + // Call iter with a function that would set called to true + leaf.iter(func(n node) bool { + called = true + return true + }) + require_False(t, called) // Should never be called since leaf.iter is a no-op + + // Call iter again with a function that returns false + leaf.iter(func(n node) bool { + called = true + return false + }) + require_False(t, called) // Still should never be called + + // Verify the leaf itself is not affected + require_True(t, leaf.match(b("test"))) + require_Equal(t, leaf.value, 42) + + // Also test through the node interface to ensure coverage + var n node = leaf + called = false + n.iter(func(child node) bool { + called = true + return true + }) + require_False(t, called) // Still should never be called +} + +func TestDeleteChildEdgeCasesMore(t *testing.T) { + // Test the edge case in node10 deleteChild where we don't swap (last element) + n10 := newNode10(b("prefix")) + n10.addChild('a', newLeaf(b("1"), 1)) + n10.addChild('b', newLeaf(b("2"), 2)) + n10.addChild('c', newLeaf(b("3"), 3)) + + // Delete the last child + n10.deleteChild('c') + require_Equal(t, n10.size, uint16(2)) + + // Test the edge case in node16 deleteChild where we don't swap (last element) + n16 := newNode16(b("prefix")) + n16.addChild('a', newLeaf(b("1"), 1)) + n16.addChild('b', newLeaf(b("2"), 2)) + n16.addChild('c', newLeaf(b("3"), 3)) + + // Delete the last child + n16.deleteChild('c') + require_Equal(t, n16.size, uint16(2)) +} + +func TestMatchPartsMoreEdgeCases(t *testing.T) { + // Test the remaining 2.6% of matchParts + // Case where frag is empty + parts := genParts(b("foo.*"), nil) + remaining, matched := matchParts(parts, "") + require_True(t, matched) + require_Equal(t, len(remaining), len(parts)) +} + +func TestInsertComplexEdgeCases(t *testing.T) { + st := NewSubjectTree[int]() + + // Test the recursive insert case with same pivot + // This requires a very specific setup + // First, create a tree structure that will trigger the recursive path + st.Insert(b("a"), 1) + st.Insert(b("aa"), 2) // This will create a split + + // Now insert something that has the same pivot after split + st.Insert(b("aaa"), 3) // This should trigger the recursive insert path + + require_Equal(t, st.Size(), 3) + + // Verify all values can be found + v, found := st.Find(b("a")) + require_True(t, found) + require_Equal(t, *v, 1) + + v, found = st.Find(b("aa")) + require_True(t, found) + require_Equal(t, *v, 2) + + v, found = st.Find(b("aaa")) + require_True(t, found) + require_Equal(t, *v, 3) +} + +func TestDeleteNilNodePointer(t *testing.T) { + st := NewSubjectTree[int]() + // Test delete with nil node + var n node + val, deleted := st.delete(&n, b("foo"), 0) + require_False(t, deleted) + require_True(t, val == nil) +} + +func TestMatchComplexEdgeCases(t *testing.T) { + st := NewSubjectTree[int]() + + // Build a complex tree to test the 2.2% uncovered in match + st.Insert(b("foo.bar.baz"), 1) + st.Insert(b("foo.bar.qux"), 2) + st.Insert(b("foo.baz.bar"), 3) + st.Insert(b("bar.foo.baz"), 4) + + // Test with terminal fwc but no remaining parts + count := 0 + st.Match(b("foo.bar.>"), func(subject []byte, val *int) { + count++ + }) + require_Equal(t, count, 2) +} + +func TestIterComplexTree(t *testing.T) { + st := NewSubjectTree[int]() + + // Build a deeper tree to test the remaining iter cases + for i := 0; i < 20; i++ { + st.Insert([]byte(fmt.Sprintf("level1.level2.level3.item%d", i)), i) + } + + // This should create multiple node types and test more paths + count := 0 + st.IterOrdered(func(subject []byte, val *int) bool { + count++ + return true + }) + require_Equal(t, count, 20) +} diff --git a/server/stree/util.go b/server/stree/util.go new file mode 100644 index 0000000..173899a --- /dev/null +++ b/server/stree/util.go @@ -0,0 +1,47 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stree + +// For subject matching. +const ( + pwc = '*' + fwc = '>' + tsep = '.' +) + +// Determine index of common prefix. No match at all is 0, etc. +func commonPrefixLen(s1, s2 []byte) int { + limit := min(len(s1), len(s2)) + var i int + for ; i < limit; i++ { + if s1[i] != s2[i] { + break + } + } + return i +} + +type position interface{ int | uint16 } + +// No pivot available. +const noPivot = byte(127) + +// Can return 127 (DEL) if we have all the subject as prefixes. +// We used to use 0, but when that was in the subject would cause infinite recursion in some situations. +func pivot[N position](subject []byte, pos N) byte { + if int(pos) >= len(subject) { + return noPivot + } + return subject[pos] +} diff --git a/server/subject_fuzz_test.go b/server/subject_fuzz_test.go new file mode 100644 index 0000000..482791c --- /dev/null +++ b/server/subject_fuzz_test.go @@ -0,0 +1,71 @@ +// Copyright 2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import "testing" + +// FuzzSubjectsCollide performs fuzz testing on the NATS subject collision detection logic. +// It verifies the behavior of the SubjectsCollide function which determines if two NATS +// subjects/subscriptions could potentially overlap in the NATS pub-sub system. +func FuzzSubjectsCollide(f *testing.F) { + corpuses := []struct { + s1 string + s2 string + }{ + // SubjectsCollide true + {s1: "", s2: ""}, + {s1: "a", s2: "a"}, + {s1: "a.b.c", s2: "a.b.c"}, + {s1: "$JS.b.c", s2: "$JS.b.c"}, + {s1: "a.b.c", s2: "a.*.c"}, + {s1: "a.b.*", s2: "a.*.c"}, + {s1: "aaa.bbb.ccc", s2: "aaa.bbb.ccc"}, + {s1: "aaa.*.ccc", s2: "*.bbb.ccc"}, + {s1: "*", s2: "*"}, + {s1: "**", s2: "*"}, + {s1: "", s2: ">"}, + {s1: ">", s2: ">"}, + {s1: ">>", s2: ">"}, + {s1: "a", s2: ">"}, + {s1: "a.b.c", s2: ">"}, + {s1: "a.b.c.>", s2: "a.b.>"}, + {s1: "a.b.c.d.*", s2: "a.b.c.*.e"}, + {s1: "a.*.*.d.>", s2: "a.bbb.ccc.*.e"}, + + // SubjectsCollide false + {s1: "a", s2: ""}, + {s1: "a.b", s2: "b.a"}, + {s1: "a.bbbbb.*.d", s2: "a.b.>"}, + {s1: "a.b", s2: "a.b.c"}, + {s1: "a.b.c", s2: "a.b"}, + {s1: "a.b", s2: ""}, + {s1: "a.*.*.d.e.>", s2: "a.bbb.ccc.*.e"}, + } + + for _, crp := range corpuses { + f.Add(crp.s1, crp.s2) + } + + f.Fuzz(func(t *testing.T, s1, s2 string) { + if !IsValidSubject(s1) { + return + } + + if !IsValidSubject(s2) { + return + } + + SubjectsCollide(s1, s2) + }) +} diff --git a/server/subject_transform.go b/server/subject_transform.go new file mode 100644 index 0000000..f2bae9d --- /dev/null +++ b/server/subject_transform.go @@ -0,0 +1,688 @@ +// Copyright 2023-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "fmt" + "hash/fnv" + "math" + "math/rand/v2" + "regexp" + "slices" + "strconv" + "strings" +) + +// Subject mapping and transform setups. +var ( + commaSeparatorRegEx = regexp.MustCompile(`,\s*`) + partitionMappingFunctionRegEx = regexp.MustCompile(`{{\s*[pP]artition\s*\((.*)\)\s*}}`) + wildcardMappingFunctionRegEx = regexp.MustCompile(`{{\s*[wW]ildcard\s*\((.*)\)\s*}}`) + splitFromLeftMappingFunctionRegEx = regexp.MustCompile(`{{\s*[sS]plit[fF]rom[lL]eft\s*\((.*)\)\s*}}`) + splitFromRightMappingFunctionRegEx = regexp.MustCompile(`{{\s*[sS]plit[fF]rom[rR]ight\s*\((.*)\)\s*}}`) + sliceFromLeftMappingFunctionRegEx = regexp.MustCompile(`{{\s*[sS]lice[fF]rom[lL]eft\s*\((.*)\)\s*}}`) + sliceFromRightMappingFunctionRegEx = regexp.MustCompile(`{{\s*[sS]lice[fF]rom[rR]ight\s*\((.*)\)\s*}}`) + splitMappingFunctionRegEx = regexp.MustCompile(`{{\s*[sS]plit\s*\((.*)\)\s*}}`) + leftMappingFunctionRegEx = regexp.MustCompile(`{{\s*[lL]eft\s*\((.*)\)\s*}}`) + rightMappingFunctionRegEx = regexp.MustCompile(`{{\s*[rR]ight\s*\((.*)\)\s*}}`) + randomMappingFunctionRegEx = regexp.MustCompile(`{{\s*[rR]andom\s*\((.*)\)\s*}}`) +) + +// Enum for the subject mapping subjectTransform function types +const ( + NoTransform int16 = iota + BadTransform + Partition + Wildcard + SplitFromLeft + SplitFromRight + SliceFromLeft + SliceFromRight + Split + Left + Right + Random +) + +// Transforms for arbitrarily mapping subjects from one to another for maps, tees and filters. +// These can also be used for proper mapping on wildcard exports/imports. +// These will be grouped and caching and locking are assumed to be in the upper layers. +type subjectTransform struct { + src, dest string + dtoks []string // destination tokens + stoks []string // source tokens + dtokmftypes []int16 // destination token mapping function types + dtokmftokindexesargs [][]int // destination token mapping function array of source token index arguments + dtokmfintargs []int32 // destination token mapping function int32 arguments + dtokmfstringargs []string // destination token mapping function string arguments +} + +// SubjectTransformer transforms subjects using mappings +// +// This API is not part of the public API and not subject to SemVer protections +type SubjectTransformer interface { + // TODO(dlc) - We could add in client here to allow for things like foo -> foo.$ACCOUNT + Match(string) (string, error) + TransformSubject(subject string) string + TransformTokenizedSubject(tokens []string) string +} + +func NewSubjectTransformWithStrict(src, dest string, strict bool) (*subjectTransform, error) { + // strict = true for import subject mappings that need to be reversible + // (meaning can only use the Wildcard function and must use all the pwcs that are present in the source) + // No source given is equivalent to the source being ">" + + if dest == _EMPTY_ { + return nil, nil + } + + if src == _EMPTY_ { + src = fwcs + } + + // Both entries need to be valid subjects. + sv, stokens, npwcs, hasFwc := subjectInfo(src) + dv, dtokens, dnpwcs, dHasFwc := subjectInfo(dest) + + // Make sure both are valid, match fwc if present and there are no pwcs in the dest subject. + if !sv || !dv || dnpwcs > 0 || hasFwc != dHasFwc { + return nil, ErrBadSubject + } + + var dtokMappingFunctionTypes []int16 + var dtokMappingFunctionTokenIndexes [][]int + var dtokMappingFunctionIntArgs []int32 + var dtokMappingFunctionStringArgs []string + + // If the src has partial wildcards then the dest needs to have the token place markers. + if npwcs > 0 || hasFwc { + // We need to count to make sure that the dest has token holders for the pwcs. + sti := make(map[int]int) + for i, token := range stokens { + if len(token) == 1 && token[0] == pwc { + sti[len(sti)+1] = i + } + } + + nphs := 0 + for _, token := range dtokens { + tranformType, transformArgWildcardIndexes, transfomArgInt, transformArgString, err := indexPlaceHolders(token) + if err != nil { + return nil, err + } + + if strict { + if tranformType != NoTransform && tranformType != Wildcard { + return nil, &mappingDestinationErr{token, ErrMappingDestinationNotSupportedForImport} + } + } + + if tranformType == NoTransform { + dtokMappingFunctionTypes = append(dtokMappingFunctionTypes, NoTransform) + dtokMappingFunctionTokenIndexes = append(dtokMappingFunctionTokenIndexes, []int{-1}) + dtokMappingFunctionIntArgs = append(dtokMappingFunctionIntArgs, -1) + dtokMappingFunctionStringArgs = append(dtokMappingFunctionStringArgs, _EMPTY_) + } else if tranformType == Random { + dtokMappingFunctionTypes = append(dtokMappingFunctionTypes, Random) + dtokMappingFunctionTokenIndexes = append(dtokMappingFunctionTokenIndexes, []int{}) + dtokMappingFunctionIntArgs = append(dtokMappingFunctionIntArgs, transfomArgInt) + dtokMappingFunctionStringArgs = append(dtokMappingFunctionStringArgs, _EMPTY_) + } else { + nphs += len(transformArgWildcardIndexes) + // Now build up our runtime mapping from dest to source tokens. + var stis []int + for _, wildcardIndex := range transformArgWildcardIndexes { + if wildcardIndex > npwcs { + return nil, &mappingDestinationErr{fmt.Sprintf("%s: [%d]", token, wildcardIndex), ErrMappingDestinationIndexOutOfRange} + } + stis = append(stis, sti[wildcardIndex]) + } + dtokMappingFunctionTypes = append(dtokMappingFunctionTypes, tranformType) + dtokMappingFunctionTokenIndexes = append(dtokMappingFunctionTokenIndexes, stis) + dtokMappingFunctionIntArgs = append(dtokMappingFunctionIntArgs, transfomArgInt) + dtokMappingFunctionStringArgs = append(dtokMappingFunctionStringArgs, transformArgString) + + } + } + if strict && nphs < npwcs { + // not all wildcards are being used in the destination + return nil, &mappingDestinationErr{dest, ErrMappingDestinationNotUsingAllWildcards} + } + } else { + // no wildcards used in the source: check that no transform functions are used in the destination + for _, token := range dtokens { + tranformType, _, transfomArgInt, _, err := indexPlaceHolders(token) + if err != nil { + return nil, err + } + + if tranformType == NoTransform { + dtokMappingFunctionTypes = append(dtokMappingFunctionTypes, NoTransform) + dtokMappingFunctionTokenIndexes = append(dtokMappingFunctionTokenIndexes, []int{-1}) + dtokMappingFunctionIntArgs = append(dtokMappingFunctionIntArgs, -1) + dtokMappingFunctionStringArgs = append(dtokMappingFunctionStringArgs, _EMPTY_) + } else if tranformType == Random || tranformType == Partition { + dtokMappingFunctionTypes = append(dtokMappingFunctionTypes, tranformType) + dtokMappingFunctionTokenIndexes = append(dtokMappingFunctionTokenIndexes, []int{}) + dtokMappingFunctionIntArgs = append(dtokMappingFunctionIntArgs, transfomArgInt) + dtokMappingFunctionStringArgs = append(dtokMappingFunctionStringArgs, _EMPTY_) + } else { + return nil, &mappingDestinationErr{token, ErrMappingDestinationIndexOutOfRange} + } + } + } + + return &subjectTransform{ + src: src, + dest: dest, + dtoks: dtokens, + stoks: stokens, + dtokmftypes: dtokMappingFunctionTypes, + dtokmftokindexesargs: dtokMappingFunctionTokenIndexes, + dtokmfintargs: dtokMappingFunctionIntArgs, + dtokmfstringargs: dtokMappingFunctionStringArgs, + }, nil +} + +func NewSubjectTransform(src, dest string) (*subjectTransform, error) { + return NewSubjectTransformWithStrict(src, dest, false) +} + +func NewSubjectTransformStrict(src, dest string) (*subjectTransform, error) { + return NewSubjectTransformWithStrict(src, dest, true) +} + +func getMappingFunctionArgs(functionRegEx *regexp.Regexp, token string) []string { + commandStrings := functionRegEx.FindStringSubmatch(token) + if len(commandStrings) > 1 { + return commaSeparatorRegEx.Split(commandStrings[1], -1) + } + return nil +} + +// Helper for mapping functions that take a wildcard index and an integer as arguments +func transformIndexIntArgsHelper(token string, args []string, transformType int16) (int16, []int, int32, string, error) { + if len(args) < 2 { + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationNotEnoughArgs} + } + if len(args) > 2 { + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationTooManyArgs} + } + i, err := strconv.Atoi(strings.Trim(args[0], " ")) + if err != nil { + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg} + } + mappingFunctionIntArg, err := strconv.ParseInt(strings.Trim(args[1], " "), 10, 32) + if err != nil { + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg} + } + + return transformType, []int{i}, int32(mappingFunctionIntArg), _EMPTY_, nil +} + +// Helper to ingest and index the subjectTransform destination token (e.g. $x or {{}}) in the token +// returns a transformation type, and three function arguments: an array of source subject token indexes, +// and a single number (e.g. number of partitions, or a slice size), and a string (e.g.a split delimiter) +func indexPlaceHolders(token string) (int16, []int, int32, string, error) { + length := len(token) + if length > 1 { + // old $1, $2, etc... mapping format still supported to maintain backwards compatibility + if token[0] == '$' { // simple non-partition mapping + tp, err := strconv.Atoi(token[1:]) + if err != nil { + // other things rely on tokens starting with $ so not an error just leave it as is + return NoTransform, []int{-1}, -1, _EMPTY_, nil + } + return Wildcard, []int{tp}, -1, _EMPTY_, nil + } + + // New 'mustache' style mapping + if length > 4 && token[0] == '{' && token[1] == '{' && token[length-2] == '}' && token[length-1] == '}' { + // wildcard(wildcard token index) (equivalent to $) + args := getMappingFunctionArgs(wildcardMappingFunctionRegEx, token) + if args != nil { + if len(args) == 1 && args[0] == _EMPTY_ { + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationNotEnoughArgs} + } + if len(args) == 1 { + tokenIndex, err := strconv.Atoi(strings.Trim(args[0], " ")) + if err != nil { + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg} + } + return Wildcard, []int{tokenIndex}, -1, _EMPTY_, nil + } else { + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationTooManyArgs} + } + } + + // partition(number of partitions, token1, token2, ...) + args = getMappingFunctionArgs(partitionMappingFunctionRegEx, token) + if args != nil { + if len(args) < 1 { + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationNotEnoughArgs} + } + if len(args) == 1 { + mappingFunctionIntArg, err := strconv.Atoi(strings.Trim(args[0], " ")) + if err != nil || mappingFunctionIntArg > math.MaxInt32 { + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg} + } + return Partition, []int{}, int32(mappingFunctionIntArg), _EMPTY_, nil + } + if len(args) >= 2 { + mappingFunctionIntArg, err := strconv.Atoi(strings.Trim(args[0], " ")) + if err != nil || mappingFunctionIntArg > math.MaxInt32 { + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg} + } + var numPositions = len(args[1:]) + tokenIndexes := make([]int, numPositions) + for ti, t := range args[1:] { + i, err := strconv.Atoi(strings.Trim(t, " ")) + if err != nil { + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg} + } + tokenIndexes[ti] = i + } + + return Partition, tokenIndexes, int32(mappingFunctionIntArg), _EMPTY_, nil + } + } + + // SplitFromLeft(token, position) + args = getMappingFunctionArgs(splitFromLeftMappingFunctionRegEx, token) + if args != nil { + return transformIndexIntArgsHelper(token, args, SplitFromLeft) + } + + // SplitFromRight(token, position) + args = getMappingFunctionArgs(splitFromRightMappingFunctionRegEx, token) + if args != nil { + return transformIndexIntArgsHelper(token, args, SplitFromRight) + } + + // SliceFromLeft(token, position) + args = getMappingFunctionArgs(sliceFromLeftMappingFunctionRegEx, token) + if args != nil { + return transformIndexIntArgsHelper(token, args, SliceFromLeft) + } + + // SliceFromRight(token, position) + args = getMappingFunctionArgs(sliceFromRightMappingFunctionRegEx, token) + if args != nil { + return transformIndexIntArgsHelper(token, args, SliceFromRight) + } + + // Right(token, length) + args = getMappingFunctionArgs(rightMappingFunctionRegEx, token) + if args != nil { + return transformIndexIntArgsHelper(token, args, Right) + } + + // Left(token, length) + args = getMappingFunctionArgs(leftMappingFunctionRegEx, token) + if args != nil { + return transformIndexIntArgsHelper(token, args, Left) + } + + // split(token, deliminator) + args = getMappingFunctionArgs(splitMappingFunctionRegEx, token) + if args != nil { + if len(args) < 2 { + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationNotEnoughArgs} + } + if len(args) > 2 { + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationTooManyArgs} + } + i, err := strconv.Atoi(strings.Trim(args[0], " ")) + if err != nil { + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg} + } + if strings.Contains(args[1], " ") || strings.Contains(args[1], tsep) { + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token: token, err: ErrMappingDestinationInvalidArg} + } + + return Split, []int{i}, -1, args[1], nil + } + + // Random(max) + args = getMappingFunctionArgs(randomMappingFunctionRegEx, token) + if args != nil { + if len(args) != 1 { + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationNotEnoughArgs} + } + mappingFunctionIntArg, err := strconv.Atoi(strings.Trim(args[0], " ")) + if err != nil || mappingFunctionIntArg > math.MaxInt32 { + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg} + } + return Random, []int{}, int32(mappingFunctionIntArg), _EMPTY_, nil + } + + return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrUnknownMappingDestinationFunction} + } + } + return NoTransform, []int{-1}, -1, _EMPTY_, nil +} + +// Helper function to tokenize subjects with partial wildcards into formal transform destinations. +// e.g. "foo.*.*" -> "foo.$1.$2" +func transformTokenize(subject string) string { + // We need to make the appropriate markers for the wildcards etc. + i := 1 + var nda []string + for token := range strings.SplitSeq(subject, tsep) { + if token == pwcs { + nda = append(nda, fmt.Sprintf("$%d", i)) + i++ + } else { + nda = append(nda, token) + } + } + return strings.Join(nda, tsep) +} + +// Helper function to go from transform destination to a subject with partial wildcards and ordered list of placeholders +// E.g.: +// +// "bar" -> "bar", [] +// "foo.$2.$1" -> "foo.*.*", ["$2","$1"] +// "foo.{{wildcard(2)}}.{{wildcard(1)}}" -> "foo.*.*", ["{{wildcard(2)}}","{{wildcard(1)}}"] +func transformUntokenize(subject string) (string, []string) { + var phs []string + var nda []string + + for token := range strings.SplitSeq(subject, tsep) { + if args := getMappingFunctionArgs(wildcardMappingFunctionRegEx, token); (len(token) > 1 && token[0] == '$' && token[1] >= '1' && token[1] <= '9') || (len(args) == 1 && args[0] != _EMPTY_) { + phs = append(phs, token) + nda = append(nda, pwcs) + } else { + nda = append(nda, token) + } + } + return strings.Join(nda, tsep), phs +} + +func tokenizeSubject(subject string) []string { + // Tokenize the subject. + tsa := [32]string{} + tts := tsa[:0] + start := 0 + for i := 0; i < len(subject); i++ { + if subject[i] == btsep { + tts = append(tts, subject[start:i]) + start = i + 1 + } + } + tts = append(tts, subject[start:]) + return tts +} + +// Match will take a literal published subject that is associated with a client and will match and subjectTransform +// the subject if possible. +// +// This API is not part of the public API and not subject to SemVer protections +func (tr *subjectTransform) Match(subject string) (string, error) { + // Special case: matches any and no no-op subjectTransform. May not be legal config for some features + // but specific validations made at subjectTransform create time + if (tr.src == fwcs || tr.src == _EMPTY_) && (tr.dest == fwcs || tr.dest == _EMPTY_) { + return subject, nil + } + + tts := tokenizeSubject(subject) + + // TODO(jnm): optimization -> not sure this is actually needed but was there in initial code + if !isValidLiteralSubject(slices.Values(tts)) { + return _EMPTY_, ErrBadSubject + } + + if (tr.src == _EMPTY_ || tr.src == fwcs) || isSubsetMatch(tts, tr.src) { + return tr.TransformTokenizedSubject(tts), nil + } + return _EMPTY_, ErrNoTransforms +} + +// TransformSubject transforms a subject +// +// This API is not part of the public API and not subject to SemVer protection +func (tr *subjectTransform) TransformSubject(subject string) string { + return tr.TransformTokenizedSubject(tokenizeSubject(subject)) +} + +func (tr *subjectTransform) getRandomPartition(ceiling int) string { + // Avoid an integer divide by zero panic below. + if ceiling == 0 { + return "0" + } + + return strconv.Itoa(int(rand.Int32()) % ceiling) +} + +func (tr *subjectTransform) getHashPartition(key []byte, numBuckets int) string { + // Avoid an integer divide by zero panic below. + if numBuckets == 0 { + return "0" + } + + h := fnv.New32a() + _, _ = h.Write(key) + + return strconv.Itoa(int(h.Sum32() % uint32(numBuckets))) +} + +// Do a subjectTransform on the subject to the dest subject. +func (tr *subjectTransform) TransformTokenizedSubject(tokens []string) string { + if len(tr.dtokmftypes) == 0 { + return tr.dest + } + + var b strings.Builder + + // We need to walk destination tokens and create the mapped subject pulling tokens or mapping functions + li := len(tr.dtokmftypes) - 1 + for i, mfType := range tr.dtokmftypes { + if mfType == NoTransform { + // Break if fwc + if len(tr.dtoks[i]) == 1 && tr.dtoks[i][0] == fwc { + break + } + b.WriteString(tr.dtoks[i]) + } else { + switch mfType { + case Partition: + var ( + _buffer [64]byte + keyForHashing = _buffer[:0] + ) + if len(tr.dtokmftokindexesargs[i]) > 0 { + // When token positions are specified. + for _, sourceToken := range tr.dtokmftokindexesargs[i] { + keyForHashing = append(keyForHashing, []byte(tokens[sourceToken])...) + } + } else { + // When using the shorthand partition(n). + keyForHashing = append(keyForHashing, strings.Join(tokens, ".")...) + } + b.WriteString(tr.getHashPartition(keyForHashing, int(tr.dtokmfintargs[i]))) + case Wildcard: // simple substitution + switch { + case len(tr.dtokmftokindexesargs) < i: + break + case len(tr.dtokmftokindexesargs[i]) < 1: + break + case len(tokens) <= tr.dtokmftokindexesargs[i][0]: + break + default: + b.WriteString(tokens[tr.dtokmftokindexesargs[i][0]]) + } + case SplitFromLeft: + sourceToken := tokens[tr.dtokmftokindexesargs[i][0]] + sourceTokenLen := len(sourceToken) + position := int(tr.dtokmfintargs[i]) + if position > 0 && position < sourceTokenLen { + b.WriteString(sourceToken[:position]) + b.WriteString(tsep) + b.WriteString(sourceToken[position:]) + } else { // too small to split at the requested position: don't split + b.WriteString(sourceToken) + } + case SplitFromRight: + sourceToken := tokens[tr.dtokmftokindexesargs[i][0]] + sourceTokenLen := len(sourceToken) + position := int(tr.dtokmfintargs[i]) + if position > 0 && position < sourceTokenLen { + b.WriteString(sourceToken[:sourceTokenLen-position]) + b.WriteString(tsep) + b.WriteString(sourceToken[sourceTokenLen-position:]) + } else { // too small to split at the requested position: don't split + b.WriteString(sourceToken) + } + case SliceFromLeft: + sourceToken := tokens[tr.dtokmftokindexesargs[i][0]] + sourceTokenLen := len(sourceToken) + sliceSize := int(tr.dtokmfintargs[i]) + if sliceSize > 0 && sliceSize < sourceTokenLen { + for i := 0; i+sliceSize <= sourceTokenLen; i += sliceSize { + if i != 0 { + b.WriteString(tsep) + } + b.WriteString(sourceToken[i : i+sliceSize]) + if i+sliceSize != sourceTokenLen && i+sliceSize+sliceSize > sourceTokenLen { + b.WriteString(tsep) + b.WriteString(sourceToken[i+sliceSize:]) + break + } + } + } else { // too small to slice at the requested size: don't slice + b.WriteString(sourceToken) + } + case SliceFromRight: + sourceToken := tokens[tr.dtokmftokindexesargs[i][0]] + sourceTokenLen := len(sourceToken) + sliceSize := int(tr.dtokmfintargs[i]) + if sliceSize > 0 && sliceSize < sourceTokenLen { + remainder := sourceTokenLen % sliceSize + if remainder > 0 { + b.WriteString(sourceToken[:remainder]) + b.WriteString(tsep) + } + for i := remainder; i+sliceSize <= sourceTokenLen; i += sliceSize { + b.WriteString(sourceToken[i : i+sliceSize]) + if i+sliceSize < sourceTokenLen { + b.WriteString(tsep) + } + } + } else { // too small to slice at the requested size: don't slice + b.WriteString(sourceToken) + } + case Split: + sourceToken := tokens[tr.dtokmftokindexesargs[i][0]] + splits := strings.Split(sourceToken, tr.dtokmfstringargs[i]) + for j, split := range splits { + if split != _EMPTY_ { + b.WriteString(split) + } + if j < len(splits)-1 && splits[j+1] != _EMPTY_ && !(j == 0 && split == _EMPTY_) { + b.WriteString(tsep) + } + } + case Left: + sourceToken := tokens[tr.dtokmftokindexesargs[i][0]] + sourceTokenLen := len(sourceToken) + sliceSize := int(tr.dtokmfintargs[i]) + if sliceSize > 0 && sliceSize < sourceTokenLen { + b.WriteString(sourceToken[0:sliceSize]) + } else { // too small to slice at the requested size: don't slice + b.WriteString(sourceToken) + } + case Right: + sourceToken := tokens[tr.dtokmftokindexesargs[i][0]] + sourceTokenLen := len(sourceToken) + sliceSize := int(tr.dtokmfintargs[i]) + if sliceSize > 0 && sliceSize < sourceTokenLen { + b.WriteString(sourceToken[sourceTokenLen-sliceSize : sourceTokenLen]) + } else { // too small to slice at the requested size: don't slice + b.WriteString(sourceToken) + } + case Random: + b.WriteString(tr.getRandomPartition(int(tr.dtokmfintargs[i]))) + } + } + + if i < li { + b.WriteByte(btsep) + } + } + + // We may have more source tokens available. This happens with ">". + if tr.dtoks[len(tr.dtoks)-1] == fwcs { + for sli, i := len(tokens)-1, len(tr.stoks)-1; i < len(tokens); i++ { + b.WriteString(tokens[i]) + if i < sli { + b.WriteByte(btsep) + } + } + } + return b.String() +} + +// Reverse a subjectTransform. +func (tr *subjectTransform) reverse() *subjectTransform { + if len(tr.dtokmftokindexesargs) == 0 { + rtr, _ := NewSubjectTransformStrict(tr.dest, tr.src) + return rtr + } + // If we are here we need to dynamically get the correct reverse + // of this subjectTransform. + nsrc, phs := transformUntokenize(tr.dest) + var nda []string + for _, token := range tr.stoks { + if token == pwcs { + if len(phs) == 0 { + // TODO(dlc) - Should not happen + return nil + } + nda = append(nda, phs[0]) + phs = phs[1:] + } else { + nda = append(nda, token) + } + } + ndest := strings.Join(nda, tsep) + rtr, _ := NewSubjectTransformStrict(nsrc, ndest) + return rtr +} + +// Will share relevant info regarding the subject. +// Returns valid, tokens, num pwcs, has fwc. +func subjectInfo(subject string) (bool, []string, int, bool) { + if subject == "" { + return false, nil, 0, false + } + npwcs := 0 + sfwc := false + tokens := strings.Split(subject, tsep) + for _, t := range tokens { + if len(t) == 0 || sfwc { + return false, nil, 0, false + } + if len(t) > 1 { + continue + } + switch t[0] { + case fwc: + sfwc = true + case pwc: + npwcs++ + } + } + return true, tokens, npwcs, sfwc +} diff --git a/server/subject_transform_test.go b/server/subject_transform_test.go new file mode 100644 index 0000000..12c67d0 --- /dev/null +++ b/server/subject_transform_test.go @@ -0,0 +1,261 @@ +// Copyright 2023-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "errors" + "fmt" + "math" + "reflect" + "slices" + "testing" +) + +func TestPlaceHolderIndex(t *testing.T) { + testString := "$1" + transformType, indexes, nbPartitions, _, err := indexPlaceHolders(testString) + var position int32 + + if err != nil || transformType != Wildcard || len(indexes) != 1 || indexes[0] != 1 || nbPartitions != -1 { + t.Fatalf("Error parsing %s", testString) + } + + testString = "{{partition(10,1,2,3)}}" + + transformType, indexes, nbPartitions, _, err = indexPlaceHolders(testString) + + if err != nil || transformType != Partition || !reflect.DeepEqual(indexes, []int{1, 2, 3}) || nbPartitions != 10 { + t.Fatalf("Error parsing %s", testString) + } + + testString = "{{ Partition (10,1,2,3) }}" + + transformType, indexes, nbPartitions, _, err = indexPlaceHolders(testString) + + if err != nil || transformType != Partition || !reflect.DeepEqual(indexes, []int{1, 2, 3}) || nbPartitions != 10 { + t.Fatalf("Error parsing %s", testString) + } + + testString = "{{wildcard(2)}}" + transformType, indexes, nbPartitions, _, err = indexPlaceHolders(testString) + + if err != nil || transformType != Wildcard || len(indexes) != 1 || indexes[0] != 2 || nbPartitions != -1 { + t.Fatalf("Error parsing %s", testString) + } + + testString = "{{SplitFromLeft(2,1)}}" + transformType, indexes, position, _, err = indexPlaceHolders(testString) + + if err != nil || transformType != SplitFromLeft || len(indexes) != 1 || indexes[0] != 2 || position != 1 { + t.Fatalf("Error parsing %s", testString) + } + + testString = "{{SplitFromRight(3,2)}}" + transformType, indexes, position, _, err = indexPlaceHolders(testString) + + if err != nil || transformType != SplitFromRight || len(indexes) != 1 || indexes[0] != 3 || position != 2 { + t.Fatalf("Error parsing %s", testString) + } + + testString = "{{SliceFromLeft(2,2)}}" + transformType, indexes, sliceSize, _, err := indexPlaceHolders(testString) + + if err != nil || transformType != SliceFromLeft || len(indexes) != 1 || indexes[0] != 2 || sliceSize != 2 { + t.Fatalf("Error parsing %s", testString) + } + + testString = "{{Left(3,2)}}" + transformType, indexes, position, _, err = indexPlaceHolders(testString) + + if err != nil || transformType != Left || len(indexes) != 1 || indexes[0] != 3 || position != 2 { + t.Fatalf("Error parsing %s", testString) + } + + testString = "{{Right(3,2)}}" + transformType, indexes, position, _, err = indexPlaceHolders(testString) + + if err != nil || transformType != Right || len(indexes) != 1 || indexes[0] != 3 || position != 2 { + t.Fatalf("Error parsing %s", testString) + } +} + +func TestSubjectTransformHelpers(t *testing.T) { + equals := func(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i, v := range a { + if v != b[i] { + return false + } + } + return true + } + + filter, placeHolders := transformUntokenize("bar") + if filter != "bar" || len(placeHolders) != 0 { + t.Fatalf("transformUntokenize for not returning expected result") + } + + filter, placeHolders = transformUntokenize("foo.$2.$1") + if filter != "foo.*.*" || !equals(placeHolders, []string{"$2", "$1"}) { + t.Fatalf("transformUntokenize for not returning expected result") + } + + filter, placeHolders = transformUntokenize("foo.{{wildcard(2)}}.{{wildcard(1)}}") + if filter != "foo.*.*" || !equals(placeHolders, []string{"{{wildcard(2)}}", "{{wildcard(1)}}"}) { + t.Fatalf("transformUntokenize for not returning expected result") + } + + newReversibleTransform := func(src, dest string) *subjectTransform { + tr, err := NewSubjectTransformStrict(src, dest) + if err != nil { + t.Fatalf("Error getting reversible transform: %s to %s", src, dest) + } + return tr + } + + tr := newReversibleTransform("foo.*.*", "bar.$2.{{Wildcard(1)}}") + subject := "foo.b.a" + transformed := tr.TransformSubject(subject) + reverse := tr.reverse() + if reverse.TransformSubject(transformed) != subject { + t.Fatal("Reversed transform subject not matching") + } +} + +func TestSubjectTransforms(t *testing.T) { + shouldErr := func(src, dest string, strict bool) { + t.Helper() + if _, err := NewSubjectTransformWithStrict(src, dest, strict); err != ErrBadSubject && !errors.Is(err, ErrInvalidMappingDestination) { + t.Fatalf("Did not get an error for src=%q and dest=%q", src, dest) + } + } + + // Must be valid subjects. + shouldErr("foo..", "bar", false) + + // Wildcards are allowed in src, but must be matched by token placements on the other side. + // e.g. foo.* -> bar.$1. + // Need to have as many pwcs as placements on other side + + shouldErr("foo.*", "bar.*", false) + shouldErr("foo.*", "bar.$2", false) // Bad pwc token identifier + shouldErr("foo.*", "bar.$1.>", false) // fwcs have to match. + shouldErr("foo.>", "bar.baz", false) // fwcs have to match. + shouldErr("foo.*.*", "bar.$2", true) // Must place all pwcs. + shouldErr("foo.*", "foo.$foo", true) // invalid $ value + shouldErr("foo.*", "bar.{{Partition(2,1)}}", true) // can only use Wildcard function (and old-style $x) in import transform + shouldErr("foo.*", "foo.{{wildcard(2)}}", false) // Mapping function being passed an out of range wildcard index + shouldErr("foo.*", "foo.{{unimplemented(1)}}", false) // Mapping trying to use an unknown mapping function + shouldErr("foo.*", "foo.{{partition()}}", false) // Not enough arguments passed to the mapping function + shouldErr("foo.*", "foo.{{random()}}", false) // Not enough arguments passed to the random function + shouldErr("foo.*", "foo.{{wildcard(foo)}}", false) // Invalid argument passed to the mapping function + shouldErr("foo.*", "foo.{{wildcard()}}", false) // Not enough arguments passed to the mapping function + shouldErr("foo.*", "foo.{{wildcard(1,2)}}", false) // Too many arguments passed to the mapping function + shouldErr("foo.*", "foo.{{ wildcard5) }}", false) // Bad mapping function + shouldErr("foo.*", "foo.{{splitLeft(2,2}}", false) // arg out of range + shouldErr("foo", "bla.{{wildcard(1)}}", false) // arg out of range with no wildcard in the source + + shouldErr("foo.*", fmt.Sprintf("foo.{{partition(%d)}}", math.MaxInt32+1), false) // Larger than int32 + shouldErr("foo.*", fmt.Sprintf("foo.{{random(%d)}}", math.MaxInt32+1), false) // Larger than int32 + + shouldBeOK := func(src, dest string, strict bool) *subjectTransform { + t.Helper() + tr, err := NewSubjectTransformWithStrict(src, dest, strict) + if err != nil { + t.Fatalf("Got an error %v for src=%q and dest=%q", err, src, dest) + } + return tr + } + + shouldBeOK("foo.*", "bar.{{Wildcard(1)}}", true) + + shouldBeOK("foo.*.*", "bar.$2", false) // don't have to use all pwcs. + shouldBeOK("foo.*.*", "bar.{{wildcard(1)}}", false) // don't have to use all pwcs. + shouldBeOK("foo.*.*", "bar.{{partition(1)}}", false) + shouldBeOK("foo.*.*", "bar.{{random(5)}}", false) + shouldBeOK("foo", "bar", false) + shouldBeOK("foo.*.bar.*.baz", "req.$2.$1", false) + shouldBeOK("baz.>", "mybaz.>", false) + shouldBeOK("*", "{{splitfromleft(1,1)}}", false) + shouldBeOK("", "prefix.>", false) + shouldBeOK("*.*", "{{partition(10,1,2)}}", false) + shouldBeOK("foo.*.*", "foo.{{wildcard(1)}}.{{wildcard(2)}}.{{partition(5,1,2)}}", false) + + shouldBeOK("foo.*", fmt.Sprintf("foo.{{partition(%d)}}", math.MaxInt32), false) // Exactly int32 + shouldBeOK("foo.*", fmt.Sprintf("foo.{{random(%d)}}", math.MaxInt32), false) // Exactly int32 + shouldBeOK("foo.bar", fmt.Sprintf("foo.{{random(%d)}}", math.MaxInt32), false) // Exactly int32 + + shouldMatch := func(src, dest, sample string, expected ...string) { + t.Helper() + tr := shouldBeOK(src, dest, false) + if tr != nil { + s, err := tr.Match(sample) + if err != nil { + t.Fatalf("Got an error %v when expecting a match for %q to %q", err, sample, expected) + } + if !slices.Contains(expected, s) { + t.Fatalf("Dest does not match what was expected. Got %q, expected %q", s, expected) + } + } + } + + shouldMatch("", "prefix.>", "foo", "prefix.foo") + shouldMatch("foo", "", "foo", "foo") + shouldMatch("foo", "bar", "foo", "bar") + shouldMatch("foo.*.bar.*.baz", "req.$2.$1", "foo.A.bar.B.baz", "req.B.A") + shouldMatch("foo.*.bar.*.baz", "req.{{wildcard(2)}}.{{wildcard(1)}}", "foo.A.bar.B.baz", "req.B.A") + shouldMatch("baz.>", "my.pre.>", "baz.1.2.3", "my.pre.1.2.3") + shouldMatch("baz.>", "foo.bar.>", "baz.1.2.3", "foo.bar.1.2.3") + shouldMatch("*", "foo.bar.$1", "foo", "foo.bar.foo") + shouldMatch("*", "{{splitfromleft(1,3)}}", "12345", "123.45") + shouldMatch("*", "{{SplitFromRight(1,3)}}", "12345", "12.345") + shouldMatch("*", "{{SliceFromLeft(1,3)}}", "1234567890", "123.456.789.0") + shouldMatch("*", "{{SliceFromRight(1,3)}}", "1234567890", "1.234.567.890") + shouldMatch("*", "{{split(1,-)}}", "-abc-def--ghi-", "abc.def.ghi") + shouldMatch("*", "{{split(1,-)}}", "abc-def--ghi-", "abc.def.ghi") + shouldMatch("*.*", "{{split(2,-)}}.{{splitfromleft(1,2)}}", "foo.-abc-def--ghij-", "abc.def.ghij.fo.o") // combo + checks split for multiple instance of deliminator and deliminator being at the start or end + shouldMatch("*", "{{right(1,1)}}", "1234", "4") + shouldMatch("*", "{{right(1,3)}}", "1234", "234") + shouldMatch("*", "{{right(1,6)}}", "1234", "1234") + shouldMatch("*", "{{left(1,1)}}", "1234", "1") + shouldMatch("*", "{{left(1,3)}}", "1234", "123") + shouldMatch("*", "{{left(1,6)}}", "1234", "1234") + shouldMatch("*", "bar.{{partition(0)}}", "baz", "bar.0") + shouldMatch("*", "bar.{{partition(10, 0)}}", "foo", "bar.3") + shouldMatch("*.*", "bar.{{partition(10)}}", "foo.bar", "bar.6") + shouldMatch("*", "bar.{{partition(10)}}", "foo", "bar.3") + shouldMatch("*", "bar.{{partition(10)}}", "baz", "bar.0") + shouldMatch("*", "bar.{{partition(10)}}", "qux", "bar.9") + shouldMatch("*", "bar.{{random(0)}}", "qux", "bar.0") + for range 100 { + shouldMatch("*", "bar.{{random(6)}}", "qux", "bar.0", "bar.1", "bar.2", "bar.3", "bar.4", "bar.5") + } + shouldBeOK("foo.bar", "baz.{{partition(10)}}", false) + shouldMatch("foo.bar", "baz.{{partition(10)}}", "foo.bar", "baz.6") + shouldMatch("foo.baz", "qux.{{partition(10)}}", "foo.baz", "qux.4") + shouldMatch("test.subject", "result.{{partition(5)}}", "test.subject", "result.0") +} + +func TestSubjectTransformDoesntPanicTransformingMissingToken(t *testing.T) { + defer func() { + p := recover() + require_True(t, p == nil) + }() + + tr, err := NewSubjectTransform("foo.*", "one.two.{{wildcard(1)}}") + require_NoError(t, err) + require_Equal(t, tr.TransformTokenizedSubject([]string{"foo"}), "one.two.") +} diff --git a/server/sublist.go b/server/sublist.go new file mode 100644 index 0000000..f26bbc1 --- /dev/null +++ b/server/sublist.go @@ -0,0 +1,1744 @@ +// Copyright 2016-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "errors" + "iter" + "strings" + "sync" + "sync/atomic" + "unicode/utf8" +) + +// Sublist is a routing mechanism to handle subject distribution and +// provides a facility to match subjects from published messages to +// interested subscribers. Subscribers can have wildcard subjects to +// match multiple published subjects. + +// Common byte variables for wildcards and token separator. +const ( + pwc = '*' + pwcs = "*" + fwc = '>' + fwcs = ">" + tsep = "." + btsep = '.' +) + +// Sublist related errors +var ( + ErrInvalidSubject = errors.New("sublist: invalid subject") + ErrNotFound = errors.New("sublist: no matches found") + ErrNilChan = errors.New("sublist: nil channel") + ErrAlreadyRegistered = errors.New("sublist: notification already registered") +) + +const ( + // cacheMax is used to bound limit the frontend cache + slCacheMax = 1024 + // If we run a sweeper we will drain to this count. + slCacheSweep = 256 + // plistMin is our lower bounds to create a fast plist for Match. + plistMin = 256 +) + +// SublistResult is a result structure better optimized for queue subs. +type SublistResult struct { + psubs []*subscription + qsubs [][]*subscription // don't make this a map, too expensive to iterate +} + +// A Sublist stores and efficiently retrieves subscriptions. +type Sublist struct { + sync.RWMutex + genid uint64 + matches uint64 + cacheHits uint64 + inserts uint64 + removes uint64 + root *level + cache map[string]*SublistResult + ccSweep int32 + notify *notifyMaps + count uint32 +} + +// notifyMaps holds maps of arrays of channels for notifications +// on a change of interest. +type notifyMaps struct { + insert map[string][]chan<- bool + remove map[string][]chan<- bool +} + +// A node contains subscriptions and a pointer to the next level. +type node struct { + next *level + psubs map[*subscription]struct{} + qsubs map[string]map[*subscription]struct{} + plist []*subscription +} + +// A level represents a group of nodes and special pointers to +// wildcard nodes. +type level struct { + nodes map[string]*node + pwc, fwc *node +} + +// Create a new default node. +func newNode() *node { + return &node{psubs: make(map[*subscription]struct{})} +} + +// Create a new default level. +func newLevel() *level { + return &level{nodes: make(map[string]*node)} +} + +// In general caching is recommended however in some extreme cases where +// interest changes are high, suppressing the cache can help. +// https://github.com/nats-io/nats-server/issues/941 +// FIXME(dlc) - should be more dynamic at some point based on cache thrashing. + +// NewSublist will create a default sublist with caching enabled per the flag. +func NewSublist(enableCache bool) *Sublist { + if enableCache { + return &Sublist{root: newLevel(), cache: make(map[string]*SublistResult)} + } + return &Sublist{root: newLevel()} +} + +// NewSublistForServer will create a default sublist with caching enabled determined +// by the server options. +func NewSublistForServer(srv *Server) *Sublist { + if srv == nil { + return NewSublistNoCache() // Probably just unit tests. + } + if opts := srv.getOpts(); opts != nil { + return NewSublist(!opts.NoSublistCache) + } + return NewSublistNoCache() +} + +// NewSublistWithCache will create a default sublist with caching enabled. +func NewSublistWithCache() *Sublist { + return NewSublist(true) +} + +// NewSublistNoCache will create a default sublist with caching disabled. +func NewSublistNoCache() *Sublist { + return NewSublist(false) +} + +// CacheEnabled returns whether or not caching is enabled for this sublist. +func (s *Sublist) CacheEnabled() bool { + s.RLock() + enabled := s.cache != nil + s.RUnlock() + return enabled +} + +// RegisterNotification will register for notifications when interest for the given +// subject changes. The subject must be a literal publish type subject. +// The notification is true for when the first interest for a subject is inserted, +// and false when all interest in the subject is removed. Note that this interest +// needs to be exact and that wildcards will not trigger the notifications. The sublist +// will not block when trying to send the notification. Its up to the caller to make +// sure the channel send will not block. +func (s *Sublist) RegisterNotification(subject string, notify chan<- bool) error { + return s.registerNotification(subject, _EMPTY_, notify) +} + +func (s *Sublist) RegisterQueueNotification(subject, queue string, notify chan<- bool) error { + return s.registerNotification(subject, queue, notify) +} + +func (s *Sublist) registerNotification(subject, queue string, notify chan<- bool) error { + if subjectHasWildcard(subject) { + return ErrInvalidSubject + } + if notify == nil { + return ErrNilChan + } + + var hasInterest bool + r := s.Match(subject) + + if len(r.psubs)+len(r.qsubs) > 0 { + if queue == _EMPTY_ { + for _, sub := range r.psubs { + if string(sub.subject) == subject { + hasInterest = true + break + } + } + } else { + for _, qsub := range r.qsubs { + qs := qsub[0] + if string(qs.subject) == subject && string(qs.queue) == queue { + hasInterest = true + break + } + } + } + } + + key := keyFromSubjectAndQueue(subject, queue) + var err error + + s.Lock() + if s.notify == nil { + s.notify = ¬ifyMaps{ + insert: make(map[string][]chan<- bool), + remove: make(map[string][]chan<- bool), + } + } + // Check which list to add us to. + if hasInterest { + err = s.addRemoveNotify(key, notify) + } else { + err = s.addInsertNotify(key, notify) + } + s.Unlock() + + if err == nil { + sendNotification(notify, hasInterest) + } + return err +} + +// Lock should be held. +func chkAndRemove(key string, notify chan<- bool, ms map[string][]chan<- bool) bool { + chs := ms[key] + for i, ch := range chs { + if ch == notify { + chs[i] = chs[len(chs)-1] + chs = chs[:len(chs)-1] + if len(chs) == 0 { + delete(ms, key) + } + return true + } + } + return false +} + +func (s *Sublist) ClearNotification(subject string, notify chan<- bool) bool { + return s.clearNotification(subject, _EMPTY_, notify) +} + +func (s *Sublist) ClearQueueNotification(subject, queue string, notify chan<- bool) bool { + return s.clearNotification(subject, queue, notify) +} + +func (s *Sublist) clearNotification(subject, queue string, notify chan<- bool) bool { + s.Lock() + if s.notify == nil { + s.Unlock() + return false + } + key := keyFromSubjectAndQueue(subject, queue) + // Check both, start with remove. + didRemove := chkAndRemove(key, notify, s.notify.remove) + didRemove = didRemove || chkAndRemove(key, notify, s.notify.insert) + // Check if everything is gone + if len(s.notify.remove)+len(s.notify.insert) == 0 { + s.notify = nil + } + s.Unlock() + return didRemove +} + +func sendNotification(ch chan<- bool, hasInterest bool) { + select { + case ch <- hasInterest: + default: + } +} + +// Add a new channel for notification in insert map. +// Write lock should be held. +func (s *Sublist) addInsertNotify(subject string, notify chan<- bool) error { + return s.addNotify(s.notify.insert, subject, notify) +} + +// Add a new channel for notification in removal map. +// Write lock should be held. +func (s *Sublist) addRemoveNotify(subject string, notify chan<- bool) error { + return s.addNotify(s.notify.remove, subject, notify) +} + +// Add a new channel for notification. +// Write lock should be held. +func (s *Sublist) addNotify(m map[string][]chan<- bool, subject string, notify chan<- bool) error { + chs := m[subject] + if len(chs) > 0 { + // Check to see if this chan is already registered. + for _, ch := range chs { + if ch == notify { + return ErrAlreadyRegistered + } + } + } + + m[subject] = append(chs, notify) + return nil +} + +// To generate a key from subject and queue. We just add spc. +func keyFromSubjectAndQueue(subject, queue string) string { + if len(queue) == 0 { + return subject + } + var sb strings.Builder + sb.WriteString(subject) + sb.WriteString(" ") + sb.WriteString(queue) + return sb.String() +} + +// chkForInsertNotification will check to see if we need to notify on this subject. +// Write lock should be held. +func (s *Sublist) chkForInsertNotification(subject, queue string) { + key := keyFromSubjectAndQueue(subject, queue) + + // All notify subjects are also literal so just do a hash lookup here. + if chs := s.notify.insert[key]; len(chs) > 0 { + for _, ch := range chs { + sendNotification(ch, true) + } + // Move from the insert map to the remove map. + s.notify.remove[key] = append(s.notify.remove[key], chs...) + delete(s.notify.insert, key) + } +} + +// chkForRemoveNotification will check to see if we need to notify on this subject. +// Write lock should be held. +func (s *Sublist) chkForRemoveNotification(subject, queue string) { + key := keyFromSubjectAndQueue(subject, queue) + if chs := s.notify.remove[key]; len(chs) > 0 { + // We need to always check that we have no interest anymore. + var hasInterest bool + r := s.matchNoLock(subject) + + if len(r.psubs)+len(r.qsubs) > 0 { + if queue == _EMPTY_ { + for _, sub := range r.psubs { + if string(sub.subject) == subject { + hasInterest = true + break + } + } + } else { + for _, qsub := range r.qsubs { + qs := qsub[0] + if string(qs.subject) == subject && string(qs.queue) == queue { + hasInterest = true + break + } + } + } + } + if !hasInterest { + for _, ch := range chs { + sendNotification(ch, false) + } + // Move from the remove map to the insert map. + s.notify.insert[key] = append(s.notify.insert[key], chs...) + delete(s.notify.remove, key) + } + } +} + +// Insert adds a subscription into the sublist +func (s *Sublist) Insert(sub *subscription) error { + // copy the subject since we hold this and this might be part of a large byte slice. + subject := string(sub.subject) + + s.Lock() + + var sfwc, haswc, isnew bool + var n *node + l := s.root + + for t := range strings.SplitSeq(subject, tsep) { + lt := len(t) + if lt == 0 || sfwc { + s.Unlock() + return ErrInvalidSubject + } + + if lt > 1 { + n = l.nodes[t] + } else { + switch t[0] { + case pwc: + n = l.pwc + haswc = true + case fwc: + n = l.fwc + haswc, sfwc = true, true + default: + n = l.nodes[t] + } + } + if n == nil { + n = newNode() + if lt > 1 { + l.nodes[t] = n + } else { + switch t[0] { + case pwc: + l.pwc = n + case fwc: + l.fwc = n + default: + l.nodes[t] = n + } + } + } + if n.next == nil { + n.next = newLevel() + } + l = n.next + } + if sub.queue == nil { + n.psubs[sub] = struct{}{} + isnew = len(n.psubs) == 1 + if n.plist != nil { + n.plist = append(n.plist, sub) + } else if len(n.psubs) > plistMin { + n.plist = make([]*subscription, 0, len(n.psubs)) + // Populate + for psub := range n.psubs { + n.plist = append(n.plist, psub) + } + } + } else { + if n.qsubs == nil { + n.qsubs = make(map[string]map[*subscription]struct{}) + } + qname := string(sub.queue) + // This is a queue subscription + subs, ok := n.qsubs[qname] + if !ok { + subs = make(map[*subscription]struct{}) + n.qsubs[qname] = subs + isnew = true + } + subs[sub] = struct{}{} + } + + s.count++ + s.inserts++ + + s.addToCache(subject, sub) + atomic.AddUint64(&s.genid, 1) + + if s.notify != nil && isnew && !haswc && len(s.notify.insert) > 0 { + s.chkForInsertNotification(subject, string(sub.queue)) + } + s.Unlock() + + return nil +} + +// Deep copy +func copyResult(r *SublistResult) *SublistResult { + nr := &SublistResult{} + nr.psubs = append([]*subscription(nil), r.psubs...) + for _, qr := range r.qsubs { + nqr := append([]*subscription(nil), qr...) + nr.qsubs = append(nr.qsubs, nqr) + } + return nr +} + +// Adds a new sub to an existing result. +func (r *SublistResult) addSubToResult(sub *subscription) *SublistResult { + // Copy since others may have a reference. + nr := copyResult(r) + if sub.queue == nil { + nr.psubs = append(nr.psubs, sub) + } else { + if i := findQSlot(sub.queue, nr.qsubs); i >= 0 { + nr.qsubs[i] = append(nr.qsubs[i], sub) + } else { + nr.qsubs = append(nr.qsubs, []*subscription{sub}) + } + } + return nr +} + +// addToCache will add the new entry to the existing cache +// entries if needed. Assumes write lock is held. +// Assumes write lock is held. +func (s *Sublist) addToCache(subject string, sub *subscription) { + if s.cache == nil { + return + } + // If literal we can direct match. + if subjectIsLiteral(subject) { + if r := s.cache[subject]; r != nil { + s.cache[subject] = r.addSubToResult(sub) + } + return + } + for key, r := range s.cache { + if matchLiteral(key, subject) { + s.cache[key] = r.addSubToResult(sub) + } + } +} + +// removeFromCache will remove the sub from any active cache entries. +// Assumes write lock is held. +func (s *Sublist) removeFromCache(subject string) { + if s.cache == nil { + return + } + // If literal we can direct match. + if subjectIsLiteral(subject) { + delete(s.cache, subject) + return + } + // Wildcard here. + for key := range s.cache { + if matchLiteral(key, subject) { + delete(s.cache, key) + } + } +} + +// a place holder for an empty result. +var emptyResult = &SublistResult{} + +// Match will match all entries to the literal subject. +// It will return a set of results for both normal and queue subscribers. +func (s *Sublist) Match(subject string) *SublistResult { + return s.match(subject, true, false) +} + +// MatchBytes will match all entries to the literal subject. +// It will return a set of results for both normal and queue subscribers. +func (s *Sublist) MatchBytes(subject []byte) *SublistResult { + return s.match(bytesToString(subject), true, true) +} + +// HasInterest will return whether or not there is any interest in the subject. +// In cases where more detail is not required, this may be faster than Match. +func (s *Sublist) HasInterest(subject string) bool { + return s.hasInterest(subject, true, nil, nil) +} + +// NumInterest will return the number of subs/qsubs interested in the subject. +// In cases where more detail is not required, this may be faster than Match. +func (s *Sublist) NumInterest(subject string) (np, nq int) { + s.hasInterest(subject, true, &np, &nq) + return +} + +func (s *Sublist) matchNoLock(subject string) *SublistResult { + return s.match(subject, false, false) +} + +func (s *Sublist) match(subject string, doLock bool, doCopyOnCache bool) *SublistResult { + atomic.AddUint64(&s.matches, 1) + + // Check cache first. + if doLock { + s.RLock() + } + cacheEnabled := s.cache != nil + r, ok := s.cache[subject] + if doLock { + s.RUnlock() + } + if ok { + atomic.AddUint64(&s.cacheHits, 1) + return r + } + + tsa := [32]string{} + tokens := tsa[:0] + start := 0 + for i := 0; i < len(subject); i++ { + if subject[i] == btsep { + if i-start == 0 { + return emptyResult + } + tokens = append(tokens, subject[start:i]) + start = i + 1 + } + } + if start >= len(subject) { + return emptyResult + } + tokens = append(tokens, subject[start:]) + + // FIXME(dlc) - Make shared pool between sublist and client readLoop? + result := &SublistResult{} + + // Get result from the main structure and place into the shared cache. + // Hold the read lock to avoid race between match and store. + var n int + + if doLock { + if cacheEnabled { + s.Lock() + } else { + s.RLock() + } + } + + matchLevel(s.root, tokens, result) + // Check for empty result. + if len(result.psubs) == 0 && len(result.qsubs) == 0 { + result = emptyResult + } + if cacheEnabled { + if doCopyOnCache { + subject = copyString(subject) + } + s.cache[subject] = result + n = len(s.cache) + } + if doLock { + if cacheEnabled { + s.Unlock() + } else { + s.RUnlock() + } + } + + // Reduce the cache count if we have exceeded our set maximum. + if cacheEnabled && n > slCacheMax && atomic.CompareAndSwapInt32(&s.ccSweep, 0, 1) { + go s.reduceCacheCount() + } + + return result +} + +func (s *Sublist) hasInterest(subject string, doLock bool, np, nq *int) bool { + // Check cache first. + if doLock { + s.RLock() + } + var matched, ok bool + if s.cache != nil { + var r *SublistResult + if r, ok = s.cache[subject]; ok { + if np != nil && nq != nil { + *np += len(r.psubs) + for _, qsub := range r.qsubs { + *nq += len(qsub) + } + } + matched = len(r.psubs)+len(r.qsubs) > 0 + } + } + if doLock { + s.RUnlock() + } + if ok { + atomic.AddUint64(&s.cacheHits, 1) + return matched + } + + tsa := [32]string{} + tokens := tsa[:0] + start := 0 + for i := 0; i < len(subject); i++ { + if subject[i] == btsep { + if i-start == 0 { + return false + } + tokens = append(tokens, subject[start:i]) + start = i + 1 + } + } + if start >= len(subject) { + return false + } + tokens = append(tokens, subject[start:]) + + if doLock { + s.RLock() + defer s.RUnlock() + } + return matchLevelForAny(s.root, tokens, np, nq) +} + +// Remove entries in the cache until we are under the maximum. +// TODO(dlc) this could be smarter now that its not inline. +func (s *Sublist) reduceCacheCount() { + defer atomic.StoreInt32(&s.ccSweep, 0) + // If we are over the cache limit randomly drop until under the limit. + s.Lock() + for key := range s.cache { + delete(s.cache, key) + if len(s.cache) <= slCacheSweep { + break + } + } + s.Unlock() +} + +// Helper function for auto-expanding remote qsubs. +func isRemoteQSub(sub *subscription) bool { + return sub != nil && sub.queue != nil && sub.client != nil && (sub.client.kind == ROUTER || sub.client.kind == LEAF) +} + +// UpdateRemoteQSub should be called when we update the weight of an existing +// remote queue sub. +func (s *Sublist) UpdateRemoteQSub(sub *subscription) { + // We could search to make sure we find it, but probably not worth + // it unless we are thrashing the cache. Just remove from our L2 and update + // the genid so L1 will be flushed. + s.Lock() + s.removeFromCache(string(sub.subject)) + atomic.AddUint64(&s.genid, 1) + s.Unlock() +} + +// This will add in a node's results to the total results. +func addNodeToResults(n *node, results *SublistResult) { + // Normal subscriptions + if n.plist != nil { + results.psubs = append(results.psubs, n.plist...) + } else { + for psub := range n.psubs { + results.psubs = append(results.psubs, psub) + } + } + // Queue subscriptions + for qname, qr := range n.qsubs { + if len(qr) == 0 { + continue + } + // Need to find matching list in results + var i int + if i = findQSlot([]byte(qname), results.qsubs); i < 0 { + i = len(results.qsubs) + nqsub := make([]*subscription, 0, len(qr)) + results.qsubs = append(results.qsubs, nqsub) + } + for sub := range qr { + if isRemoteQSub(sub) { + ns := atomic.LoadInt32(&sub.qw) + // Shadow these subscriptions + for n := 0; n < int(ns); n++ { + results.qsubs[i] = append(results.qsubs[i], sub) + } + } else { + results.qsubs[i] = append(results.qsubs[i], sub) + } + } + } +} + +// We do not use a map here since we want iteration to be past when +// processing publishes in L1 on client. So we need to walk sequentially +// for now. Keep an eye on this in case we start getting large number of +// different queue subscribers for the same subject. +func findQSlot(queue []byte, qsl [][]*subscription) int { + if queue == nil { + return -1 + } + for i, qr := range qsl { + if len(qr) > 0 && bytes.Equal(queue, qr[0].queue) { + return i + } + } + return -1 +} + +// matchLevel is used to recursively descend into the trie. +func matchLevel(l *level, toks []string, results *SublistResult) { + var pwc, n *node + for i, t := range toks { + if l == nil { + return + } + if l.fwc != nil { + addNodeToResults(l.fwc, results) + } + if pwc = l.pwc; pwc != nil { + matchLevel(pwc.next, toks[i+1:], results) + } + n = l.nodes[t] + if n != nil { + l = n.next + } else { + l = nil + } + } + if n != nil { + addNodeToResults(n, results) + } + if pwc != nil { + addNodeToResults(pwc, results) + } +} + +func matchLevelForAny(l *level, toks []string, np, nq *int) bool { + var pwc, n *node + for i, t := range toks { + if l == nil { + return false + } + if l.fwc != nil { + if np != nil && nq != nil { + *np += len(l.fwc.psubs) + for _, qsub := range l.fwc.qsubs { + *nq += len(qsub) + } + } + return true + } + if pwc = l.pwc; pwc != nil { + if match := matchLevelForAny(pwc.next, toks[i+1:], np, nq); match { + return true + } + } + n = l.nodes[t] + if n != nil { + l = n.next + } else { + l = nil + } + } + if n != nil { + if np != nil && nq != nil { + *np += len(n.psubs) + for _, qsub := range n.qsubs { + *nq += len(qsub) + } + } + if len(n.plist) > 0 || len(n.psubs) > 0 || len(n.qsubs) > 0 { + return true + } + } + if pwc != nil { + if np != nil && nq != nil { + *np += len(pwc.psubs) + for _, qsub := range pwc.qsubs { + *nq += len(qsub) + } + } + return len(pwc.plist) > 0 || len(pwc.psubs) > 0 || len(pwc.qsubs) > 0 + } + return false +} + +// lnt is used to track descent into levels for a removal for pruning. +type lnt struct { + l *level + n *node + t string +} + +// Raw low level remove, can do batches with lock held outside. +func (s *Sublist) remove(sub *subscription, shouldLock bool, doCacheUpdates bool) error { + subject := string(sub.subject) + + if shouldLock { + s.Lock() + defer s.Unlock() + } + + var sfwc, haswc bool + var n *node + l := s.root + + // Track levels for pruning + var lnts [32]lnt + levels := lnts[:0] + + for t := range strings.SplitSeq(subject, tsep) { + lt := len(t) + if lt == 0 || sfwc { + return ErrInvalidSubject + } + if l == nil { + return ErrNotFound + } + if lt > 1 { + n = l.nodes[t] + } else { + switch t[0] { + case pwc: + n = l.pwc + haswc = true + case fwc: + n = l.fwc + haswc, sfwc = true, true + default: + n = l.nodes[t] + } + } + if n != nil { + levels = append(levels, lnt{l, n, t}) + l = n.next + } else { + l = nil + } + } + removed, last := s.removeFromNode(n, sub) + if !removed { + return ErrNotFound + } + + s.count-- + s.removes++ + + for i := len(levels) - 1; i >= 0; i-- { + l, n, t := levels[i].l, levels[i].n, levels[i].t + if n.isEmpty() { + l.pruneNode(n, t) + } + } + if doCacheUpdates { + s.removeFromCache(subject) + atomic.AddUint64(&s.genid, 1) + } + + if s.notify != nil && last && !haswc && len(s.notify.remove) > 0 { + s.chkForRemoveNotification(subject, string(sub.queue)) + } + + return nil +} + +// Remove will remove a subscription. +func (s *Sublist) Remove(sub *subscription) error { + return s.remove(sub, true, true) +} + +// RemoveBatch will remove a list of subscriptions. +func (s *Sublist) RemoveBatch(subs []*subscription) error { + if len(subs) == 0 { + return nil + } + + s.Lock() + defer s.Unlock() + + // TODO(dlc) - We could try to be smarter here for a client going away but the account + // has a large number of subscriptions compared to this client. Quick and dirty testing + // though said just disabling all the time best for now. + + // Turn off our cache if enabled. + wasEnabled := s.cache != nil + s.cache = nil + // We will try to remove all subscriptions but will report the first that caused + // an error. In other words, we don't bail out at the first error which would + // possibly leave a bunch of subscriptions that could have been removed. + var err error + for _, sub := range subs { + if lerr := s.remove(sub, false, false); lerr != nil && err == nil { + err = lerr + } + } + // Turn caching back on here. + atomic.AddUint64(&s.genid, 1) + if wasEnabled { + s.cache = make(map[string]*SublistResult) + } + return err +} + +// pruneNode is used to prune an empty node from the tree. +func (l *level) pruneNode(n *node, t string) { + if n == nil { + return + } + if n == l.fwc { + l.fwc = nil + } else if n == l.pwc { + l.pwc = nil + } else { + delete(l.nodes, t) + } +} + +// isEmpty will test if the node has any entries. Used +// in pruning. +func (n *node) isEmpty() bool { + if len(n.psubs) == 0 && len(n.qsubs) == 0 { + if n.next == nil || n.next.numNodes() == 0 { + return true + } + } + return false +} + +// Return the number of nodes for the given level. +func (l *level) numNodes() int { + if l == nil { + return 0 + } + num := len(l.nodes) + if l.pwc != nil { + num++ + } + if l.fwc != nil { + num++ + } + return num +} + +// Remove the sub for the given node. +func (s *Sublist) removeFromNode(n *node, sub *subscription) (found, last bool) { + if n == nil { + return false, true + } + if sub.queue == nil { + _, found = n.psubs[sub] + delete(n.psubs, sub) + if found && n.plist != nil { + // This will brute force remove the plist to perform + // correct behavior. Will get re-populated on a call + // to Match as needed. + n.plist = nil + } + return found, len(n.psubs) == 0 + } + + // We have a queue group subscription here + qsub := n.qsubs[string(sub.queue)] + _, found = qsub[sub] + delete(qsub, sub) + if len(qsub) == 0 { + // This is the last queue subscription interest when len(qsub) == 0, not + // when n.qsubs is empty. + last = true + delete(n.qsubs, string(sub.queue)) + } + return found, last +} + +// Count returns the number of subscriptions. +func (s *Sublist) Count() uint32 { + s.RLock() + defer s.RUnlock() + return s.count +} + +// CacheCount returns the number of result sets in the cache. +func (s *Sublist) CacheCount() int { + s.RLock() + cc := len(s.cache) + s.RUnlock() + return cc +} + +// SublistStats are public stats for the sublist +type SublistStats struct { + NumSubs uint32 `json:"num_subscriptions"` + NumCache uint32 `json:"num_cache"` + NumInserts uint64 `json:"num_inserts"` + NumRemoves uint64 `json:"num_removes"` + NumMatches uint64 `json:"num_matches"` + CacheHitRate float64 `json:"cache_hit_rate"` + MaxFanout uint32 `json:"max_fanout"` + AvgFanout float64 `json:"avg_fanout"` + totFanout int + cacheCnt int + cacheHits uint64 +} + +func (s *SublistStats) add(stat *SublistStats) { + s.NumSubs += stat.NumSubs + s.NumCache += stat.NumCache + s.NumInserts += stat.NumInserts + s.NumRemoves += stat.NumRemoves + s.NumMatches += stat.NumMatches + s.cacheHits += stat.cacheHits + if s.MaxFanout < stat.MaxFanout { + s.MaxFanout = stat.MaxFanout + } + + // ignore slStats.AvgFanout, collect the values + // it's based on instead + s.totFanout += stat.totFanout + s.cacheCnt += stat.cacheCnt + if s.totFanout > 0 { + s.AvgFanout = float64(s.totFanout) / float64(s.cacheCnt) + } + if s.NumMatches > 0 { + s.CacheHitRate = float64(s.cacheHits) / float64(s.NumMatches) + } +} + +// Stats will return a stats structure for the current state. +func (s *Sublist) Stats() *SublistStats { + st := &SublistStats{} + + s.RLock() + cache := s.cache + cc := len(s.cache) + st.NumSubs = s.count + st.NumInserts = s.inserts + st.NumRemoves = s.removes + s.RUnlock() + + st.NumCache = uint32(cc) + st.NumMatches = atomic.LoadUint64(&s.matches) + st.cacheHits = atomic.LoadUint64(&s.cacheHits) + if st.NumMatches > 0 { + st.CacheHitRate = float64(st.cacheHits) / float64(st.NumMatches) + } + + // whip through cache for fanout stats, this can be off if cache is full and doing evictions. + // If this is called frequently, which it should not be, this could hurt performance. + if cache != nil { + tot, max, clen := 0, 0, 0 + s.RLock() + for _, r := range s.cache { + clen++ + l := len(r.psubs) + len(r.qsubs) + tot += l + if l > max { + max = l + } + } + s.RUnlock() + st.totFanout = tot + st.cacheCnt = clen + st.MaxFanout = uint32(max) + if tot > 0 { + st.AvgFanout = float64(tot) / float64(clen) + } + } + return st +} + +// numLevels will return the maximum number of levels +// contained in the Sublist tree. +func (s *Sublist) numLevels() int { + return visitLevel(s.root, 0) +} + +// visitLevel is used to descend the Sublist tree structure +// recursively. +func visitLevel(l *level, depth int) int { + if l == nil || l.numNodes() == 0 { + return depth + } + + depth++ + maxDepth := depth + + for _, n := range l.nodes { + if n == nil { + continue + } + newDepth := visitLevel(n.next, depth) + if newDepth > maxDepth { + maxDepth = newDepth + } + } + if l.pwc != nil { + pwcDepth := visitLevel(l.pwc.next, depth) + if pwcDepth > maxDepth { + maxDepth = pwcDepth + } + } + if l.fwc != nil { + fwcDepth := visitLevel(l.fwc.next, depth) + if fwcDepth > maxDepth { + maxDepth = fwcDepth + } + } + return maxDepth +} + +// Determine if a subject has any wildcard tokens. +func subjectHasWildcard(subject string) bool { + // This one exits earlier then !subjectIsLiteral(subject) + for i, c := range subject { + if c == pwc || c == fwc { + if (i == 0 || subject[i-1] == btsep) && + (i+1 == len(subject) || subject[i+1] == btsep) { + return true + } + } + } + return false +} + +// Determine if the subject has any wildcards. Fast version, does not check for +// valid subject. Used in caching layer. +func subjectIsLiteral(subject string) bool { + for i, c := range subject { + if c == pwc || c == fwc { + if (i == 0 || subject[i-1] == btsep) && + (i+1 == len(subject) || subject[i+1] == btsep) { + return false + } + } + } + return true +} + +// IsValidPublishSubject returns true if a subject is valid and a literal, false otherwise +func IsValidPublishSubject(subject string) bool { + return IsValidSubject(subject) && subjectIsLiteral(subject) +} + +// IsValidSubject returns true if a subject is valid, false otherwise +func IsValidSubject(subject string) bool { + return isValidSubject(subject, false) +} + +func isValidSubject(subject string, checkRunes bool) bool { + if subject == _EMPTY_ { + return false + } + if checkRunes { + // Check if we have embedded nulls. + if bytes.IndexByte(stringToBytes(subject), 0) >= 0 { + return false + } + // Since casting to a string will always produce valid UTF-8, we need to look for replacement runes. + // This signals something is off or corrupt. + for _, r := range subject { + if r == utf8.RuneError { + return false + } + } + } + sfwc := false + for t := range strings.SplitSeq(subject, tsep) { + length := len(t) + if length == 0 || sfwc { + return false + } + if length > 1 { + if strings.ContainsAny(t, "\t\n\f\r ") { + return false + } + continue + } + switch t[0] { + case fwc: + sfwc = true + case ' ', '\t', '\n', '\r', '\f': + return false + } + } + return true +} + +// IsValidLiteralSubject returns true if a subject is valid and literal (no wildcards), false otherwise +func IsValidLiteralSubject(subject string) bool { + return isValidLiteralSubject(strings.SplitSeq(subject, tsep)) +} + +// isValidLiteralSubject returns true if the tokens are valid and literal (no wildcards), false otherwise +func isValidLiteralSubject(tokens iter.Seq[string]) bool { + for t := range tokens { + if len(t) == 0 { + return false + } + if len(t) > 1 { + continue + } + switch t[0] { + case pwc, fwc: + return false + } + } + return true +} + +// ValidateMapping returns nil error if the subject is a valid subject mapping destination subject +func ValidateMapping(src string, dest string) error { + if dest == _EMPTY_ { + return nil + } + sfwc := false + for t := range strings.SplitSeq(dest, tsep) { + length := len(t) + if length == 0 || sfwc { + return &mappingDestinationErr{t, ErrInvalidMappingDestinationSubject} + } + + // if it looks like it contains a mapping function, it should be a valid mapping function + if length > 4 && t[0] == '{' && t[1] == '{' && t[length-2] == '}' && t[length-1] == '}' { + if !partitionMappingFunctionRegEx.MatchString(t) && + !wildcardMappingFunctionRegEx.MatchString(t) && + !splitFromLeftMappingFunctionRegEx.MatchString(t) && + !splitFromRightMappingFunctionRegEx.MatchString(t) && + !sliceFromLeftMappingFunctionRegEx.MatchString(t) && + !sliceFromRightMappingFunctionRegEx.MatchString(t) && + !splitMappingFunctionRegEx.MatchString(t) && + !leftMappingFunctionRegEx.MatchString(t) && + !rightMappingFunctionRegEx.MatchString(t) && + !randomMappingFunctionRegEx.MatchString(t) { + return &mappingDestinationErr{t, ErrUnknownMappingDestinationFunction} + } else { + continue + } + } + + if length == 1 && t[0] == fwc { + sfwc = true + } else if strings.ContainsAny(t, "\t\n\f\r ") { + return ErrInvalidMappingDestinationSubject + } + } + + // Finally, verify that the transform can actually be created from the source and destination + _, err := NewSubjectTransform(src, dest) + return err +} + +// Will check tokens and report back if the have any partial or full wildcards. +func analyzeTokens(tokens []string) (hasPWC, hasFWC bool) { + for _, t := range tokens { + if lt := len(t); lt == 0 || lt > 1 { + continue + } + switch t[0] { + case pwc: + hasPWC = true + case fwc: + hasFWC = true + } + } + return +} + +// Check on a token basis if they could match. +func tokensCanMatch(t1, t2 string) bool { + if len(t1) == 0 || len(t2) == 0 { + return false + } + t1c, t2c := t1[0], t2[0] + if t1c == pwc || t2c == pwc || t1c == fwc || t2c == fwc { + return true + } + return t1 == t2 +} + +// SubjectsCollide will determine if two subjects could both match a single literal subject. +func SubjectsCollide(subj1, subj2 string) bool { + if subj1 == subj2 { + return true + } + tsa, tsb := [32]string{}, [32]string{} + toks1 := tokenizeSubjectIntoSlice(tsa[:0], subj1) + toks2 := tokenizeSubjectIntoSlice(tsb[:0], subj2) + pwc1, fwc1 := analyzeTokens(toks1) + pwc2, fwc2 := analyzeTokens(toks2) + // if both literal just string compare. + l1, l2 := !(pwc1 || fwc1), !(pwc2 || fwc2) + if l1 && l2 { + return subj1 == subj2 + } + // So one or both have wildcards. If one is literal than we can do subset matching. + if l1 && !l2 { + return isSubsetMatchTokenized(toks1, toks2) + } else if l2 && !l1 { + return isSubsetMatchTokenized(toks2, toks1) + } + // Both have wildcards. + // If they only have partials then the lengths must match. + if !fwc1 && !fwc2 && len(toks1) != len(toks2) { + return false + } + if lt1, lt2 := len(toks1), len(toks2); lt1 != lt2 { + // If the shorter one only has partials then these will not collide. + if lt1 < lt2 && !fwc1 || lt2 < lt1 && !fwc2 { + return false + } + } + + stop := len(toks1) + if len(toks2) < stop { + stop = len(toks2) + } + + // We look for reasons to say no. + for i := 0; i < stop; i++ { + t1, t2 := toks1[i], toks2[i] + if !tokensCanMatch(t1, t2) { + return false + } + } + + return true +} + +// Returns number of tokens in the subject. +func numTokens(subject string) int { + var numTokens int + if len(subject) == 0 { + return 0 + } + for i := 0; i < len(subject); i++ { + if subject[i] == btsep { + numTokens++ + } + } + return numTokens + 1 +} + +// Fast way to return an indexed token. +// This is one based, so first token is TokenAt(subject, 1) +func tokenAt(subject string, index uint8) string { + ti, start := uint8(1), 0 + for i := 0; i < len(subject); i++ { + if subject[i] == btsep { + if ti == index { + return subject[start:i] + } + start = i + 1 + ti++ + } + } + if ti == index { + return subject[start:] + } + return _EMPTY_ +} + +// use similar to append. meaning, the updated slice will be returned +func tokenizeSubjectIntoSlice(tts []string, subject string) []string { + start := 0 + for i := 0; i < len(subject); i++ { + if subject[i] == btsep { + tts = append(tts, subject[start:i]) + start = i + 1 + } + } + tts = append(tts, subject[start:]) + return tts +} + +// SubjectMatchesFilter returns true if the subject matches the provided +// filter or false otherwise. +func SubjectMatchesFilter(subject, filter string) bool { + return subjectIsSubsetMatch(subject, filter) +} + +// Calls into the function isSubsetMatch() +func subjectIsSubsetMatch(subject, test string) bool { + tsa := [32]string{} + tts := tokenizeSubjectIntoSlice(tsa[:0], subject) + return isSubsetMatch(tts, test) +} + +// This will test a subject as an array of tokens against a test subject +// Calls into the function isSubsetMatchTokenized +func isSubsetMatch(tokens []string, test string) bool { + tsa := [32]string{} + tts := tokenizeSubjectIntoSlice(tsa[:0], test) + return isSubsetMatchTokenized(tokens, tts) +} + +// This will test a subject as an array of tokens against a test subject (also encoded as array of tokens) +// and determine if the tokens are matched. Both test subject and tokens +// may contain wildcards. So foo.* is a subset match of [">", "*.*", "foo.*"], +// but not of foo.bar, etc. +func isSubsetMatchTokenized(tokens, test []string) bool { + // Walk the target tokens + for i, t2 := range test { + if i >= len(tokens) { + return false + } + l := len(t2) + if l == 0 { + return false + } + if t2[0] == fwc && l == 1 { + return true + } + t1 := tokens[i] + + l = len(t1) + if l == 0 || t1[0] == fwc && l == 1 { + return false + } + + if t1[0] == pwc && len(t1) == 1 { + m := t2[0] == pwc && len(t2) == 1 + if !m { + return false + } + if i >= len(test) { + return true + } + continue + } + if t2[0] != pwc && strings.Compare(t1, t2) != 0 { + return false + } + } + return len(tokens) == len(test) +} + +// matchLiteral is used to test literal subjects, those that do not have any +// wildcards, with a target subject. This is used in the cache layer. +func matchLiteral(literal, subject string) bool { + li := 0 + ll := len(literal) + ls := len(subject) + for i := 0; i < ls; i++ { + if li >= ll { + return false + } + // This function has been optimized for speed. + // For instance, do not set b:=subject[i] here since + // we may bump `i` in this loop to avoid `continue` or + // skipping common test in a particular test. + // Run Benchmark_SublistMatchLiteral before making any change. + switch subject[i] { + case pwc: + // NOTE: This is not testing validity of a subject, instead ensures + // that wildcards are treated as such if they follow some basic rules, + // namely that they are a token on their own. + if i == 0 || subject[i-1] == btsep { + if i == ls-1 { + // There is no more token in the subject after this wildcard. + // Skip token in literal and expect to not find a separator. + for { + // End of literal, this is a match. + if li >= ll { + return true + } + // Presence of separator, this can't be a match. + if literal[li] == btsep { + return false + } + li++ + } + } else if subject[i+1] == btsep { + // There is another token in the subject after this wildcard. + // Skip token in literal and expect to get a separator. + for { + // We found the end of the literal before finding a separator, + // this can't be a match. + if li >= ll { + return false + } + if literal[li] == btsep { + break + } + li++ + } + // Bump `i` since we know there is a `.` following, we are + // safe. The common test below is going to check `.` with `.` + // which is good. A `continue` here is too costly. + i++ + } + } + case fwc: + // For `>` to be a wildcard, it means being the only or last character + // in the string preceded by a `.` + if (i == 0 || subject[i-1] == btsep) && i == ls-1 { + return true + } + } + if subject[i] != literal[li] { + return false + } + li++ + } + // Make sure we have processed all of the literal's chars.. + return li >= ll +} + +func addLocalSub(sub *subscription, subs *[]*subscription, includeLeafHubs bool) { + if sub != nil && sub.client != nil { + kind := sub.client.kind + if kind == CLIENT || kind == SYSTEM || kind == JETSTREAM || kind == ACCOUNT || + (includeLeafHubs && sub.client.isHubLeafNode() /* implied kind==LEAF */) { + *subs = append(*subs, sub) + } + } +} + +func (s *Sublist) addNodeToSubs(n *node, subs *[]*subscription, includeLeafHubs bool) { + // Normal subscriptions + if n.plist != nil { + for _, sub := range n.plist { + addLocalSub(sub, subs, includeLeafHubs) + } + } else { + for sub := range n.psubs { + addLocalSub(sub, subs, includeLeafHubs) + } + } + // Queue subscriptions + for _, qr := range n.qsubs { + for sub := range qr { + addLocalSub(sub, subs, includeLeafHubs) + } + } +} + +func (s *Sublist) collectLocalSubs(l *level, subs *[]*subscription, includeLeafHubs bool) { + for _, n := range l.nodes { + s.addNodeToSubs(n, subs, includeLeafHubs) + s.collectLocalSubs(n.next, subs, includeLeafHubs) + } + if l.pwc != nil { + s.addNodeToSubs(l.pwc, subs, includeLeafHubs) + s.collectLocalSubs(l.pwc.next, subs, includeLeafHubs) + } + if l.fwc != nil { + s.addNodeToSubs(l.fwc, subs, includeLeafHubs) + s.collectLocalSubs(l.fwc.next, subs, includeLeafHubs) + } +} + +// Return all local client subscriptions. Use the supplied slice. +func (s *Sublist) localSubs(subs *[]*subscription, includeLeafHubs bool) { + s.RLock() + s.collectLocalSubs(s.root, subs, includeLeafHubs) + s.RUnlock() +} + +// All is used to collect all subscriptions. +func (s *Sublist) All(subs *[]*subscription) { + s.RLock() + s.collectAllSubs(s.root, subs) + s.RUnlock() +} + +func (s *Sublist) addAllNodeToSubs(n *node, subs *[]*subscription) { + // Normal subscriptions + if n.plist != nil { + *subs = append(*subs, n.plist...) + } else { + for sub := range n.psubs { + *subs = append(*subs, sub) + } + } + // Queue subscriptions + for _, qr := range n.qsubs { + for sub := range qr { + *subs = append(*subs, sub) + } + } +} + +func (s *Sublist) collectAllSubs(l *level, subs *[]*subscription) { + for _, n := range l.nodes { + s.addAllNodeToSubs(n, subs) + s.collectAllSubs(n.next, subs) + } + if l.pwc != nil { + s.addAllNodeToSubs(l.pwc, subs) + s.collectAllSubs(l.pwc.next, subs) + } + if l.fwc != nil { + s.addAllNodeToSubs(l.fwc, subs) + s.collectAllSubs(l.fwc.next, subs) + } +} + +// For a given subject (which may contain wildcards), this call returns all +// subscriptions that would match that subject. For instance, suppose that +// the sublist contains: foo.bar, foo.bar.baz and foo.baz, ReverseMatch("foo.*") +// would return foo.bar and foo.baz. +// This is used in situations where the sublist is likely to contain only +// literals and one wants to get all the subjects that would have been a match +// to a subscription on `subject`. +func (s *Sublist) ReverseMatch(subject string) *SublistResult { + tsa := [32]string{} + tokens := tsa[:0] + start := 0 + for i := 0; i < len(subject); i++ { + if subject[i] == btsep { + tokens = append(tokens, subject[start:i]) + start = i + 1 + } + } + tokens = append(tokens, subject[start:]) + + result := &SublistResult{} + + s.RLock() + reverseMatchLevel(s.root, tokens, nil, result) + // Check for empty result. + if len(result.psubs) == 0 && len(result.qsubs) == 0 { + result = emptyResult + } + s.RUnlock() + + return result +} + +func reverseMatchLevel(l *level, toks []string, n *node, results *SublistResult) { + if l == nil { + return + } + for i, t := range toks { + if len(t) == 1 { + if t[0] == fwc { + getAllNodes(l, results) + return + } else if t[0] == pwc { + for _, n := range l.nodes { + reverseMatchLevel(n.next, toks[i+1:], n, results) + } + if l.pwc != nil { + reverseMatchLevel(l.pwc.next, toks[i+1:], n, results) + } + if l.fwc != nil { + getAllNodes(l, results) + } + return + } + } + // If the sub tree has a fwc at this position, match as well. + if l.fwc != nil { + getAllNodes(l, results) + return + } else if l.pwc != nil { + reverseMatchLevel(l.pwc.next, toks[i+1:], n, results) + } + n = l.nodes[t] + if n == nil { + break + } + l = n.next + } + if n != nil { + addNodeToResults(n, results) + } +} + +func getAllNodes(l *level, results *SublistResult) { + if l == nil { + return + } + if l.pwc != nil { + addNodeToResults(l.pwc, results) + } + if l.fwc != nil { + addNodeToResults(l.fwc, results) + } + for _, n := range l.nodes { + addNodeToResults(n, results) + getAllNodes(n.next, results) + } +} diff --git a/server/sublist_test.go b/server/sublist_test.go new file mode 100644 index 0000000..aacca89 --- /dev/null +++ b/server/sublist_test.go @@ -0,0 +1,2489 @@ +// Copyright 2016-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "errors" + "flag" + "fmt" + "math/rand/v2" + "os" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nuid" +) + +func stackFatalf(t *testing.T, f string, args ...any) { + lines := make([]string, 0, 32) + msg := fmt.Sprintf(f, args...) + lines = append(lines, msg) + + // Generate the Stack of callers: Skip us and verify* frames. + for i := 2; true; i++ { + _, file, line, ok := runtime.Caller(i) + if !ok { + break + } + msg := fmt.Sprintf("%d - %s:%d", i, file, line) + lines = append(lines, msg) + } + t.Fatalf("%s", strings.Join(lines, "\n")) +} + +func verifyCount(s *Sublist, count uint32, t *testing.T) { + t.Helper() + if s.Count() != count { + t.Fatalf("Count is %d, should be %d", s.Count(), count) + } +} + +func verifyLen(r []*subscription, l int, t *testing.T) { + t.Helper() + if len(r) != l { + t.Fatalf("Results len is %d, should be %d", len(r), l) + } +} + +func verifyQLen(r [][]*subscription, l int, t *testing.T) { + t.Helper() + if len(r) != l { + t.Fatalf("Queue Results len is %d, should be %d", len(r), l) + } +} + +func verifyNumLevels(s *Sublist, expected int, t *testing.T) { + t.Helper() + dl := s.numLevels() + if dl != expected { + t.Fatalf("NumLevels is %d, should be %d", dl, expected) + } +} + +func verifyQMember(qsubs [][]*subscription, val *subscription, t *testing.T) { + t.Helper() + verifyMember(qsubs[findQSlot(val.queue, qsubs)], val, t) +} + +func verifyMember(r []*subscription, val *subscription, t *testing.T) { + t.Helper() + for _, v := range r { + if v == nil { + continue + } + if v == val { + return + } + } + t.Fatalf("Subscription (%p) for [%s : %s] not found in results", val, val.subject, val.queue) +} + +// Helpers to generate test subscriptions. +func newSub(subject string) *subscription { + c := &client{kind: CLIENT} + return &subscription{client: c, subject: []byte(subject)} +} + +func newQSub(subject, queue string) *subscription { + if queue != "" { + return &subscription{subject: []byte(subject), queue: []byte(queue)} + } + return newSub(subject) +} + +func newRemoteQSub(subject, queue string, num int32) *subscription { + if queue != "" { + c := &client{kind: ROUTER} + return &subscription{client: c, subject: []byte(subject), queue: []byte(queue), qw: num} + } + return newSub(subject) +} + +func TestSublistInit(t *testing.T) { + s := NewSublistWithCache() + verifyCount(s, 0, t) +} + +func TestSublistForServerNoOpts(t *testing.T) { + s := NewSublistForServer(&Server{}) + if s == nil { + t.Fatal("Expected a sublist") + } + if s.CacheEnabled() { + t.Fatal("Expected cache to be disabled when server options are nil") + } +} + +func TestSetAccountSublistNoOpts(t *testing.T) { + s := &Server{} + acc := NewAccount("foo") + + s.setAccountSublist(acc) + + if acc.sl == nil { + t.Fatal("Expected account sublist to be initialized") + } + if acc.sl.CacheEnabled() { + t.Fatal("Expected cache to be disabled when server options are nil") + } +} + +func TestSublistInsertCount(t *testing.T) { + testSublistInsertCount(t, NewSublistWithCache()) +} + +func TestSublistInsertCountNoCache(t *testing.T) { + testSublistInsertCount(t, NewSublistNoCache()) +} + +func testSublistInsertCount(t *testing.T, s *Sublist) { + s.Insert(newSub("foo")) + s.Insert(newSub("bar")) + s.Insert(newSub("foo.bar")) + verifyCount(s, 3, t) +} + +func TestSublistSimple(t *testing.T) { + testSublistSimple(t, NewSublistWithCache()) +} + +func TestSublistSimpleNoCache(t *testing.T) { + testSublistSimple(t, NewSublistNoCache()) +} + +func testSublistSimple(t *testing.T, s *Sublist) { + subject := "foo" + sub := newSub(subject) + s.Insert(sub) + r := s.Match(subject) + verifyLen(r.psubs, 1, t) + verifyMember(r.psubs, sub, t) +} + +func TestSublistSimpleMultiTokens(t *testing.T) { + testSublistSimpleMultiTokens(t, NewSublistWithCache()) +} + +func TestSublistSimpleMultiTokensNoCache(t *testing.T) { + testSublistSimpleMultiTokens(t, NewSublistNoCache()) +} + +func testSublistSimpleMultiTokens(t *testing.T, s *Sublist) { + subject := "foo.bar.baz" + sub := newSub(subject) + s.Insert(sub) + r := s.Match(subject) + verifyLen(r.psubs, 1, t) + verifyMember(r.psubs, sub, t) +} + +func TestSublistPartialWildcard(t *testing.T) { + testSublistPartialWildcard(t, NewSublistWithCache()) +} + +func TestSublistPartialWildcardNoCache(t *testing.T) { + testSublistPartialWildcard(t, NewSublistNoCache()) +} + +func testSublistPartialWildcard(t *testing.T, s *Sublist) { + lsub := newSub("a.b.c") + psub := newSub("a.*.c") + s.Insert(lsub) + s.Insert(psub) + r := s.Match("a.b.c") + verifyLen(r.psubs, 2, t) + verifyMember(r.psubs, lsub, t) + verifyMember(r.psubs, psub, t) +} + +func TestSublistPartialWildcardAtEnd(t *testing.T) { + testSublistPartialWildcardAtEnd(t, NewSublistWithCache()) +} + +func TestSublistPartialWildcardAtEndNoCache(t *testing.T) { + testSublistPartialWildcardAtEnd(t, NewSublistNoCache()) +} + +func testSublistPartialWildcardAtEnd(t *testing.T, s *Sublist) { + lsub := newSub("a.b.c") + psub := newSub("a.b.*") + s.Insert(lsub) + s.Insert(psub) + r := s.Match("a.b.c") + verifyLen(r.psubs, 2, t) + verifyMember(r.psubs, lsub, t) + verifyMember(r.psubs, psub, t) +} + +func TestSublistFullWildcard(t *testing.T) { + testSublistFullWildcard(t, NewSublistWithCache()) +} + +func TestSublistFullWildcardNoCache(t *testing.T) { + testSublistFullWildcard(t, NewSublistNoCache()) +} + +func testSublistFullWildcard(t *testing.T, s *Sublist) { + lsub := newSub("a.b.c") + fsub := newSub("a.>") + s.Insert(lsub) + s.Insert(fsub) + r := s.Match("a.b.c") + verifyLen(r.psubs, 2, t) + verifyMember(r.psubs, lsub, t) + verifyMember(r.psubs, fsub, t) + + r = s.Match("a.>") + verifyLen(r.psubs, 1, t) + verifyMember(r.psubs, fsub, t) +} + +func TestSublistRemove(t *testing.T) { + testSublistRemove(t, NewSublistWithCache()) +} + +func TestSublistRemoveNoCache(t *testing.T) { + testSublistRemove(t, NewSublistNoCache()) +} + +func testSublistRemove(t *testing.T, s *Sublist) { + subject := "a.b.c.d" + sub := newSub(subject) + s.Insert(sub) + verifyCount(s, 1, t) + r := s.Match(subject) + verifyLen(r.psubs, 1, t) + s.Remove(newSub("a.b.c")) + verifyCount(s, 1, t) + s.Remove(sub) + verifyCount(s, 0, t) + r = s.Match(subject) + verifyLen(r.psubs, 0, t) +} + +func TestSublistRemoveWildcard(t *testing.T) { + testSublistRemoveWildcard(t, NewSublistWithCache()) +} + +func TestSublistRemoveWildcardNoCache(t *testing.T) { + testSublistRemoveWildcard(t, NewSublistNoCache()) +} + +func testSublistRemoveWildcard(t *testing.T, s *Sublist) { + subject := "a.b.c.d" + sub := newSub(subject) + psub := newSub("a.b.*.d") + fsub := newSub("a.b.>") + s.Insert(sub) + s.Insert(psub) + s.Insert(fsub) + verifyCount(s, 3, t) + r := s.Match(subject) + verifyLen(r.psubs, 3, t) + s.Remove(sub) + verifyCount(s, 2, t) + s.Remove(fsub) + verifyCount(s, 1, t) + s.Remove(psub) + verifyCount(s, 0, t) + r = s.Match(subject) + verifyLen(r.psubs, 0, t) +} + +func TestSublistRemoveCleanup(t *testing.T) { + testSublistRemoveCleanup(t, NewSublistWithCache()) +} + +func TestSublistRemoveCleanupNoCache(t *testing.T) { + testSublistRemoveCleanup(t, NewSublistNoCache()) +} + +func testSublistRemoveCleanup(t *testing.T, s *Sublist) { + literal := "a.b.c.d.e.f" + depth := len(strings.Split(literal, tsep)) + sub := newSub(literal) + verifyNumLevels(s, 0, t) + s.Insert(sub) + verifyNumLevels(s, depth, t) + s.Remove(sub) + verifyNumLevels(s, 0, t) +} + +func TestSublistRemoveCleanupWildcards(t *testing.T) { + testSublistRemoveCleanupWildcards(t, NewSublistWithCache()) +} + +func TestSublistRemoveCleanupWildcardsNoCache(t *testing.T) { + testSublistRemoveCleanupWildcards(t, NewSublistNoCache()) +} + +func testSublistRemoveCleanupWildcards(t *testing.T, s *Sublist) { + subject := "a.b.*.d.e.>" + depth := len(strings.Split(subject, tsep)) + sub := newSub(subject) + verifyNumLevels(s, 0, t) + s.Insert(sub) + verifyNumLevels(s, depth, t) + s.Remove(sub) + verifyNumLevels(s, 0, t) +} + +func TestSublistRemoveWithLargeSubs(t *testing.T) { + testSublistRemoveWithLargeSubs(t, NewSublistWithCache()) +} + +func TestSublistRemoveWithLargeSubsNoCache(t *testing.T) { + testSublistRemoveWithLargeSubs(t, NewSublistNoCache()) +} + +func testSublistRemoveWithLargeSubs(t *testing.T, s *Sublist) { + subject := "foo" + for i := 0; i < plistMin*2; i++ { + sub := newSub(subject) + s.Insert(sub) + } + r := s.Match(subject) + verifyLen(r.psubs, plistMin*2, t) + // Remove one that is in the middle + s.Remove(r.psubs[plistMin]) + // Remove first one + s.Remove(r.psubs[0]) + // Remove last one + s.Remove(r.psubs[len(r.psubs)-1]) + // Check len again + r = s.Match(subject) + verifyLen(r.psubs, plistMin*2-3, t) +} + +func TestSublistInvalidSubjectsInsert(t *testing.T) { + testSublistInvalidSubjectsInsert(t, NewSublistWithCache()) +} + +func TestSublistInvalidSubjectsInsertNoCache(t *testing.T) { + testSublistInvalidSubjectsInsert(t, NewSublistNoCache()) +} + +func TestSublistNoCacheRemoveBatch(t *testing.T) { + s := NewSublistNoCache() + s.Insert(newSub("foo")) + sub := newSub("bar") + s.Insert(sub) + s.RemoveBatch([]*subscription{sub}) + // Now test that this did not turn on cache + for i := 0; i < 10; i++ { + s.Match("foo") + } + if s.CacheEnabled() { + t.Fatalf("Cache should not be enabled") + } +} + +func TestSublistRemoveBatchWithError(t *testing.T) { + s := NewSublistNoCache() + sub1 := newSub("foo") + sub2 := newSub("bar") + sub3 := newSub("baz") + s.Insert(sub1) + s.Insert(sub2) + s.Insert(sub3) + subNotPresent := newSub("not.inserted") + // Try to remove all subs, but include the sub that has not been inserted. + err := s.RemoveBatch([]*subscription{subNotPresent, sub1, sub3}) + // We expect an error to be returned, but sub1,2 and 3 to have been removed. + require_Error(t, err, ErrNotFound) + // Make sure that we have only sub2 present + verifyCount(s, 1, t) + r := s.Match("bar") + verifyLen(r.psubs, 1, t) + verifyMember(r.psubs, sub2, t) + r = s.Match("foo") + verifyLen(r.psubs, 0, t) + r = s.Match("baz") + verifyLen(r.psubs, 0, t) +} + +func testSublistInvalidSubjectsInsert(t *testing.T, s *Sublist) { + // Insert, or subscriptions, can have wildcards, but not empty tokens, + // and can not have a FWC that is not the terminal token. + + // beginning empty token + if err := s.Insert(newSub(".foo")); err != ErrInvalidSubject { + t.Fatal("Expected invalid subject error") + } + + // trailing empty token + if err := s.Insert(newSub("foo.")); err != ErrInvalidSubject { + t.Fatal("Expected invalid subject error") + } + // empty middle token + if err := s.Insert(newSub("foo..bar")); err != ErrInvalidSubject { + t.Fatal("Expected invalid subject error") + } + // empty middle token #2 + if err := s.Insert(newSub("foo.bar..baz")); err != ErrInvalidSubject { + t.Fatal("Expected invalid subject error") + } + // fwc not terminal + if err := s.Insert(newSub("foo.>.bar")); err != ErrInvalidSubject { + t.Fatal("Expected invalid subject error") + } +} + +func TestSublistCache(t *testing.T) { + s := NewSublistWithCache() + + // Test add a remove logistics + subject := "a.b.c.d" + sub := newSub(subject) + psub := newSub("a.b.*.d") + fsub := newSub("a.b.>") + s.Insert(sub) + r := s.Match(subject) + verifyLen(r.psubs, 1, t) + s.Insert(psub) + s.Insert(fsub) + verifyCount(s, 3, t) + r = s.Match(subject) + verifyLen(r.psubs, 3, t) + s.Remove(sub) + verifyCount(s, 2, t) + s.Remove(fsub) + verifyCount(s, 1, t) + s.Remove(psub) + verifyCount(s, 0, t) + + // Check that cache is now empty + if cc := s.CacheCount(); cc != 0 { + t.Fatalf("Cache should be zero, got %d\n", cc) + } + + r = s.Match(subject) + verifyLen(r.psubs, 0, t) + + for i := 0; i < 2*slCacheMax; i++ { + s.Match(fmt.Sprintf("foo-%d\n", i)) + } + + checkFor(t, 2*time.Second, 10*time.Millisecond, func() error { + if cc := s.CacheCount(); cc > slCacheMax { + return fmt.Errorf("Cache should be constrained by cacheMax, got %d for current count", cc) + } + return nil + }) + + // Test that adding to a wildcard properly adds to the cache. + s = NewSublistWithCache() + s.Insert(newSub("foo.*")) + s.Insert(newSub("foo.bar")) + r = s.Match("foo.baz") + verifyLen(r.psubs, 1, t) + r = s.Match("foo.bar") + verifyLen(r.psubs, 2, t) + s.Insert(newSub("foo.>")) + r = s.Match("foo.bar") + verifyLen(r.psubs, 3, t) +} + +func TestSublistBasicQueueResults(t *testing.T) { + testSublistBasicQueueResults(t, NewSublistWithCache()) +} + +func TestSublistBasicQueueResultsNoCache(t *testing.T) { + testSublistBasicQueueResults(t, NewSublistNoCache()) +} + +func testSublistBasicQueueResults(t *testing.T, s *Sublist) { + // Test some basics + subject := "foo" + sub := newSub(subject) + sub1 := newQSub(subject, "bar") + sub2 := newQSub(subject, "baz") + + s.Insert(sub1) + r := s.Match(subject) + verifyLen(r.psubs, 0, t) + verifyQLen(r.qsubs, 1, t) + verifyLen(r.qsubs[0], 1, t) + verifyQMember(r.qsubs, sub1, t) + + s.Insert(sub2) + r = s.Match(subject) + verifyLen(r.psubs, 0, t) + verifyQLen(r.qsubs, 2, t) + verifyLen(r.qsubs[0], 1, t) + verifyLen(r.qsubs[1], 1, t) + verifyQMember(r.qsubs, sub1, t) + verifyQMember(r.qsubs, sub2, t) + + s.Insert(sub) + r = s.Match(subject) + verifyLen(r.psubs, 1, t) + verifyQLen(r.qsubs, 2, t) + verifyLen(r.qsubs[0], 1, t) + verifyLen(r.qsubs[1], 1, t) + verifyQMember(r.qsubs, sub1, t) + verifyQMember(r.qsubs, sub2, t) + verifyMember(r.psubs, sub, t) + + sub3 := newQSub(subject, "bar") + sub4 := newQSub(subject, "baz") + + s.Insert(sub3) + s.Insert(sub4) + + r = s.Match(subject) + verifyLen(r.psubs, 1, t) + verifyQLen(r.qsubs, 2, t) + verifyLen(r.qsubs[0], 2, t) + verifyLen(r.qsubs[1], 2, t) + verifyQMember(r.qsubs, sub1, t) + verifyQMember(r.qsubs, sub2, t) + verifyQMember(r.qsubs, sub3, t) + verifyQMember(r.qsubs, sub4, t) + verifyMember(r.psubs, sub, t) + + // Now removal + s.Remove(sub) + + r = s.Match(subject) + verifyLen(r.psubs, 0, t) + verifyQLen(r.qsubs, 2, t) + verifyLen(r.qsubs[0], 2, t) + verifyLen(r.qsubs[1], 2, t) + verifyQMember(r.qsubs, sub1, t) + verifyQMember(r.qsubs, sub2, t) + + s.Remove(sub1) + r = s.Match(subject) + verifyLen(r.psubs, 0, t) + verifyQLen(r.qsubs, 2, t) + verifyLen(r.qsubs[findQSlot(sub1.queue, r.qsubs)], 1, t) + verifyLen(r.qsubs[findQSlot(sub2.queue, r.qsubs)], 2, t) + verifyQMember(r.qsubs, sub2, t) + verifyQMember(r.qsubs, sub3, t) + verifyQMember(r.qsubs, sub4, t) + + s.Remove(sub3) // Last one + r = s.Match(subject) + verifyLen(r.psubs, 0, t) + verifyQLen(r.qsubs, 1, t) + verifyLen(r.qsubs[0], 2, t) // this is sub2/baz now + verifyQMember(r.qsubs, sub2, t) + + s.Remove(sub2) + s.Remove(sub4) + r = s.Match(subject) + verifyLen(r.psubs, 0, t) + verifyQLen(r.qsubs, 0, t) +} + +func checkBool(b, expected bool, t *testing.T) { + t.Helper() + if b != expected { + t.Fatalf("Expected %v, but got %v\n", expected, b) + } +} + +func checkError(err, expected error, t *testing.T) { + t.Helper() + if err != expected && err != nil && !errors.Is(err, expected) { + t.Fatalf("Expected %v, but got %v\n", expected, err) + } +} + +func TestSublistValidLiteralSubjects(t *testing.T) { + checkBool(IsValidLiteralSubject("foo"), true, t) + checkBool(IsValidLiteralSubject(".foo"), false, t) + checkBool(IsValidLiteralSubject("foo."), false, t) + checkBool(IsValidLiteralSubject("foo..bar"), false, t) + checkBool(IsValidLiteralSubject("foo.bar.*"), false, t) + checkBool(IsValidLiteralSubject("foo.bar.>"), false, t) + checkBool(IsValidLiteralSubject("*"), false, t) + checkBool(IsValidLiteralSubject(">"), false, t) + // The followings have widlcards characters but are not + // considered as such because they are not individual tokens. + checkBool(IsValidLiteralSubject("foo*"), true, t) + checkBool(IsValidLiteralSubject("foo**"), true, t) + checkBool(IsValidLiteralSubject("foo.**"), true, t) + checkBool(IsValidLiteralSubject("foo*bar"), true, t) + checkBool(IsValidLiteralSubject("foo.*bar"), true, t) + checkBool(IsValidLiteralSubject("foo*.bar"), true, t) + checkBool(IsValidLiteralSubject("*bar"), true, t) + checkBool(IsValidLiteralSubject("foo>"), true, t) + checkBool(IsValidLiteralSubject("foo>>"), true, t) + checkBool(IsValidLiteralSubject("foo.>>"), true, t) + checkBool(IsValidLiteralSubject("foo>bar"), true, t) + checkBool(IsValidLiteralSubject("foo.>bar"), true, t) + checkBool(IsValidLiteralSubject("foo>.bar"), true, t) + checkBool(IsValidLiteralSubject(">bar"), true, t) +} + +func TestSublistValidSubjects(t *testing.T) { + checkBool(IsValidSubject("."), false, t) + checkBool(IsValidSubject(".foo"), false, t) + checkBool(IsValidSubject("foo."), false, t) + checkBool(IsValidSubject("foo..bar"), false, t) + checkBool(IsValidSubject(">.bar"), false, t) + checkBool(IsValidSubject("foo.>.bar"), false, t) + checkBool(IsValidSubject("foo"), true, t) + checkBool(IsValidSubject("foo.bar.*"), true, t) + checkBool(IsValidSubject("foo.bar.>"), true, t) + checkBool(IsValidSubject("*"), true, t) + checkBool(IsValidSubject(">"), true, t) + checkBool(IsValidSubject("foo*"), true, t) + checkBool(IsValidSubject("foo**"), true, t) + checkBool(IsValidSubject("foo.**"), true, t) + checkBool(IsValidSubject("foo*bar"), true, t) + checkBool(IsValidSubject("foo.*bar"), true, t) + checkBool(IsValidSubject("foo*.bar"), true, t) + checkBool(IsValidSubject("*bar"), true, t) + checkBool(IsValidSubject("foo>"), true, t) + checkBool(IsValidSubject("foo.>>"), true, t) + checkBool(IsValidSubject("foo>bar"), true, t) + checkBool(IsValidSubject("foo.>bar"), true, t) + checkBool(IsValidSubject("foo>.bar"), true, t) + checkBool(IsValidSubject(">bar"), true, t) + + // Check for embedded nulls. + subj := []byte("foo.bar.baz.") + subj = append(subj, 0) + checkBool(isValidSubject(string(subj), true), false, t) +} + +func TestSublistMatchLiterals(t *testing.T) { + checkBool(matchLiteral("foo", "foo"), true, t) + checkBool(matchLiteral("foo", "bar"), false, t) + checkBool(matchLiteral("foo", "*"), true, t) + checkBool(matchLiteral("foo", ">"), true, t) + checkBool(matchLiteral("foo.bar", ">"), true, t) + checkBool(matchLiteral("foo.bar", "foo.>"), true, t) + checkBool(matchLiteral("foo.bar", "bar.>"), false, t) + checkBool(matchLiteral("stats.test.22", "stats.>"), true, t) + checkBool(matchLiteral("stats.test.22", "stats.*.*"), true, t) + checkBool(matchLiteral("foo.bar", "foo"), false, t) + checkBool(matchLiteral("stats.test.foos", "stats.test.foos"), true, t) + checkBool(matchLiteral("stats.test.foos", "stats.test.foo"), false, t) + checkBool(matchLiteral("stats.test", "stats.test.*"), false, t) + checkBool(matchLiteral("stats.test.foos", "stats.*"), false, t) + checkBool(matchLiteral("stats.test.foos", "stats.*.*.foos"), false, t) + + // These are cases where wildcards characters should not be considered + // wildcards since they do not follow the rules of wildcards. + checkBool(matchLiteral("*bar", "*bar"), true, t) + checkBool(matchLiteral("foo*", "foo*"), true, t) + checkBool(matchLiteral("foo*bar", "foo*bar"), true, t) + checkBool(matchLiteral("foo.***.bar", "foo.***.bar"), true, t) + checkBool(matchLiteral(">bar", ">bar"), true, t) + checkBool(matchLiteral("foo>", "foo>"), true, t) + checkBool(matchLiteral("foo>bar", "foo>bar"), true, t) + checkBool(matchLiteral("foo.>>>.bar", "foo.>>>.bar"), true, t) +} + +func TestSubjectIsLiteral(t *testing.T) { + checkBool(subjectIsLiteral("foo"), true, t) + checkBool(subjectIsLiteral("foo.bar"), true, t) + checkBool(subjectIsLiteral("foo*.bar"), true, t) + checkBool(subjectIsLiteral("*"), false, t) + checkBool(subjectIsLiteral(">"), false, t) + checkBool(subjectIsLiteral("foo.*"), false, t) + checkBool(subjectIsLiteral("foo.>"), false, t) + checkBool(subjectIsLiteral("foo.*.>"), false, t) + checkBool(subjectIsLiteral("foo.*.bar"), false, t) + checkBool(subjectIsLiteral("foo.bar.>"), false, t) +} + +func TestValidateDestinationSubject(t *testing.T) { + checkError(ValidateMapping("bar", "foo"), nil, t) + checkError(ValidateMapping("foo", "foo.bar"), nil, t) + checkError(ValidateMapping("*", "{{wildcard(1)}}"), nil, t) + checkError(ValidateMapping("foo.>", ">"), nil, t) + checkError(ValidateMapping("foo.*", "bar.{{wildcard(1)}}"), nil, t) + checkError(ValidateMapping("foo.>", "bar.>"), nil, t) + checkError(ValidateMapping("foo.*.>", "bar.{{wildcard(1)}}.>"), nil, t) + checkError(ValidateMapping("foo.*.bar", "bar.{{wildcard(1)}}.foo"), nil, t) + checkError(ValidateMapping("foo.bar.>", "foo.bar.foo.>"), nil, t) + checkError(ValidateMapping("*", "foo.{{wildcard(1)}}"), nil, t) + checkError(ValidateMapping("*", "foo.{{ wildcard(1) }}"), nil, t) + checkError(ValidateMapping("*", "foo.{{wildcard( 1 )}}"), nil, t) + checkError(ValidateMapping("*", "foo.{{partition(2,1)}}"), nil, t) + checkError(ValidateMapping("*.*", "foo.{{SplitFromLeft(2,1)}}"), nil, t) + checkError(ValidateMapping("*.*", "foo.{{SplitFromRight(2,1)}}"), nil, t) + checkError(ValidateMapping("*.*", "foo.{{SliceFromLeft(2,1)}}"), nil, t) + checkError(ValidateMapping("*.*", "foo.{{SliceFromRight(2,1)}}"), nil, t) + checkError(ValidateMapping("*.*", "foo.{{split(1,-)}}"), nil, t) + checkError(ValidateMapping("*", "foo.{{random(1)}}"), nil, t) + checkError(ValidateMapping("*", "foo.{{left(1,2)}}"), nil, t) + checkError(ValidateMapping("*", "foo.{{Left(1,2)}}"), nil, t) + checkError(ValidateMapping("*", "foo.{{right(1,2)}}"), nil, t) + checkError(ValidateMapping("*", "foo.{{Right(1,2)}}"), nil, t) + checkError(ValidateMapping("*", "foo.{{ left( 1 , 2 ) }}"), nil, t) + checkError(ValidateMapping("*", "foo.{{unknown(1)}}"), ErrInvalidMappingDestination, t) + checkError(ValidateMapping("foo", "foo..}"), ErrInvalidMappingDestination, t) + checkError(ValidateMapping("foo", "foo. bar}"), ErrInvalidMappingDestinationSubject, t) +} + +func TestSubjectToken(t *testing.T) { + checkToken := func(token, expected string) { + t.Helper() + if token != expected { + t.Fatalf("Expected token of %q, got %q", expected, token) + } + } + checkToken(tokenAt("foo.bar.baz.*", 0), "") + checkToken(tokenAt("foo.bar.baz.*", 1), "foo") + checkToken(tokenAt("foo.bar.baz.*", 2), "bar") + checkToken(tokenAt("foo.bar.baz.*", 3), "baz") + checkToken(tokenAt("foo.bar.baz.*", 4), "*") + checkToken(tokenAt("foo.bar.baz.*", 5), "") +} + +func TestSublistBadSubjectOnRemove(t *testing.T) { + testSublistBadSubjectOnRemove(t, NewSublistWithCache()) +} + +func TestSublistBadSubjectOnRemoveNoCache(t *testing.T) { + testSublistBadSubjectOnRemove(t, NewSublistNoCache()) +} + +func testSublistBadSubjectOnRemove(t *testing.T, s *Sublist) { + bad := "a.b..d" + sub := newSub(bad) + + if err := s.Insert(sub); err != ErrInvalidSubject { + t.Fatalf("Expected ErrInvalidSubject, got %v\n", err) + } + + if err := s.Remove(sub); err != ErrInvalidSubject { + t.Fatalf("Expected ErrInvalidSubject, got %v\n", err) + } + + badfwc := "a.>.b" + if err := s.Remove(newSub(badfwc)); err != ErrInvalidSubject { + t.Fatalf("Expected ErrInvalidSubject, got %v\n", err) + } +} + +// This is from bug report #18 +func TestSublistTwoTokenPubMatchSingleTokenSub(t *testing.T) { + testSublistTwoTokenPubMatchSingleTokenSub(t, NewSublistWithCache()) +} + +func TestSublistTwoTokenPubMatchSingleTokenSubNoCache(t *testing.T) { + testSublistTwoTokenPubMatchSingleTokenSub(t, NewSublistNoCache()) +} + +func testSublistTwoTokenPubMatchSingleTokenSub(t *testing.T, s *Sublist) { + sub := newSub("foo") + s.Insert(sub) + r := s.Match("foo") + verifyLen(r.psubs, 1, t) + verifyMember(r.psubs, sub, t) + r = s.Match("foo.bar") + verifyLen(r.psubs, 0, t) +} + +func TestSublistInsertWithWildcardsAsLiterals(t *testing.T) { + testSublistInsertWithWildcardsAsLiterals(t, NewSublistWithCache()) +} + +func TestSublistInsertWithWildcardsAsLiteralsNoCache(t *testing.T) { + testSublistInsertWithWildcardsAsLiterals(t, NewSublistNoCache()) +} + +func testSublistInsertWithWildcardsAsLiterals(t *testing.T, s *Sublist) { + subjects := []string{"foo.*-", "foo.>-"} + for _, subject := range subjects { + sub := newSub(subject) + s.Insert(sub) + // Should find no match + r := s.Match("foo.bar") + verifyLen(r.psubs, 0, t) + // Should find a match + r = s.Match(subject) + verifyLen(r.psubs, 1, t) + } +} + +func TestSublistRemoveWithWildcardsAsLiterals(t *testing.T) { + testSublistRemoveWithWildcardsAsLiterals(t, NewSublistWithCache()) +} + +func TestSublistRemoveWithWildcardsAsLiteralsNoCache(t *testing.T) { + testSublistRemoveWithWildcardsAsLiterals(t, NewSublistNoCache()) +} + +func testSublistRemoveWithWildcardsAsLiterals(t *testing.T, s *Sublist) { + subjects := []string{"foo.*-", "foo.>-"} + for _, subject := range subjects { + sub := newSub(subject) + s.Insert(sub) + // Should find no match + rsub := newSub("foo.bar") + s.Remove(rsub) + if c := s.Count(); c != 1 { + t.Fatalf("Expected sublist to still contain sub, got %v", c) + } + s.Remove(sub) + if c := s.Count(); c != 0 { + t.Fatalf("Expected sublist to be empty, got %v", c) + } + } +} + +func TestSublistRaceOnRemove(t *testing.T) { + testSublistRaceOnRemove(t, NewSublistWithCache()) +} + +func TestSublistRaceOnRemoveNoCache(t *testing.T) { + testSublistRaceOnRemove(t, NewSublistNoCache()) +} + +func testSublistRaceOnRemove(t *testing.T, s *Sublist) { + var ( + total = 100 + subs = make(map[int]*subscription, total) // use map for randomness + ) + for i := 0; i < total; i++ { + sub := newQSub("foo", "bar") + subs[i] = sub + } + + for i := 0; i < 2; i++ { + for _, sub := range subs { + s.Insert(sub) + } + // Call Match() once or twice, to make sure we get from cache + if i == 1 { + s.Match("foo") + } + // This will be from cache when i==1 + r := s.Match("foo") + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + for _, sub := range subs { + s.Remove(sub) + } + wg.Done() + }() + for _, qsub := range r.qsubs { + for i := 0; i < len(qsub); i++ { + sub := qsub[i] + if string(sub.queue) != "bar" { + t.Fatalf("Queue name should be bar, got %s", qsub[i].queue) + } + } + } + wg.Wait() + } + + // Repeat tests with regular subs + for i := 0; i < total; i++ { + sub := newSub("foo") + subs[i] = sub + } + + for i := 0; i < 2; i++ { + for _, sub := range subs { + s.Insert(sub) + } + // Call Match() once or twice, to make sure we get from cache + if i == 1 { + s.Match("foo") + } + // This will be from cache when i==1 + r := s.Match("foo") + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + for _, sub := range subs { + s.Remove(sub) + } + wg.Done() + }() + for i := 0; i < len(r.psubs); i++ { + sub := r.psubs[i] + if string(sub.subject) != "foo" { + t.Fatalf("Subject should be foo, got %s", sub.subject) + } + } + wg.Wait() + } +} + +func TestSublistRaceOnInsert(t *testing.T) { + testSublistRaceOnInsert(t, NewSublistWithCache()) +} + +func TestSublistRaceOnInsertNoCache(t *testing.T) { + testSublistRaceOnInsert(t, NewSublistNoCache()) +} + +func testSublistRaceOnInsert(t *testing.T, s *Sublist) { + var ( + total = 100 + subs = make(map[int]*subscription, total) // use map for randomness + wg sync.WaitGroup + ) + for i := 0; i < total; i++ { + sub := newQSub("foo", "bar") + subs[i] = sub + } + wg.Add(1) + go func() { + for _, sub := range subs { + s.Insert(sub) + } + wg.Done() + }() + for i := 0; i < 1000; i++ { + r := s.Match("foo") + for _, qsubs := range r.qsubs { + for _, qsub := range qsubs { + if string(qsub.queue) != "bar" { + t.Fatalf("Expected queue name to be bar, got %v", string(qsub.queue)) + } + } + } + } + wg.Wait() + + // Repeat the test with plain subs + for i := 0; i < total; i++ { + sub := newSub("foo") + subs[i] = sub + } + wg.Add(1) + go func() { + for _, sub := range subs { + s.Insert(sub) + } + wg.Done() + }() + for i := 0; i < 1000; i++ { + r := s.Match("foo") + for _, sub := range r.psubs { + if string(sub.subject) != "foo" { + t.Fatalf("Expected subject to be foo, got %v", string(sub.subject)) + } + } + } + wg.Wait() +} + +func TestSublistRaceOnMatch(t *testing.T) { + s := NewSublistNoCache() + s.Insert(newQSub("foo.*", "workers")) + s.Insert(newQSub("foo.bar", "workers")) + s.Insert(newSub("foo.*")) + s.Insert(newSub("foo.bar")) + + wg := sync.WaitGroup{} + wg.Add(2) + errCh := make(chan error, 2) + f := func() { + defer wg.Done() + for i := 0; i < 10; i++ { + r := s.Match("foo.bar") + for _, sub := range r.psubs { + if !strings.HasPrefix(string(sub.subject), "foo.") { + errCh <- fmt.Errorf("Wrong subject: %s", sub.subject) + return + } + } + for _, qsub := range r.qsubs { + for _, sub := range qsub { + if string(sub.queue) != "workers" { + errCh <- fmt.Errorf("Wrong queue name: %s", sub.queue) + return + } + } + } + } + } + go f() + go f() + wg.Wait() + select { + case e := <-errCh: + t.Fatal(e.Error()) + default: + } +} + +// Remote subscriptions for queue subscribers will be weighted such that a single subscription +// is received, but represents all of the queue subscribers on the remote side. +func TestSublistRemoteQueueSubscriptions(t *testing.T) { + testSublistRemoteQueueSubscriptions(t, NewSublistWithCache()) +} + +func TestSublistRemoteQueueSubscriptionsNoCache(t *testing.T) { + testSublistRemoteQueueSubscriptions(t, NewSublistNoCache()) +} + +func testSublistRemoteQueueSubscriptions(t *testing.T, s *Sublist) { + // Normals + s1 := newQSub("foo", "bar") + s2 := newQSub("foo", "bar") + s.Insert(s1) + s.Insert(s2) + + // Now do weighted remotes. + rs1 := newRemoteQSub("foo", "bar", 10) + s.Insert(rs1) + rs2 := newRemoteQSub("foo", "bar", 10) + s.Insert(rs2) + + // These are just shadowed in results, so should appear as 4 subs. + verifyCount(s, 4, t) + + r := s.Match("foo") + verifyLen(r.psubs, 0, t) + verifyQLen(r.qsubs, 1, t) + verifyLen(r.qsubs[0], 22, t) + + s.Remove(s1) + s.Remove(rs1) + + verifyCount(s, 2, t) + + // Now make sure our shadowed results are correct after a removal. + r = s.Match("foo") + verifyLen(r.psubs, 0, t) + verifyQLen(r.qsubs, 1, t) + verifyLen(r.qsubs[0], 11, t) + + // Now do an update to an existing remote sub to update its weight. + rs2.qw = 1 + s.UpdateRemoteQSub(rs2) + + // Results should reflect new weight. + r = s.Match("foo") + verifyLen(r.psubs, 0, t) + verifyQLen(r.qsubs, 1, t) + verifyLen(r.qsubs[0], 2, t) +} + +func TestSublistSharedEmptyResult(t *testing.T) { + s := NewSublistWithCache() + r1 := s.Match("foo") + verifyLen(r1.psubs, 0, t) + verifyQLen(r1.qsubs, 0, t) + + r2 := s.Match("bar") + verifyLen(r2.psubs, 0, t) + verifyQLen(r2.qsubs, 0, t) + + if r1 != r2 { + t.Fatalf("Expected empty result to be a shared result set") + } +} + +func TestSublistNoCacheStats(t *testing.T) { + s := NewSublistNoCache() + s.Insert(newSub("foo")) + s.Insert(newSub("bar")) + s.Insert(newSub("baz")) + s.Insert(newSub("foo.bar.baz")) + s.Match("a.b.c") + s.Match("bar") + stats := s.Stats() + if stats.NumCache != 0 { + t.Fatalf("Expected 0 for NumCache stat, got %d", stats.NumCache) + } +} + +func TestSublistAll(t *testing.T) { + s := NewSublistNoCache() + subs := []*subscription{ + newSub("foo.bar.baz"), + newSub("foo"), + newSub("baz"), + } + // alter client's kind + subs[0].client.kind = LEAF + for _, sub := range subs { + s.Insert(sub) + } + + var buf [32]*subscription + output := buf[:0] + s.All(&output) + if len(output) != len(subs) { + t.Fatalf("Expected %d for All, got %d", len(subs), len(output)) + } +} + +func TestIsSubsetMatch(t *testing.T) { + for _, test := range []struct { + subject string + test string + result bool + }{ + {"foo.bar", "foo.bar", true}, + {"foo.*", ">", true}, + {"foo.*", "*.*", true}, + {"foo.*", "foo.*", true}, + {"foo.*", "foo.bar", false}, + {"foo.>", ">", true}, + {"foo.>", "*.>", true}, + {"foo.>", "foo.>", true}, + {"foo.>", "foo.bar", false}, + {"foo..bar", "foo.*", false}, // Bad subject, we return false + {"foo.*", "foo..bar", false}, // Bad subject, we return false + } { + t.Run("", func(t *testing.T) { + if res := subjectIsSubsetMatch(test.subject, test.test); res != test.result { + t.Fatalf("Subject %q subset match of %q, should be %v, got %v", + test.test, test.subject, test.result, res) + } + }) + } +} + +func TestSublistRegisterInterestNotification(t *testing.T) { + s := NewSublistWithCache() + ch := make(chan bool, 1) + + expectErr := func(subject string) { + if err := s.RegisterNotification(subject, ch); err != ErrInvalidSubject { + t.Fatalf("Expected err, got %v", err) + } + } + + // Test that we require a literal subject. + expectErr("foo.*") + expectErr(">") + + // Chan needs to be non-nil + if err := s.RegisterNotification("foo", nil); err != ErrNilChan { + t.Fatalf("Expected err, got %v", err) + } + + // Clearing one that is not there will return false. + if s.ClearNotification("foo", ch) { + t.Fatalf("Expected to return false on non-existent notification entry") + } + + // This should work properly. + if err := s.RegisterNotification("foo", ch); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + tt := time.NewTimer(time.Second) + expectBoolWithCh := func(ch chan bool, b bool) { + t.Helper() + tt.Reset(time.Second) + defer tt.Stop() + select { + case v := <-ch: + if v != b { + t.Fatalf("Expected %v, got %v", b, v) + } + case <-tt.C: + t.Fatalf("Timeout waiting for expected value") + } + } + expectBool := func(b bool) { + t.Helper() + expectBoolWithCh(ch, b) + } + expectFalse := func() { + t.Helper() + expectBool(false) + } + expectTrue := func() { + t.Helper() + expectBool(true) + } + expectNone := func() { + t.Helper() + if lch := len(ch); lch != 0 { + t.Fatalf("Expected no notifications, had %d and first was %v", lch, <-ch) + } + } + expectOneWithCh := func(ch chan bool) { + t.Helper() + if len(ch) != 1 { + t.Fatalf("Expected 1 notification") + } + } + expectOne := func() { + t.Helper() + expectOneWithCh(ch) + } + + expectOne() + expectFalse() + sub := newSub("foo") + s.Insert(sub) + expectTrue() + + sub2 := newSub("foo") + s.Insert(sub2) + expectNone() + + if err := s.RegisterNotification("bar", ch); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + expectFalse() + + sub3 := newSub("foo") + s.Insert(sub3) + expectNone() + + // Now remove literals. + s.Remove(sub) + expectNone() + s.Remove(sub2) + expectNone() + s.Remove(sub3) + expectFalse() + + if err := s.RegisterNotification("test.node", ch); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + expectOne() + expectFalse() + + tnSub1 := newSub("test.node.already.exist") + s.Insert(tnSub1) + expectNone() + + tnSub2 := newSub("test.node") + s.Insert(tnSub2) + expectTrue() + + tnSub3 := newSub("test.node") + s.Insert(tnSub3) + expectNone() + + s.Remove(tnSub1) + expectNone() + s.Remove(tnSub2) + expectNone() + s.Remove(tnSub3) + expectFalse() + + if !s.ClearNotification("test.node", ch) { + t.Fatalf("Expected to return true") + } + + sub4 := newSub("bar") + s.Insert(sub4) + expectTrue() + + if !s.ClearNotification("bar", ch) { + t.Fatalf("Expected to return true") + } + s.RLock() + lnr := len(s.notify.remove) + s.RUnlock() + if lnr != 0 { + t.Fatalf("Expected zero entries for remove notify, got %d", lnr) + } + if !s.ClearNotification("foo", ch) { + t.Fatalf("Expected to return true") + } + s.RLock() + notifyMap := s.notify + s.RUnlock() + if notifyMap != nil { + t.Fatalf("Expected the notify map to be nil") + } + + // Let's do some wildcard checks. + // Wildcards will not trigger interest. + subpwc := newSub("*") + s.Insert(subpwc) + expectNone() + + if err := s.RegisterNotification("foo", ch); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + expectFalse() + + s.Insert(sub) + expectTrue() + + s.Remove(sub) + expectFalse() + + s.Remove(subpwc) + expectNone() + + subfwc := newSub(">") + s.Insert(subfwc) + expectNone() + + s.Insert(subpwc) + expectNone() + + s.Remove(subpwc) + expectNone() + + s.Remove(subfwc) + expectNone() + + // Test batch + subs := []*subscription{sub, sub2, sub3, sub4, subpwc, subfwc} + for _, sub := range subs { + s.Insert(sub) + } + expectTrue() + + s.RemoveBatch(subs) + expectOne() + expectFalse() + + // Test queue subs + // We know make sure that if you have qualified a queue group it has to match, etc. + // Also if you do not specify one they will not trigger. + qsub := newQSub("foo.bar.baz", "1") + s.Insert(qsub) + expectNone() + + if err := s.RegisterNotification("foo.bar.baz", ch); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + expectFalse() + + wcqsub := newQSub("foo.bar.>", "1") + s.Insert(wcqsub) + expectNone() + + s.Remove(qsub) + expectNone() + + s.Remove(wcqsub) + expectNone() + + s.Insert(wcqsub) + expectNone() + + if err := s.RegisterQueueNotification("queue.test.node", "q22", ch); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + expectOne() + expectFalse() + + qsub1 := newQSub("queue.test.node.already.exist", "queue") + s.Insert(qsub1) + expectNone() + + qsub2 := newQSub("queue.test.node", "q22") + s.Insert(qsub2) + expectTrue() + + qsub3 := newQSub("queue.test.node", "otherqueue") + s.Insert(qsub3) + expectNone() + + qsub4 := newQSub("queue.different.node", "q22") + s.Insert(qsub4) + expectNone() + + qsub5 := newQSub("queue.test.node", "q22") + s.Insert(qsub5) + expectNone() + + s.Remove(qsub3) + expectNone() + s.Remove(qsub1) + expectNone() + s.Remove(qsub2) + expectNone() + s.Remove(qsub4) + expectNone() + s.Remove(qsub5) + expectFalse() + + if !s.ClearQueueNotification("queue.test.node", "q22", ch) { + t.Fatalf("Expected to return true") + } + + if err := s.RegisterQueueNotification("some.subject", "queue1", ch); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + expectOne() + expectFalse() + + qsub1 = newQSub("some.subject", "queue1") + s.Insert(qsub1) + expectTrue() + + // Create a second channel for this other queue + ch2 := make(chan bool, 1) + if err := s.RegisterQueueNotification("some.subject", "queue2", ch2); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + expectOneWithCh(ch2) + expectBoolWithCh(ch2, false) + + qsub2 = newQSub("some.subject", "queue2") + s.Insert(qsub2) + expectBoolWithCh(ch2, true) + + // But we should not get notification on queue1 + expectNone() + + s.Remove(qsub1) + expectFalse() + s.Remove(qsub2) + expectBoolWithCh(ch2, false) + + if !s.ClearQueueNotification("some.subject", "queue1", ch) { + t.Fatalf("Expected to return true") + } + if !s.ClearQueueNotification("some.subject", "queue2", ch2) { + t.Fatalf("Expected to return true") + } + + // Test non-blocking notifications. + if err := s.RegisterNotification("bar", ch); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + if err := s.RegisterNotification("baz", ch); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + s.Insert(newSub("baz")) + s.Insert(newSub("bar")) + s.Insert(subpwc) + expectOne() + expectFalse() +} + +func TestSublistReverseMatch(t *testing.T) { + s := NewSublistWithCache() + fooSub := newSub("foo") + barSub := newSub("bar") + fooBarSub := newSub("foo.bar") + fooBazSub := newSub("foo.baz") + fooBarBazSub := newSub("foo.bar.baz") + s.Insert(fooSub) + s.Insert(barSub) + s.Insert(fooBarSub) + s.Insert(fooBazSub) + s.Insert(fooBarBazSub) + + r := s.ReverseMatch("foo") + verifyLen(r.psubs, 1, t) + verifyMember(r.psubs, fooSub, t) + + r = s.ReverseMatch("bar") + verifyLen(r.psubs, 1, t) + verifyMember(r.psubs, barSub, t) + + r = s.ReverseMatch("*") + verifyLen(r.psubs, 2, t) + verifyMember(r.psubs, fooSub, t) + verifyMember(r.psubs, barSub, t) + + r = s.ReverseMatch("baz") + verifyLen(r.psubs, 0, t) + + r = s.ReverseMatch("foo.*") + verifyLen(r.psubs, 2, t) + verifyMember(r.psubs, fooBarSub, t) + verifyMember(r.psubs, fooBazSub, t) + + r = s.ReverseMatch("*.*") + verifyLen(r.psubs, 2, t) + verifyMember(r.psubs, fooBarSub, t) + verifyMember(r.psubs, fooBazSub, t) + + r = s.ReverseMatch("*.bar") + verifyLen(r.psubs, 1, t) + verifyMember(r.psubs, fooBarSub, t) + + r = s.ReverseMatch("*.baz") + verifyLen(r.psubs, 1, t) + verifyMember(r.psubs, fooBazSub, t) + + r = s.ReverseMatch("bar.*") + verifyLen(r.psubs, 0, t) + + r = s.ReverseMatch("*.bat") + verifyLen(r.psubs, 0, t) + + r = s.ReverseMatch("foo.>") + verifyLen(r.psubs, 3, t) + verifyMember(r.psubs, fooBarSub, t) + verifyMember(r.psubs, fooBazSub, t) + verifyMember(r.psubs, fooBarBazSub, t) + + r = s.ReverseMatch(">") + verifyLen(r.psubs, 5, t) + verifyMember(r.psubs, fooSub, t) + verifyMember(r.psubs, barSub, t) + verifyMember(r.psubs, fooBarSub, t) + verifyMember(r.psubs, fooBazSub, t) + verifyMember(r.psubs, fooBarBazSub, t) +} + +func TestSublistReverseMatchWider(t *testing.T) { + s := NewSublistWithCache() + sub := newSub("uplink.*.*.>") + s.Insert(sub) + + r := s.ReverseMatch("uplink.1.*.*.>") + verifyLen(r.psubs, 1, t) + verifyMember(r.psubs, sub, t) + + r = s.ReverseMatch("uplink.1.2.3.>") + verifyLen(r.psubs, 1, t) + verifyMember(r.psubs, sub, t) +} + +func TestSublistMatchWithEmptyTokens(t *testing.T) { + for _, test := range []struct { + name string + cache bool + }{ + {"cache", true}, + {"no cache", false}, + } { + t.Run(test.name, func(t *testing.T) { + sl := NewSublist(true) + sub1 := newSub(">") + sub2 := newQSub(">", "queue") + sl.Insert(sub1) + sl.Insert(sub2) + + for _, subj := range []string{".foo", "..foo", "foo..", "foo.", "foo..bar", "foo...bar"} { + t.Run(subj, func(t *testing.T) { + r := sl.Match(subj) + verifyLen(r.psubs, 0, t) + verifyQLen(r.qsubs, 0, t) + }) + } + }) + } +} + +func TestSublistSubjectCollide(t *testing.T) { + require_False(t, SubjectsCollide("foo.*", "foo.*.bar.>")) + require_False(t, SubjectsCollide("foo.*.bar.>", "foo.*")) + require_True(t, SubjectsCollide("foo.*", "foo.foo")) + require_True(t, SubjectsCollide("foo.*", "*.foo")) + require_True(t, SubjectsCollide("foo.bar.>", "*.bar.foo")) +} + +func TestSublistAddCacheHitRate(t *testing.T) { + sl1 := NewSublistWithCache() + fooSub := newSub("foo") + sl1.Insert(fooSub) + for i := 0; i < 4; i++ { + sl1.Match("foo") + } + stats1 := sl1.Stats() + require_True(t, stats1.CacheHitRate == 0.75) + + sl2 := NewSublistWithCache() + barSub := newSub("bar") + sl2.Insert(barSub) + for i := 0; i < 4; i++ { + sl2.Match("bar") + } + stats2 := sl2.Stats() + require_True(t, stats2.CacheHitRate == 0.75) + + ts := &SublistStats{} + ts.add(stats1) + ts.add(stats2) + require_True(t, ts.CacheHitRate == 0.75) +} + +// -- Benchmarks Setup -- + +var benchSublistSubs []*subscription +var benchSublistSl = NewSublistWithCache() + +// https://github.com/golang/go/issues/31859 +func TestMain(m *testing.M) { + flag.StringVar(&testDefaultClusterCompression, "cluster_compression", _EMPTY_, "Test with this compression level as the default") + flag.StringVar(&testDefaultLeafNodeCompression, "leafnode_compression", _EMPTY_, "Test with this compression level as the default") + flag.Parse() + initSublist := false + flag.Visit(func(f *flag.Flag) { + if f.Name == "test.bench" { + initSublist = true + } + }) + if initSublist { + benchSublistSubs = make([]*subscription, 0, 256*1024) + toks := []string{"synadia", "nats", "jetstream", "nkeys", "jwt", "deny", "auth", "drain"} + subsInit("", toks) + for i := 0; i < len(benchSublistSubs); i++ { + benchSublistSl.Insert(benchSublistSubs[i]) + } + addWildcards() + } + os.Exit(m.Run()) +} + +func TestSublistHasInterest(t *testing.T) { + sl := NewSublistWithCache() + fooSub := newSub("foo") + sl.Insert(fooSub) + + // Expect to find that "foo" matches but "bar" doesn't. + // At this point nothing should be in the cache. + require_True(t, sl.HasInterest("foo")) + require_False(t, sl.HasInterest("bar")) + require_Equal(t, sl.cacheHits, 0) + + // Now call Match(), which will populate the cache. + sl.Match("foo") + require_Equal(t, sl.cacheHits, 0) + + // Future calls to HasInterest() should hit the cache now. + for i := uint64(1); i <= 5; i++ { + require_True(t, sl.HasInterest("foo")) + require_Equal(t, sl.cacheHits, i) + } + + // Call Match on a subject we know there is no match. + hits := sl.cacheHits + sl.Match("bar") + require_False(t, sl.HasInterest("bar")) + // The sublist caches "negative" results as well, expect a cache hit. + require_Equal(t, sl.cacheHits, hits+1) + + // Remove fooSub and check interest again + sl.Remove(fooSub) + require_False(t, sl.HasInterest("foo")) + + // Try with some wildcards + sub := newSub("foo.*") + sl.Insert(sub) + require_False(t, sl.HasInterest("foo")) + require_True(t, sl.HasInterest("foo.bar")) + require_False(t, sl.HasInterest("foo.bar.baz")) + + // Remove sub, there should be no interest + sl.Remove(sub) + require_False(t, sl.HasInterest("foo")) + require_False(t, sl.HasInterest("foo.bar")) + require_False(t, sl.HasInterest("foo.bar.baz")) + + sub = newSub("foo.>") + sl.Insert(sub) + require_False(t, sl.HasInterest("foo")) + require_True(t, sl.HasInterest("foo.bar")) + require_True(t, sl.HasInterest("foo.bar.baz")) + + sl.Remove(sub) + require_False(t, sl.HasInterest("foo")) + require_False(t, sl.HasInterest("foo.bar")) + require_False(t, sl.HasInterest("foo.bar.baz")) + + sub = newSub("*.>") + sl.Insert(sub) + require_False(t, sl.HasInterest("foo")) + require_True(t, sl.HasInterest("foo.bar")) + require_True(t, sl.HasInterest("foo.baz")) + sl.Remove(sub) + + sub = newSub("*.bar") + sl.Insert(sub) + require_False(t, sl.HasInterest("foo")) + require_True(t, sl.HasInterest("foo.bar")) + require_False(t, sl.HasInterest("foo.baz")) + sl.Remove(sub) + + sub = newSub("*") + sl.Insert(sub) + require_True(t, sl.HasInterest("foo")) + require_False(t, sl.HasInterest("foo.bar")) + sl.Remove(sub) + + // Try with queues now. + qsub := newQSub("foo", "bar") + sl.Insert(qsub) + require_True(t, sl.HasInterest("foo")) + require_False(t, sl.HasInterest("foo.bar")) + + qsub2 := newQSub("foo", "baz") + sl.Insert(qsub2) + require_True(t, sl.HasInterest("foo")) + require_False(t, sl.HasInterest("foo.bar")) + + // Remove first queue + sl.Remove(qsub) + require_True(t, sl.HasInterest("foo")) + require_False(t, sl.HasInterest("foo.bar")) + + // Remove last. + sl.Remove(qsub2) + require_False(t, sl.HasInterest("foo")) + require_False(t, sl.HasInterest("foo.bar")) + + // With wildcards now + qsub = newQSub("foo.*", "bar") + sl.Insert(qsub) + require_False(t, sl.HasInterest("foo")) + require_True(t, sl.HasInterest("foo.bar")) + require_False(t, sl.HasInterest("foo.bar.baz")) + + // Add another queue to the group + qsub2 = newQSub("foo.*", "baz") + sl.Insert(qsub2) + require_False(t, sl.HasInterest("foo")) + require_True(t, sl.HasInterest("foo.bar")) + require_False(t, sl.HasInterest("foo.bar.baz")) + + // Remove first queue + sl.Remove(qsub) + require_False(t, sl.HasInterest("foo")) + require_True(t, sl.HasInterest("foo.bar")) + require_False(t, sl.HasInterest("foo.bar.baz")) + + // Remove last + sl.Remove(qsub2) + require_False(t, sl.HasInterest("foo")) + require_False(t, sl.HasInterest("foo.bar")) + require_False(t, sl.HasInterest("foo.bar.baz")) + + qsub = newQSub("foo.>", "bar") + sl.Insert(qsub) + require_False(t, sl.HasInterest("foo")) + require_True(t, sl.HasInterest("foo.bar")) + require_True(t, sl.HasInterest("foo.bar.baz")) + + // Add another queue to the group + qsub2 = newQSub("foo.>", "baz") + sl.Insert(qsub2) + require_False(t, sl.HasInterest("foo")) + require_True(t, sl.HasInterest("foo.bar")) + require_True(t, sl.HasInterest("foo.bar.baz")) + + // Remove first queue + sl.Remove(qsub) + require_False(t, sl.HasInterest("foo")) + require_True(t, sl.HasInterest("foo.bar")) + require_True(t, sl.HasInterest("foo.bar.baz")) + + // Remove last + sl.Remove(qsub2) + require_False(t, sl.HasInterest("foo")) + require_False(t, sl.HasInterest("foo.bar")) + require_False(t, sl.HasInterest("foo.bar.baz")) + + qsub = newQSub("*.>", "bar") + sl.Insert(qsub) + require_False(t, sl.HasInterest("foo")) + require_True(t, sl.HasInterest("foo.bar")) + require_True(t, sl.HasInterest("foo.baz")) + sl.Remove(qsub) + + qsub = newQSub("*.bar", "bar") + sl.Insert(qsub) + require_False(t, sl.HasInterest("foo")) + require_True(t, sl.HasInterest("foo.bar")) + require_False(t, sl.HasInterest("foo.baz")) + sl.Remove(qsub) + + qsub = newQSub("*", "bar") + sl.Insert(qsub) + require_True(t, sl.HasInterest("foo")) + require_False(t, sl.HasInterest("foo.bar")) + sl.Remove(qsub) +} + +func TestSublistHasInterestOverlapping(t *testing.T) { + sl := NewSublistWithCache() + require_NoError(t, sl.Insert(newSub("stream.A.child"))) + require_NoError(t, sl.Insert(newSub("stream.*"))) + require_True(t, sl.HasInterest("stream.A.child")) + require_True(t, sl.HasInterest("stream.A")) +} + +func TestSublistNumInterest(t *testing.T) { + sl := NewSublistWithCache() + fooSub := newSub("foo") + sl.Insert(fooSub) + + require_NumInterest := func(t *testing.T, subj string, wnp, wnq int) { + t.Helper() + np, nq := sl.NumInterest(subj) + require_Equal(t, np, wnp) + require_Equal(t, nq, wnq) + } + + // Expect to find that "foo" matches but "bar" doesn't. + // At this point nothing should be in the cache. + require_NumInterest(t, "foo", 1, 0) + require_NumInterest(t, "bar", 0, 0) + require_Equal(t, sl.cacheHits, 0) + + // Now call Match(), which will populate the cache. + sl.Match("foo") + require_Equal(t, sl.cacheHits, 0) + + // Future calls to HasInterest() should hit the cache now. + for i := uint64(1); i <= 5; i++ { + require_NumInterest(t, "foo", 1, 0) + require_Equal(t, sl.cacheHits, i) + } + + // Call Match on a subject we know there is no match. + hits := sl.cacheHits + sl.Match("bar") + require_NumInterest(t, "bar", 0, 0) + // The sublist caches "negative" results as well, expect a cache hit. + require_Equal(t, sl.cacheHits, hits+1) + + // Remove fooSub and check interest again + sl.Remove(fooSub) + require_NumInterest(t, "foo", 0, 0) + + // Try with some wildcards + sub := newSub("foo.*") + sl.Insert(sub) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 1, 0) + require_NumInterest(t, "foo.bar.baz", 0, 0) + + // Remove sub, there should be no interest + sl.Remove(sub) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 0, 0) + require_NumInterest(t, "foo.bar.baz", 0, 0) + + sub = newSub("foo.>") + sl.Insert(sub) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 1, 0) + require_NumInterest(t, "foo.bar.baz", 1, 0) + + sl.Remove(sub) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 0, 0) + require_NumInterest(t, "foo.bar.baz", 0, 0) + + sub = newSub("*.>") + sl.Insert(sub) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 1, 0) + require_NumInterest(t, "foo.bar.baz", 1, 0) + sl.Remove(sub) + + sub = newSub("*.bar") + sl.Insert(sub) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 1, 0) + require_NumInterest(t, "foo.bar.baz", 0, 0) + sl.Remove(sub) + + sub = newSub("*") + sl.Insert(sub) + require_NumInterest(t, "foo", 1, 0) + require_NumInterest(t, "foo.bar", 0, 0) + sl.Remove(sub) + + // Try with queues now. + qsub := newQSub("foo", "bar") + sl.Insert(qsub) + require_NumInterest(t, "foo", 0, 1) + require_NumInterest(t, "foo.bar", 0, 0) + + qsub2 := newQSub("foo", "baz") + sl.Insert(qsub2) + require_NumInterest(t, "foo", 0, 2) + require_NumInterest(t, "foo.bar", 0, 0) + + // Add a second qsub to the second queue group + qsub3 := newQSub("foo", "baz") + sl.Insert(qsub3) + require_NumInterest(t, "foo", 0, 3) + require_NumInterest(t, "foo.bar", 0, 0) + + // Remove first queue + sl.Remove(qsub) + require_NumInterest(t, "foo", 0, 2) + require_NumInterest(t, "foo.bar", 0, 0) + + // Remove second + sl.Remove(qsub2) + require_NumInterest(t, "foo", 0, 1) + require_NumInterest(t, "foo.bar", 0, 0) + + // Remove last. + sl.Remove(qsub3) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 0, 0) + + // With wildcards now + qsub = newQSub("foo.*", "bar") + sl.Insert(qsub) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 0, 1) + require_NumInterest(t, "foo.bar.baz", 0, 0) + + // Add another queue to the group + qsub2 = newQSub("foo.*", "baz") + sl.Insert(qsub2) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 0, 2) + require_NumInterest(t, "foo.bar.baz", 0, 0) + + qsub3 = newQSub("foo.*", "baz") + sl.Insert(qsub3) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 0, 3) + require_NumInterest(t, "foo.bar.baz", 0, 0) + + // Remove first queue + sl.Remove(qsub) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 0, 2) + require_NumInterest(t, "foo.bar.baz", 0, 0) + + // Remove second + sl.Remove(qsub2) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 0, 1) + require_NumInterest(t, "foo.bar.baz", 0, 0) + + // Remove last + sl.Remove(qsub3) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 0, 0) + require_NumInterest(t, "foo.bar.baz", 0, 0) + + // With > wildcard + qsub = newQSub("foo.>", "bar") + sl.Insert(qsub) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 0, 1) + require_NumInterest(t, "foo.bar.baz", 0, 1) + + // Add another queue to the group + qsub2 = newQSub("foo.>", "baz") + sl.Insert(qsub2) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 0, 2) + require_NumInterest(t, "foo.bar.baz", 0, 2) + + // Add another queue to second group. + qsub3 = newQSub("foo.>", "baz") + sl.Insert(qsub3) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 0, 3) + require_NumInterest(t, "foo.bar.baz", 0, 3) + + // Remove first queue + sl.Remove(qsub) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 0, 2) + require_NumInterest(t, "foo.bar.baz", 0, 2) + + // Remove second + sl.Remove(qsub2) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 0, 1) + require_NumInterest(t, "foo.bar.baz", 0, 1) + + // Remove last + sl.Remove(qsub3) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 0, 0) + require_NumInterest(t, "foo.bar.baz", 0, 0) + + qsub = newQSub("*.>", "bar") + sl.Insert(qsub) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 0, 1) + require_NumInterest(t, "foo.bar.baz", 0, 1) + sl.Remove(qsub) + + qsub = newQSub("*.bar", "bar") + sl.Insert(qsub) + require_NumInterest(t, "foo", 0, 0) + require_NumInterest(t, "foo.bar", 0, 1) + require_NumInterest(t, "foo.bar.baz", 0, 0) + sl.Remove(qsub) + + qsub = newQSub("*", "bar") + sl.Insert(qsub) + require_NumInterest(t, "foo", 0, 1) + require_NumInterest(t, "foo.bar", 0, 0) + sl.Remove(qsub) +} + +func subsInit(pre string, toks []string) { + var sub string + for _, t := range toks { + if len(pre) > 0 { + sub = pre + tsep + t + } else { + sub = t + } + benchSublistSubs = append(benchSublistSubs, newSub(sub)) + if len(strings.Split(sub, tsep)) < 5 { + subsInit(sub, toks) + } + } +} + +func addWildcards() { + benchSublistSl.Insert(newSub("cloud.>")) + benchSublistSl.Insert(newSub("cloud.nats.component.>")) + benchSublistSl.Insert(newSub("cloud.*.*.nkeys.*")) +} + +// -- Benchmarks Setup End -- + +func Benchmark______________________SublistInsert(b *testing.B) { + s := NewSublistWithCache() + for i, l := 0, len(benchSublistSubs); i < b.N; i++ { + index := i % l + s.Insert(benchSublistSubs[index]) + } +} + +func Benchmark_______________SublistInsertNoCache(b *testing.B) { + s := NewSublistNoCache() + for i, l := 0, len(benchSublistSubs); i < b.N; i++ { + index := i % l + s.Insert(benchSublistSubs[index]) + } +} + +func benchSublistTokens(b *testing.B, tokens string) { + for i := 0; i < b.N; i++ { + benchSublistSl.Match(tokens) + } +} + +func Benchmark____________SublistMatchSingleToken(b *testing.B) { + benchSublistTokens(b, "synadia") +} + +func Benchmark______________SublistMatchTwoTokens(b *testing.B) { + benchSublistTokens(b, "synadia.nats") +} + +func Benchmark____________SublistMatchThreeTokens(b *testing.B) { + benchSublistTokens(b, "synadia.nats.jetstream") +} + +func Benchmark_____________SublistMatchFourTokens(b *testing.B) { + benchSublistTokens(b, "synadia.nats.jetstream.nkeys") +} + +func Benchmark_SublistMatchFourTokensSingleResult(b *testing.B) { + benchSublistTokens(b, "synadia.nats.jetstream.nkeys") +} + +func Benchmark_SublistMatchFourTokensMultiResults(b *testing.B) { + benchSublistTokens(b, "cloud.nats.component.router") +} + +func Benchmark_______SublistMissOnLastTokenOfFive(b *testing.B) { + benchSublistTokens(b, "synadia.nats.jetstream.nkeys.ZZZZ") +} + +func multiRead(b *testing.B, num int) { + var swg, fwg sync.WaitGroup + swg.Add(num) + fwg.Add(num) + s := "synadia.nats.jetstream.nkeys" + for i := 0; i < num; i++ { + go func() { + swg.Done() + swg.Wait() + n := b.N / num + for i := 0; i < n; i++ { + benchSublistSl.Match(s) + } + fwg.Done() + }() + } + swg.Wait() + b.ResetTimer() + fwg.Wait() +} + +func Benchmark____________Sublist10XMultipleReads(b *testing.B) { + multiRead(b, 10) +} + +func Benchmark___________Sublist100XMultipleReads(b *testing.B) { + multiRead(b, 100) +} + +func Benchmark__________Sublist1000XMultipleReads(b *testing.B) { + multiRead(b, 1000) +} + +func Benchmark________________SublistMatchLiteral(b *testing.B) { + cachedSubj := "foo.foo.foo.foo.foo.foo.foo.foo.foo.foo" + subjects := []string{ + "foo.foo.foo.foo.foo.foo.foo.foo.foo.foo", + "foo.foo.foo.foo.foo.foo.foo.foo.foo.>", + "foo.foo.foo.foo.foo.foo.foo.foo.>", + "foo.foo.foo.foo.foo.foo.foo.>", + "foo.foo.foo.foo.foo.foo.>", + "foo.foo.foo.foo.foo.>", + "foo.foo.foo.foo.>", + "foo.foo.foo.>", + "foo.foo.>", + "foo.>", + ">", + "foo.foo.foo.foo.foo.foo.foo.foo.foo.*", + "foo.foo.foo.foo.foo.foo.foo.foo.*.*", + "foo.foo.foo.foo.foo.foo.foo.*.*.*", + "foo.foo.foo.foo.foo.foo.*.*.*.*", + "foo.foo.foo.foo.foo.*.*.*.*.*", + "foo.foo.foo.foo.*.*.*.*.*.*", + "foo.foo.foo.*.*.*.*.*.*.*", + "foo.foo.*.*.*.*.*.*.*.*", + "foo.*.*.*.*.*.*.*.*.*", + "*.*.*.*.*.*.*.*.*.*", + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, subject := range subjects { + if !matchLiteral(cachedSubj, subject) { + b.Fatalf("Subject %q no match with %q", cachedSubj, subject) + } + } + } +} + +func Benchmark_____SublistMatch10kSubsWithNoCache(b *testing.B) { + var nsubs = 512 + s := NewSublistNoCache() + subject := "foo" + for i := 0; i < nsubs; i++ { + s.Insert(newSub(subject)) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + r := s.Match(subject) + if len(r.psubs) != nsubs { + b.Fatalf("Results len is %d, should be %d", len(r.psubs), nsubs) + } + } +} + +func removeTest(b *testing.B, singleSubject, doBatch bool, qgroup string) { + s := NewSublistWithCache() + subject := "foo" + + subs := make([]*subscription, 0, b.N) + for i := 0; i < b.N; i++ { + var sub *subscription + if singleSubject { + sub = newQSub(subject, qgroup) + } else { + sub = newQSub(fmt.Sprintf("%s.%d\n", subject, i), qgroup) + } + s.Insert(sub) + subs = append(subs, sub) + } + + // Actual test on Remove + b.ResetTimer() + if doBatch { + s.RemoveBatch(subs) + } else { + for _, sub := range subs { + s.Remove(sub) + } + } +} + +func Benchmark__________SublistRemove1TokenSingle(b *testing.B) { + removeTest(b, true, false, "") +} + +func Benchmark___________SublistRemove1TokenBatch(b *testing.B) { + removeTest(b, true, true, "") +} + +func Benchmark_________SublistRemove2TokensSingle(b *testing.B) { + removeTest(b, false, false, "") +} + +func Benchmark__________SublistRemove2TokensBatch(b *testing.B) { + removeTest(b, false, true, "") +} + +func Benchmark________SublistRemove1TokenQGSingle(b *testing.B) { + removeTest(b, true, false, "bar") +} + +func Benchmark_________SublistRemove1TokenQGBatch(b *testing.B) { + removeTest(b, true, true, "bar") +} + +func removeMultiTest(b *testing.B, singleSubject, doBatch bool) { + s := NewSublistWithCache() + subject := "foo" + var swg, fwg sync.WaitGroup + swg.Add(b.N) + fwg.Add(b.N) + + // We will have b.N go routines each with 1k subscriptions. + sc := 1000 + + for i := 0; i < b.N; i++ { + go func() { + subs := make([]*subscription, 0, sc) + for n := 0; n < sc; n++ { + var sub *subscription + if singleSubject { + sub = newSub(subject) + } else { + sub = newSub(fmt.Sprintf("%s.%d\n", subject, n)) + } + s.Insert(sub) + subs = append(subs, sub) + } + // Wait to start test + swg.Done() + swg.Wait() + // Actual test on Remove + if doBatch { + s.RemoveBatch(subs) + } else { + for _, sub := range subs { + s.Remove(sub) + } + } + fwg.Done() + }() + } + swg.Wait() + b.ResetTimer() + fwg.Wait() +} + +// Check contention rates for remove from multiple Go routines. +// Reason for BatchRemove. +func Benchmark_________SublistRemove1kSingleMulti(b *testing.B) { + removeMultiTest(b, true, false) +} + +// Batch version +func Benchmark__________SublistRemove1kBatchMulti(b *testing.B) { + removeMultiTest(b, true, true) +} + +func Benchmark__SublistRemove1kSingle2TokensMulti(b *testing.B) { + removeMultiTest(b, false, false) +} + +// Batch version +func Benchmark___SublistRemove1kBatch2TokensMulti(b *testing.B) { + removeMultiTest(b, false, true) +} + +// Cache contention tests +func cacheContentionTest(b *testing.B, numMatchers, numAdders, numRemovers int) { + var swg, fwg, mwg sync.WaitGroup + total := numMatchers + numAdders + numRemovers + swg.Add(total) + fwg.Add(total) + mwg.Add(numMatchers) + + mu := sync.RWMutex{} + subs := make([]*subscription, 0, 8192) + + quitCh := make(chan struct{}) + + // Set up a new sublist. subjects will be foo.bar.baz.N + s := NewSublistWithCache() + mu.Lock() + for i := 0; i < 10000; i++ { + sub := newSub(fmt.Sprintf("foo.bar.baz.%d", i)) + s.Insert(sub) + subs = append(subs, sub) + } + mu.Unlock() + + // Now warm up the cache + for i := 0; i < slCacheMax; i++ { + s.Match(fmt.Sprintf("foo.bar.baz.%d", i)) + } + + // Setup go routines. + + // Adders + for i := 0; i < numAdders; i++ { + go func() { + swg.Done() + swg.Wait() + for { + select { + case <-quitCh: + fwg.Done() + return + default: + mu.Lock() + next := len(subs) + subj := "foo.bar.baz." + strconv.FormatInt(int64(next), 10) + sub := newSub(subj) + subs = append(subs, sub) + mu.Unlock() + s.Insert(sub) + } + } + }() + } + + // Removers + for i := 0; i < numRemovers; i++ { + go func() { + prand := rand.New(rand.NewPCG(uint64(time.Now().UnixNano()), 0)) + swg.Done() + swg.Wait() + for { + select { + case <-quitCh: + fwg.Done() + return + default: + mu.RLock() + lh := len(subs) - 1 + index := prand.IntN(lh) + sub := subs[index] + mu.RUnlock() + s.Remove(sub) + } + } + }() + } + + // Matchers + for i := 0; i < numMatchers; i++ { + go func() { + id := nuid.New() + swg.Done() + swg.Wait() + + // We will miss on purpose to blow the cache. + n := b.N / numMatchers + for i := 0; i < n; i++ { + subj := "foo.bar.baz." + id.Next() + s.Match(subj) + } + mwg.Done() + fwg.Done() + }() + } + + swg.Wait() + b.ResetTimer() + mwg.Wait() + b.StopTimer() + close(quitCh) + fwg.Wait() +} + +func Benchmark____SublistCacheContention10M10A10R(b *testing.B) { + cacheContentionTest(b, 10, 10, 10) +} + +func Benchmark_SublistCacheContention100M100A100R(b *testing.B) { + cacheContentionTest(b, 100, 100, 100) +} + +func Benchmark____SublistCacheContention1kM1kA1kR(b *testing.B) { + cacheContentionTest(b, 1024, 1024, 1024) +} + +func Benchmark_SublistCacheContention10kM10kA10kR(b *testing.B) { + cacheContentionTest(b, 10*1024, 10*1024, 10*1024) +} + +func Benchmark______________IsValidLiteralSubject(b *testing.B) { + for i := 0; i < b.N; i++ { + IsValidLiteralSubject("foo.bar.baz.22") + } +} + +func Benchmark___________________subjectIsLiteral(b *testing.B) { + for i := 0; i < b.N; i++ { + subjectIsLiteral("foo.bar.baz.22") + } +} + +// Test to determine the fastest way to match a filter +// with wildcards against a literal subject. +func Benchmark_SubjectFilterMatchers(b *testing.B) { + cases := []struct { + name string + subject string + filter string + want bool + }{ + { + name: "short_single_wc_match", + subject: "foo.baz.12345", + filter: "foo.baz.*", + want: true, + }, + { + name: "short_mixed_wc_match", + subject: "foo.baz.12345", + filter: "foo.*.>", + want: true, + }, + { + name: "long_single_wc_match", + subject: "foo.alpha.beta.gamma.delta.epsilon.zeta.12345", + filter: "foo.alpha.beta.gamma.delta.epsilon.zeta.*", + want: true, + }, + { + name: "long_many_wc_late_mismatch", + subject: "foo.alpha.beta.gamma.delta.epsilon.zeta.12345", + filter: "foo.*.*.*.*.*.*.99999", + want: false, + }, + } + + for _, tc := range cases { + b.Run(tc.name, func(b *testing.B) { + b.Run("subjectIsSubsetMatch", func(b *testing.B) { + var matched bool + for i := 0; i < b.N; i++ { + matched = subjectIsSubsetMatch(tc.subject, tc.filter) + } + if matched != tc.want { + b.Fatalf("unexpected result: got %v, want %v", matched, tc.want) + } + }) + b.Run("matchLiteral", func(b *testing.B) { + var matched bool + for i := 0; i < b.N; i++ { + matched = matchLiteral(tc.subject, tc.filter) + } + if matched != tc.want { + b.Fatalf("unexpected result: got %v, want %v", matched, tc.want) + } + }) + }) + } +} diff --git a/server/sysmem/mem_bsd.go b/server/sysmem/mem_bsd.go new file mode 100644 index 0000000..46aaf99 --- /dev/null +++ b/server/sysmem/mem_bsd.go @@ -0,0 +1,20 @@ +// Copyright 2019-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build freebsd || openbsd || dragonfly || netbsd + +package sysmem + +func Memory() int64 { + return sysctlInt64("hw.physmem") +} diff --git a/server/sysmem/mem_darwin.go b/server/sysmem/mem_darwin.go new file mode 100644 index 0000000..e2dee28 --- /dev/null +++ b/server/sysmem/mem_darwin.go @@ -0,0 +1,20 @@ +// Copyright 2019-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build darwin + +package sysmem + +func Memory() int64 { + return sysctlInt64("hw.memsize") +} diff --git a/server/sysmem/mem_linux.go b/server/sysmem/mem_linux.go new file mode 100644 index 0000000..2d15e5b --- /dev/null +++ b/server/sysmem/mem_linux.go @@ -0,0 +1,27 @@ +// Copyright 2019-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package sysmem + +import "syscall" + +func Memory() int64 { + var info syscall.Sysinfo_t + err := syscall.Sysinfo(&info) + if err != nil { + return 0 + } + return int64(info.Totalram) * int64(info.Unit) +} diff --git a/server/sysmem/mem_solaris.go b/server/sysmem/mem_solaris.go new file mode 100644 index 0000000..afba036 --- /dev/null +++ b/server/sysmem/mem_solaris.go @@ -0,0 +1,37 @@ +// Copyright 2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build illumos || solaris + +package sysmem + +import ( + "golang.org/x/sys/unix" +) + +const ( + _SC_PHYS_PAGES = 500 + _SC_PAGESIZE = 11 +) + +func Memory() int64 { + pages, err := unix.Sysconf(_SC_PHYS_PAGES) + if err != nil { + return 0 + } + pageSize, err := unix.Sysconf(_SC_PAGESIZE) + if err != nil { + return 0 + } + return int64(pages) * int64(pageSize) +} diff --git a/server/sysmem/mem_wasm.go b/server/sysmem/mem_wasm.go new file mode 100644 index 0000000..bde3586 --- /dev/null +++ b/server/sysmem/mem_wasm.go @@ -0,0 +1,21 @@ +// Copyright 2022-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build wasm + +package sysmem + +func Memory() int64 { + // TODO: We don't know the system memory + return 0 +} diff --git a/server/sysmem/mem_windows.go b/server/sysmem/mem_windows.go new file mode 100644 index 0000000..9fcb0ea --- /dev/null +++ b/server/sysmem/mem_windows.go @@ -0,0 +1,51 @@ +// Copyright 2019-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package sysmem + +import ( + "unsafe" + + "golang.org/x/sys/windows" +) + +var winKernel32 = windows.NewLazySystemDLL("kernel32.dll") +var winGlobalMemoryStatusEx = winKernel32.NewProc("GlobalMemoryStatusEx") + +func init() { + if err := winKernel32.Load(); err != nil { + panic(err) + } + if err := winGlobalMemoryStatusEx.Find(); err != nil { + panic(err) + } +} + +// https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/ns-sysinfoapi-memorystatusex +type _memoryStatusEx struct { + dwLength uint32 + dwMemoryLoad uint32 + ullTotalPhys uint64 + unused [6]uint64 // ignore rest of struct +} + +func Memory() int64 { + msx := &_memoryStatusEx{dwLength: 64} + res, _, _ := winGlobalMemoryStatusEx.Call(uintptr(unsafe.Pointer(msx))) + if res == 0 { + return 0 + } + return int64(msx.ullTotalPhys) +} diff --git a/server/sysmem/mem_zos.go b/server/sysmem/mem_zos.go new file mode 100644 index 0000000..f8db5c5 --- /dev/null +++ b/server/sysmem/mem_zos.go @@ -0,0 +1,21 @@ +// Copyright 2022-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build zos + +package sysmem + +func Memory() int64 { + // TODO: We don't know the system memory + return 0 +} diff --git a/server/sysmem/sysctl.go b/server/sysmem/sysctl.go new file mode 100644 index 0000000..fd6a41d --- /dev/null +++ b/server/sysmem/sysctl.go @@ -0,0 +1,33 @@ +// Copyright 2019-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build darwin || freebsd || openbsd || dragonfly || netbsd + +package sysmem + +import ( + "syscall" + "unsafe" +) + +func sysctlInt64(name string) int64 { + s, err := syscall.Sysctl(name) + if err != nil { + return 0 + } + // Make sure it's 8 bytes when we do the cast below. + // We were getting fatal error: checkptr: converted pointer straddles multiple allocations in go 1.22.1 on darwin. + var b [8]byte + copy(b[:], s) + return *(*int64)(unsafe.Pointer(&b[0])) +} diff --git a/server/test_test.go b/server/test_test.go new file mode 100644 index 0000000..96bfe12 --- /dev/null +++ b/server/test_test.go @@ -0,0 +1,398 @@ +// Copyright 2019-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "cmp" + "fmt" + "math/rand/v2" + "net/url" + "os" + "reflect" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/internal/antithesis" +) + +// DefaultTestOptions are default options for the unit tests. +var DefaultTestOptions = Options{ + Host: "127.0.0.1", + Port: 4222, + NoLog: true, + NoSigs: true, + MaxControlLine: 4096, + DisableShortFirstPing: true, + NoJetStreamStrict: false, +} + +func testDefaultClusterOptionsForLeafNodes() *Options { + o := DefaultTestOptions + o.Port = -1 + o.Cluster.Host = o.Host + o.Cluster.Port = -1 + o.Gateway.Host = o.Host + o.Gateway.Port = -1 + o.LeafNode.Host = o.Host + o.LeafNode.Port = -1 + return &o +} + +func RunRandClientPortServer(t *testing.T) *Server { + opts := DefaultTestOptions + opts.Port = -1 + opts.StoreDir = t.TempDir() + return RunServer(&opts) +} + +func require_True(t testing.TB, b bool) { + t.Helper() + if !b { + antithesis.AssertUnreachable(t, "Failed require_True check", nil) + t.Fatalf("require true, but got false") + } +} + +func require_False(t testing.TB, b bool) { + t.Helper() + if b { + antithesis.AssertUnreachable(t, "Failed require_False check", nil) + t.Fatalf("require false, but got true") + } +} + +func require_NoError(t testing.TB, err error) { + t.Helper() + if err != nil { + antithesis.AssertUnreachable(t, "Failed require_NoError check", map[string]any{ + "error": err.Error(), + }) + t.Fatalf("require no error, but got: %v", err) + } +} + +// Must be used in a defer call. +func require_NoPanic(t testing.TB) { + t.Helper() + if err := recover(); err != nil { + t.Fatal(err) + } +} + +func require_NotNil[T any](t testing.TB, vs ...T) { + t.Helper() + for _, v := range vs { + r := reflect.ValueOf(v) + switch k := r.Kind(); k { + case reflect.Pointer, reflect.Interface, reflect.Slice, + reflect.Map, reflect.Chan, reflect.Func: + if r.IsNil() { + antithesis.AssertUnreachable(t, "Failed require_NotNil check", nil) + t.Fatalf("require not nil, but got nil") + } + } + } +} + +func require_Contains(t *testing.T, s string, subStrs ...string) { + t.Helper() + for _, subStr := range subStrs { + if !strings.Contains(s, subStr) { + antithesis.AssertUnreachable(t, "Failed require_Contains check", map[string]any{ + "string": s, + "sub_strings": subStr, + }) + t.Fatalf("require %q to be contained in %q", subStr, s) + } + } +} + +func require_Error(t testing.TB, err error, expected ...error) { + t.Helper() + if err == nil { + antithesis.AssertUnreachable(t, "Failed require_Error check (nil error)", nil) + t.Fatalf("require error, but got none") + } + if len(expected) == 0 { + return + } + // Try to strip nats prefix from Go library if present. + const natsErrPre = "nats: " + eStr := err.Error() + if strings.HasPrefix(eStr, natsErrPre) { + eStr = strings.Replace(eStr, natsErrPre, _EMPTY_, 1) + } + + for _, e := range expected { + if err == e || strings.Contains(eStr, e.Error()) || strings.Contains(e.Error(), eStr) { + return + } + } + + antithesis.AssertUnreachable(t, "Failed require_Error check (unexpected error)", map[string]any{ + "error": err.Error(), + }) + t.Fatalf("Expected one of %v, got '%v'", expected, err) +} + +func require_Equal[T comparable](t testing.TB, a, b T) { + t.Helper() + if a != b { + antithesis.AssertUnreachable(t, "Failed require_Equal check", nil) + t.Fatalf("require %T equal, but got: %v != %v", a, a, b) + } +} + +func require_NotEqual[T comparable](t testing.TB, a, b T) { + t.Helper() + if a == b { + antithesis.AssertUnreachable(t, "Failed require_NotEqual check", nil) + t.Fatalf("require %T not equal, but got: %v == %v", a, a, b) + } +} + +func require_Len(t testing.TB, a, b int) { + t.Helper() + if a != b { + antithesis.AssertUnreachable(t, "Failed require_Len check", nil) + t.Fatalf("require len, but got: %v != %v", a, b) + } +} + +func require_LessThan[T cmp.Ordered](t *testing.T, a, b T) { + t.Helper() + if a >= b { + antithesis.AssertUnreachable(t, "Failed require_LessThan check", nil) + t.Fatalf("require %v to be less than %v", a, b) + } +} + +func require_ChanRead[T any](t *testing.T, ch chan T, timeout time.Duration) T { + t.Helper() + select { + case v := <-ch: + return v + case <-time.After(timeout): + antithesis.AssertUnreachable(t, "Failed require_ChanRead check", nil) + t.Fatalf("require read from channel within %v but didn't get anything", timeout) + } + panic("this shouldn't be possible") +} + +func require_NoChanRead[T any](t *testing.T, ch chan T, timeout time.Duration) { + t.Helper() + select { + case <-ch: + antithesis.AssertUnreachable(t, "Failed require_NoChanRead check", nil) + t.Fatalf("require no read from channel within %v but got something", timeout) + case <-time.After(timeout): + } +} + +func checkNatsError(t *testing.T, e *ApiError, id ErrorIdentifier) { + t.Helper() + ae, ok := ApiErrors[id] + if !ok { + t.Fatalf("Unknown error ID identifier: %d", id) + } + + if e.ErrCode != ae.ErrCode { + t.Fatalf("Did not get NATS Error %d: %+v", e.ErrCode, e) + } +} + +// Creates a full cluster with numServers and given name and makes sure its well formed. +// Will have Gateways and Leaf Node connections active. +func createClusterWithName(t *testing.T, clusterName string, numServers int, connectTo ...*cluster) *cluster { + t.Helper() + return createClusterEx(t, false, 5*time.Millisecond, true, clusterName, numServers, connectTo...) +} + +// Creates a cluster and optionally additional accounts and users. +// Will have Gateways and Leaf Node connections active. +func createClusterEx(t *testing.T, doAccounts bool, gwSolicit time.Duration, waitOnGWs bool, clusterName string, numServers int, connectTo ...*cluster) *cluster { + t.Helper() + + if clusterName == "" || numServers < 1 { + t.Fatalf("Bad params") + } + + // Setup some accounts and users. + // $SYS is always the system account. And we have default FOO and BAR accounts, as well + // as DLC and NGS which do a service import. + createAccountsAndUsers := func() ([]*Account, []*User) { + if !doAccounts { + return []*Account{NewAccount("$SYS")}, nil + } + + sys := NewAccount("$SYS") + ngs := NewAccount("NGS") + dlc := NewAccount("DLC") + foo := NewAccount("FOO") + bar := NewAccount("BAR") + + accounts := []*Account{sys, foo, bar, ngs, dlc} + + ngs.AddServiceExport("ngs.usage.*", nil) + dlc.AddServiceImport(ngs, "ngs.usage", "ngs.usage.dlc") + + // Setup users + users := []*User{ + {Username: "dlc", Password: "pass", Permissions: nil, Account: dlc}, + {Username: "ngs", Password: "pass", Permissions: nil, Account: ngs}, + {Username: "foo", Password: "pass", Permissions: nil, Account: foo}, + {Username: "bar", Password: "pass", Permissions: nil, Account: bar}, + {Username: "sys", Password: "pass", Permissions: nil, Account: sys}, + } + return accounts, users + } + + bindGlobal := func(s *Server) { + ngs, err := s.LookupAccount("NGS") + if err != nil { + return + } + // Bind global to service import + gacc, _ := s.LookupAccount("$G") + gacc.AddServiceImport(ngs, "ngs.usage", "ngs.usage.$G") + } + + // If we are going to connect to another cluster set that up now for options. + var gws []*RemoteGatewayOpts + for _, c := range connectTo { + // Gateways autodiscover here too, so just need one address from the set. + gwAddr := fmt.Sprintf("nats-gw://%s:%d", c.opts[0].Gateway.Host, c.opts[0].Gateway.Port) + gwurl, _ := url.Parse(gwAddr) + gws = append(gws, &RemoteGatewayOpts{Name: c.name, URLs: []*url.URL{gwurl}}) + } + + // Make the GWs form faster for the tests. + SetGatewaysSolicitDelay(gwSolicit) + defer ResetGatewaysSolicitDelay() + + // Create seed first. + o := testDefaultClusterOptionsForLeafNodes() + o.Gateway.Name = clusterName + o.Gateway.Gateways = gws + // All of these need system accounts. + o.Accounts, o.Users = createAccountsAndUsers() + o.SystemAccount = "$SYS" + o.ServerName = fmt.Sprintf("%s1", clusterName) + // Run the server + s := RunServer(o) + bindGlobal(s) + + c := &cluster{servers: make([]*Server, 0, numServers), opts: make([]*Options, 0, numServers), name: clusterName} + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + + // For connecting to seed server above. + routeAddr := fmt.Sprintf("nats-route://%s:%d", o.Cluster.Host, o.Cluster.Port) + rurl, _ := url.Parse(routeAddr) + routes := []*url.URL{rurl} + + for i := 1; i < numServers; i++ { + o := testDefaultClusterOptionsForLeafNodes() + o.Gateway.Name = clusterName + o.Gateway.Gateways = gws + o.Routes = routes + // All of these need system accounts. + o.Accounts, o.Users = createAccountsAndUsers() + o.SystemAccount = "$SYS" + o.ServerName = fmt.Sprintf("%s%d", clusterName, i+1) + s := RunServer(o) + bindGlobal(s) + + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + } + checkClusterFormed(t, c.servers...) + + if waitOnGWs { + // Wait on gateway connections if we were asked to connect to other gateways. + if numGWs := len(connectTo); numGWs > 0 { + for _, s := range c.servers { + waitForOutboundGateways(t, s, numGWs, 2*time.Second) + } + } + } + c.t = t + return c +} + +func (c *cluster) shutdown() { + if c == nil { + return + } + // Stop any proxies. + for _, np := range c.nproxies { + np.stop() + } + // Shutdown and cleanup servers. + for i, s := range c.servers { + sd := s.StoreDir() + s.Shutdown() + s.WaitForShutdown() + if cf := c.opts[i].ConfigFile; cf != _EMPTY_ { + os.Remove(cf) + } + if sd != _EMPTY_ { + sd = strings.TrimSuffix(sd, JetStreamStoreDir) + os.RemoveAll(sd) + } + } +} + +func shutdownCluster(c *cluster) { + c.shutdown() +} + +func (c *cluster) randomServer() *Server { + return c.randomServerFromCluster(c.name) +} + +func (c *cluster) randomServerFromCluster(cname string) *Server { + // Since these can be randomly shutdown in certain tests make sure they are running first. + // Copy our servers list and shuffle then walk looking for first running server. + cs := append(c.servers[:0:0], c.servers...) + rand.Shuffle(len(cs), func(i, j int) { cs[i], cs[j] = cs[j], cs[i] }) + + for _, s := range cs { + if s.Running() && s.ClusterName() == cname { + return s + } + } + return nil +} + +func runSolicitLeafServer(lso *Options) (*Server, *Options) { + return runSolicitLeafServerToURL(fmt.Sprintf("nats-leaf://%s:%d", lso.LeafNode.Host, lso.LeafNode.Port)) +} + +func runSolicitLeafServerToURL(surl string) (*Server, *Options) { + o := DefaultTestOptions + o.Port = -1 + o.NoSystemAccount = true + rurl, _ := url.Parse(surl) + o.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{rurl}}} + o.LeafNode.ReconnectInterval = 100 * time.Millisecond + return RunServer(&o), &o +} + +func skipIfBuildkite(t *testing.T) { + if os.Getenv("BUILDKITE") == "true" { + t.Skip("skipping test on Buildkite CI") + } +} diff --git a/server/thw/helper_test.go b/server/thw/helper_test.go new file mode 100644 index 0000000..0c5bcb1 --- /dev/null +++ b/server/thw/helper_test.go @@ -0,0 +1,57 @@ +// Copyright 2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package thw + +import ( + "strings" + "testing" +) + +func require_NoError(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatalf("require no error, but got: %v", err) + } +} + +func require_Error(t *testing.T, err error, expected ...error) { + t.Helper() + if err == nil { + t.Fatalf("require error, but got none") + } + if len(expected) == 0 { + return + } + eStr := err.Error() + for _, e := range expected { + if err == e || strings.Contains(eStr, e.Error()) || strings.Contains(e.Error(), eStr) { + return + } + } + t.Fatalf("Expected one of %v, got '%v'", expected, err) +} + +func require_True(t *testing.T, b bool) { + t.Helper() + if !b { + t.Fatalf("require true, but got false") + } +} + +func require_Equal[T comparable](t *testing.T, a, b T) { + t.Helper() + if a != b { + t.Fatalf("require %T equal, but got: %v != %v", a, a, b) + } +} diff --git a/server/thw/thw.go b/server/thw/thw.go new file mode 100644 index 0000000..1ed6014 --- /dev/null +++ b/server/thw/thw.go @@ -0,0 +1,244 @@ +// Copyright 2024-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package thw + +import ( + "encoding/binary" + "errors" + "io" + "math" + "time" +) + +// Error for when we can not locate a task for removal or updates. +var ErrTaskNotFound = errors.New("thw: task not found") + +// Error for when we try to decode a binary-encoded THW with an unknown version number. +var ErrInvalidVersion = errors.New("thw: encoded version not known") + +const ( + tickDuration = int64(time.Second) // Tick duration in nanoseconds. + wheelBits = 12 // 2^12 = 4096 slots. + wheelSize = 1 << wheelBits // Number of slots in the wheel. + wheelMask = wheelSize - 1 // Mask for calculating position. + headerLen = 17 // 1 byte magic + 2x uint64s +) + +// slot represents a single slot in the wheel. +type slot struct { + entries map[uint64]int64 // Map of sequence to expires. + lowest int64 // Lowest expiration time in this slot. +} + +// HashWheel represents the timing wheel. +type HashWheel struct { + wheel []*slot // Array of slots. + lowest int64 // Track the lowest expiration time across all slots. + count uint64 // How many entries are present? +} + +// HashWheelEntry represents a single entry in the wheel. +type HashWheelEntry struct { + Seq uint64 + Expires int64 +} + +// NewHashWheel initializes a new HashWheel. +func NewHashWheel() *HashWheel { + return &HashWheel{ + wheel: make([]*slot, wheelSize), + lowest: math.MaxInt64, + } +} + +// getPosition calculates the slot position for a given expiration time. +func (hw *HashWheel) getPosition(expires int64) int64 { + return (expires / tickDuration) & wheelMask +} + +// newSlot creates a new slot. +func newSlot() *slot { + return &slot{ + entries: make(map[uint64]int64), + lowest: math.MaxInt64, + } +} + +// Add schedules a new timer task. +func (hw *HashWheel) Add(seq uint64, expires int64) error { + pos := hw.getPosition(expires) + // Initialize the slot lazily. + if hw.wheel[pos] == nil { + hw.wheel[pos] = newSlot() + } + if _, ok := hw.wheel[pos].entries[seq]; !ok { + hw.count++ + } + hw.wheel[pos].entries[seq] = expires + + // Update slot's lowest expiration if this is earlier. + if expires < hw.wheel[pos].lowest { + hw.wheel[pos].lowest = expires + // Update global lowest if this is now the earliest. + if expires < hw.lowest { + hw.lowest = expires + } + } + + return nil +} + +// Remove removes a timer task. +func (hw *HashWheel) Remove(seq uint64, expires int64) error { + pos := hw.getPosition(expires) + s := hw.wheel[pos] + if s == nil { + return ErrTaskNotFound + } + if _, exists := s.entries[seq]; !exists { + return ErrTaskNotFound + } + delete(s.entries, seq) + hw.count-- + + // If the slot is empty, we can set it to nil to free memory. + if len(s.entries) == 0 { + hw.wheel[pos] = nil + } + return nil +} + +// Update updates the expiration time of an existing timer task. +func (hw *HashWheel) Update(seq uint64, oldExpires int64, newExpires int64) error { + // Remove from old position. + if err := hw.Remove(seq, oldExpires); err != nil { + return err + } + // Add to new position. + return hw.Add(seq, newExpires) +} + +// ExpireTasks processes all expired tasks using a callback, but only expires a task if the callback returns true. +func (hw *HashWheel) ExpireTasks(callback func(seq uint64, expires int64) bool) { + now := time.Now().UnixNano() + hw.expireTasks(now, callback) +} + +func (hw *HashWheel) expireTasks(ts int64, callback func(seq uint64, expires int64) bool) { + // Quick return if nothing is expired. + if hw.lowest > ts { + return + } + + globalLowest := int64(math.MaxInt64) + for pos, s := range hw.wheel { + // Skip s if nothing to expire. + if s == nil || s.lowest > ts { + if s != nil && s.lowest < globalLowest { + globalLowest = s.lowest + } + continue + } + + // Track new lowest while processing expirations + slotLowest := int64(math.MaxInt64) + for seq, expires := range s.entries { + if expires <= ts && callback(seq, expires) { + // Only remove if not done so already by the callback. + if _, ok := s.entries[seq]; ok { + delete(s.entries, seq) + hw.count-- + } + continue + } + if expires < slotLowest { + slotLowest = expires + } + } + + // Nil out if we are empty. + if len(s.entries) == 0 { + hw.wheel[pos] = nil + } else { + s.lowest = slotLowest + if slotLowest < globalLowest { + globalLowest = slotLowest + } + } + } + hw.lowest = globalLowest +} + +// GetNextExpiration returns the earliest expiration time before the given time. +// Returns math.MaxInt64 if no expirations exist before the specified time. +func (hw *HashWheel) GetNextExpiration(before int64) int64 { + if hw.lowest < before { + return hw.lowest + } + return math.MaxInt64 +} + +// Count returns the amount of tasks in the THW. +func (hw *HashWheel) Count() uint64 { + return hw.count +} + +// Encode writes out the contents of the THW into a binary snapshot +// and returns it. The high seq number is included in the snapshot and will +// be returned on decode. +func (hw *HashWheel) Encode(highSeq uint64) []byte { + b := make([]byte, 0, headerLen+(hw.count*(2*binary.MaxVarintLen64))) + b = append(b, 1) // Magic version + b = binary.LittleEndian.AppendUint64(b, hw.count) // Entry count + b = binary.LittleEndian.AppendUint64(b, highSeq) // Stamp + for _, slot := range hw.wheel { + if slot == nil || slot.entries == nil { + continue + } + for v, ts := range slot.entries { + b = binary.AppendVarint(b, ts) + b = binary.AppendUvarint(b, v) + } + } + return b +} + +// Decode snapshots a binary-encoded THW and replaces the contents of this +// THW with them. Returns the high seq number from the snapshot. +func (hw *HashWheel) Decode(b []byte) (uint64, error) { + if len(b) < headerLen { + return 0, io.ErrShortBuffer + } + if b[0] != 1 { + return 0, ErrInvalidVersion + } + hw.wheel = make([]*slot, wheelSize) + hw.lowest = math.MaxInt64 + count := binary.LittleEndian.Uint64(b[1:]) + stamp := binary.LittleEndian.Uint64(b[9:]) + b = b[headerLen:] + for i := uint64(0); i < count; i++ { + ts, tn := binary.Varint(b) + if tn <= 0 { + return 0, io.ErrUnexpectedEOF + } + v, vn := binary.Uvarint(b[tn:]) + if vn <= 0 { + return 0, io.ErrUnexpectedEOF + } + hw.Add(v, ts) + b = b[tn+vn:] + } + return stamp, nil +} diff --git a/server/thw/thw_test.go b/server/thw/thw_test.go new file mode 100644 index 0000000..9e89417 --- /dev/null +++ b/server/thw/thw_test.go @@ -0,0 +1,387 @@ +// Copyright 2024-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package thw + +import ( + "encoding/binary" + "io" + "math" + "testing" + "time" +) + +func TestHashWheelBasics(t *testing.T) { + hw := NewHashWheel() + now := time.Now() + + // Add a sequence. + seq, expires := uint64(1), now.Add(5*time.Second).UnixNano() + require_NoError(t, hw.Add(seq, expires)) + require_Equal(t, hw.count, 1) + + // Try to remove non-existent sequence. + require_Error(t, hw.Remove(999, expires), ErrTaskNotFound) + require_Equal(t, hw.count, 1) + + // Remove the sequence properly. + require_NoError(t, hw.Remove(seq, expires)) + require_Equal(t, hw.count, 0) + + // Verify it's gone. + require_Error(t, hw.Remove(seq, expires), ErrTaskNotFound) + require_Equal(t, hw.count, 0) +} + +func TestHashWheelUpdate(t *testing.T) { + hw := NewHashWheel() + now := time.Now() + oldExpires := now.Add(5 * time.Second).UnixNano() + newExpires := now.Add(10 * time.Second).UnixNano() + + // Add initial sequence. + require_NoError(t, hw.Add(1, oldExpires)) + require_Equal(t, hw.count, 1) + + // Update expiration. + require_NoError(t, hw.Update(1, oldExpires, newExpires)) + require_Equal(t, hw.count, 1) + + // Verify old expiration is gone. + require_Error(t, hw.Remove(1, oldExpires), ErrTaskNotFound) + require_Equal(t, hw.count, 1) + + // Verify new expiration exists + require_NoError(t, hw.Remove(1, newExpires)) + require_Equal(t, hw.count, 0) +} + +func TestHashWheelExpiration(t *testing.T) { + hw := NewHashWheel() + now := time.Now() + + // Add sequences with different expiration times. + seqs := map[uint64]int64{ + 1: now.Add(-1 * time.Second).UnixNano(), // Already expired + 2: now.Add(1 * time.Second).UnixNano(), // Expires soon + 3: now.Add(10 * time.Second).UnixNano(), // Expires later + 4: now.Add(60 * time.Second).UnixNano(), // Expires much later + } + + for seq, expires := range seqs { + require_NoError(t, hw.Add(seq, expires)) + } + require_Equal(t, hw.count, uint64(len(seqs))) + + // Process expired tasks. + expired := make(map[uint64]bool) + hw.ExpireTasks(func(seq uint64, expires int64) bool { + expired[seq] = true + return true + }) + + // Verify only sequence 1 expired. + require_Equal(t, len(expired), 1) + require_True(t, expired[1]) + require_Equal(t, hw.count, 3) +} + +func TestHashWheelManualExpiration(t *testing.T) { + hw := NewHashWheel() + now := time.Now().UnixNano() + + for seq := uint64(1); seq <= 4; seq++ { + require_NoError(t, hw.Add(seq, now)) + } + require_Equal(t, hw.count, 4) + + // Loop over expired multiple times, but without removing them. + expired := make(map[uint64]uint64) + for i := uint64(0); i <= 1; i++ { + hw.ExpireTasks(func(seq uint64, expires int64) bool { + expired[seq]++ + return false + }) + + require_Equal(t, len(expired), 4) + require_Equal(t, expired[1], 1+i) + require_Equal(t, expired[2], 1+i) + require_Equal(t, expired[3], 1+i) + require_Equal(t, expired[4], 1+i) + require_Equal(t, hw.count, 4) + } + + // Only remove even sequences. + for i := uint64(0); i <= 1; i++ { + hw.ExpireTasks(func(seq uint64, expires int64) bool { + expired[seq]++ + return seq%2 == 0 + }) + + // Verify even sequences are removed. + require_Equal(t, expired[1], 3+i) + require_Equal(t, expired[2], 3) + require_Equal(t, expired[3], 3+i) + require_Equal(t, expired[4], 3) + require_Equal(t, hw.count, 2) + } + + // Manually remove last items. + require_NoError(t, hw.Remove(1, now)) + require_NoError(t, hw.Remove(3, now)) + require_Equal(t, hw.count, 0) +} + +func TestHashWheelExpirationLargerThanWheel(t *testing.T) { + hw := NewHashWheel() + + // Add sequences such that they can be expired immediately. + seqs := map[uint64]int64{ + 1: 0, + 2: int64(time.Second), + } + for seq, expires := range seqs { + require_NoError(t, hw.Add(seq, expires)) + } + require_Equal(t, hw.count, 2) + + // Pick a timestamp such that the expiration needs to wrap around the whole wheel. + now := int64(time.Second) * wheelMask + + // Process expired tasks. + expired := make(map[uint64]bool) + hw.expireTasks(now, func(seq uint64, expires int64) bool { + expired[seq] = true + return true + }) + + // Verify both sequences are expired. + require_Equal(t, len(expired), 2) + require_Equal(t, hw.count, 0) +} + +func TestHashWheelNextExpiration(t *testing.T) { + hw := NewHashWheel() + now := time.Now() + + // Add sequences with different expiration times. + seqs := map[uint64]int64{ + 1: now.Add(5 * time.Second).UnixNano(), + 2: now.Add(3 * time.Second).UnixNano(), // Earliest + 3: now.Add(10 * time.Second).UnixNano(), + } + + for seq, expires := range seqs { + require_NoError(t, hw.Add(seq, expires)) + } + require_Equal(t, hw.count, uint64(len(seqs))) + + // Test GetNextExpiration. + nextExternalTick := now.Add(6 * time.Second).UnixNano() + // Should return sequence 2's expiration + require_Equal(t, hw.GetNextExpiration(nextExternalTick), seqs[2]) + + // Test with empty wheel. + empty := NewHashWheel() + require_Equal(t, empty.GetNextExpiration(now.Add(1*time.Second).UnixNano()), math.MaxInt64) +} + +func TestHashWheelStress(t *testing.T) { + hw := NewHashWheel() + now := time.Now() + + // Add many sequences. + numSequences := 100_000 + for seq := 0; seq < numSequences; seq++ { + expires := now.Add(time.Duration(seq) * time.Second).UnixNano() + require_NoError(t, hw.Add(uint64(seq), expires)) + } + + // Update many sequences. + for seq := 0; seq < numSequences; seq += 2 { // Update every other sequence + oldExpires := now.Add(time.Duration(seq) * time.Second).UnixNano() + newExpires := now.Add(time.Duration(seq+numSequences) * time.Second).UnixNano() + require_NoError(t, hw.Update(uint64(seq), oldExpires, newExpires)) + } + + // Remove many sequences. + for seq := 1; seq < numSequences; seq += 2 { // Remove odd-numbered sequences + expires := now.Add(time.Duration(seq) * time.Second).UnixNano() + require_NoError(t, hw.Remove(uint64(seq), expires)) + } +} + +func TestHashWheelEncodeDecode(t *testing.T) { + hw := NewHashWheel() + now := time.Now() + + // Add many sequences. + numSequences := 100_000 + for seq := 0; seq < numSequences; seq++ { + expires := now.Add(time.Duration(seq) * time.Second).UnixNano() + require_NoError(t, hw.Add(uint64(seq), expires)) + } + + b := hw.Encode(12345) + require_True(t, len(b) > 17) // Bigger than just the header + + nhw := NewHashWheel() + stamp, err := nhw.Decode(b) + require_NoError(t, err) + require_Equal(t, stamp, 12345) + require_Equal(t, hw.GetNextExpiration(math.MaxInt64), nhw.GetNextExpiration(math.MaxInt64)) + + for s, slot := range hw.wheel { + nslot := nhw.wheel[s] + require_Equal(t, slot.lowest, nslot.lowest) + require_Equal(t, len(slot.entries), len(nslot.entries)) + for v, ts := range slot.entries { + nts, ok := nslot.entries[v] + require_True(t, ok) + require_Equal(t, ts, nts) + } + } +} + +func TestHashWheelDecodeRejectsMalformed(t *testing.T) { + // Build a valid header that claims the given number of entries. + header := func(count uint64) []byte { + b := make([]byte, headerLen) + b[0] = 1 // Magic version + binary.LittleEndian.PutUint64(b[1:], count) // Entry count + binary.LittleEndian.PutUint64(b[9:], 0) // High sequence stamp + return b + } + + for _, test := range []struct { + title string + buf []byte + err error + }{ + {title: "ShortHeader", buf: make([]byte, headerLen-1), err: io.ErrShortBuffer}, + {title: "BadVersion", buf: func() []byte { b := header(0); b[0] = 2; return b }(), err: ErrInvalidVersion}, + // Claims one entry but the buffer ends right after the header. + {title: "TruncatedAtVarints", buf: header(1), err: io.ErrUnexpectedEOF}, + // Has the timestamp varint but the seq varint is missing. + {title: "TruncatedSeq", buf: append(header(1), binary.AppendVarint(nil, 12345)...), err: io.ErrUnexpectedEOF}, + } { + t.Run(test.title, func(t *testing.T) { + hw := NewHashWheel() + _, err := hw.Decode(test.buf) + require_Error(t, err, test.err) + }) + } +} + +// Benchmarks + +func BenchmarkHashWheel_Add(b *testing.B) { + hw := NewHashWheel() + now := time.Now() + + // Create different ranges for expires to spread across slots + expires := make([]int64, b.N) + for i := 0; i < b.N; i++ { + expires[i] = now.Add(time.Duration(i%3600) * time.Second).UnixNano() + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + hw.Add(uint64(i), expires[i]) + } +} + +func BenchmarkHashWheel_Add_SameSlot(b *testing.B) { + hw := NewHashWheel() + expires := time.Now().Add(time.Second).UnixNano() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + hw.Add(uint64(i), expires) + } +} + +func BenchmarkHashWheel_Update(b *testing.B) { + hw := NewHashWheel() + now := time.Now() + + // First add N items + expires := make([]int64, b.N) + for i := 0; i < b.N; i++ { + expires[i] = now.Add(time.Duration(i%3600) * time.Second).UnixNano() + hw.Add(uint64(i), expires[i]) + } + + newExpires := now.Add(2 * time.Hour).UnixNano() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + hw.Update(uint64(i), expires[i], newExpires) + } +} + +func BenchmarkHashWheel_Update_SameSlot(b *testing.B) { + hw := NewHashWheel() + now := time.Now() + oldExpires := now.Add(time.Second).UnixNano() + newExpires := now.Add(2 * time.Second).UnixNano() + + // First add N items + for i := 0; i < b.N; i++ { + hw.Add(uint64(i), oldExpires) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + hw.Update(uint64(i), oldExpires, newExpires) + } +} + +// Benchmark memory allocation +func BenchmarkHashWheel_Add_Memory(b *testing.B) { + b.ReportAllocs() + hw := NewHashWheel() + now := time.Now() + + expires := make([]int64, b.N) + for i := 0; i < b.N; i++ { + expires[i] = now.Add(time.Duration(i%3600) * time.Second).UnixNano() + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + hw.Add(uint64(i), expires[i]) + } +} + +// Large scale test +func BenchmarkHashWheel_LargeScale(b *testing.B) { + hw := NewHashWheel() + now := time.Now() + + // Pre-populate with 100k items + for i := 0; i < 100_000; i++ { + expires := now.Add(time.Duration(i%3600) * time.Second).UnixNano() + hw.Add(uint64(i), expires) + } + + expires := make([]int64, b.N) + for i := 0; i < b.N; i++ { + expires[i] = now.Add(time.Duration(i%3600) * time.Second).UnixNano() + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + hw.Add(uint64(i+100000), expires[i]) + } +} diff --git a/server/tpm/js_ek_tpm_other.go b/server/tpm/js_ek_tpm_other.go new file mode 100644 index 0000000..a1ed593 --- /dev/null +++ b/server/tpm/js_ek_tpm_other.go @@ -0,0 +1,23 @@ +// Copyright 2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !windows + +package tpm + +import "fmt" + +// LoadJetStreamEncryptionKeyFromTPM here is a stub for unsupported platforms. +func LoadJetStreamEncryptionKeyFromTPM(srkPassword, jsKeyFile, jsKeyPassword string, pcr int) (string, error) { + return "", fmt.Errorf("TPM functionality is not supported on this platform") +} diff --git a/server/tpm/js_ek_tpm_test.go b/server/tpm/js_ek_tpm_test.go new file mode 100644 index 0000000..3fa3deb --- /dev/null +++ b/server/tpm/js_ek_tpm_test.go @@ -0,0 +1,80 @@ +// Copyright 2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package tpm + +import ( + "os" + "testing" +) + +func getTempFile(t *testing.T) string { + return t.TempDir() + "/jskeys.json" +} + +func TestLoadJetStreamEncryptionKeyFromTPM(t *testing.T) { + testFile := getTempFile(t) + type args struct { + srkPassword string + jsKeyFile string + jsKeyPassword string + pcr int + } + tests := []struct { + name string + args args + clear bool + wantErr bool + }{ + {"TestLoadJetStreamEncryptionKeyFromTPM-Load", args{"", testFile, "password", 22}, true, false}, + {"TestLoadJetStreamEncryptionKeyFromTPM-Read", args{"", testFile, "password", 22}, false, false}, + {"TestLoadJetStreamEncryptionKeyFromTPM-BadPass", args{"", testFile, "badpass", 22}, false, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.clear { + os.Remove(tt.args.jsKeyFile) + } + _, err := LoadJetStreamEncryptionKeyFromTPM(tt.args.srkPassword, tt.args.jsKeyFile, tt.args.jsKeyPassword, tt.args.pcr) + if (err != nil) != tt.wantErr { + t.Errorf("LoadJetStreamEncryptionKeyFromTPM() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +// TestLoadJetStreamEncryptionKeyFromTPMBasic tests the basic functionality. +// The first pass will create the keys and generate the js encryption key. +// the second pass will read the keys from disk, decrypt with the TPM (unseal), +// and return the same key. +func TestLoadJetStreamEncryptionKeyFromTPMBasic(t *testing.T) { + testFile := getTempFile(t) + + // Create the key file. + key1, err := LoadJetStreamEncryptionKeyFromTPM("", testFile, "password", 22) + if err != nil { + t.Errorf("LoadJetStreamEncryptionKeyFromTPM() failed: %v", err) + } + + // Now obtain the newly generated key from the file. + key2, err := LoadJetStreamEncryptionKeyFromTPM("", testFile, "password", 22) + if err != nil { + t.Errorf("LoadJetStreamEncryptionKeyFromTPM() failed: %v", err) + } + if key1 != key2 { + t.Errorf("Keys should match") + } +} diff --git a/server/tpm/js_ek_tpm_windows.go b/server/tpm/js_ek_tpm_windows.go new file mode 100644 index 0000000..5e40168 --- /dev/null +++ b/server/tpm/js_ek_tpm_windows.go @@ -0,0 +1,281 @@ +// Copyright 2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package tpm + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/google/go-tpm/legacy/tpm2" + "github.com/google/go-tpm/tpmutil" + "github.com/nats-io/nkeys" +) + +var ( + // Version of the NATS TPM JS implmentation + JsKeyTPMVersion = 1 +) + +// How this works: +// Create a Storage Root Key (SRK) in the TPM. +// If existing JS Encryption keys do not exist on disk. +// - Create a JetStream encryption key (js key) and seal it to the SRK +// using a provided js encryption key password. +// - Save the public and private blobs to a file on disk. +// - Return the new js encryption key (the private portion of the nkey) +// Otherwise (keys exist on disk) +// - Read the public and private blobs from disk +// - Load them into the TPM +// - Unseal the js key using the TPM, and the provided js encryption keys password. +// +// Note: a SRK password for the SRK is supported but not tested here. + +// Gets/Regenerates the Storage Root Key (SRK) from the TPM. Caller MUST flush this handle when done. +func regenerateSRK(rwc io.ReadWriteCloser, srkPassword string) (tpmutil.Handle, error) { + // Default EK template defined in: + // https://trustedcomputinggroup.org/wp-content/uploads/Credential_Profile_EK_V2.0_R14_published.pdf + // Shared SRK template based off of EK template and specified in: + // https://trustedcomputinggroup.org/wp-content/uploads/TCG-TPM-v2.0-Provisioning-Guidance-Published-v1r1.pdf + srkTemplate := tpm2.Public{ + Type: tpm2.AlgRSA, + NameAlg: tpm2.AlgSHA256, + Attributes: tpm2.FlagFixedTPM | tpm2.FlagFixedParent | tpm2.FlagSensitiveDataOrigin | tpm2.FlagUserWithAuth | tpm2.FlagRestricted | tpm2.FlagDecrypt | tpm2.FlagNoDA, + AuthPolicy: nil, + // We must use RSA 2048 for the intel TSS2 stack + RSAParameters: &tpm2.RSAParams{ + Symmetric: &tpm2.SymScheme{ + Alg: tpm2.AlgAES, + KeyBits: 128, + Mode: tpm2.AlgCFB, + }, + KeyBits: 2048, + ModulusRaw: make([]byte, 256), + }, + } + // Create the parent key against which to seal the data + srkHandle, _, err := tpm2.CreatePrimary(rwc, tpm2.HandleOwner, tpm2.PCRSelection{}, "", srkPassword, srkTemplate) + return srkHandle, err +} + +type natsTPMPersistedKeys struct { + Version int `json:"version"` + PrivateKey []byte `json:"private_key"` + PublicKey []byte `json:"public_key"` +} + +// Writes the private and public blobs to disk in a single file. If the directory does +// not exist, it will be created. If the file already exists it will be overwritten. +func writeTPMKeysToFile(filename string, privateBlob []byte, publicBlob []byte) error { + keyDir := filepath.Dir(filename) + if err := os.MkdirAll(keyDir, 0750); err != nil { + return fmt.Errorf("unable to create/access directory %q: %v", keyDir, err) + } + + // Create a new set of persisted keys. Note that the private key doesn't necessarily + // need to be protected as the TPM password is required to use unseal, although it's + // a good idea to put this in a secure location accessible to the server. + tpmKeys := natsTPMPersistedKeys{ + Version: JsKeyTPMVersion, + PrivateKey: make([]byte, base64.StdEncoding.EncodedLen(len(privateBlob))), + PublicKey: make([]byte, base64.StdEncoding.EncodedLen(len(publicBlob))), + } + base64.StdEncoding.Encode(tpmKeys.PrivateKey, privateBlob) + base64.StdEncoding.Encode(tpmKeys.PublicKey, publicBlob) + // Convert to JSON + keysJSON, err := json.Marshal(tpmKeys) + if err != nil { + return fmt.Errorf("unable to marshal keys to JSON: %v", err) + } + // Write the JSON to a file + if err := os.WriteFile(filename, keysJSON, 0640); err != nil { + return fmt.Errorf("unable to write keys file to %q: %v", filename, err) + } + return nil +} + +// Reads the private and public blobs from a single file. If the file does not exist, +// or the file cannot be read and the keys decoded, an error is returned. +func readTPMKeysFromFile(filename string) ([]byte, []byte, error) { + keysJSON, err := os.ReadFile(filename) + if err != nil { + return nil, nil, err + } + + var tpmKeys natsTPMPersistedKeys + if err := json.Unmarshal(keysJSON, &tpmKeys); err != nil { + return nil, nil, fmt.Errorf("unable to unmarshal TPM file keys JSON from %s: %v", filename, err) + } + + // Placeholder for future-proofing. Here is where we would + // check the current version against tpmKeys.Version and + // handle any changes. + + // Base64 decode the private and public blobs. + privateBlob := make([]byte, base64.StdEncoding.DecodedLen(len(tpmKeys.PrivateKey))) + publicBlob := make([]byte, base64.StdEncoding.DecodedLen(len(tpmKeys.PublicKey))) + prn, err := base64.StdEncoding.Decode(privateBlob, tpmKeys.PrivateKey) + if err != nil { + return nil, nil, fmt.Errorf("unable to decode privateBlob from base64: %v", err) + } + pun, err := base64.StdEncoding.Decode(publicBlob, tpmKeys.PublicKey) + if err != nil { + return nil, nil, fmt.Errorf("unable to decode publicBlob from base64: %v", err) + } + return publicBlob[:pun], privateBlob[:prn], nil +} + +// Creates a new JetStream encryption key, seals it to the TPM, and saves the public and +// private blobs to disk in a JSON encoded file. The key is returned as a string. +func createAndSealJsEncryptionKey(rwc io.ReadWriteCloser, srkHandle tpmutil.Handle, srkPassword, jsKeyFile, jsKeyPassword string, pcr int) (string, error) { + // Get the authorization policy that will protect the data to be sealed + sessHandle, policy, err := policyPCRPasswordSession(rwc, pcr) + if err != nil { + return "", fmt.Errorf("unable to get policy: %v", err) + } + if err := tpm2.FlushContext(rwc, sessHandle); err != nil { + return "", fmt.Errorf("unable to flush session: %v", err) + } + // Seal the data to the parent key and the policy + user, err := nkeys.CreateUser() + if err != nil { + return "", fmt.Errorf("unable to create seed: %v", err) + } + // We'll use the seed to represent the encryption key. + jsStoreKey, err := user.Seed() + if err != nil { + return "", fmt.Errorf("unable to get seed: %v", err) + } + privateArea, publicArea, err := tpm2.Seal(rwc, srkHandle, srkPassword, jsKeyPassword, policy, jsStoreKey) + if err != nil { + return "", fmt.Errorf("unable to seal data: %v", err) + } + err = writeTPMKeysToFile(jsKeyFile, privateArea, publicArea) + if err != nil { + return "", fmt.Errorf("unable to write key file: %v", err) + } + return string(jsStoreKey), nil +} + +// Unseals the JetStream encryption key from the TPM with the provided keys. +// The key is returned as a string. +func unsealJsEncrpytionKey(rwc io.ReadWriteCloser, pcr int, srkHandle tpmutil.Handle, srkPassword, objectPassword string, publicBlob, privateBlob []byte) (string, error) { + // Load the public/private blobs into the TPM for decryption. + objectHandle, _, err := tpm2.Load(rwc, srkHandle, srkPassword, publicBlob, privateBlob) + if err != nil { + return "", fmt.Errorf("unable to load data: %v", err) + } + defer tpm2.FlushContext(rwc, objectHandle) + + // Create the authorization session with TPM. + sessHandle, _, err := policyPCRPasswordSession(rwc, pcr) + if err != nil { + return "", fmt.Errorf("unable to get auth session: %v", err) + } + defer func() { + tpm2.FlushContext(rwc, sessHandle) + }() + // Unseal the data we've loaded into the TPM with the object (js key) password. + unsealedData, err := tpm2.UnsealWithSession(rwc, sessHandle, objectHandle, objectPassword) + if err != nil { + return "", fmt.Errorf("unable to unseal data: %v", err) + } + return string(unsealedData), nil +} + +// Returns session handle and policy digest. +func policyPCRPasswordSession(rwc io.ReadWriteCloser, pcr int) (sessHandle tpmutil.Handle, policy []byte, retErr error) { + sessHandle, _, err := tpm2.StartAuthSession( + rwc, + tpm2.HandleNull, /*tpmKey*/ + tpm2.HandleNull, /*bindKey*/ + make([]byte, 16), /*nonceCaller*/ + nil, /*secret*/ + tpm2.SessionPolicy, + tpm2.AlgNull, + tpm2.AlgSHA256) + if err != nil { + return tpm2.HandleNull, nil, fmt.Errorf("unable to start session: %v", err) + } + defer func() { + if sessHandle != tpm2.HandleNull && err != nil { + if err := tpm2.FlushContext(rwc, sessHandle); err != nil { + retErr = fmt.Errorf("%v\nunable to flush session: %v", retErr, err) + } + } + }() + + pcrSelection := tpm2.PCRSelection{ + Hash: tpm2.AlgSHA256, + PCRs: []int{pcr}, + } + if err := tpm2.PolicyPCR(rwc, sessHandle, nil, pcrSelection); err != nil { + return sessHandle, nil, fmt.Errorf("unable to bind PCRs to auth policy: %v", err) + } + if err := tpm2.PolicyPassword(rwc, sessHandle); err != nil { + return sessHandle, nil, fmt.Errorf("unable to require password for auth policy: %v", err) + } + policy, err = tpm2.PolicyGetDigest(rwc, sessHandle) + if err != nil { + return sessHandle, nil, fmt.Errorf("unable to get policy digest: %v", err) + } + return sessHandle, policy, nil +} + +// LoadJetStreamEncryptionKeyFromTPM loads the JetStream encryption key from the TPM. +// If the keyfile does not exist, a key will be created and sealed. Public and private blobs +// used to decrypt the key in future sessions will be saved to disk in the file provided. +// The key will be unsealed and returned only with the correct password and PCR value. +func LoadJetStreamEncryptionKeyFromTPM(srkPassword, jsKeyFile, jsKeyPassword string, pcr int) (string, error) { + rwc, err := tpm2.OpenTPM() + if err != nil { + return "", fmt.Errorf("could not open the TPM: %v", err) + } + defer rwc.Close() + + // Load the key from the TPM + srkHandle, err := regenerateSRK(rwc, srkPassword) + defer func() { + tpm2.FlushContext(rwc, srkHandle) + }() + if err != nil { + return "", fmt.Errorf("unable to regenerate SRK from the TPM: %v", err) + } + // Read the keys from the key file. If the filed doesn't exist it means we need to create + // a new js encrytpion key. + publicBlob, privateBlob, err := readTPMKeysFromFile(jsKeyFile) + if err != nil { + if os.IsNotExist(err) { + jsek, err := createAndSealJsEncryptionKey(rwc, srkHandle, srkPassword, jsKeyFile, jsKeyPassword, pcr) + if err != nil { + return "", fmt.Errorf("unable to generate new key from the TPM: %v", err) + } + // we've created and sealed the JS Encryption key, now we just return it. + return jsek, nil + } + return "", fmt.Errorf("unable to load key from TPM: %v", err) + } + + // Unseal the JetStream encryption key using the TPM. + jsek, err := unsealJsEncrpytionKey(rwc, pcr, srkHandle, srkPassword, jsKeyPassword, publicBlob, privateBlob) + if err != nil { + return "", fmt.Errorf("unable to unseal key from the TPM: %v", err) + } + return jsek, nil +} diff --git a/server/trust_test.go b/server/trust_test.go new file mode 100644 index 0000000..951a9ca --- /dev/null +++ b/server/trust_test.go @@ -0,0 +1,119 @@ +// Copyright 2018-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "fmt" + "strings" + "testing" +) + +const ( + t1 = "OBYEOZQ46VZMFMNETBAW2H6VGDSOBLP67VUEZJ5LPR3PIZBWWRIY4UI4" + t2 = "OAHC7NGAHG3YVPTD6QOUFZGPM2OMU6EOS67O2VHBUOA6BJLPTWFHGLKU" +) + +func TestStampedTrustedKeys(t *testing.T) { + opts := DefaultOptions() + defer func() { trustedKeys = "" }() + + // Set this to a bad key. We require valid operator public keys. + trustedKeys = "bad" + if s := New(opts); s != nil { + s.Shutdown() + t.Fatalf("Expected a bad trustedKeys to return nil server") + } + + trustedKeys = t1 + s := New(opts) + if s == nil { + t.Fatalf("Expected non-nil server") + } + if len(s.trustedKeys) != 1 || s.trustedKeys[0] != t1 { + t.Fatalf("Trusted Nkeys not setup properly") + } + trustedKeys = strings.Join([]string{t1, t2}, " ") + if s = New(opts); s == nil { + t.Fatalf("Expected non-nil server") + } + if len(s.trustedKeys) != 2 || s.trustedKeys[0] != t1 || s.trustedKeys[1] != t2 { + t.Fatalf("Trusted Nkeys not setup properly") + } + + opts.TrustedKeys = []string{"OVERRIDE ME"} + if s = New(opts); s != nil { + t.Fatalf("Expected opts.TrustedKeys to return nil server") + } +} + +func TestTrustedKeysOptions(t *testing.T) { + trustedKeys = "" + opts := DefaultOptions() + opts.TrustedKeys = []string{"bad"} + if s := New(opts); s != nil { + s.Shutdown() + t.Fatalf("Expected a bad opts.TrustedKeys to return nil server") + } + opts.TrustedKeys = []string{t1} + s := New(opts) + if s == nil { + t.Fatalf("Expected non-nil server") + } + if len(s.trustedKeys) != 1 || s.trustedKeys[0] != t1 { + t.Fatalf("Trusted Nkeys not setup properly via options") + } + opts.TrustedKeys = []string{t1, t2} + if s = New(opts); s == nil { + t.Fatalf("Expected non-nil server") + } + if len(s.trustedKeys) != 2 || s.trustedKeys[0] != t1 || s.trustedKeys[1] != t2 { + t.Fatalf("Trusted Nkeys not setup properly via options") + } +} + +func TestTrustConfigOption(t *testing.T) { + confFileName := createConfFile(t, []byte(fmt.Sprintf("trusted = %q", t1))) + opts, err := ProcessConfigFile(confFileName) + if err != nil { + t.Fatalf("Error parsing config: %v", err) + } + if l := len(opts.TrustedKeys); l != 1 { + t.Fatalf("Expected 1 trusted key, got %d", l) + } + if opts.TrustedKeys[0] != t1 { + t.Fatalf("Expected trusted key to be %q, got %q", t1, opts.TrustedKeys[0]) + } + + confFileName = createConfFile(t, []byte(fmt.Sprintf("trusted = [%q, %q]", t1, t2))) + opts, err = ProcessConfigFile(confFileName) + if err != nil { + t.Fatalf("Error parsing config: %v", err) + } + if l := len(opts.TrustedKeys); l != 2 { + t.Fatalf("Expected 2 trusted key, got %d", l) + } + if opts.TrustedKeys[0] != t1 { + t.Fatalf("Expected trusted key to be %q, got %q", t1, opts.TrustedKeys[0]) + } + if opts.TrustedKeys[1] != t2 { + t.Fatalf("Expected trusted key to be %q, got %q", t2, opts.TrustedKeys[1]) + } + + // Now do a bad one. + confFileName = createConfFile(t, []byte(fmt.Sprintf("trusted = [%q, %q]", t1, "bad"))) + _, err = ProcessConfigFile(confFileName) + if err == nil { + t.Fatalf("Expected an error parsing trust keys with a bad key") + } +} diff --git a/server/util.go b/server/util.go new file mode 100644 index 0000000..89330dc --- /dev/null +++ b/server/util.go @@ -0,0 +1,408 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "math" + "math/bits" + "net" + "net/url" + "reflect" + "runtime" + "strconv" + "strings" + "time" +) + +// This map is used to store URLs string as the key with a reference count as +// the value. This is used to handle gossiped URLs such as connect_urls, etc.. +type refCountedUrlSet map[string]int + +// Ascii numbers 0-9 +const ( + asciiZero = 48 + asciiNine = 57 +) + +func versionComponents(version string) (major, minor, patch int, err error) { + m := semVerRe.FindStringSubmatch(version) + if len(m) == 0 { + return 0, 0, 0, errors.New("invalid semver") + } + major, err = strconv.Atoi(m[1]) + if err != nil { + return -1, -1, -1, err + } + minor, err = strconv.Atoi(m[2]) + if err != nil { + return -1, -1, -1, err + } + patch, err = strconv.Atoi(m[3]) + if err != nil { + return -1, -1, -1, err + } + return major, minor, patch, err +} + +func versionAtLeastCheckError(version string, emajor, eminor, epatch int) (bool, error) { + major, minor, patch, err := versionComponents(version) + if err != nil { + return false, err + } + if major > emajor || + (major == emajor && minor > eminor) || + (major == emajor && minor == eminor && patch >= epatch) { + return true, nil + } + return false, err +} + +func versionAtLeast(version string, emajor, eminor, epatch int) bool { + res, _ := versionAtLeastCheckError(version, emajor, eminor, epatch) + return res +} + +// parseSize expects decimal positive numbers. We +// return -1 to signal error. +func parseSize(d []byte) (n int) { + const maxParseSizeLen = 9 //999M + + l := len(d) + if l == 0 || l > maxParseSizeLen { + return -1 + } + var ( + i int + dec byte + ) + + // Note: Use `goto` here to avoid for loop in order + // to have the function be inlined. + // See: https://github.com/golang/go/issues/14768 +loop: + dec = d[i] + if dec < asciiZero || dec > asciiNine { + return -1 + } + n = n*10 + (int(dec) - asciiZero) + + i++ + if i < l { + goto loop + } + return n +} + +// parseInt64 expects decimal positive numbers. We +// return -1 to signal error +func parseInt64(d []byte) (n int64) { + if len(d) == 0 { + return -1 + } + for _, dec := range d { + if dec < asciiZero || dec > asciiNine { + return -1 + } + n = n*10 + (int64(dec) - asciiZero) + } + return n +} + +// parseUint64 expects decimal positive numbers. Returns the value and true on success, +// or 0 and false on invalid input or overflow. +func parseUint64(d []byte) (uint64, bool) { + if len(d) == 0 { + return 0, false + } + var n uint64 + for _, dec := range d { + if dec < asciiZero || dec > asciiNine { + return 0, false + } + digit := uint64(dec) - asciiZero + if n > math.MaxUint64/10 || (n == math.MaxUint64/10 && digit > math.MaxUint64%10) { + return 0, false + } + n = n*10 + digit + } + return n, true +} + +// Helper to move from float seconds to time.Duration +func secondsToDuration(seconds float64) time.Duration { + ttl := seconds * float64(time.Second) + return time.Duration(ttl) +} + +// Parse a host/port string with a default port to use +// if none (or 0 or -1) is specified in `hostPort` string. +func parseHostPort(hostPort string, defaultPort int) (host string, port int, err error) { + if hostPort != "" { + host, sPort, err := net.SplitHostPort(hostPort) + if ae, ok := err.(*net.AddrError); ok && strings.Contains(ae.Err, "missing port") { + // try appending the current port + host, sPort, err = net.SplitHostPort(fmt.Sprintf("%s:%d", hostPort, defaultPort)) + } + if err != nil { + return "", -1, err + } + port, err = strconv.Atoi(strings.TrimSpace(sPort)) + if err != nil { + return "", -1, err + } + if port == 0 || port == -1 { + port = defaultPort + } + return strings.TrimSpace(host), port, nil + } + return "", -1, errors.New("no hostport specified") +} + +// Returns true if URL u1 represents the same URL than u2, +// false otherwise. +func urlsAreEqual(u1, u2 *url.URL) bool { + return reflect.DeepEqual(u1, u2) +} + +// comma produces a string form of the given number in base 10 with +// commas after every three orders of magnitude. +// +// e.g. comma(834142) -> 834,142 +// +// This function was copied from the github.com/dustin/go-humanize +// package (MIT License) and is Copyright Dustin Sallings +func comma(v int64) string { + sign := "" + + // Min int64 can't be negated to a usable value, so it has to be special cased. + if v == math.MinInt64 { + return "-9,223,372,036,854,775,808" + } + + if v < 0 { + sign = "-" + v = 0 - v + } + + parts := []string{"", "", "", "", "", "", ""} + j := len(parts) - 1 + + for v > 999 { + parts[j] = strconv.FormatInt(v%1000, 10) + switch len(parts[j]) { + case 2: + parts[j] = "0" + parts[j] + case 1: + parts[j] = "00" + parts[j] + } + v = v / 1000 + j-- + } + parts[j] = strconv.Itoa(int(v)) + return sign + strings.Join(parts[j:], ",") +} + +// Adds urlStr to the given map. If the string was already present, simply +// bumps the reference count. +// Returns true only if it was added for the first time. +func (m refCountedUrlSet) addUrl(urlStr string) bool { + m[urlStr]++ + return m[urlStr] == 1 +} + +// Removes urlStr from the given map. If the string is not present, nothing +// is done and false is returned. +// If the string was present, its reference count is decreased. Returns true +// if this was the last reference, false otherwise. +func (m refCountedUrlSet) removeUrl(urlStr string) bool { + removed := false + if ref, ok := m[urlStr]; ok { + if ref == 1 { + removed = true + delete(m, urlStr) + } else { + m[urlStr]-- + } + } + return removed +} + +// Returns the unique URLs in this map as a slice +func (m refCountedUrlSet) getAsStringSlice() []string { + a := make([]string, 0, len(m)) + for u := range m { + a = append(a, u) + } + return a +} + +// natsListenConfig provides a common configuration to match the one used by +// net.Listen() but with our own defaults. +// Go 1.13 introduced default-on TCP keepalives with aggressive timings and +// there's no sane portable way in Go with stdlib to split the initial timer +// from the retry timer. Linux/BSD defaults are 2hrs/75s and Go sets both +// to 15s; the issue re making them indepedently tunable has been open since +// 2014 and this code here is being written in 2020. +// The NATS protocol has its own L7 PING/PONG keepalive system and the Go +// defaults are inappropriate for IoT deployment scenarios. +// Replace any NATS-protocol calls to net.Listen(...) with +// natsListenConfig.Listen(ctx,...) or use natsListen(); leave calls for HTTP +// monitoring, etc, on the default. +var natsListenConfig = &net.ListenConfig{ + KeepAlive: -1, +} + +// natsListen() is the same as net.Listen() except that TCP keepalives are +// disabled (to match Go's behavior before Go 1.13). +func natsListen(network, address string) (net.Listener, error) { + return natsListenConfig.Listen(context.Background(), network, address) +} + +// natsDialTimeout is the same as net.DialTimeout() except the TCP keepalives +// are disabled (to match Go's behavior before Go 1.13). +func natsDialTimeout(network, address string, timeout time.Duration) (net.Conn, error) { + d := net.Dialer{ + Timeout: timeout, + KeepAlive: -1, + } + return d.Dial(network, address) +} + +// redactURLList() returns a copy of a list of URL pointers where each item +// in the list will either be the same pointer if the URL does not contain a +// password, or to a new object if there is a password. +// The intended use-case is for logging lists of URLs safely. +func redactURLList(unredacted []*url.URL) []*url.URL { + r := make([]*url.URL, len(unredacted)) + // In the common case of no passwords, if we don't let the new object leave + // this function then GC should be easier. + needCopy := false + for i := range unredacted { + if unredacted[i] == nil { + r[i] = nil + continue + } + if _, has := unredacted[i].User.Password(); !has { + r[i] = unredacted[i] + continue + } + needCopy = true + ru := *unredacted[i] + ru.User = url.UserPassword(ru.User.Username(), "xxxxx") + r[i] = &ru + } + if needCopy { + return r + } + return unredacted +} + +// redactURLString() attempts to redact a URL string. +func redactURLString(raw string) string { + if !strings.ContainsRune(raw, '@') { + return raw + } + u, err := url.Parse(raw) + if err != nil { + return raw + } + return u.Redacted() +} + +// getURLsAsString returns a slice of u.Host from the given slice of url.URL's +func getURLsAsString(urls []*url.URL) []string { + a := make([]string, 0, len(urls)) + for _, u := range urls { + a = append(a, u.Host) + } + return a +} + +// copyBytes make a new slice of the same size as `src` and copy its content. +// If `src` is nil or its length is 0, then this returns `nil` +func copyBytes(src []byte) []byte { + if len(src) == 0 { + return nil + } + dst := make([]byte, len(src)) + copy(dst, src) + return dst +} + +// copyStrings make a new slice of the same size than `src` and copy its content. +// If `src` is nil, then this returns `nil` +func copyStrings(src []string) []string { + if src == nil { + return nil + } + dst := make([]string, len(src)) + copy(dst, src) + return dst +} + +// Returns a byte slice for the INFO protocol. +func generateInfoJSON(info *Info) []byte { + b, _ := json.Marshal(info) + pcs := [][]byte{[]byte("INFO"), b, []byte(CR_LF)} + return bytes.Join(pcs, []byte(" ")) +} + +// parallelTaskQueue starts a number of goroutines and returns a channel +// which functions can be sent to for queued parallel execution. The +// goroutines will stop running when the returned channel is closed and +// all queued tasks have completed. The passed in mp limits concurrency, +// or a value <= 0 will default to GOMAXPROCS. +func parallelTaskQueue(mp int) chan<- func() { + if rmp := runtime.GOMAXPROCS(-1); mp <= 0 { + mp = rmp + } else { + mp = max(rmp, mp) + } + tq := make(chan func(), mp) + for range mp { + go func() { + for fn := range tq { + fn() + } + }() + } + return tq +} + +// addSaturate returns a + b, saturating at the maximum value of T. +// Signed inputs must be non-negative. +func addSaturate[T ~int64 | ~uint64](a, b T) T { + if c := a + b; c >= a { + return c + } + if ^T(0) < 0 { + return T(^uint64(0) >> 1) + } + return ^T(0) +} + +// mulSaturate returns a * b, saturating at math.MaxInt64. +// Both a and b must be non-negative. +func mulSaturate(a, b int64) int64 { + hi, lo := bits.Mul64(uint64(a), uint64(b)) + if hi != 0 || lo > uint64(math.MaxInt64) { + return math.MaxInt64 + } + return int64(lo) +} diff --git a/server/util_test.go b/server/util_test.go new file mode 100644 index 0000000..e79447e --- /dev/null +++ b/server/util_test.go @@ -0,0 +1,311 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "math" + "math/rand/v2" + "net/url" + "reflect" + "strconv" + "sync" + "testing" + "time" +) + +func TestParseSize(t *testing.T) { + if parseSize(nil) != -1 { + t.Fatal("Should error on nil byte slice") + } + n := []byte("12345678") + if pn := parseSize(n); pn != 12345678 { + t.Fatalf("Did not parse %q correctly, res=%d", n, pn) + } + + n = []byte("12345invalid678") + if pn := parseSize(n); pn != -1 { + t.Fatalf("Should error on %q, res=%d", n, pn) + } +} + +func TestParseSInt64(t *testing.T) { + if parseInt64(nil) != -1 { + t.Fatal("Should error on nil byte slice") + } + n := []byte("12345678") + if pn := parseInt64(n); pn != 12345678 { + t.Fatalf("Did not parse %q correctly, res=%d", n, pn) + } + + n = []byte("12345invalid678") + if pn := parseInt64(n); pn != -1 { + t.Fatalf("Should error on %q, res=%d", n, pn) + } +} + +func TestParseHostPort(t *testing.T) { + check := func(hostPort string, defaultPort int, expectedHost string, expectedPort int, expectedErr bool) { + h, p, err := parseHostPort(hostPort, defaultPort) + if expectedErr { + if err == nil { + stackFatalf(t, "Expected an error, did not get one") + } + // expected error, so we are done + return + } + if !expectedErr && err != nil { + stackFatalf(t, "Unexpected error: %v", err) + } + if expectedHost != h { + stackFatalf(t, "Expected host %q, got %q", expectedHost, h) + } + if expectedPort != p { + stackFatalf(t, "Expected port %d, got %d", expectedPort, p) + } + } + check("addr:1234", 5678, "addr", 1234, false) + check(" addr:1234 ", 5678, "addr", 1234, false) + check(" addr : 1234 ", 5678, "addr", 1234, false) + check("addr", 5678, "addr", 5678, false) + check(" addr ", 5678, "addr", 5678, false) + check("addr:-1", 5678, "addr", 5678, false) + check(" addr:-1 ", 5678, "addr", 5678, false) + check(" addr : -1 ", 5678, "addr", 5678, false) + check("addr:0", 5678, "addr", 5678, false) + check(" addr:0 ", 5678, "addr", 5678, false) + check(" addr : 0 ", 5678, "addr", 5678, false) + check("addr:addr", 0, "", 0, true) + check("addr:::1234", 0, "", 0, true) + check("", 0, "", 0, true) +} + +func TestURLsAreEqual(t *testing.T) { + check := func(t *testing.T, u1Str, u2Str string, expectedSame bool) { + t.Helper() + u1, err := url.Parse(u1Str) + if err != nil { + t.Fatalf("Error parsing url %q: %v", u1Str, err) + } + u2, err := url.Parse(u2Str) + if err != nil { + t.Fatalf("Error parsing url %q: %v", u2Str, err) + } + same := urlsAreEqual(u1, u2) + if expectedSame && !same { + t.Fatalf("Expected %v and %v to be the same, they were not", u1, u2) + } else if !expectedSame && same { + t.Fatalf("Expected %v and %v to be different, they were not", u1, u2) + } + } + check(t, "nats://localhost:4222", "nats://localhost:4222", true) + check(t, "nats://ivan:pwd@localhost:4222", "nats://ivan:pwd@localhost:4222", true) + check(t, "nats://ivan@localhost:4222", "nats://ivan@localhost:4222", true) + check(t, "nats://ivan:@localhost:4222", "nats://ivan:@localhost:4222", true) + check(t, "nats://host1:4222", "nats://host2:4222", false) +} + +func TestComma(t *testing.T) { + type testList []struct { + name, got, exp string + } + + l := testList{ + {"0", comma(0), "0"}, + {"10", comma(10), "10"}, + {"100", comma(100), "100"}, + {"1,000", comma(1000), "1,000"}, + {"10,000", comma(10000), "10,000"}, + {"100,000", comma(100000), "100,000"}, + {"10,000,000", comma(10000000), "10,000,000"}, + {"10,100,000", comma(10100000), "10,100,000"}, + {"10,010,000", comma(10010000), "10,010,000"}, + {"10,001,000", comma(10001000), "10,001,000"}, + {"123,456,789", comma(123456789), "123,456,789"}, + {"maxint", comma(9.223372e+18), "9,223,372,000,000,000,000"}, + {"math.maxint", comma(math.MaxInt64), "9,223,372,036,854,775,807"}, + {"math.minint", comma(math.MinInt64), "-9,223,372,036,854,775,808"}, + {"minint", comma(-9.223372e+18), "-9,223,372,000,000,000,000"}, + {"-123,456,789", comma(-123456789), "-123,456,789"}, + {"-10,100,000", comma(-10100000), "-10,100,000"}, + {"-10,010,000", comma(-10010000), "-10,010,000"}, + {"-10,001,000", comma(-10001000), "-10,001,000"}, + {"-10,000,000", comma(-10000000), "-10,000,000"}, + {"-100,000", comma(-100000), "-100,000"}, + {"-10,000", comma(-10000), "-10,000"}, + {"-1,000", comma(-1000), "-1,000"}, + {"-100", comma(-100), "-100"}, + {"-10", comma(-10), "-10"}, + } + + failed := false + for _, test := range l { + if test.got != test.exp { + t.Errorf("On %v, expected '%v', but got '%v'", + test.name, test.exp, test.got) + failed = true + } + } + if failed { + t.FailNow() + } +} + +func TestURLRedaction(t *testing.T) { + redactionFromTo := []struct { + Full string + Safe string + }{ + {"nats://foo:bar@example.org", "nats://foo:xxxxx@example.org"}, + {"nats://foo@example.org", "nats://foo@example.org"}, + {"nats://example.org", "nats://example.org"}, + {"nats://example.org/foo?bar=1", "nats://example.org/foo?bar=1"}, + } + var err error + listFull := make([]*url.URL, len(redactionFromTo)) + listSafe := make([]*url.URL, len(redactionFromTo)) + for i := range redactionFromTo { + r := redactURLString(redactionFromTo[i].Full) + if r != redactionFromTo[i].Safe { + t.Fatalf("Redacting URL [index %d] %q, expected %q got %q", i, redactionFromTo[i].Full, redactionFromTo[i].Safe, r) + } + if listFull[i], err = url.Parse(redactionFromTo[i].Full); err != nil { + t.Fatalf("Redacting URL index %d parse Full failed: %v", i, err) + } + if listSafe[i], err = url.Parse(redactionFromTo[i].Safe); err != nil { + t.Fatalf("Redacting URL index %d parse Safe failed: %v", i, err) + } + } + results := redactURLList(listFull) + if !reflect.DeepEqual(results, listSafe) { + t.Fatalf("Redacting URL list did not compare equal, even after each URL did") + } +} + +func TestVersionAtLeast(t *testing.T) { + for _, test := range []struct { + version string + major int + minor int + update int + result bool + }{ + {"2.0.0-beta", 1, 9, 9, true}, + {"2.0.0", 1, 99, 9, true}, + {"2.2.0", 2, 1, 9, true}, + {"2.2.2", 2, 2, 2, true}, + {"2.2.2", 2, 2, 3, false}, + {"2.2.2", 2, 3, 2, false}, + {"2.2.2", 3, 2, 2, false}, + {"2.22.2", 3, 0, 0, false}, + {"2.2.22", 2, 3, 0, false}, + {"bad.version", 1, 2, 3, false}, + } { + t.Run(_EMPTY_, func(t *testing.T) { + if res := versionAtLeast(test.version, test.major, test.minor, test.update); res != test.result { + t.Fatalf("For check version %q at least %d.%d.%d result should have been %v, got %v", + test.version, test.major, test.minor, test.update, test.result, res) + } + }) + } +} + +func BenchmarkParseInt(b *testing.B) { + b.SetBytes(1) + n := "12345678" + for i := 0; i < b.N; i++ { + strconv.ParseInt(n, 10, 0) + } +} + +func BenchmarkParseSize(b *testing.B) { + b.SetBytes(1) + n := []byte("12345678") + for i := 0; i < b.N; i++ { + parseSize(n) + } +} + +func deferUnlock(mu *sync.Mutex) { + mu.Lock() + defer mu.Unlock() + // see noDeferUnlock + if false { + return + } +} + +func BenchmarkDeferMutex(b *testing.B) { + var mu sync.Mutex + b.SetBytes(1) + for i := 0; i < b.N; i++ { + deferUnlock(&mu) + } +} + +func noDeferUnlock(mu *sync.Mutex) { + mu.Lock() + // prevent staticcheck warning about empty critical section + if false { + return + } + mu.Unlock() +} + +func BenchmarkNoDeferMutex(b *testing.B) { + var mu sync.Mutex + b.SetBytes(1) + for i := 0; i < b.N; i++ { + noDeferUnlock(&mu) + } +} + +func createTestSub() *subscription { + return &subscription{ + subject: []byte("foo"), + queue: []byte("bar"), + sid: []byte("22"), + } +} + +func BenchmarkArrayRand(b *testing.B) { + b.StopTimer() + r := rand.New(rand.NewPCG(uint64(time.Now().UnixNano()), 0)) + // Create an array of 10 items + subs := []*subscription{} + for i := 0; i < 10; i++ { + subs = append(subs, createTestSub()) + } + b.StartTimer() + + for i := 0; i < b.N; i++ { + index := r.IntN(len(subs)) + _ = subs[index] + } +} + +func BenchmarkMapRange(b *testing.B) { + b.StopTimer() + // Create an map of 10 items + subs := map[int]*subscription{} + for i := 0; i < 10; i++ { + subs[i] = createTestSub() + } + b.StartTimer() + + for i := 0; i < b.N; i++ { + for range subs { + break + } + } +} diff --git a/server/websocket.go b/server/websocket.go new file mode 100644 index 0000000..9eaed7e --- /dev/null +++ b/server/websocket.go @@ -0,0 +1,1689 @@ +// Copyright 2020-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "crypto/fips140" + crand "crypto/rand" + "crypto/sha1" + "crypto/tls" + "encoding/base64" + "encoding/binary" + "errors" + "fmt" + "io" + "log" + mrand "math/rand/v2" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + "unicode/utf8" + + "github.com/klauspost/compress/flate" +) + +type wsOpCode int + +const ( + // From https://tools.ietf.org/html/rfc6455#section-5.2 + wsTextMessage = wsOpCode(1) + wsBinaryMessage = wsOpCode(2) + wsCloseMessage = wsOpCode(8) + wsPingMessage = wsOpCode(9) + wsPongMessage = wsOpCode(10) + + wsFinalBit = 1 << 7 + wsRsv1Bit = 1 << 6 // Used for compression, from https://tools.ietf.org/html/rfc7692#section-6 + wsRsv2Bit = 1 << 5 + wsRsv3Bit = 1 << 4 + + wsMaskBit = 1 << 7 + + wsContinuationFrame = 0 + wsMaxFrameHeaderSize = 14 // Since LeafNode may need to behave as a client + wsMaxControlPayloadSize = 125 + wsFrameSizeForBrowsers = 4096 // From experiment, webrowsers behave better with limited frame size + wsCompressThreshold = 64 // Don't compress for small buffer(s) + wsMaxMsgPayloadMultiple = 8 + wsMaxMsgPayloadLimit = 64 * 1024 * 1024 + wsCloseSatusSize = 2 + + // From https://tools.ietf.org/html/rfc6455#section-11.7 + wsCloseStatusNormalClosure = 1000 + wsCloseStatusGoingAway = 1001 + wsCloseStatusProtocolError = 1002 + wsCloseStatusUnsupportedData = 1003 + wsCloseStatusNoStatusReceived = 1005 + wsCloseStatusInvalidPayloadData = 1007 + wsCloseStatusPolicyViolation = 1008 + wsCloseStatusMessageTooBig = 1009 + wsCloseStatusInternalSrvError = 1011 + wsCloseStatusTLSHandshake = 1015 + + wsFirstFrame = true + wsContFrame = false + wsFinalFrame = true + wsUncompressedFrame = false + + wsSchemePrefix = "ws" + wsSchemePrefixTLS = "wss" + + wsNoMaskingHeader = "Nats-No-Masking" + wsNoMaskingValue = "true" + wsXForwardedForHeader = "X-Forwarded-For" + wsNoMaskingFullResponse = wsNoMaskingHeader + ": " + wsNoMaskingValue + CR_LF + wsPMCExtension = "permessage-deflate" // per-message compression + wsPMCSrvNoCtx = "server_no_context_takeover" + wsPMCCliNoCtx = "client_no_context_takeover" + wsPMCReqHeaderValue = wsPMCExtension + "; " + wsPMCSrvNoCtx + "; " + wsPMCCliNoCtx + wsPMCFullResponse = "Sec-WebSocket-Extensions: " + wsPMCExtension + "; " + wsPMCSrvNoCtx + "; " + wsPMCCliNoCtx + _CRLF_ + wsSecProto = "Sec-Websocket-Protocol" + wsMQTTSecProtoVal = "mqtt" + wsMQTTSecProto = wsSecProto + ": " + wsMQTTSecProtoVal + CR_LF +) + +var decompressorPool sync.Pool +var compressLastBlock = []byte{0x00, 0x00, 0xff, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff} + +// From https://tools.ietf.org/html/rfc6455#section-1.3 +var wsGUID = []byte("258EAFA5-E914-47DA-95CA-C5AB0DC85B11") + +// Concatenate the key sent by the client with the GUID, then computes the SHA1 hash +// and returns it as a based64 encoded string. +func wsAcceptKey(key string) string { + var r []byte + fips140.WithoutEnforcement(func() { + h := sha1.New() + h.Write([]byte(key)) + h.Write(wsGUID) + r = h.Sum(nil) + }) + return base64.StdEncoding.EncodeToString(r) +} + +// Test can enable this so that server does not support "no-masking" requests. +var wsTestRejectNoMasking = false + +type websocket struct { + frames net.Buffers + fs int64 + closeMsg []byte + compress bool + closeSent bool + browser bool + nocompfrag bool // No fragment for compressed frames + maskread bool + maskwrite bool + compressor *flate.Writer + cookieJwt string + cookieUsername string + cookiePassword string + cookieToken string + clientIP string +} + +type srvWebsocket struct { + mu sync.RWMutex + server *http.Server + listener net.Listener + listenerErr error + allowedOrigins map[string][]*allowedOrigin // host will be the key + sameOrigin bool + connectURLs []string + connectURLsMap refCountedUrlSet + authOverride bool // indicate if there is auth override in websocket config + rawHeaders string // raw headers to be used in the upgrade response. + + // These are immutable and can be accessed without lock. + // This is the case when generating the client INFO. + tls bool // True if TLS is required (TLSConfig is specified). + host string // Host/IP the webserver is listening on (shortcut to opts.Websocket.Host). + port int // Port the webserver is listening on. This is after an ephemeral port may have been selected (shortcut to opts.Websocket.Port). +} + +type allowedOrigin struct { + scheme string + port string +} + +type wsUpgradeResult struct { + conn net.Conn + ws *websocket + kind int +} + +type wsReadInfo struct { + rem uint64 + fs bool + ff bool + fc bool + mask bool // Incoming leafnode connections may not have masking. + mkpos byte + mkey [4]byte + cbufs [][]byte + coff int + csz uint64 +} + +func (r *wsReadInfo) init() { + r.fs, r.ff = true, true +} + +func (r *wsReadInfo) resetCompressedState() { + r.fs = true + r.ff = true + r.fc = false + r.rem = 0 + r.cbufs = nil + r.coff = 0 + r.csz = 0 +} + +// Compressed WebSocket messages have to be accumulated before they can be +// decompressed and handed to the parser, so this transport limit needs to +// allow batching several max_payload-sized NATS operations while still +// capping resource usage on the buffered compressed path. +func wsMaxMessageSize(mpay int) uint64 { + if mpay <= 0 { + mpay = MAX_PAYLOAD_SIZE + } + limit := uint64(mpay) * wsMaxMsgPayloadMultiple + if limit > wsMaxMsgPayloadLimit { + limit = wsMaxMsgPayloadLimit + } + return limit +} + +// Returns a slice containing `needed` bytes from the given buffer `buf` +// starting at position `pos`, and possibly read from the given reader `r`. +// When bytes are present in `buf`, the `pos` is incremented by the number +// of bytes found up to `needed` and the new position is returned. If not +// enough bytes are found, the bytes found in `buf` are copied to the returned +// slice and the remaning bytes are read from `r`. +func wsGet(r io.Reader, buf []byte, pos, needed uint64) ([]byte, uint64, error) { + avail := uint64(len(buf)) - pos + if avail >= needed { + return buf[pos : pos+needed], pos + needed, nil + } + b := make([]byte, needed) + start := uint64(copy(b, buf[pos:])) + for start != needed { + n, err := r.Read(b[start:cap(b)]) + if err != nil { + return nil, 0, err + } + start += uint64(n) + } + return b, pos + avail, nil +} + +// Returns true if this connection is from a Websocket client. +// Lock held on entry. +func (c *client) isWebsocket() bool { + return c.ws != nil +} + +// Returns a slice of byte slices corresponding to payload of websocket frames. +// The byte slice `buf` is filled with bytes from the connection's read loop. +// This function will decode the frame headers and unmask the payload(s). +// It is possible that the returned slices point to the given `buf` slice, so +// `buf` should not be overwritten until the returned slices have been parsed. +// +// Client lock MUST NOT be held on entry. +func (c *client) wsRead(r *wsReadInfo, ior io.Reader, buf []byte) ([][]byte, error) { + var bufs [][]byte + err := c.wsReadLoop(r, ior, buf, func(b []byte, compressed, final bool) error { + if compressed { + return errors.New("compressed websocket frames require wsReadAndParse") + } + bufs = append(bufs, b) + return nil + }) + return bufs, err +} + +func (c *client) wsReadAndParse(r *wsReadInfo, ior io.Reader, buf []byte) error { + mpay := int(atomic.LoadInt32(&c.mpay)) + if mpay <= 0 { + mpay = MAX_PAYLOAD_SIZE + } + return c.wsReadLoop(r, ior, buf, func(b []byte, compressed, final bool) error { + if compressed { + if err := c.wsDecompressAndParse(r, b, final, mpay); err != nil { + r.resetCompressedState() + return err + } + if final { + r.fc = false + } + return nil + } + return c.parse(b) + }) +} + +func (c *client) wsReadLoop(r *wsReadInfo, ior io.Reader, buf []byte, handle func([]byte, bool, bool) error) error { + var ( + tmpBuf []byte + err error + pos uint64 + max = uint64(len(buf)) + ) + for pos != max { + if r.fs { + b0 := buf[pos] + frameType := wsOpCode(b0 & 0xF) + final := b0&wsFinalBit != 0 + compressed := b0&wsRsv1Bit != 0 + if b0&(wsRsv2Bit|wsRsv3Bit) != 0 { + return c.wsHandleProtocolError("RSV2 and RSV3 must be clear") + } + if compressed && !c.ws.compress { + return c.wsHandleProtocolError("compressed frame received without negotiated permessage-deflate") + } + pos++ + + tmpBuf, pos, err = wsGet(ior, buf, pos, 1) + if err != nil { + return err + } + b1 := tmpBuf[0] + + // Clients MUST set the mask bit. If not set, reject. + // However, LEAF by default will not have masking, unless they are forced to, by configuration. + if r.mask && b1&wsMaskBit == 0 { + return c.wsHandleProtocolError("mask bit missing") + } + + // Store size in case it is < 125 + r.rem = uint64(b1 & 0x7F) + + switch frameType { + case wsPingMessage, wsPongMessage, wsCloseMessage: + if r.rem > wsMaxControlPayloadSize { + return c.wsHandleProtocolError( + fmt.Sprintf("control frame length bigger than maximum allowed of %v bytes", + wsMaxControlPayloadSize)) + } + if !final { + return c.wsHandleProtocolError("control frame does not have final bit set") + } + if compressed { + return c.wsHandleProtocolError("control frame must not be compressed") + } + case wsTextMessage, wsBinaryMessage: + if !r.ff { + return c.wsHandleProtocolError("new message started before final frame for previous message was received") + } + r.ff = final + r.fc = compressed + case wsContinuationFrame: + // Compressed bit must be only set in the first frame + if r.ff || compressed { + return c.wsHandleProtocolError("invalid continuation frame") + } + r.ff = final + default: + return c.wsHandleProtocolError(fmt.Sprintf("unknown opcode %v", frameType)) + } + + switch r.rem { + case 126: + tmpBuf, pos, err = wsGet(ior, buf, pos, 2) + if err != nil { + return err + } + r.rem = uint64(binary.BigEndian.Uint16(tmpBuf)) + case 127: + tmpBuf, pos, err = wsGet(ior, buf, pos, 8) + if err != nil { + return err + } + if r.rem = binary.BigEndian.Uint64(tmpBuf); r.rem&(uint64(1)<<63) != 0 { + return c.wsHandleProtocolError("invalid 64-bit payload length") + } + } + + if r.mask { + // Read masking key + tmpBuf, pos, err = wsGet(ior, buf, pos, 4) + if err != nil { + return err + } + copy(r.mkey[:], tmpBuf) + r.mkpos = 0 + } + + // Handle control messages in place... + if wsIsControlFrame(frameType) { + pos, err = c.wsHandleControlFrame(r, frameType, ior, buf, pos) + if err != nil { + return err + } + continue + } + + // Done with the frame header + r.fs = false + } + if pos < max { + n := r.rem + if pos+n > max { + n = max - pos + } + b := buf[pos : pos+n] + pos += n + r.rem -= n + // If needed, unmask the buffer + if r.mask { + r.unmask(b) + } + if err := handle(b, r.fc, r.ff && r.rem == 0); err != nil { + return err + } + if r.rem == 0 { + r.fs = true + } + } + } + return nil +} + +func (r *wsReadInfo) Read(dst []byte) (int, error) { + if len(dst) == 0 { + return 0, nil + } + if len(r.cbufs) == 0 { + return 0, io.EOF + } + copied := 0 + rem := len(dst) + for buf := r.cbufs[0]; buf != nil && rem > 0; { + n := len(buf[r.coff:]) + if n > rem { + n = rem + } + copy(dst[copied:], buf[r.coff:r.coff+n]) + copied += n + rem -= n + r.coff += n + buf = r.nextCBuf() + } + return copied, nil +} + +func (r *wsReadInfo) nextCBuf() []byte { + // We still have remaining data in the first buffer + if r.coff != len(r.cbufs[0]) { + return r.cbufs[0] + } + // We read the full first buffer. Reset offset. + r.coff = 0 + // We were at the last buffer, so we are done. + if len(r.cbufs) == 1 { + r.cbufs = nil + return nil + } + // Here we move to the next buffer. + r.cbufs = r.cbufs[1:] + return r.cbufs[0] +} + +func (r *wsReadInfo) ReadByte() (byte, error) { + for len(r.cbufs) > 0 && len(r.cbufs[0]) == 0 { + r.nextCBuf() + } + if len(r.cbufs) == 0 { + return 0, io.EOF + } + b := r.cbufs[0][r.coff] + r.coff++ + r.nextCBuf() + return b, nil +} + +func (c *client) wsDecompressAndParse(r *wsReadInfo, b []byte, final bool, mpay int) error { + limit := wsMaxMessageSize(mpay) + if len(b) > 0 { + if r.csz+uint64(len(b)) > limit { + return ErrMaxPayload + } + r.cbufs = append(r.cbufs, append([]byte(nil), b...)) + r.csz += uint64(len(b)) + } + if !final { + return nil + } + if r.csz+uint64(len(compressLastBlock)) > limit { + return ErrMaxPayload + } + r.cbufs = append(r.cbufs, compressLastBlock) + r.csz += uint64(len(compressLastBlock)) + r.coff = 0 + d, _ := decompressorPool.Get().(io.ReadCloser) + if d == nil { + d = flate.NewReader(r) + } else { + d.(flate.Resetter).Reset(r, nil) + } + defer func() { + d.Close() + decompressorPool.Put(d) + r.cbufs = nil + r.coff = 0 + r.csz = 0 + }() + lr := io.LimitedReader{R: d, N: int64(mpay + 1)} + buf := make([]byte, 32*1024) + total := 0 + for { + n, err := lr.Read(buf) + if n > 0 { + pn := n + if total+n > mpay { + pn = mpay - total + } + if pn > 0 { + if err := c.parse(buf[:pn]); err != nil { + return err + } + } + total += n + if total > mpay { + return ErrMaxPayload + } + } + if err == nil { + continue + } + if err == io.EOF { + return nil + } + return err + } +} + +// Handles the PING, PONG and CLOSE websocket control frames. +// +// Client lock MUST NOT be held on entry. +func (c *client) wsHandleControlFrame(r *wsReadInfo, frameType wsOpCode, nc io.Reader, buf []byte, pos uint64) (uint64, error) { + var payload []byte + var err error + + if r.rem > 0 { + payload, pos, err = wsGet(nc, buf, pos, r.rem) + if err != nil { + return pos, err + } + if r.mask { + r.unmask(payload) + } + r.rem = 0 + } + switch frameType { + case wsCloseMessage: + status := wsCloseStatusNoStatusReceived + var body string + lp := len(payload) + if lp == 1 { + return pos, c.wsHandleProtocolError("close frame payload cannot be 1 byte") + } + // If there is a payload, the status is represented as a 2-byte + // unsigned integer (in network byte order). Then, there may be an + // optional body. + hasStatus, hasBody := lp >= wsCloseSatusSize, lp > wsCloseSatusSize + if hasStatus { + // Decode the status + status = int(binary.BigEndian.Uint16(payload[:wsCloseSatusSize])) + if !wsIsValidCloseStatus(status) { + return pos, c.wsHandleProtocolError(fmt.Sprintf("invalid close status code %v", status)) + } + // Now if there is a body, capture it and make sure this is a valid UTF-8. + if hasBody { + body = string(payload[wsCloseSatusSize:]) + if !utf8.ValidString(body) { + // https://tools.ietf.org/html/rfc6455#section-5.5.1 + // If body is present, it must be a valid utf8 + status = wsCloseStatusInvalidPayloadData + body = "invalid utf8 body in close frame" + } + } + } + // If the status indicates that nothing was received, then we don't + // send anything back. + // From https://datatracker.ietf.org/doc/html/rfc6455#section-7.4 + // it says that code 1005 is a reserved value and MUST NOT be set as a + // status code in a Close control frame by an endpoint. It is + // designated for use in applications expecting a status code to indicate + // that no status code was actually present. + var clm []byte + if status != wsCloseStatusNoStatusReceived { + clm = wsCreateCloseMessage(status, body) + } + c.wsEnqueueControlMessage(wsCloseMessage, clm) + if len(clm) > 0 { + nbPoolPut(clm) // wsEnqueueControlMessage has taken a copy. + } + // Return io.EOF so that readLoop will close the connection as ClientClosed + // after processing pending buffers. + return pos, io.EOF + case wsPingMessage: + c.wsEnqueueControlMessage(wsPongMessage, payload) + case wsPongMessage: + // Nothing to do.. + } + return pos, nil +} + +// Unmask the given slice. +func (r *wsReadInfo) unmask(buf []byte) { + p := int(r.mkpos) + if len(buf) < 16 { + for i := 0; i < len(buf); i++ { + buf[i] ^= r.mkey[p&3] + p++ + } + r.mkpos = byte(p & 3) + return + } + var k [8]byte + for i := 0; i < 8; i++ { + k[i] = r.mkey[(p+i)&3] + } + km := binary.BigEndian.Uint64(k[:]) + n := (len(buf) / 8) * 8 + for i := 0; i < n; i += 8 { + tmp := binary.BigEndian.Uint64(buf[i : i+8]) + tmp ^= km + binary.BigEndian.PutUint64(buf[i:], tmp) + } + buf = buf[n:] + for i := 0; i < len(buf); i++ { + buf[i] ^= r.mkey[p&3] + p++ + } + r.mkpos = byte(p & 3) +} + +// Returns true if the op code corresponds to a control frame. +func wsIsControlFrame(frameType wsOpCode) bool { + return frameType >= wsCloseMessage +} + +// Create the frame header. +// Encodes the frame type and optional compression flag, and the size of the payload. +func wsCreateFrameHeader(useMasking, compressed bool, frameType wsOpCode, l int) ([]byte, []byte) { + fh := nbPoolGet(wsMaxFrameHeaderSize)[:wsMaxFrameHeaderSize] + n, key := wsFillFrameHeader(fh, useMasking, wsFirstFrame, wsFinalFrame, compressed, frameType, l) + return fh[:n], key +} + +func wsFillFrameHeader(fh []byte, useMasking, first, final, compressed bool, frameType wsOpCode, l int) (int, []byte) { + var n int + var b byte + if first { + b = byte(frameType) + } + if final { + b |= wsFinalBit + } + if compressed { + b |= wsRsv1Bit + } + b1 := byte(0) + if useMasking { + b1 |= wsMaskBit + } + switch { + case l <= 125: + n = 2 + fh[0] = b + fh[1] = b1 | byte(l) + case l < 65536: + n = 4 + fh[0] = b + fh[1] = b1 | 126 + binary.BigEndian.PutUint16(fh[2:], uint16(l)) + default: + n = 10 + fh[0] = b + fh[1] = b1 | 127 + binary.BigEndian.PutUint64(fh[2:], uint64(l)) + } + var key []byte + if useMasking { + var keyBuf [4]byte + if _, err := io.ReadFull(crand.Reader, keyBuf[:4]); err != nil { + kv := mrand.Int32() + binary.LittleEndian.PutUint32(keyBuf[:4], uint32(kv)) + } + copy(fh[n:], keyBuf[:4]) + key = fh[n : n+4] + n += 4 + } + return n, key +} + +// Invokes wsEnqueueControlMessageLocked under client lock. +// +// Client lock MUST NOT be held on entry +func (c *client) wsEnqueueControlMessage(controlMsg wsOpCode, payload []byte) { + c.mu.Lock() + c.wsEnqueueControlMessageLocked(controlMsg, payload) + c.mu.Unlock() +} + +// Mask the buffer with the given key +func wsMaskBuf(key, buf []byte) { + for i := 0; i < len(buf); i++ { + buf[i] ^= key[i&3] + } +} + +// Mask the buffers, as if they were contiguous, with the given key +func wsMaskBufs(key []byte, bufs [][]byte) { + pos := 0 + for i := 0; i < len(bufs); i++ { + buf := bufs[i] + for j := 0; j < len(buf); j++ { + buf[j] ^= key[pos&3] + pos++ + } + } +} + +// Enqueues a websocket control message. +// If the control message is a wsCloseMessage, then marks this client +// has having sent the close message (since only one should be sent). +// This will prevent the generic closeConnection() to enqueue one. +// +// Client lock held on entry. +func (c *client) wsEnqueueControlMessageLocked(controlMsg wsOpCode, payload []byte) { + // Control messages are never compressed and their size will be + // less than wsMaxControlPayloadSize, which means the frame header + // will be only 2 or 6 bytes. + useMasking := c.ws.maskwrite + sz := 2 + if useMasking { + sz += 4 + } + cm := nbPoolGet(sz + len(payload)) + cm = cm[:cap(cm)] + n, key := wsFillFrameHeader(cm, useMasking, wsFirstFrame, wsFinalFrame, wsUncompressedFrame, controlMsg, len(payload)) + cm = cm[:n] + // Note that payload is optional. + if len(payload) > 0 { + cm = append(cm, payload...) + if useMasking { + wsMaskBuf(key, cm[n:]) + } + } + c.out.pb += int64(len(cm)) + if controlMsg == wsCloseMessage { + // We can't add the close message to the frames buffers + // now. It will be done on a flushOutbound() when there + // are no more pending buffers to send. + c.ws.closeSent = true + c.ws.closeMsg = cm + } else { + c.ws.frames = append(c.ws.frames, cm) + c.ws.fs += int64(len(cm)) + } + c.flushSignal() +} + +// Enqueues a websocket close message with a status mapped from the given `reason`. +// +// Client lock held on entry +func (c *client) wsEnqueueCloseMessage(reason ClosedState) { + var status int + switch reason { + case ClientClosed: + status = wsCloseStatusNormalClosure + case AuthenticationTimeout, AuthenticationViolation, SlowConsumerPendingBytes, SlowConsumerWriteDeadline, + MaxAccountConnectionsExceeded, MaxConnectionsExceeded, MaxControlLineExceeded, MaxSubscriptionsExceeded, + MissingAccount, AuthenticationExpired, Revocation: + status = wsCloseStatusPolicyViolation + case TLSHandshakeError: + status = wsCloseStatusTLSHandshake + case ParseError, ProtocolViolation, BadClientProtocolVersion: + status = wsCloseStatusProtocolError + case MaxPayloadExceeded: + status = wsCloseStatusMessageTooBig + case WriteError, ReadError, StaleConnection, ServerShutdown: + // We used to have WriteError, ReadError and StaleConnection result in + // code 1006, which the spec says that it must not be used to set the + // status in the close message. So using this one instead. + status = wsCloseStatusGoingAway + default: + status = wsCloseStatusInternalSrvError + } + body := wsCreateCloseMessage(status, reason.String()) + c.wsEnqueueControlMessageLocked(wsCloseMessage, body) + nbPoolPut(body) // wsEnqueueControlMessageLocked has taken a copy. +} + +// Create and then enqueue a close message with a protocol error and the +// given message. This is invoked when parsing websocket frames. +// +// Lock MUST NOT be held on entry. +func (c *client) wsHandleProtocolError(message string) error { + buf := wsCreateCloseMessage(wsCloseStatusProtocolError, message) + c.wsEnqueueControlMessage(wsCloseMessage, buf) + nbPoolPut(buf) // wsEnqueueControlMessage has taken a copy. + return errors.New(message) +} + +func wsIsValidCloseStatus(code int) bool { + switch code { + case wsCloseStatusNoStatusReceived, 1004, 1006, wsCloseStatusTLSHandshake: + return false + } + if code < 1000 || code >= 5000 { + return false + } + // 1016-2999 are currently reserved. + if code >= 1016 && code <= 2999 { + return false + } + return true +} + +// Create a close message with the given `status` and `body`. +// If the `body` is more than the maximum allows control frame payload size, +// it is truncated and "..." is added at the end (as a hint that message +// is not complete). +func wsCreateCloseMessage(status int, body string) []byte { + // Since a control message payload is limited in size, we + // will limit the text and add trailing "..." if truncated. + // The body of a Close Message must be preceded with 2 bytes, + // so take that into account for limiting the body length. + if len(body) > wsMaxControlPayloadSize-2 { + body = body[:wsMaxControlPayloadSize-5] + body += "..." + } + buf := nbPoolGet(2 + len(body))[:2+len(body)] + // We need to have a 2 byte unsigned int that represents the error status code + // https://tools.ietf.org/html/rfc6455#section-5.5.1 + binary.BigEndian.PutUint16(buf[:2], uint16(status)) + copy(buf[2:], []byte(body)) + return buf +} + +// Process websocket client handshake. On success, returns the raw net.Conn that +// will be used to create a *client object. +// Invoked from the HTTP server listening on websocket port. +func (s *Server) wsUpgrade(w http.ResponseWriter, r *http.Request) (*wsUpgradeResult, error) { + kind := CLIENT + if r.URL != nil { + ep := r.URL.EscapedPath() + if strings.HasSuffix(ep, leafNodeWSPath) { + kind = LEAF + } else if strings.HasSuffix(ep, mqttWSPath) { + kind = MQTT + } + } + + opts := s.getOpts() + + // Reject MQTT-over-WebSocket upgrades unless MQTT is enabled. + if kind == MQTT && opts.MQTT.Port == 0 { + return nil, wsReturnHTTPError(w, r, http.StatusNotFound, "mqtt websocket endpoint not enabled") + } + + // From https://tools.ietf.org/html/rfc6455#section-4.2.1 + // Point 1. + if r.Method != "GET" { + return nil, wsReturnHTTPError(w, r, http.StatusMethodNotAllowed, "request method must be GET") + } + // Point 2. + if r.Host == _EMPTY_ { + return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "'Host' missing in request") + } + // Point 3. + if !wsHeaderContains(r.Header, "Upgrade", "websocket") { + return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "invalid value for header 'Upgrade'") + } + // Point 4. + if !wsHeaderContains(r.Header, "Connection", "Upgrade") { + return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "invalid value for header 'Connection'") + } + // Point 5. + key := r.Header.Get("Sec-Websocket-Key") + if key == _EMPTY_ { + return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "key missing") + } + decoded, err := base64.StdEncoding.DecodeString(key) + if err != nil || len(decoded) != 16 { + return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "invalid websocket key") + } + // Point 6. + if !wsHeaderContains(r.Header, "Sec-Websocket-Version", "13") { + return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "invalid version") + } + // Others are optional + // Point 7. + if err := s.websocket.checkOrigin(r); err != nil { + return nil, wsReturnHTTPError(w, r, http.StatusForbidden, fmt.Sprintf("origin not allowed: %v", err)) + } + // Point 8. + // We don't have protocols, so ignore. + // Point 9. + // Extensions, only support for compression at the moment + compress := opts.Websocket.Compression + if compress { + // Simply check if permessage-deflate extension is present. + compress, _ = wsPMCExtensionSupport(r.Header, true) + } + // We will do masking if asked (unless we reject for tests) + noMasking := r.Header.Get(wsNoMaskingHeader) == wsNoMaskingValue && !wsTestRejectNoMasking + + h, ok := w.(http.Hijacker) + if !ok { + return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "websocket upgrade not supported") + } + conn, brw, err := h.Hijack() + if err != nil { + if conn != nil { + conn.Close() + } + return nil, wsReturnHTTPError(w, r, http.StatusInternalServerError, err.Error()) + } + if brw.Reader.Buffered() > 0 { + conn.Close() + return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "client sent data before handshake is complete") + } + + var buf [1024]byte + p := buf[:0] + + // From https://tools.ietf.org/html/rfc6455#section-4.2.2 + p = append(p, "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: "...) + p = append(p, wsAcceptKey(key)...) + p = append(p, _CRLF_...) + if compress { + p = append(p, wsPMCFullResponse...) + } + if noMasking { + p = append(p, wsNoMaskingFullResponse...) + } + if kind == MQTT { + p = append(p, wsMQTTSecProto...) + } + if s.websocket.rawHeaders != _EMPTY_ { + p = append(p, s.websocket.rawHeaders...) + } + p = append(p, _CRLF_...) + + if _, err = conn.Write(p); err != nil { + conn.Close() + return nil, err + } + // If there was a deadline set for the handshake, clear it now. + if opts.Websocket.HandshakeTimeout > 0 { + conn.SetDeadline(time.Time{}) + } + // Server always expect "clients" to send masked payload, unless the option + // "no-masking" has been enabled. + ws := &websocket{compress: compress, maskread: !noMasking} + + // Check for X-Forwarded-For header + if cips, ok := r.Header[wsXForwardedForHeader]; ok { + if len(cips) > 0 { + cip := cips[0] + if net.ParseIP(cip) != nil { + ws.clientIP = cip + } + } + } + + if kind == CLIENT || kind == MQTT { + // Indicate if this is likely coming from a browser. + if ua := r.Header.Get("User-Agent"); ua != _EMPTY_ && strings.HasPrefix(ua, "Mozilla/") { + ws.browser = true + // Disable fragmentation of compressed frames for Safari browsers. + // Unfortunately, you could be running Chrome on macOS and this + // string will contain "Safari/" (along "Chrome/"). However, what + // I have found is that actual Safari browser also have "Version/". + // So make the combination of the two. + ws.nocompfrag = ws.compress && strings.Contains(ua, "Version/") && strings.Contains(ua, "Safari/") + } + + if cookies := r.Cookies(); len(cookies) > 0 { + ows := &opts.Websocket + for _, c := range cookies { + if ows.JWTCookie == c.Name { + ws.cookieJwt = c.Value + } else if ows.UsernameCookie == c.Name { + ws.cookieUsername = c.Value + } else if ows.PasswordCookie == c.Name { + ws.cookiePassword = c.Value + } else if ows.TokenCookie == c.Name { + ws.cookieToken = c.Value + } + } + } + } + return &wsUpgradeResult{conn: conn, ws: ws, kind: kind}, nil +} + +// Returns true if the header named `name` contains a token with value `value`. +func wsHeaderContains(header http.Header, name string, value string) bool { + for _, s := range header[name] { + tokens := strings.Split(s, ",") + for _, t := range tokens { + t = strings.Trim(t, " \t") + if strings.EqualFold(t, value) { + return true + } + } + } + return false +} + +func wsPMCExtensionSupport(header http.Header, checkPMCOnly bool) (bool, bool) { + for _, extensionList := range header["Sec-Websocket-Extensions"] { + extensions := strings.Split(extensionList, ",") + for _, extension := range extensions { + extension = strings.Trim(extension, " \t") + params := strings.Split(extension, ";") + for i, p := range params { + p = strings.Trim(p, " \t") + if strings.EqualFold(p, wsPMCExtension) { + if checkPMCOnly { + return true, false + } + var snc bool + var cnc bool + for j := i + 1; j < len(params); j++ { + p = params[j] + p = strings.Trim(p, " \t") + if strings.EqualFold(p, wsPMCSrvNoCtx) { + snc = true + } else if strings.EqualFold(p, wsPMCCliNoCtx) { + cnc = true + } + if snc && cnc { + return true, true + } + } + return true, false + } + } + } + } + return false, false +} + +// Send an HTTP error with the given `status` to the given http response writer `w`. +// Return an error created based on the `reason` string. +func wsReturnHTTPError(w http.ResponseWriter, r *http.Request, status int, reason string) error { + err := fmt.Errorf("%s - websocket handshake error: %s", r.RemoteAddr, reason) + w.Header().Set("Sec-Websocket-Version", "13") + http.Error(w, http.StatusText(status), status) + return err +} + +// If the server is configured to accept any origin, then this function returns +// `nil` without checking if the Origin is present and valid. This is also +// the case if the request does not have the Origin header. +// Otherwise, this will check that the Origin matches the same origin or +// any origin in the allowed list. +func (w *srvWebsocket) checkOrigin(r *http.Request) error { + w.mu.RLock() + checkSame := w.sameOrigin + listEmpty := len(w.allowedOrigins) == 0 + w.mu.RUnlock() + if !checkSame && listEmpty { + return nil + } + origin := r.Header.Get("Origin") + if origin == _EMPTY_ { + origin = r.Header.Get("Sec-Websocket-Origin") + } + // If the header is not present, we will accept. + // From https://datatracker.ietf.org/doc/html/rfc6455#section-1.6 + // "Naturally, when the WebSocket Protocol is used by a dedicated client + // directly (i.e., not from a web page through a web browser), the origin + // model is not useful, as the client can provide any arbitrary origin string." + if origin == _EMPTY_ { + return nil + } + u, err := url.ParseRequestURI(origin) + if err != nil { + return err + } + oh, op, err := wsGetHostAndPort(u.Scheme == "https", u.Host) + if err != nil { + return err + } + // If checking same origin, compare with the http's request's Host. + if checkSame { + rh, rp, err := wsGetHostAndPort(r.TLS != nil, r.Host) + if err != nil { + return err + } + rs := "http" + if r.TLS != nil { + rs = "https" + } + if oh != rh || op != rp || !strings.EqualFold(u.Scheme, rs) { + return errors.New("not same origin") + } + // I guess it is possible to have cases where one wants to check + // same origin, but also that the origin is in the allowed list. + // So continue with the next check. + } + if !listEmpty { + w.mu.RLock() + origins := w.allowedOrigins[oh] + w.mu.RUnlock() + var allowed bool + for _, ao := range origins { + if u.Scheme == ao.scheme && op == ao.port { + allowed = true + break + } + } + if !allowed { + return errors.New("not in the allowed list") + } + } + return nil +} + +func wsGetHostAndPort(tls bool, hostport string) (string, string, error) { + host, port, err := net.SplitHostPort(hostport) + if err != nil { + // If error is missing port, then use defaults based on the scheme + if ae, ok := err.(*net.AddrError); ok && strings.Contains(ae.Err, "missing port") { + err = nil + host = hostport + if tls { + port = "443" + } else { + port = "80" + } + } + } + return strings.ToLower(host), port, err +} + +func wsMakeChallengeKey() (string, error) { + p := make([]byte, 16) + if _, err := io.ReadFull(crand.Reader, p); err != nil { + return _EMPTY_, err + } + return base64.StdEncoding.EncodeToString(p), nil +} + +// Validate the websocket related options. +func validateWebsocketOptions(o *Options) error { + wo := &o.Websocket + // If no port is defined, we don't care about other options + if wo.Port == 0 { + return nil + } + // Enforce TLS... unless NoTLS is set to true. + if wo.TLSConfig == nil && !wo.NoTLS { + return errors.New("websocket requires TLS configuration") + } + // Make sure that allowed origins, if specified, can be parsed. + for _, ao := range wo.AllowedOrigins { + u, err := url.ParseRequestURI(ao) + if err != nil { + return fmt.Errorf("unable to parse allowed origin: %v", err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("unable to parse allowed origin %q: allowed origins must be absolute URLs with http or https scheme", ao) + } + if u.Host == _EMPTY_ { + return fmt.Errorf("unable to parse allowed origin %q: host is required", ao) + } + if _, _, err := wsGetHostAndPort(u.Scheme == "https", u.Host); err != nil { + return fmt.Errorf("unable to parse allowed origin: %v", err) + } + } + // If there is a NoAuthUser, we need to have Users defined and + // the user to be present. + if wo.NoAuthUser != _EMPTY_ { + if err := validateNoAuthUser(o, wo.NoAuthUser); err != nil { + return err + } + } + // Token/Username not possible if there are users/nkeys + if len(o.Users) > 0 || len(o.Nkeys) > 0 { + if wo.Username != _EMPTY_ { + return fmt.Errorf("websocket authentication username not compatible with presence of users/nkeys") + } + if wo.Token != _EMPTY_ { + return fmt.Errorf("websocket authentication token not compatible with presence of users/nkeys") + } + } + // Using JWT requires Trusted Keys + if wo.JWTCookie != _EMPTY_ { + if len(o.TrustedOperators) == 0 && len(o.TrustedKeys) == 0 { + return fmt.Errorf("trusted operators or trusted keys configuration is required for JWT authentication via cookie %q", wo.JWTCookie) + } + } + if err := validatePinnedCerts(wo.TLSPinnedCerts); err != nil { + return fmt.Errorf("websocket: %v", err) + } + + // Check for invalid headers here. + for key := range wo.Headers { + k := strings.ToLower(key) + switch k { + case "host", + "content-length", + "connection", + "upgrade", + "nats-no-masking": + return fmt.Errorf("websocket: invalid header %q not allowed", key) + } + + if strings.HasPrefix(k, "sec-websocket-") { + return fmt.Errorf("websocket: invalid header %q, \"Sec-WebSocket-\" prefix not allowed", key) + } + } + + return nil +} + +// Creates or updates the existing map +func (s *Server) wsSetOriginOptions(o *WebsocketOpts) { + ws := &s.websocket + ws.mu.Lock() + defer ws.mu.Unlock() + // Copy over the option's same origin boolean + ws.sameOrigin = o.SameOrigin + // Reset the map. Will help for config reload if/when we support it. + ws.allowedOrigins = nil + if o.AllowedOrigins == nil { + return + } + for _, ao := range o.AllowedOrigins { + // We have previously checked (during options validation) that the urls + // are parseable, but if we get an error, report and skip. + u, err := url.ParseRequestURI(ao) + if err != nil { + s.Errorf("error parsing allowed origin: %v", err) + continue + } + h, p, _ := wsGetHostAndPort(u.Scheme == "https", u.Host) + if ws.allowedOrigins == nil { + ws.allowedOrigins = make(map[string][]*allowedOrigin, len(o.AllowedOrigins)) + } + ws.allowedOrigins[h] = append(ws.allowedOrigins[h], &allowedOrigin{scheme: u.Scheme, port: p}) + } +} + +// Calculate the raw headers for websocket upgrade response. +func (s *Server) wsSetHeadersOptions(o *WebsocketOpts) { + var sb strings.Builder + for k, v := range o.Headers { + sb.WriteString(k) + sb.WriteString(": ") + sb.WriteString(v) + sb.WriteString(_CRLF_) + } + ws := &s.websocket + ws.mu.Lock() + defer ws.mu.Unlock() + ws.rawHeaders = sb.String() +} + +// Given the websocket options, we check if any auth configuration +// has been provided. If so, possibly create users/nkey users and +// store them in s.websocket.users/nkeys. +// Also update a boolean that indicates if auth is required for +// websocket clients. +// Server lock is held on entry. +func (s *Server) wsConfigAuth(opts *WebsocketOpts) { + ws := &s.websocket + // If any of those is specified, we consider that there is an override. + ws.authOverride = opts.Username != _EMPTY_ || opts.Token != _EMPTY_ || opts.NoAuthUser != _EMPTY_ +} + +func (s *Server) startWebsocketServer() { + if s.isShuttingDown() { + return + } + + sopts := s.getOpts() + o := &sopts.Websocket + + s.wsSetOriginOptions(o) + s.wsSetHeadersOptions(o) + + var hl net.Listener + var proto string + var err error + + port := o.Port + if port == -1 { + port = 0 + } + hp := net.JoinHostPort(o.Host, strconv.Itoa(port)) + + // We are enforcing (when validating the options) the use of TLS, but the + // code was originally supporting both modes. The reason for TLS only is + // that we expect users to send JWTs with bearer tokens and we want to + // avoid the possibility of it being "intercepted". + + s.mu.Lock() + // Do not check o.NoTLS here. If a TLS configuration is available, use it, + // regardless of NoTLS. If we don't have a TLS config, it means that the + // user has configured NoTLS because otherwise the server would have failed + // to start due to options validation. + var config *tls.Config + if o.TLSConfig != nil { + proto = wsSchemePrefixTLS + config = o.TLSConfig.Clone() + config.GetConfigForClient = s.wsGetTLSConfig + } else { + proto = wsSchemePrefix + } + hl, err = natsListen("tcp", hp) + s.websocket.listenerErr = err + if err != nil { + s.mu.Unlock() + s.Fatalf("Unable to listen for websocket connections: %v", err) + return + } + if config != nil { + hl = tls.NewListener(hl, config) + } + if port == 0 { + o.Port = hl.Addr().(*net.TCPAddr).Port + } + s.Noticef("Listening for websocket clients on %s://%s:%d", proto, o.Host, o.Port) + if proto == wsSchemePrefix { + s.Warnf("Websocket not configured with TLS. DO NOT USE IN PRODUCTION!") + } + + // These 3 are immutable and will be accessed without lock by the client + // when generating/sending the INFO protocols. + s.websocket.tls = proto == wsSchemePrefixTLS + s.websocket.host, s.websocket.port = o.Host, o.Port + + // This will be updated when/if the cluster changes. + s.websocket.connectURLs, err = s.getConnectURLs(o.Advertise, o.Host, o.Port) + if err != nil { + s.Fatalf("Unable to get websocket connect URLs: %v", err) + hl.Close() + s.mu.Unlock() + return + } + hasLeaf := sopts.LeafNode.Port != 0 + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + res, err := s.wsUpgrade(w, r) + if err != nil { + s.Errorf(err.Error()) + return + } + switch res.kind { + case CLIENT: + s.createWSClient(res.conn, res.ws) + case MQTT: + s.createMQTTClient(res.conn, res.ws) + case LEAF: + if !hasLeaf { + s.Errorf("Not configured to accept leaf node connections") + // Silently close for now. If we want to send an error back, we would + // need to create the leafnode client anyway, so that is handling websocket + // frames, then send the error to the remote. + res.conn.Close() + return + } + s.createLeafNode(res.conn, nil, nil, res.ws) + } + }) + hs := &http.Server{ + Addr: hp, + Handler: mux, + ReadTimeout: o.HandshakeTimeout, + ErrorLog: log.New(&captureHTTPServerLog{s, "websocket: "}, _EMPTY_, 0), + } + s.websocket.mu.Lock() + s.websocket.server = hs + s.websocket.listener = hl + s.websocket.mu.Unlock() + go func() { + if err := hs.Serve(hl); err != http.ErrServerClosed { + s.Fatalf("websocket listener error: %v", err) + } + if s.isLameDuckMode() { + // Signal that we are not accepting new clients + s.ldmCh <- true + // Now wait for the Shutdown... + <-s.quitCh + return + } + s.done <- true + }() + s.mu.Unlock() +} + +// The TLS configuration is passed to the listener when the websocket +// "server" is setup. That prevents TLS configuration updates on reload +// from being used. By setting this function in tls.Config.GetConfigForClient +// we instruct the TLS handshake to ask for the tls configuration to be +// used for a specific client. We don't care which client, we always use +// the same TLS configuration. +func (s *Server) wsGetTLSConfig(_ *tls.ClientHelloInfo) (*tls.Config, error) { + opts := s.getOpts() + return opts.Websocket.TLSConfig, nil +} + +// This is similar to createClient() but has some modifications +// specific to handle websocket clients. +// The comments have been kept to minimum to reduce code size. +// Check createClient() for more details. +func (s *Server) createWSClient(conn net.Conn, ws *websocket) *client { + opts := s.getOpts() + + maxPay := int32(opts.MaxPayload) + maxSubs := int32(opts.MaxSubs) + if maxSubs == 0 { + maxSubs = -1 + } + now := time.Now().UTC() + + c := &client{srv: s, nc: conn, opts: defaultOpts, mpay: maxPay, msubs: maxSubs, start: now, last: now, ws: ws} + + c.registerWithAccount(s.globalAccount()) + + var info Info + var authRequired bool + + s.mu.Lock() + info = s.copyInfo() + // Check auth, override if applicable. + if !info.AuthRequired { + // Set info.AuthRequired since this is what is sent to the client. + info.AuthRequired = s.websocket.authOverride + } + if s.nonceRequired() { + var raw [nonceLen]byte + nonce := raw[:] + s.generateNonce(nonce) + info.Nonce = string(nonce) + } + c.nonce = []byte(info.Nonce) + authRequired = info.AuthRequired + + s.totalClients++ + s.mu.Unlock() + + c.mu.Lock() + if authRequired { + c.flags.set(expectConnect) + } + c.initClient() + c.Debugf("Client connection created") + c.sendProtoNow(c.generateClientInfoJSON(info, true)) + c.mu.Unlock() + + s.mu.Lock() + if !s.isRunning() || s.ldm { + if s.isShuttingDown() { + conn.Close() + } + s.mu.Unlock() + return c + } + + if opts.MaxConn < 0 || (opts.MaxConn > 0 && len(s.clients) >= opts.MaxConn) { + s.mu.Unlock() + c.maxConnExceeded() + return nil + } + s.clients[c.cid] = c + s.mu.Unlock() + + c.mu.Lock() + // Websocket clients do TLS in the websocket http server. + // So no TLS initiation here... + if _, ok := conn.(*tls.Conn); ok { + c.flags.set(handshakeComplete) + } + + if c.isClosed() { + c.mu.Unlock() + c.closeConnection(WriteError) + return nil + } + + if authRequired { + timeout := opts.AuthTimeout + // Possibly override with Websocket specific value. + if opts.Websocket.AuthTimeout != 0 { + timeout = opts.Websocket.AuthTimeout + } + c.setAuthTimer(secondsToDuration(timeout)) + } + + c.setPingTimer() + + s.startGoRoutine(func() { c.readLoop(nil) }) + s.startGoRoutine(func() { c.writeLoop() }) + + c.mu.Unlock() + + return c +} + +func (c *client) wsCollapsePtoNB() (net.Buffers, int64) { + nb := c.out.nb + var mfs int + var usz int + if c.ws.browser { + mfs = wsFrameSizeForBrowsers + } + mask := c.ws.maskwrite + // Start with possible already framed buffers (that we could have + // got from partials or control messages such as ws pings or pongs). + bufs := c.ws.frames + compress := c.ws.compress + if compress && len(nb) > 0 { + // First, make sure we don't compress for very small cumulative buffers. + for _, b := range nb { + usz += len(b) + } + if usz <= wsCompressThreshold { + compress = false + if cp := c.ws.compressor; cp != nil { + cp.Reset(nil) + } + } + } + if compress && len(nb) > 0 { + // Overwrite mfs if this connection does not support fragmented compressed frames. + if mfs > 0 && c.ws.nocompfrag { + mfs = 0 + } + seed := nbPoolGet(usz) + buf := bytes.NewBuffer(seed) + cp := c.ws.compressor + if cp == nil { + c.ws.compressor, _ = flate.NewWriter(buf, flate.BestSpeed) + cp = c.ws.compressor + } else { + cp.Reset(buf) + } + var csz int + for i, b := range nb { + for len(b) > 0 { + n, err := cp.Write(b) + if err != nil { + // Whatever this error is, it'll be handled by the cp.Flush() + // call below, as the same error will be returned there. + // Let the outer loop return all the buffers back to the pool + // and fall through naturally. + break + } + b = b[n:] + } + // Use original slice since capacity will change to zero + // in the loop after consuming the buffer, which will make + // nbPoolPut discard it. + nbPoolPut(nb[i]) + } + if err := cp.Flush(); err != nil { + c.Errorf("Error during compression: %v", err) + c.markConnAsClosed(WriteError) + cp.Reset(nil) + nbPoolPut(seed) + return nil, 0 + } + + // Recycle seed buffer already if capacity changed. + b := buf.Bytes() + if cap(b) > cap(seed) { + nbPoolPut(seed) + } + p := b[:len(b)-4] + if mfs > 0 && len(p) > mfs { + for first, final := true, false; len(p) > 0; first = false { + lp := len(p) + if lp > mfs { + lp = mfs + } else { + final = true + } + // Only the first frame should be marked as compressed, so pass + // `first` for the compressed boolean. + fh := nbPoolGet(wsMaxFrameHeaderSize)[:wsMaxFrameHeaderSize] + n, key := wsFillFrameHeader(fh, mask, first, final, first, wsBinaryMessage, lp) + if mask { + wsMaskBuf(key, p[:lp]) + } + bufs = append(bufs, fh[:n], append(nbPoolGet(lp), p[:lp]...)) + csz += n + lp + p = p[lp:] + } + nbPoolPut(b) + } else { + ol := len(p) + h, key := wsCreateFrameHeader(mask, true, wsBinaryMessage, ol) + if mask { + wsMaskBuf(key, p) + } + if ol > 0 { + bufs = append(bufs, h, p) + } + csz = len(h) + ol + } + // Make sure that the compressor no longer holds a reference to + // the bytes.Buffer, so that the underlying memory gets cleaned + // up after flushOutbound/flushAndClose. For this to be safe, we + // always cp.Reset(...) before reusing the compressor again. + cp.Reset(nil) + // Add to pb the compressed data size (including headers), but + // remove the original uncompressed data size that was added + // during the queueing. + c.out.pb += int64(csz) - int64(usz) + c.ws.fs += int64(csz) + } else if len(nb) > 0 { + var total int + if mfs > 0 { + // We are limiting the frame size. + startFrame := func() int { + bufs = append(bufs, nbPoolGet(wsMaxFrameHeaderSize)) + return len(bufs) - 1 + } + endFrame := func(idx, size int) { + bufs[idx] = bufs[idx][:wsMaxFrameHeaderSize] + n, key := wsFillFrameHeader(bufs[idx], mask, wsFirstFrame, wsFinalFrame, wsUncompressedFrame, wsBinaryMessage, size) + bufs[idx] = bufs[idx][:n] + c.out.pb += int64(n) + c.ws.fs += int64(n + size) + if mask { + wsMaskBufs(key, bufs[idx+1:]) + } + } + + fhIdx := startFrame() + for i := 0; i < len(nb); i++ { + b := nb[i] + if total+len(b) <= mfs { + buf := nbPoolGet(len(b)) + bufs = append(bufs, append(buf, b...)) + total += len(b) + nbPoolPut(nb[i]) + continue + } + for len(b) > 0 { + endStart := total != 0 + if endStart { + endFrame(fhIdx, total) + } + total = len(b) + if total >= mfs { + total = mfs + } + if endStart { + fhIdx = startFrame() + } + buf := nbPoolGet(total) + bufs = append(bufs, append(buf, b[:total]...)) + b = b[total:] + } + nbPoolPut(nb[i]) // No longer needed as copied into smaller frames. + } + if total > 0 { + endFrame(fhIdx, total) + } + } else { + // If there is no limit on the frame size, create a single frame for + // all pending buffers. + for _, b := range nb { + total += len(b) + } + wsfh, key := wsCreateFrameHeader(mask, false, wsBinaryMessage, total) + c.out.pb += int64(len(wsfh)) + bufs = append(bufs, wsfh) + idx := len(bufs) + bufs = append(bufs, nb...) + if mask { + wsMaskBufs(key, bufs[idx:]) + } + c.ws.fs += int64(len(wsfh) + total) + } + } + if len(c.ws.closeMsg) > 0 { + bufs = append(bufs, c.ws.closeMsg) + c.ws.fs += int64(len(c.ws.closeMsg)) + c.ws.closeMsg = nil + c.ws.compressor = nil + } + c.ws.frames = nil + return bufs, c.ws.fs +} + +func isWSURL(u *url.URL) bool { + return strings.HasPrefix(strings.ToLower(u.Scheme), wsSchemePrefix) +} + +func isWSSURL(u *url.URL) bool { + return strings.HasPrefix(strings.ToLower(u.Scheme), wsSchemePrefixTLS) +} diff --git a/test/accounts_cycles_test.go b/test/accounts_cycles_test.go new file mode 100644 index 0000000..5abd541 --- /dev/null +++ b/test/accounts_cycles_test.go @@ -0,0 +1,663 @@ +// Copyright 2020-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "strconv" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" +) + +func TestAccountCycleService(t *testing.T) { + conf := createConfFile(t, []byte(` + accounts { + A { + exports [ { service: help } ] + imports [ { service { subject: help, account: B } } ] + } + B { + exports [ { service: help } ] + imports [ { service { subject: help, account: A } } ] + } + } + `)) + + if _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { + t.Fatalf("Expected an error on cycle service import, got none") + } + + conf = createConfFile(t, []byte(` + accounts { + A { + exports [ { service: * } ] + imports [ { service { subject: help, account: B } } ] + } + B { + exports [ { service: help } ] + imports [ { service { subject: *, account: A } } ] + } + } + `)) + + if _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { + t.Fatalf("Expected an error on cycle service import, got none") + } + + conf = createConfFile(t, []byte(` + accounts { + A { + exports [ { service: * } ] + imports [ { service { subject: help, account: B } } ] + } + B { + exports [ { service: help } ] + imports [ { service { subject: help, account: C } } ] + } + C { + exports [ { service: * } ] + imports [ { service { subject: *, account: A } } ] + } + } + `)) + + if _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { + t.Fatalf("Expected an error on cycle service import, got none") + } +} + +func TestAccountCycleStream(t *testing.T) { + conf := createConfFile(t, []byte(` + accounts { + A { + exports [ { stream: strm } ] + imports [ { stream { subject: strm, account: B } } ] + } + B { + exports [ { stream: strm } ] + imports [ { stream { subject: strm, account: A } } ] + } + } + `)) + if _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { + t.Fatalf("Expected an error on cyclic import, got none") + } +} + +func TestAccountCycleStreamWithMapping(t *testing.T) { + conf := createConfFile(t, []byte(` + accounts { + A { + exports [ { stream: * } ] + imports [ { stream { subject: bar, account: B } } ] + } + B { + exports [ { stream: bar } ] + imports [ { stream { subject: foo, account: A }, to: bar } ] + } + } + `)) + if _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { + t.Fatalf("Expected an error on cyclic import, got none") + } +} + +func TestAccountCycleNonCycleStreamWithMapping(t *testing.T) { + conf := createConfFile(t, []byte(` + accounts { + A { + exports [ { stream: foo } ] + imports [ { stream { subject: bar, account: B } } ] + } + B { + exports [ { stream: bar } ] + imports [ { stream { subject: baz, account: C }, to: bar } ] + } + C { + exports [ { stream: baz } ] + imports [ { stream { subject: foo, account: A }, to: bar } ] + } + } + `)) + if _, err := server.ProcessConfigFile(conf); err != nil { + t.Fatalf("Expected no error but got %s", err) + } +} + +func TestAccountCycleServiceCycleWithMapping(t *testing.T) { + conf := createConfFile(t, []byte(` + accounts { + A { + exports [ { service: a } ] + imports [ { service { subject: b, account: B }, to: a } ] + } + B { + exports [ { service: b } ] + imports [ { service { subject: a, account: A }, to: b } ] + } + } + `)) + if _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { + t.Fatalf("Expected an error on cycle service import, got none") + } +} + +func TestAccountCycleServiceNonCycle(t *testing.T) { + conf := createConfFile(t, []byte(` + accounts { + A { + exports [ { service: * } ] + imports [ { service { subject: help, account: B } } ] + } + B { + exports [ { service: help } ] + imports [ { service { subject: nohelp, account: C } } ] + } + C { + exports [ { service: * } ] + imports [ { service { subject: *, account: A } } ] + } + } + `)) + + if _, err := server.ProcessConfigFile(conf); err != nil { + t.Fatalf("Expected no error but got %s", err) + } +} + +func TestAccountCycleServiceNonCycleChain(t *testing.T) { + conf := createConfFile(t, []byte(` + accounts { + A { + exports [ { service: help } ] + imports [ { service { subject: help, account: B } } ] + } + B { + exports [ { service: help } ] + imports [ { service { subject: help, account: C } } ] + } + C { + exports [ { service: help } ] + imports [ { service { subject: help, account: D } } ] + } + D { + exports [ { service: help } ] + } + } + `)) + + if _, err := server.ProcessConfigFile(conf); err != nil { + t.Fatalf("Expected no error but got %s", err) + } +} + +// bug: https://github.com/nats-io/nats-server/issues/1769 +func TestServiceImportReplyMatchCycle(t *testing.T) { + conf := createConfFile(t, []byte(` + port: -1 + accounts { + A { + users: [{user: d, pass: x}] + imports [ {service: {account: B, subject: ">" }}] + } + B { + users: [{user: x, pass: x}] + exports [ { service: ">" } ] + } + } + no_auth_user: d + `)) + + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + nc1 := clientConnectToServerWithUP(t, opts, "x", "x") + defer nc1.Close() + + msg := []byte("HELLO") + _, err := nc1.Subscribe("foo", func(m *nats.Msg) { + m.Respond(msg) + }) + if err != nil { + t.Fatal(err) + } + + // Ensure the subscription is known by the server we're connected to. + err = nc1.Flush() + if err != nil { + t.Fatal(err) + } + checkSubInterest(t, s, "B", "foo", time.Second) + + nc2 := clientConnectToServer(t, s) + defer nc2.Close() + + resp, err := nc2.Request("foo", nil, time.Second) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if resp == nil || string(resp.Data) != string(msg) { + t.Fatalf("Wrong or empty response") + } +} + +func TestServiceImportReplyMatchCycleMultiHops(t *testing.T) { + conf := createConfFile(t, []byte(` + port: -1 + accounts { + A { + users: [{user: d, pass: x}] + imports [ {service: {account: B, subject: ">" }}] + } + B { + exports [ { service: ">" } ] + imports [ {service: {account: C, subject: ">" }}] + } + C { + users: [{user: x, pass: x}] + exports [ { service: ">" } ] + } + } + no_auth_user: d + `)) + + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + nc1 := clientConnectToServerWithUP(t, opts, "x", "x") + defer nc1.Close() + + msg := []byte("HELLO") + _, err := nc1.Subscribe("foo", func(m *nats.Msg) { + m.Respond(msg) + }) + if err != nil { + t.Fatal(err) + } + + // Ensure the subscription is known by the server we're connected to. + err = nc1.Flush() + if err != nil { + t.Fatal(err) + } + checkSubInterest(t, s, "B", "foo", time.Second) + + nc2 := clientConnectToServer(t, s) + defer nc2.Close() + + resp, err := nc2.Request("foo", nil, time.Second) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if resp == nil || string(resp.Data) != string(msg) { + t.Fatalf("Wrong or empty response") + } +} + +// Go's stack are infinite sans memory, but not call depth. However its good to limit. +func TestAccountCycleDepthLimit(t *testing.T) { + var last *server.Account + chainLen := server.MaxAccountCycleSearchDepth + 1 + + // Services + for i := 1; i <= chainLen; i++ { + acc := server.NewAccount(fmt.Sprintf("ACC-%d", i)) + if err := acc.AddServiceExport("*", nil); err != nil { + t.Fatalf("Error adding service export to '*': %v", err) + } + if last != nil { + err := acc.AddServiceImport(last, "foo", "foo") + switch i { + case chainLen: + if err != server.ErrCycleSearchDepth { + t.Fatalf("Expected last import to fail with '%v', but got '%v'", server.ErrCycleSearchDepth, err) + } + default: + if err != nil { + t.Fatalf("Error adding service import to 'foo': %v", err) + } + } + } + last = acc + } + + last = nil + + // Streams + for i := 1; i <= chainLen; i++ { + acc := server.NewAccount(fmt.Sprintf("ACC-%d", i)) + if err := acc.AddStreamExport("foo", nil); err != nil { + t.Fatalf("Error adding stream export to '*': %v", err) + } + if last != nil { + err := acc.AddStreamImport(last, "foo", "") + switch i { + case chainLen: + if err != server.ErrCycleSearchDepth { + t.Fatalf("Expected last import to fail with '%v', but got '%v'", server.ErrCycleSearchDepth, err) + } + default: + if err != nil { + t.Fatalf("Error adding stream import to 'foo': %v", err) + } + } + } + last = acc + } +} + +// Test token and partition subject mapping within an account +func TestAccountSubjectMapping(t *testing.T) { + conf := createConfFile(t, []byte(` + port: -1 + mappings = { + "foo.*.*" : "foo.$1.{{wildcard(2)}}.{{partition(10,1,2)}}" + } + `)) + + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + nc1 := clientConnectToServer(t, s) + defer nc1.Close() + + numMessages := 100 + subjectsReceived := make(chan string) + + msg := []byte("HELLO") + sub1, err := nc1.Subscribe("foo.*.*.*", func(m *nats.Msg) { + subjectsReceived <- m.Subject + }) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + sub1.AutoUnsubscribe(numMessages * 2) + nc1.Flush() + + nc2 := clientConnectToServer(t, s) + defer nc2.Close() + + // publish numMessages with an increasing id (should map to partition numbers with the range of 10 partitions) - twice + for j := 0; j < 2; j++ { + for i := 0; i < numMessages; i++ { + err = nc2.Publish(fmt.Sprintf("foo.%d.%d", i, numMessages-i), msg) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + } + } + + // verify all the partition numbers are in the expected range + partitionsReceived := make([]int, numMessages) + + for i := 0; i < numMessages; i++ { + var subject string + select { + case subject = <-subjectsReceived: + case <-time.After(5 * time.Second): + t.Fatal("Timed out waiting for messages") + } + sTokens := strings.Split(subject, ".") + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + t1, _ := strconv.Atoi(sTokens[1]) + t2, _ := strconv.Atoi(sTokens[2]) + partitionsReceived[i], err = strconv.Atoi(sTokens[3]) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + if partitionsReceived[i] > 9 || partitionsReceived[i] < 0 || t1 != i || t2 != numMessages-i { + t.Fatalf("Error received unexpected %d.%d to partition %d", t1, t2, partitionsReceived[i]) + } + } + + // verify hashing is deterministic by checking it produces the same exact result twice + for i := 0; i < numMessages; i++ { + subject := <-subjectsReceived + partitionNumber, err := strconv.Atoi(strings.Split(subject, ".")[3]) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if partitionsReceived[i] != partitionNumber { + t.Fatalf("Error: same id mapped to two different partitions") + } + } +} + +// test token subject mapping within an account +// Alice imports from Bob with subject mapping +func TestAccountImportSubjectMapping(t *testing.T) { + conf := createConfFile(t, []byte(` + port: -1 + accounts { + A { + users: [{user: a, pass: x}] + imports [ {stream: {account: B, subject: "foo.*.*"}, to : "foo.$1.{{wildcard(2)}}"}] + } + B { + users: [{user: b, pass x}] + exports [ { stream: ">" } ] + } + } + `)) + + s, opts := RunServerWithConfig(conf) + + defer s.Shutdown() + ncA := clientConnectToServerWithUP(t, opts, "a", "x") + defer ncA.Close() + + numMessages := 100 + subjectsReceived := make(chan string) + + msg := []byte("HELLO") + sub1, err := ncA.Subscribe("foo.*.*", func(m *nats.Msg) { + subjectsReceived <- m.Subject + }) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + sub1.AutoUnsubscribe(numMessages) + ncA.Flush() + + ncB := clientConnectToServerWithUP(t, opts, "b", "x") + defer ncB.Close() + + // publish numMessages with an increasing id + + for i := 0; i < numMessages; i++ { + err = ncB.Publish(fmt.Sprintf("foo.%d.%d", i, numMessages-i), msg) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + } + + for i := 0; i < numMessages; i++ { + var subject string + select { + case subject = <-subjectsReceived: + case <-time.After(1 * time.Second): + t.Fatal("Timed out waiting for messages") + } + sTokens := strings.Split(subject, ".") + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + t1, _ := strconv.Atoi(sTokens[1]) + t2, _ := strconv.Atoi(sTokens[2]) + + if t1 != i || t2 != numMessages-i { + t.Fatalf("Error received unexpected %d.%d", t1, t2) + } + } +} + +// bug: https://github.com/nats-io/nats-server/issues/1789 +func TestAccountCycleWithRenaming(t *testing.T) { + conf := createConfFile(t, []byte(` + accounts { + A { + exports [ { service: * } ] + imports [ { service { subject: foo, account: B }, to: foo } ] + } + B { + exports [ { service: foo } ] + imports [ { service { subject: *, account: A }, to: "$1" } ] + } + } + `)) + if _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { + t.Fatalf("Expected an error on cycle service import, got none") + } + + conf = createConfFile(t, []byte(` + accounts { + A { + exports [ { stream: * } ] + imports [ { stream { subject: foo, account: B }, to: foo } ] + } + B { + exports [ { stream: foo } ] + imports [ { stream { subject: *, account: A }, to: "$1" } ] + } + } + `)) + if _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { + t.Fatalf("Expected an error on cycle service import, got none") + } +} + +// https://github.com/nats-io/nats-server/issues/5752 +func TestAccountCycleFalsePositiveSubjectMapping(t *testing.T) { + conf := createConfFile(t, []byte(` + accounts { + A { + exports [ { service: "a.*" } ] + imports [ { service { subject: "a.*", account: B }, to: "b.*" } ] + } + B { + exports [ { service: "a.*" } ] + imports [ { service { subject: "a.foo", account: A }, to: "c.foo" } ] + } + } + `)) + if _, err := server.ProcessConfigFile(conf); err != nil { + t.Fatalf("Expected no errors on cycle service import, got error") + } +} + +// TestAccountCycleServiceMissedDueToFromMutation reproduces a bug where +// checkServiceImportsForCycles mutates the `from` loop variable via +// `from = si.from`, affecting subsequent map iterations non-deterministically. +// +// Cycle: E -> A -> D -> E +// A also has a non-cyclic import from C. When C's map key is iterated first, +// `from` is narrowed from "*" to "narrow", causing D's import to be skipped +// (SubjectsCollide("narrow","other") is false), and the cycle is missed. +func TestAccountCycleServiceMissedDueToFromMutation(t *testing.T) { + // Run multiple times because Go map iteration order is non-deterministic. + // The bug only manifests when the non-cyclic import is visited first. + for i := 0; i < 100; i++ { + conf := createConfFile(t, []byte(` + accounts { + A { + exports [ { service: * } ] + imports [ + { service { subject: narrow, account: C } } + { service { subject: other, account: D } } + ] + } + C { + exports [ { service: narrow } ] + } + D { + exports [ { service: other } ] + imports [ { service { subject: other, account: E } } ] + } + E { + exports [ { service: * } ] + imports [ { service { subject: *, account: A } } ] + } + } + `)) + _, err := server.ProcessConfigFile(conf) + if err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { + t.Fatalf("Iteration %d: Expected cycle detection error for E->A->D->E, got: %v", i, err) + } + } +} + +// TestAccountCycleStreamMissedDueToToMutation reproduces a bug where the `to` +// parameter is mutated inside the loop in checkStreamImportsForCycles. +// Stream imports use a slice (deterministic order), so listing the narrowing +// import first guarantees the bug triggers every time. +// +// Cycle: E → A → D → E (via subject "other") +// A also imports "narrow" from C (leaf, no cycle) declared first. +// After processing the first import, `to` is narrowed from "*" to "narrow", +// causing SubjectsCollide("narrow","other") to return false and miss the cycle. +func TestAccountCycleStreamMissedDueToToMutation(t *testing.T) { + conf := createConfFile(t, []byte(` + accounts { + A { + exports [ { stream: * } ] + imports [ + { stream { subject: narrow, account: C }, to: narrow } + { stream { subject: other, account: D }, to: other } + ] + } + C { + exports [ { stream: narrow } ] + } + D { + exports [ { stream: other } ] + imports [ { stream { subject: other, account: E }, to: other } ] + } + E { + exports [ { stream: * } ] + imports [ { stream { subject: *, account: A } } ] + } + } + `)) + if _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { + t.Fatalf("Expected cycle detection error for E->A->D->E, got: %v", err) + } +} + +func clientConnectToServer(t *testing.T, s *server.Server) *nats.Conn { + t.Helper() + nc, err := nats.Connect(s.ClientURL(), + nats.Name("JS-TEST"), + nats.ReconnectWait(5*time.Millisecond), + nats.MaxReconnects(-1)) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + return nc +} + +func clientConnectToServerWithUP(t *testing.T, opts *server.Options, user, pass string) *nats.Conn { + curl := fmt.Sprintf("nats://%s:%s@%s:%d", user, pass, opts.Host, opts.Port) + nc, err := nats.Connect(curl, nats.Name("JS-UP-TEST"), nats.ReconnectWait(5*time.Millisecond), nats.MaxReconnects(-1)) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + return nc +} diff --git a/test/auth_test.go b/test/auth_test.go new file mode 100644 index 0000000..0f60f11 --- /dev/null +++ b/test/auth_test.go @@ -0,0 +1,236 @@ +// Copyright 2012-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "encoding/json" + "fmt" + "net" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" +) + +func doAuthConnect(t tLogger, c net.Conn, token, user, pass string) { + cs := fmt.Sprintf("CONNECT {\"verbose\":true,\"auth_token\":\"%s\",\"user\":\"%s\",\"pass\":\"%s\"}\r\n", token, user, pass) + sendProto(t, c, cs) +} + +func testInfoForAuth(t tLogger, infojs []byte) bool { + var sinfo server.Info + err := json.Unmarshal(infojs, &sinfo) + if err != nil { + t.Fatalf("Could not unmarshal INFO json: %v\n", err) + } + return sinfo.AuthRequired +} + +func expectAuthRequired(t tLogger, c net.Conn) { + buf := expectResult(t, c, infoRe) + infojs := infoRe.FindAllSubmatch(buf, 1)[0][1] + if !testInfoForAuth(t, infojs) { + t.Fatalf("Expected server to require authorization: '%s'", infojs) + } +} + +//////////////////////////////////////////////////////////// +// The authorization token version +//////////////////////////////////////////////////////////// + +const AUTH_PORT = 10422 +const AUTH_TOKEN = "_YZZ22_" + +func runAuthServerWithToken() *server.Server { + opts := DefaultTestOptions + opts.Port = AUTH_PORT + opts.Authorization = AUTH_TOKEN + return RunServer(&opts) +} + +func TestNoAuthClient(t *testing.T) { + s := runAuthServerWithToken() + defer s.Shutdown() + c := createClientConn(t, "127.0.0.1", AUTH_PORT) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, "", "", "") + expectResult(t, c, errRe) +} + +func TestAuthClientBadToken(t *testing.T) { + s := runAuthServerWithToken() + defer s.Shutdown() + c := createClientConn(t, "127.0.0.1", AUTH_PORT) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, "ZZZ", "", "") + expectResult(t, c, errRe) +} + +func TestAuthClientNoConnect(t *testing.T) { + s := runAuthServerWithToken() + defer s.Shutdown() + c := createClientConn(t, "127.0.0.1", AUTH_PORT) + defer c.Close() + expectAuthRequired(t, c) + // This is timing dependent.. + time.Sleep(server.AUTH_TIMEOUT) + expectResult(t, c, errRe) +} + +func TestAuthClientGoodConnect(t *testing.T) { + s := runAuthServerWithToken() + defer s.Shutdown() + c := createClientConn(t, "127.0.0.1", AUTH_PORT) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, AUTH_TOKEN, "", "") + expectResult(t, c, okRe) +} + +func TestAuthClientFailOnEverythingElse(t *testing.T) { + s := runAuthServerWithToken() + defer s.Shutdown() + c := createClientConn(t, "127.0.0.1", AUTH_PORT) + defer c.Close() + expectAuthRequired(t, c) + sendProto(t, c, "PUB foo 2\r\nok\r\n") + expectResult(t, c, errRe) +} + +//////////////////////////////////////////////////////////// +// The username/password version +//////////////////////////////////////////////////////////// + +const AUTH_USER = "derek" +const AUTH_PASS = "foobar" + +func runAuthServerWithUserPass() *server.Server { + opts := DefaultTestOptions + opts.Port = AUTH_PORT + opts.Username = AUTH_USER + opts.Password = AUTH_PASS + return RunServer(&opts) +} + +func TestNoUserOrPasswordClient(t *testing.T) { + s := runAuthServerWithUserPass() + defer s.Shutdown() + c := createClientConn(t, "127.0.0.1", AUTH_PORT) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, "", "", "") + expectResult(t, c, errRe) +} + +func TestBadUserClient(t *testing.T) { + s := runAuthServerWithUserPass() + defer s.Shutdown() + c := createClientConn(t, "127.0.0.1", AUTH_PORT) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, "", "derekzz", AUTH_PASS) + expectResult(t, c, errRe) +} + +func TestBadPasswordClient(t *testing.T) { + s := runAuthServerWithUserPass() + defer s.Shutdown() + c := createClientConn(t, "127.0.0.1", AUTH_PORT) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, "", AUTH_USER, "ZZ") + expectResult(t, c, errRe) +} + +func TestPasswordClientGoodConnect(t *testing.T) { + s := runAuthServerWithUserPass() + defer s.Shutdown() + c := createClientConn(t, "127.0.0.1", AUTH_PORT) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, "", AUTH_USER, AUTH_PASS) + expectResult(t, c, okRe) +} + +//////////////////////////////////////////////////////////// +// The bcrypt username/password version +//////////////////////////////////////////////////////////// + +// Generated with nats server passwd (Cost 4 because of cost of --race, default is 11) +const BCRYPT_AUTH_PASS = "IW@$6v(y1(t@fhPDvf!5^%" +const BCRYPT_AUTH_HASH = "$2a$04$Q.CgCP2Sl9pkcTXEZHazaeMwPaAkSHk7AI51HkyMt5iJQQyUA4qxq" + +func runAuthServerWithBcryptUserPass() *server.Server { + opts := DefaultTestOptions + opts.Port = AUTH_PORT + opts.Username = AUTH_USER + opts.Password = BCRYPT_AUTH_HASH + return RunServer(&opts) +} + +func TestBadBcryptPassword(t *testing.T) { + s := runAuthServerWithBcryptUserPass() + defer s.Shutdown() + c := createClientConn(t, "127.0.0.1", AUTH_PORT) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, "", AUTH_USER, BCRYPT_AUTH_HASH) + expectResult(t, c, errRe) +} + +func TestGoodBcryptPassword(t *testing.T) { + s := runAuthServerWithBcryptUserPass() + defer s.Shutdown() + c := createClientConn(t, "127.0.0.1", AUTH_PORT) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, "", AUTH_USER, BCRYPT_AUTH_PASS) + expectResult(t, c, okRe) +} + +//////////////////////////////////////////////////////////// +// The bcrypt authorization token version +//////////////////////////////////////////////////////////// + +const BCRYPT_AUTH_TOKEN = "0uhJOSr3GW7xvHvtd^K6pa" +const BCRYPT_AUTH_TOKEN_HASH = "$2a$04$u5ZClXpcjHgpfc61Ee0VKuwI1K3vTC4zq7SjphjnlHMeb1Llkb5Y6" + +func runAuthServerWithBcryptToken() *server.Server { + opts := DefaultTestOptions + opts.Port = AUTH_PORT + opts.Authorization = BCRYPT_AUTH_TOKEN_HASH + return RunServer(&opts) +} + +func TestBadBcryptToken(t *testing.T) { + s := runAuthServerWithBcryptToken() + defer s.Shutdown() + c := createClientConn(t, "127.0.0.1", AUTH_PORT) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, BCRYPT_AUTH_TOKEN_HASH, "", "") + expectResult(t, c, errRe) +} + +func TestGoodBcryptToken(t *testing.T) { + s := runAuthServerWithBcryptToken() + defer s.Shutdown() + c := createClientConn(t, "127.0.0.1", AUTH_PORT) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, BCRYPT_AUTH_TOKEN, "", "") + expectResult(t, c, okRe) +} diff --git a/test/bench_results.txt b/test/bench_results.txt new file mode 100644 index 0000000..89850ca --- /dev/null +++ b/test/bench_results.txt @@ -0,0 +1,79 @@ +2017 iMac Pro 3Ghz (Turbo 4Ghz) 10-Core Skylake +OSX High Sierra 10.13.2 + +=================== +Go version go1.9.2 +=================== + +Benchmark_____Pub0b_Payload-20 30000000 55.1 ns/op 199.78 MB/s +Benchmark_____Pub8b_Payload-20 30000000 55.8 ns/op 340.21 MB/s +Benchmark____Pub32b_Payload-20 20000000 63.4 ns/op 694.34 MB/s +Benchmark___Pub128B_Payload-20 20000000 79.8 ns/op 1766.47 MB/s +Benchmark___Pub256B_Payload-20 20000000 98.1 ns/op 2741.51 MB/s +Benchmark_____Pub1K_Payload-20 5000000 283 ns/op 3660.72 MB/s +Benchmark_____Pub4K_Payload-20 1000000 1395 ns/op 2945.30 MB/s +Benchmark_____Pub8K_Payload-20 500000 2846 ns/op 2882.35 MB/s +Benchmark_AuthPub0b_Payload-20 10000000 126 ns/op 86.82 MB/s +Benchmark____________PubSub-20 10000000 135 ns/op +Benchmark____PubSubTwoConns-20 10000000 136 ns/op +Benchmark____PubTwoQueueSub-20 10000000 152 ns/op +Benchmark___PubFourQueueSub-20 10000000 152 ns/op +Benchmark__PubEightQueueSub-20 10000000 152 ns/op +Benchmark___RoutedPubSub_0b-20 5000000 385 ns/op +Benchmark___RoutedPubSub_1K-20 1000000 1076 ns/op +Benchmark_RoutedPubSub_100K-20 20000 78501 ns/op + + +2015 iMac5k 4Ghz i7 Haswell +OSX El Capitan 10.11.3 + +=================== +Go version go1.6 +=================== + +Benchmark____PubNo_Payload-8 20000000 88.6 ns/op 124.11 MB/s +Benchmark____Pub8b_Payload-8 20000000 89.8 ns/op 211.63 MB/s +Benchmark___Pub32b_Payload-8 20000000 97.3 ns/op 452.20 MB/s +Benchmark__Pub256B_Payload-8 10000000 129 ns/op 2078.43 MB/s +Benchmark____Pub1K_Payload-8 5000000 216 ns/op 4791.00 MB/s +Benchmark____Pub4K_Payload-8 1000000 1123 ns/op 3657.53 MB/s +Benchmark____Pub8K_Payload-8 500000 2309 ns/op 3553.09 MB/s +Benchmark___________PubSub-8 10000000 210 ns/op +Benchmark___PubSubTwoConns-8 10000000 205 ns/op +Benchmark___PubTwoQueueSub-8 10000000 231 ns/op +Benchmark__PubFourQueueSub-8 10000000 233 ns/op +Benchmark_PubEightQueueSub-8 5000000 231 ns/op + +OSX Yosemite 10.10.5 + +=================== +Go version go1.4.2 +=================== + +Benchmark___PubNo_Payload 10000000 133 ns/op 82.44 MB/s +Benchmark___Pub8b_Payload 10000000 135 ns/op 140.27 MB/s +Benchmark__Pub32b_Payload 10000000 147 ns/op 297.56 MB/s +Benchmark_Pub256B_Payload 10000000 211 ns/op 1273.82 MB/s +Benchmark___Pub1K_Payload 3000000 447 ns/op 2321.55 MB/s +Benchmark___Pub4K_Payload 1000000 1677 ns/op 2450.43 MB/s +Benchmark___Pub8K_Payload 300000 3670 ns/op 2235.80 MB/s +Benchmark__________PubSub 5000000 263 ns/op +Benchmark__PubSubTwoConns 5000000 268 ns/op +Benchmark__PubTwoQueueSub 2000000 936 ns/op +Benchmark_PubFourQueueSub 1000000 1103 ns/op + +=================== +Go version go1.5.0 +=================== + +Benchmark___PubNo_Payload-8 10000000 122 ns/op 89.94 MB/s +Benchmark___Pub8b_Payload-8 10000000 124 ns/op 152.72 MB/s +Benchmark__Pub32b_Payload-8 10000000 135 ns/op 325.73 MB/s +Benchmark_Pub256B_Payload-8 10000000 159 ns/op 1685.78 MB/s +Benchmark___Pub1K_Payload-8 5000000 256 ns/op 4047.90 MB/s +Benchmark___Pub4K_Payload-8 1000000 1164 ns/op 3530.77 MB/s +Benchmark___Pub8K_Payload-8 500000 2444 ns/op 3357.34 MB/s +Benchmark__________PubSub-8 5000000 254 ns/op +Benchmark__PubSubTwoConns-8 5000000 245 ns/op +Benchmark__PubTwoQueueSub-8 2000000 845 ns/op +Benchmark_PubFourQueueSub-8 1000000 1004 ns/op diff --git a/test/bench_test.go b/test/bench_test.go new file mode 100644 index 0000000..d40d4d8 --- /dev/null +++ b/test/bench_test.go @@ -0,0 +1,1569 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Please note that these tests will stress a system and they need generous +// amounts of CPU, Memory and network sockets. Make sure the 'open files' +// setting for your platform is at least 8192. On linux and MacOSX you can +// do this via 'ulimit -n 8192' +// + +package test + +import ( + "bufio" + "fmt" + "math/rand/v2" + "net" + "net/url" + "sync" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" +) + +const PERF_PORT = 8422 + +// For Go routine based server. +func runBenchServer() *server.Server { + opts := DefaultTestOptions + opts.Port = PERF_PORT + opts.DisableShortFirstPing = true + return RunServer(&opts) +} + +const defaultRecBufSize = 32768 +const defaultSendBufSize = 32768 + +func flushConnection(b *testing.B, c net.Conn) { + buf := make([]byte, 32) + c.Write([]byte("PING\r\n")) + c.SetReadDeadline(time.Now().Add(5 * time.Second)) + n, err := c.Read(buf) + c.SetReadDeadline(time.Time{}) + if err != nil { + b.Fatalf("Failed read: %v\n", err) + } + if n != 6 && buf[0] != 'P' && buf[1] != 'O' { + b.Fatalf("Failed read of PONG: %s\n", buf) + } +} + +func benchPub(b *testing.B, subject, payload string) { + b.StopTimer() + s := runBenchServer() + c := createClientConn(b, "127.0.0.1", PERF_PORT) + doDefaultConnect(b, c) + bw := bufio.NewWriterSize(c, defaultSendBufSize) + sendOp := []byte(fmt.Sprintf("PUB %s %d\r\n%s\r\n", subject, len(payload), payload)) + b.SetBytes(int64(len(sendOp))) + b.StartTimer() + for i := 0; i < b.N; i++ { + bw.Write(sendOp) + } + bw.Flush() + flushConnection(b, c) + b.StopTimer() + c.Close() + s.Shutdown() +} + +var ch = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@$#%^&*()") + +func sizedBytes(sz int) []byte { + b := make([]byte, sz) + for i := range b { + b[i] = ch[rand.IntN(len(ch))] + } + return b +} + +func sizedString(sz int) string { + return string(sizedBytes(sz)) +} + +// Publish subject for pub benchmarks. +var psub = "a" + +func Benchmark______Pub0b_Payload(b *testing.B) { + benchPub(b, psub, "") +} + +func Benchmark______Pub8b_Payload(b *testing.B) { + b.StopTimer() + s := sizedString(8) + benchPub(b, psub, s) +} + +func Benchmark_____Pub32b_Payload(b *testing.B) { + b.StopTimer() + s := sizedString(32) + benchPub(b, psub, s) +} + +func Benchmark____Pub128B_Payload(b *testing.B) { + b.StopTimer() + s := sizedString(128) + benchPub(b, psub, s) +} + +func Benchmark____Pub256B_Payload(b *testing.B) { + b.StopTimer() + s := sizedString(256) + benchPub(b, psub, s) +} + +func Benchmark______Pub1K_Payload(b *testing.B) { + b.StopTimer() + s := sizedString(1024) + benchPub(b, psub, s) +} + +func Benchmark______Pub4K_Payload(b *testing.B) { + b.StopTimer() + s := sizedString(4 * 1024) + benchPub(b, psub, s) +} + +func Benchmark______Pub8K_Payload(b *testing.B) { + b.StopTimer() + s := sizedString(8 * 1024) + benchPub(b, psub, s) +} + +func Benchmark_____Pub32K_Payload(b *testing.B) { + b.StopTimer() + s := sizedString(32 * 1024) + benchPub(b, psub, s) +} + +func drainConnection(b *testing.B, c net.Conn, ch chan bool, expected int) { + drainConnectionWithFlowControl(b, c, ch, expected, nil, 0) +} + +func drainConnectionAfter(b *testing.B, c net.Conn, ch chan bool, expected int, deadlineAfter <-chan struct{}) { + drainConnectionWithFlowControlAfter(b, c, ch, expected, nil, 0, deadlineAfter) +} + +func drainConnectionWithFlowControl(b *testing.B, c net.Conn, ch chan bool, expected int, credits chan struct{}, creditBytes int) { + drainConnectionWithFlowControlAfter(b, c, ch, expected, credits, creditBytes, nil) +} + +func drainConnectionWithFlowControlAfter(b *testing.B, c net.Conn, ch chan bool, expected int, credits chan struct{}, creditBytes int, deadlineAfter <-chan struct{}) { + buf := make([]byte, defaultRecBufSize) + bytes, pendingCreditBytes := 0, 0 + deadlineActive := deadlineAfter == nil + if !deadlineActive { + go func() { + <-deadlineAfter + c.SetReadDeadline(time.Now().Add(30 * time.Second)) + }() + } + + for { + if !deadlineActive { + select { + case <-deadlineAfter: + deadlineActive = true + default: + } + } + if deadlineActive { + c.SetReadDeadline(time.Now().Add(30 * time.Second)) + } + n, err := c.Read(buf) + if err != nil { + b.Errorf("Error on read: %v\n", err) + break + } + bytes += n + if credits != nil { + pendingCreditBytes += n + for pendingCreditBytes >= creditBytes { + credits <- struct{}{} + pendingCreditBytes -= creditBytes + } + } + if bytes >= expected { + break + } + } + if bytes != expected { + b.Errorf("Did not receive all bytes: %d vs %d\n", bytes, expected) + } + ch <- true +} + +// Benchmark the authorization code path. +func Benchmark__AuthPub0b_Payload(b *testing.B) { + b.StopTimer() + + srv, opts := RunServerWithConfig("./configs/authorization.conf") + opts.DisableShortFirstPing = true + defer srv.Shutdown() + + c := createClientConn(b, opts.Host, opts.Port) + defer c.Close() + expectAuthRequired(b, c) + + cs := fmt.Sprintf("CONNECT {\"verbose\":false,\"user\":\"%s\",\"pass\":\"%s\"}\r\n", "bench", DefaultPass) + sendProto(b, c, cs) + + bw := bufio.NewWriterSize(c, defaultSendBufSize) + sendOp := []byte("PUB a 0\r\n\r\n") + b.SetBytes(int64(len(sendOp))) + b.StartTimer() + for i := 0; i < b.N; i++ { + bw.Write(sendOp) + } + bw.Flush() + flushConnection(b, c) + b.StopTimer() +} + +func Benchmark_____________PubSub(b *testing.B) { + b.StopTimer() + s := runBenchServer() + c := createClientConn(b, "127.0.0.1", PERF_PORT) + doDefaultConnect(b, c) + sendProto(b, c, "SUB foo 1\r\n") + bw := bufio.NewWriterSize(c, defaultSendBufSize) + sendOp := []byte("PUB foo 2\r\nok\r\n") + ch := make(chan bool) + expected := len("MSG foo 1 2\r\nok\r\n") * b.N + go drainConnection(b, c, ch, expected) + b.StartTimer() + + for i := 0; i < b.N; i++ { + _, err := bw.Write(sendOp) + if err != nil { + b.Errorf("Received error on PUB write: %v\n", err) + } + } + err := bw.Flush() + if err != nil { + b.Errorf("Received error on FLUSH write: %v\n", err) + } + + // Wait for connection to be drained + <-ch + + b.StopTimer() + c.Close() + s.Shutdown() +} + +func Benchmark_____PubSubTwoConns(b *testing.B) { + b.StopTimer() + s := runBenchServer() + c := createClientConn(b, "127.0.0.1", PERF_PORT) + doDefaultConnect(b, c) + bw := bufio.NewWriterSize(c, defaultSendBufSize) + + c2 := createClientConn(b, "127.0.0.1", PERF_PORT) + doDefaultConnect(b, c2) + sendProto(b, c2, "SUB foo 1\r\n") + flushConnection(b, c2) + + sendOp := []byte("PUB foo 2\r\nok\r\n") + ch := make(chan bool) + + expected := len("MSG foo 1 2\r\nok\r\n") * b.N + go drainConnection(b, c2, ch, expected) + + b.StartTimer() + for i := 0; i < b.N; i++ { + bw.Write(sendOp) + } + err := bw.Flush() + if err != nil { + b.Errorf("Received error on FLUSH write: %v\n", err) + } + + // Wait for connection to be drained + <-ch + + b.StopTimer() + c.Close() + c2.Close() + s.Shutdown() +} + +func benchDefaultOptionsForAccounts() *server.Options { + o := DefaultTestOptions + o.Host = "127.0.0.1" + o.Port = -1 + o.Cluster.Host = o.Host + o.Cluster.Port = -1 + o.DisableShortFirstPing = true + fooAcc := server.NewAccount("$foo") + fooAcc.AddStreamExport("foo", nil) + barAcc := server.NewAccount("$bar") + barAcc.AddStreamImport(fooAcc, "foo", "") + o.Accounts = []*server.Account{fooAcc, barAcc} + o.Users = []*server.User{ + {Username: "$foo", Password: DefaultPass, Account: fooAcc}, + {Username: "$bar", Password: DefaultPass, Account: barAcc}, + } + + return &o +} + +func createClientWithAccount(b *testing.B, account, host string, port int) net.Conn { + c := createClientConn(b, host, port) + checkInfoMsg(b, c) + cs := fmt.Sprintf("CONNECT {\"verbose\":%v,\"pedantic\":%v,\"tls_required\":%v,\"user\":%q,\"pass\":%q}\r\n", false, false, false, account, DefaultPass) + sendProto(b, c, cs) + flushConnection(b, c) + return c +} + +func benchOptionsForServiceImports() *server.Options { + o := DefaultTestOptions + o.Host = "127.0.0.1" + o.Port = -1 + o.DisableShortFirstPing = true + foo := server.NewAccount("$foo") + bar := server.NewAccount("$bar") + o.Accounts = []*server.Account{foo, bar} + o.Users = []*server.User{ + {Username: "$foo", Password: DefaultPass, Account: foo}, + {Username: "$bar", Password: DefaultPass, Account: bar}, + } + + return &o +} + +func addServiceImports(b *testing.B, s *server.Server) { + // Add a bunch of service exports with wildcards, similar to JS. + var exports = []string{ + server.JSApiAccountInfo, + server.JSApiStreamCreate, + server.JSApiStreamUpdate, + server.JSApiStreams, + server.JSApiStreamInfo, + server.JSApiStreamDelete, + server.JSApiStreamPurge, + server.JSApiMsgDelete, + server.JSApiConsumerCreate, + server.JSApiConsumers, + server.JSApiConsumerInfo, + server.JSApiConsumerDelete, + } + foo, _ := s.LookupAccount("$foo") + bar, _ := s.LookupAccount("$bar") + + for _, export := range exports { + if err := bar.AddServiceExport(export, nil); err != nil { + b.Fatalf("Could not add service export: %v", err) + } + if err := foo.AddServiceImport(bar, export, ""); err != nil { + b.Fatalf("Could not add service import: %v", err) + } + } +} + +func Benchmark__PubServiceImports(b *testing.B) { + o := benchOptionsForServiceImports() + s := RunServer(o) + defer s.Shutdown() + + addServiceImports(b, s) + + c := createClientWithAccount(b, "$foo", o.Host, o.Port) + defer c.Close() + + bw := bufio.NewWriterSize(c, defaultSendBufSize) + sendOp := []byte("PUB foo 2\r\nok\r\n") + + b.ResetTimer() + for i := 0; i < b.N; i++ { + bw.Write(sendOp) + } + err := bw.Flush() + if err != nil { + b.Errorf("Received error on FLUSH write: %v\n", err) + } + b.StopTimer() +} + +func Benchmark___PubSubAccsImport(b *testing.B) { + o := benchDefaultOptionsForAccounts() + s := RunServer(o) + defer s.Shutdown() + + pub := createClientWithAccount(b, "$foo", o.Host, o.Port) + defer pub.Close() + + sub := createClientWithAccount(b, "$bar", o.Host, o.Port) + defer sub.Close() + + sendProto(b, sub, "SUB foo 1\r\n") + flushConnection(b, sub) + ch := make(chan bool) + expected := len("MSG foo 1 2\r\nok\r\n") * b.N + go drainConnection(b, sub, ch, expected) + + bw := bufio.NewWriterSize(pub, defaultSendBufSize) + sendOp := []byte("PUB foo 2\r\nok\r\n") + + b.ResetTimer() + for i := 0; i < b.N; i++ { + bw.Write(sendOp) + } + err := bw.Flush() + if err != nil { + b.Errorf("Received error on FLUSH write: %v\n", err) + } + + // Wait for connection to be drained + <-ch + b.StopTimer() +} + +func Benchmark_____PubTwoQueueSub(b *testing.B) { + b.StopTimer() + s := runBenchServer() + c := createClientConn(b, "127.0.0.1", PERF_PORT) + doDefaultConnect(b, c) + sendProto(b, c, "SUB foo group1 1\r\n") + sendProto(b, c, "SUB foo group1 2\r\n") + bw := bufio.NewWriterSize(c, defaultSendBufSize) + sendOp := []byte("PUB foo 2\r\nok\r\n") + ch := make(chan bool) + expected := len("MSG foo 1 2\r\nok\r\n") * b.N + go drainConnection(b, c, ch, expected) + b.StartTimer() + + for i := 0; i < b.N; i++ { + _, err := bw.Write(sendOp) + if err != nil { + b.Fatalf("Received error on PUB write: %v\n", err) + } + } + err := bw.Flush() + if err != nil { + b.Fatalf("Received error on FLUSH write: %v\n", err) + } + + // Wait for connection to be drained + <-ch + + b.StopTimer() + c.Close() + s.Shutdown() +} + +func Benchmark____PubFourQueueSub(b *testing.B) { + b.StopTimer() + s := runBenchServer() + c := createClientConn(b, "127.0.0.1", PERF_PORT) + doDefaultConnect(b, c) + sendProto(b, c, "SUB foo group1 1\r\n") + sendProto(b, c, "SUB foo group1 2\r\n") + sendProto(b, c, "SUB foo group1 3\r\n") + sendProto(b, c, "SUB foo group1 4\r\n") + bw := bufio.NewWriterSize(c, defaultSendBufSize) + sendOp := []byte("PUB foo 2\r\nok\r\n") + ch := make(chan bool) + expected := len("MSG foo 1 2\r\nok\r\n") * b.N + go drainConnection(b, c, ch, expected) + b.StartTimer() + + for i := 0; i < b.N; i++ { + _, err := bw.Write(sendOp) + if err != nil { + b.Fatalf("Received error on PUB write: %v\n", err) + } + } + err := bw.Flush() + if err != nil { + b.Fatalf("Received error on FLUSH write: %v\n", err) + } + + // Wait for connection to be drained + <-ch + + b.StopTimer() + c.Close() + s.Shutdown() +} + +func Benchmark___PubEightQueueSub(b *testing.B) { + b.StopTimer() + s := runBenchServer() + c := createClientConn(b, "127.0.0.1", PERF_PORT) + doDefaultConnect(b, c) + sendProto(b, c, "SUB foo group1 1\r\n") + sendProto(b, c, "SUB foo group1 2\r\n") + sendProto(b, c, "SUB foo group1 3\r\n") + sendProto(b, c, "SUB foo group1 4\r\n") + sendProto(b, c, "SUB foo group1 5\r\n") + sendProto(b, c, "SUB foo group1 6\r\n") + sendProto(b, c, "SUB foo group1 7\r\n") + sendProto(b, c, "SUB foo group1 8\r\n") + bw := bufio.NewWriterSize(c, defaultSendBufSize) + sendOp := []byte("PUB foo 2\r\nok\r\n") + ch := make(chan bool) + expected := len("MSG foo 1 2\r\nok\r\n") * b.N + go drainConnection(b, c, ch, expected) + b.StartTimer() + + for i := 0; i < b.N; i++ { + _, err := bw.Write(sendOp) + if err != nil { + b.Fatalf("Received error on PUB write: %v\n", err) + } + } + err := bw.Flush() + if err != nil { + b.Fatalf("Received error on FLUSH write: %v\n", err) + } + + // Wait for connection to be drained + <-ch + + b.StopTimer() + c.Close() + s.Shutdown() +} + +func Benchmark_PubSub512kTwoConns(b *testing.B) { + b.StopTimer() + s := runBenchServer() + c := createClientConn(b, "127.0.0.1", PERF_PORT) + doDefaultConnect(b, c) + bw := bufio.NewWriterSize(c, defaultSendBufSize) + + c2 := createClientConn(b, "127.0.0.1", PERF_PORT) + doDefaultConnect(b, c2) + sendProto(b, c2, "SUB foo 1\r\n") + flushConnection(b, c2) + + sz := 1024 * 512 + payload := sizedString(sz) + + sendOp := []byte(fmt.Sprintf("PUB foo %d\r\n%s\r\n", sz, payload)) + ch := make(chan bool) + + expected := len(fmt.Sprintf("MSG foo 1 %d\r\n%s\r\n", sz, payload)) * b.N + go drainConnection(b, c2, ch, expected) + + b.StartTimer() + for i := 0; i < b.N; i++ { + bw.Write(sendOp) + } + err := bw.Flush() + if err != nil { + b.Errorf("Received error on FLUSH write: %v\n", err) + } + + // Wait for connection to be drained + <-ch + + b.StopTimer() + c.Close() + c2.Close() + s.Shutdown() +} + +func Benchmark__DenyMsgNoWCPubSub(b *testing.B) { + s, opts := RunServerWithConfig("./configs/authorization.conf") + opts.DisableShortFirstPing = true + defer s.Shutdown() + + c := createClientConn(b, opts.Host, opts.Port) + defer c.Close() + + expectAuthRequired(b, c) + cs := fmt.Sprintf("CONNECT {\"verbose\":false,\"pedantic\":false,\"user\":\"%s\",\"pass\":\"%s\"}\r\n", "bench-deny", DefaultPass) + sendProto(b, c, cs) + + sendProto(b, c, "SUB foo 1\r\n") + bw := bufio.NewWriterSize(c, defaultSendBufSize) + sendOp := []byte("PUB foo 2\r\nok\r\n") + ch := make(chan bool) + expected := len("MSG foo 1 2\r\nok\r\n") * b.N + go drainConnection(b, c, ch, expected) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := bw.Write(sendOp) + if err != nil { + b.Errorf("Received error on PUB write: %v\n", err) + } + } + err := bw.Flush() + if err != nil { + b.Errorf("Received error on FLUSH write: %v\n", err) + } + + // Wait for connection to be drained + <-ch + + // To not count defer cleanup of client and server. + b.StopTimer() +} + +func Benchmark_DenyMsgYesWCPubSub(b *testing.B) { + s, opts := RunServerWithConfig("./configs/authorization.conf") + opts.DisableShortFirstPing = true + defer s.Shutdown() + + c := createClientConn(b, opts.Host, opts.Port) + defer c.Close() + + expectAuthRequired(b, c) + cs := fmt.Sprintf("CONNECT {\"verbose\":false,\"pedantic\":false,\"user\":\"%s\",\"pass\":\"%s\"}\r\n", "bench-deny", DefaultPass) + sendProto(b, c, cs) + + sendProto(b, c, "SUB * 1\r\n") + bw := bufio.NewWriterSize(c, defaultSendBufSize) + sendOp := []byte("PUB foo 2\r\nok\r\n") + ch := make(chan bool) + expected := len("MSG foo 1 2\r\nok\r\n") * b.N + go drainConnection(b, c, ch, expected) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := bw.Write(sendOp) + if err != nil { + b.Errorf("Received error on PUB write: %v\n", err) + } + } + err := bw.Flush() + if err != nil { + b.Errorf("Received error on FLUSH write: %v\n", err) + } + + // Wait for connection to be drained + <-ch + + // To not count defer cleanup of client and server. + b.StopTimer() +} + +func routePubSub(b *testing.B, size int) { + b.StopTimer() + + s1, o1 := RunServerWithConfig("./configs/srv_a.conf") + o1.DisableShortFirstPing = true + defer s1.Shutdown() + s2, o2 := RunServerWithConfig("./configs/srv_b.conf") + o2.DisableShortFirstPing = true + defer s2.Shutdown() + + sub := createClientConn(b, o1.Host, o1.Port) + doDefaultConnect(b, sub) + sendProto(b, sub, "SUB foo 1\r\n") + flushConnection(b, sub) + + payload := sizedString(size) + + pub := createClientConn(b, o2.Host, o2.Port) + doDefaultConnect(b, pub) + bw := bufio.NewWriterSize(pub, defaultSendBufSize) + + ch := make(chan bool) + sendOp := []byte(fmt.Sprintf("PUB foo %d\r\n%s\r\n", len(payload), payload)) + expected := len(fmt.Sprintf("MSG foo 1 %d\r\n%s\r\n", len(payload), payload)) * b.N + go drainConnection(b, sub, ch, expected) + b.StartTimer() + + for i := 0; i < b.N; i++ { + _, err := bw.Write(sendOp) + if err != nil { + b.Fatalf("Received error on PUB write: %v\n", err) + } + + } + err := bw.Flush() + if err != nil { + b.Errorf("Received error on FLUSH write: %v\n", err) + } + + // Wait for connection to be drained + <-ch + + b.StopTimer() + pub.Close() + sub.Close() +} + +func Benchmark____RoutedPubSub_0b(b *testing.B) { + routePubSub(b, 2) +} + +func Benchmark____RoutedPubSub_1K(b *testing.B) { + routePubSub(b, 1024) +} + +func Benchmark__RoutedPubSub_100K(b *testing.B) { + routePubSub(b, 100*1024) +} + +func routeQueue(b *testing.B, numQueueSubs, size int) { + s1, o1 := RunServerWithConfig("./configs/srv_a.conf") + o1.DisableShortFirstPing = true + defer s1.Shutdown() + s2, o2 := RunServerWithConfig("./configs/srv_b.conf") + o2.DisableShortFirstPing = true + defer s2.Shutdown() + + sub := createClientConn(b, o1.Host, o1.Port) + defer sub.Close() + doDefaultConnect(b, sub) + for i := 0; i < numQueueSubs; i++ { + sendProto(b, sub, fmt.Sprintf("SUB foo bar %d\r\n", 100+i)) + } + flushConnection(b, sub) + + payload := sizedString(size) + + pub := createClientConn(b, o2.Host, o2.Port) + defer pub.Close() + doDefaultConnect(b, pub) + bw := bufio.NewWriterSize(pub, defaultSendBufSize) + + ch := make(chan bool) + sendOp := []byte(fmt.Sprintf("PUB foo %d\r\n%s\r\n", len(payload), payload)) + expected := len(fmt.Sprintf("MSG foo 100 %d\r\n%s\r\n", len(payload), payload)) * b.N + go drainConnection(b, sub, ch, expected) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := bw.Write(sendOp) + if err != nil { + b.Fatalf("Received error on PUB write: %v\n", err) + } + + } + err := bw.Flush() + if err != nil { + b.Errorf("Received error on FLUSH write: %v\n", err) + } + + // Wait for connection to be drained + <-ch + + b.StopTimer() +} + +func Benchmark____Routed2QueueSub(b *testing.B) { + routeQueue(b, 2, 2) +} + +func Benchmark____Routed4QueueSub(b *testing.B) { + routeQueue(b, 4, 2) +} + +func Benchmark____Routed8QueueSub(b *testing.B) { + routeQueue(b, 8, 2) +} + +func Benchmark___Routed16QueueSub(b *testing.B) { + routeQueue(b, 16, 2) +} + +func doS2CompressBench(b *testing.B, compress string) { + b.StopTimer() + conf1 := createConfFile(b, []byte(fmt.Sprintf(` + port: -1 + cluster { + name: "local" + port: -1 + pool_size: -1 + compression: %s + } + `, compress))) + s1, o1 := RunServerWithConfig(conf1) + defer s1.Shutdown() + + conf2 := createConfFile(b, []byte(fmt.Sprintf(` + port: -1 + cluster { + name: "local" + port: -1 + pool_size: -1 + compression: %s + routes: ["nats://127.0.0.1:%d"] + } + `, compress, o1.Cluster.Port))) + s2, _ := RunServerWithConfig(conf2) + defer s2.Shutdown() + + checkClusterFormed(b, s1, s2) + + nc2, err := nats.Connect(s2.ClientURL()) + if err != nil { + b.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + ch := make(chan struct{}, 1) + var count int + nc2.Subscribe("foo", func(_ *nats.Msg) { + if count++; count == b.N { + select { + case ch <- struct{}{}: + default: + } + } + }) + + checkSubInterest(b, s1, "$G", "foo", time.Second) + + nc1, err := nats.Connect(s1.ClientURL()) + if err != nil { + b.Fatalf("Error on connect: %v", err) + } + defer nc1.Close() + + // This one is easily compressible. + payload1 := make([]byte, 128) + // Make it random so that compression code has more to do. + payload2 := make([]byte, 256) + for i := 0; i < len(payload); i++ { + payload2[i] = byte(rand.IntN(26) + 'A') + } + b.StartTimer() + + for i := 0; i < b.N; i++ { + if i%2 == 0 { + nc1.Publish("foo", payload1) + } else { + nc1.Publish("foo", payload2) + } + } + + select { + case <-ch: + return + case <-time.After(10 * time.Second): + b.Fatal("Timeout waiting to receive all messages") + } +} + +func Benchmark____________RouteCompressOff(b *testing.B) { + doS2CompressBench(b, server.CompressionOff) +} + +func Benchmark_RouteCompressS2Uncompressed(b *testing.B) { + doS2CompressBench(b, server.CompressionS2Uncompressed) +} + +func Benchmark_________RouteCompressS2Fast(b *testing.B) { + doS2CompressBench(b, server.CompressionS2Fast) +} + +func Benchmark_______RouteCompressS2Better(b *testing.B) { + doS2CompressBench(b, server.CompressionS2Better) +} + +func Benchmark_________RouteCompressS2Best(b *testing.B) { + doS2CompressBench(b, server.CompressionS2Best) +} + +func doFanout(b *testing.B, numServers, numConnections, subsPerConnection int, subject, payload string) { + var s1, s2 *server.Server + var o1, o2 *server.Options + + switch numServers { + case 1: + s1, o1 = RunServerWithConfig("./configs/srv_a.conf") + o1.DisableShortFirstPing = true + defer s1.Shutdown() + s2, o2 = s1, o1 + case 2: + s1, o1 = RunServerWithConfig("./configs/srv_a.conf") + o1.DisableShortFirstPing = true + defer s1.Shutdown() + s2, o2 = RunServerWithConfig("./configs/srv_b.conf") + o2.DisableShortFirstPing = true + defer s2.Shutdown() + default: + b.Fatalf("%d servers not supported for this test\n", numServers) + } + + // To get a consistent length sid in MSG sent to us for drainConnection. + var sidFloor int + switch { + case subsPerConnection <= 100: + sidFloor = 100 + case subsPerConnection <= 1000: + sidFloor = 1000 + case subsPerConnection <= 10000: + sidFloor = 10000 + default: + b.Fatalf("Unsupported SubsPerConnection argument of %d\n", subsPerConnection) + } + + msgOp := fmt.Sprintf("MSG %s %d %d\r\n%s\r\n", subject, sidFloor, len(payload), payload) + expected := len(msgOp) * subsPerConnection * b.N + + // Client connections and subscriptions. + clients := make([]chan bool, 0, numConnections) + for i := 0; i < numConnections; i++ { + c := createClientConn(b, o2.Host, o2.Port) + doDefaultConnect(b, c) + defer c.Close() + + ch := make(chan bool) + clients = append(clients, ch) + + for s := 0; s < subsPerConnection; s++ { + subOp := fmt.Sprintf("SUB %s %d\r\n", subject, sidFloor+s) + sendProto(b, c, subOp) + } + flushConnection(b, c) + go drainConnection(b, c, ch, expected) + } + // Publish Connection + c := createClientConn(b, o1.Host, o1.Port) + doDefaultConnect(b, c) + flushConnection(b, c) + + bw := bufio.NewWriterSize(c, defaultSendBufSize) + sendOp := []byte(fmt.Sprintf("PUB %s %d\r\n%s\r\n", subject, len(payload), payload)) + + b.SetBytes(int64(len(sendOp) + (len(msgOp) * numConnections * subsPerConnection))) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := bw.Write(sendOp) + if err != nil { + b.Fatalf("Received error on PUB write: %v\n", err) + } + } + err := bw.Flush() + if err != nil { + b.Fatalf("Received error on FLUSH write: %v\n", err) + } + + // Wait for connections to be drained + for i := 0; i < numConnections; i++ { + <-clients[i] + } + b.StopTimer() +} + +var sub = "x" +var payload = "12345678" + +func Benchmark______FanOut_8x1x10(b *testing.B) { + doFanout(b, 1, 1, 10, sub, payload) +} + +func Benchmark_____FanOut_8x1x100(b *testing.B) { + doFanout(b, 1, 1, 100, sub, payload) +} + +func Benchmark____FanOut_8x10x100(b *testing.B) { + doFanout(b, 1, 10, 100, sub, payload) +} + +func Benchmark___FanOut_8x10x1000(b *testing.B) { + doFanout(b, 1, 10, 1000, sub, payload) +} + +func Benchmark___FanOut_8x100x100(b *testing.B) { + doFanout(b, 1, 100, 100, sub, payload) +} + +func Benchmark__FanOut_8x100x1000(b *testing.B) { + doFanout(b, 1, 100, 1000, sub, payload) +} + +func Benchmark__FanOut_8x10x10000(b *testing.B) { + doFanout(b, 1, 10, 10000, sub, payload) +} + +func Benchmark___FanOut_8x500x100(b *testing.B) { + doFanout(b, 1, 500, 100, sub, payload) +} + +func Benchmark___FanOut_128x1x100(b *testing.B) { + doFanout(b, 1, 1, 100, sub, sizedString(128)) +} + +func Benchmark__FanOut_128x10x100(b *testing.B) { + doFanout(b, 1, 10, 100, sub, sizedString(128)) +} + +func Benchmark_FanOut_128x10x1000(b *testing.B) { + doFanout(b, 1, 10, 1000, sub, sizedString(128)) +} + +func Benchmark_FanOut_128x100x100(b *testing.B) { + doFanout(b, 1, 100, 100, sub, sizedString(128)) +} + +func BenchmarkFanOut_128x100x1000(b *testing.B) { + doFanout(b, 1, 100, 1000, sub, sizedString(128)) +} + +func BenchmarkFanOut_128x10x10000(b *testing.B) { + doFanout(b, 1, 10, 10000, sub, sizedString(128)) +} + +func BenchmarkFanOut__128x500x100(b *testing.B) { + doFanout(b, 1, 500, 100, sub, sizedString(128)) +} + +func Benchmark_FanOut_512x100x100(b *testing.B) { + doFanout(b, 1, 100, 100, sub, sizedString(512)) +} + +func Benchmark__FanOut_512x100x1k(b *testing.B) { + doFanout(b, 1, 100, 1000, sub, sizedString(512)) +} + +func Benchmark____FanOut_1kx10x1k(b *testing.B) { + doFanout(b, 1, 10, 1000, sub, sizedString(1024)) +} + +func Benchmark__FanOut_1kx100x100(b *testing.B) { + doFanout(b, 1, 100, 100, sub, sizedString(1024)) +} + +func Benchmark_____RFanOut_8x1x10(b *testing.B) { + doFanout(b, 2, 1, 10, sub, payload) +} + +func Benchmark____RFanOut_8x1x100(b *testing.B) { + doFanout(b, 2, 1, 100, sub, payload) +} + +func Benchmark___RFanOut_8x10x100(b *testing.B) { + doFanout(b, 2, 10, 100, sub, payload) +} + +func Benchmark__RFanOut_8x10x1000(b *testing.B) { + doFanout(b, 2, 10, 1000, sub, payload) +} + +func Benchmark__RFanOut_8x100x100(b *testing.B) { + doFanout(b, 2, 100, 100, sub, payload) +} + +func Benchmark_RFanOut_8x100x1000(b *testing.B) { + doFanout(b, 2, 100, 1000, sub, payload) +} + +func Benchmark_RFanOut_8x10x10000(b *testing.B) { + doFanout(b, 2, 10, 10000, sub, payload) +} + +func Benchmark_RFanOut_1kx10x1000(b *testing.B) { + doFanout(b, 2, 10, 1000, sub, sizedString(1024)) +} + +func doFanIn(b *testing.B, numServers, numPublishers, numSubscribers int, subject, payload string) { + b.Helper() + if b.N < numPublishers { + return + } + // Don't check for number of subscribers being lower than the number of publishers. + // We also use this bench to show the performance impact of increased number of publishers, + // and for those tests, the number of publishers will start at 1 and increase to 10, + // while the number of subscribers will always be 3. + if numSubscribers > 10 { + b.Fatalf("numSubscribers should be <= 10") + } + + var s1, s2 *server.Server + var o1, o2 *server.Options + + switch numServers { + case 1: + s1, o1 = RunServerWithConfig("./configs/srv_a.conf") + o1.DisableShortFirstPing = true + defer s1.Shutdown() + s2, o2 = s1, o1 + case 2: + s1, o1 = RunServerWithConfig("./configs/srv_a.conf") + o1.DisableShortFirstPing = true + defer s1.Shutdown() + s2, o2 = RunServerWithConfig("./configs/srv_b.conf") + o2.DisableShortFirstPing = true + defer s2.Shutdown() + default: + b.Fatalf("%d servers not supported for this test\n", numServers) + } + + msgOp := fmt.Sprintf("MSG %s %d %d\r\n%s\r\n", subject, 9, len(payload), payload) + sendOp := []byte(fmt.Sprintf("PUB %s %d\r\n%s\r\n", subject, len(payload), payload)) + l := b.N / numPublishers + expected := len(msgOp) * l * numPublishers + + // Keep the amount of data in flight bounded below the server's default + // max_pending limit. Without this, a large payload sent concurrently by + // all publishers can cause the benchmark subscriber to be closed as a + // slow consumer before its drain goroutine gets enough CPU time. + const ( + creditTargetBytes = 256 * 1024 + numCredits = 128 + ) + batchMsgs := creditTargetBytes / len(sendOp) + if batchMsgs == 0 { + batchMsgs = 1 + } + credits := make(chan struct{}, numCredits) + for i := 0; i < numCredits; i++ { + credits <- struct{}{} + } + creditBytes := batchMsgs * len(msgOp) + + // Client connections and subscriptions. For fan in these are smaller then numPublishers. + clients := make([]chan bool, 0, numSubscribers) + for i := 0; i < numSubscribers; i++ { + c := createClientConn(b, o2.Host, o2.Port) + doDefaultConnect(b, c) + defer c.Close() + + ch := make(chan bool) + clients = append(clients, ch) + + subOp := fmt.Sprintf("SUB %s %d\r\n", subject, i) + sendProto(b, c, subOp) + flushConnection(b, c) + if i == 0 { + go drainConnectionWithFlowControl(b, c, ch, expected, credits, creditBytes) + } else { + go drainConnection(b, c, ch, expected) + } + } + + startCh := make(chan bool) + + pubLoop := func(c net.Conn, ch chan bool) { + bw := bufio.NewWriterSize(c, defaultSendBufSize) + + // Signal we are ready + close(ch) + + // Wait to start up actual sends. + <-startCh + + for sent := 0; sent < l; { + <-credits + batch := min(batchMsgs, l-sent) + for i := 0; i < batch; i++ { + if _, err := bw.Write(sendOp); err != nil { + b.Errorf("Received error on PUB write: %v\n", err) + return + } + } + sent += batch + } + err := bw.Flush() + if err != nil { + b.Errorf("Received error on FLUSH write: %v\n", err) + return + } + } + + // Publish Connections SPINUP + for i := 0; i < numPublishers; i++ { + c := createClientConn(b, o1.Host, o1.Port) + doDefaultConnect(b, c) + flushConnection(b, c) + ch := make(chan bool) + + go pubLoop(c, ch) + <-ch + } + + b.SetBytes(int64(len(sendOp) + len(msgOp))) + b.ResetTimer() + + // Closing this will start all publishers at once (roughly) + close(startCh) + + // Wait for connections to be drained + for i := 0; i < len(clients); i++ { + <-clients[i] + } + b.StopTimer() +} + +func Benchmark_____FanIn_1kx100x1(b *testing.B) { + doFanIn(b, 1, 100, 1, sub, sizedString(1024)) +} + +func Benchmark_____FanIn_4kx100x1(b *testing.B) { + doFanIn(b, 1, 100, 1, sub, sizedString(4096)) +} + +func Benchmark_____FanIn_8kx100x1(b *testing.B) { + doFanIn(b, 1, 100, 1, sub, sizedString(8192)) +} + +func Benchmark____FanIn_16kx100x1(b *testing.B) { + doFanIn(b, 1, 100, 1, sub, sizedString(16384)) +} + +func Benchmark____FanIn_64kx100x1(b *testing.B) { + doFanIn(b, 1, 100, 1, sub, sizedString(65536)) +} + +func Benchmark___FanIn_128kx100x1(b *testing.B) { + doFanIn(b, 1, 100, 1, sub, sizedString(65536*2)) +} + +func Benchmark___BumpPubCount_1x3(b *testing.B) { + doFanIn(b, 1, 1, 3, sub, sizedString(128)) +} + +func Benchmark___BumpPubCount_2x3(b *testing.B) { + doFanIn(b, 1, 2, 3, sub, sizedString(128)) +} + +func Benchmark___BumpPubCount_5x3(b *testing.B) { + doFanIn(b, 1, 5, 3, sub, sizedString(128)) +} + +func Benchmark__BumpPubCount_10x3(b *testing.B) { + doFanIn(b, 1, 10, 3, sub, sizedString(128)) +} + +func testDefaultBenchOptionsForGateway(name string) *server.Options { + opts := testDefaultOptionsForGateway(name) + opts.DisableShortFirstPing = true + return opts +} + +func gatewaysBench(b *testing.B, optimisticMode bool, payload string, numPublishers int, subInterest bool) { + b.Helper() + if b.N < numPublishers { + return + } + + ob := testDefaultBenchOptionsForGateway("B") + sb := RunServer(ob) + defer sb.Shutdown() + + server.SetGatewaysSolicitDelay(10 * time.Millisecond) + defer server.ResetGatewaysSolicitDelay() + + gwbURL, err := url.Parse(fmt.Sprintf("nats://%s:%d", ob.Gateway.Host, ob.Gateway.Port)) + if err != nil { + b.Fatalf("Error parsing url: %v", err) + } + oa := testDefaultBenchOptionsForGateway("A") + oa.Gateway.Gateways = []*server.RemoteGatewayOpts{ + { + Name: "B", + URLs: []*url.URL{gwbURL}, + }, + } + sa := RunServer(oa) + defer sa.Shutdown() + + sub := createClientConn(b, ob.Host, ob.Port) + defer sub.Close() + doDefaultConnect(b, sub) + sendProto(b, sub, "SUB end.test 1\r\n") + if subInterest { + sendProto(b, sub, "SUB foo 2\r\n") + } + flushConnection(b, sub) + + // If not optimisticMode, make B switch GW connection + // to interest mode only + if !optimisticMode { + pub := createClientConn(b, oa.Host, oa.Port) + doDefaultConnect(b, pub) + // has to be more that defaultGatewayMaxRUnsubBeforeSwitch + for i := 0; i < 2000; i++ { + sendProto(b, pub, fmt.Sprintf("PUB reject.me.%d 2\r\nok\r\n", i+1)) + } + flushConnection(b, pub) + pub.Close() + } + + ch := make(chan bool) + publishersDone := make(chan struct{}) + var publishers sync.WaitGroup + var msgOp string + var expected int + l := b.N / numPublishers + if subInterest { + msgOp = fmt.Sprintf("MSG foo 2 %d\r\n%s\r\n", len(payload), payload) + expected = len(msgOp) * l * numPublishers + } + // Last message sent to end.test + lastMsg := "MSG end.test 1 2\r\nok\r\n" + expected += len(lastMsg) * numPublishers + go drainConnectionAfter(b, sub, ch, expected, publishersDone) + + sendOp := []byte(fmt.Sprintf("PUB foo %d\r\n%s\r\n", len(payload), payload)) + startCh := make(chan bool) + + lastMsgSendOp := []byte("PUB end.test 2\r\nok\r\n") + + pubLoop := func(c net.Conn, ch chan bool) { + defer publishers.Done() + bw := bufio.NewWriterSize(c, defaultSendBufSize) + + // Signal we are ready + close(ch) + + // Wait to start up actual sends. + <-startCh + + for i := 0; i < l; i++ { + if _, err := bw.Write(sendOp); err != nil { + b.Errorf("Received error on PUB write: %v\n", err) + return + } + } + if _, err := bw.Write(lastMsgSendOp); err != nil { + b.Errorf("Received error on PUB write: %v\n", err) + return + } + if err := bw.Flush(); err != nil { + b.Errorf("Received error on FLUSH write: %v\n", err) + return + } + flushConnection(b, c) + } + + // Publish Connections SPINUP + for i := 0; i < numPublishers; i++ { + c := createClientConn(b, oa.Host, oa.Port) + defer c.Close() + doDefaultConnect(b, c) + flushConnection(b, c) + ch := make(chan bool) + + publishers.Add(1) + go pubLoop(c, ch) + <-ch + } + go func() { + publishers.Wait() + close(publishersDone) + }() + + // To report the number of bytes: + // from publisher to server on cluster A: + numBytes := len(sendOp) + if subInterest { + // from server in cluster A to server on cluster B: + // RMSG $G foo \r\n + numBytes += len("RMSG $G foo xxxx ") + len(payload) + 2 + + // From server in cluster B to sub: + numBytes += len(msgOp) + } + b.SetBytes(int64(numBytes)) + b.ResetTimer() + + // Closing this will start all publishers at once (roughly) + close(startCh) + + // Wait for end of test + <-ch + <-publishersDone + + b.StopTimer() +} + +func Benchmark____GWs_Opt_1kx01x0(b *testing.B) { + gatewaysBench(b, true, sizedString(1024), 1, false) +} + +func Benchmark____GWs_Opt_2kx01x0(b *testing.B) { + gatewaysBench(b, true, sizedString(2048), 1, false) +} + +func Benchmark____GWs_Opt_4kx01x0(b *testing.B) { + gatewaysBench(b, true, sizedString(4096), 1, false) +} + +func Benchmark____GWs_Opt_1kx10x0(b *testing.B) { + gatewaysBench(b, true, sizedString(1024), 10, false) +} + +func Benchmark____GWs_Opt_2kx10x0(b *testing.B) { + gatewaysBench(b, true, sizedString(2048), 10, false) +} + +func Benchmark____GWs_Opt_4kx10x0(b *testing.B) { + gatewaysBench(b, true, sizedString(4096), 10, false) +} + +func Benchmark____GWs_Opt_1kx01x1(b *testing.B) { + gatewaysBench(b, true, sizedString(1024), 1, true) +} + +func Benchmark____GWs_Opt_2kx01x1(b *testing.B) { + gatewaysBench(b, true, sizedString(2048), 1, true) +} + +func Benchmark____GWs_Opt_4kx01x1(b *testing.B) { + gatewaysBench(b, true, sizedString(4096), 1, true) +} + +func Benchmark____GWs_Opt_1kx10x1(b *testing.B) { + gatewaysBench(b, true, sizedString(1024), 10, true) +} + +func Benchmark____GWs_Opt_2kx10x1(b *testing.B) { + gatewaysBench(b, true, sizedString(2048), 10, true) +} + +func Benchmark____GWs_Opt_4kx10x1(b *testing.B) { + gatewaysBench(b, true, sizedString(4096), 10, true) +} + +func Benchmark____GWs_Int_1kx01x0(b *testing.B) { + gatewaysBench(b, false, sizedString(1024), 1, false) +} + +func Benchmark____GWs_Int_2kx01x0(b *testing.B) { + gatewaysBench(b, false, sizedString(2048), 1, false) +} + +func Benchmark____GWs_Int_4kx01x0(b *testing.B) { + gatewaysBench(b, false, sizedString(4096), 1, false) +} + +func Benchmark____GWs_Int_1kx10x0(b *testing.B) { + gatewaysBench(b, false, sizedString(1024), 10, false) +} + +func Benchmark____GWs_Int_2kx10x0(b *testing.B) { + gatewaysBench(b, false, sizedString(2048), 10, false) +} + +func Benchmark____GWs_Int_4kx10x0(b *testing.B) { + gatewaysBench(b, false, sizedString(4096), 10, false) +} + +func Benchmark____GWs_Int_1kx01x1(b *testing.B) { + gatewaysBench(b, false, sizedString(1024), 1, true) +} + +func Benchmark____GWs_Int_2kx01x1(b *testing.B) { + gatewaysBench(b, false, sizedString(2048), 1, true) +} + +func Benchmark____GWs_Int_4kx01x1(b *testing.B) { + gatewaysBench(b, false, sizedString(4096), 1, true) +} + +func Benchmark____GWs_Int_1kx10x1(b *testing.B) { + gatewaysBench(b, false, sizedString(1024), 10, true) +} + +func Benchmark____GWs_Int_2kx10x1(b *testing.B) { + gatewaysBench(b, false, sizedString(2048), 10, true) +} + +func Benchmark____GWs_Int_4kx10x1(b *testing.B) { + gatewaysBench(b, false, sizedString(4096), 10, true) +} + +// This bench only sends the requests to verify impact +// of reply mapping in GW code. +func gatewaySendRequestsBench(b *testing.B, singleReplySub bool) { + server.SetGatewaysSolicitDelay(10 * time.Millisecond) + defer server.ResetGatewaysSolicitDelay() + + ob := testDefaultBenchOptionsForGateway("B") + sb := RunServer(ob) + defer sb.Shutdown() + + gwbURL, err := url.Parse(fmt.Sprintf("nats://%s:%d", ob.Gateway.Host, ob.Gateway.Port)) + if err != nil { + b.Fatalf("Error parsing url: %v", err) + } + oa := testDefaultBenchOptionsForGateway("A") + oa.Gateway.Gateways = []*server.RemoteGatewayOpts{ + { + Name: "B", + URLs: []*url.URL{gwbURL}, + }, + } + sa := RunServer(oa) + defer sa.Shutdown() + + sub := createClientConn(b, ob.Host, ob.Port) + defer sub.Close() + doDefaultConnect(b, sub) + sendProto(b, sub, "SUB foo 1\r\n") + flushConnection(b, sub) + + lenMsg := len("MSG foo reply.xxxxxxxxxx 1 2\r\nok\r\n") + expected := b.N * lenMsg + ch := make(chan bool, 1) + go drainConnection(b, sub, ch, expected) + + c := createClientConn(b, oa.Host, oa.Port) + defer c.Close() + doDefaultConnect(b, c) + flushConnection(b, c) + + // From pub to server in cluster A: + numBytes := len("PUB foo reply.0123456789 2\r\nok\r\n") + if !singleReplySub { + // Add the preceding SUB + numBytes += len("SUB reply.0123456789 0123456789\r\n") + // And UNSUB... + numBytes += len("UNSUB 0123456789\r\n") + } + // From server in cluster A to cluster B + numBytes += len("RMSG $G foo reply.0123456789 2\r\nok\r\n") + // If mapping of reply... + if !singleReplySub { + // the mapping uses about 24 more bytes. So add them + // for RMSG from server to server. + numBytes += 24 + } + // From server in cluster B to sub + numBytes += lenMsg + b.SetBytes(int64(numBytes)) + + bw := bufio.NewWriterSize(c, defaultSendBufSize) + var subStr string + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if !singleReplySub { + subStr = fmt.Sprintf("SUB reply.%010d %010d\r\n", i+1, i+1) + } + bw.Write([]byte(fmt.Sprintf("%sPUB foo reply.%010d 2\r\nok\r\n", subStr, i+1))) + // Simulate that we are doing actual request/reply and therefore + // unsub'ing the subs on the reply subject. + if !singleReplySub && i > 1000 { + bw.Write([]byte(fmt.Sprintf("UNSUB %010d\r\n", (i - 1000)))) + } + } + bw.Flush() + flushConnection(b, c) + + <-ch +} + +func Benchmark__GWs_Reqs_1_SubAll(b *testing.B) { + gatewaySendRequestsBench(b, true) +} + +func Benchmark__GWs_Reqs_1SubEach(b *testing.B) { + gatewaySendRequestsBench(b, false) +} diff --git a/test/client_auth_test.go b/test/client_auth_test.go new file mode 100644 index 0000000..7402960 --- /dev/null +++ b/test/client_auth_test.go @@ -0,0 +1,550 @@ +// Copyright 2016-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "encoding/base64" + "fmt" + "net" + "os" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nkeys" +) + +func TestMultipleUserAuth(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/multi_user.conf") + defer srv.Shutdown() + + if opts.Users == nil { + t.Fatal("Expected a user array that is not nil") + } + if len(opts.Users) != 2 { + t.Fatal("Expected a user array that had 2 users") + } + + // Test first user + url := fmt.Sprintf("nats://%s:%s@%s:%d/", + opts.Users[0].Username, + opts.Users[0].Password, + opts.Host, opts.Port) + + nc, err := nats.Connect(url) + if err != nil { + t.Fatalf("Expected a successful connect, got %v\n", err) + } + defer nc.Close() + + if !nc.AuthRequired() { + t.Fatal("Expected auth to be required for the server") + } + + // Test second user + url = fmt.Sprintf("nats://%s:%s@%s:%d/", + opts.Users[1].Username, + opts.Users[1].Password, + opts.Host, opts.Port) + + nc, err = nats.Connect(url) + if err != nil { + t.Fatalf("Expected a successful connect, got %v\n", err) + } + defer nc.Close() +} + +// Resolves to "test" +const testToken = "$2a$05$3sSWEVA1eMCbV0hWavDjXOx.ClBjI6u1CuUdLqf22cbJjXsnzz8/." + +func TestTokenInConfig(t *testing.T) { + content := ` + listen: 127.0.0.1:4567 + authorization={ + token: ` + testToken + ` + timeout: 5 + }` + confFile := createConfFile(t, []byte(content)) + if err := os.WriteFile(confFile, []byte(content), 0666); err != nil { + t.Fatalf("Error writing config file: %v", err) + } + s, opts := RunServerWithConfig(confFile) + defer s.Shutdown() + + url := fmt.Sprintf("nats://test@%s:%d/", opts.Host, opts.Port) + nc, err := nats.Connect(url) + if err != nil { + t.Fatalf("Expected a successful connect, got %v\n", err) + } + defer nc.Close() + if !nc.AuthRequired() { + t.Fatal("Expected auth to be required for the server") + } +} + +func TestClientConnectInfo(t *testing.T) { + for _, test := range []struct { + name string + sys string + hasSys bool + protocol int + }{ + {"async info support with explicit system account", "system_account: SYS", true, server.ClientProtoInfo}, + {"async info support without explicit system account", "", false, server.ClientProtoInfo}, + {"no async info support with explicit system account", "system_account: SYS", true, server.ClientProtoZero}, + {"no async info support without explicit system account", "", false, server.ClientProtoZero}, + } { + t.Run(test.name, func(t *testing.T) { + conf := createConfFile(t, []byte(fmt.Sprintf(` + port: -1 + %s + accounts { + SYS: { users: [{ user: sys, password: pwd}] } + A: { users: [{ user: a, password: pwd}] } + B: { users: [{ user: b, password: pwd}] } + } + `, test.sys))) + hub, oHub := RunServerWithConfig(conf) + defer hub.Shutdown() + + checkInfoOnConnect := func(user, acc string, isSys bool) { + t.Helper() + cc := createClientConn(t, oHub.Host, oHub.Port) + defer cc.Close() + + checkInfoMsg(t, cc) + sendProto(t, cc, fmt.Sprintf("CONNECT {\"user\":%q,\"pass\":\"pwd\",\"pedantic\":false,\"verbose\":false,\"protocol\":%d}\r\nPING\r\n", user, test.protocol)) + // Since the PONG and INFO may be receive as a single TCP read, + // make sure we consume the pong alone here. + pongBuf := make([]byte, len("PONG\r\n")) + cc.SetReadDeadline(time.Now().Add(2 * time.Second)) + n, err := cc.Read(pongBuf) + cc.SetReadDeadline(time.Time{}) + if n <= 0 && err != nil { + t.Fatalf("Error reading from conn: %v\n", err) + } + if !pongRe.Match(pongBuf) { + t.Fatalf("Response did not match expected: \n\tReceived:'%q'\n\tExpected:'%s'\n", pongBuf, pongRe) + } + if test.protocol < server.ClientProtoInfo { + expectNothing(t, cc) + } else { + info := checkInfoMsg(t, cc) + if !info.ConnectInfo { + t.Fatal("Expected ConnectInfo to be true") + } + if an := info.RemoteAccount; an != acc { + t.Fatalf("Expected account %q, got %q", acc, info.RemoteAccount) + } + if ais := info.IsSystemAccount; ais != isSys { + t.Fatalf("Expected IsSystemAccount to be %v, got %v", isSys, ais) + } + } + } + checkInfoOnConnect("a", "A", false) + checkInfoOnConnect("sys", "SYS", test.hasSys) + checkInfoOnConnect("b", "B", false) + }) + } +} + +type captureProxiesReloadLogger struct { + dummyLogger + ch chan string +} + +func (l *captureProxiesReloadLogger) Noticef(format string, args ...any) { + msg := fmt.Sprintf(format, args...) + if strings.Contains(msg, "proxies trusted keys") { + l.ch <- msg + } +} + +func TestProxyKeyVerification(t *testing.T) { + u1, _ := nkeys.CreateUser() + u1Pub, _ := u1.PublicKey() + + u2, _ := nkeys.CreateUser() + u2Pub, _ := u2.PublicKey() + + u3, _ := nkeys.CreateUser() + u3Pub, _ := u3.PublicKey() + + tmpl := ` + listen: "127.0.0.1:-1" + proxies { + trusted [ + {key: %q} + {key: %q} + ] + } + leafnodes { + listen: "127.0.0.1:-1" + } + ` + conf := createConfFile(t, fmt.Appendf(nil, tmpl, u1Pub, u2Pub)) + s, o := RunServerWithConfig(conf) + defer s.Shutdown() + + varz, err := s.Varz(nil) + if err != nil { + t.Fatalf("Error getting varz: %v", err) + } + vproxies := varz.Proxies + if vproxies == nil { + t.Fatal("Expected proxies to be not nil") + } + trusted := vproxies.Trusted + if len(trusted) != 2 { + t.Fatalf("Expected 2 trusted proxies, got %+v", trusted) + } + if k := trusted[0].Key; k != u1Pub { + t.Fatalf("Expected key to be %q, got %q", u1Pub, k) + } + if k := trusted[1].Key; k != u2Pub { + t.Fatalf("Expected key to be %q, got %q", u2Pub, k) + } + + var currentCID uint64 + connect := func(t *testing.T, kp nkeys.KeyPair, bsig bool, port int, isClient bool, user string) (expectFun, net.Conn) { + c := createClientConn(t, "127.0.0.1", port) + expect := expectCommand(t, c) + info := checkInfoMsg(t, c) + currentCID = info.CID + if info.Nonce == "" { + c.Close() + t.Fatalf("Expected nonce from INFO, got %+v", info) + } + if info.JSApiLevel != server.JSApiLevel { + c.Close() + t.Fatalf("Expected JSApiLevel to be set to %v, got %v", server.JSApiLevel, info.JSApiLevel) + } + sig, err := kp.Sign([]byte(info.Nonce)) + if err != nil { + c.Close() + t.Fatalf("Error signing nonce %q: %v", info.Nonce, err) + } + encodedSig := make([]byte, base64.RawURLEncoding.EncodedLen(len(sig))) + base64.RawURLEncoding.Encode(encodedSig, sig) + if bsig { + encodedSig[0] = '*' + } + sendProto(t, c, + fmt.Sprintf("CONNECT {\"pedantic\":false,\"verbose\":false,\"user\":%q,\"pass\":\"pwd\",\"proxy_sig\":\"%s\"}\r\n", + user, string(encodedSig))) + if isClient { + sendProto(t, c, "PING\r\n") + } + return expect, c + } + + checkClosedState := func(t *testing.T, cid uint64, reason server.ClosedState) { + // Closed connection getting in the closed state is done in a go routine + // so make sure we wait for it. + checkFor(t, time.Second, 100*time.Millisecond, func() error { + connz, err := s.Connz(&server.ConnzOptions{ + CID: cid, + State: server.ConnClosed, + }) + if err != nil { + return fmt.Errorf("Error getting connz for closed connection %v: %v", cid, err) + } + if len(connz.Conns) != 1 { + return fmt.Errorf("Expected 1 connection, got %v", connz.Conns) + } + conn := connz.Conns[0] + if conn.Reason != reason.String() { + return fmt.Errorf("Expected reason %q, got %q", reason.String(), conn.Reason) + } + return nil + }) + } + + for _, test := range []struct { + name string + kp nkeys.KeyPair + bsig bool + ok bool + }{ + {"key present first", u1, false, true}, + {"key present last", u2, false, true}, + {"key not present", u3, false, false}, + {"bad signature", u2, true, false}, + } { + t.Run(test.name, func(t *testing.T) { + for _, subTest := range []struct { + name string + port int + isClient bool + }{ + {"client", o.Port, true}, + {"leafnodes", o.LeafNode.Port, false}, + } { + t.Run(subTest.name, func(t *testing.T) { + expect, c := connect(t, test.kp, test.bsig, subTest.port, subTest.isClient, "user") + defer c.Close() + if test.ok { + var p *server.ProxyInfo + if subTest.isClient { + expect(pongRe) + + connz, err := s.Connz(nil) + if err != nil { + t.Fatalf("Error getting connz: %v", err) + } + if n := len(connz.Conns); n != 1 { + t.Fatalf("Expected 1 ConnInfo, got %v", n) + } + ci := connz.Conns[0] + p = ci.Proxy + } else { + expect(infoRe) + + leafz, err := s.Leafz(nil) + if err != nil { + t.Fatalf("Error getting leafz: %v", err) + } + if n := len(leafz.Leafs); n != 1 { + t.Fatalf("Expected 1 LeafInfo, got %v", n) + } + li := leafz.Leafs[0] + if li.Proxy == nil { + t.Fatalf("Required Proxy to be present, was not: %+v", li) + } + p = li.Proxy + } + if p == nil { + t.Fatalf("Required Proxy to be present, was not") + } + pub, _ := test.kp.PublicKey() + if p.Key != pub { + t.Fatalf("Expected public key to be %q, got %q", pub, p.Key) + } + } else { + expect(errRe) + checkClosedState(t, currentCID, server.ProxyNotTrusted) + } + }) + } + }) + } + + // Create a connection using u1Pub and we will config reload and remove + // that key from the list. The client should get disconnected. + expectc, c := connect(t, u1, false, o.Port, true, "user") + defer c.Close() + expectc(pongRe) + // Capture this connection CID before the next connect. + cid1 := currentCID + + expectl, l := connect(t, u1, false, o.LeafNode.Port, false, "user") + defer l.Close() + expectl(infoRe) + cid2 := currentCID + checkLeafNodeConnected(t, s) + + logger := &captureProxiesReloadLogger{ch: make(chan string, 10)} + s.SetLogger(logger, false, false) + + os.WriteFile(conf, fmt.Appendf(nil, tmpl, u3Pub, u2Pub), 0660) + if err := s.Reload(); err != nil { + t.Fatalf("Reload failed: %v", err) + } + for range 2 { + select { + case str := <-logger.ch: + if strings.Contains(str, "removed") { + if !strings.Contains(str, u1Pub) { + t.Fatalf("Expected removed trace to include %q, it did not: %s", u1Pub, str) + } + } else if strings.Contains(str, "added") { + if !strings.Contains(str, u3Pub) { + t.Fatalf("Expected added trace to include %q, it did not: %s", u3Pub, str) + } + } else { + t.Fatalf("Unexpected log: %q", str) + } + default: + t.Fatal("Expected a log, did not get one") + } + } + + // Connections should get disconnected. + // We need to consume what is sent by the server, but for leaf we may + // get some LS+, etc... so just consumer until we get the io.EOF + readAll := func(c net.Conn) { + for { + var buf [1024]byte + if _, err := c.Read(buf[:]); err != nil { + break + } + } + } + readAll(c) + expectDisconnect(t, c) + readAll(l) + expectDisconnect(t, l) + c.Close() + l.Close() + + // Now check that the connection with CID `cid` was closed because the + // proxy is (no longer) trusted. + checkClosedState(t, cid1, server.ProxyNotTrusted) + checkClosedState(t, cid2, server.ProxyNotTrusted) + + varz, err = s.Varz(nil) + if err != nil { + t.Fatalf("Error getting varz: %v", err) + } + vproxies = varz.Proxies + if vproxies == nil { + t.Fatal("Expected proxies to be not nil") + } + trusted = vproxies.Trusted + if len(trusted) != 2 { + t.Fatalf("Expected 2 trusted proxies, got %+v", trusted) + } + if k := trusted[0].Key; k != u3Pub { + t.Fatalf("Expected key to be %q, got %q", u3Pub, k) + } + if k := trusted[1].Key; k != u2Pub { + t.Fatalf("Expected key to be %q, got %q", u2Pub, k) + } + + // And we should be able to connect using u3 key pair. + expectc, c = connect(t, u3, false, o.Port, true, "user") + expectc(pongRe) + c.Close() + expectl, l = connect(t, u3, false, o.LeafNode.Port, false, "user") + expectl(infoRe) + l.Close() + + // But now using u1 key pair should fail. + expectc, c = connect(t, u1, false, o.Port, true, "user") + expectc(errRe) + c.Close() + expectl, l = connect(t, u1, false, o.LeafNode.Port, false, "user") + expectl(errRe) + l.Close() + + s.Shutdown() + + // Now check that if no trusted proxy are provided, a proxy connection is + // accepted. Use leafnodes since we always send NONCE for this type of connection. + conf = createConfFile(t, []byte(` + port: -1 + leafnodes { + port: -1 + } + `)) + s, o = RunServerWithConfig(conf) + defer s.Shutdown() + + for _, test := range []struct { + name string + bsig bool + }{ + {"no trusted proxy and correct signature", false}, + {"no trusted proxy and invalid signature", true}, + } { + t.Run(test.name, func(t *testing.T) { + expect, c := connect(t, u1, test.bsig, o.LeafNode.Port, false, "user") + defer c.Close() + // Does not matter if bad signature, since there is no trusted keys + // in the server, we accept without signature check. + expect(infoRe) + }) + } + + s.Shutdown() + + // Finally, check that if there are no configured trusted proxies + // and a user requires a proxy connection, then the connection will fail. + conf = createConfFile(t, []byte(` + port: -1 + leafnodes { + port: -1 + authorization { + users [ + {user: u1, password: pwd} + {user: u2, password: pwd, proxy_required: true} + ] + } + } + `)) + s, o = RunServerWithConfig(conf) + defer s.Shutdown() + + for _, test := range []struct { + name string + user string + ok bool + }{ + {"no trusted proxy configured no proxy required", "u1", true}, + {"no trusted proxy configured and proxy required", "u2", false}, + } { + t.Run(test.name, func(t *testing.T) { + // We can use any key pair here, it is not important. + expect, c := connect(t, u1, false, o.LeafNode.Port, false, test.user) + defer c.Close() + if test.ok { + expect(infoRe) + } else { + expect(errRe) + // Check that the reason is "proxy required" + checkClosedState(t, currentCID, server.ProxyRequired) + } + }) + } + + // Same test, but the "proxy_required" is at the authorization{} to-level, + // so any user should fail. + conf = createConfFile(t, []byte(` + port: -1 + leafnodes { + port: -1 + authorization { + users [ + {user: u1, password: pwd} + {user: u2, password: pwd} + ] + proxy_required: true + } + } + `)) + s, o = RunServerWithConfig(conf) + defer s.Shutdown() + + t.Run("all users required to use trusted proxy", func(t *testing.T) { + for _, user := range []string{"u1", "u2"} { + // We can use any key pair here, it is not important. + expect, c := connect(t, u1, false, o.LeafNode.Port, false, user) + defer c.Close() + expect(errRe) + // Check that the reason is "proxy required" + checkClosedState(t, currentCID, server.ProxyRequired) + } + }) + + varz, err = s.Varz(nil) + if err != nil { + t.Fatalf("Error getting varz: %v", err) + } + vproxies = varz.Proxies + if vproxies != nil { + t.Fatalf("Expected proxies to be nil, got %+v", vproxies) + } +} diff --git a/test/client_cluster_test.go b/test/client_cluster_test.go new file mode 100644 index 0000000..3e29f82 --- /dev/null +++ b/test/client_cluster_test.go @@ -0,0 +1,467 @@ +// Copyright 2013-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "math/rand/v2" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +func TestServerRestartReSliceIssue(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + + urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, optsA.Port) + urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, optsB.Port) + + // msg to send.. + msg := []byte("Hello World") + + servers := []string{urlA, urlB} + + opts := nats.GetDefaultOptions() + opts.Timeout = (5 * time.Second) + opts.ReconnectWait = (50 * time.Millisecond) + opts.MaxReconnect = 1000 + + numClients := 20 + + reconnects := int32(0) + reconnectsDone := make(chan bool, numClients) + opts.ReconnectedCB = func(nc *nats.Conn) { + atomic.AddInt32(&reconnects, 1) + reconnectsDone <- true + } + + clients := make([]*nats.Conn, numClients) + + // Create 20 random clients. + // Half connected to A and half to B.. + for i := 0; i < numClients; i++ { + opts.Url = servers[i%2] + nc, err := opts.Connect() + if err != nil { + t.Fatalf("Failed to create connection: %v\n", err) + } + clients[i] = nc + defer nc.Close() + + // Create 10 subscriptions each.. + for x := 0; x < 10; x++ { + subject := fmt.Sprintf("foo.%d", (rand.Int()%50)+1) + nc.Subscribe(subject, func(m *nats.Msg) { + // Just eat it.. + }) + } + // Pick one subject to send to.. + subject := fmt.Sprintf("foo.%d", (rand.Int()%50)+1) + go func() { + time.Sleep(10 * time.Millisecond) + for i := 1; i <= 100; i++ { + if err := nc.Publish(subject, msg); err != nil { + return + } + if i%10 == 0 { + time.Sleep(time.Millisecond) + } + } + }() + } + + // Wait for a short bit.. + time.Sleep(20 * time.Millisecond) + + // Restart SrvB + srvB.Shutdown() + srvB = RunServer(optsB) + defer srvB.Shutdown() + + // Check that all expected clients have reconnected + done := false + for i := 0; i < numClients/2 && !done; i++ { + select { + case <-reconnectsDone: + done = true + case <-time.After(3 * time.Second): + t.Fatalf("Expected %d reconnects, got %d\n", numClients/2, reconnects) + } + } + + // Since srvB was restarted, its defer Shutdown() was last, so will + // exectue first, which would cause clients that have reconnected to + // it to try to reconnect (causing delays on Windows). So let's + // explicitly close them here. + // NOTE: With fix of NATS GO client (reconnect loop yields to Close()), + // this change would not be required, however, it still speeeds up + // the test, from more than 7s to less than one. + for i := 0; i < numClients; i++ { + nc := clients[i] + nc.Close() + } +} + +// This will test queue subscriber semantics across a cluster in the presence +// of server restarts. +func TestServerRestartAndQueueSubs(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, optsA.Port) + urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, optsB.Port) + + // Client options + opts := nats.GetDefaultOptions() + opts.Timeout = 5 * time.Second + opts.ReconnectWait = 20 * time.Millisecond + opts.MaxReconnect = 1000 + opts.NoRandomize = true + + // Allow us to block on a reconnect completion. + reconnectsDone := make(chan bool) + opts.ReconnectedCB = func(nc *nats.Conn) { + reconnectsDone <- true + } + + // Helper to wait on a reconnect. + waitOnReconnect := func() { + t.Helper() + select { + case <-reconnectsDone: + case <-time.After(2 * time.Second): + t.Fatalf("Expected a reconnect, timedout!\n") + } + } + + // Create two clients.. + opts.Servers = []string{urlA, urlB} + c1, err := opts.Connect() + if err != nil { + t.Fatalf("Failed to create connection for c1: %v\n", err) + } + defer c1.Close() + + opts.Servers = []string{urlB, urlA} + c2, err := opts.Connect() + if err != nil { + t.Fatalf("Failed to create connection for c2: %v\n", err) + } + defer c2.Close() + + // Flusher helper function. + flush := func() { + // Wait for processing. + c1.Flush() + c2.Flush() + // Wait for a short bit for cluster propagation. + time.Sleep(50 * time.Millisecond) + } + + // To hold queue results. + results := make(map[int]int) + var mu sync.Mutex + + // This corresponds to the subsriptions below. + const ExpectedMsgCount = 3 + + // Make sure we got what we needed, 1 msg only and all seqnos accounted for.. + checkResults := func(numSent int) { + mu.Lock() + defer mu.Unlock() + + for i := 0; i < numSent; i++ { + if results[i] != ExpectedMsgCount { + t.Fatalf("Received incorrect number of messages, [%d] vs [%d] for seq: %d\n", results[i], ExpectedMsgCount, i) + } + } + + // Auto reset results map + results = make(map[int]int) + } + + subj := "foo.bar" + qgroup := "workers" + + cb := func(msg *nats.Msg) { + mu.Lock() + defer mu.Unlock() + seqno, _ := strconv.Atoi(string(msg.Data)) + results[seqno] = results[seqno] + 1 + } + + // Create queue subscribers + c1.QueueSubscribe(subj, qgroup, cb) + c2.QueueSubscribe(subj, qgroup, cb) + + // Do a wildcard subscription. + c1.Subscribe("foo.*", cb) + c2.Subscribe("foo.*", cb) + + // Wait for processing. + flush() + + sendAndCheckMsgs := func(numToSend int) { + for i := 0; i < numToSend; i++ { + if i%2 == 0 { + c1.Publish(subj, []byte(strconv.Itoa(i))) + } else { + c2.Publish(subj, []byte(strconv.Itoa(i))) + } + } + // Wait for processing. + flush() + // Check Results + checkResults(numToSend) + } + + //////////////////////////////////////////////////////////////////////////// + // Base Test + //////////////////////////////////////////////////////////////////////////// + + // Make sure subscriptions are propagated in the cluster + if err := checkExpectedSubs(4, srvA, srvB); err != nil { + t.Fatalf("%v", err) + } + + // Now send 10 messages, from each client.. + sendAndCheckMsgs(10) + + //////////////////////////////////////////////////////////////////////////// + // Now restart SrvA and srvB, re-run test + //////////////////////////////////////////////////////////////////////////// + + srvA.Shutdown() + // Wait for client on A to reconnect to B. + waitOnReconnect() + + srvA = RunServer(optsA) + defer srvA.Shutdown() + + srvB.Shutdown() + // Now both clients should reconnect to A. + waitOnReconnect() + waitOnReconnect() + + srvB = RunServer(optsB) + defer srvB.Shutdown() + + // Make sure the cluster is reformed + checkClusterFormed(t, srvA, srvB) + + // Make sure subscriptions are propagated in the cluster + // Clients will be connected to srvA, so that will be 4, + // but srvB will only have 2 now since we coaelsce. + if err := checkExpectedSubs(4, srvA); err != nil { + t.Fatalf("%v", err) + } + if err := checkExpectedSubs(2, srvB); err != nil { + t.Fatalf("%v", err) + } + + // Now send another 10 messages, from each client.. + sendAndCheckMsgs(10) + + // Since servers are restarted after all client's close defer calls, + // their defer Shutdown() are last, and so will be executed first, + // which would cause clients to try to reconnect on exit, causing + // delays on Windows. So let's explicitly close them here. + c1.Close() + c2.Close() +} + +// This will test request semantics across a route +func TestRequestsAcrossRoutes(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, optsA.Port) + urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, optsB.Port) + + nc1, err := nats.Connect(urlA) + if err != nil { + t.Fatalf("Failed to create connection for nc1: %v\n", err) + } + defer nc1.Close() + + nc2, err := nats.Connect(urlB) + if err != nil { + t.Fatalf("Failed to create connection for nc2: %v\n", err) + } + defer nc2.Close() + + response := []byte("I will help you") + + // Connect responder to srvA + nc1.Subscribe("foo-req", func(m *nats.Msg) { + nc1.Publish(m.Reply, response) + }) + // Make sure the route and the subscription are propagated. + nc1.Flush() + + if err = checkExpectedSubs(1, srvA, srvB); err != nil { + t.Fatal(err.Error()) + } + + for i := 0; i < 100; i++ { + if _, err = nc2.Request("foo-req", []byte(strconv.Itoa(i)), 250*time.Millisecond); err != nil { + t.Fatalf("Received an error on Request test [%d]: %s", i, err) + } + } +} + +// This will test request semantics across a route to queues +func TestRequestsAcrossRoutesToQueues(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, optsA.Port) + urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, optsB.Port) + + nc1, err := nats.Connect(urlA) + if err != nil { + t.Fatalf("Failed to create connection for nc1: %v\n", err) + } + defer nc1.Close() + + nc2, err := nats.Connect(urlB) + if err != nil { + t.Fatalf("Failed to create connection for nc2: %v\n", err) + } + defer nc2.Close() + + response := []byte("I will help you") + + // Connect one responder to srvA + nc1.QueueSubscribe("foo-req", "booboo", func(m *nats.Msg) { + nc1.Publish(m.Reply, response) + }) + // Make sure the route and the subscription are propagated. + nc1.Flush() + + // Connect the other responder to srvB + nc2.QueueSubscribe("foo-req", "booboo", func(m *nats.Msg) { + nc2.Publish(m.Reply, response) + }) + + if err = checkExpectedSubs(2, srvA, srvB); err != nil { + t.Fatal(err.Error()) + } + + for i := 0; i < 100; i++ { + if _, err = nc2.Request("foo-req", []byte(strconv.Itoa(i)), 500*time.Millisecond); err != nil { + t.Fatalf("Received an error on Request test [%d]: %s", i, err) + } + } + + for i := 0; i < 100; i++ { + if _, err = nc1.Request("foo-req", []byte(strconv.Itoa(i)), 500*time.Millisecond); err != nil { + t.Fatalf("Received an error on Request test [%d]: %s", i, err) + } + } +} + +// This is in response to Issue #1144 +// https://github.com/nats-io/nats-server/issues/1144 +func TestQueueDistributionAcrossRoutes(t *testing.T) { + srvA, srvB, _, _ := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + checkClusterFormed(t, srvA, srvB) + + urlA := srvA.ClientURL() + urlB := srvB.ClientURL() + + nc1, err := nats.Connect(urlA) + if err != nil { + t.Fatalf("Failed to create connection for nc1: %v\n", err) + } + defer nc1.Close() + + nc2, err := nats.Connect(urlB) + if err != nil { + t.Fatalf("Failed to create connection for nc2: %v\n", err) + } + defer nc2.Close() + + var qsubs []*nats.Subscription + + // Connect queue subscriptions as mentioned in the issue. 2(A) - 6(B) - 4(A) + for i := 0; i < 2; i++ { + sub, _ := nc1.QueueSubscribeSync("foo", "bar") + qsubs = append(qsubs, sub) + } + nc1.Flush() + for i := 0; i < 6; i++ { + sub, _ := nc2.QueueSubscribeSync("foo", "bar") + qsubs = append(qsubs, sub) + } + nc2.Flush() + for i := 0; i < 4; i++ { + sub, _ := nc1.QueueSubscribeSync("foo", "bar") + qsubs = append(qsubs, sub) + } + nc1.Flush() + + if err := checkExpectedSubs(7, srvA, srvB); err != nil { + t.Fatalf("%v", err) + } + + send := 10000 + for i := 0; i < send; i++ { + nc2.Publish("foo", nil) + } + nc2.Flush() + + tp := func() int { + var total int + for i := 0; i < len(qsubs); i++ { + pending, _, _ := qsubs[i].Pending() + total += pending + } + return total + } + + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if total := tp(); total != send { + return fmt.Errorf("Number of total received %d", total) + } + return nil + }) + + // The bug is essentially that when we deliver across a route, we + // prefer locals, but if we randomize to a block of bounce backs, then + // we walk to the end and find the same local for all the remote options. + // So what you will see in this case is a large value at #9 (2+6, next one local). + + avg := send / len(qsubs) + for i := 0; i < len(qsubs); i++ { + total, _, _ := qsubs[i].Pending() + if total > avg+(avg*3/10) { + if i == 8 { + t.Fatalf("Qsub in 8th position gets majority of the messages (prior 6 spots) in this test") + } + t.Fatalf("Received too high, %d vs %d", total, avg) + } + } +} diff --git a/test/cluster_test.go b/test/cluster_test.go new file mode 100644 index 0000000..bdd99e7 --- /dev/null +++ b/test/cluster_test.go @@ -0,0 +1,684 @@ +// Copyright 2013-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "errors" + "fmt" + "math/rand/v2" + "runtime" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" +) + +// Helper function to check that a cluster is formed +func checkClusterFormed(t testing.TB, servers ...*server.Server) { + t.Helper() + expectedNumRoutes := len(servers) - 1 + checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { + for _, s := range servers { + if numRoutes := s.NumRoutes(); numRoutes != expectedNumRoutes { + return fmt.Errorf("Expected %d routes for server %q, got %d", expectedNumRoutes, s.ID(), numRoutes) + } + } + return nil + }) +} + +func checkNumRoutes(t *testing.T, s *server.Server, expected int) { + t.Helper() + checkFor(t, 5*time.Second, 15*time.Millisecond, func() error { + if nr := s.NumRoutes(); nr != expected { + return fmt.Errorf("Expected %v routes, got %v", expected, nr) + } + return nil + }) +} + +// Helper function to check that a server (or list of servers) have the +// expected number of subscriptions. +func checkExpectedSubs(expected int, servers ...*server.Server) error { + var err string + maxTime := time.Now().Add(10 * time.Second) + for time.Now().Before(maxTime) { + err = "" + for _, s := range servers { + if numSubs := int(s.NumSubscriptions()); numSubs != expected { + err = fmt.Sprintf("Expected %d subscriptions for server %q, got %d", expected, s.ID(), numSubs) + break + } + } + if err != "" { + time.Sleep(10 * time.Millisecond) + } else { + break + } + } + if err != "" { + return errors.New(err) + } + return nil +} + +func checkSubInterest(t testing.TB, s *server.Server, accName, subject string, timeout time.Duration) { + t.Helper() + checkFor(t, timeout, 15*time.Millisecond, func() error { + acc, err := s.LookupAccount(accName) + if err != nil { + return fmt.Errorf("error looking up account %q: %v", accName, err) + } + if acc.SubscriptionInterest(subject) { + return nil + } + return fmt.Errorf("no subscription interest for account %q on %q", accName, subject) + }) +} + +func checkNoSubInterest(t *testing.T, s *server.Server, accName, subject string, timeout time.Duration) { + t.Helper() + acc, err := s.LookupAccount(accName) + if err != nil { + t.Fatalf("error looking up account %q: %v", accName, err) + } + + start := time.Now() + for time.Now().Before(start.Add(timeout)) { + if acc.SubscriptionInterest(subject) { + t.Fatalf("Did not expect interest for %q", subject) + } + time.Sleep(5 * time.Millisecond) + } +} + +func runThreeServers(t *testing.T) (srvA, srvB, srvC *server.Server, optsA, optsB, optsC *server.Options) { + srvA, optsA = RunServerWithConfig("./configs/srv_a.conf") + srvB, optsB = RunServerWithConfig("./configs/srv_b.conf") + srvC, optsC = RunServerWithConfig("./configs/srv_c.conf") + + checkClusterFormed(t, srvA, srvB, srvC) + return +} + +func runServers(t *testing.T) (srvA, srvB *server.Server, optsA, optsB *server.Options) { + srvA, optsA = RunServerWithConfig("./configs/srv_a.conf") + srvB, optsB = RunServerWithConfig("./configs/srv_b.conf") + + checkClusterFormed(t, srvA, srvB) + return +} + +func TestProperServerWithRoutesShutdown(t *testing.T) { + before := runtime.NumGoroutine() + srvA, srvB, _, _ := runServers(t) + srvA.Shutdown() + srvB.Shutdown() + time.Sleep(100 * time.Millisecond) + + after := runtime.NumGoroutine() + delta := after - before + // There may be some finalizers or IO, but in general more than + // 2 as a delta represents a problem. + if delta > 2 { + t.Fatalf("Expected same number of goroutines, %d vs %d\n", before, after) + } +} + +func TestDoubleRouteConfig(t *testing.T) { + srvA, srvB, _, _ := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() +} + +func TestBasicClusterPubSub(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + sendA, expectA := setupConn(t, clientA) + sendA("SUB foo 22\r\n") + sendA("PING\r\n") + expectA(pongRe) + + if err := checkExpectedSubs(1, srvA, srvB); err != nil { + t.Fatalf("%v", err) + } + + sendB, expectB := setupConn(t, clientB) + sendB("PUB foo 2\r\nok\r\n") + sendB("PING\r\n") + expectB(pongRe) + + expectMsgs := expectMsgsCommand(t, expectA) + + matches := expectMsgs(1) + checkMsg(t, matches[0], "foo", "22", "", "2", "ok") +} + +func TestClusterQueueSubs(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + sendA, expectA := setupConn(t, clientA) + sendB, expectB := setupConn(t, clientB) + + expectMsgsA := expectMsgsCommand(t, expectA) + expectMsgsB := expectMsgsCommand(t, expectB) + + // Capture sids for checking later. + qg1SidsA := []string{"1", "2", "3"} + + // Three queue subscribers + for _, sid := range qg1SidsA { + sendA(fmt.Sprintf("SUB foo qg1 %s\r\n", sid)) + } + sendA("PING\r\n") + expectA(pongRe) + + // Make sure the subs have propagated to srvB before continuing + // New cluster proto this will only be 1. + if err := checkExpectedSubs(1, srvB); err != nil { + t.Fatalf("%v", err) + } + + sendB("PUB foo 2\r\nok\r\n") + sendB("PING\r\n") + expectB(pongRe) + + // Make sure we get only 1. + matches := expectMsgsA(1) + checkMsg(t, matches[0], "foo", "", "", "2", "ok") + + // Capture sids for checking later. + pSids := []string{"4", "5", "6"} + + // Create 3 normal subscribers + for _, sid := range pSids { + sendA(fmt.Sprintf("SUB foo %s\r\n", sid)) + } + + // Create a FWC Subscriber + pSids = append(pSids, "7") + sendA("SUB > 7\r\n") + sendA("PING\r\n") + expectA(pongRe) + + // Make sure the subs have propagated to srvB before continuing + // Normal foo and the queue group will be one a piece, so 2 + wc == 3 + if err := checkExpectedSubs(3, srvB); err != nil { + t.Fatalf("%v", err) + } + + // Send to B + sendB("PUB foo 2\r\nok\r\n") + sendB("PING\r\n") + expectB(pongRe) + + // Give plenty of time for the messages to flush, so that we don't + // accidentally only read some of them. + time.Sleep(time.Millisecond * 250) + + // Should receive 5. + matches = expectMsgsA(5) + checkForQueueSid(t, matches, qg1SidsA) + checkForPubSids(t, matches, pSids) + + // Send to A + sendA("PUB foo 2\r\nok\r\n") + + // Give plenty of time for the messages to flush, so that we don't + // accidentally only read some of them. + time.Sleep(time.Millisecond * 250) + + // Should receive 5. + matches = expectMsgsA(5) + checkForQueueSid(t, matches, qg1SidsA) + checkForPubSids(t, matches, pSids) + + // Now add queue subscribers to B + qg2SidsB := []string{"1", "2", "3"} + for _, sid := range qg2SidsB { + sendB(fmt.Sprintf("SUB foo qg2 %s\r\n", sid)) + } + sendB("PING\r\n") + expectB(pongRe) + + // Make sure the subs have propagated to srvA before continuing + // This will be all the subs on A and just 1 from B that gets coalesced. + if err := checkExpectedSubs(len(qg1SidsA)+len(pSids)+1, srvA); err != nil { + t.Fatalf("%v", err) + } + + // Send to B + sendB("PUB foo 2\r\nok\r\n") + + // Give plenty of time for the messages to flush, so that we don't + // accidentally only read some of them. + time.Sleep(time.Millisecond * 250) + + // Should receive 1 from B. + matches = expectMsgsB(1) + checkForQueueSid(t, matches, qg2SidsB) + + // Should receive 5 still from A. + matches = expectMsgsA(5) + checkForQueueSid(t, matches, qg1SidsA) + checkForPubSids(t, matches, pSids) + + // Now drop queue subscribers from A + for _, sid := range qg1SidsA { + sendA(fmt.Sprintf("UNSUB %s\r\n", sid)) + } + sendA("PING\r\n") + expectA(pongRe) + + // Make sure the subs have propagated to srvB before continuing + if err := checkExpectedSubs(1+1+len(qg2SidsB), srvB); err != nil { + t.Fatalf("%v", err) + } + + // Send to B + sendB("PUB foo 2\r\nok\r\n") + + // Should receive 1 from B. + matches = expectMsgsB(1) + checkForQueueSid(t, matches, qg2SidsB) + + sendB("PING\r\n") + expectB(pongRe) + + // Should receive 4 now. + matches = expectMsgsA(4) + checkForPubSids(t, matches, pSids) + + // Send to A + sendA("PUB foo 2\r\nok\r\n") + + // Give plenty of time for the messages to flush, so that we don't + // accidentally only read some of them. + time.Sleep(time.Millisecond * 250) + + // Should receive 4 now. + matches = expectMsgsA(4) + checkForPubSids(t, matches, pSids) +} + +// Issue #22 +func TestClusterDoubleMsgs(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + clientA1 := createClientConn(t, optsA.Host, optsA.Port) + defer clientA1.Close() + + clientA2 := createClientConn(t, optsA.Host, optsA.Port) + defer clientA2.Close() + + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + sendA1, expectA1 := setupConn(t, clientA1) + sendA2, expectA2 := setupConn(t, clientA2) + sendB, expectB := setupConn(t, clientB) + + expectMsgsA1 := expectMsgsCommand(t, expectA1) + expectMsgsA2 := expectMsgsCommand(t, expectA2) + + // Capture sids for checking later. + qg1SidsA := []string{"1", "2", "3"} + + // Three queue subscribers + for _, sid := range qg1SidsA { + sendA1(fmt.Sprintf("SUB foo qg1 %s\r\n", sid)) + } + sendA1("PING\r\n") + expectA1(pongRe) + + // Make sure the subs have propagated to srvB before continuing + if err := checkExpectedSubs(1, srvB); err != nil { + t.Fatalf("%v", err) + } + + sendB("PUB foo 2\r\nok\r\n") + sendB("PING\r\n") + expectB(pongRe) + + // Make sure we get only 1. + matches := expectMsgsA1(1) + checkMsg(t, matches[0], "foo", "", "", "2", "ok") + checkForQueueSid(t, matches, qg1SidsA) + + // Add a FWC subscriber on A2 + sendA2("SUB > 1\r\n") + sendA2("SUB foo 2\r\n") + sendA2("PING\r\n") + expectA2(pongRe) + pSids := []string{"1", "2"} + + // Make sure the subs have propagated to srvB before continuing + if err := checkExpectedSubs(1+2, srvB); err != nil { + t.Fatalf("%v", err) + } + + sendB("PUB foo 2\r\nok\r\n") + sendB("PING\r\n") + expectB(pongRe) + + matches = expectMsgsA1(1) + checkMsg(t, matches[0], "foo", "", "", "2", "ok") + checkForQueueSid(t, matches, qg1SidsA) + + matches = expectMsgsA2(2) + checkMsg(t, matches[0], "foo", "", "", "2", "ok") + checkForPubSids(t, matches, pSids) + + // Close ClientA1 + clientA1.Close() + + sendB("PUB foo 2\r\nok\r\n") + sendB("PING\r\n") + expectB(pongRe) + + time.Sleep(10 * time.Millisecond) + + matches = expectMsgsA2(2) + checkMsg(t, matches[0], "foo", "", "", "2", "ok") + checkForPubSids(t, matches, pSids) +} + +// This will test that we drop remote sids correctly. +func TestClusterDropsRemoteSids(t *testing.T) { + srvA, srvB, optsA, _ := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConn(t, clientA) + + // Add a subscription + sendA("SUB foo 1\r\n") + sendA("PING\r\n") + expectA(pongRe) + + // Wait for propagation. + time.Sleep(100 * time.Millisecond) + + if sc := srvA.NumSubscriptions(); sc != 1 { + t.Fatalf("Expected one subscription for srvA, got %d\n", sc) + } + if sc := srvB.NumSubscriptions(); sc != 1 { + t.Fatalf("Expected one subscription for srvB, got %d\n", sc) + } + + // Add another subscription + sendA("SUB bar 2\r\n") + sendA("PING\r\n") + expectA(pongRe) + + // Wait for propagation. + time.Sleep(100 * time.Millisecond) + + if sc := srvA.NumSubscriptions(); sc != 2 { + t.Fatalf("Expected two subscriptions for srvA, got %d\n", sc) + } + if sc := srvB.NumSubscriptions(); sc != 2 { + t.Fatalf("Expected two subscriptions for srvB, got %d\n", sc) + } + + // unsubscription + sendA("UNSUB 1\r\n") + sendA("PING\r\n") + expectA(pongRe) + + // Wait for propagation. + time.Sleep(100 * time.Millisecond) + + if sc := srvA.NumSubscriptions(); sc != 1 { + t.Fatalf("Expected one subscription for srvA, got %d\n", sc) + } + if sc := srvB.NumSubscriptions(); sc != 1 { + t.Fatalf("Expected one subscription for srvB, got %d\n", sc) + } + + // Close the client and make sure we remove subscription state. + clientA.Close() + + // Wait for propagation. + time.Sleep(100 * time.Millisecond) + if sc := srvA.NumSubscriptions(); sc != 0 { + t.Fatalf("Expected no subscriptions for srvA, got %d\n", sc) + } + if sc := srvB.NumSubscriptions(); sc != 0 { + t.Fatalf("Expected no subscriptions for srvB, got %d\n", sc) + } +} + +// This will test that we drop remote sids correctly. +func TestAutoUnsubscribePropagation(t *testing.T) { + srvA, srvB, optsA, _ := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConn(t, clientA) + expectMsgs := expectMsgsCommand(t, expectA) + + // We will create subscriptions that will auto-unsubscribe and make sure + // we are not accumulating orphan subscriptions on the other side. + for i := 1; i <= 100; i++ { + sub := fmt.Sprintf("SUB foo %d\r\n", i) + auto := fmt.Sprintf("UNSUB %d 1\r\n", i) + sendA(sub) + sendA(auto) + // This will trip the auto-unsubscribe + sendA("PUB foo 2\r\nok\r\n") + expectMsgs(1) + } + + sendA("PING\r\n") + expectA(pongRe) + + time.Sleep(50 * time.Millisecond) + + // Make sure number of subscriptions on B is correct + if subs := srvB.NumSubscriptions(); subs != 0 { + t.Fatalf("Expected no subscriptions on remote server, got %d\n", subs) + } +} + +func TestAutoUnsubscribePropagationOnClientDisconnect(t *testing.T) { + srvA, srvB, optsA, _ := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + cluster := []*server.Server{srvA, srvB} + + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConn(t, clientA) + + // No subscriptions. Ready to test. + if err := checkExpectedSubs(0, cluster...); err != nil { + t.Fatalf("%v", err) + } + + sendA("SUB foo 1\r\n") + sendA("UNSUB 1 1\r\n") + sendA("PING\r\n") + expectA(pongRe) + + // Waiting cluster subs propagation + if err := checkExpectedSubs(1, cluster...); err != nil { + t.Fatalf("%v", err) + } + + clientA.Close() + + // No subs should be on the cluster when all clients is disconnected + if err := checkExpectedSubs(0, cluster...); err != nil { + t.Fatalf("%v", err) + } +} + +func TestClusterNameOption(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + cluster { + name: MyCluster + listen: 127.0.0.1:-1 + } + `)) + + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + si := checkInfoMsg(t, c) + if si.Cluster != "MyCluster" { + t.Fatalf("Expected a cluster name of %q, got %q", "MyCluster", si.Cluster) + } +} + +func TestEphemeralClusterName(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + cluster { + listen: 127.0.0.1:-1 + } + `)) + + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + si := checkInfoMsg(t, c) + if si.Cluster == "" { + t.Fatalf("Expected an ephemeral cluster name to be set") + } +} + +type captureErrLogger struct { + dummyLogger + ch chan string +} + +func (c *captureErrLogger) Errorf(format string, v ...any) { + msg := fmt.Sprintf(format, v...) + select { + case c.ch <- msg: + default: + } +} + +func TestClusterNameConflictsDropRoutes(t *testing.T) { + ll := &captureErrLogger{ch: make(chan string, 4)} + + conf := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + cluster { + name: MyCluster33 + listen: 127.0.0.1:5244 + } + `)) + + s1, _ := RunServerWithConfig(conf) + defer s1.Shutdown() + s1.SetLogger(ll, false, false) + + conf2 := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + cluster { + name: MyCluster22 + listen: 127.0.0.1:-1 + routes = [nats-route://127.0.0.1:5244] + } + `)) + + s2, _ := RunServerWithConfig(conf2) + defer s2.Shutdown() + s2.SetLogger(ll, false, false) + + select { + case msg := <-ll.ch: + if !strings.Contains(msg, "Rejecting connection") || !strings.Contains(msg, "does not match") { + t.Fatalf("Got bad error about cluster name mismatch") + } + case <-time.After(time.Second): + t.Fatalf("Expected an error, timed out") + } +} + +func TestClusterNameDynamicNegotiation(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + cluster {listen: 127.0.0.1:5244} + `)) + + seed, _ := RunServerWithConfig(conf) + defer seed.Shutdown() + + oconf := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + cluster { + listen: 127.0.0.1:-1 + routes = [nats-route://127.0.0.1:5244] + } + `)) + + // Create a random number of additional servers, up to 20. + numServers := rand.IntN(20) + 1 + servers := make([]*server.Server, 0, numServers+1) + servers = append(servers, seed) + + for i := 0; i < numServers; i++ { + s, _ := RunServerWithConfig(oconf) + defer s.Shutdown() + servers = append(servers, s) + } + + // If this passes we should have all the same name. + checkClusterFormed(t, servers...) + + clusterName := seed.ClusterName() + for _, s := range servers { + if s.ClusterName() != clusterName { + t.Fatalf("Expected the cluster names to all be the same as %q, got %q", clusterName, s.ClusterName()) + } + } +} diff --git a/test/cluster_tls_test.go b/test/cluster_tls_test.go new file mode 100644 index 0000000..cc363a3 --- /dev/null +++ b/test/cluster_tls_test.go @@ -0,0 +1,175 @@ +// Copyright 2013-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" +) + +func runTLSServers(t *testing.T) (srvA, srvB *server.Server, optsA, optsB *server.Options) { + srvA, optsA = RunServerWithConfig("./configs/srv_a_tls.conf") + srvB, optsB = RunServerWithConfig("./configs/srv_b_tls.conf") + checkClusterFormed(t, srvA, srvB) + return +} + +func TestTLSClusterConfig(t *testing.T) { + srvA, srvB, _, _ := runTLSServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() +} + +func TestBasicTLSClusterPubSub(t *testing.T) { + srvA, srvB, optsA, optsB := runTLSServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + sendA, expectA := setupConn(t, clientA) + sendA("SUB foo 22\r\n") + sendA("PING\r\n") + expectA(pongRe) + + if err := checkExpectedSubs(1, srvA, srvB); err != nil { + t.Fatalf("%v", err) + } + + sendB, expectB := setupConn(t, clientB) + sendB("PUB foo 2\r\nok\r\n") + sendB("PING\r\n") + expectB(pongRe) + + expectMsgs := expectMsgsCommand(t, expectA) + + matches := expectMsgs(1) + checkMsg(t, matches[0], "foo", "22", "", "2", "ok") +} + +type captureTLSError struct { + dummyLogger + ch chan struct{} +} + +func (c *captureTLSError) Errorf(format string, v ...any) { + msg := fmt.Sprintf(format, v...) + if strings.Contains(msg, "handshake error") { + select { + case c.ch <- struct{}{}: + default: + } + } +} + +type captureClusterTLSInsecureLogger struct { + dummyLogger + ch chan struct{} +} + +func (c *captureClusterTLSInsecureLogger) Warnf(format string, v ...any) { + msg := fmt.Sprintf(format, v...) + if strings.Contains(msg, "solicited routes will not be verified") { + select { + case c.ch <- struct{}{}: + default: + } + } +} + +func TestClusterTLSInsecure(t *testing.T) { + confA := createConfFile(t, []byte(` + port: -1 + cluster { + name: "xyz" + listen: "127.0.0.1:-1" + pool_size: -1 + compression: "disabled" + tls { + cert_file: "./configs/certs/server-noip.pem" + key_file: "./configs/certs/server-key-noip.pem" + ca_file: "./configs/certs/ca.pem" + timeout: 2 + } + } + `)) + srvA, optsA := RunServerWithConfig(confA) + defer srvA.Shutdown() + + l := &captureTLSError{ch: make(chan struct{}, 1)} + srvA.SetLogger(l, false, false) + + bConfigTemplate := ` + port: -1 + cluster { + name: "xyz" + listen: "127.0.0.1:-1" + pool_size: -1 + compression: "disabled" + tls { + cert_file: "./configs/certs/server-noip.pem" + key_file: "./configs/certs/server-key-noip.pem" + ca_file: "./configs/certs/ca.pem" + timeout: 2 + %s + } + routes [ + "nats://%s:%d" + ] + } + ` + confB := createConfFile(t, []byte(fmt.Sprintf(bConfigTemplate, + "", optsA.Cluster.Host, optsA.Cluster.Port))) + srvB, _ := RunServerWithConfig(confB) + defer srvB.Shutdown() + + // We should get errors + select { + case <-l.ch: + case <-time.After(2 * time.Second): + t.Fatalf("Did not get handshake error") + } + + // Set a logger that will capture the warning + wl := &captureClusterTLSInsecureLogger{ch: make(chan struct{}, 1)} + srvB.SetLogger(wl, false, false) + + // Need to add "insecure: true" and reload + if err := os.WriteFile(confB, + []byte(fmt.Sprintf(bConfigTemplate, "insecure: true", optsA.Cluster.Host, optsA.Cluster.Port)), + 0666); err != nil { + t.Fatalf("Error rewriting file: %v", err) + } + if err := srvB.Reload(); err != nil { + t.Fatalf("Error on reload: %v", err) + } + + checkClusterFormed(t, srvA, srvB) + + // Make sure we have the tracing + select { + case <-wl.ch: + case <-time.After(2 * time.Second): + t.Fatalf("Did not get warning about using cluster's insecure setting") + } +} diff --git a/test/configs/certs/ca.pem b/test/configs/certs/ca.pem new file mode 100644 index 0000000..911c486 --- /dev/null +++ b/test/configs/certs/ca.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEkDCCA3igAwIBAgIUSZwW7btc9EUbrMWtjHpbM0C2bSEwDQYJKoZIhvcNAQEL +BQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM +B1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl +IEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw +MjMwMlowcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV +BAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmlj +YXRlIEF1dGhvcml0eSAyMDIyLTA4LTI3MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAqilVqyY8rmCpTwAsLF7DEtWEq37KbljBWVjmlp2Wo6TgMd3b537t +6iO8+SbI8KH75i63RcxV3Uzt1/L9Yb6enDXF52A/U5ugmDhaa+Vsoo2HBTbCczmp +qndp7znllQqn7wNLv6aGSvaeIUeYS5Dmlh3kt7Vqbn4YRANkOUTDYGSpMv7jYKSu +1ee05Rco3H674zdwToYto8L8V7nVMrky42qZnGrJTaze+Cm9tmaIyHCwUq362CxS +dkmaEuWx11MOIFZvL80n7ci6pveDxe5MIfwMC3/oGn7mbsSqidPMcTtjw6ey5NEu +Z0UrC/2lL1FtF4gnVMKUSaEhU2oKjj0ZAQIDAQABo4IBHjCCARowHQYDVR0OBBYE +FP7Pfz4u7sSt6ltviEVsx4hIFIs6MIGuBgNVHSMEgaYwgaOAFP7Pfz4u7sSt6ltv +iEVsx4hIFIs6oXWkczBxMQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5p +YTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UECwwHbmF0cy5pbzEpMCcGA1UEAwwg +Q2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMjItMDgtMjeCFEmcFu27XPRFG6zFrYx6 +WzNAtm0hMAwGA1UdEwQFMAMBAf8wOgYJYIZIAYb4QgENBC0WK25hdHMuaW8gbmF0 +cy1zZXJ2ZXIgdGVzdC1zdWl0ZSB0cmFuc2llbnQgQ0EwDQYJKoZIhvcNAQELBQAD +ggEBAHDCHLQklYZlnzHDaSwxgGSiPUrCf2zhk2DNIYSDyBgdzrIapmaVYQRrCBtA +j/4jVFesgw5WDoe4TKsyha0QeVwJDIN8qg2pvpbmD8nOtLApfl0P966vcucxDwqO +dQWrIgNsaUdHdwdo0OfvAlTfG0v/y2X0kbL7h/el5W9kWpxM/rfbX4IHseZL2sLq +FH69SN3FhMbdIm1ldrcLBQVz8vJAGI+6B9hSSFQWljssE0JfAX+8VW/foJgMSx7A +vBTq58rLkAko56Jlzqh/4QT+ckayg9I73v1Q5/44jP1mHw35s5ZrzpDQt2sVv4l5 +lwRPJFXMwe64flUs9sM+/vqJaIY= +-----END CERTIFICATE----- diff --git a/test/configs/certs/client-cert.pem b/test/configs/certs/client-cert.pem new file mode 100644 index 0000000..1c748f0 --- /dev/null +++ b/test/configs/certs/client-cert.pem @@ -0,0 +1,99 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 38:4c:16:24:9b:04:1c:b3:db:e0:4c:3c:ed:b7:40:7d:68:b5:fa:1f + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27 + Validity + Not Before: Aug 27 20:23:02 2022 GMT + Not After : Aug 24 20:23:02 2032 GMT + Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (2048 bit) + Modulus: + 00:ac:9c:3e:9d:3b:7a:12:56:85:78:ca:df:9c:fc: + 0c:7e:5e:f2:4f:22:33:46:81:38:53:d7:a7:25:8f: + d7:ee:16:13:e2:67:49:88:f6:94:99:f0:a9:a6:db: + fe:7a:17:c9:e3:df:31:73:71:38:70:3a:96:1e:99: + 7b:5d:07:e3:63:e4:e8:bf:99:f7:3d:5c:27:f5:b7: + 37:29:da:ee:82:80:00:d4:c8:d3:1b:36:0d:8b:d3: + 8a:9b:8e:12:a1:4d:0c:c5:22:f8:56:3b:6a:1a:fb: + e9:3d:08:1e:13:7f:55:6e:2e:65:93:9a:90:54:03: + 6d:0d:e6:44:d6:f7:c0:d7:d8:e1:c7:1e:c2:9b:a3: + 6e:88:f1:7c:58:08:a2:9f:13:cc:5b:b9:11:2c:1d: + 23:6f:3a:ae:47:9a:0f:6a:ce:e5:80:34:09:e6:e3: + fd:76:4a:cf:5a:18:bb:9c:c5:c1:74:49:67:77:1b: + ba:28:86:31:a6:fc:12:af:4a:85:1b:73:5b:f4:d6: + 42:ff:0c:1c:49:e7:31:f2:5a:2a:1e:cd:87:cb:22: + ff:70:1c:48:ed:ba:e0:be:f0:bc:9e:e0:dc:59:db: + a5:74:25:58:b3:61:04:f6:33:28:6b:07:25:60:0f: + 72:93:16:6c:9f:b0:ad:4a:18:f7:9e:29:1e:b7:61: + 34:17 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Comment: + nats.io nats-server test-suite certificate + X509v3 Subject Key Identifier: + 1F:14:EF:2B:53:AB:28:4A:93:42:98:AE:85:06:0F:B4:7D:DC:36:AE + X509v3 Authority Key Identifier: + keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A + DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27 + serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21 + + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1, email:derek@nats.io + Netscape Cert Type: + SSL Client + X509v3 Key Usage: + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Client Authentication + Signature Algorithm: sha256WithRSAEncryption + 60:43:0b:c6:11:0b:96:ae:03:dc:77:26:9a:4a:bd:6a:d7:03: + ec:43:16:2d:ba:8c:e5:50:fa:57:a9:1f:2f:a4:15:c3:a8:13: + b9:d3:59:2a:97:7c:ae:ce:a9:f8:44:e4:97:ee:7d:09:dc:74: + 38:80:94:cf:47:e0:84:52:2a:91:44:8a:85:55:da:42:6a:f1: + 91:1a:6e:5a:63:e6:0b:61:3c:0d:b0:aa:17:b8:77:94:32:20: + 4d:20:8f:84:56:64:ae:ef:d8:8d:42:b5:52:4d:b0:1c:46:97: + bc:4c:77:8c:3f:a3:73:43:87:27:71:62:e7:fe:02:de:a1:27: + 77:be:86:29:8f:62:a1:d9:e7:ea:61:33:73:f4:1f:0a:12:14: + 68:eb:7d:8c:71:5b:42:e7:48:10:c9:df:30:3b:5b:eb:69:29: + b6:95:bc:09:fc:01:b0:be:fc:9f:ee:c4:f3:df:a0:01:c5:68: + 20:f5:2f:f8:e7:1c:a5:4c:a8:a8:a2:20:a1:d2:0f:f6:f6:c4: + 0d:f5:26:fd:ea:8b:b5:06:a9:9e:17:35:47:f7:fd:6e:78:3d: + 5f:7a:87:ed:21:b2:4e:e9:6a:d1:d9:ed:0e:cf:43:61:83:7c: + fe:0d:b1:ad:ff:fa:2d:2b:36:9d:99:9c:20:48:21:0d:36:c8: + dd:b6:0a:d8 +-----BEGIN CERTIFICATE----- +MIIE5zCCA8+gAwIBAgIUOEwWJJsEHLPb4Ew87bdAfWi1+h8wDQYJKoZIhvcNAQEL +BQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM +B1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl +IEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw +MjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV +BAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z +dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKycPp07ehJWhXjK35z8 +DH5e8k8iM0aBOFPXpyWP1+4WE+JnSYj2lJnwqabb/noXyePfMXNxOHA6lh6Ze10H +42Pk6L+Z9z1cJ/W3Nyna7oKAANTI0xs2DYvTipuOEqFNDMUi+FY7ahr76T0IHhN/ +VW4uZZOakFQDbQ3mRNb3wNfY4ccewpujbojxfFgIop8TzFu5ESwdI286rkeaD2rO +5YA0Cebj/XZKz1oYu5zFwXRJZ3cbuiiGMab8Eq9KhRtzW/TWQv8MHEnnMfJaKh7N +h8si/3AcSO264L7wvJ7g3FnbpXQlWLNhBPYzKGsHJWAPcpMWbJ+wrUoY954pHrdh +NBcCAwEAAaOCAYwwggGIMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu +aW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU +HxTvK1OrKEqTQpiuhQYPtH3cNq4wga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I +RWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh +MRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD +ZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb +M0C2bSEwOwYDVR0RBDQwMoIJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAA +AAABgQ1kZXJla0BuYXRzLmlvMBEGCWCGSAGG+EIBAQQEAwIHgDALBgNVHQ8EBAMC +BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDQYJKoZIhvcNAQELBQADggEBAGBDC8YR +C5auA9x3JppKvWrXA+xDFi26jOVQ+lepHy+kFcOoE7nTWSqXfK7OqfhE5JfufQnc +dDiAlM9H4IRSKpFEioVV2kJq8ZEablpj5gthPA2wqhe4d5QyIE0gj4RWZK7v2I1C +tVJNsBxGl7xMd4w/o3NDhydxYuf+At6hJ3e+himPYqHZ5+phM3P0HwoSFGjrfYxx +W0LnSBDJ3zA7W+tpKbaVvAn8AbC+/J/uxPPfoAHFaCD1L/jnHKVMqKiiIKHSD/b2 +xA31Jv3qi7UGqZ4XNUf3/W54PV96h+0hsk7patHZ7Q7PQ2GDfP4Nsa3/+i0rNp2Z +nCBIIQ02yN22Ctg= +-----END CERTIFICATE----- diff --git a/test/configs/certs/client-id-auth-cert.pem b/test/configs/certs/client-id-auth-cert.pem new file mode 100644 index 0000000..5786fef --- /dev/null +++ b/test/configs/certs/client-id-auth-cert.pem @@ -0,0 +1,99 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 74:4b:fa:47:3b:79:81:c8:07:a4:c8:d9:70:df:d2:98:bc:a1:27:76 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27 + Validity + Not Before: Aug 27 20:23:02 2022 GMT + Not After : Aug 24 20:23:02 2032 GMT + Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (2048 bit) + Modulus: + 00:e4:2f:ec:ba:66:87:36:a8:09:ab:66:9b:cc:77: + 24:50:e0:29:4d:c9:98:c8:9f:08:1c:a5:66:0a:b6: + 83:38:8a:3a:f3:3c:16:64:fe:13:b6:02:bd:50:64: + 7d:bb:19:a5:ec:e1:1c:2f:1c:30:be:cd:07:c4:33: + 1f:ae:e2:7f:c5:11:f0:d3:1d:2a:4b:28:66:c1:a0: + 09:52:77:64:75:b7:86:a9:63:cf:a4:29:c8:ad:87: + 3d:79:71:6b:19:34:86:23:80:ed:ed:e6:eb:29:c4: + 6f:2c:9a:c8:ef:da:6a:4e:e5:c9:ce:ef:48:87:26: + 11:d3:f2:2c:4f:d2:a8:b6:26:b1:1d:48:49:9f:f5: + 5a:d4:8a:30:a8:67:1c:8a:b9:3a:13:5d:04:82:e4: + 89:aa:4b:ec:13:59:f7:92:32:33:8f:28:df:50:98: + 65:92:0a:35:cc:c5:04:e3:86:ac:69:cc:6a:84:30: + 91:af:af:44:4a:e8:9f:26:0a:f6:1b:78:f6:90:81: + b1:eb:11:e6:85:1e:42:0e:09:15:e5:e3:36:86:b2: + 46:38:b7:81:c2:c4:16:72:b9:a7:55:89:03:78:28: + e0:2e:29:17:ce:58:2f:16:80:34:b2:5e:02:00:11: + 01:2c:35:ac:bb:0e:2d:41:00:6d:0c:85:94:1a:f6: + 7a:8b + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Comment: + nats.io nats-server test-suite certificate + X509v3 Subject Key Identifier: + 86:48:10:75:95:44:01:FC:76:A0:96:E9:35:52:55:A6:38:8C:5F:37 + X509v3 Authority Key Identifier: + keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A + DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27 + serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21 + + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1, email:derek@nats.io + Netscape Cert Type: + SSL Client + X509v3 Key Usage: + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Client Authentication + Signature Algorithm: sha256WithRSAEncryption + 0e:37:24:36:db:25:73:c2:8d:71:e0:44:05:c1:2f:ac:e7:a4: + 20:26:2a:15:69:20:2f:ea:4c:fb:0c:74:4b:29:e0:92:0f:12: + 8a:a0:4a:73:78:ba:81:eb:4d:91:1c:3f:8e:58:cb:3f:30:77: + 6e:95:5d:a6:a6:22:b3:f9:20:0a:75:93:bc:95:6c:b3:1c:c2: + 6a:72:9a:a6:62:1f:ed:a1:df:9c:2f:d7:28:02:08:49:07:c5: + 3b:5b:58:0f:f6:7b:b5:39:de:05:c7:be:c7:89:bf:62:77:56: + eb:96:b9:74:d6:bf:ce:20:8d:ef:0c:08:bb:ec:0d:b2:55:6d: + cb:94:83:88:89:a5:47:c4:95:91:72:10:c5:3c:6a:be:e5:86: + bb:be:96:74:fd:67:d2:08:55:4e:df:e9:96:c4:62:58:93:6a: + 0c:2c:68:a1:93:b9:61:38:98:eb:c4:4a:1c:3f:9f:9d:c2:3c: + b0:eb:62:7f:a0:aa:e9:53:a3:f7:55:5d:43:1d:5a:63:8b:97: + 32:4c:dc:b8:11:0b:65:35:de:2b:f9:af:8e:28:8c:6f:dd:4d: + 13:3e:b9:f8:7b:7f:27:0d:43:a9:da:b6:59:10:d5:9c:a6:97: + 79:97:19:94:f0:8d:0e:13:f7:c1:ee:50:f2:4c:81:6a:e6:0f: + d7:42:c3:67 +-----BEGIN CERTIFICATE----- +MIIE5zCCA8+gAwIBAgIUdEv6Rzt5gcgHpMjZcN/SmLyhJ3YwDQYJKoZIhvcNAQEL +BQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM +B1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl +IEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw +MjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV +BAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z +dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOQv7LpmhzaoCatmm8x3 +JFDgKU3JmMifCBylZgq2gziKOvM8FmT+E7YCvVBkfbsZpezhHC8cML7NB8QzH67i +f8UR8NMdKksoZsGgCVJ3ZHW3hqljz6QpyK2HPXlxaxk0hiOA7e3m6ynEbyyayO/a +ak7lyc7vSIcmEdPyLE/SqLYmsR1ISZ/1WtSKMKhnHIq5OhNdBILkiapL7BNZ95Iy +M48o31CYZZIKNczFBOOGrGnMaoQwka+vREronyYK9ht49pCBsesR5oUeQg4JFeXj +NoayRji3gcLEFnK5p1WJA3go4C4pF85YLxaANLJeAgARASw1rLsOLUEAbQyFlBr2 +eosCAwEAAaOCAYwwggGIMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu +aW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU +hkgQdZVEAfx2oJbpNVJVpjiMXzcwga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I +RWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh +MRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD +ZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb +M0C2bSEwOwYDVR0RBDQwMoIJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAA +AAABgQ1kZXJla0BuYXRzLmlvMBEGCWCGSAGG+EIBAQQEAwIHgDALBgNVHQ8EBAMC +BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDQYJKoZIhvcNAQELBQADggEBAA43JDbb +JXPCjXHgRAXBL6znpCAmKhVpIC/qTPsMdEsp4JIPEoqgSnN4uoHrTZEcP45Yyz8w +d26VXaamIrP5IAp1k7yVbLMcwmpymqZiH+2h35wv1ygCCEkHxTtbWA/2e7U53gXH +vseJv2J3VuuWuXTWv84gje8MCLvsDbJVbcuUg4iJpUfElZFyEMU8ar7lhru+lnT9 +Z9IIVU7f6ZbEYliTagwsaKGTuWE4mOvEShw/n53CPLDrYn+gqulTo/dVXUMdWmOL +lzJM3LgRC2U13iv5r44ojG/dTRM+ufh7fycNQ6natlkQ1Zyml3mXGZTwjQ4T98Hu +UPJMgWrmD9dCw2c= +-----END CERTIFICATE----- diff --git a/test/configs/certs/client-id-auth-key.pem b/test/configs/certs/client-id-auth-key.pem new file mode 100644 index 0000000..d88ed6a --- /dev/null +++ b/test/configs/certs/client-id-auth-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDkL+y6Zoc2qAmr +ZpvMdyRQ4ClNyZjInwgcpWYKtoM4ijrzPBZk/hO2Ar1QZH27GaXs4RwvHDC+zQfE +Mx+u4n/FEfDTHSpLKGbBoAlSd2R1t4apY8+kKcithz15cWsZNIYjgO3t5uspxG8s +msjv2mpO5cnO70iHJhHT8ixP0qi2JrEdSEmf9VrUijCoZxyKuToTXQSC5ImqS+wT +WfeSMjOPKN9QmGWSCjXMxQTjhqxpzGqEMJGvr0RK6J8mCvYbePaQgbHrEeaFHkIO +CRXl4zaGskY4t4HCxBZyuadViQN4KOAuKRfOWC8WgDSyXgIAEQEsNay7Di1BAG0M +hZQa9nqLAgMBAAECggEBAI3c3aflJhrszVYqLKIpUAKX2hXqR3oypLBqg84VOe9k +wNGHgcS30TlO6rOYRjKT93wVV5hSRlvYzANGZWQsnJLAXKBjeW/QZlHqVOFYKLSm +rKmSy/ybnY+EjMt7n8HDzcE03rcQ4RLOdO+eK14yw/TZF2X3jXe4S00hOjtGFG1o +5BvIaN1HMnrV4G5WsbD2n4gE17lLGLlmn1FhbZaI3/uJgSvM/RgRzzSOSZL+XOzo +pyoMAm9DTI8U020qMuU23KEnjY2Wg68bMNMF38H7z73W+whRVQTyHq/mw0HhD4K7 +P15eWaBUytmRsAkeoybO1hEfGQbUYVdPOtqrvxCCp8ECgYEA83Jcmeiv5DRaH8Sk +/K+O0LM1CDfdWoM/w+Z2frooKqjHKsSa8B8fqI/2AIafgLorksVb2WA8nzzhBvT+ +cIO9NiFbFkFVeLbrGc7supzPVjT1ZAFj7Qj9T0FmrMK937lnOmRNp6DU9t0hEJuF +4wUly0smzVZiBiFRfXsPzANX2T8CgYEA7/Qg401pp99rQfjgCC70n/jVgI1cOFKh +pAdJHU+64GiozP4adMN/fd3fqqG3XnvFG8mRViwgL57iiIvxbbhx5pXUIZZO5LKC +xwuMBzFhX8f4JAR7KANetOIsNF/ML7wTNvGhlUgc3CYXc/Fa1Wh78Xf4vhSd04KA +OdkYuuVk37UCgYEAhCb4jbP6h27D3arpxSGn7TLa/vMUfiXxX26jtHdphn7IXzcK +xH6guOgtKmvp+f8V2D66dW4AepCZtyUXWgypkdDZmWMt+rGRPhlN+J9XDf0BmKAI +lovitjtSeUXdvKzwlIoOfYiZEslHQbSrIWmR8qGBFsZlv94mVm+PS7gk4BECgYEA +ts6t054hlXSAGYXK4FEtq0Z/Ge4YSQyi+v1V9Y/NlrQFjg81Bqn+Ul0bzrpfogr3 +cyEQqa76Ym1QtqivKWEw3XReZaxGtLNPMOeaKcy0G62UXZRQY36Vw4bgGJK3U9Kc +bOqPqNSEsDARBBLnmdh9PMyi4+V/DCnLGMdNsO2c+VUCgYEAqz79qg3S+GON2woM +Npe72bNBJC9UCciA4XpWkFB0f4AiPtFU0VW8fjZ4ZT6rd4sxvl727Y/FYgPcl3Dg +r0zWWZAsaZ4LQRFk7EG9kMYe3Qsc/1211c5fg5dwmJwEybgW1gVvx/cb9qnbBifo +hyWhzlSGwd/pB4skREo/fbNM3hA= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/client-key.pem b/test/configs/certs/client-key.pem new file mode 100644 index 0000000..9f02b6c --- /dev/null +++ b/test/configs/certs/client-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCsnD6dO3oSVoV4 +yt+c/Ax+XvJPIjNGgThT16clj9fuFhPiZ0mI9pSZ8Kmm2/56F8nj3zFzcThwOpYe +mXtdB+Nj5Oi/mfc9XCf1tzcp2u6CgADUyNMbNg2L04qbjhKhTQzFIvhWO2oa++k9 +CB4Tf1VuLmWTmpBUA20N5kTW98DX2OHHHsKbo26I8XxYCKKfE8xbuREsHSNvOq5H +mg9qzuWANAnm4/12Ss9aGLucxcF0SWd3G7oohjGm/BKvSoUbc1v01kL/DBxJ5zHy +WioezYfLIv9wHEjtuuC+8Lye4NxZ26V0JVizYQT2MyhrByVgD3KTFmyfsK1KGPee +KR63YTQXAgMBAAECggEBAKc6FHt2NPTxOAxn2C6aDmycBftesfiblnu8EWaVrmgu +oYMV+CsmYZ+mhmZu+mNFCsam5JzoUvp/+BKbNeZSjx2nl0qRmvOqhdhLcbkuLybl +ZmjAS64wNv2Bq+a6xRfaswWGtLuugkS0TCph4+mV0qmVb7mJ5ExQqWXu8kCl9QHn +uKacp1wVFok9rmEI+byL1+Z01feKrkf/hcF6dk62U7zHNPajViJFTDww7hiHyfUH +6qsxIe1UWSNKtE61haEHkzqbDIDAy79jX4t3JobLToeVNCbJ7BSPf2IQSPJxELVL +sidIJhndEjsbDR2CLpIF/EjsiSIaP7jh2zC9fxFpgSkCgYEA1qH0PH1JD5FqRV/p +n9COYa6EifvSymGo4u/2FHgtX7wNSIQvqAVXenrQs41mz9E65womeqFXT/AZglaM +1PEjjwcFlDuLvUEYYJNgdXrIC515ZXS6TdvJ0JpQJLx28GzZ7h31tZXfwn68C3/i +UGEHp+nN1BfBBQnsqvmGFFvHZFUCgYEAzeDlZHHijBlgHU+kGzKm7atJfAGsrv6/ +tw7CIMEsL+z/y7pl3nwDLdZF+mLIvGuKlwIRajEzbYcEuVymCyG2/SmPMQEUf6j+ +C1OmorX9CW8OwHmVCajkIgKn0ICFsF9iFv6aYZmm1kG48AIuYiQ7HOvY/MlilqFs +1p8sw6ZpQrsCgYEAj7Z9fQs+omfxymYAXnwc+hcKtAGkENL3bIzULryRVSrrkgTA +jDaXbnFR0Qf7MWedkxnezfm+Js5TpkwhnGuiLaC8AZclaCFwGypTShZeYDifEmno +XT2vkjfhNdfjo/Ser6vr3BxwaSDG9MQ6Wyu9HpeUtFD7c05D4++T8YnKpskCgYEA +pCkcoIAStcWSFy0m3K0B3+dBvAiVyh/FfNDeyEFf24Mt4CPsEIBwBH+j4ugbyeoy +YwC6JCPBLyeHA8q1d5DVmX4m+Fs1HioBD8UOzRUyA/CzIZSQ21f5OIlHiIDCmQUl +cNJpBUQAfT2AmpgSphzfqcsBhWeLHjLvVx8rEYLC0fsCgYAiHdPZ3C0f7rWZP93N +gY4DuldiO4d+KVsWAdBxeNgPznisUI7/ZZ/9NvCxGvA5NynyZr0qlpiKzVvtFJG8 +1ZPUuFFRMAaWn9h5C+CwMPgk65tFC6lw/el0hpmcocSXVdiJEbkV0rnv9iGh0CYX +HMACGrYlyZdDYM0CH/JAM+K/QQ== +-----END PRIVATE KEY----- diff --git a/test/configs/certs/ocsp/ca-cert.pem b/test/configs/certs/ocsp/ca-cert.pem new file mode 100644 index 0000000..b84dccd --- /dev/null +++ b/test/configs/certs/ocsp/ca-cert.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDqDCCApCgAwIBAgIBATANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe +Fw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMG0xCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJDQTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzEQMA4GA1UECgwHU3lu +YWRpYTEQMA4GA1UECwwHbmF0cy5pbzEVMBMGA1UEAwwMbG9jYWxob3N0IGNhMIIB +IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwy+fupDc9MZldhetmGqPJtuM +sp5VV6W9amlzkTck15B9Vc3laC6ph7Ble7FrT2L0sjG3U94MwU9/AHTXOmZdmbjM +FpkjkLIVdFkbcWiErXYWDBHdA6dzOu+dagn0OyxRDjfqo1QUVKYVNu8Jw6MyWHXJ +gljFl2ymHaQEhta/87tSvPULZ7gcEZ5CPFLENHWOlJPtQrPhJHDKjS8XHlbE1uXp +i8kHqPCkImlv/s7Jw/QRIknV/kiAXAWGJCMbqLDG9JEatp7ektytcwMCr9pz9VzF +6O/4LvOC8UCbu50eW7OudppN8G18IF3cMgH9jWsJpgVmXfJR+VZNe92/6ePTgQID +AQABo1MwUTAdBgNVHQ4EFgQU7upCnRG44j5THcgKd28H4ESXBFkwHwYDVR0jBBgw +FoAU7upCnRG44j5THcgKd28H4ESXBFkwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG +9w0BAQsFAAOCAQEAfBHCa8sm0e767+oIZj3JIRi9MWN24hB9i4lVjDrwdOMaapMC +YLLj5urqIgjOULjdsxBMzdNgNgH1vPenRYUUvIQcq7tk1q8DpfvmHEg2DHajpTAC +DroutE5fYtlmFPSQ5UGG1if237osd6pDarVhGAdxex4YhwM+y+OXgpLqk6oC85oI +fatf+hcovwFOlNeOTUqNZW6fEC+iFdH5g4+dtlx2LAJLpW57+5z25iTH7z16nUwB +Vi76fezpaGA3xwkP/NMujgD4MbpVpF22a0YdK5fjUjXFwRI4Vu1zAjyJFhVuOWCS +yT9yNzidtD5pho+Iv3JMzu54VWSq7nSUoPmKHQ== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp/ca-key.pem b/test/configs/certs/ocsp/ca-key.pem new file mode 100644 index 0000000..fdd021e --- /dev/null +++ b/test/configs/certs/ocsp/ca-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAwy+fupDc9MZldhetmGqPJtuMsp5VV6W9amlzkTck15B9Vc3l +aC6ph7Ble7FrT2L0sjG3U94MwU9/AHTXOmZdmbjMFpkjkLIVdFkbcWiErXYWDBHd +A6dzOu+dagn0OyxRDjfqo1QUVKYVNu8Jw6MyWHXJgljFl2ymHaQEhta/87tSvPUL +Z7gcEZ5CPFLENHWOlJPtQrPhJHDKjS8XHlbE1uXpi8kHqPCkImlv/s7Jw/QRIknV +/kiAXAWGJCMbqLDG9JEatp7ektytcwMCr9pz9VzF6O/4LvOC8UCbu50eW7OudppN +8G18IF3cMgH9jWsJpgVmXfJR+VZNe92/6ePTgQIDAQABAoIBAC5RXtYnCkgLzIf5 +lnhU0SOndfvtFtN1wT0/SO1s6JE++H8kHQxcBl7svShdMdnk4axnn9mHF//HnZu+ +HlT9dbjE4al7LbVojS7O9nQzGUkQfKrgklILqoyR0AkZ05s3KQT2v/eCPFDaGK6w +iuCiGZBkYy1LY5hLcCAYi/pze5mar1m6S2ZqvfpZWjlrlZOVi73cZOq91Es2g/iU +TYaiR4HGHJ0McXFNjL6q7DxCBkRLWb6i+Xy+9+84XFZAVzRBAo1EaFJBPg0p88EC +VKxQt4X0jgWYgLRABAqoQ+DkNEEoaCEQuUV67aIcIyV++ddn1FsegPtGYo38z7mr +M+fzUAECgYEA/XL53y3eNnZ/U90gfr1HYIvyo6WShXEoIFw/s9QJnd5/ZkxcQFWr +wUtMDNyjKFPnTTSLPr+vc3CGYqh8wFNxfids9KP95bMpVN8XGoTI99QqYK2tIgFl +T8q46igOTcrg0f49ecqtQjL9F/dnZqzMlh5nJLMVGqRpNoZz9YpiS0kCgYEAxSaH +uvaW7WxiBC+xf2vvgf585Wn9jh6QDX+MQjA7Ao7Tk4ZjDwAPMJkYd475BWp+DHSl +b74x/nbRHwdLAfkxvYEsv4KrPR0yzdzGrXVATWcOP2bftEGPYVQnlgjjzMpLVXdt +QErRS1vVnkavJLsxOHw++qiiyGENQk8LR2yOTnkCgYBBzy30dluBtskfBIbggdNb +zVrmhSKDhbtOk8VyszcAB/r6nA9EITqkySFpIY039nlTwbX6SBmNlwU97tPduIz6 +ndAbwc02bIvp3reICjyIpU2PpukSsFwXGONk4Zu9NVWlESfzTN4qF0VCiNoPfgTt +Yd2UWO+86D3ti4HmmtUlCQKBgDRFYvdPKfUJJ3O0sXr3QylUMAkjcPadY9QwXR+v +afXjqHUUzG7NtTlNXg9U+PFWqtTimHpoExlEp21yoZCEYYu9FAAyxPQPKckrIAId +dE8RY9WrkORZ/Ynwpg5BjSRe/lpKr8y8CYHRd3Hfi9BRUVuIlaofzAkUsk9CZdsq +DREBAoGBAKfvd2PpW9B8Wu248h/zxkK4M58XQUubkFTvUm6ErH0GWFBwfq8b8Q5Y +7/KnzZ3BefqQcNQKyoM06bTDT/YjlPLZdZtgN8UulGZQjBbLV9EIZJwRj+AUwMQV +X+U8NAibD6yFqnOoJ0P5r8rlKPMg9+BkdZfnbhaCXb/KygYxqGcH +-----END RSA PRIVATE KEY----- diff --git a/test/configs/certs/ocsp/client-cert.pem b/test/configs/certs/ocsp/client-cert.pem new file mode 100644 index 0000000..0f96aec --- /dev/null +++ b/test/configs/certs/ocsp/client-cert.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIID0jCCArqgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe +Fw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMHExCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJDQTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzEQMA4GA1UECgwHU3lu +YWRpYTEQMA4GA1UECwwHbmF0cy5pbzEZMBcGA1UEAwwQbG9jYWxob3N0IGNsaWVu +dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMn1VyxBY4AkODPmOxK5 +VG3F2qQ+0jNFeikwcgJPHvFamqn3cA5AIJIUVmMtBiUfjnperHVKeuPfmW1bJw4E +ne3V2eccDySoAR/BTX4kw0SPtIO3hnHyhOLX4bY4/Xw5OWgw2HMEwEwuoWxd+jpc +GGzXY49J9gRKqxJFXR9tXD6T+1ABZPynqrTm3SYYCJoWq/C6feTSkf13HvnTnf8k +fWcFum1Y5FegAObqbPqJwA0TGiuXSFkqw5oV0uAZzRQ7zqB6V8MB3W1U1pw86F1h +09EN78PrW1yXX1LZrLKwlqPTVh53Y1HuT+mwJkdQjFbOGXwh3x7rmp8+A3QLD4pR +5tsCAwEAAaN5MHcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBeAwHQYDVR0lBBYwFAYI +KwYBBQUHAwEGCCsGAQUFBwMCMD4GA1UdEQQ3MDWHBH8AAAGHEAAAAAAAAAAAAAAA +AAAAAAGCCWxvY2FsaG9zdIIQY2xpZW50LmxvY2FsaG9zdDANBgkqhkiG9w0BAQsF +AAOCAQEAWie6Pz2iJP6F9HfVH7anKVHeIXecwXJj4iLgEONaIcOyMcLPU4cthx1S +OdvKAh+D9tT2PhVaIeDyYTUgFg/aaZUqI/W3odRH5HwQmE2YJDfXQusRtdFDTAUV +XDqFkkNoJo4w3OQmlnQGm6QVReedyQ3jMTvqDRV+pa8gx6aH64jhP9fQRS4WkpYX +d0HjWarV9/GzCP/+vGVZhwrhRG9p4F2ZCsflBzTx0YMGdo+vLDCSjwMbIT9t0T6/ +mt07Q70QSk8M3QAClrqarvLk+5z5XSZjtM06s/Z6opyqK2X8KYcOYX4WQyNFbOpy +0YHy3iqmx/Ii0Zn5XZUXzAVGyJk5Yg== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp/client-key.pem b/test/configs/certs/ocsp/client-key.pem new file mode 100644 index 0000000..4d98188 --- /dev/null +++ b/test/configs/certs/ocsp/client-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAyfVXLEFjgCQ4M+Y7ErlUbcXapD7SM0V6KTByAk8e8Vqaqfdw +DkAgkhRWYy0GJR+Oel6sdUp649+ZbVsnDgSd7dXZ5xwPJKgBH8FNfiTDRI+0g7eG +cfKE4tfhtjj9fDk5aDDYcwTATC6hbF36OlwYbNdjj0n2BEqrEkVdH21cPpP7UAFk +/KeqtObdJhgImhar8Lp95NKR/Xce+dOd/yR9ZwW6bVjkV6AA5ups+onADRMaK5dI +WSrDmhXS4BnNFDvOoHpXwwHdbVTWnDzoXWHT0Q3vw+tbXJdfUtmssrCWo9NWHndj +Ue5P6bAmR1CMVs4ZfCHfHuuanz4DdAsPilHm2wIDAQABAoIBAHu5DodxI5i8F6ZL +1EK7QOri+/gE+FcqqBUVtbKOcCFh5UBc8sv4Izv6s5WcXphdhbaXy0UrtK9nKyIg +ZoOi9nFewlhgCzLkrZObo3K06N9Wvjq3MukZrqkdogw1S61PjUi0K9YCwh+prYCq +7gHUq636IecFY27ro3PVBKCdKZa3jwNIg4Oy5H0dsLgA1ma+tJaZMxDQZGyXvPqD +wGXdkjdIefo7WQHi3yyujSs3Bu76ChmkGQfma48clWiu4HNlWI3Uc8aLBwDWVmjF +S4T8s9HNw+nDWNHvnD45dkgKyTNAGA4rmD01OtuiC44GfZkRtupXcI0xA7d0lzd+ +yvMBlfkCgYEA/9zpUgL2Uw24H28pDXiV6Lc0PmCmHVLPQxQpDzsudX/ukaUl0tMC +tOL9575sluXOXDSkuoMX5nDX/8C6fPNSbi4AzrqDxQ7Dc05W41MvlJ9DdIzlozEm +LFJ4RfXccxcVCXRrgNDNHpkMKQkk3Z+zmlZeNbbhWKs2CgZGrvyFnW8CgYEAyhEJ +Z1aRC6XrP+GbMCiN7w2X6ZcDNm89o8jXR/94hTDGvyyQ4XIs0245XjHNYVP/ghzi +iFpIRlph4QpKZebWiqgkwlsNsPKQmiBn5uqobo3VsUt+rFIhaX50GyM5cxSbSRkl +/i9Z7ZZdj/dcsviURthKZiBn2uw+aROLCgKm71UCgYEA102h2K09cm4c/fagaQGL +xCRGBid2IT7Jwfx5AKQgWCerLUv3JA0EPgq09gm7fs8qc1SpOXmO5w8V89TOGM74 +ElcLvuocb/oYZjMJ0ojxhPLv5Geb5VM6eBl9tAFL3F0UCry4qdEKijDnlrBnIUd9 +7uW2qSSXQ/Huq0jUufMszGkCgYEAnayLnPJkviUbG77svLh4gHgn+SNYY2qMO7il +nE3R+oRkIZsh9nmEVvtkkobkDzVfZGUrs2BXk2ZFiDfic/+bm5i3Dl3EojW09j+h +NAQZqCLPA8i4MLjpz4rYCLEEzDLhNToFdoH2dzllCsjnsdPcyCdQbr6Mq7y6un2A +ejA1mP0CgYABUHeLemQTvsciXVAy7ZggDYmFIVPvxwYejAXWAeUdq2FEFWIqmywy +gTISeMrWpaCpwRr97Rez5bLgYe1Crqujd03uzYzoMtuiE0027XE6CaqeDGEk7jWQ +bnbanxoy7Ax6yjcEbqyaaG4ZbVXu5EbsCMPFE3mws3AqyF1/54YmbQ== +-----END RSA PRIVATE KEY----- diff --git a/test/configs/certs/ocsp/desgsign/ca-cert.pem b/test/configs/certs/ocsp/desgsign/ca-cert.pem new file mode 100644 index 0000000..cdd05b8 --- /dev/null +++ b/test/configs/certs/ocsp/desgsign/ca-cert.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEITCCAwmgAwIBAgIUBAbcj/g36HWhhWkPhGuGuEczfycwDQYJKoZIhvcNAQEL +BQAwgZ8xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEWMBQGA1UEBwwNU2FuIEZy +YW5jaXNjbzETMBEGA1UECgwKTG9jYWwgSG9zdDEiMCAGA1UECwwZTG9jYWwgSG9z +dCBUcnVzdCBTZXJ2aWNlczEVMBMGA1UEAwwMY2EubG9jYWxob3N0MRswGQYJKoZI +hvcNAQkBFgxjYUBsb2NhbGhvc3QwHhcNMjExMTE2MTgwMzMwWhcNMjExMjE2MTgw +MzMwWjCBnzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4g +RnJhbmNpc2NvMRMwEQYDVQQKDApMb2NhbCBIb3N0MSIwIAYDVQQLDBlMb2NhbCBI +b3N0IFRydXN0IFNlcnZpY2VzMRUwEwYDVQQDDAxjYS5sb2NhbGhvc3QxGzAZBgkq +hkiG9w0BCQEWDGNhQGxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAMK9Y88QNOVAyN43ESTQVCFJkBnpgAup/1oFAAv77pLuV0Wl1UVO8gKM +W5k1qTahVSZXkX+gL0uoRe7n7XZb40iIcza1emEdMKcIOxSQ/jzj8xikziBdZrtk +cM/MkpkDCg/45wuPUuq4umjXn7sJ3XwpKmR87PTNcSVRG7YCj8Vqze/KBhCDmTPa +DPs0GGWBqlGHjx8LmG9WYPIxbV9wi5SYUQ4Iww29xEoZBcTPd1YluaAzF9OsYsFu +NIuaYjh/XpZFbxGA99BfoYtIK1/X+CO1OEkCTcfS2DRwPzGV9uj39yGEmd/cOji3 +Nqj55DglphjbdGR87CTKiwinKaLwXzMCAwEAAaNTMFEwHQYDVR0OBBYEFHTL54rz +mJCUnDGtWCqne0MRIS0iMB8GA1UdIwQYMBaAFHTL54rzmJCUnDGtWCqne0MRIS0i +MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAJinrS32gInu3tTB +u0Q3nXj99YMjHfyIO7kFrvyoZo4IEJhcPefhtt40Gbmnzo8ZJsx6HjAVDMnkj57b +XkdRH23JKpcBeTHz1xqIcf93ij9HjO6AcVDCl5Ew3Tir2F81j/skXKqUZL4KeD0F +gF5nQTEkpH9vlIg35T0eWygchRumKmRb+aw6/QvAmnfEWZskghV0zzPvG51B99bp +pZPaqZ25GyxqxwBcA6pUdbmBKoXRATyDwGcLhX40Rb6nCBeb2vQQTJ51lrEsCurh +rU67Tcf+r9nS9fIWPoasmdaamEXFZNoZbw2g/EeH1OTXfR87sGCXZpBXIgsg9QIH +tCztUKY= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp/desgsign/ca-chain-cert.pem b/test/configs/certs/ocsp/desgsign/ca-chain-cert.pem new file mode 100644 index 0000000..7f3e3ef --- /dev/null +++ b/test/configs/certs/ocsp/desgsign/ca-chain-cert.pem @@ -0,0 +1,52 @@ +-----BEGIN CERTIFICATE----- +MIIEmTCCA4GgAwIBAgIBATANBgkqhkiG9w0BAQsFADCBnzELMAkGA1UEBhMCVVMx +CzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNpc2NvMRMwEQYDVQQKDApM +b2NhbCBIb3N0MSIwIAYDVQQLDBlMb2NhbCBIb3N0IFRydXN0IFNlcnZpY2VzMRUw +EwYDVQQDDAxjYS5sb2NhbGhvc3QxGzAZBgkqhkiG9w0BCQEWDGNhQGxvY2FsaG9z +dDAeFw0yMTExMTYxODAzMzBaFw0yOTEwMTQxMzU3MjZaMIGsMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEzARBgNVBAoM +CkxvY2FsIEhvc3QxITAfBgNVBAsMGExvY2FsIEhvc3QgVHJ1c3QgU2VydmljZTEc +MBoGA1UEAwwTY2EtaW50ZXJtLmxvY2FsaG9zdDEiMCAGCSqGSIb3DQEJARYTY2Et +aW50ZXJtQGxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALljsfg9PNMYfmCgDQ4z+7VsfqUEHB+t/6NkACkBQi6NjNKhXxD1APfI6K5ZeCMo +jeSTmjmIDB7Shu9oU8Db5LFBxe7PE+dbHpX0/SAzBkW5UyCoX691MHhb2VbrS94q +ttBtl0U/DgtCwfVxHWf1GBhsJqsPUnBLRInxB/BMXlzhw8FSS6B5PkHmEyJgiUoa +AyN60sEOUswa8IDzHlH/mDuYDUJKCHIgenVH7N8qZZfkK68bJfSjNHoPWT3sWPYq +NEyPgvLqsb1ewTSAsKWTAEefhgE/DzVpsxEw613VvW/n/Tl9jay4IpQ38cuL8HhP +1d7pF5kpg+tCDIjK6zjCCm8CAwEAAaOB0DCBzTAPBgNVHRMBAf8EBTADAQH/MB0G +A1UdDgQWBBRMb9Of/qEuaW3cAAI66e/fsBYy2zALBgNVHQ8EBAMCAcYwHwYDVR0j +BBgwFoAUdMvnivOYkJScMa1YKqd7QxEhLSIwKgYDVR0lAQH/BCAwHgYIKwYBBQUH +AwkGCCsGAQUFBwMBBggrBgEFBQcDAjBBBgNVHREEOjA4hwR/AAABhxAAAAAAAAAA +AAAAAAAAAAABghNjYS1pbnRlcm0ubG9jYWxob3N0gglsb2NhbGhvc3QwDQYJKoZI +hvcNAQELBQADggEBACjDl6q3l6F1YouwalK2sQJyn8gBjx2W5w95n+zuQrw4UGkp +JE+wOVCmlj7mv77GOjZUhTPqW6cDKQfjCDiLWiw2Gw0HiLxgxiyOB3hS9hGCSiPx +W7MLJ81dINq+ogEO22gC8fb0BoyqdkqteyLsjzz6aKaUnCB8UCMD/Ysjm/beVvPy +/i1K7Ki1NnqeiCGOO8WTHdNOn2YWrt6Exbh/nsFMB7/wE8poQs2ynotTsFNBwuTr +y6iwdnVnJLyQd6AzJiE5gOdvqRobKhUzh6C1d7wwDl3V4WAydsPXjTboRzJ0lFS+ +Tp0aK3KjLHaWIehFoVgLuZhaLwXKpyRlyK54z14= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEITCCAwmgAwIBAgIUBAbcj/g36HWhhWkPhGuGuEczfycwDQYJKoZIhvcNAQEL +BQAwgZ8xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEWMBQGA1UEBwwNU2FuIEZy +YW5jaXNjbzETMBEGA1UECgwKTG9jYWwgSG9zdDEiMCAGA1UECwwZTG9jYWwgSG9z +dCBUcnVzdCBTZXJ2aWNlczEVMBMGA1UEAwwMY2EubG9jYWxob3N0MRswGQYJKoZI +hvcNAQkBFgxjYUBsb2NhbGhvc3QwHhcNMjExMTE2MTgwMzMwWhcNMjExMjE2MTgw +MzMwWjCBnzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4g +RnJhbmNpc2NvMRMwEQYDVQQKDApMb2NhbCBIb3N0MSIwIAYDVQQLDBlMb2NhbCBI +b3N0IFRydXN0IFNlcnZpY2VzMRUwEwYDVQQDDAxjYS5sb2NhbGhvc3QxGzAZBgkq +hkiG9w0BCQEWDGNhQGxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAMK9Y88QNOVAyN43ESTQVCFJkBnpgAup/1oFAAv77pLuV0Wl1UVO8gKM +W5k1qTahVSZXkX+gL0uoRe7n7XZb40iIcza1emEdMKcIOxSQ/jzj8xikziBdZrtk +cM/MkpkDCg/45wuPUuq4umjXn7sJ3XwpKmR87PTNcSVRG7YCj8Vqze/KBhCDmTPa +DPs0GGWBqlGHjx8LmG9WYPIxbV9wi5SYUQ4Iww29xEoZBcTPd1YluaAzF9OsYsFu +NIuaYjh/XpZFbxGA99BfoYtIK1/X+CO1OEkCTcfS2DRwPzGV9uj39yGEmd/cOji3 +Nqj55DglphjbdGR87CTKiwinKaLwXzMCAwEAAaNTMFEwHQYDVR0OBBYEFHTL54rz +mJCUnDGtWCqne0MRIS0iMB8GA1UdIwQYMBaAFHTL54rzmJCUnDGtWCqne0MRIS0i +MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAJinrS32gInu3tTB +u0Q3nXj99YMjHfyIO7kFrvyoZo4IEJhcPefhtt40Gbmnzo8ZJsx6HjAVDMnkj57b +XkdRH23JKpcBeTHz1xqIcf93ij9HjO6AcVDCl5Ew3Tir2F81j/skXKqUZL4KeD0F +gF5nQTEkpH9vlIg35T0eWygchRumKmRb+aw6/QvAmnfEWZskghV0zzPvG51B99bp +pZPaqZ25GyxqxwBcA6pUdbmBKoXRATyDwGcLhX40Rb6nCBeb2vQQTJ51lrEsCurh +rU67Tcf+r9nS9fIWPoasmdaamEXFZNoZbw2g/EeH1OTXfR87sGCXZpBXIgsg9QIH +tCztUKY= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp/desgsign/ca-interm-cert.pem b/test/configs/certs/ocsp/desgsign/ca-interm-cert.pem new file mode 100644 index 0000000..8d2ffe2 --- /dev/null +++ b/test/configs/certs/ocsp/desgsign/ca-interm-cert.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEmTCCA4GgAwIBAgIBATANBgkqhkiG9w0BAQsFADCBnzELMAkGA1UEBhMCVVMx +CzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNpc2NvMRMwEQYDVQQKDApM +b2NhbCBIb3N0MSIwIAYDVQQLDBlMb2NhbCBIb3N0IFRydXN0IFNlcnZpY2VzMRUw +EwYDVQQDDAxjYS5sb2NhbGhvc3QxGzAZBgkqhkiG9w0BCQEWDGNhQGxvY2FsaG9z +dDAeFw0yMTExMTYxODAzMzBaFw0yOTEwMTQxMzU3MjZaMIGsMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEzARBgNVBAoM +CkxvY2FsIEhvc3QxITAfBgNVBAsMGExvY2FsIEhvc3QgVHJ1c3QgU2VydmljZTEc +MBoGA1UEAwwTY2EtaW50ZXJtLmxvY2FsaG9zdDEiMCAGCSqGSIb3DQEJARYTY2Et +aW50ZXJtQGxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALljsfg9PNMYfmCgDQ4z+7VsfqUEHB+t/6NkACkBQi6NjNKhXxD1APfI6K5ZeCMo +jeSTmjmIDB7Shu9oU8Db5LFBxe7PE+dbHpX0/SAzBkW5UyCoX691MHhb2VbrS94q +ttBtl0U/DgtCwfVxHWf1GBhsJqsPUnBLRInxB/BMXlzhw8FSS6B5PkHmEyJgiUoa +AyN60sEOUswa8IDzHlH/mDuYDUJKCHIgenVH7N8qZZfkK68bJfSjNHoPWT3sWPYq +NEyPgvLqsb1ewTSAsKWTAEefhgE/DzVpsxEw613VvW/n/Tl9jay4IpQ38cuL8HhP +1d7pF5kpg+tCDIjK6zjCCm8CAwEAAaOB0DCBzTAPBgNVHRMBAf8EBTADAQH/MB0G +A1UdDgQWBBRMb9Of/qEuaW3cAAI66e/fsBYy2zALBgNVHQ8EBAMCAcYwHwYDVR0j +BBgwFoAUdMvnivOYkJScMa1YKqd7QxEhLSIwKgYDVR0lAQH/BCAwHgYIKwYBBQUH +AwkGCCsGAQUFBwMBBggrBgEFBQcDAjBBBgNVHREEOjA4hwR/AAABhxAAAAAAAAAA +AAAAAAAAAAABghNjYS1pbnRlcm0ubG9jYWxob3N0gglsb2NhbGhvc3QwDQYJKoZI +hvcNAQELBQADggEBACjDl6q3l6F1YouwalK2sQJyn8gBjx2W5w95n+zuQrw4UGkp +JE+wOVCmlj7mv77GOjZUhTPqW6cDKQfjCDiLWiw2Gw0HiLxgxiyOB3hS9hGCSiPx +W7MLJ81dINq+ogEO22gC8fb0BoyqdkqteyLsjzz6aKaUnCB8UCMD/Ysjm/beVvPy +/i1K7Ki1NnqeiCGOO8WTHdNOn2YWrt6Exbh/nsFMB7/wE8poQs2ynotTsFNBwuTr +y6iwdnVnJLyQd6AzJiE5gOdvqRobKhUzh6C1d7wwDl3V4WAydsPXjTboRzJ0lFS+ +Tp0aK3KjLHaWIehFoVgLuZhaLwXKpyRlyK54z14= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp/desgsign/ca-interm-key.pem b/test/configs/certs/ocsp/desgsign/ca-interm-key.pem new file mode 100644 index 0000000..95eaee2 --- /dev/null +++ b/test/configs/certs/ocsp/desgsign/ca-interm-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAuWOx+D080xh+YKANDjP7tWx+pQQcH63/o2QAKQFCLo2M0qFf +EPUA98jorll4IyiN5JOaOYgMHtKG72hTwNvksUHF7s8T51selfT9IDMGRblTIKhf +r3UweFvZVutL3iq20G2XRT8OC0LB9XEdZ/UYGGwmqw9ScEtEifEH8ExeXOHDwVJL +oHk+QeYTImCJShoDI3rSwQ5SzBrwgPMeUf+YO5gNQkoIciB6dUfs3ypll+Qrrxsl +9KM0eg9ZPexY9io0TI+C8uqxvV7BNICwpZMAR5+GAT8PNWmzETDrXdW9b+f9OX2N +rLgilDfxy4vweE/V3ukXmSmD60IMiMrrOMIKbwIDAQABAoIBAHrcuHCk3O+llvOg +fqaACvvkaFYiUCUqonX9ayHBxMOnacGZ0rAhPz/39US+5KCgWtE2hQpFwIqYyXZW +dNMmp/xVc8DdmfpE6BNHpo21Yx3IQXAuRiO5DaNmc7ZoAAK/2iJtzsfEjyElX/ey +vGNOCmb9He9WGzNynnywsasuYYc5sr6+2b5AN0yMxCggJY9tdyxAK5Tgh0ahMumf +gmNNziSqwRhJ8V9natT7TzFqheQe8u6NuC1uSZ7ARqHsf9v9nfjq3jscBjkcQexx +kuBSXS43DoksMangyDRK7epbQfQf0KmmrScdlCy3vJquKWG4wsSaob+Wsyg/XvOa +k/bCsuECgYEA55wfpvuzB4J5S9nes7D9Fd3mGi/HBjEflWFXIv0uns4zljDTEd1k +/Ye13UfvIyVcKMiLzmVsGpf6sESdDo6Fgd/g6gjOfeGerc3t/MZLYsXVukheABhF +YJX+4xvwVp7QxEXcftSbZjcoTL4HdTYDeKyfn0KftTm3C26MHJhron8CgYEAzOmI +q8rqW/6CbGb12zxDNKnKgANS0MNFeLUKdJqwrX8LxYTXmXypdlwOJ66D1pAvhblR +j7A7QBItbkvr0Qwv56a9fa2F2Nuot5VmJyHz6mL9cRGrhIZL6aFIVZ3C1iqhQKoS +/6mAxPSWjD3u7eCFBwKeZwmWdSx3zM9VaefDwBECgYEAwWDycJqRJUEEA5faQNAS +z/IhEFY552qWg0Pt3DHmfgOOwOTtJmpiyuhHqYVJHmAwLYEccezusNmaHxh6xc+r +mv+RK/bEagg6U8Wv4jCyerrRs6J+kbeyHW2/jmIibkBV8Lqf2mmrglGlXUYAthWu +GlCPSgr3i/mvYmUfqTR+EgMCgYEAvpX7GyWpIpUug1qkExwSufmuMbBlp2vnwqRI +LDnwV/4RWc37pXNwPnjSZZAIaVlEChFaTdWw1h/SB4MvuwilycSo/CqXkiKD4vRe +xcjrj7YwWakAqUsrcgojOBZ6sC2IO5e0AfyKmyWOnLPB9Zfcwq9p2xXszeDlMCYr +IEyDIbECgYBT8aN5dimyIduEk4XTQ377l5i/Ng9EVOaxsuX4MhHUUScaWogo18Fq +eggKOwFXD08J0xVk0WORdkyt01/EEwhUmUUPolya8HZK2rpikpp0eIX68qSCMB6q +IEuHf42ip+4d8IbQp7i8SFh8LNaxkUPdJi92fFuwbQ4WeNRiL4CoVg== +-----END RSA PRIVATE KEY----- diff --git a/test/configs/certs/ocsp/desgsign/ca-key.pem b/test/configs/certs/ocsp/desgsign/ca-key.pem new file mode 100644 index 0000000..6eac9dd --- /dev/null +++ b/test/configs/certs/ocsp/desgsign/ca-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAwr1jzxA05UDI3jcRJNBUIUmQGemAC6n/WgUAC/vuku5XRaXV +RU7yAoxbmTWpNqFVJleRf6AvS6hF7uftdlvjSIhzNrV6YR0wpwg7FJD+POPzGKTO +IF1mu2Rwz8ySmQMKD/jnC49S6ri6aNefuwndfCkqZHzs9M1xJVEbtgKPxWrN78oG +EIOZM9oM+zQYZYGqUYePHwuYb1Zg8jFtX3CLlJhRDgjDDb3EShkFxM93ViW5oDMX +06xiwW40i5piOH9elkVvEYD30F+hi0grX9f4I7U4SQJNx9LYNHA/MZX26Pf3IYSZ +39w6OLc2qPnkOCWmGNt0ZHzsJMqLCKcpovBfMwIDAQABAoIBAANogyWtl+9nOdzB +w/gL+Vz3x6DceFQ77m8p5T1QdHV33GkopNnors2inTvKN6LwIehg4fgE2q8NS+QJ +hRtsMkcjtDiB5plqhF54A2ixvPFd3/RPdhgU6mZfNKY+Y1ZwOi3bYlfOc1oOT5zk +ITSJ0KmKouZNVWxXaCKKD90YoGGJHCxZGPkAQlX/GT3h7g8h6QUYovgKwn788EVi +n78ldLq8AeqSiLZNK76DpnniKtaIwshslUkqPDW22EWk8YM6uvTNQQMuZzV1oBlQ +Ta5rJizOuWcIi+Mu2/QAhSVjhMSjXMyLoIrO5WkHpj9LzRwys9SQM7A1njZ9Xatl +iWWPmKECgYEA6pm1qgMG84ZhsFTxojYkrR/bs7LuLCSAIUfEoyypXJTrWNIJv5eb +nZ/HL5o90KQHYN5T/LpZJfe/Etz+4npS6ahTXM71o51EpIN9DhNzJNduPSDaBqHP +LiomPg+ipgPJ3ziFu4m2dK2y2TW/u9IO8+6Ah/2ozkXX4/kf+DFq+GMCgYEA1IDf +KybhAgWImgf6m3GoZMDQOQLUUcn1cAagUUe2eNUa24lXx67ZmpJZpoHqixYylhfx +Bp8fmMD3q/OGe+j427OlVcomzlDTqnahJ9m2KFfn2tkovfJKYqZqx8r7F7kuqlgM +r9DOAkR5s50tHz+EuQZqC1VZ796YcqnuIVoWbvECgYAhzJdxsRH1T+0PHI3bkvVh +w+9BSowp6/BR2ycnYy3bWtE1cL1azxrqcLSf1RcG0jsF58It7SMe5zyuGQzX0EvV +whyQiHi3Y9cZ5J/FwWObcTY+tFb1EabpvcTYuCP0yyLweBI8XLDeyo+z35yKEM96 +sWfvL8p8PW/HNoM2nNgOhQKBgQCkxHPForSue5nqTKt84Yi+7l3FBrOX4y8iOJEP +1LngQOQ9OuXMF3/0AOvwViWEyKZaiJ/DEZhPObgQJJeu9foXZ9iXh1HFgRhNwQO9 +wWojJ93Ha7/SX85bZUvANFuyjkxnmjPkEtPZIDz5DrLQ2tBPInEQ7pH4kjDEH4xb +Yd2pEQKBgATSX2zyg3POs3958Ls/4oegfnVclFeUf8zQpwGtsDxP6NtpVti9YGgM +rO63dfeCuAfapxyX+I6iNw5pbDWK3TC8STjWnhhzAUZnZSoIGRB8q0yKCK8TZSMG +3rMw+2LuO+br5tu/Hq+d6YlHbhdYW8JJFEF6QfNT31ltGE5+Nr6D +-----END RSA PRIVATE KEY----- diff --git a/test/configs/certs/ocsp/desgsign/server-01-cert.pem b/test/configs/certs/ocsp/desgsign/server-01-cert.pem new file mode 100644 index 0000000..d638409 --- /dev/null +++ b/test/configs/certs/ocsp/desgsign/server-01-cert.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEiTCCA3GgAwIBAgIBAjANBgkqhkiG9w0BAQsFADCBrDELMAkGA1UEBhMCVVMx +CzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNpc2NvMRMwEQYDVQQKDApM +b2NhbCBIb3N0MSEwHwYDVQQLDBhMb2NhbCBIb3N0IFRydXN0IFNlcnZpY2UxHDAa +BgNVBAMME2NhLWludGVybS5sb2NhbGhvc3QxIjAgBgkqhkiG9w0BCQEWE2NhLWlu +dGVybUBsb2NhbGhvc3QwHhcNMjExMTE2MTgwMzMwWhcNMjkxMDE0MTM1NzI2WjCB +rDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNp +c2NvMRMwEQYDVQQKDApMb2NhbCBIb3N0MSEwHwYDVQQLDBhMb2NhbCBIb3N0IFRy +dXN0IFNlcnZpY2UxHDAaBgNVBAMME3NlcnZlci0wMS5sb2NhbGhvc3QxIjAgBgkq +hkiG9w0BCQEWE3NlcnZlci0wMUBsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQDF2MDDwVEZqC/dje8+soPRapPa9C4V8hmHuCQS9sg27HZv +bJlywJMJE0PKm+612JoVB9p5tsXlV29gKcIK7zLEhesGlhiW9d2/7MOrPBSwqJfo +IeabwlUfUTZ2M3XWGvU5Z65cK+cK7f0FABUzwTqb4/M3UR41cRvNtSK95nwl48Kb +x4Dew4MIqvIN/H1qmjkNM8lsDHrCV2IyyYc1W1zDqZVU1JdpjO7tBBL6pkwlM31k +8xjMr+hiVA64N+fESCauf4xArnqcHdhAkfnX+8DWCfSLFjP1lBZZTw1ByLEBBeNr +kHt22RhO1mvBUHnML78k7TEFIWeKuNmOv1RGksfzAgMBAAGjgbMwgbAwDAYDVR0T +AQH/BAIwADALBgNVHQ8EBAMCBeAwMQYIKwYBBQUHAQEEJTAjMCEGCCsGAQUFBzAB +hhVodHRwOi8vMTI3LjAuMC4xOjg4ODgwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsG +AQUFBwMCMEEGA1UdEQQ6MDiHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAGCE3NlcnZl +ci0wMS5sb2NhbGhvc3SCCWxvY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAngLW +TZx4PVsbAQ+saPRJErZU4JfA8Csx1MpdOkmvqbTwpIvnkgE6mER6DA31z+Fml2J1 +q+TpGh83mZTaH2yivF1i7/EU4h/lDhCXOns+yPto1t8GbIGNRTvG9VwJjhjgbILm +5l5cZQY7NeIryU4LmbELJCwPoHt52HnK0pfbhjCbD3Q6ugEx7xzESeqIwVS4gP5M +so/S+lvlQi1YWn1JJrj+SM8E5k/7x1kjTQZ/sHXAsUTwdtE6Qk/xbT5dbZYuhhPU +TF6sSfQpIO3Ju5Z+MTbcaCbJmqSe/y6J21e0Di1ve1BORKAy833F4leyPX8Ziv1x +iB8gZs4dMh88tJHqgw== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp/desgsign/server-01-key.pem b/test/configs/certs/ocsp/desgsign/server-01-key.pem new file mode 100644 index 0000000..fe567d9 --- /dev/null +++ b/test/configs/certs/ocsp/desgsign/server-01-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAxdjAw8FRGagv3Y3vPrKD0WqT2vQuFfIZh7gkEvbINux2b2yZ +csCTCRNDypvutdiaFQfaebbF5VdvYCnCCu8yxIXrBpYYlvXdv+zDqzwUsKiX6CHm +m8JVH1E2djN11hr1OWeuXCvnCu39BQAVM8E6m+PzN1EeNXEbzbUiveZ8JePCm8eA +3sODCKryDfx9apo5DTPJbAx6wldiMsmHNVtcw6mVVNSXaYzu7QQS+qZMJTN9ZPMY +zK/oYlQOuDfnxEgmrn+MQK56nB3YQJH51/vA1gn0ixYz9ZQWWU8NQcixAQXja5B7 +dtkYTtZrwVB5zC+/JO0xBSFnirjZjr9URpLH8wIDAQABAoIBAA/+b8bjb9z1Hbl5 +aefVopZhWUaQCtG3Wp0AI9psnM7j2sczLkx6iOho6EgOxwNoWTxuABlqzyC1KsnJ +Vmhv4djFQrSrmZ1Kjvye9Up4duu5FV6srunUkfEQLajsjBAc6coUOaI14l2d06B9 +2zjt3AESMm08X4bOeALvK7nSUiL13CAzKfv/ufM+bqi3nXxzNCBhtd51sHDaYqr7 +nbhFMrX1frhRb0a5k0Q0UPf3V8pUf04difa0MZJ8tQTW8HYLrEVjOE8NPp5yGQj0 +UX0d4XqS5LVDQl6Y1DAOQ7IsDq91gGKa8sk48HVvnwRjez0igAtbTq5I2DTydFDf +D19fpCECgYEA8eyHbdojrawzQM9P5QADRTLN+r2rk8Tv0cW2CdJTmxZiU6HhVqxU +NrP5G9FpCkn0jOikzhGdJalTgd3TKWMbZS9gPdezpj1viIT7v7NsK7Gnc8EcF+v8 +4H+Rj7jsxiu6ryNoNPmXZSsIyfGIvbOE9SwUki1Cc2NVRYf0g6aLfXsCgYEA0Vuw +2DRg7FCP9Ck08eburI163OEKE2sNIc2z4O6I0gy/u/U86Y/bLBrpRypiSEAf1r/p +yEEzzFqSNf7UNCLumdKk4LXUk3FMPygMZY/D7s1j/MVW/6HWeMeMy7IKKqOclTT1 +R1nNjbjRkorBRyHDkYqDrAGLuQUleWLrqSzIyekCgYBjxm5wcvgmB5A32YiU5LV0 +k6h6EkGyNxXFiWozkMgkfU3eOjRqf7ZXvVAvFeXhdXDjsItP5dnPD4++TtNpDVPe +HnTt7IlONaZLQrVlccVaG/H4/prsjsqDeHl7MgSNErnyw4KV3p5+/gmo5/HCc0iR +qTVuuDXgywX/IDxLE6QSAwKBgQChDDWXu4MrhjWWjvRJeWn0lskCjKJhmaH6dPCA +gT4CxwffIKGA0ca5wOHeer4r8hgL7Il8IJwmAS4kFylKCe0dqypmKbmiyi7rDnWq +0tLYKmtWEMAB2Y7QTkECmKy1bDKRnLFp96zl7lxYrCBOBa0ZkkID2RSQeWMAY5YB +B5BVsQKBgQDRIIKN0UhGqHZRcHb26c5lE5Rs/8nX2nyFgxKlwno6Dk5I0Cdo9CDh +z8f5klwm6Rgmsvr41crTg4IOimvCYGu94vyJTOKo+ln8Ky+02mdReomCVKoFdIB/ +WR+1aFvk1+r9aRr7MyPcHUGuIdUDJBnWv0pfNH1TUyvjpE3rMoaIqg== +-----END RSA PRIVATE KEY----- diff --git a/test/configs/certs/ocsp/desgsign/server-02-cert.pem b/test/configs/certs/ocsp/desgsign/server-02-cert.pem new file mode 100644 index 0000000..1d0bddd --- /dev/null +++ b/test/configs/certs/ocsp/desgsign/server-02-cert.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEiTCCA3GgAwIBAgIBAzANBgkqhkiG9w0BAQsFADCBrDELMAkGA1UEBhMCVVMx +CzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNpc2NvMRMwEQYDVQQKDApM +b2NhbCBIb3N0MSEwHwYDVQQLDBhMb2NhbCBIb3N0IFRydXN0IFNlcnZpY2UxHDAa +BgNVBAMME2NhLWludGVybS5sb2NhbGhvc3QxIjAgBgkqhkiG9w0BCQEWE2NhLWlu +dGVybUBsb2NhbGhvc3QwHhcNMjExMTE2MTgwMzMwWhcNMjkxMDE0MTM1NzI2WjCB +rDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNp +c2NvMRMwEQYDVQQKDApMb2NhbCBIb3N0MSEwHwYDVQQLDBhMb2NhbCBIb3N0IFRy +dXN0IFNlcnZpY2UxHDAaBgNVBAMME3NlcnZlci0wMi5sb2NhbGhvc3QxIjAgBgkq +hkiG9w0BCQEWE3NlcnZlci0wMkBsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQDWzAyE9A/EzJv5TcQlM9o4C/UOIQD4Rzs8S80x4ckGyF00 +8xF5FMmGzGMka4a9FNPDBzbBM41hlhJXcDhxRfvQodzvpOUXqi8Siha2SsyFzhD5 +o3exxV8ETkHMtST1nW/bwuYqQbDf3PLpVV5qZgv4jIByyGE4nCPDqHMKX0h1xXIQ +E3fv2qTcs5j72c3jcTlINFJ9aqTaFgSvvGwg+0KLrOjs1Og3408zTIHYSjTScFHd +heDLHjfmIx8oqXj4+g3F+5BgLNvO2E6if9YYGOFjxzRCPOrRw4JB8nveI1gZhTD9 ++OmUkIMoNLX/+4Kbq+a47/kqrXv5GxLxwola8do3AgMBAAGjgbMwgbAwDAYDVR0T +AQH/BAIwADALBgNVHQ8EBAMCBeAwMQYIKwYBBQUHAQEEJTAjMCEGCCsGAQUFBzAB +hhVodHRwOi8vMTI3LjAuMC4xOjg4ODgwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsG +AQUFBwMCMEEGA1UdEQQ6MDiHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAGCE3NlcnZl +ci0wMi5sb2NhbGhvc3SCCWxvY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAH9jr +P5EepmTiJzxYsrGE9q5BqhspBYqomgCbLWH3vDX50MwmxZ4ETe1icpHbXiddDA1k +dfBxn/1HXEWXxwHCTgoRsSzufgacHdk/LgXCxkFB0JBZ9c+ewOFdUQihxFUx1rMt +Sz0QnO40aopTk7PN8YlinihAvil/JC8fN54T0myCSjtE54dtZq2wwFiiorxR6hyi +hLL4zYKi/kCFa6DG0qmR6Yl2VsKiq3yp+6QxZ8w5srXTeeOoTOEyUkYKCTND7U1G +5vyCdMFNIDWHSB35I+7e0D1QFaH5XQlFztUfmLoOMSMLbOuzG2cFO6J+Y0AgTP5h +pZ5mw1ixmVdNmkSf3g== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp/desgsign/server-02-key.pem b/test/configs/certs/ocsp/desgsign/server-02-key.pem new file mode 100644 index 0000000..96882cb --- /dev/null +++ b/test/configs/certs/ocsp/desgsign/server-02-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpgIBAAKCAQEA1swMhPQPxMyb+U3EJTPaOAv1DiEA+Ec7PEvNMeHJBshdNPMR +eRTJhsxjJGuGvRTTwwc2wTONYZYSV3A4cUX70KHc76TlF6ovEooWtkrMhc4Q+aN3 +scVfBE5BzLUk9Z1v28LmKkGw39zy6VVeamYL+IyAcshhOJwjw6hzCl9IdcVyEBN3 +79qk3LOY+9nN43E5SDRSfWqk2hYEr7xsIPtCi6zo7NToN+NPM0yB2Eo00nBR3YXg +yx435iMfKKl4+PoNxfuQYCzbzthOon/WGBjhY8c0Qjzq0cOCQfJ73iNYGYUw/fjp +lJCDKDS1//uCm6vmuO/5Kq17+RsS8cKJWvHaNwIDAQABAoIBAQChTG2CUbydroDa +P9mxjjSqreACEiqmRudmqg2XDdRl8YR8xKM0Z7XGMimpKc3uo3s6E2q8vrfGtmlj +m/PmrAUjcMl2dD9M/BGJPIU9swHO4SXCMU0rA+oMU6/5x3XPs8BSKROqW9Y05bjM +G71g7OzEYs36ZBsN5cK9pPtuqIrDVHfPQI+Y3eP7wBnLscK/AqUoapbc6oQqarl4 +9rdHY4IHsab3xTxt/riSooia17R6xrrSpDcg+tdx8CphjD53zLN2Y8jHpl6LVJEN +rRFud68sDEtdU5EOj0WiQIke0Z8yBOgAt2YKqrU+O+Wuh2ad9pBhhp7oLvaRi0C8 +s+URKM2BAoGBAO4LtDgpWyMC3FfMDuSGT7hwp5+TtKM9Js2mZsQYZbJRX3BVK/G1 ++/LhZzpqcKqGAarZ/yz1cLUEprbEXX7+TFbPKKpBsEwTOQXxlvhmjSYxyHVfBCbB ++o43EolRey0Zdnnawk2OpqOVWSQafrp5JyJumyvfwES3Oc2iT8288bXhAoGBAOb/ +cwLtk5I9Bx0BFUDFNgn8g8QXfWJ0axyCUQE2UGn/tiXodpkrvFF2QrpEQ+OG+iKB +0d6I9lLNThw/7uKvecYN1HliWnJ9dzUe59nguwG+PNZJgSmBhnPs4Wg/gsG6h7K1 +JZ3VPeyNaSRK+pCn8sWkcQjumImZR/PHKzsDdOMXAoGBANp18UoNYi9qY69LfWtq +e+Unth30HzYkW+UlznAud75DgZQFBlRIkFWhWOw6XPWSEBus/stS4MGv6BQZeDig +xoxwh8BgkpvulEmJIuUKsIUZ8P8OWS/8m6ZCkodlOOb11E4WXnVw8it0V5+TlTQr +Fag627tTGA+4G5tFV3nX6ffhAoGBANloGOXrlhVvzK8WotsYATk66QT4mrC8I0ds +uzKp3Ns2qUdaV6znhdEhvcGzmDWfhvJNPqn0O+lIgziBT6MYRkMKJyyrTbctsLFV +Sh88rKUCWB3Shnb7CgE3NBq6k6Ujmq2uYh3/Yc2udgOLcfINr6cmkqA2d2gh1J9y +l5RuN2e3AoGBAN9FEe+OLRbq+qnMHkEoozoshxgEqv/M/BgXvoX8kelcp1zyBaWX +t5yelDn7UjANhJNBsripzvaLilMxracm1Oy0fCtRJ3dvwm6djUelhSPBV9HJSEN8 +t2Sv1z45SNq3l2cz8pVvoQDNmUbh0QCQnScrLN8L3vBJSibAJgSdml51 +-----END RSA PRIVATE KEY----- diff --git a/test/configs/certs/ocsp/gen.sh b/test/configs/certs/ocsp/gen.sh new file mode 100755 index 0000000..64b25f1 --- /dev/null +++ b/test/configs/certs/ocsp/gen.sh @@ -0,0 +1,346 @@ +#!/usr/bin/env bash +set -euo pipefail + +# gen.sh generates certificates used in OCSP tests. It generates a CA, client +# certs, and a few different types of server certs with different OCSP +# settings. This requires OpenSSL, not LibreSSL. +# +# usage: ./gen.sh + +################################################################################ +# Setup CA +################################################################################ +mkdir -p ./demoCA/newcerts +rm -f demoCA/index.txt +touch demoCA/index.txt +echo "01" > demoCA/serial + +prefix="ca" +openssl genrsa -out ${prefix}-key.pem +openssl req -new -key ${prefix}-key.pem -out ${prefix}-csr.pem \ + -config <(echo " + [ req ] + prompt = no + distinguished_name = req_distinguished_name + string_mask = utf8only + utf8 = yes + x509_extensions = v3_ca + + [ req_distinguished_name ] + C = US + ST = CA + L = San Francisco + O = Synadia + OU = nats.io + CN = localhost ca + + [ v3_ca ] + subjectKeyIdentifier=hash + authorityKeyIdentifier=keyid:always,issuer + basicConstraints = critical,CA:true + ") +openssl ca -batch -keyfile ${prefix}-key.pem -selfsign -notext \ + -config <(echo " + [ ca ] + default_ca = ca_default + + [ ca_default ] + dir = ./demoCA + database = ./demoCA/index.txt + new_certs_dir = ./demoCA/newcerts + serial = ./demoCA/serial + default_md = default + policy = policy_anything + x509_extensions = v3_ca + default_md = sha256 + + default_enddate = 20291014135726Z + copy_extensions = copy + + [ policy_anything ] + countryName = optional + stateOrProvinceName = optional + localityName = optional + organizationName = optional + organizationalUnitName = optional + commonName = supplied + emailAddress = optional + + [ v3_ca ] + subjectKeyIdentifier=hash + authorityKeyIdentifier=keyid:always,issuer + basicConstraints = critical,CA:true + ") \ + -out ${prefix}-cert.pem -infiles ${prefix}-csr.pem + +################################################################################ +# Client cert +################################################################################ +prefix="client" +openssl genrsa -out ${prefix}-key.pem +openssl req -new -key ${prefix}-key.pem -out ${prefix}-csr.pem \ + -config <(echo " + [ req ] + prompt = no + distinguished_name = req_distinguished_name + req_extensions = v3_req + string_mask = utf8only + utf8 = yes + + [ req_distinguished_name ] + C = US + ST = CA + L = San Francisco + O = Synadia + OU = nats.io + CN = localhost client + + [ v3_req ] + subjectAltName = @alt_names + + [ alt_names ] + IP.1 = 127.0.0.1 + IP.2 = 0:0:0:0:0:0:0:1 + DNS.1 = localhost + DNS.2 = client.localhost + ") +openssl ca -batch -keyfile ca-key.pem -cert ca-cert.pem -notext \ + -config <(echo " + [ ca ] + default_ca = ca_default + + [ ca_default ] + dir = ./demoCA + database = ./demoCA/index.txt + new_certs_dir = ./demoCA/newcerts + serial = ./demoCA/serial + default_md = default + policy = policy_anything + x509_extensions = ext_ca + default_md = sha256 + + default_enddate = 20291014135726Z + copy_extensions = copy + + [ policy_anything ] + countryName = optional + stateOrProvinceName = optional + localityName = optional + organizationName = optional + organizationalUnitName = optional + commonName = supplied + emailAddress = optional + + [ ext_ca ] + basicConstraints = CA:FALSE + keyUsage = nonRepudiation, digitalSignature, keyEncipherment + extendedKeyUsage = serverAuth, clientAuth + ") \ + -out ${prefix}-cert.pem -infiles ${prefix}-csr.pem + +################################################################################ +# Server cert +################################################################################ +prefix="server" +openssl genrsa -out ${prefix}-key.pem +openssl req -new -key ${prefix}-key.pem -out ${prefix}-csr.pem \ + -config <(echo " + [ req ] + prompt = no + distinguished_name = req_distinguished_name + req_extensions = v3_req + string_mask = utf8only + utf8 = yes + + [ req_distinguished_name ] + C = US + ST = CA + L = San Francisco + O = Synadia + OU = nats.io + CN = localhost server + + [ v3_req ] + subjectAltName = @alt_names + + [ alt_names ] + IP.1 = 127.0.0.1 + IP.2 = 0:0:0:0:0:0:0:1 + DNS.1 = localhost + DNS.2 = server.localhost + ") +openssl ca -batch -keyfile ca-key.pem -cert ca-cert.pem -notext \ + -config <(echo " + [ ca ] + default_ca = ca_default + + [ ca_default ] + dir = ./demoCA + database = ./demoCA/index.txt + new_certs_dir = ./demoCA/newcerts + serial = ./demoCA/serial + default_md = default + policy = policy_anything + x509_extensions = ext_ca + default_md = sha256 + + default_enddate = 20291014135726Z + copy_extensions = copy + + [ policy_anything ] + countryName = optional + stateOrProvinceName = optional + localityName = optional + organizationName = optional + organizationalUnitName = optional + commonName = supplied + emailAddress = optional + + [ ext_ca ] + basicConstraints = CA:FALSE + keyUsage = nonRepudiation, digitalSignature, keyEncipherment + extendedKeyUsage = serverAuth, clientAuth + ") \ + -out ${prefix}-cert.pem -infiles ${prefix}-csr.pem + +################################################################################ +# Server cert (tlsfeature) +################################################################################ +prefix="server-status-request" +openssl genrsa -out ${prefix}-key.pem +openssl req -new -key ${prefix}-key.pem -out ${prefix}-csr.pem \ + -config <(echo " + [ req ] + prompt = no + distinguished_name = req_distinguished_name + req_extensions = v3_req + string_mask = utf8only + utf8 = yes + + [ req_distinguished_name ] + C = US + ST = CA + L = San Francisco + O = Synadia + OU = nats.io + CN = localhost server status request + + [ v3_req ] + subjectAltName = @alt_names + + [ alt_names ] + IP.1 = 127.0.0.1 + IP.2 = 0:0:0:0:0:0:0:1 + DNS.1 = localhost + DNS.2 = server-status-request.localhost + ") +openssl ca -batch -keyfile ca-key.pem -cert ca-cert.pem -notext \ + -config <(echo " + [ ca ] + default_ca = ca_default + + [ ca_default ] + dir = ./demoCA + database = ./demoCA/index.txt + new_certs_dir = ./demoCA/newcerts + serial = ./demoCA/serial + default_md = default + policy = policy_anything + x509_extensions = ext_ca + default_md = sha256 + + default_enddate = 20291014135726Z + copy_extensions = copy + + [ policy_anything ] + countryName = optional + stateOrProvinceName = optional + localityName = optional + organizationName = optional + organizationalUnitName = optional + commonName = supplied + emailAddress = optional + + [ ext_ca ] + basicConstraints = CA:FALSE + keyUsage = nonRepudiation, digitalSignature, keyEncipherment + tlsfeature = status_request + extendedKeyUsage = serverAuth, clientAuth + ") \ + -out ${prefix}-cert.pem -infiles ${prefix}-csr.pem + +################################################################################ +# Server cert (authorityInfoAccess and tlsfeature) +################################################################################ +for n in {01..08}; do + prefix="server-status-request-url-${n}" + + openssl genrsa -out ${prefix}-key.pem + openssl req -new -key ${prefix}-key.pem -out ${prefix}-csr.pem \ + -config <(echo " + [ req ] + prompt = no + distinguished_name = req_distinguished_name + req_extensions = v3_req + string_mask = utf8only + utf8 = yes + + [ req_distinguished_name ] + C = US + ST = CA + L = San Francisco + O = Synadia + OU = nats.io + CN = localhost ${prefix} + + [ v3_req ] + subjectAltName = @alt_names + + [ alt_names ] + IP.1 = 127.0.0.1 + IP.2 = 0:0:0:0:0:0:0:1 + DNS.1 = localhost + DNS.2 = ${prefix}.localhost + ") + openssl ca -batch -keyfile ca-key.pem -cert ca-cert.pem -notext \ + -config <(echo " + [ ca ] + default_ca = ca_default + + [ ca_default ] + dir = ./demoCA + database = ./demoCA/index.txt + new_certs_dir = ./demoCA/newcerts + serial = ./demoCA/serial + default_md = default + policy = policy_anything + x509_extensions = ext_ca + default_md = sha256 + + default_enddate = 20291014135726Z + copy_extensions = copy + + [ policy_anything ] + countryName = optional + stateOrProvinceName = optional + localityName = optional + organizationName = optional + organizationalUnitName = optional + commonName = supplied + emailAddress = optional + + [ ext_ca ] + basicConstraints = CA:FALSE + keyUsage = nonRepudiation, digitalSignature, keyEncipherment + authorityInfoAccess = OCSP;URI:http://127.0.0.1:8888 + tlsfeature = status_request + extendedKeyUsage = serverAuth, clientAuth + ") \ + -out ${prefix}-cert.pem -infiles ${prefix}-csr.pem +done + +################################################################################ +# Clean up +################################################################################ +rm -f *-csr.pem +rm -rf ./demoCA diff --git a/test/configs/certs/ocsp/server-cert.pem b/test/configs/certs/ocsp/server-cert.pem new file mode 100644 index 0000000..f7cc0f8 --- /dev/null +++ b/test/configs/certs/ocsp/server-cert.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIID0jCCArqgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe +Fw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMHExCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJDQTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzEQMA4GA1UECgwHU3lu +YWRpYTEQMA4GA1UECwwHbmF0cy5pbzEZMBcGA1UEAwwQbG9jYWxob3N0IHNlcnZl +cjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAN/v4lwsEQqEr8gqG6Xu +UjNfvUBN/enc/26FqtsAF6ms0r4oHcyR3RZQGQj+Z0RF3Wu0Kq9692gk7FD/QulE +hYJTjq6lEwvETuUHbkNmIAppNJW1JvgLsTOfm38VorBVU5PUMbrcfsVsFijXVACj +9VMZ23So4dxtlvnqrd5/fVx0Pql5EjY87bJEKH5Zngy1v+AR5kybZaorOX9T4/Nl +e0P184GwGs15hKAokoQMPm9uIhG527JMyhQh5J/2wooY2DBZ9jDt5FVXNpb0C+nr +M+AULk5QHQsobTtmC3RSNHiNw5B5w+gmauhGziurq8gcx0DctqAslKFBkCLkL9fc +F30CAwEAAaN5MHcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBeAwHQYDVR0lBBYwFAYI +KwYBBQUHAwEGCCsGAQUFBwMCMD4GA1UdEQQ3MDWHBH8AAAGHEAAAAAAAAAAAAAAA +AAAAAAGCCWxvY2FsaG9zdIIQc2VydmVyLmxvY2FsaG9zdDANBgkqhkiG9w0BAQsF +AAOCAQEATM/K671w3aHt665HBMawzMIZZPq/ZoBfEUkSUW9KdnQHgTxatHcZonsL +aFn4XZBYQ0Pqkz7H1w39mHdvpURQ5ZMnsmn4jH3LECsOtQ4ztrLk2fhLSoMQBVdb +UjdYhrM8AuILKRCzOBNsDm/ZB/vPSlmYhnaEBUjO0t+I/A0X1z5eDcYPLl578kfJ +WjlvRluWr7Uku1DaZUy7TByYvUuOjP4c33DAnbZ5Sldx18repZ20REASxsCpa/CW +tptxVfUvLcGRHIY0FxOn+5Pfm1QDo2uh6yVYHgsOCh1qW8FHfJvgnrMlvvXniKXu +5H6A5GeyCkIVvAENDfl1cN9LaV5eQg== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp/server-key.pem b/test/configs/certs/ocsp/server-key.pem new file mode 100644 index 0000000..edfd16e --- /dev/null +++ b/test/configs/certs/ocsp/server-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA3+/iXCwRCoSvyCobpe5SM1+9QE396dz/boWq2wAXqazSvigd +zJHdFlAZCP5nREXda7Qqr3r3aCTsUP9C6USFglOOrqUTC8RO5QduQ2YgCmk0lbUm ++AuxM5+bfxWisFVTk9Qxutx+xWwWKNdUAKP1UxnbdKjh3G2W+eqt3n99XHQ+qXkS +NjztskQoflmeDLW/4BHmTJtlqis5f1Pj82V7Q/XzgbAazXmEoCiShAw+b24iEbnb +skzKFCHkn/bCihjYMFn2MO3kVVc2lvQL6esz4BQuTlAdCyhtO2YLdFI0eI3DkHnD +6CZq6EbOK6uryBzHQNy2oCyUoUGQIuQv19wXfQIDAQABAoIBAC5rWl/C3rFwecOj +PuHxeeaeVOuMfzLIFcbCPH1zEnSgl3rFdA/guJSUe+wKWDulw9U8npCLi9dxt+6+ +Sw9xnb87NNts6nrI8ZW2KZwdAk1GK5lQ2MgYHF5YGnKIeJXWyiFdngVfCYRA+IL0 +x7vuQL8+H+iZzV/U87PQesQhZ0oPh03n/4OtSo1bnPOoiPLAC+GVVpSSKoh13beb +GpIRXhGNXonGPWnj1t/oBHQwLDfDndnljkRaWTzxeCZGcQ1wYucf6jne5dI1E21q +vrdyK8GL4SqzHC2SHnJO24aKam/Xr7YqOM2T+XsVywKxE4I1icT2YhQAgRuN8Nlg +7CELdAECgYEA+CEhY7JYfmRb/+PIvogRr67uaYtxYygvDcYfse5O/V/HOzotIXIT +t99deD8XOFpWhJHdDzDD/TStb/rmwu5kavMQzmQafyesu7okk1NtB+fZtMLQfF9/ +0+bW9uJQFyc1EVHKgyya3UidXTcmKcExKQZT+jdgFz614zndcOKl0H0CgYEA5wpO +WIjjxVqke5ucrVSRsm37l0BZYeea2vxLAzUV4txVw3hcOgJo+FhEGFy6EjNSV2OR +y589gZDEPZ/LNeaj7nbNUqf2xZV7MAXBgUCtj7fJqMbVhbXZB0w0hIknDL+td2WZ +bocuVExTwRwXyuSCgUFo0Bz++L0cR3JK6kNBEwECgYEAyOfKaTbWgEAyXZbJy7vQ +1jcFw1+sh2TZ9IUe1KroOi962XHZaOM9I/wvalVrL621r9GK8+nARxyH8cttXRg5 +Jn94dCSJb7toGPg29TLvbR9FHx8+P/XzQlf+ZhgIUTbluQhIuL09Bz7sa7VjqRtL ++rOs+0QrAac9DqajretV5uECgYEAgaXe4P+wEQcUVei0uu9B8waUsAOEJNR6qXf6 +AArCBVPvLIlV95dyoCmnzKP8Jkp2YmOVZNYvBY3fEVWiCtUqGJ7CCSgH6kg/oGsa +cxWAT62qk/M/zpCFAPtaXSU5rIXDKcTxnHxvGw7Z0PuavlgMg8vYrTAYRCyaud0A +/QRQeAECgYBuRDM+mk5EDbsi6MOD24x6krRDHr5/Ch5xCQXK2FhU8zcQ8P24UixU +Re71LBsYLBHkhB/slofGdBvgeiVHwJyVWA9c3+kb+IwSilNRLV7IwxrfLg3xSVBu +0KEwWLSXlJmPnGWObpBmz62HrfquyMME4srQrNfW1q+Qm8OlZIeInw== +-----END RSA PRIVATE KEY----- diff --git a/test/configs/certs/ocsp/server-status-request-cert.pem b/test/configs/certs/ocsp/server-status-request-cert.pem new file mode 100644 index 0000000..7caa779 --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-cert.pem @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIIEBjCCAu6gAwIBAgIBBDANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe +Fw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMIGAMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKDAmBgNVBAMMH2xvY2FsaG9zdCBzZXJ2 +ZXIgc3RhdHVzIHJlcXVlc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQDViockMMFbHDOsiD+JLalZc1g3rd10xbXwHFHsgFQUdr+g6guGnnQhKMHEsQWf +bGmn5G+K0QG2jxpz2IpfXA2UpMIrYLhIcf5faEbUgmwd8zaNXNs0I6sZZk8GrKYw +PLHhXam94FUfVwsecJ/V+sK7limOB/T1AuPamBhgqIgubz40N3Qmkh9J/rkhq2b1 +ffgr2v6qKjQ8bIyPPDh9OB10KffjqoaN8ogXuE3hZdQTniRWW1nT38NqQdJg6T+N +cEheH0H4pYUSx/TdF6AHXMxWFD6lX9nLU9UGpXLAHY63rVJCv2KIMMsCGBUPKkKK +TujdFu2KHh52CstdasQ3gpnRAgMBAAGjgZwwgZkwCQYDVR0TBAIwADALBgNVHQ8E +BAMCBeAwEQYIKwYBBQUHARgEBTADAgEFMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggr +BgEFBQcDAjBNBgNVHREERjBEhwR/AAABhxAAAAAAAAAAAAAAAAAAAAABgglsb2Nh +bGhvc3SCH3NlcnZlci1zdGF0dXMtcmVxdWVzdC5sb2NhbGhvc3QwDQYJKoZIhvcN +AQELBQADggEBAIObz14Fkbn0Cp6qo1Sbvv8owyKxPULnV3i6qx9abThlAusbMvjt +I4MXVmgn+mNV8raivSauqKKSp0QWKVrToMUIYurJOHqDBf7idL/g6ZP9u0RIcxE2 +b4gK54xdfh2gnc4BS+5LITB0bS96zFouz5gfz+pj7Yoe8dteylSDeG7rlO6g6qgI +Y2EkS9eayCTfr99joCfvHhuJxqWFQq++OAPujbMSC03CKb87Jg7jk3WKq8RzFgut +tuTtJjPGvDgv15axrQk1zktIzzfMG3//gOAVH185AUbIA5tsiUmXY+q3mHxJejPS +sr1dXay2Kw2scEgWeYiu4X7SrJg2C9ksCtw= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp/server-status-request-key.pem b/test/configs/certs/ocsp/server-status-request-key.pem new file mode 100644 index 0000000..53f470c --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA1YqHJDDBWxwzrIg/iS2pWXNYN63ddMW18BxR7IBUFHa/oOoL +hp50ISjBxLEFn2xpp+RvitEBto8ac9iKX1wNlKTCK2C4SHH+X2hG1IJsHfM2jVzb +NCOrGWZPBqymMDyx4V2pveBVH1cLHnCf1frCu5Ypjgf09QLj2pgYYKiILm8+NDd0 +JpIfSf65Iatm9X34K9r+qio0PGyMjzw4fTgddCn346qGjfKIF7hN4WXUE54kVltZ +09/DakHSYOk/jXBIXh9B+KWFEsf03RegB1zMVhQ+pV/Zy1PVBqVywB2Ot61SQr9i +iDDLAhgVDypCik7o3Rbtih4edgrLXWrEN4KZ0QIDAQABAoIBAQCS0tXKr28y+vgY +p/Gvm2K2a1V7qyL6eDp5Zq95gl7NVzy5IlNcczR73C8m02R/UiZvVuTOuJiJ8mkI +wBmcKYn46zPKaY0r2p1A8kzJLwexrBmVJwRdHC03oJ2zhCAcSI8x2pmsQUOl8c5m +Vp+/m9Qq8LuPua7Wi+8ozKzuQNKpwXcdPlD8jXbQNVmjbFAzSnFukxzHDxkEV2MO +9seOf0b+2ByoBsAEKW6whdHX7qEJ68OjAbv6lYu0dJmAA3v9rXdKk1tYKorvGRms +sRcSgvjGol+SwM7daa1V+Pa4c8DszSfieocCWJt1wde4BEMqmdgZVRXxjJVHzEPa +YkZsLTCxAoGBAPfUARuZlH6yfVi2kK3rpjjqToAzSTQW3GRiEAb1/UeE3XKUor/N +bJxHwB8Zxkk06Qfjc1Y7MP45qTfDUwk6pLJSCYsdX/c1rtKPHKTGKrg7qy6vYqNN +sNv/E2C8spTCe0FD3hOFnRPBFtZrfRuWl8r7eI/uiTzES0E2QX/s1u/tAoGBANyV +GJSyvs7vZHXTjaTd5prfpcGZFgdB0DivGjFISfTJ4cqFlxb7doDUh0//xIhoKMP9 +BnWpV7c+FN5sOReB+wkbqiWqRaflm8BhEJmHddYfpz3kgBBrfOHAA6reeG1kBJ5h +NqSagt3VMmmkHSrGtuzeZAHF/owH/cAqgEIaE2z1AoGAIgWrxUM3PJF6XcRqZkX0 +gtm/vx/LS+hbhzhjJOF+TOQzlnhLQ3OLFoVPHbXnH8OwvhF+kvb7SdtWnL4m1xyC +4awbfUqiEwj+oA9fikteL/6ZCIaxTuPqhLkmyt/80ClGzHXptdpg7wZSAUuuWCw7 +WHVfXrsLghkcj65IHazA8R0CgYEA0mQPPv1CS4RAULYIewD/zCaXJhHK1f8rCHXT +SMBHcgkAqLdExjHw9K5BpccxgF3AzDbRa3aq6Gd5ZjDZP5fFhglx/1zp9VtmdFbg +i1+NwD8OuFTy5TZwta38kYSCXuwwD5Rvlw6c1dNcszBKdZt1rHXt81cTFCMnH5wq +G2JdtuUCgYBtUJVhPpUn23Lnmc/UEH+kVHAx+YytfOngUbJoPBe8HgNUnB4Gczii +uQtKfJMTEXz/e45tAmNTaA2MwTAgbD2sH1gJDnALqUTxoWfFW4qTwl5ZB4CVue2X +imQj/a/PallX0KENE+0VCQRhSwdhO60tMHJhK1KEycorHo+57QHJrA== +-----END RSA PRIVATE KEY----- diff --git a/test/configs/certs/ocsp/server-status-request-url-01-cert.pem b/test/configs/certs/ocsp/server-status-request-url-01-cert.pem new file mode 100644 index 0000000..df50c10 --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-url-01-cert.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIERzCCAy+gAwIBAgIBBTANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe +Fw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2 +ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEA3yzlI0YFwf0JWbT1kOtRc2HbTfU4ktVmfYAgNHYry1ARxw9MT5rT +8qSV/xZR7EAj6lmE2AtYFeUI8WIVIYd8yrEegM4U5Foytbjem+xgZOJ4n7Mr/BqU +BQQ/7IjPJ/EymIdvTJ6LEtZQNWyuHgNL2Z1MCJPPUBLwSujS+cPebKxrYMIWzgxr +UzT8EK8VPJQGo/GJdQLk53GVS15Khtn+eymviFoNYEToLY9geaoewy1DWxUrWFwN +jngB7iMBvrIjq+Pn9exObVHSpbdPHYDqKvdEykzpRl+bPS5O371z59/IQbxFfzwH +4k3blC/M4JSN/HRsZQvnERyjHTjjwgadowIDAQABo4HWMIHTMAkGA1UdEwQCMAAw +CwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov +LzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr +BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA +AAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAxLmxv +Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAPyxQodHHHRAW9t9A3jR1LE9FkEzN +FmzFb3sZpYIzw/wTmD0KvbW3D8YVe/lj5e/LxhtHzun+AnFa/JXoK0t5uLzItges +zGEyOGFj3zP+E9Chr1fr2X8sHJjqnUrknC9dQPoiBKlNRCWTP2eeKC+yODgEmgzS +fEdcuuqGuM7MFbf3eVwtkWNCv1R14GZN4cuSeQn3xA8/8aLcGEHV2N1GLVxK55x0 +hbB1hjTaq/t9a66ByIiaeEex832H1MRRkPMhEQxAIx8HZFQJSX+yf9CtTJTc18Z+ +I/1utSjX9VW+4YW/vN7kvYgQ3YQUbMSGjqkROe/hb12TCTSbSt6O7P3JTw== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp/server-status-request-url-01-key.pem b/test/configs/certs/ocsp/server-status-request-url-01-key.pem new file mode 100644 index 0000000..ca73e8c --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-url-01-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA3yzlI0YFwf0JWbT1kOtRc2HbTfU4ktVmfYAgNHYry1ARxw9M +T5rT8qSV/xZR7EAj6lmE2AtYFeUI8WIVIYd8yrEegM4U5Foytbjem+xgZOJ4n7Mr +/BqUBQQ/7IjPJ/EymIdvTJ6LEtZQNWyuHgNL2Z1MCJPPUBLwSujS+cPebKxrYMIW +zgxrUzT8EK8VPJQGo/GJdQLk53GVS15Khtn+eymviFoNYEToLY9geaoewy1DWxUr +WFwNjngB7iMBvrIjq+Pn9exObVHSpbdPHYDqKvdEykzpRl+bPS5O371z59/IQbxF +fzwH4k3blC/M4JSN/HRsZQvnERyjHTjjwgadowIDAQABAoIBADIx0seANENROg0n +GiQiCQu+cqiyAqtju812R8gE3Ay6UYVH943f8NOeGO5qgrp0Ip/OyoYc5w7lmIFR +4hbMdlDQTWAC+cfyUX11uvuuym9mUGL61WW6RJhsQERe8Ni28v5DV9EPx4/RC/E4 +JGEztJeZPUSgmfwUhl8SxoDOgPRH18ilJ2kkBAC1UrS3jFTTSiIENXfpCkXcaBQN ++DFstygHragvXXIRjpSAYo1hv/YNJHAvdsPDGfEYvL0G5MTwLtFVBMZVzjQqkriZ +pTYke2epm1cNY2+PrW3yrZhgaM+dZkVAaRVqsAKbP6L6z48PlqSOIKJtceOKYVMT +1awxrUECgYEA8P3qOt7R6ICu9taxW+pVxScLr4UUtpvERXigUjbTCW2VTXBS18oe +2MwihcRtRa5DI+4qCNC+mIRIKWKLqTyqiL733fVykEaXJQGnUtbHe1TmwT5TipQ3 +cc9XJ+8di7BsygVGrw6S3Yu20HjSbg8pTF3CqpHfGYd8hUl5Fb5bBCsCgYEA7RLv +em1T/KlfrsT2jNp7aIH/frhPZu701UTHEPw6xbr/wXrMRQ25rKB0BrjGhA2IDmHO +C3qXc/6J6uyl+Q7UnioPJ5uPIWQO9zmOqzUeaqpZi4ZDzKbZOcvDXGEh75SAcA2F +G8rWgUsSG1MJrvRyc6LrMFflzFboNlM1RFKZuGkCgYAPcMVcJkCeu/f52sGcWQRL +0HegNE+ib5KAvPzQp7CXzwyc3JCCQPH/A/1GQtKZf1PP7oeE7xL3JKqW9DhsjHWE +s2+gKgTAF7XBajy1Qwue7E6onPyvSVXQFe/IoDptY81kmDfyKWXhif0ZmFQrNw93 +VB6P3S7FLPgedhmq+fpM5wKBgQDQxmYi2Hin/riGfmnpLNaU+DZ57/Hs4e62ibUM +3jMbwkxMpMmS3j1vKnZDHSGlVQ1GLRd4wGL0AWqPegvHwym2h6+E39cyPp13Vjav +yz2YDaCB4KGAbkbKQQkwJ1HtXEenZiCckwpIHkY5zeyvKBJKh2S61A1I4BfrD2fP +XTaGUQKBgQCNb+1O1VNwkxAXYTSo8VDoRAcN+RagRGRPIWk38zAirN7+RZ6LzWQe +8co0SD2Ex8eI5U7bHGObFIzNKgWNkZFX1CkyQMV96GWMfwdaIygOpXG9kbyaJBLo +/oZMeSRMIN+OSMOsE7G+93y23sU5s/fI1sXn+U89n1frpnyVO2d9Zw== +-----END RSA PRIVATE KEY----- diff --git a/test/configs/certs/ocsp/server-status-request-url-02-cert.pem b/test/configs/certs/ocsp/server-status-request-url-02-cert.pem new file mode 100644 index 0000000..d4d4b24 --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-url-02-cert.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIERzCCAy+gAwIBAgIBBjANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe +Fw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2 +ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAvpS9BNTijovvNY6lTOYqAdsYKjraohGUTHgtt+6NL5S9bnm3bLi/ +F1PWpqaUrR6pT6YtHBIO6acjYc0XgyvW2ECStpmpiDW1GS11A4vtnJwdk1ZojJbB +bt2didXHM4I82hwf1Zn75syqIffNGz2re8X+H4MRPeLmoS0ubZBYk2+WlEUhGlWP +rNRIWi9OxKPNXX7WBBzaF4QFnbVvkAlNfyOgH8QryRxVTUeLvO/QbOutPVyB8rIC +SUkq/PAqNrMbhpV4Wro3zhl3JIfnXKjLTITEpGcf2z7VgdZxqtZOARtY6lmCUUmm +Bc9voyT7QjSXalynFFjU70UguNc7soEAZQIDAQABo4HWMIHTMAkGA1UdEwQCMAAw +CwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov +LzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr +BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA +AAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAyLmxv +Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAjbMrXjneMWFNr4IHuVsjso86Ay83 +6E8ltbjOeqsyWI6uyWzaJvBlec2Gyr4rib+wgFOAGFDAGPB/hvyveas6wg4JUbQ/ +P3e8a+0Ls6eTYz6ijrwUGFxVsEu2VQTF7oy7SEbfabsC2g9sQSiaWJ0Js1txIhaH +bqR2bzRDorgQ3HoLjiKbVGNvifg6qjLE2M8AhJ7FhnOpUpewsanBn4p0BCjfG8Ne +v2EaxdJxK+kHiWX4D7ybOeIKRzX9E0HMmCh1KyEkV+qGsOrYq3ErWPcoPODOT/f3 +CLrfKmdHHGdxCc+NTxxSyc/CMZceMnV8yplX2ixPY7o1gFQyT3CvJ40Vnw== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp/server-status-request-url-02-key.pem b/test/configs/certs/ocsp/server-status-request-url-02-key.pem new file mode 100644 index 0000000..84f22f5 --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-url-02-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAvpS9BNTijovvNY6lTOYqAdsYKjraohGUTHgtt+6NL5S9bnm3 +bLi/F1PWpqaUrR6pT6YtHBIO6acjYc0XgyvW2ECStpmpiDW1GS11A4vtnJwdk1Zo +jJbBbt2didXHM4I82hwf1Zn75syqIffNGz2re8X+H4MRPeLmoS0ubZBYk2+WlEUh +GlWPrNRIWi9OxKPNXX7WBBzaF4QFnbVvkAlNfyOgH8QryRxVTUeLvO/QbOutPVyB +8rICSUkq/PAqNrMbhpV4Wro3zhl3JIfnXKjLTITEpGcf2z7VgdZxqtZOARtY6lmC +UUmmBc9voyT7QjSXalynFFjU70UguNc7soEAZQIDAQABAoIBAC1hE0T0P+H7VgEl +rxyJbZU1iyJ7ExYmI2616wTx63JZ82U73D4qG249i03xOlOiimQpyHH3ps9h+d3O +zPtx0914Orb6DdEeILoXBdbLWc/BqJUtrQVU3Eg+wsVQPvd32m+A5N+io8WIFDa0 +X4VOAOge8+yi89cNkSbtsDFsfnrUXQf/CzeNKutEHWqC3GmKtfxMyEVT+KbpV82b +ghoe+MrhILyMoMppTz1QOoo2ESBHzLz3WxhsJFMqhoECFmatNsq7etQ2fw3h5B6G +oOStbFB7AqYALuwYNdL4UkLA6k0hifM8ew/iNNSl9E9s+GYfVJfAE7lneI9PDwJ9 +9OLGNWECgYEA4qNCay5zV1MuJFdEFbZV8v7Nd+B7tIHE/kA14+rxNkhhMee8zLsc +Ve79UWWtVLhDsZSTkdUw/4r+DF7/TEl1q423aKD5Ar2jyPpW8rWOelB9GZ9R6Eui +RutJclSWSBO40epOpIRIQjgFzN+gDyfvM1rQo3ZJ7dZy6bPaEHudlH0CgYEA10Wc +JKEFd3xk62uYNv+nBZ0AucrGqlYLsaqUiEFRVEJkVng/le3DNNJt/kIvi9vhunQK +ksZCHjupsyfMfYS2sgkzzRfyZsgZH5DTNexX1dE9ijngfh1EO9rPQN+nzN60fEIy +XWBiLx0G/b1YqhqSHyiNtmQlNs/9PXDzZtF0aAkCgYBPngjkPFI6uDb6f0mk0wRI +tCicV1k95WS96PLFmbCZYD382SsjRQxESAvnv29v1re5N7fLwHhRHZP35puLQjc7 +SDLJZ9tykgpqvT80ToG6CHKaQLT2hTOB9IA90OdmdL81xzlPZEU6NhIbkefZyy6l +/N5UvmZkTjTaUttldPe7xQKBgEiL9vEJKtR3oXMNEYEHavwjSwlR4t2oncxEFkZM +7OGedj4FzDf0pqJ4gAT9vRQ/B7VUQfPwyHtz097CfNGYFhGttD1b4p7stDrFDcjQ +W1F9cGXS12ro5gPd25abSOtr6hsuG9cIEk6aU67TTrwUtEW33vomibwNH+TC2eQ+ +liIxAoGAd+hpl56dAFSEFgaHn/cq+nKgrT/wkJjjIukAL7X9GCM/+qsXQbKjCh+B +pSi8G8ClSOoLLW0Of/j4uMrD3yNW/3Mww6Rkrl8Pemlno4+mt7ZJW0jGImM1tiFk +QBwTY2HA9w7HhaOn2eDJ/pG1FSdbCeDBTuNMrAtVKsHbJFvWTdw= +-----END RSA PRIVATE KEY----- diff --git a/test/configs/certs/ocsp/server-status-request-url-03-cert.pem b/test/configs/certs/ocsp/server-status-request-url-03-cert.pem new file mode 100644 index 0000000..d84ba21 --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-url-03-cert.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIERzCCAy+gAwIBAgIBBzANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe +Fw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2 +ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAzMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAydHQHjnTS8Anevj4KyqNrionnZgRcwh275OIwJ5o9v8qxatSMQdn +RvQBehShr1R2QGtPPVvXxlIYoKr3H+EGXZEpcyYPuaZ1dKhSZPQpLo+gIQbNXDzY +9754HhCxBLHwx0wtb+flKbb+pgsWQI9t954LWnrNqZUtzPfKh9qg3ODG5sqnuk8Q +R/E7reifsWs2x/iiza8HavTJXZlvED8r7fUEVoxA4UCSHvWogR2VZxCPprbLLVYy +dXthGepvWNSGGZKOMrnJspe0d1zJXUTrzGFc2G6wm+LOJDl02XsfUCP1jHBIoYTK +FpWsPhMZJp5v6Ucw8UPKVvdD3FMoN4XYnwIDAQABo4HWMIHTMAkGA1UdEwQCMAAw +CwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov +LzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr +BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA +AAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAzLmxv +Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAM0XD1w1gEzD/1AubpOF5r0zvRGl3 +Cr9ucLLAXA7/7kq0KsL3FKDU4ybVRZauZVEQYVKXAJGu8mGHM0VIRXtvEjriGw8o +8acKXoMWfKVriAzaKPzHDgPfc1Cq6ejsLrsFMge4BqSua7OVmMNedshEU+Z7bvRD +w+ikh+S0DxWcZxFVnKQqn9WSGvlCF+n0RG0yVjOHt2tLygfzcUVAifAuVs3ktyeH +enX54T/drMIUaUBRjwlMOBXwehRBfOnPpx+RZ/W3IIpf4Pi2XfTEAHVXiDSPz3vM +l+Kqp4ntmJII1XJfoC2+NS7sR2OGDJoIFlrpmetFWhUkSMojPxoeKieR9g== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp/server-status-request-url-03-key.pem b/test/configs/certs/ocsp/server-status-request-url-03-key.pem new file mode 100644 index 0000000..6138e30 --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-url-03-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAydHQHjnTS8Anevj4KyqNrionnZgRcwh275OIwJ5o9v8qxatS +MQdnRvQBehShr1R2QGtPPVvXxlIYoKr3H+EGXZEpcyYPuaZ1dKhSZPQpLo+gIQbN +XDzY9754HhCxBLHwx0wtb+flKbb+pgsWQI9t954LWnrNqZUtzPfKh9qg3ODG5sqn +uk8QR/E7reifsWs2x/iiza8HavTJXZlvED8r7fUEVoxA4UCSHvWogR2VZxCPprbL +LVYydXthGepvWNSGGZKOMrnJspe0d1zJXUTrzGFc2G6wm+LOJDl02XsfUCP1jHBI +oYTKFpWsPhMZJp5v6Ucw8UPKVvdD3FMoN4XYnwIDAQABAoIBADTGqzfcQ0cB3tEn +ni4bmKU83NM3WwjL495OpGpKgoRkwdijLoEnwHgrgt9b2dQxsCK0bSpMhCaWfV42 +lx09CR8awM4d2+refsSc578CompMoMFCOB2ww8Q1iM/D6aaiqaZUY2VqLOE1u23M +ZGtJlqY0LB/jETkRi8KX4dyY7YwUvHE2aDIkz0BKxWudmSbVNT5C0Ykujrm8EJpr +KtRPUoBtw1+gGzEjuIRLxKJbWb5NS8+WA9ePQsS/10u9/qxH6VZ2SHJ2BzBaTHJr +7z0VLtu7VqwXz+N8/N/K6roSlJWihRufPwNcaPagBzWVCURyodyWcSr65VnJB7t4 +9tcnnJECgYEA9a8sIAsmNMb+Kb3AESpxmdFxgX/GJrP8EPis+PlFqTx6EM6J9dbd +e6r9hXAB9tvVouEEi56hRwfv3f0OEhlwdYMK5ar3T2mKrzFoOAT53YwULPj4moH0 +M6hta8B+YBzkWeOC/S6NRzGEkB1ORFI25nZeUtqJ3WvAxDXumxEerDMCgYEA0ksl +GLCbxCefXT9fPySAUafdnyNtBFJPeP3ykSoXrBqbXmdc+1hRUS/weZz7xVtazMrt +Ueg52+wtMhj81RKZbW8wpfrU4tAQcb02DyJFzGwvZT0F4ChCTuNJn3HBtCxNCm9K +4k83BV+4JOrH5VQXncaOMz3U9R/ROqkpJ6mT9eUCgYB/D7d8YwfBZ+Du6Ym51v+l +k6JmXRS304HbrSCYKyMR4YsnwgmrsRIe8VHofwMlDpCwijt1kfbK65nsbwGl7q4w +uDMckI0S7vygmqbRwhEPuXB9yc2Y6vjG9qaQgZ9aSdb3fiiylC7Q0RVEC0P9JZ/r +FPC3XPrMHvw0/ceHPxVj7QKBgBfGGQaGiMWPOSwAixMHXF5e3OYtyhhP+d4Sz4Zr +vMtfIqt0ggWEQYUtJ0GIZsoz/rriQhwdZdUgCSf9vS+Vb7T4egZ5qfGOVh2Vp3Pk +iwGGRYFreauSwZRLi7oz0RM+YuNIG64kCHNaE+ZQiJK7hAP5O4A9gELJ+wxnrVhn +imulAoGAOtDcYdA2gb7rKHTWwVhMEmFtztPGDOzQOMsKJjilRpab88syGtH/Eycd +1nD11MsLGQjrxaZo4eFWEDnomtMVsCixXzr2mmciDw3ZdRdlBrEUfeUjTSBtxSWA +DSx94pwGGVaT+C+cDaFAOms9k3B96AwjomWnbH94w5aufOVy+ak= +-----END RSA PRIVATE KEY----- diff --git a/test/configs/certs/ocsp/server-status-request-url-04-cert.pem b/test/configs/certs/ocsp/server-status-request-url-04-cert.pem new file mode 100644 index 0000000..2488a7d --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-url-04-cert.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIERzCCAy+gAwIBAgIBCDANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe +Fw0yMTA3MDExNTU3MDBaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2 +ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAsSTz1GlWezoGY6Soz2hGAP4gvsWnWdMl8xoYdmUtorcEsauKVv2x +m+9XRX/OxjR9Kq9IyFooMBHJE/Eavft2LkT0BQ33g+NAp/NUGew5hQ/jlN2Dm/C3 +dAbHBeHAvwR9z2XUE6lf58BVsK83xPJau1PuVWC3yxiRC2B08tWm1Ign2soE6XQw +G7tEjPHbgldtgD/dxcXuuovU5UsFB4gB0rRu06M2j2J5sfWvLarWV8sJ2PXZs6Cp +LmkAVf5EtRnfugqbCJ0Xsb5rgfMJ4iWWyYrIVEWbQ6JmNFDuBEWakh3fKSfvBK1I +gbWYeLik5hQUt4WZVZ8q9jjtyewgBgZDvQIDAQABo4HWMIHTMAkGA1UdEwQCMAAw +CwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov +LzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr +BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA +AAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA0Lmxv +Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAGWF7FwDv6NE0oEBsjmOoLXNw9qyJ +TEiTfrWgHfJ2XFDIbpj8dmquX3TmSneWTFRS8LUwR73zl/JJ3Op6Q5ISBpwSXCh4 +8V9XWOB02fhJN1NhXOqxPgNH4EM2m+x/C1wdpZuOrmZjuH3uAW4lxi3lXS+H2AQu +ixD5dXgzrR8l7LIo6RAibqUs2aNG393ck19BF5ghAL+iZtZs0klTrsAyQnzHIGMP +Za2DnK9yjir0M7n4AXspTvtvFj8Zr3WzCAwKW4w025iGxrbJ7Nu4ED9p65s/KZr0 +u8ES5UVaVM1v+7Y+fLzt0v7EX5bSx4okWbNbcMJ3e14JqUGoGMhB8o7IPw== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp/server-status-request-url-04-key.pem b/test/configs/certs/ocsp/server-status-request-url-04-key.pem new file mode 100644 index 0000000..b2bcbf3 --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-url-04-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAsSTz1GlWezoGY6Soz2hGAP4gvsWnWdMl8xoYdmUtorcEsauK +Vv2xm+9XRX/OxjR9Kq9IyFooMBHJE/Eavft2LkT0BQ33g+NAp/NUGew5hQ/jlN2D +m/C3dAbHBeHAvwR9z2XUE6lf58BVsK83xPJau1PuVWC3yxiRC2B08tWm1Ign2soE +6XQwG7tEjPHbgldtgD/dxcXuuovU5UsFB4gB0rRu06M2j2J5sfWvLarWV8sJ2PXZ +s6CpLmkAVf5EtRnfugqbCJ0Xsb5rgfMJ4iWWyYrIVEWbQ6JmNFDuBEWakh3fKSfv +BK1IgbWYeLik5hQUt4WZVZ8q9jjtyewgBgZDvQIDAQABAoIBABJ6PETHyBGKd+gk +4MbhpYus9lVv4IjqEWdOLYcL/rqeRIsTzZZNWB1f1caWEkdyaivtpLXhUZfSAxtl +ZtiyRh2Fa76rOkozhib+pqMdlNJgWejJQlUdwsUDf/cJGUXTfhwIDxsN68cMtON9 +I2ATt4sash2NvR4eLeL16Lz7tC8u6ANGleI2PjG+17qtTSnSoy5wO10M2aK+SrBn +V0C0pT1BqapSTM6NkZ9Q/Z8gPH38Q5Eqc1iabYh0tHop6DLgudh+Au+z9q9NITgL +fkEYxBIcBZ+1b/6HT5XPd1j5sh+VbG7y/xtoIU6YCguWkcxem/JFJWSgSVTyfK57 +Bg284rECgYEA59RnZbyEC2VALPrHAnW7+Ik6LM6fhRED2MlIy7swzrjoT5s5u9OS +li83xgVsSRhwWCf3UoDTyG7ArIF87hwaPXbH8PfZEsV2K8P0ZyaMIz2vQcotT9+A +QmbXbKiXSg6MoSSJic+MrnCspF4c2YpCRnc3h285wUwH9ZvOVfkc6GsCgYEAw5z7 +XpV9iO0eA5dfIeHsTwbFz6AcbPkTPZ5nf4zn+o6HaAV7oyV6f7bF0JYf26guvvg4 +oSYAU9VLRQrw/zzXAyJ16mnJBdlqO6dZU1G7HI6zPZlovuJvze2Q+tNUBZGD6Ouf +0PhWzQtxL+LcmpCul4sDCP7mUCdG7aiClNg5LncCgYBRL8JSD5XSg8/YsK4W/3In +lK8p1+ZnbFEDj2IN7u0lx/2bO0oZq/tO2xRWJboJUySsyrpDS1hffeG9x88fd/n8 +gmd8zN9ZQouiwgoKQaacBNMmYA5ERYoeNvPEWro4tiWrnScJewTSZiUfntHNoSya +js+Ef4AjYGP9MGYvF4F3JwKBgQCEl7kPwNbp5IhuuMFkTyaF3rpg5U+/UgoHv4K6 +Q8HO5aPFD8phqPri7PojTE9l7hdZnRmNqhuYt8CgmS0IZa380vQIpBH95ASNUP9M +ad8iGVxHyd/lW8mbVYfrbSnL6Hn4fRbEaEE2FaZwZh9QqfeegzzYcSeedzEt0QZK +bFI+OwKBgQCDqh0JwrL1WLXlor4N45EImHbeMbFzIZ4ARQuGptQ/m66MKN0cdMG2 +qCysRlj7eac6rEJo4gsQ/ASR0NLPZAxDnHk63PP15ObgALDUppa7fQgTpUgNKgD6 +LqWoCGvCm5etr5WUu621mlZoZpi4IWPU5zJfSoxNwGOJKy0ExNws5A== +-----END RSA PRIVATE KEY----- diff --git a/test/configs/certs/ocsp/server-status-request-url-05-cert.pem b/test/configs/certs/ocsp/server-status-request-url-05-cert.pem new file mode 100644 index 0000000..dcc2571 --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-url-05-cert.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIERzCCAy+gAwIBAgIBCTANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe +Fw0yMTA3MDExNTU3MDBaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2 +ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA1MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEApT8gUAdz5Zgt6Zn+MEqRVn+6R/K9RNKZn7w+KXix7Khfi/67ZOxt +pZnWtUdjgGqPj/Zq9r93OrSb4ZXkquFox1XYdVfyWnq1seIOJfWea2s9iK5UOZEA +MvFHbQOVhV3b2OAtnJEEpuvtbwJsvIRZGYZKxwE2E4rXXeXRzWOWM91KV0Ynoi+V +u1klX/UVB4NPH/ktsm5lzay1otybAEUwy5W2NOY2NhayigV3NHD8vsDnRolD0DF0 +mmlaLfinrzyR8/mPrWkCf58sACwIuc1+f8eaQ+j6MxRZHJzX/lzAXlMMDIGAKlld ++ThiCbbOIvTQDdmQnwI8+iRzQLS8SgnTXwIDAQABo4HWMIHTMAkGA1UdEwQCMAAw +CwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov +LzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr +BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA +AAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA1Lmxv +Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAkmk/YJoaCGip+h+kUjmUQHT929TP +Ix3Jd/nA1LT1QQ7b8rP+jImw51CSLbWwNGihVRKkOOR/kkKZLgWdsa77jySKvqnf +2QX4YcpaSzXitctiVoJAXhyJwEo4y6sRJS27IwFOh+X3gHzPdz7y5KYRfj6OHutC +ArqD6Ohy5dRg1Ixwf2+2go7I1pZIEfzhFHzFUylFmh9ko5EyTQ7cQw8I/D3UI9TB +3TCPUrlSNGOHuREsGMtF7An5Idxytx6Rsx45sJq89tB1eipS//Pk/obRGCZYIb2v +ZmsOSYeppa+97tr2j3ox1cXgXlIvt7ANroZzVHrPhiOMCztrlkFhlqUibA== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp/server-status-request-url-05-key.pem b/test/configs/certs/ocsp/server-status-request-url-05-key.pem new file mode 100644 index 0000000..85d07ce --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-url-05-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEApT8gUAdz5Zgt6Zn+MEqRVn+6R/K9RNKZn7w+KXix7Khfi/67 +ZOxtpZnWtUdjgGqPj/Zq9r93OrSb4ZXkquFox1XYdVfyWnq1seIOJfWea2s9iK5U +OZEAMvFHbQOVhV3b2OAtnJEEpuvtbwJsvIRZGYZKxwE2E4rXXeXRzWOWM91KV0Yn +oi+Vu1klX/UVB4NPH/ktsm5lzay1otybAEUwy5W2NOY2NhayigV3NHD8vsDnRolD +0DF0mmlaLfinrzyR8/mPrWkCf58sACwIuc1+f8eaQ+j6MxRZHJzX/lzAXlMMDIGA +Klld+ThiCbbOIvTQDdmQnwI8+iRzQLS8SgnTXwIDAQABAoIBAH6YfgjRrNiYXZQ5 +haM5fV7nO68jKB0xur/FV1ouPSExXQHDhY0sFFRqJzN6HaCkApgoEIo45P70nveT +/jtrtPoBqnM7jVAD8m704CH0qglhfyLXa28uxxhAAJhjxokF/weHt2nbL9UxqhUB +poxGWHxVhbQwGV2fYJ9vEexn/wp7ok1/gU7E/TVuKAal/PGMjOgkWijDUWHFRzZi +EbaEfXBhLROOX7P3+rCjd7pfPw8oc6ktGYBpQXJmSS6jiDGypPm5XGjvb5lU0wTM +hVEUlKIiZgwtdUieM3vwENoX8z5s3PXkJouGGWhUN63o3HTMD7WppGmYitSTO93b +1u8uRgECgYEA204MDJkLZt10r95+d5N9tnD2gjENJHQzoaziKOOBoTDaTP2+69CI +NpAYOKXcbxY7oOp1a0FbQEcmbF9QsMMFFPzm/+9kn8ekrPIxPmO6o1DB8PTcWOBD +br7x3OgyNtClRa+91D++/CkxBLgZz11uo/VW4UDl5KApuMEzGs4FpKcCgYEAwOV8 +RgmZRZMbS03VOvJsZVhybHVRnyQqtvsm8o7dE1URvHwxon3o9sryGLbtWLh43DcJ +Og87B1CchuiydB9QAof32nVZmAcW0snxmg1/fhDiDO0Nmggm69097INwdypQJNGV +Bo3c7T3uExHQgFVjjyA2hNj8lnLz9H8PJhQOWokCgYB2em3MCIq+Yfn+YYeVdls+ +Sz0CDRn6jcSvHcV5LaAEw23MlnGk2J4eTC6pvAGhCjPgtYoGDeMMkOPTPJNNS67d +LdxmfKCyKZtvy9CK0josrR553O+GHHKRzBrCq7clIgeH5G/70QyPEnnnAMcA3os9 +jPgI7ZTFtTmVFNtVKeSmlQKBgGJr9TfLKABe6dtHcFfuX65qLZ/3UFkx2/WXOi0j +BdwaWL1iZPTy544cyOAhXgMZxNkf375XDRhTYjpXus4TOADCnY1CuR8f5t5Qmcyv +vHB3bs+HmJwSMsHAFht6iUTUOyY+JZq9gY8vPS1PtqH0b6MUnZy1FlQxfRYwSmt0 +akp5AoGAJMvbFeQtX+1v8ronYwSZ1RK5RrvhUp9myP+4naeG9RvvlmmhRf7y+ohe +IRXrvANTbbOTabygbLvuIVCmBus6fbtkH6ez4cLn+CnUpHCbpzrrXKLT2T9r9tl+ +IIgVtavl+Ags+X5XvmP5eGViFZ4oqSECM5Se1oqBX52ljZ/FQmU= +-----END RSA PRIVATE KEY----- diff --git a/test/configs/certs/ocsp/server-status-request-url-06-cert.pem b/test/configs/certs/ocsp/server-status-request-url-06-cert.pem new file mode 100644 index 0000000..ff2a714 --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-url-06-cert.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIERzCCAy+gAwIBAgIBCjANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe +Fw0yMTA3MDExNTU3MDBaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2 +ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA2MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAoVGwsG/awAsfsLb0Sy/GPapkjGnAbumZh9c/9tEKa5ZmbKJnWZVQ +/Qju30IlEiGtPrvBsdpQKowExT6aSK64DytS/aJndT9K9lvzv+Gs4m8eSx1K5Yez +G3QoIxp8TtZOOPMQtSVesUjDOcgmrYXtg4qHayKsKTFzs30t0rzGfjX4V8lYirw+ +hhVQAPRAyTtNOG+Xh66t+nUMtOYbM5QWIn5WDPPqzMU1H+DZd7HrOX008/LgL2Pa +z2FNSGhxz6y8re1hF+G+CC68kMayftYRy3B2NX2E6mLI+Rq0mW7iQR/FNicEtGLd +w5LYzIJ460FUc1iRXyO9We1+IgLaLKbeuwIDAQABo4HWMIHTMAkGA1UdEwQCMAAw +CwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov +LzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr +BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA +AAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA2Lmxv +Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAYGlcBAP3Hdc7sruYdxXk8FYFIBrh +vqLLElpR2qQZHfnXBTDjRf4ZyHbrEPsv7JWDlQMXI3U5SBReiqvv4LYgQ3W6PvZ/ +W3nKUlClu5L64E7ZGGQvGh2lyNWEdG21dlpves3Sb5UNtZdAuOBBJihHcJsUJGSE +P2+lKy4O6lTTba0H9wt0uOiqwUVkwHMF6rqirncRLihXaLcAOFb+NSuFDOMMEX7D +bAZBFDkk38Y7fBlWQ3sR2EvWpXp1uzg0ug8zl+JbFnlDswGSom1Y/NXBzIDYx87c +BlPrhizNUKrCYT1OFz+a6VIYTNM01vyNZ2QEMYgcL4Ug3wYeb88HxEEeMA== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp/server-status-request-url-06-key.pem b/test/configs/certs/ocsp/server-status-request-url-06-key.pem new file mode 100644 index 0000000..943435c --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-url-06-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAoVGwsG/awAsfsLb0Sy/GPapkjGnAbumZh9c/9tEKa5ZmbKJn +WZVQ/Qju30IlEiGtPrvBsdpQKowExT6aSK64DytS/aJndT9K9lvzv+Gs4m8eSx1K +5YezG3QoIxp8TtZOOPMQtSVesUjDOcgmrYXtg4qHayKsKTFzs30t0rzGfjX4V8lY +irw+hhVQAPRAyTtNOG+Xh66t+nUMtOYbM5QWIn5WDPPqzMU1H+DZd7HrOX008/Lg +L2Paz2FNSGhxz6y8re1hF+G+CC68kMayftYRy3B2NX2E6mLI+Rq0mW7iQR/FNicE +tGLdw5LYzIJ460FUc1iRXyO9We1+IgLaLKbeuwIDAQABAoIBADCLDXk6X4AUmNer +Zj3NxYC+6nzaQb7eaoXW/vd9lf4kBvQ/tovdg75w1ri2BcO8AI71HucvA5qaAcF9 +Ah5G/ToLVnDeUvWUMzGJIAMonMaSm3INAYGqYPos7PsLOEbc9DLs6aUo9qKwe5bC +pOjoeT1xDSjvCRFoQSHM1GIqziwiBSYl+30wB6PtE/ww2HtLm1i7z3u8AOB4MHYY +hpDm8r0VMys290goZGSUgz4OD7lXV3Is3F4DO1KJAcP06XDhJhUhIn54xhGJsj85 +24p7hYx1D1gFnbsCpWN2MPj1ZcXgYHR0brW2kEvf4stq5uYqDuQyT1VtvP/31jtW +kmTZBIkCgYEAzY9fsDT8c6DU+Pk2kAiDyKWfEFwSVy0FY1Zs1H0DzL0go7szQGAY +B3bzuuHhvHV450zwLVBec3H8iooi/2H50nnQg6IAjSjvu1qWjmuK6zS/jMcJ1ONX +r7mscyPmeYgnEG4k1AcKnbBVQ0mCYw2u1nJH/jxpavp8BXjfAdhAfqcCgYEAyOc9 +uKZk2WwC+sHXC/gUfjiorFql2KVqJUG/K7hL4Yv/no6HPnI8ksHF9TUFtOGMRR36 +bDd+DwFuaix8IZJw0M8XU7koOrh81iDifbfkljaY35qMlqrVdKIa+fUUh8hgzBWj +DLRj0PoymzguaSoOrnjj+lNhR2hZs0tDayPsVc0CgYEAyJhLFpb+bI/chQIdfrzw +bVNCQCK5Ox0SDvh9x3YfySIeweiigQXFLTOlcQ+Qy+oPDBiGoJG3Og1YFpHqyTEB +K6X4Fxx2UUpLd5dVKLJFpHWbH3P0Yi1gmnkkkk/MT3jQNjZl9grRD0TOT5ViiesG +XPq0yqKFdQHvGR/078XqWi0CgYAjs2P2cistr9H/uX57dARAQdVHe3xJOWvZXwuX +u1JQrE4qYO2LnUVCVwjUgC2ZbRM+HQupO1s9U2XJnEoatSkzEfn1OMv7U1lru8BD +0u/d+anE4nPlOkpgRYZBsNmLH4KEAbxNv6iVNEDV8G/e5EhjnLv2eeaph9OY1uEO +Iv4WaQKBgQDElaNA6WQXsZAH1u1LLC6EWAKAJ/SyJzhMzrs+51MXXB7dIFlpM0BO +DEmXkXSUO5aZQ+fsBgvDJlQhp6yheoLb9K+eCLgELh40sChg9jDFKdVRDShJFh5O +UZYOrHYUSdy/gSzhaF47c+MZrNwwMfE6fbLSOtQrGYS4w5Hh5i+RHA== +-----END RSA PRIVATE KEY----- diff --git a/test/configs/certs/ocsp/server-status-request-url-07-cert.pem b/test/configs/certs/ocsp/server-status-request-url-07-cert.pem new file mode 100644 index 0000000..185645f --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-url-07-cert.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIERzCCAy+gAwIBAgIBCzANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe +Fw0yMTA3MDExNTU3MDBaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2 +ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA3MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAqAFsSyRNQXcB3wCzvNycy+hkNORi9gjJBM0lu0BnxVgvlnroEDT/ +/ZXwWF4Z/D5jbqCqi9iharH1O2wa0/zofAs0DYxLuSEp9efF/CXCURcQviaNHlkg +iar8KeIB0r31ump0KwRUJgZZuDUgCkgNpn2J6QtjwIe7EB/0wssQE5IemHbMyyCh +MwSoMzwDGtpY2tDRfORPN8WMTnIh6Rfl4naszQ2gvveW0z6Ill8O5OmhOZA1niQB +h80f77MYfC9let5wV+yqHFA6++ro4yiSY4+VLAVEIxw7KZt5nmFn5h+v6Gam3dHv +t8MW7yBzA1vDvDO1+fCywV7F3MGpq/bRTwIDAQABo4HWMIHTMAkGA1UdEwQCMAAw +CwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov +LzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr +BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA +AAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA3Lmxv +Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAUcf3kUOmWALEcw/2DPQWzY7nn0mv +RGlgQP4xl3W0MLXg2IywAqpOsrdR8sx9sNBA7WYtUT0fdR4gs1PWbddQeJqIwjLZ +kOxjNwKVCp4DWiKStYA6bIdzNnxXvMKD4FHAUMBo7jsEGzvln0IfutNeE8WKloAB +nezVDTefi+kbB8npk93yag3HXQldAKjUt8VZwebJKu1TvCSLiq4BXXV2DjGIrmTb +0Zhdbar1HeEhf1IsnUyxfuCS+eXrVmF6XesRiaWux7Y79SGD37bBAH+dMwEuKH7+ +025W+IXGls4RvwfoXjVe2GNT9G5aLzzKwY2SE+A3GeeAqf8WCeCAA10nCg== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp/server-status-request-url-07-key.pem b/test/configs/certs/ocsp/server-status-request-url-07-key.pem new file mode 100644 index 0000000..c9367cc --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-url-07-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAqAFsSyRNQXcB3wCzvNycy+hkNORi9gjJBM0lu0BnxVgvlnro +EDT//ZXwWF4Z/D5jbqCqi9iharH1O2wa0/zofAs0DYxLuSEp9efF/CXCURcQviaN +Hlkgiar8KeIB0r31ump0KwRUJgZZuDUgCkgNpn2J6QtjwIe7EB/0wssQE5IemHbM +yyChMwSoMzwDGtpY2tDRfORPN8WMTnIh6Rfl4naszQ2gvveW0z6Ill8O5OmhOZA1 +niQBh80f77MYfC9let5wV+yqHFA6++ro4yiSY4+VLAVEIxw7KZt5nmFn5h+v6Gam +3dHvt8MW7yBzA1vDvDO1+fCywV7F3MGpq/bRTwIDAQABAoIBAC13dR9Ay8eTtWXs +grfx3F0ynyvRQxbiecRa4IM0guDdZMOMr5DzRvQRx+GiVdX17GLcCEugbM67KmrS +btA/+YrYXVXtSfDoELsD5oi+jz/yxg0v0bEM3clkBKoxB91Im1+/v4CEw2EnBtDb +T7h4l3kfMFtpQa2xDebnPw4YdllVGcKEwG5xqMe/EqMg+QpXbqsjvC4lCTJutdHD +NpdHiz/GVq/UdrsM2Wsw1nlBokbt92Gn0LMx/R9kiegwRr0b2Mf1hxIQ/7lStAwB +ZIaWPqQn/nzCZrdn3cx3yzqKDmYKeXSME6SNsKEY465wO1iq0iTdPTdDN40bFZrb +8g31GAECgYEAz2NCn+p+12G9EoeHm1ozdahaF6MZfTp/5cgJwPWeY5TYy3qFFCvE +XPqaOo3EpdEYWjmNBgBqFrlgkuUpOsZekMEyAm6YmG+EPDK9XFuRNTBRaPaDp8bK +RMYh3oDToDzzYVig7MJKi1a05rbo/oGogqex0kMEdToAFIcL56PL588CgYEAz2Lz +sx/hTw0YjaviL+6aSVomttJE4gwrUcMheWUgGQDOK12wSsNqW1UWXdKtLmjqlwbz +y1fxVsCLIdDFdvp7397+xkC9PNPoF78Nq9Kbj6BiC870dGGm/V6o9Bw1FZpDwT0h +aJNecMCHP7PUeGtw0U7x3eBBXB/BCQHa+/iXXoECgYBjDm85QOE68RVFL3UHMAta +TIJTvfSjyvhiAa0e/HKd7++pKSk9XDZbjttx0ls2NGxkVA7W6gXCOuM05r5Ns4hh +rdW8MaUzUjigJEAsSBRQinaoIu75iUr9lIGC7JeodtRtD881pwvCCDU56e3Z+oZJ +ed3Gp3oOoBh/tY1rI+J2IwKBgQCEtMw/NkpF/JpWpcOyalueTqrxWDIt+B2MT3JP +LS/R8Br90xDpdozLbvJGDXc2eHqNNCyzVU4g9krR07dYQEceZaLgmDLABtXAxEfq +SHW0/atf5Qm2o3ppLbatppMthK6QrB6BvbO0MO8bC2cNu6rQtVS+Zy9L1SIAMoi/ +rj6mgQKBgQDDx62JtXHSfaiIOEkpfZW1kOvzYdkjwYT92mJgrpso9ZDr/FhJuAj+ +NxPcO/e6Iie3pijTKbLpJQdE6n2srJ9GEdLPdWUC94ecwmKBA/m0GIO94hm4oIka ++B5U5rkTWxn9xUtOfX4O4MBXpSvgf2a2ZrD34JNftYwzbB7pCkRxGg== +-----END RSA PRIVATE KEY----- diff --git a/test/configs/certs/ocsp/server-status-request-url-08-cert.pem b/test/configs/certs/ocsp/server-status-request-url-08-cert.pem new file mode 100644 index 0000000..c9db72e --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-url-08-cert.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIERzCCAy+gAwIBAgIBDDANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe +Fw0yMTA3MDExNTU3MDBaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 +bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2 +ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA4MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAuDPg2JZudC65wAhHA8hQ/PydGDk8V7ocO1RDKFTp7Lw0Jb5Uz/kP +F5PnZ763kKUA8AIp88AKTxFw4+GTiQJRV5Mg4mvprHzHHkqk1fTSfvWQU4NjancD +RmevCNf9k5H272saSUTNi6wE7vay8nKNEgsWlsJUVGrQHreSXc8VKgNtSl3Qnc9m +umtPHPfS98DvtAWZzCRfTMpeJoAXbntV+TmsSKu4eJPR57u9ZDbMcNnYSB2gwrUV +W6FkRhhrgfCPhBkRw4/WaZcBMoDdg/fjf4LmGeqdrJIwzcoaeAZQhW2+HzsU7zcJ +u0oKFLtTWlySevVPZnOwi06z4uBGv5f/bwIDAQABo4HWMIHTMAkGA1UdEwQCMAAw +CwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov +LzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr +BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA +AAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA4Lmxv +Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAgIB/wKMd38ZiSGP+Pj3t/h7H3uDV +onmoqGw5+By0B555k3g8AHaGzRw2DTWz+1ncxkDZlZvt/GNDX+gg3B/MxA/HKlQp +nf6Ctbo4MFkHtw2DlLPZSo4KHn5TZnBw26R6SApCIJspGBTjCd71l2i50LsaLC9S +fX5OWl2Abgz5xjX7iGBGbR/4S5tWOc/JmSQJtawLGF1w6JDyCHIjWlmz0EjzKu5d +w0+8BGzR8s116TA9PNn/BMf7lxD3bEM2Vh0wchikliFNO5Bdtzz8CCakvBrHhUK4 +Pu28NEST4868VmxhBpn0Im3/3QFfaKuQN2AeAZAojdLorqXwzDZ8wc4XHA== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp/server-status-request-url-08-key.pem b/test/configs/certs/ocsp/server-status-request-url-08-key.pem new file mode 100644 index 0000000..7d8b934 --- /dev/null +++ b/test/configs/certs/ocsp/server-status-request-url-08-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAuDPg2JZudC65wAhHA8hQ/PydGDk8V7ocO1RDKFTp7Lw0Jb5U +z/kPF5PnZ763kKUA8AIp88AKTxFw4+GTiQJRV5Mg4mvprHzHHkqk1fTSfvWQU4Nj +ancDRmevCNf9k5H272saSUTNi6wE7vay8nKNEgsWlsJUVGrQHreSXc8VKgNtSl3Q +nc9mumtPHPfS98DvtAWZzCRfTMpeJoAXbntV+TmsSKu4eJPR57u9ZDbMcNnYSB2g +wrUVW6FkRhhrgfCPhBkRw4/WaZcBMoDdg/fjf4LmGeqdrJIwzcoaeAZQhW2+HzsU +7zcJu0oKFLtTWlySevVPZnOwi06z4uBGv5f/bwIDAQABAoIBAEW39QFlJpc43DtP +prlKx0SJxFfXyfwWR/s4sFaClStyLHagrG8vAERXSb2dlNLdcRZma0SDyiMVIdpO +lc3En876s/afC4mqjJ7td6g3irhc72x8jVNNimLKeZhqT+Lb20/RBNj4fqe/yvo7 +S42yR6ytn7YeBVcSOoV1y6NP7t3AW2L84snfXKBqddOmwoaodV13IDBAEsmwjhVy +XkwRhiuPGCA+A/9x8YWqC+/BJMXQ0zEaAtE9tPnebWbbY8aHhCEhEpo36uG5Wypk +lDS6Bg/w34IBbuUho0NFFVLXkM0/yGVS8UQW8eqEEWLJq6H2t8xwDWNNbXhUgl95 +B85B7bECgYEA60ZLMjYC3BWl+EDlxxxv7DT3FkhFAblWH5IRfCc4ilMJaQxPdwYj +n2Zx/jjrpbk86AiuI4YdE1jM4FIxrkAAqiD91/uTSxX9U81jujuR7GDvBxY73SLY +JvinO+6jpyyDrik03EZh2vZYr5VlwK0qTt588GeA5JGatVxfF6ID3JcCgYEAyG3a +4+/qHW804XkfLrcM5esmBwPateleY1IeQPgv5W3ODlb5uS5C+8eLToM5MCkxvUaN +92siQk/o5+yrNuVUB3Ao4et2iBGpASC2UZ5i5v0mctYOafFz2t1rhPyISI2ivbSe +q272qY7P2qtPMMeyHe3xF33nHcc3JnSiu62C1ukCgYEAttYPxdt3aXVhX4V/i7ar +u9KFWkmbZvWS2kH3WJZaOBTDsWEhuAuLT2qbl8bASi+kB2YHfg/RNKHDxWfat+GB +IrU0z83v72ANWDy7DZURl7LUzpsWtolHlTGTNN4FS/sp8gSP0cbYcQMUdI8TXiK0 +SEpaqbCl1/rXUa2RMJp1ic0CgYBJX64ac+IJFIUPZf/8YhbetM8fElIm9mAPjCh3 +MGSYYTJmHYEeQclT0yE0hOWStAH5gLLIOPg5vndNMF8doaa5cl4FFuY1ugFc4FTe +XFVoqpRAxgxQzIvVO+n4rVpW9UL2oADWUbELMbT2IHDFMtYKDumL6BOL2zpdYaWR +f1u+wQKBgDk9d7b/D53ThOnW+0vYMjo4YMxwg1dcdOBB6p1TrpEBjoqZSPOaY4Ht +OC/cYzvxh5H7YOsfCYzTEea+y3r5lpmpQi6EfvNpbCVsdKVCHFLivFb9/6mirdJq +rUMrTPqCM+Q6aI2/GAwFc//3ItktChuwOxJtH0w/AirvFYR/NHaP +-----END RSA PRIVATE KEY----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/caocsp/caocsp_cert.pem b/test/configs/certs/ocsp_peer/mini-ca/caocsp/caocsp_cert.pem new file mode 100644 index 0000000..b6d024a --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/caocsp/caocsp_cert.pem @@ -0,0 +1,91 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 17:37:00:a1:ce:35:e0:84:dd:e9:30:0c:a7:12:b9:50:88:9c:16:07 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:02:58 2023 GMT + Not After : Apr 28 19:02:58 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=CA OCSP Responder + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:d6:10:15:61:34:1e:97:0d:c6:c2:7d:f2:0f:9a: + 35:56:54:7a:9b:9e:a3:0b:ff:31:0d:db:49:4b:98: + e0:64:3a:3c:7f:4f:4b:d0:a8:01:80:c9:68:4e:76: + 3b:be:7b:d9:56:8d:d4:fd:bf:e1:6f:d0:5c:88:07: + 3f:05:a8:83:b3:7e:0b:ba:e0:36:f6:1c:e0:75:fd: + be:38:26:33:1b:42:96:4e:62:0b:88:36:ef:cc:14: + e3:97:86:dd:c2:78:d3:05:b7:4d:cd:2b:52:f2:11: + 16:d2:7e:8f:f3:47:8c:f9:0f:1e:cd:5e:f7:a4:1c: + 62:34:03:70:74:89:6b:bc:75:e3:30:82:c1:5b:67: + f3:d1:ca:81:13:10:d8:c5:d8:20:05:6d:d1:e7:51: + 19:ac:03:96:2a:a1:21:ff:88:2e:d2:e9:67:79:cf: + ef:17:b5:2b:7c:10:1f:5e:79:3e:08:98:7f:42:bb: + 8a:13:17:2d:9a:1a:8d:ff:36:c2:e9:c0:07:ea:cb: + 4f:72:35:f7:f2:d9:86:d2:ab:6b:70:2b:57:82:c8: + 02:93:aa:04:aa:00:3a:53:23:3d:61:82:32:0e:68: + 33:7e:5f:03:52:c9:53:db:e3:26:46:8a:ab:e0:e5: + 54:57:0d:e3:e3:24:b8:d9:69:92:0a:fb:bd:51:25: + 89:fd + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + D2:00:A2:C3:AA:00:76:1C:E7:67:37:96:89:77:38:69:C5:1B:5E:45 + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Key Usage: critical + Digital Signature + X509v3 Extended Key Usage: critical + OCSP Signing + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + b0:36:29:84:91:de:14:e5:db:bf:55:fc:d8:0a:81:b5:df:84: + e4:5c:ae:e2:3c:1d:05:09:8a:85:7a:9e:f4:82:61:1b:7b:8a: + 0f:1d:e3:ad:b0:60:45:12:2e:38:6d:9c:95:d2:42:fe:2e:1a: + d2:a5:2c:82:40:1e:6c:4b:35:d1:3c:a6:4c:1c:73:c9:d0:32: + e9:47:c9:9a:fa:d0:1a:ef:86:c7:1e:49:ca:62:f1:81:9d:4e: + 38:35:56:1b:53:fe:4a:f4:4c:91:31:8f:32:70:64:ee:91:f7: + 4e:fe:ab:c5:1e:84:d1:43:cd:af:f6:5d:2a:b1:4f:b1:f4:1f: + 5a:9d:33:7a:48:94:c8:88:23:e5:b9:c8:a1:4d:51:4c:d5:3b: + 5f:f7:e8:e5:e1:53:a6:de:c8:95:14:32:e0:52:db:43:d6:c9: + 2f:7f:96:07:fb:87:0a:f0:53:3d:ce:e1:56:6f:dc:0e:84:f3: + e2:ef:dc:17:0f:59:1f:1a:70:d5:7f:08:36:3d:7e:8e:f8:1f: + 55:47:9a:96:1b:11:25:d9:27:7f:bf:e1:65:e5:16:ca:d9:bc: + 6f:5c:5e:a6:4c:d0:7a:24:8d:42:c4:dc:b5:4a:75:4a:7c:88: + da:21:5e:27:e1:0c:36:64:69:10:58:81:3d:cd:74:df:50:85: + c2:71:fe:43 +-----BEGIN CERTIFICATE----- +MIIEGzCCAwOgAwIBAgIUFzcAoc414ITd6TAMpxK5UIicFgcwDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDI1OFoXDTMzMDQyODE5MDI1OFowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFD +QSBPQ1NQIFJlc3BvbmRlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ANYQFWE0HpcNxsJ98g+aNVZUepueowv/MQ3bSUuY4GQ6PH9PS9CoAYDJaE52O757 +2VaN1P2/4W/QXIgHPwWog7N+C7rgNvYc4HX9vjgmMxtClk5iC4g278wU45eG3cJ4 +0wW3Tc0rUvIRFtJ+j/NHjPkPHs1e96QcYjQDcHSJa7x14zCCwVtn89HKgRMQ2MXY +IAVt0edRGawDliqhIf+ILtLpZ3nP7xe1K3wQH155PgiYf0K7ihMXLZoajf82wunA +B+rLT3I19/LZhtKra3ArV4LIApOqBKoAOlMjPWGCMg5oM35fA1LJU9vjJkaKq+Dl +VFcN4+MkuNlpkgr7vVElif0CAwEAAaOB4jCB3zAdBgNVHQ4EFgQU0gCiw6oAdhzn +ZzeWiXc4acUbXkUwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwDAYD +VR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwFgYDVR0lAQH/BAwwCgYIKwYBBQUH +AwkwMwYDVR0fBCwwKjAooCagJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3Rf +Y3JsLmRlcjAyBggrBgEFBQcBAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcu +MC4wLjE6ODg4OC8wDQYJKoZIhvcNAQELBQADggEBALA2KYSR3hTl279V/NgKgbXf +hORcruI8HQUJioV6nvSCYRt7ig8d462wYEUSLjhtnJXSQv4uGtKlLIJAHmxLNdE8 +pkwcc8nQMulHyZr60BrvhsceScpi8YGdTjg1VhtT/kr0TJExjzJwZO6R907+q8Ue +hNFDza/2XSqxT7H0H1qdM3pIlMiII+W5yKFNUUzVO1/36OXhU6beyJUUMuBS20PW +yS9/lgf7hwrwUz3O4VZv3A6E8+Lv3BcPWR8acNV/CDY9fo74H1VHmpYbESXZJ3+/ +4WXlFsrZvG9cXqZM0HokjULE3LVKdUp8iNohXifhDDZkaRBYgT3NdN9QhcJx/kM= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/caocsp/private/caocsp_keypair.pem b/test/configs/certs/ocsp_peer/mini-ca/caocsp/private/caocsp_keypair.pem new file mode 100644 index 0000000..e3ac9b3 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/caocsp/private/caocsp_keypair.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDWEBVhNB6XDcbC +ffIPmjVWVHqbnqML/zEN20lLmOBkOjx/T0vQqAGAyWhOdju+e9lWjdT9v+Fv0FyI +Bz8FqIOzfgu64Db2HOB1/b44JjMbQpZOYguINu/MFOOXht3CeNMFt03NK1LyERbS +fo/zR4z5Dx7NXvekHGI0A3B0iWu8deMwgsFbZ/PRyoETENjF2CAFbdHnURmsA5Yq +oSH/iC7S6Wd5z+8XtSt8EB9eeT4ImH9Cu4oTFy2aGo3/NsLpwAfqy09yNffy2YbS +q2twK1eCyAKTqgSqADpTIz1hgjIOaDN+XwNSyVPb4yZGiqvg5VRXDePjJLjZaZIK ++71RJYn9AgMBAAECggEACnoECdtaqervMOKoH7Jc7Oo6i/ZCJZqqRHLYjf4f8VfW +USbI35/xXuO8mqZ3uxVlqDJN29NxzZ6lgLTWFUlPlM/U9CL4HaiBJdUy452e/7UN +FS4AQXzq1JKrJuXfYZ63OT7k7Gcz6owCkW/HTNFSKXhfeg6tURdgiQooDVQSdUk6 +xX4gVEK3skozRXf4mrjTaNnFCOk2+sZdqrRn19ZAUGRisv6ECf8/wQlh3+ySfPYV +u+BHQqzntToYP0HUZAO6rezcTVayW25E+AaOqNdNmSqcOX218ohVCzwzFpzIk8LW +jYLyGQBhHHcw+RHJeitcHrDuTTpOZFznQxzHiGH3AwKBgQD91TNHx9Y9jUBkISSi +XylSiZEAOjPl4VrhRfI5OUx1l3XTqB/e3xBYLwxEpXjs7m3qyXhCe+rVuIwSjLzc +mLCspPZw/fxdRefWW5B+v1HbHxC3/lBOhqaDfLL6x4A/q3n/itG9X0GjpfvRkdJY +GYOJea/2rJuMsFs3atX160p4cwKBgQDX4/VXJWxxWUJbObwoxxABG9VTZdI6Dsqr +8tgg+7NPqw3PAo5W+XLsGZCSWQfJTD49AHcHBon5IfEDa5srfKsOXFXoiNEdCjIG +zJ9mNtGMokXOWLKgxMoqHz+WnqWgxi9D7QwWWNq5hWnACJUqeqelRMzoNkmr96DX +NloqHREHzwKBgQC0jKnlLOfe8FIU5t5AAKBL7T4Og1fW8+zIwBADVBZmrk1JOBUz +Wkct8okvauQQ46ebkaLQ54OqcZJwv1q3LoS8yLnitUaEseyuNIMbJMr8qaQiu+oz +cOOQM2q7ppw6raYhdoSpxs/Rr4bnEmoj8EH3z26ybyRVdjvrtzppqetWsQKBgQCa +YogGA9siy6PqPMVTm9bUFCVfeEb4Aa/pesYYACbgaAB98uP7SnNmZ3m9TjGFQCKZ +2QVFXuW35Q/HVGIonQRuRpWgroZr7+iKeDXdEIKVwU2OHFvRICk6KhJ9EYJ8EH2o +Y5HrQStY1BElpH2XXRMZ2rN1s6zHb1Pz0whzaUnOfQKBgQCpfJYh1Yzpryb0hkfa +MAL2Rsw+mpYeJ27Bmv+taW5iEVMQr2AEYNJhQx1SjNZOml2mqY6un4UPwhUwqAqg +SOgWNQGD5g6xoM6Hom+nZG03QacCYUOaD6xDmVKTY0LnzVBwspfvrIgLKgZ7IWBx +KlqvY5FJ+NXg3wHLNwgGzkgVPg== +-----END PRIVATE KEY----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/client1/System_bundle.pem b/test/configs/certs/ocsp_peer/mini-ca/client1/System_bundle.pem new file mode 100644 index 0000000..e896906 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/client1/System_bundle.pem @@ -0,0 +1,186 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 7c:43:65:c7:cf:27:e3:83:ae:2f:60:ac:03:e5:f2:b6:22:88:bc:a2 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Validity + Not Before: May 1 19:37:36 2023 GMT + Not After : Apr 28 19:37:36 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=System + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:a3:21:2f:74:34:c1:1b:41:90:b6:4e:41:72:e0: + 3f:9e:49:94:55:ec:02:4c:dd:14:80:b8:3d:c6:c7: + 47:bb:a5:59:c3:35:86:89:17:08:ce:fe:71:e6:2f: + 9c:c1:db:d2:7e:14:24:da:61:30:3a:e7:6e:b1:e3: + 21:38:81:bc:47:df:b2:7f:1f:60:be:3d:c5:ed:76: + 03:94:e3:c4:b3:3e:bf:f8:43:ba:c2:54:bc:bb:66: + 59:98:a3:f9:aa:e3:10:e8:c3:88:dc:1a:18:6f:dd: + 90:eb:6f:a3:4b:d4:af:34:5c:43:20:d5:5b:e7:98: + a5:7c:7b:a9:15:86:bb:28:bf:ba:e0:bb:f7:1c:08: + c4:26:eb:c1:ac:05:1f:74:4f:05:11:57:e0:12:77: + 17:9e:89:dd:a5:38:ee:cf:cf:67:be:0c:5e:6a:4a: + 74:61:21:79:8e:c3:28:f1:e2:06:00:2d:ea:3a:6d: + e2:a6:25:fd:2d:8b:f5:82:36:91:8a:21:f0:6a:93: + 19:d6:76:08:fd:cd:ee:90:a9:a9:cf:99:30:71:46: + 57:ea:fb:c5:65:4f:7c:86:5c:9d:d7:b4:c3:27:3c: + eb:27:dd:bc:55:76:1f:25:0d:cb:6f:43:9a:9f:ba: + de:54:c1:90:03:9e:e5:0d:d9:cd:84:d4:58:74:63: + be:59 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + A0:FA:B5:24:42:70:DF:E1:BB:E6:10:62:BE:FE:F5:81:13:2F:31:9B + X509v3 Authority Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, S/MIME + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Client Authentication, E-mail Protection + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:18888/intermediate1_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:18888/ + X509v3 Subject Alternative Name: + email:System@user.net + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + ad:00:40:7a:34:ad:07:e9:ed:fa:8f:1f:48:08:79:81:a8:3c: + 90:da:05:95:74:05:51:9c:17:a8:5c:03:09:c8:f8:2c:09:64: + e2:7c:fc:69:e1:c0:5d:8a:d9:f0:f3:e4:cd:2c:5e:43:77:71: + f8:58:20:88:8f:63:e1:b4:86:db:7a:54:df:ce:be:01:e2:55: + a2:70:a8:89:64:cf:2a:13:78:91:de:83:ed:d6:74:24:00:ca: + 3d:67:4a:cd:e3:82:b9:56:a3:3a:b4:80:b2:ac:61:e9:75:6c: + 30:1c:81:96:2f:f0:99:b2:7b:73:b5:45:b0:3c:20:ed:54:b3: + 87:37:9f:5e:07:c4:8a:72:94:53:4e:a2:a0:83:bc:fb:61:59: + ff:8c:91:1c:db:ad:7a:e0:12:e3:a3:b1:91:97:d4:c7:ed:02: + 6e:7e:01:d8:d6:d5:6d:81:a2:32:ca:8c:6d:32:91:40:97:e5: + a1:ad:22:7d:af:ab:ce:68:0b:69:52:53:8a:80:dd:f3:9f:a8: + 1f:34:a7:1f:37:58:cb:6c:da:54:cf:cc:0b:67:95:e9:6e:30: + a4:ce:12:c4:5a:e0:d4:92:fb:0b:67:a8:51:ad:dc:4a:d0:ad: + fb:92:77:85:a5:9d:84:ff:99:50:ca:15:4f:d4:30:c8:85:ca: + 95:a0:88:62 +-----BEGIN CERTIFICATE----- +MIIEXTCCA0WgAwIBAgIUfENlx88n44OuL2CsA+XytiKIvKIwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe +Fw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP +MA0GA1UEAwwGU3lzdGVtMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +oyEvdDTBG0GQtk5BcuA/nkmUVewCTN0UgLg9xsdHu6VZwzWGiRcIzv5x5i+cwdvS +fhQk2mEwOuduseMhOIG8R9+yfx9gvj3F7XYDlOPEsz6/+EO6wlS8u2ZZmKP5quMQ +6MOI3BoYb92Q62+jS9SvNFxDINVb55ilfHupFYa7KL+64Lv3HAjEJuvBrAUfdE8F +EVfgEncXnondpTjuz89nvgxeakp0YSF5jsMo8eIGAC3qOm3ipiX9LYv1gjaRiiHw +apMZ1nYI/c3ukKmpz5kwcUZX6vvFZU98hlyd17TDJzzrJ928VXYfJQ3Lb0Oan7re +VMGQA57lDdnNhNRYdGO+WQIDAQABo4IBJDCCASAwHQYDVR0OBBYEFKD6tSRCcN/h +u+YQYr7+9YETLzGbMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG +A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd +BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs +aHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI +KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4 +LzAaBgNVHREEEzARgQ9TeXN0ZW1AdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB +AK0AQHo0rQfp7fqPH0gIeYGoPJDaBZV0BVGcF6hcAwnI+CwJZOJ8/GnhwF2K2fDz +5M0sXkN3cfhYIIiPY+G0htt6VN/OvgHiVaJwqIlkzyoTeJHeg+3WdCQAyj1nSs3j +grlWozq0gLKsYel1bDAcgZYv8Jmye3O1RbA8IO1Us4c3n14HxIpylFNOoqCDvPth +Wf+MkRzbrXrgEuOjsZGX1MftAm5+AdjW1W2BojLKjG0ykUCX5aGtIn2vq85oC2lS +U4qA3fOfqB80px83WMts2lTPzAtnleluMKTOEsRa4NSS+wtnqFGt3ErQrfuSd4Wl +nYT/mVDKFU/UMMiFypWgiGI= +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:15 2023 GMT + Not After : Apr 28 19:01:15 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: + d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: + 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: + b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: + a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: + 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: + d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: + 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: + 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: + 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: + 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: + 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: + a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: + c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: + 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: + 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: + 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: + 40:27 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: + a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: + d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: + 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: + 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: + c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: + 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: + 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: + 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: + be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: + 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: + 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: + ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: + 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: + 91:5c:43:a9 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC +YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL +3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy +n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu +hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX +Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 ++bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp +LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr +uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN +zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg +zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX +KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/client1/System_cert.pem b/test/configs/certs/ocsp_peer/mini-ca/client1/System_cert.pem new file mode 100644 index 0000000..3354853 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/client1/System_cert.pem @@ -0,0 +1,97 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 7c:43:65:c7:cf:27:e3:83:ae:2f:60:ac:03:e5:f2:b6:22:88:bc:a2 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Validity + Not Before: May 1 19:37:36 2023 GMT + Not After : Apr 28 19:37:36 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=System + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:a3:21:2f:74:34:c1:1b:41:90:b6:4e:41:72:e0: + 3f:9e:49:94:55:ec:02:4c:dd:14:80:b8:3d:c6:c7: + 47:bb:a5:59:c3:35:86:89:17:08:ce:fe:71:e6:2f: + 9c:c1:db:d2:7e:14:24:da:61:30:3a:e7:6e:b1:e3: + 21:38:81:bc:47:df:b2:7f:1f:60:be:3d:c5:ed:76: + 03:94:e3:c4:b3:3e:bf:f8:43:ba:c2:54:bc:bb:66: + 59:98:a3:f9:aa:e3:10:e8:c3:88:dc:1a:18:6f:dd: + 90:eb:6f:a3:4b:d4:af:34:5c:43:20:d5:5b:e7:98: + a5:7c:7b:a9:15:86:bb:28:bf:ba:e0:bb:f7:1c:08: + c4:26:eb:c1:ac:05:1f:74:4f:05:11:57:e0:12:77: + 17:9e:89:dd:a5:38:ee:cf:cf:67:be:0c:5e:6a:4a: + 74:61:21:79:8e:c3:28:f1:e2:06:00:2d:ea:3a:6d: + e2:a6:25:fd:2d:8b:f5:82:36:91:8a:21:f0:6a:93: + 19:d6:76:08:fd:cd:ee:90:a9:a9:cf:99:30:71:46: + 57:ea:fb:c5:65:4f:7c:86:5c:9d:d7:b4:c3:27:3c: + eb:27:dd:bc:55:76:1f:25:0d:cb:6f:43:9a:9f:ba: + de:54:c1:90:03:9e:e5:0d:d9:cd:84:d4:58:74:63: + be:59 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + A0:FA:B5:24:42:70:DF:E1:BB:E6:10:62:BE:FE:F5:81:13:2F:31:9B + X509v3 Authority Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, S/MIME + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Client Authentication, E-mail Protection + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:18888/intermediate1_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:18888/ + X509v3 Subject Alternative Name: + email:System@user.net + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + ad:00:40:7a:34:ad:07:e9:ed:fa:8f:1f:48:08:79:81:a8:3c: + 90:da:05:95:74:05:51:9c:17:a8:5c:03:09:c8:f8:2c:09:64: + e2:7c:fc:69:e1:c0:5d:8a:d9:f0:f3:e4:cd:2c:5e:43:77:71: + f8:58:20:88:8f:63:e1:b4:86:db:7a:54:df:ce:be:01:e2:55: + a2:70:a8:89:64:cf:2a:13:78:91:de:83:ed:d6:74:24:00:ca: + 3d:67:4a:cd:e3:82:b9:56:a3:3a:b4:80:b2:ac:61:e9:75:6c: + 30:1c:81:96:2f:f0:99:b2:7b:73:b5:45:b0:3c:20:ed:54:b3: + 87:37:9f:5e:07:c4:8a:72:94:53:4e:a2:a0:83:bc:fb:61:59: + ff:8c:91:1c:db:ad:7a:e0:12:e3:a3:b1:91:97:d4:c7:ed:02: + 6e:7e:01:d8:d6:d5:6d:81:a2:32:ca:8c:6d:32:91:40:97:e5: + a1:ad:22:7d:af:ab:ce:68:0b:69:52:53:8a:80:dd:f3:9f:a8: + 1f:34:a7:1f:37:58:cb:6c:da:54:cf:cc:0b:67:95:e9:6e:30: + a4:ce:12:c4:5a:e0:d4:92:fb:0b:67:a8:51:ad:dc:4a:d0:ad: + fb:92:77:85:a5:9d:84:ff:99:50:ca:15:4f:d4:30:c8:85:ca: + 95:a0:88:62 +-----BEGIN CERTIFICATE----- +MIIEXTCCA0WgAwIBAgIUfENlx88n44OuL2CsA+XytiKIvKIwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe +Fw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP +MA0GA1UEAwwGU3lzdGVtMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +oyEvdDTBG0GQtk5BcuA/nkmUVewCTN0UgLg9xsdHu6VZwzWGiRcIzv5x5i+cwdvS +fhQk2mEwOuduseMhOIG8R9+yfx9gvj3F7XYDlOPEsz6/+EO6wlS8u2ZZmKP5quMQ +6MOI3BoYb92Q62+jS9SvNFxDINVb55ilfHupFYa7KL+64Lv3HAjEJuvBrAUfdE8F +EVfgEncXnondpTjuz89nvgxeakp0YSF5jsMo8eIGAC3qOm3ipiX9LYv1gjaRiiHw +apMZ1nYI/c3ukKmpz5kwcUZX6vvFZU98hlyd17TDJzzrJ928VXYfJQ3Lb0Oan7re +VMGQA57lDdnNhNRYdGO+WQIDAQABo4IBJDCCASAwHQYDVR0OBBYEFKD6tSRCcN/h +u+YQYr7+9YETLzGbMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG +A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd +BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs +aHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI +KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4 +LzAaBgNVHREEEzARgQ9TeXN0ZW1AdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB +AK0AQHo0rQfp7fqPH0gIeYGoPJDaBZV0BVGcF6hcAwnI+CwJZOJ8/GnhwF2K2fDz +5M0sXkN3cfhYIIiPY+G0htt6VN/OvgHiVaJwqIlkzyoTeJHeg+3WdCQAyj1nSs3j +grlWozq0gLKsYel1bDAcgZYv8Jmye3O1RbA8IO1Us4c3n14HxIpylFNOoqCDvPth +Wf+MkRzbrXrgEuOjsZGX1MftAm5+AdjW1W2BojLKjG0ykUCX5aGtIn2vq85oC2lS +U4qA3fOfqB80px83WMts2lTPzAtnleluMKTOEsRa4NSS+wtnqFGt3ErQrfuSd4Wl +nYT/mVDKFU/UMMiFypWgiGI= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem b/test/configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem new file mode 100644 index 0000000..a27daa1 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem @@ -0,0 +1,186 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 5c:a1:af:d5:7c:bb:16:ef:c2:c7:e6:53:fc:94:1a:ed:24:bb:b4:17 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Validity + Not Before: May 1 19:37:36 2023 GMT + Not After : Apr 28 19:37:36 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserA1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b4:eb:22:e2:c4:ba:7f:33:aa:57:ab:13:f1:69: + 09:98:28:3c:7d:a7:e2:41:2a:28:2f:f9:85:a1:6c: + 94:ee:0a:eb:4d:01:4c:28:7c:9d:05:4d:d8:10:7f: + b7:cf:13:c2:a6:de:11:0c:97:38:97:cd:6d:11:fd: + 16:76:c0:eb:5a:b7:7b:17:13:45:9d:4b:00:4f:26: + c5:b1:9b:67:93:2c:d6:d5:33:37:e1:50:1d:7b:0d: + be:8c:cb:bd:29:99:8f:54:f6:7e:04:84:82:2a:28: + ee:71:3e:8d:5f:72:b2:6a:77:6b:47:3e:ba:4d:b3: + e2:96:14:71:0a:1e:26:16:8f:6c:1b:07:2a:ac:15: + 89:1e:88:63:c3:81:3b:91:e9:f3:43:1b:f0:ec:08: + 24:96:46:27:21:2a:56:25:2c:b6:cc:d9:02:70:77: + 9d:e4:7c:44:8c:93:04:85:a3:09:0a:8e:f5:e7:21: + fa:bd:56:28:b7:52:20:09:ec:9a:c4:d4:d7:8a:19: + 4e:7a:10:e9:b2:10:36:68:ce:ce:78:8b:79:3f:6f: + 70:3b:75:6d:70:59:3a:c9:85:a8:f8:23:d4:ab:44: + c2:ae:f5:1c:6e:38:11:e1:5f:cc:8f:e2:43:f5:b3: + 0e:09:17:b3:c6:ee:47:fb:39:c4:58:62:ba:e3:a8: + c5:ef + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 70:55:CA:CA:A5:8F:4D:73:39:47:E2:97:A3:1F:F6:3E:33:C9:7A:BF + X509v3 Authority Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, S/MIME + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Client Authentication, E-mail Protection + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:18888/intermediate1_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:18888/ + X509v3 Subject Alternative Name: + email:UserA1@user.net + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 99:81:61:3a:f1:c2:de:05:ad:ab:f3:fd:e0:d5:97:5b:fe:b2: + fa:e2:5f:ab:41:9d:71:1d:10:54:0b:bc:b5:c9:8d:26:91:a9: + 45:71:51:14:61:a7:3c:ef:1d:f7:db:71:2f:1f:c1:d7:80:96: + 03:5d:0d:69:81:fa:be:ca:f7:56:70:7b:89:ca:8f:b6:16:ee: + 4a:83:fc:70:2e:4b:0c:50:ba:c6:06:5e:58:bb:25:d6:19:40: + 82:b4:18:57:16:5f:f2:98:3e:5d:9d:72:7a:8f:20:de:25:c2: + 06:a7:46:b2:cc:4c:f9:da:a7:43:f5:a0:92:e4:e2:05:49:43: + 9d:58:9f:20:5d:e2:88:77:f1:10:0c:f5:fc:a2:85:b6:41:0a: + 1a:12:75:1e:47:3b:b3:4f:c9:45:71:99:b6:14:e9:6b:7d:7a: + 98:ee:82:dd:59:f6:af:fa:a5:d1:1c:24:db:66:e7:82:bb:53: + 70:4f:27:96:dc:19:c0:9e:2d:df:da:00:2f:c3:22:9e:71:9c: + b3:89:da:0a:79:c3:f6:e3:9b:ca:b7:db:b6:5c:8f:e9:29:cb: + d0:9c:e3:0e:0f:7c:2c:b5:b0:36:a9:13:38:d2:8e:6f:6a:6c: + 0a:7f:3f:dd:af:b1:e2:ea:c6:de:1d:b0:97:c9:36:1d:85:81: + aa:42:9f:53 +-----BEGIN CERTIFICATE----- +MIIEXTCCA0WgAwIBAgIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBcwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe +Fw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP +MA0GA1UEAwwGVXNlckExMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +tOsi4sS6fzOqV6sT8WkJmCg8fafiQSooL/mFoWyU7grrTQFMKHydBU3YEH+3zxPC +pt4RDJc4l81tEf0WdsDrWrd7FxNFnUsATybFsZtnkyzW1TM34VAdew2+jMu9KZmP +VPZ+BISCKijucT6NX3KyandrRz66TbPilhRxCh4mFo9sGwcqrBWJHohjw4E7kenz +Qxvw7AgklkYnISpWJSy2zNkCcHed5HxEjJMEhaMJCo715yH6vVYot1IgCeyaxNTX +ihlOehDpshA2aM7OeIt5P29wO3VtcFk6yYWo+CPUq0TCrvUcbjgR4V/Mj+JD9bMO +CRezxu5H+znEWGK646jF7wIDAQABo4IBJDCCASAwHQYDVR0OBBYEFHBVysqlj01z +OUfil6Mf9j4zyXq/MB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG +A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd +BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs +aHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI +KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4 +LzAaBgNVHREEEzARgQ9Vc2VyQTFAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB +AJmBYTrxwt4Fravz/eDVl1v+svriX6tBnXEdEFQLvLXJjSaRqUVxURRhpzzvHffb +cS8fwdeAlgNdDWmB+r7K91Zwe4nKj7YW7kqD/HAuSwxQusYGXli7JdYZQIK0GFcW +X/KYPl2dcnqPIN4lwganRrLMTPnap0P1oJLk4gVJQ51YnyBd4oh38RAM9fyihbZB +ChoSdR5HO7NPyUVxmbYU6Wt9epjugt1Z9q/6pdEcJNtm54K7U3BPJ5bcGcCeLd/a +AC/DIp5xnLOJ2gp5w/bjm8q327Zcj+kpy9Cc4w4PfCy1sDapEzjSjm9qbAp/P92v +seLqxt4dsJfJNh2FgapCn1M= +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:15 2023 GMT + Not After : Apr 28 19:01:15 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: + d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: + 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: + b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: + a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: + 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: + d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: + 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: + 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: + 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: + 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: + 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: + a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: + c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: + 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: + 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: + 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: + 40:27 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: + a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: + d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: + 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: + 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: + c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: + 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: + 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: + 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: + be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: + 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: + 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: + ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: + 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: + 91:5c:43:a9 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC +YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL +3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy +n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu +hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX +Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 ++bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp +LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr +uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN +zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg +zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX +KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem b/test/configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem new file mode 100644 index 0000000..a2c5078 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem @@ -0,0 +1,97 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 5c:a1:af:d5:7c:bb:16:ef:c2:c7:e6:53:fc:94:1a:ed:24:bb:b4:17 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Validity + Not Before: May 1 19:37:36 2023 GMT + Not After : Apr 28 19:37:36 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserA1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b4:eb:22:e2:c4:ba:7f:33:aa:57:ab:13:f1:69: + 09:98:28:3c:7d:a7:e2:41:2a:28:2f:f9:85:a1:6c: + 94:ee:0a:eb:4d:01:4c:28:7c:9d:05:4d:d8:10:7f: + b7:cf:13:c2:a6:de:11:0c:97:38:97:cd:6d:11:fd: + 16:76:c0:eb:5a:b7:7b:17:13:45:9d:4b:00:4f:26: + c5:b1:9b:67:93:2c:d6:d5:33:37:e1:50:1d:7b:0d: + be:8c:cb:bd:29:99:8f:54:f6:7e:04:84:82:2a:28: + ee:71:3e:8d:5f:72:b2:6a:77:6b:47:3e:ba:4d:b3: + e2:96:14:71:0a:1e:26:16:8f:6c:1b:07:2a:ac:15: + 89:1e:88:63:c3:81:3b:91:e9:f3:43:1b:f0:ec:08: + 24:96:46:27:21:2a:56:25:2c:b6:cc:d9:02:70:77: + 9d:e4:7c:44:8c:93:04:85:a3:09:0a:8e:f5:e7:21: + fa:bd:56:28:b7:52:20:09:ec:9a:c4:d4:d7:8a:19: + 4e:7a:10:e9:b2:10:36:68:ce:ce:78:8b:79:3f:6f: + 70:3b:75:6d:70:59:3a:c9:85:a8:f8:23:d4:ab:44: + c2:ae:f5:1c:6e:38:11:e1:5f:cc:8f:e2:43:f5:b3: + 0e:09:17:b3:c6:ee:47:fb:39:c4:58:62:ba:e3:a8: + c5:ef + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 70:55:CA:CA:A5:8F:4D:73:39:47:E2:97:A3:1F:F6:3E:33:C9:7A:BF + X509v3 Authority Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, S/MIME + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Client Authentication, E-mail Protection + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:18888/intermediate1_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:18888/ + X509v3 Subject Alternative Name: + email:UserA1@user.net + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 99:81:61:3a:f1:c2:de:05:ad:ab:f3:fd:e0:d5:97:5b:fe:b2: + fa:e2:5f:ab:41:9d:71:1d:10:54:0b:bc:b5:c9:8d:26:91:a9: + 45:71:51:14:61:a7:3c:ef:1d:f7:db:71:2f:1f:c1:d7:80:96: + 03:5d:0d:69:81:fa:be:ca:f7:56:70:7b:89:ca:8f:b6:16:ee: + 4a:83:fc:70:2e:4b:0c:50:ba:c6:06:5e:58:bb:25:d6:19:40: + 82:b4:18:57:16:5f:f2:98:3e:5d:9d:72:7a:8f:20:de:25:c2: + 06:a7:46:b2:cc:4c:f9:da:a7:43:f5:a0:92:e4:e2:05:49:43: + 9d:58:9f:20:5d:e2:88:77:f1:10:0c:f5:fc:a2:85:b6:41:0a: + 1a:12:75:1e:47:3b:b3:4f:c9:45:71:99:b6:14:e9:6b:7d:7a: + 98:ee:82:dd:59:f6:af:fa:a5:d1:1c:24:db:66:e7:82:bb:53: + 70:4f:27:96:dc:19:c0:9e:2d:df:da:00:2f:c3:22:9e:71:9c: + b3:89:da:0a:79:c3:f6:e3:9b:ca:b7:db:b6:5c:8f:e9:29:cb: + d0:9c:e3:0e:0f:7c:2c:b5:b0:36:a9:13:38:d2:8e:6f:6a:6c: + 0a:7f:3f:dd:af:b1:e2:ea:c6:de:1d:b0:97:c9:36:1d:85:81: + aa:42:9f:53 +-----BEGIN CERTIFICATE----- +MIIEXTCCA0WgAwIBAgIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBcwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe +Fw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP +MA0GA1UEAwwGVXNlckExMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +tOsi4sS6fzOqV6sT8WkJmCg8fafiQSooL/mFoWyU7grrTQFMKHydBU3YEH+3zxPC +pt4RDJc4l81tEf0WdsDrWrd7FxNFnUsATybFsZtnkyzW1TM34VAdew2+jMu9KZmP +VPZ+BISCKijucT6NX3KyandrRz66TbPilhRxCh4mFo9sGwcqrBWJHohjw4E7kenz +Qxvw7AgklkYnISpWJSy2zNkCcHed5HxEjJMEhaMJCo715yH6vVYot1IgCeyaxNTX +ihlOehDpshA2aM7OeIt5P29wO3VtcFk6yYWo+CPUq0TCrvUcbjgR4V/Mj+JD9bMO +CRezxu5H+znEWGK646jF7wIDAQABo4IBJDCCASAwHQYDVR0OBBYEFHBVysqlj01z +OUfil6Mf9j4zyXq/MB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG +A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd +BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs +aHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI +KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4 +LzAaBgNVHREEEzARgQ9Vc2VyQTFAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB +AJmBYTrxwt4Fravz/eDVl1v+svriX6tBnXEdEFQLvLXJjSaRqUVxURRhpzzvHffb +cS8fwdeAlgNdDWmB+r7K91Zwe4nKj7YW7kqD/HAuSwxQusYGXli7JdYZQIK0GFcW +X/KYPl2dcnqPIN4lwganRrLMTPnap0P1oJLk4gVJQ51YnyBd4oh38RAM9fyihbZB +ChoSdR5HO7NPyUVxmbYU6Wt9epjugt1Z9q/6pdEcJNtm54K7U3BPJ5bcGcCeLd/a +AC/DIp5xnLOJ2gp5w/bjm8q327Zcj+kpy9Cc4w4PfCy1sDapEzjSjm9qbAp/P92v +seLqxt4dsJfJNh2FgapCn1M= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/client1/UserA2_bundle.pem b/test/configs/certs/ocsp_peer/mini-ca/client1/UserA2_bundle.pem new file mode 100644 index 0000000..a181550 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/client1/UserA2_bundle.pem @@ -0,0 +1,186 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 7a:3d:fa:5b:9b:df:69:55:6e:9c:53:4c:fc:86:75:65:bc:78:4c:24 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Validity + Not Before: May 1 19:37:36 2023 GMT + Not After : Apr 28 19:37:36 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserA2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:a6:7c:40:80:2b:44:00:33:11:c6:c2:9d:67:3e: + 87:8e:7e:40:d3:f5:d3:27:b6:7d:18:3c:c0:86:ac: + 96:3a:ad:d8:c3:cb:ab:72:5e:4c:b7:24:45:da:c7: + a8:cc:74:b8:21:75:62:9e:81:88:96:54:6e:db:f9: + 8c:2f:4c:97:0d:ce:21:42:2f:92:57:7f:34:2b:02: + 43:4c:22:ae:14:ca:fc:b2:2c:d0:67:0e:52:e0:6d: + 61:96:a6:3b:cc:4f:6a:d6:ef:45:9c:74:92:25:6c: + 0a:10:62:1b:22:2b:11:6b:d1:52:4d:da:8d:c3:4a: + e6:74:a7:1b:1e:ef:8a:f4:96:88:02:0d:b7:57:35: + 9f:a3:ff:a2:2c:b7:0e:27:4e:79:2f:cf:0c:f1:91: + 0e:bf:01:d7:a2:71:2c:b7:0e:4b:7e:50:91:89:71: + c2:17:aa:cb:29:80:9e:d7:2b:fa:33:41:e8:82:d1: + 3a:97:3d:6c:de:66:9b:b4:ea:1a:eb:94:be:6e:c0: + 66:e8:77:3d:72:d5:5c:a5:e8:ab:3b:33:f4:b3:c2: + 26:49:bc:08:55:cf:16:b6:12:22:91:fe:c1:5a:b2: + d7:77:e3:f4:47:bc:c4:77:6b:f5:7f:c3:e8:48:99: + b9:a8:ea:b1:ae:e6:cc:3a:12:fa:4d:2f:5f:0f:a8: + fd:8d + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + B8:8E:4F:76:F1:F8:3C:A5:23:C5:8F:A1:2E:64:3E:48:53:02:CD:6B + X509v3 Authority Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, S/MIME + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Client Authentication, E-mail Protection + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:18888/intermediate1_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:18888/ + X509v3 Subject Alternative Name: + email:UserA2@user.net + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 7c:1b:ae:98:16:42:f3:b2:a6:66:e9:a4:4f:61:04:a8:23:d5: + 55:ea:d4:68:b5:98:fd:66:ff:10:dc:54:b7:01:78:4f:fc:e1: + 75:e8:09:6d:ad:ac:57:b0:33:41:26:3d:ac:b0:17:46:c4:6f: + 5b:c7:fa:ad:d2:94:13:ef:5e:bb:f5:ad:2d:39:85:d3:af:ff: + 56:8e:f6:d1:20:12:03:86:cd:e8:ad:38:49:30:fb:98:de:3a: + 5f:61:5a:08:37:a9:c3:10:ed:a3:60:3c:46:68:30:d8:4a:ac: + 5d:eb:fd:d9:5d:90:b1:f0:b8:a8:68:5e:c8:41:6f:de:eb:a1: + cc:33:98:2d:06:17:26:c4:24:bf:62:82:a9:13:04:71:3e:6e: + ca:20:cf:5c:c5:47:67:f5:db:2e:56:60:4c:52:0c:4e:59:16: + da:6a:e3:b2:e4:cb:d6:65:26:df:26:2e:e0:f4:11:b1:36:92: + 7c:ab:c3:c3:97:a5:06:26:54:5c:c1:35:a1:2f:e5:0f:2f:91: + 2d:cd:c5:dd:a7:f2:4c:e1:4d:0d:5c:bd:25:4f:c8:52:79:c2: + 29:78:ef:88:10:43:a4:c4:df:97:48:22:09:db:48:19:85:01: + 48:39:28:20:69:1d:31:b5:4f:97:e0:ea:38:6d:e0:98:4b:78: + a4:b7:fd:c2 +-----BEGIN CERTIFICATE----- +MIIEXTCCA0WgAwIBAgIUej36W5vfaVVunFNM/IZ1Zbx4TCQwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe +Fw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP +MA0GA1UEAwwGVXNlckEyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +pnxAgCtEADMRxsKdZz6Hjn5A0/XTJ7Z9GDzAhqyWOq3Yw8urcl5MtyRF2seozHS4 +IXVinoGIllRu2/mML0yXDc4hQi+SV380KwJDTCKuFMr8sizQZw5S4G1hlqY7zE9q +1u9FnHSSJWwKEGIbIisRa9FSTdqNw0rmdKcbHu+K9JaIAg23VzWfo/+iLLcOJ055 +L88M8ZEOvwHXonEstw5LflCRiXHCF6rLKYCe1yv6M0HogtE6lz1s3mabtOoa65S+ +bsBm6Hc9ctVcpeirOzP0s8ImSbwIVc8WthIikf7BWrLXd+P0R7zEd2v1f8PoSJm5 +qOqxrubMOhL6TS9fD6j9jQIDAQABo4IBJDCCASAwHQYDVR0OBBYEFLiOT3bx+Dyl +I8WPoS5kPkhTAs1rMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG +A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd +BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs +aHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI +KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4 +LzAaBgNVHREEEzARgQ9Vc2VyQTJAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB +AHwbrpgWQvOypmbppE9hBKgj1VXq1Gi1mP1m/xDcVLcBeE/84XXoCW2trFewM0Em +PaywF0bEb1vH+q3SlBPvXrv1rS05hdOv/1aO9tEgEgOGzeitOEkw+5jeOl9hWgg3 +qcMQ7aNgPEZoMNhKrF3r/dldkLHwuKhoXshBb97rocwzmC0GFybEJL9igqkTBHE+ +bsogz1zFR2f12y5WYExSDE5ZFtpq47Lky9ZlJt8mLuD0EbE2knyrw8OXpQYmVFzB +NaEv5Q8vkS3Nxd2n8kzhTQ1cvSVPyFJ5wil474gQQ6TE35dIIgnbSBmFAUg5KCBp +HTG1T5fg6jht4JhLeKS3/cI= +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:15 2023 GMT + Not After : Apr 28 19:01:15 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: + d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: + 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: + b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: + a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: + 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: + d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: + 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: + 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: + 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: + 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: + 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: + a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: + c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: + 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: + 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: + 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: + 40:27 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: + a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: + d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: + 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: + 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: + c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: + 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: + 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: + 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: + be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: + 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: + 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: + ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: + 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: + 91:5c:43:a9 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC +YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL +3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy +n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu +hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX +Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 ++bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp +LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr +uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN +zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg +zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX +KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/client1/UserA2_cert.pem b/test/configs/certs/ocsp_peer/mini-ca/client1/UserA2_cert.pem new file mode 100644 index 0000000..19b70a4 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/client1/UserA2_cert.pem @@ -0,0 +1,97 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 7a:3d:fa:5b:9b:df:69:55:6e:9c:53:4c:fc:86:75:65:bc:78:4c:24 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Validity + Not Before: May 1 19:37:36 2023 GMT + Not After : Apr 28 19:37:36 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserA2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:a6:7c:40:80:2b:44:00:33:11:c6:c2:9d:67:3e: + 87:8e:7e:40:d3:f5:d3:27:b6:7d:18:3c:c0:86:ac: + 96:3a:ad:d8:c3:cb:ab:72:5e:4c:b7:24:45:da:c7: + a8:cc:74:b8:21:75:62:9e:81:88:96:54:6e:db:f9: + 8c:2f:4c:97:0d:ce:21:42:2f:92:57:7f:34:2b:02: + 43:4c:22:ae:14:ca:fc:b2:2c:d0:67:0e:52:e0:6d: + 61:96:a6:3b:cc:4f:6a:d6:ef:45:9c:74:92:25:6c: + 0a:10:62:1b:22:2b:11:6b:d1:52:4d:da:8d:c3:4a: + e6:74:a7:1b:1e:ef:8a:f4:96:88:02:0d:b7:57:35: + 9f:a3:ff:a2:2c:b7:0e:27:4e:79:2f:cf:0c:f1:91: + 0e:bf:01:d7:a2:71:2c:b7:0e:4b:7e:50:91:89:71: + c2:17:aa:cb:29:80:9e:d7:2b:fa:33:41:e8:82:d1: + 3a:97:3d:6c:de:66:9b:b4:ea:1a:eb:94:be:6e:c0: + 66:e8:77:3d:72:d5:5c:a5:e8:ab:3b:33:f4:b3:c2: + 26:49:bc:08:55:cf:16:b6:12:22:91:fe:c1:5a:b2: + d7:77:e3:f4:47:bc:c4:77:6b:f5:7f:c3:e8:48:99: + b9:a8:ea:b1:ae:e6:cc:3a:12:fa:4d:2f:5f:0f:a8: + fd:8d + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + B8:8E:4F:76:F1:F8:3C:A5:23:C5:8F:A1:2E:64:3E:48:53:02:CD:6B + X509v3 Authority Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, S/MIME + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Client Authentication, E-mail Protection + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:18888/intermediate1_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:18888/ + X509v3 Subject Alternative Name: + email:UserA2@user.net + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 7c:1b:ae:98:16:42:f3:b2:a6:66:e9:a4:4f:61:04:a8:23:d5: + 55:ea:d4:68:b5:98:fd:66:ff:10:dc:54:b7:01:78:4f:fc:e1: + 75:e8:09:6d:ad:ac:57:b0:33:41:26:3d:ac:b0:17:46:c4:6f: + 5b:c7:fa:ad:d2:94:13:ef:5e:bb:f5:ad:2d:39:85:d3:af:ff: + 56:8e:f6:d1:20:12:03:86:cd:e8:ad:38:49:30:fb:98:de:3a: + 5f:61:5a:08:37:a9:c3:10:ed:a3:60:3c:46:68:30:d8:4a:ac: + 5d:eb:fd:d9:5d:90:b1:f0:b8:a8:68:5e:c8:41:6f:de:eb:a1: + cc:33:98:2d:06:17:26:c4:24:bf:62:82:a9:13:04:71:3e:6e: + ca:20:cf:5c:c5:47:67:f5:db:2e:56:60:4c:52:0c:4e:59:16: + da:6a:e3:b2:e4:cb:d6:65:26:df:26:2e:e0:f4:11:b1:36:92: + 7c:ab:c3:c3:97:a5:06:26:54:5c:c1:35:a1:2f:e5:0f:2f:91: + 2d:cd:c5:dd:a7:f2:4c:e1:4d:0d:5c:bd:25:4f:c8:52:79:c2: + 29:78:ef:88:10:43:a4:c4:df:97:48:22:09:db:48:19:85:01: + 48:39:28:20:69:1d:31:b5:4f:97:e0:ea:38:6d:e0:98:4b:78: + a4:b7:fd:c2 +-----BEGIN CERTIFICATE----- +MIIEXTCCA0WgAwIBAgIUej36W5vfaVVunFNM/IZ1Zbx4TCQwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe +Fw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP +MA0GA1UEAwwGVXNlckEyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +pnxAgCtEADMRxsKdZz6Hjn5A0/XTJ7Z9GDzAhqyWOq3Yw8urcl5MtyRF2seozHS4 +IXVinoGIllRu2/mML0yXDc4hQi+SV380KwJDTCKuFMr8sizQZw5S4G1hlqY7zE9q +1u9FnHSSJWwKEGIbIisRa9FSTdqNw0rmdKcbHu+K9JaIAg23VzWfo/+iLLcOJ055 +L88M8ZEOvwHXonEstw5LflCRiXHCF6rLKYCe1yv6M0HogtE6lz1s3mabtOoa65S+ +bsBm6Hc9ctVcpeirOzP0s8ImSbwIVc8WthIikf7BWrLXd+P0R7zEd2v1f8PoSJm5 +qOqxrubMOhL6TS9fD6j9jQIDAQABo4IBJDCCASAwHQYDVR0OBBYEFLiOT3bx+Dyl +I8WPoS5kPkhTAs1rMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG +A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd +BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs +aHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI +KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4 +LzAaBgNVHREEEzARgQ9Vc2VyQTJAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB +AHwbrpgWQvOypmbppE9hBKgj1VXq1Gi1mP1m/xDcVLcBeE/84XXoCW2trFewM0Em +PaywF0bEb1vH+q3SlBPvXrv1rS05hdOv/1aO9tEgEgOGzeitOEkw+5jeOl9hWgg3 +qcMQ7aNgPEZoMNhKrF3r/dldkLHwuKhoXshBb97rocwzmC0GFybEJL9igqkTBHE+ +bsogz1zFR2f12y5WYExSDE5ZFtpq47Lky9ZlJt8mLuD0EbE2knyrw8OXpQYmVFzB +NaEv5Q8vkS3Nxd2n8kzhTQ1cvSVPyFJ5wil474gQQ6TE35dIIgnbSBmFAUg5KCBp +HTG1T5fg6jht4JhLeKS3/cI= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/client1/certfile.pem b/test/configs/certs/ocsp_peer/mini-ca/client1/certfile.pem new file mode 100644 index 0000000..719d751 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/client1/certfile.pem @@ -0,0 +1,175 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:15 2023 GMT + Not After : Apr 28 19:01:15 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: + d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: + 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: + b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: + a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: + 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: + d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: + 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: + 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: + 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: + 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: + 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: + a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: + c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: + 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: + 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: + 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: + 40:27 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: + a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: + d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: + 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: + 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: + c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: + 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: + 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: + 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: + be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: + 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: + 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: + ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: + 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: + 91:5c:43:a9 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC +YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL +3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy +n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu +hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX +Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 ++bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp +LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr +uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN +zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg +zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX +KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 18:57:57 2023 GMT + Not After : Apr 28 18:57:57 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68: + c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b: + 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f: + d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce: + f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b: + 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19: + d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08: + 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c: + 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8: + 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1: + 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05: + 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11: + f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a: + 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01: + fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07: + 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91: + c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65: + 76:d7 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5: + ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60: + d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59: + 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed: + 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a: + f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d: + 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7: + 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28: + 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58: + af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f: + 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95: + 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95: + 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82: + d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8: + dc:29:ac:17 +-----BEGIN CERTIFICATE----- +MIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4 +NTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS +b290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i ++1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi +YM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY +Y0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv +XbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ +7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC +Wz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220 +4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD +AQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3 +LjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5 +Xfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr +UFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I +4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y +VPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb +hpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp +MElNbuoFHtjcKawX +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/client1/private/System_keypair.pem b/test/configs/certs/ocsp_peer/mini-ca/client1/private/System_keypair.pem new file mode 100644 index 0000000..cc779be --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/client1/private/System_keypair.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCjIS90NMEbQZC2 +TkFy4D+eSZRV7AJM3RSAuD3Gx0e7pVnDNYaJFwjO/nHmL5zB29J+FCTaYTA6526x +4yE4gbxH37J/H2C+PcXtdgOU48SzPr/4Q7rCVLy7ZlmYo/mq4xDow4jcGhhv3ZDr +b6NL1K80XEMg1VvnmKV8e6kVhrsov7rgu/ccCMQm68GsBR90TwURV+ASdxeeid2l +OO7Pz2e+DF5qSnRhIXmOwyjx4gYALeo6beKmJf0ti/WCNpGKIfBqkxnWdgj9ze6Q +qanPmTBxRlfq+8VlT3yGXJ3XtMMnPOsn3bxVdh8lDctvQ5qfut5UwZADnuUN2c2E +1Fh0Y75ZAgMBAAECggEAGJh8EGwU0pB56nbVmOW1Sd8jsanGNgMeYIMG83Xf+6uk +Y1GqcXiK4DTOhQuYOcV0UQSmAtQlAriawNDzVRMAiaCxh8e6HSzwrws8YoJOCc2U +AbFqkvrWQvYdW62bive1+LZkp/T6SsGQJGNebmRIr18a0vRAaWSjTOfTOFbqWKwD +640JDw2KmJmba6JtOaEL4QWrvbugTNwh3OEHugBVTCiRTdruVpCpLSxW1yZEpwB2 +BmxQxHvbtIjiOmHuNrsh21jzi7IEx+TFawJ0EV6Wm9XCbjX62XETraILg5bWVGv7 +X+TIDE2JBCC9GZMm9Qj1EfCojRmKfxopv7sA1yBYRQKBgQC5K5NzQzk14G64tyvW +61BteydWlBzFbgiMjYUq9wqgf2WbDVONBUB5x3MOArOmYuJOy0Xbt+pF3pF8wrkl +hMt/hZqKzDtdDWo3+PnNFgcWmB+T76Jei9khIk3D9ENcGaNwGAS3l8sxqXNLVVBJ +u5qHKeKFreXSra7xlXOuw5IMbQKBgQDhh1QqpQMloXOPCQWWzPtrEa8BRGNMFQTU +yZFHeetQjjX5opxxMbbXU/wNz/dRgdfe2VLVo9e4dtQbzCKenuwWeivuDxd4YOsF +Von9XDOzVWoXuP01MxDcU+sRoBLwWbpCWMe7r4Ny98C78+/5GssvkFUo+hd2vPo6 +U20pVZfuHQKBgGYD1eZooLpH/XgSojpzxgmrEc8nJnq21krpJPa4x8gIp+e2fdNx +k0YEViTf5C3EyL10S/Zy6sS3jBvaA7rh4GNPLgdN4V6wp1ZS+vy8KAeQo8US/rds +AKG6jnFovzucfGijMuYa4L1ph7V3ORaGHupcbwoK9lUNjxZVqjgcUvg5AoGBANOU +zpWjcaxgJ7XNVP0BGe59DJ43tqCuJ3YqFK3l56oPgPvOXs6jQVIKbLHYpcJF+mwL +nvbnW36nnJ7niKMfnYYI4CXa6r34zwSXB6Y2Vhqsy3euCX9bhTnvUN2cO6hZxbBw +8hFWvA+j96FdXYlqZa0dz4c9+b1f1bHaitL4hizRAoGAVlH2lJr6s+ZmzkDY7D+Y +6YKyjXaxhHBIqB2oLK1KuxGMiQnRADs9UOC4x2PQPOfemjVTJ3eN3rwxdqSh+Y2v +K+RejHBJzbd4JIv0QRxpPAm9sezaNEHa7ss387cLZEBEYUI9HkIuPunKX+2lHITn +WpVRyzYjVkFUUcRe3DyTlh8= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem b/test/configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem new file mode 100644 index 0000000..1f4a6df --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC06yLixLp/M6pX +qxPxaQmYKDx9p+JBKigv+YWhbJTuCutNAUwofJ0FTdgQf7fPE8Km3hEMlziXzW0R +/RZ2wOtat3sXE0WdSwBPJsWxm2eTLNbVMzfhUB17Db6My70pmY9U9n4EhIIqKO5x +Po1fcrJqd2tHPrpNs+KWFHEKHiYWj2wbByqsFYkeiGPDgTuR6fNDG/DsCCSWRich +KlYlLLbM2QJwd53kfESMkwSFowkKjvXnIfq9Vii3UiAJ7JrE1NeKGU56EOmyEDZo +zs54i3k/b3A7dW1wWTrJhaj4I9SrRMKu9RxuOBHhX8yP4kP1sw4JF7PG7kf7OcRY +YrrjqMXvAgMBAAECggEATFRQOaCKlpQzsB0rotSQCbQgIVutZ5Tjs6nwqTRoeS3+ +LFT5zrMUhGJdYEiiQimyHDjgtJEwfUtcUxSWX6/xHCsBMbEd08kK7loLWm2Ye02V +rgmX7+WfKoWX+UsUGfMBt/TvIfTN/f+a6ghcGQMJJ0YO6tYaQCI+3NbvAjfKFgXi +nWWZA+ipjh+Nu3YhVAy/uMInMi0qGWmpomU1yS+04E3OQksKYc3OmER7zFwbmNbF +0LanWlLURUeHIS1BY+V4yXw6yBJaCDUpVA37mfLqRQshGtGjmWLtMt/AuSFokwHd +yewoORlpVkZnE4Igv1JDggFdEI5lZ4PTmOjEXfntYQKBgQDH6sBr24OMUceNWyvf +k03pqUaoiJkivAcUI/krfY7mdSLkiqs+UPuRrikGbvKT3R+iJVbTB4dXzGG6nzBc +es7xwvzDGNHHXe0KAFhyIXwNMZmLGTmsNVfnKPAQ1BfKG9MtD5ck2gI1L1DkpaRz +X57YONvG05HYmY7TaV2VOKK1iQKBgQDnq/zW6P9WHpIHBjZRN0V4yFl1dMfn2VwZ +c3QWBd+kTwVBBhlJlqYeRIt4kmwExPnd3OX8Y7N18RttIc+k4dZgTA4w8G3xzvgk +0sHgf3EBbrkUuS23BJ2IPIb4LmWckH6+KJkvBrlZOfoLBj8uxQwz/wmWA2IoQgKv +CvDNr6G5twKBgECWSgZOjAhoX1T+0ITRvUkxJB/MydSb9JmAKb7wOJuh2l0Fo99l +IHFnV9+5Nmuo89BZydwxwXsPD7/QMDqgfn1C5pBNU3Damnsxs2FkCgTlMlrrEmPd +dAG9ixmUu/7S0H3tXIJOYIo4OCU2kpOnn9TxQafRsHvO2ILatp5ABukpAoGBALgP +KJ4GF3bwaswx302/P+6qHoj28yv8wPNnir9Eg14jeeUjV0vj6K77fmOY0UEozeu6 +6O4QuC/oEwYtaq9wzcVMJ6oyGueWrAd1eptGJR4iPeF9DhjuDcqDbCgZlJlDI68o +yitWiEOfkEzZ9bDO1NcqtQ7+OSoK597yLkb8Vt0ZAoGAN2dHPkTiNFlbzCefv/EP +A4xQUAUiwfQ9ZlhMtD9Tlea8cMAD901rxy52YrgCvBPxw3HmKG2H0NOpa7BwrgA8 +uODxi6xBRExRhvaqZe1aP1xn4XKw2VVsMlIlJQj2Wmuxeknfm9R1sfRD797c4nuN +ntLUOPAWtDkLoJLrTd9EqFk= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/client1/private/UserA2_keypair.pem b/test/configs/certs/ocsp_peer/mini-ca/client1/private/UserA2_keypair.pem new file mode 100644 index 0000000..8485c7c --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/client1/private/UserA2_keypair.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCmfECAK0QAMxHG +wp1nPoeOfkDT9dMntn0YPMCGrJY6rdjDy6tyXky3JEXax6jMdLghdWKegYiWVG7b ++YwvTJcNziFCL5JXfzQrAkNMIq4UyvyyLNBnDlLgbWGWpjvMT2rW70WcdJIlbAoQ +YhsiKxFr0VJN2o3DSuZ0pxse74r0logCDbdXNZ+j/6Istw4nTnkvzwzxkQ6/Adei +cSy3Dkt+UJGJccIXqsspgJ7XK/ozQeiC0TqXPWzeZpu06hrrlL5uwGbodz1y1Vyl +6Ks7M/SzwiZJvAhVzxa2EiKR/sFastd34/RHvMR3a/V/w+hImbmo6rGu5sw6EvpN +L18PqP2NAgMBAAECggEAHsf4UPou326RydLvsUgRXhofuFDKEpyd8l5BJmVAfWbp +HgJJF6Mxwea196ZUokCuTplae33tmAXSXV99OL2LbCUBZzBOeVjud0k60hfTYcrJ +/9NjULqIPjBbC7R+d97zHPwuPagb4UlhbvgElkOqO+n+sqBG96WgiE7hJ84YPfJO +Y+Is9vRbESkMVK1TH5PxfDE0Yu/i2vm/Fv+Ekgqe+GiZgbDw+L98D7ZX1xb8W2Ix +WnM2Skd22pit1ftpixuHbdHcPX2NNdhFKy8r/ZYK0SCLwkb8yJhDLQF8Q01YXd6q +FHtuE+MGXsr7dkcqYtc2QvigJdHs72WCjZwcpA+vgQKBgQC9qt3AIXPjPckhTEEK +97tg0zqFVPHyhiy23qsKJ/egMIhYESQngLOPcQ0Q/bG5OJqe5sx31rmKQ368QUSX +lIPG9WrRxCh3BTo7nOOEmAh4uGnKtvDJbTRP56fPQhkKlDywua8vKs0moUdcact7 +jjXYxXSPGEqHQrjPkuurPJvc7QKBgQDgtd4/kYGM9R2ltSLWm/TZnE6LM1EtBWrA +HNAYV7WxxKdUvTtxBIXDKer0RAbDKHIoZ6HI3lon5siuBVtIFoq2VLxS1jm3rEJv +qV6USxxDnEkbLla6Jzmd5eqFPZErWfNqmdXP1sqC8fs5q1PUJXNtEIdJulXQnHP2 +5lJxq8ovIQKBgQC1HBerg0YZ08HfHeVuB6jRiGH1N2vhXeYMqQtCI2/9ctp+3b9c +STUs35LOirHOYBKlcVYFiPCa6mB2ewx4gcRjk61wqJLLNB6rFeDbmCFexRmgDJhY +fwLY2igPbNpkk7BwQJ7bt082eAKgaBV54g3g9IucqGFiT4ASFgUb+kAK8QKBgFYJ +rJgAWW8kJv7clQNA4YY0j+pCctFfIpl+LrszUhFHr54Fem3ygljQgvKV3VT59oO7 +8jkb0b83YR0oVeQLJX9cgGLjPWQzI5jna5wyChdlDqTGoFRUUn4/mwT7JstHfKkT +T8dtgUqT5lIVZFp1IHXg/zveiZ7/WHNvip+VXCuhAoGAE3aA/rPBYEHJFLHaSgcR +E+ggLP3HjQPN6347NXBZSaWSXqXtdLDlgnnysRnEd+JHSDI3lNuFa6d6nY7x7Mc0 +Bn54Tf3KLLHcyUrwQTCjY230212gYGqWXMgeaTPJRtl4K0PchWzKzZ1m9RQAZHOQ +OaBsh0IA+LCDTmsPsbzh6U4= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/client2/UserB1_bundle.pem b/test/configs/certs/ocsp_peer/mini-ca/client2/UserB1_bundle.pem new file mode 100644 index 0000000..c09aef2 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/client2/UserB1_bundle.pem @@ -0,0 +1,186 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3a:d5:76:e0:a4:4b:67:ba:da:f2:9b:15:09:4c:ff:54:58:1d:e9:92 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Validity + Not Before: May 1 19:40:31 2023 GMT + Not After : Apr 28 19:40:31 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserB1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:ba:19:65:ab:3f:2e:f2:7a:93:ea:06:eb:a2:9a: + c5:b9:20:66:2e:74:1b:94:5a:43:1c:8c:22:72:00: + 79:2d:20:18:e3:4a:35:a6:df:8a:58:33:73:2c:28: + 20:e7:d9:85:ec:f5:81:ae:44:44:55:66:65:d6:b5: + 78:71:c4:d8:c2:7b:4c:2d:8b:18:b6:86:fc:50:c0: + 7e:b6:6e:f7:76:c0:30:6c:67:09:53:2d:87:98:d6: + d4:d8:b3:a9:80:45:93:7f:33:3f:41:2a:70:f3:e1: + df:a0:85:64:4b:25:e4:91:e9:e6:c8:c3:a0:3e:b3: + ef:97:1f:ae:9d:44:84:35:26:26:4e:0c:7a:1d:c7: + ef:b6:46:8d:82:b8:b0:18:fb:25:77:04:20:8c:da: + af:fa:9e:a2:b0:67:b6:a6:5b:d7:95:a5:3c:3e:76: + b4:37:4a:48:98:34:96:9d:d2:ff:36:6a:f4:2a:cd: + 85:b3:e3:71:74:0f:e0:25:f1:06:cb:9d:53:fc:b4: + 5d:c4:8d:7a:0b:bd:16:ee:5c:58:21:ad:49:34:9f: + 9e:1b:6d:f6:47:52:1f:a0:74:00:fe:3c:4d:5f:4c: + 5a:23:4a:d5:4c:ff:3f:42:5d:85:df:f6:3b:32:c4: + ca:4b:d0:9d:4b:9e:86:a6:64:44:b8:ae:24:1a:f4: + 66:6b + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + EC:AB:7B:4D:CD:62:D6:89:63:69:FE:97:34:5A:96:58:A5:94:A6:D9 + X509v3 Authority Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, S/MIME + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Client Authentication, E-mail Protection + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:28888/intermediate2_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:28888/ + X509v3 Subject Alternative Name: + email:UserB1@user.net + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + a8:78:fa:c2:44:e0:b9:c7:af:d5:cc:b6:b4:2b:3d:74:ae:b8: + d1:e1:22:d0:63:7d:77:97:db:97:2f:f1:f0:ce:e3:9e:5e:e1: + 2a:19:54:00:38:7b:30:0b:8b:95:3a:4b:5d:83:08:80:fe:29: + 85:72:fd:c9:80:6b:c3:fd:a3:00:4f:b5:f2:34:a3:42:54:77: + 77:70:43:40:fe:1f:7a:b7:7f:55:c3:c0:e2:44:d1:95:fb:4c: + eb:f8:39:dd:b6:3d:07:27:39:8e:89:e4:a8:49:fd:02:70:65: + 72:6f:c7:d4:12:57:bd:47:ea:7d:2d:63:b4:fe:81:33:20:3c: + e0:36:a2:60:58:79:5e:ce:6c:ed:7c:97:6e:6b:52:25:8d:73: + bb:ea:b5:8b:1e:d2:97:24:88:59:ea:a4:29:a3:ea:04:45:e1: + 6a:cd:c8:b9:13:44:57:f8:7e:1a:85:34:11:71:f9:10:a4:6f: + 07:d4:7d:21:84:f1:52:6f:f9:e8:36:83:28:32:aa:ad:2a:c3: + fb:98:02:c7:2e:2c:49:08:21:af:fe:15:0e:f3:ce:e7:24:b5: + c8:08:d6:20:e8:8c:24:ce:1f:84:0b:9a:46:07:8c:05:d0:86: + 04:06:2b:a2:a8:e2:20:c1:1f:ac:07:fc:ac:e0:f5:ee:7a:c6: + 5a:e4:81:74 +-----BEGIN CERTIFICATE----- +MIIEXTCCA0WgAwIBAgIUOtV24KRLZ7ra8psVCUz/VFgd6ZIwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe +Fw0yMzA1MDExOTQwMzFaFw0zMzA0MjgxOTQwMzFaME8xCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP +MA0GA1UEAwwGVXNlckIxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +uhllqz8u8nqT6gbroprFuSBmLnQblFpDHIwicgB5LSAY40o1pt+KWDNzLCgg59mF +7PWBrkREVWZl1rV4ccTYwntMLYsYtob8UMB+tm73dsAwbGcJUy2HmNbU2LOpgEWT +fzM/QSpw8+HfoIVkSyXkkenmyMOgPrPvlx+unUSENSYmTgx6HcfvtkaNgriwGPsl +dwQgjNqv+p6isGe2plvXlaU8Pna0N0pImDSWndL/Nmr0Ks2Fs+NxdA/gJfEGy51T +/LRdxI16C70W7lxYIa1JNJ+eG232R1IfoHQA/jxNX0xaI0rVTP8/Ql2F3/Y7MsTK +S9CdS56GpmREuK4kGvRmawIDAQABo4IBJDCCASAwHQYDVR0OBBYEFOyre03NYtaJ +Y2n+lzRallillKbZMB8GA1UdIwQYMBaAFHVV4o7nraXdgD3JMwssold37RWsMAwG +A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd +BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs +aHR0cDovLzEyNy4wLjAuMToyODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYI +KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4 +LzAaBgNVHREEEzARgQ9Vc2VyQjFAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB +AKh4+sJE4LnHr9XMtrQrPXSuuNHhItBjfXeX25cv8fDO455e4SoZVAA4ezALi5U6 +S12DCID+KYVy/cmAa8P9owBPtfI0o0JUd3dwQ0D+H3q3f1XDwOJE0ZX7TOv4Od22 +PQcnOY6J5KhJ/QJwZXJvx9QSV71H6n0tY7T+gTMgPOA2omBYeV7ObO18l25rUiWN +c7vqtYse0pckiFnqpCmj6gRF4WrNyLkTRFf4fhqFNBFx+RCkbwfUfSGE8VJv+eg2 +gygyqq0qw/uYAscuLEkIIa/+FQ7zzucktcgI1iDojCTOH4QLmkYHjAXQhgQGK6Ko +4iDBH6wH/Kzg9e56xlrkgXQ= +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:43 2023 GMT + Not After : Apr 28 19:01:43 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: + 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: + ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: + f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: + ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: + fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: + 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: + c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: + 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: + 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: + d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: + af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: + 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: + 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: + 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: + 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: + d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: + 08:41 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: + dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: + 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: + 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: + 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: + 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: + 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: + ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: + 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: + e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: + 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: + 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: + 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: + 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: + f1:5e:60:55 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ +97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh +bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn +qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW +wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 +1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A +PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 +ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG +m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 +J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc +jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz +EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem b/test/configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem new file mode 100644 index 0000000..5105011 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem @@ -0,0 +1,97 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3a:d5:76:e0:a4:4b:67:ba:da:f2:9b:15:09:4c:ff:54:58:1d:e9:92 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Validity + Not Before: May 1 19:40:31 2023 GMT + Not After : Apr 28 19:40:31 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserB1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:ba:19:65:ab:3f:2e:f2:7a:93:ea:06:eb:a2:9a: + c5:b9:20:66:2e:74:1b:94:5a:43:1c:8c:22:72:00: + 79:2d:20:18:e3:4a:35:a6:df:8a:58:33:73:2c:28: + 20:e7:d9:85:ec:f5:81:ae:44:44:55:66:65:d6:b5: + 78:71:c4:d8:c2:7b:4c:2d:8b:18:b6:86:fc:50:c0: + 7e:b6:6e:f7:76:c0:30:6c:67:09:53:2d:87:98:d6: + d4:d8:b3:a9:80:45:93:7f:33:3f:41:2a:70:f3:e1: + df:a0:85:64:4b:25:e4:91:e9:e6:c8:c3:a0:3e:b3: + ef:97:1f:ae:9d:44:84:35:26:26:4e:0c:7a:1d:c7: + ef:b6:46:8d:82:b8:b0:18:fb:25:77:04:20:8c:da: + af:fa:9e:a2:b0:67:b6:a6:5b:d7:95:a5:3c:3e:76: + b4:37:4a:48:98:34:96:9d:d2:ff:36:6a:f4:2a:cd: + 85:b3:e3:71:74:0f:e0:25:f1:06:cb:9d:53:fc:b4: + 5d:c4:8d:7a:0b:bd:16:ee:5c:58:21:ad:49:34:9f: + 9e:1b:6d:f6:47:52:1f:a0:74:00:fe:3c:4d:5f:4c: + 5a:23:4a:d5:4c:ff:3f:42:5d:85:df:f6:3b:32:c4: + ca:4b:d0:9d:4b:9e:86:a6:64:44:b8:ae:24:1a:f4: + 66:6b + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + EC:AB:7B:4D:CD:62:D6:89:63:69:FE:97:34:5A:96:58:A5:94:A6:D9 + X509v3 Authority Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, S/MIME + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Client Authentication, E-mail Protection + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:28888/intermediate2_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:28888/ + X509v3 Subject Alternative Name: + email:UserB1@user.net + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + a8:78:fa:c2:44:e0:b9:c7:af:d5:cc:b6:b4:2b:3d:74:ae:b8: + d1:e1:22:d0:63:7d:77:97:db:97:2f:f1:f0:ce:e3:9e:5e:e1: + 2a:19:54:00:38:7b:30:0b:8b:95:3a:4b:5d:83:08:80:fe:29: + 85:72:fd:c9:80:6b:c3:fd:a3:00:4f:b5:f2:34:a3:42:54:77: + 77:70:43:40:fe:1f:7a:b7:7f:55:c3:c0:e2:44:d1:95:fb:4c: + eb:f8:39:dd:b6:3d:07:27:39:8e:89:e4:a8:49:fd:02:70:65: + 72:6f:c7:d4:12:57:bd:47:ea:7d:2d:63:b4:fe:81:33:20:3c: + e0:36:a2:60:58:79:5e:ce:6c:ed:7c:97:6e:6b:52:25:8d:73: + bb:ea:b5:8b:1e:d2:97:24:88:59:ea:a4:29:a3:ea:04:45:e1: + 6a:cd:c8:b9:13:44:57:f8:7e:1a:85:34:11:71:f9:10:a4:6f: + 07:d4:7d:21:84:f1:52:6f:f9:e8:36:83:28:32:aa:ad:2a:c3: + fb:98:02:c7:2e:2c:49:08:21:af:fe:15:0e:f3:ce:e7:24:b5: + c8:08:d6:20:e8:8c:24:ce:1f:84:0b:9a:46:07:8c:05:d0:86: + 04:06:2b:a2:a8:e2:20:c1:1f:ac:07:fc:ac:e0:f5:ee:7a:c6: + 5a:e4:81:74 +-----BEGIN CERTIFICATE----- +MIIEXTCCA0WgAwIBAgIUOtV24KRLZ7ra8psVCUz/VFgd6ZIwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe +Fw0yMzA1MDExOTQwMzFaFw0zMzA0MjgxOTQwMzFaME8xCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP +MA0GA1UEAwwGVXNlckIxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +uhllqz8u8nqT6gbroprFuSBmLnQblFpDHIwicgB5LSAY40o1pt+KWDNzLCgg59mF +7PWBrkREVWZl1rV4ccTYwntMLYsYtob8UMB+tm73dsAwbGcJUy2HmNbU2LOpgEWT +fzM/QSpw8+HfoIVkSyXkkenmyMOgPrPvlx+unUSENSYmTgx6HcfvtkaNgriwGPsl +dwQgjNqv+p6isGe2plvXlaU8Pna0N0pImDSWndL/Nmr0Ks2Fs+NxdA/gJfEGy51T +/LRdxI16C70W7lxYIa1JNJ+eG232R1IfoHQA/jxNX0xaI0rVTP8/Ql2F3/Y7MsTK +S9CdS56GpmREuK4kGvRmawIDAQABo4IBJDCCASAwHQYDVR0OBBYEFOyre03NYtaJ +Y2n+lzRallillKbZMB8GA1UdIwQYMBaAFHVV4o7nraXdgD3JMwssold37RWsMAwG +A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd +BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs +aHR0cDovLzEyNy4wLjAuMToyODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYI +KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4 +LzAaBgNVHREEEzARgQ9Vc2VyQjFAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB +AKh4+sJE4LnHr9XMtrQrPXSuuNHhItBjfXeX25cv8fDO455e4SoZVAA4ezALi5U6 +S12DCID+KYVy/cmAa8P9owBPtfI0o0JUd3dwQ0D+H3q3f1XDwOJE0ZX7TOv4Od22 +PQcnOY6J5KhJ/QJwZXJvx9QSV71H6n0tY7T+gTMgPOA2omBYeV7ObO18l25rUiWN +c7vqtYse0pckiFnqpCmj6gRF4WrNyLkTRFf4fhqFNBFx+RCkbwfUfSGE8VJv+eg2 +gygyqq0qw/uYAscuLEkIIa/+FQ7zzucktcgI1iDojCTOH4QLmkYHjAXQhgQGK6Ko +4iDBH6wH/Kzg9e56xlrkgXQ= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/client2/UserB2_bundle.pem b/test/configs/certs/ocsp_peer/mini-ca/client2/UserB2_bundle.pem new file mode 100644 index 0000000..b28ac13 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/client2/UserB2_bundle.pem @@ -0,0 +1,186 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 1e:dc:a2:b9:fd:aa:6e:73:ae:1c:7d:8d:13:73:d1:cd:16:bb:40:90 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Validity + Not Before: May 1 19:40:31 2023 GMT + Not After : Apr 28 19:40:31 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserB2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b2:1d:92:83:be:0f:40:5c:b8:34:93:66:28:ea: + d3:85:1e:ec:66:e3:97:d0:fe:a7:2d:2c:89:c4:aa: + e0:ff:62:a2:8b:19:19:8a:1f:bb:a9:24:2f:a8:a1: + 16:95:a7:5b:42:65:2f:03:27:12:ac:44:fb:2f:e0: + 9b:19:52:32:a7:db:83:d0:1a:d6:36:d7:b7:40:0e: + 85:c6:a7:75:5c:d1:71:a9:99:d3:da:2b:70:f9:9e: + 9d:0b:a8:35:bc:3c:7f:24:1e:b5:2e:83:31:07:c9: + 9b:4a:0e:a3:32:36:bd:a6:2c:55:79:f8:71:66:6a: + 2a:8f:f9:f9:67:b0:06:21:e4:2a:02:44:b6:39:84: + 18:7a:00:5e:34:36:f4:61:0d:11:a9:e2:0c:b8:05: + ed:67:97:bc:29:e7:69:ac:48:6e:fb:78:e9:3b:38: + e3:db:09:cb:22:0f:9a:57:1c:cc:06:f1:f7:44:66: + d0:01:c4:c1:14:65:29:e5:cf:19:26:73:c9:8a:5c: + 2b:25:a9:d1:c6:3e:d8:4d:f5:f3:67:c7:23:b9:7b: + 2b:f5:97:28:89:81:99:9d:82:45:21:27:f4:ca:86: + 02:22:2f:26:4b:61:8a:cb:76:fb:b1:7b:4c:42:b6: + 25:e8:3e:cb:ab:2c:60:a7:a3:82:fb:ef:05:59:03: + a5:5b + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + C6:25:DB:6C:4E:18:89:96:67:30:E8:5F:EC:0C:03:70:A4:4C:07:98 + X509v3 Authority Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, S/MIME + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Client Authentication, E-mail Protection + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:28888/intermediate2_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:28888/ + X509v3 Subject Alternative Name: + email:UserB2@user.net + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 7d:93:8d:17:4b:fe:9e:5d:d0:4e:c3:47:dc:6c:05:1b:10:7f: + 9d:24:75:ea:30:27:c3:b1:26:2c:38:c3:c9:18:ec:21:d2:ef: + 07:b2:d4:f9:2e:a1:a2:1a:a5:68:cb:1a:14:55:7f:82:05:8a: + a3:0d:11:f0:ed:f2:e2:c0:e3:6a:1c:76:42:01:92:68:2b:f7: + 4d:98:ae:7b:02:f1:36:2e:44:67:43:39:8e:08:91:f1:f0:ab: + 9c:84:df:08:80:bf:76:6b:37:3f:e8:70:e0:d6:27:73:e9:bc: + 49:1f:c2:4a:15:51:22:c6:f3:85:52:e3:a6:93:aa:f6:c9:b4: + 96:f2:09:e6:62:53:0e:87:76:fd:7a:38:69:e2:41:54:c5:51: + 6e:cf:bc:1a:7b:0a:ef:c6:6e:be:b5:72:4d:f4:6f:fd:a5:a8: + ba:23:15:80:fa:b6:37:8d:68:d8:3e:36:c5:ae:f6:6c:22:a0: + 00:0d:93:e1:ae:41:9a:d7:35:d0:ab:98:71:1b:6b:8d:da:78: + 65:3c:97:be:9c:9e:d7:32:a1:0c:2b:60:ac:74:18:18:e4:48: + 87:40:dd:bf:eb:0e:27:17:96:a1:aa:32:a9:58:b5:ee:fc:42: + 7e:d7:71:a4:8e:a0:5b:06:6f:f1:85:27:8c:6b:20:df:e0:6b: + 13:5f:cf:4c +-----BEGIN CERTIFICATE----- +MIIEXTCCA0WgAwIBAgIUHtyiuf2qbnOuHH2NE3PRzRa7QJAwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe +Fw0yMzA1MDExOTQwMzFaFw0zMzA0MjgxOTQwMzFaME8xCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP +MA0GA1UEAwwGVXNlckIyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +sh2Sg74PQFy4NJNmKOrThR7sZuOX0P6nLSyJxKrg/2KiixkZih+7qSQvqKEWladb +QmUvAycSrET7L+CbGVIyp9uD0BrWNte3QA6Fxqd1XNFxqZnT2itw+Z6dC6g1vDx/ +JB61LoMxB8mbSg6jMja9pixVefhxZmoqj/n5Z7AGIeQqAkS2OYQYegBeNDb0YQ0R +qeIMuAXtZ5e8KedprEhu+3jpOzjj2wnLIg+aVxzMBvH3RGbQAcTBFGUp5c8ZJnPJ +ilwrJanRxj7YTfXzZ8cjuXsr9ZcoiYGZnYJFISf0yoYCIi8mS2GKy3b7sXtMQrYl +6D7Lqyxgp6OC++8FWQOlWwIDAQABo4IBJDCCASAwHQYDVR0OBBYEFMYl22xOGImW +ZzDoX+wMA3CkTAeYMB8GA1UdIwQYMBaAFHVV4o7nraXdgD3JMwssold37RWsMAwG +A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd +BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs +aHR0cDovLzEyNy4wLjAuMToyODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYI +KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4 +LzAaBgNVHREEEzARgQ9Vc2VyQjJAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB +AH2TjRdL/p5d0E7DR9xsBRsQf50kdeowJ8OxJiw4w8kY7CHS7wey1PkuoaIapWjL +GhRVf4IFiqMNEfDt8uLA42ocdkIBkmgr902YrnsC8TYuRGdDOY4IkfHwq5yE3wiA +v3ZrNz/ocODWJ3PpvEkfwkoVUSLG84VS46aTqvbJtJbyCeZiUw6Hdv16OGniQVTF +UW7PvBp7Cu/Gbr61ck30b/2lqLojFYD6tjeNaNg+NsWu9mwioAANk+GuQZrXNdCr +mHEba43aeGU8l76cntcyoQwrYKx0GBjkSIdA3b/rDicXlqGqMqlYte78Qn7XcaSO +oFsGb/GFJ4xrIN/gaxNfz0w= +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:43 2023 GMT + Not After : Apr 28 19:01:43 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: + 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: + ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: + f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: + ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: + fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: + 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: + c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: + 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: + 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: + d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: + af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: + 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: + 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: + 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: + 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: + d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: + 08:41 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: + dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: + 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: + 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: + 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: + 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: + 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: + ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: + 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: + e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: + 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: + 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: + 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: + 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: + f1:5e:60:55 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ +97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh +bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn +qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW +wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 +1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A +PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 +ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG +m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 +J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc +jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz +EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/client2/UserB2_cert.pem b/test/configs/certs/ocsp_peer/mini-ca/client2/UserB2_cert.pem new file mode 100644 index 0000000..a76cdf8 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/client2/UserB2_cert.pem @@ -0,0 +1,97 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 1e:dc:a2:b9:fd:aa:6e:73:ae:1c:7d:8d:13:73:d1:cd:16:bb:40:90 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Validity + Not Before: May 1 19:40:31 2023 GMT + Not After : Apr 28 19:40:31 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserB2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b2:1d:92:83:be:0f:40:5c:b8:34:93:66:28:ea: + d3:85:1e:ec:66:e3:97:d0:fe:a7:2d:2c:89:c4:aa: + e0:ff:62:a2:8b:19:19:8a:1f:bb:a9:24:2f:a8:a1: + 16:95:a7:5b:42:65:2f:03:27:12:ac:44:fb:2f:e0: + 9b:19:52:32:a7:db:83:d0:1a:d6:36:d7:b7:40:0e: + 85:c6:a7:75:5c:d1:71:a9:99:d3:da:2b:70:f9:9e: + 9d:0b:a8:35:bc:3c:7f:24:1e:b5:2e:83:31:07:c9: + 9b:4a:0e:a3:32:36:bd:a6:2c:55:79:f8:71:66:6a: + 2a:8f:f9:f9:67:b0:06:21:e4:2a:02:44:b6:39:84: + 18:7a:00:5e:34:36:f4:61:0d:11:a9:e2:0c:b8:05: + ed:67:97:bc:29:e7:69:ac:48:6e:fb:78:e9:3b:38: + e3:db:09:cb:22:0f:9a:57:1c:cc:06:f1:f7:44:66: + d0:01:c4:c1:14:65:29:e5:cf:19:26:73:c9:8a:5c: + 2b:25:a9:d1:c6:3e:d8:4d:f5:f3:67:c7:23:b9:7b: + 2b:f5:97:28:89:81:99:9d:82:45:21:27:f4:ca:86: + 02:22:2f:26:4b:61:8a:cb:76:fb:b1:7b:4c:42:b6: + 25:e8:3e:cb:ab:2c:60:a7:a3:82:fb:ef:05:59:03: + a5:5b + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + C6:25:DB:6C:4E:18:89:96:67:30:E8:5F:EC:0C:03:70:A4:4C:07:98 + X509v3 Authority Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, S/MIME + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Client Authentication, E-mail Protection + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:28888/intermediate2_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:28888/ + X509v3 Subject Alternative Name: + email:UserB2@user.net + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 7d:93:8d:17:4b:fe:9e:5d:d0:4e:c3:47:dc:6c:05:1b:10:7f: + 9d:24:75:ea:30:27:c3:b1:26:2c:38:c3:c9:18:ec:21:d2:ef: + 07:b2:d4:f9:2e:a1:a2:1a:a5:68:cb:1a:14:55:7f:82:05:8a: + a3:0d:11:f0:ed:f2:e2:c0:e3:6a:1c:76:42:01:92:68:2b:f7: + 4d:98:ae:7b:02:f1:36:2e:44:67:43:39:8e:08:91:f1:f0:ab: + 9c:84:df:08:80:bf:76:6b:37:3f:e8:70:e0:d6:27:73:e9:bc: + 49:1f:c2:4a:15:51:22:c6:f3:85:52:e3:a6:93:aa:f6:c9:b4: + 96:f2:09:e6:62:53:0e:87:76:fd:7a:38:69:e2:41:54:c5:51: + 6e:cf:bc:1a:7b:0a:ef:c6:6e:be:b5:72:4d:f4:6f:fd:a5:a8: + ba:23:15:80:fa:b6:37:8d:68:d8:3e:36:c5:ae:f6:6c:22:a0: + 00:0d:93:e1:ae:41:9a:d7:35:d0:ab:98:71:1b:6b:8d:da:78: + 65:3c:97:be:9c:9e:d7:32:a1:0c:2b:60:ac:74:18:18:e4:48: + 87:40:dd:bf:eb:0e:27:17:96:a1:aa:32:a9:58:b5:ee:fc:42: + 7e:d7:71:a4:8e:a0:5b:06:6f:f1:85:27:8c:6b:20:df:e0:6b: + 13:5f:cf:4c +-----BEGIN CERTIFICATE----- +MIIEXTCCA0WgAwIBAgIUHtyiuf2qbnOuHH2NE3PRzRa7QJAwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe +Fw0yMzA1MDExOTQwMzFaFw0zMzA0MjgxOTQwMzFaME8xCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP +MA0GA1UEAwwGVXNlckIyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +sh2Sg74PQFy4NJNmKOrThR7sZuOX0P6nLSyJxKrg/2KiixkZih+7qSQvqKEWladb +QmUvAycSrET7L+CbGVIyp9uD0BrWNte3QA6Fxqd1XNFxqZnT2itw+Z6dC6g1vDx/ +JB61LoMxB8mbSg6jMja9pixVefhxZmoqj/n5Z7AGIeQqAkS2OYQYegBeNDb0YQ0R +qeIMuAXtZ5e8KedprEhu+3jpOzjj2wnLIg+aVxzMBvH3RGbQAcTBFGUp5c8ZJnPJ +ilwrJanRxj7YTfXzZ8cjuXsr9ZcoiYGZnYJFISf0yoYCIi8mS2GKy3b7sXtMQrYl +6D7Lqyxgp6OC++8FWQOlWwIDAQABo4IBJDCCASAwHQYDVR0OBBYEFMYl22xOGImW +ZzDoX+wMA3CkTAeYMB8GA1UdIwQYMBaAFHVV4o7nraXdgD3JMwssold37RWsMAwG +A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd +BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs +aHR0cDovLzEyNy4wLjAuMToyODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYI +KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4 +LzAaBgNVHREEEzARgQ9Vc2VyQjJAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB +AH2TjRdL/p5d0E7DR9xsBRsQf50kdeowJ8OxJiw4w8kY7CHS7wey1PkuoaIapWjL +GhRVf4IFiqMNEfDt8uLA42ocdkIBkmgr902YrnsC8TYuRGdDOY4IkfHwq5yE3wiA +v3ZrNz/ocODWJ3PpvEkfwkoVUSLG84VS46aTqvbJtJbyCeZiUw6Hdv16OGniQVTF +UW7PvBp7Cu/Gbr61ck30b/2lqLojFYD6tjeNaNg+NsWu9mwioAANk+GuQZrXNdCr +mHEba43aeGU8l76cntcyoQwrYKx0GBjkSIdA3b/rDicXlqGqMqlYte78Qn7XcaSO +oFsGb/GFJ4xrIN/gaxNfz0w= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/client2/certfile.pem b/test/configs/certs/ocsp_peer/mini-ca/client2/certfile.pem new file mode 100644 index 0000000..a25efa0 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/client2/certfile.pem @@ -0,0 +1,175 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:43 2023 GMT + Not After : Apr 28 19:01:43 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: + 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: + ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: + f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: + ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: + fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: + 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: + c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: + 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: + 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: + d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: + af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: + 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: + 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: + 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: + 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: + d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: + 08:41 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: + dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: + 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: + 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: + 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: + 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: + 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: + ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: + 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: + e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: + 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: + 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: + 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: + 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: + f1:5e:60:55 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ +97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh +bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn +qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW +wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 +1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A +PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 +ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG +m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 +J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc +jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz +EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 18:57:57 2023 GMT + Not After : Apr 28 18:57:57 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68: + c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b: + 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f: + d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce: + f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b: + 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19: + d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08: + 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c: + 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8: + 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1: + 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05: + 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11: + f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a: + 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01: + fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07: + 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91: + c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65: + 76:d7 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5: + ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60: + d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59: + 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed: + 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a: + f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d: + 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7: + 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28: + 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58: + af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f: + 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95: + 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95: + 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82: + d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8: + dc:29:ac:17 +-----BEGIN CERTIFICATE----- +MIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4 +NTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS +b290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i ++1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi +YM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY +Y0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv +XbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ +7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC +Wz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220 +4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD +AQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3 +LjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5 +Xfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr +UFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I +4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y +VPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb +hpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp +MElNbuoFHtjcKawX +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/client2/private/UserB1_keypair.pem b/test/configs/certs/ocsp_peer/mini-ca/client2/private/UserB1_keypair.pem new file mode 100644 index 0000000..1b2df18 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/client2/private/UserB1_keypair.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC6GWWrPy7yepPq +BuuimsW5IGYudBuUWkMcjCJyAHktIBjjSjWm34pYM3MsKCDn2YXs9YGuRERVZmXW +tXhxxNjCe0wtixi2hvxQwH62bvd2wDBsZwlTLYeY1tTYs6mARZN/Mz9BKnDz4d+g +hWRLJeSR6ebIw6A+s++XH66dRIQ1JiZODHodx++2Ro2CuLAY+yV3BCCM2q/6nqKw +Z7amW9eVpTw+drQ3SkiYNJad0v82avQqzYWz43F0D+Al8QbLnVP8tF3EjXoLvRbu +XFghrUk0n54bbfZHUh+gdAD+PE1fTFojStVM/z9CXYXf9jsyxMpL0J1LnoamZES4 +riQa9GZrAgMBAAECggEAAVnSLX+MajFnQj3our9FMrZSGx4bbAhQz9dndUWc1HT4 +d4AgPFpAfqpof6vVycHx2jSnILhuseJykSGzwoHgynrVpI82T6f9EzhRmkLbK1Y5 +6t6jC9uwXDvv37RgYcW02o1avD8VdHtN+qXtO4Db22P1p7zeA6LzSscmmLjf4QcY +15O5DFUsVD6jfjI+edTKY4OgqblwD/t5EqApBI/KhAypSRD/NDzKdtHZO+K3eJW0 +apznw5wrzPVX1xk4p+1LnM5nLBRnwECqRyzlmxjX3rJr7tVVWqOkTHs807wK+7AW +o9rujmS/J8I86BtZdj938VGVyuyqhJndANF8rOh6nQKBgQD09ZFmj/SMIeJIa2Xj +MiK1JMU1rcr2h8NxYhQqZV/sj8TD+Sm/ljCDDClqyo5wAvBdIkFO689sIDEFT1W1 +vUOnE8xa4kkoSf4TVADiGAt4aLHiPiRAoX0aPqgBSy9IcXg7p/iG5qFLp72CNEFg +3vM5vgjX+xio42Hqdo6+ruE1pwKBgQDCfK4KpR2BAv6dbuGNF8qZHWkgBDpSSlug +WMEZe6c9l44EAIHgJNr4nBviVZTZAHD+H5qSC8STQ6Y4ccOZYnG4dGxAztKYnX9Z +T6R+zOkisK+Zhq9noj8veBwS6F2fGTL7cagBkj2q3SveagGtutkV6kOKUw5uu8dI +GnSxaiNpnQKBgQDrzURlVWgUST3ZdsECvq1YcIgCj0TUooYKLF67HREE2LSR7dU5 +XytdyyRHb6tDuiCFlscFYMwwCqEFuoQISaPJPq62QiQoS2nwUynyezD3fNjXr/gX +2xxhWjVB4Y0nkEssKhp8SaC1AkjUANd6l8PNLti2iDkJwrDsEaqBdjjG+wKBgAVM +Eg12K9SMuVSeZYRLRphfBbL6ioAdSFuYr0G7bXWvAA452U+6kUA+OEA05oX2jh1N +zQ73RRZhvFBDQPmXhdNpUF1/hJrlh0dudOODP0JTn6TF11cyQxhO5CzbqVkg/ZN9 +p/7K9eUGeyBmsL8DnNAM/mPxGS6I7MeY+N6wLmC9AoGBAPL97OOwtkfCqCXBzIua +eNFIPvW8cKEM1ggxUPGar36TuaKnDt8bdGL3/ZEAD28XMGCbUwo99WrW0J4r9b95 +Rrs1FzUW9iVIqB+4W35lMfSbFOC/2GsSUf95ANT4wihu2QbVQU7iqjXw+w8ZN9Vx +Qkiwv6M/K0lzm6Q1H1pb7urx +-----END PRIVATE KEY----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/client2/private/UserB2_keypair.pem b/test/configs/certs/ocsp_peer/mini-ca/client2/private/UserB2_keypair.pem new file mode 100644 index 0000000..587c454 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/client2/private/UserB2_keypair.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCyHZKDvg9AXLg0 +k2Yo6tOFHuxm45fQ/qctLInEquD/YqKLGRmKH7upJC+ooRaVp1tCZS8DJxKsRPsv +4JsZUjKn24PQGtY217dADoXGp3Vc0XGpmdPaK3D5np0LqDW8PH8kHrUugzEHyZtK +DqMyNr2mLFV5+HFmaiqP+flnsAYh5CoCRLY5hBh6AF40NvRhDRGp4gy4Be1nl7wp +52msSG77eOk7OOPbCcsiD5pXHMwG8fdEZtABxMEUZSnlzxkmc8mKXCslqdHGPthN +9fNnxyO5eyv1lyiJgZmdgkUhJ/TKhgIiLyZLYYrLdvuxe0xCtiXoPsurLGCno4L7 +7wVZA6VbAgMBAAECggEACzGbuulEMPd1DwetATtNZTbHBOoMe3vVj0A7dEiIXokG +zc2tl10Td26EVEBFvTpI5YiaqwzElYNMTo2M7TjizvTynZyGusPnisl6SoWoh0U5 +2HWIAHkSKCAww1RbGL+HbEuO5Wy3R7FMC0C6PuQPP3Bo+swVnqn1s6wf88U/zWml +Nthu0uQSj+pxW4tK/p7IoUVBnSqKExODDLG4LpO3meSaZIr36wC6bJZ8w8lZfRBy +DkPJu9NNknL6qSoVGozLzgtg1//yCkU+LX0OcDgTNeup5DlA08jglQY8p3Xo3FPn +evofoPvDnku4H1gCXT/djERRSlPdcGPEcy7xMQx12QKBgQDqdoL8hkp/DUzoKZyM +u2Vud5E1jULal3SmRB1XFzqxEiFsAT6UBH2feVweBOKTjLBqIuC+teQ+JgC5TsYP +CGbclQG/XBTYzOPfn3bBJWS4j7Jd68uXDQvkM9+RroFVaCXn75UGWEMqcbtgTNyU +wUrAVgfTtz07iHf2oUy+IreW7wKBgQDCegdlOojhn4juC+B5ROJHXzwI1qEznpJa +ftI7RERUbDFRIaucwvI6y95nduIRORO1bzpBhHZzJDPNBhZZya9wkaLElXktgi1Z +IwF6eb3m/FtOxx7DtI9daCVsuZsoPEw08NJq6UYQqeauaJ3LM5rDSMX0DN3V//2m +7tULbZn4VQKBgQCT4dwMWsdyC3mOlXBgc3IuksvL8yVPqmew1xWKcORb+wuJi99k +jNCPXYR0irA+UGaVCxqmLyOe72lVeBIEOVBnoLRRdkrP06uGyJWmjWdR4ZCnHKp0 +w43UicNhp6d7rwz5lWtxbQowIzwEKXaXfLMhTSHyr4i3nAPOUz6MTmltkQKBgB6z +ePtoFDfaIZnC0jsSvs4ZoLace3JUtDIJF1M34bmaIub188uZkvfpO0EGKYYihpP7 +7SxupuxiaLMTJPAjwMh6lUGHf0vJ4zLRLeiR04Llj9yN3rNyi7dpO49AddgSPM2W +vwEVtnPm/n3GEjMEAIiXsnhml5azBO4XghZ9xPLJAoGBALctm1sK8MdawZ+cnc1i +4P3VP2/nzGeODF29MbJefYrg0hlKHZSfKsWMKg3Dk9jDUplwsVjK5oBgN1vg/zOV +ysTtyn1q/RBbe96lYkPHzdYPWDD5Rg80/t0n6jItTOQr6QCshDLrMB3bruIQz7V9 +6PPhzvdQu3v3e07wrKDa1F3t +-----END PRIVATE KEY----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem b/test/configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem new file mode 100644 index 0000000..53786eb --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem @@ -0,0 +1,89 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:15 2023 GMT + Not After : Apr 28 19:01:15 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: + d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: + 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: + b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: + a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: + 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: + d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: + 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: + 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: + 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: + 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: + 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: + a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: + c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: + 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: + 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: + 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: + 40:27 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: + a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: + d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: + 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: + 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: + c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: + 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: + 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: + 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: + be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: + 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: + 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: + ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: + 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: + 91:5c:43:a9 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC +YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL +3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy +n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu +hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX +Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 ++bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp +LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr +uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN +zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg +zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX +KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/intermediate1/private/intermediate1_keypair.pem b/test/configs/certs/ocsp_peer/mini-ca/intermediate1/private/intermediate1_keypair.pem new file mode 100644 index 0000000..6c04954 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/intermediate1/private/intermediate1_keypair.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEuwIBADANBgkqhkiG9w0BAQEFAASCBKUwggShAgEAAoIBAQC8xoQtwqtdBddl +qOIVdNjy8VURRZOWTKXcy0T19BR+RgJZ6K54WWkhWPcWOLnCwmDYdquhOboLowMX +5KHLXRoMYnEkZLAA8G9MrwhijNxP4NfUVSzbNvypqtdYJ+SZy9wp2eo1FssuvgSy +glj05VwH2xKO4zyaXpBLxaPUIZZf4Y/3y57b4BCgbKIeMBdsMp97Q6Sf02szGxjN +pK0zSKOYsCvIInQXcdjxZCFV4TO8f3Rfpaaim1gv2+3HweU2LoYmrcb+uACFbnzt +/UrGoNmyP069+ghSyF0xE4a9P+x62DoV4nGv7ACIfqbo4Z2rV1qKH/jiTSlYU3kl +8J7ZGEAnAgMBAAECgf9Now4nMXr/fdU8+hNvCMPnuMbV5ewWCN2bzEa04K1D09BI +Tmm78MCVGwGoRoeNJBr5fdTPMMoJ/yVrG+W34iSvzqgnT4rJ/KqlA6CTwsiPyFay +RgxRQHCpVuLwp8ClyQ0wu26XQlrgJ480trAoUQdj6pC3V+ICdk90R/j0RW5JtsSu +e0ML3jNA9C4OgKlt2ia/MLqriaHXOf30EPONvtyqyKeGUFL7Un4eYKh4euRFEEMb +MKngNonefDCIdYA1wVFa3wT8bNBbpuHl3ghkokv6VpdHIVn9wC1l6HY5nPRjgmo7 +sguRI1bRa2TFkOIVwZjCJTyfANyQw14pRS6rxIkCgYEAwzSYHRpJlPHAD7wi3tls +bw7cBF9Q1P9PYKmVD9fAjx6eOjzDVOCdpGDijEkYoQoX1yYK3JaS8Vvp8V1wZ5Uh +HTTr6Y5uS6CPh37wGTJc9XhXdJpeN67fEOBZGU04FUlASVFeCiV3Ga6YX0HQ/yKd +VSc2JMX9mzxZjwhKRHmCEr0CgYEA95FFAxPxPNzYU3yHIdBlQWB1Z6AUFn+D4FgF +xeFOGmul1E+0PnPH78IlYanMjhhJ1nkc6X71rdX4ylonB/x6ldUpghWW9VWrqRGG +76S010aaZgOinwVE7+eeoelsIuma2W0QDwWrUT+RAsJBvZpGx1keo1qZEAaocs9V +R2lvHrMCgYBNMTMl7wtB9wd4MXGoploW4M1ofTi9wehl1Sm5BhyDfBwd84FawygT +pKxxxUYUCKW80rJg4Lpi73HnnIeirnpVzmOsDELZbTjU4AGaNSxFdb0/wvuXEXPs +fIs/UiXnZPwjAiYp5P7gDQb8RE6dVdbZoZPrns/W31qbETAtO8+QEQKBgQDgA710 +yYjSz+uXr+j/OflFrSjPedRzfzMvv7aJlhP8aEgH049/q3jRhNYah3Enaub1gWYe +Ctn4UNPtFqKW4WlzRw1mPm741Gqec9Or6VgSLDrt8IAocLYud2HdlMBa3xNVhxCu +5yxcOq7W1jxyerVtEUFeA07ZZ4zpRp8eHVOFbQKBgGJGU7xoJWO9P17SUGNfmSEF +6VIYFX6orA1Fi/kAJiqiFf98T4jnUWnL8LXVckt9FNw6KQqBCB6JuKXBFVkG2Bkr +f5IIhziTuDVpdLQSf0Z2i59TspgYjiKs4WEN3N0HGtCXfbyPO6Tt08d4icxL5Myt +W84T6Uof3+QQaqQnGvBE +-----END PRIVATE KEY----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem b/test/configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem new file mode 100644 index 0000000..4ca1762 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem @@ -0,0 +1,89 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:43 2023 GMT + Not After : Apr 28 19:01:43 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: + 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: + ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: + f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: + ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: + fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: + 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: + c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: + 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: + 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: + d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: + af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: + 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: + 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: + 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: + 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: + d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: + 08:41 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: + dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: + 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: + 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: + 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: + 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: + 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: + ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: + 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: + e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: + 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: + 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: + 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: + 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: + f1:5e:60:55 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ +97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh +bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn +qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW +wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 +1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A +PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 +ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG +m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 +J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc +jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz +EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/intermediate2/private/intermediate2_keypair.pem b/test/configs/certs/ocsp_peer/mini-ca/intermediate2/private/intermediate2_keypair.pem new file mode 100644 index 0000000..91e2908 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/intermediate2/private/intermediate2_keypair.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDaX/8d940anprz +K2iPwQwzBkEAyT7kGuHgcGr1L63f8+mZ7cXXqpMTN/9HqvPFife3rTpH5ZxOn4zi +Qe2kfJ2IMq71ioSfDBigs/6O3CqIavUvnIaS+ntus1p4Z1MLIWwNbIAaDh7uBsTS +5yTG5XS+Hi4XVSvlnwugWMz+v1M399yViPR3plm0uHyiS7dqZ6qE3Cnx+deJBU0L +84stUplX7W8Rnq8oo2FEwuxuf589C9z3GW0UiqW4tikCNJC0lsHLp0JGl8+NWf0X +saYnp3uKR2/6AyQcEiXuNNZc2kWYIzDhSMma3zeqG3Bssg+VOdZtPiUgqAcsSFcM +mVLLiQhBAgMBAAECggEABFew29ByrKMXSzsjgOpSmwgmjkSyPLiBIeSyZ85LK58u +18oH62Y/tvvf1nXCk7zO4YbvGANrpI+dLlmnx2PYAR+a5ZSb1wrXSYjSyNX9fYl8 +9zWqYm1bO4QTCj5pwximzKyJ7pq1yD93tgb1LwRcmjRA7+NYdGBBi66AYxd8aOo6 +QB7JoME+hzYAWB+foCOAPGAxYe7EFCPkPEyz08oxRCvDua0xa0+tWkU77MhUSCu+ +/uSq/Og9C9TfzCX0W91TNDnq8VeXbLDJoPNzgfSWIeYxSw/X5dUkYU8N2LuPLQOO +84Xv5UqU9YV22TEjg22YAL8/GMZ160K1xzXnQb1LPQKBgQDs/jOBp9NFiFlcNbJ8 +MKdfv+sktQR7onGehOaz/dEFEKiHNO8UmSlkAk+aZoTmYXKgIT4uytKRSOfZUWSl +kY64sKJ7KTvVq/Dzm4KsyH8VgYYQ3OrNbqSCSK7DiOiKJxQ+Jhm2+a+io16B8ZbM +RXLoaQ5+8oET6BgM5R6IMe4iFQKBgQDr44q7I7nhQdNz4h7OJ4HZ8X8WmQG7YpSX +EMLb5sX5wymfPi5uUDTcthUw7dpleO9js5iT93fB6+rd5yyiDPIes/dWjqULprvR +zIIr0u+cyt1TRxrNSa6dz/dJO3t/g/fTPKeM9j7ON4RvEGW4LPA+PbEUU0Q6xfSq +OZ0sZSXUfQKBgQDh8+r/rxbLsJgiRkAKEAlETSLQOJYxmkthq6yZ52ElxyAm6N0Z +cn34EAv9VclYLYiwC4HR8yaXxj7m/6dKBGFizWXcrw+RRQHSAW6xdedUhc1gvoBP +pTHL1ahqXVn4fhHav1C9F4nRMpmkosX3tC8+Twu3FVbjt+FWSgy2JYS5kQKBgD5B +6u6jaj7Skc2HA5xjfvkXrPQ44+UiCpeoW9WQHfZilQyra7O/xYPvJr6oODkJ5xzI +XN/Is7nh2zY/+l62zfxegUw+D794fR/NOxn37TfTrwB4xtEhvk12gwy3/0tTeEgv +PQWORFtG+dQaXs5yReIXhDIaG+rrLjzzQdFizM49AoGBAOulUGVDBpUFDVJn8S5r +bqss/PXW+5xj5g8b9/tzBuyfL0NJ9p3q6EWlELPRTX3zXuVRYjSe9cBUm5FXPxP2 +s1TsGUILjSw21dOtodahvXRDN3Uw2ALQy1MTDy8xLhr9Le+e6xF1T2muzg0vDT6L +VXAYfY5NPUOiPaYAj792oZk/ +-----END PRIVATE KEY----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/misc/misconfig_TestServer1_bundle.pem b/test/configs/certs/ocsp_peer/mini-ca/misc/misconfig_TestServer1_bundle.pem new file mode 100644 index 0000000..c3d1d2c --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/misc/misconfig_TestServer1_bundle.pem @@ -0,0 +1,186 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3c:c4:82:66:f8:5d:a6:b6:c7:66:e1:b2:01:3f:e0:72:fc:72:61:33 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Validity + Not Before: May 1 19:33:37 2023 GMT + Not After : Apr 28 19:33:37 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:af:26:5c:50:c0:fa:62:b5:fd:3d:c1:9e:26:51: + 58:62:04:37:b0:b5:6a:9b:6a:e3:22:3c:cd:ee:3c: + e7:8b:d3:e2:4c:08:1a:4d:63:c1:81:20:f4:53:a5: + 5d:2f:d2:71:d8:af:e3:26:95:b4:27:14:46:7f:e2: + 0a:73:12:a7:0e:ff:99:5a:29:f5:d0:65:96:b1:d1: + 96:7f:0c:43:b8:71:f2:4b:21:e1:97:6c:1b:01:e5: + 38:1a:39:44:72:d5:19:20:87:fe:90:4f:3b:97:f2: + 7d:bd:57:97:4d:9d:56:50:89:5b:79:29:7a:3a:13: + 97:08:61:c2:0c:a6:02:49:c9:8a:41:ab:8e:9f:25: + c9:33:18:f8:92:64:58:04:cc:a3:9d:cf:d4:d2:bd: + 20:ab:8b:9d:55:df:fb:5b:23:ac:95:12:fa:6f:07: + 93:3f:0e:03:86:c4:9b:25:06:21:9b:03:96:32:b8: + e0:0f:63:e2:1d:34:d1:41:35:19:09:c1:a0:dc:26: + b9:c8:66:fa:87:67:22:6e:0c:a6:e7:0f:24:64:b1: + 4f:84:05:ef:ad:8e:1b:f2:f4:38:87:d3:e3:48:a5: + 82:e0:66:89:1d:92:9a:59:67:a4:1d:03:6f:4d:a5: + fb:3b:c0:0b:73:a7:ab:8f:b4:10:25:8e:69:42:76: + 82:5f + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 43:16:E6:03:AF:37:B2:7B:BD:B3:C8:A2:9C:95:D7:FA:32:F8:9E:6F + X509v3 Authority Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, SSL Server + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:18888/intermediate1_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:18888/ + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + a3:87:9f:05:e4:38:61:f7:c4:5b:17:13:4b:2c:9d:a2:4d:e6: + ad:93:54:c5:a3:00:27:0b:5c:45:c5:bd:f8:b6:a7:5a:2a:ec: + dc:9b:59:8a:c7:59:e7:b9:86:f7:27:be:45:0d:d9:86:76:cf: + 00:71:ad:aa:cc:73:50:8c:68:63:b0:e2:3a:59:dd:85:fa:0d: + f0:82:51:05:79:e6:d5:0e:0b:bb:ed:23:65:8f:d0:8b:01:df: + 86:74:bc:3a:22:90:e4:59:44:91:d5:44:d8:21:4d:4e:10:72: + 0a:12:2e:4a:20:5f:15:e7:16:0b:6f:76:f3:04:1f:da:44:50: + 3b:c3:b3:0f:fa:05:cf:6e:64:9c:65:e2:0d:38:28:31:c3:c3: + b6:66:ef:80:d3:c4:5f:e9:f9:01:e9:ce:e6:99:46:a0:9d:ce: + 90:63:77:d2:85:21:d7:88:32:55:38:fe:10:07:69:cd:c8:06: + b7:6f:49:98:bf:cd:be:4f:ab:44:ea:78:af:ab:01:c8:3e:fa: + d9:54:bc:59:28:db:03:9b:1c:ee:e4:c3:ed:f3:97:30:c6:40: + 33:76:84:40:b2:b8:4d:b4:ca:a9:2d:d1:4d:17:92:ea:c0:c9: + cb:f6:b1:d7:d3:c7:e6:75:15:00:ff:c7:d9:54:63:27:19:5c: + 96:a5:e5:d9 +-----BEGIN CERTIFICATE----- +MIIEYjCCA0qgAwIBAgIUPMSCZvhdprbHZuGyAT/gcvxyYTMwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe +Fw0yMzA1MDExOTMzMzdaFw0zMzA0MjgxOTMzMzdaMFQxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU +MBIGA1UEAwwLVGVzdFNlcnZlcjEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQCvJlxQwPpitf09wZ4mUVhiBDewtWqbauMiPM3uPOeL0+JMCBpNY8GBIPRT +pV0v0nHYr+MmlbQnFEZ/4gpzEqcO/5laKfXQZZax0ZZ/DEO4cfJLIeGXbBsB5Tga +OURy1Rkgh/6QTzuX8n29V5dNnVZQiVt5KXo6E5cIYcIMpgJJyYpBq46fJckzGPiS +ZFgEzKOdz9TSvSCri51V3/tbI6yVEvpvB5M/DgOGxJslBiGbA5YyuOAPY+IdNNFB +NRkJwaDcJrnIZvqHZyJuDKbnDyRksU+EBe+tjhvy9DiH0+NIpYLgZokdkppZZ6Qd +A29Npfs7wAtzp6uPtBAljmlCdoJfAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUQxbm +A683snu9s8iinJXX+jL4nm8wHwYDVR0jBBgwFoAUtZFuT2S3FoR2+bS+mc5glZga +jp0wDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD +AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg +MKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ludGVybWVkaWF0ZTFfY3JsLmRl +cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 +MTg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF +AAOCAQEAo4efBeQ4YffEWxcTSyydok3mrZNUxaMAJwtcRcW9+LanWirs3JtZisdZ +57mG9ye+RQ3ZhnbPAHGtqsxzUIxoY7DiOlndhfoN8IJRBXnm1Q4Lu+0jZY/QiwHf +hnS8OiKQ5FlEkdVE2CFNThByChIuSiBfFecWC2928wQf2kRQO8OzD/oFz25knGXi +DTgoMcPDtmbvgNPEX+n5AenO5plGoJ3OkGN30oUh14gyVTj+EAdpzcgGt29JmL/N +vk+rROp4r6sByD762VS8WSjbA5sc7uTD7fOXMMZAM3aEQLK4TbTKqS3RTReS6sDJ +y/ax19PH5nUVAP/H2VRjJxlclqXl2Q== +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:43 2023 GMT + Not After : Apr 28 19:01:43 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: + 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: + ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: + f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: + ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: + fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: + 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: + c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: + 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: + 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: + d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: + af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: + 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: + 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: + 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: + 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: + d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: + 08:41 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: + dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: + 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: + 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: + 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: + 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: + 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: + ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: + 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: + e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: + 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: + 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: + 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: + 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: + f1:5e:60:55 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ +97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh +bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn +qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW +wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 +1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A +PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 +ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG +m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 +J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc +jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz +EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/misc/trust_config1_bundle.pem b/test/configs/certs/ocsp_peer/mini-ca/misc/trust_config1_bundle.pem new file mode 100644 index 0000000..f632ad5 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/misc/trust_config1_bundle.pem @@ -0,0 +1,264 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:43 2023 GMT + Not After : Apr 28 19:01:43 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: + 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: + ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: + f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: + ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: + fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: + 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: + c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: + 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: + 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: + d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: + af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: + 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: + 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: + 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: + 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: + d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: + 08:41 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: + dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: + 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: + 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: + 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: + 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: + 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: + ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: + 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: + e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: + 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: + 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: + 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: + 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: + f1:5e:60:55 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ +97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh +bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn +qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW +wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 +1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A +PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 +ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG +m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 +J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc +jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz +EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 18:57:57 2023 GMT + Not After : Apr 28 18:57:57 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68: + c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b: + 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f: + d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce: + f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b: + 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19: + d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08: + 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c: + 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8: + 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1: + 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05: + 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11: + f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a: + 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01: + fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07: + 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91: + c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65: + 76:d7 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5: + ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60: + d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59: + 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed: + 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a: + f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d: + 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7: + 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28: + 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58: + af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f: + 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95: + 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95: + 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82: + d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8: + dc:29:ac:17 +-----BEGIN CERTIFICATE----- +MIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4 +NTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS +b290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i ++1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi +YM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY +Y0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv +XbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ +7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC +Wz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220 +4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD +AQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3 +LjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5 +Xfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr +UFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I +4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y +VPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb +hpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp +MElNbuoFHtjcKawX +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:15 2023 GMT + Not After : Apr 28 19:01:15 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: + d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: + 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: + b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: + a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: + 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: + d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: + 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: + 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: + 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: + 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: + 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: + a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: + c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: + 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: + 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: + 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: + 40:27 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: + a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: + d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: + 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: + 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: + c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: + 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: + 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: + 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: + be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: + 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: + 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: + ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: + 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: + 91:5c:43:a9 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC +YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL +3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy +n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu +hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX +Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 ++bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp +LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr +uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN +zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg +zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX +KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/misc/trust_config2_bundle.pem b/test/configs/certs/ocsp_peer/mini-ca/misc/trust_config2_bundle.pem new file mode 100644 index 0000000..fb390ca --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/misc/trust_config2_bundle.pem @@ -0,0 +1,264 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:43 2023 GMT + Not After : Apr 28 19:01:43 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: + 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: + ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: + f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: + ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: + fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: + 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: + c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: + 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: + 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: + d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: + af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: + 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: + 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: + 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: + 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: + d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: + 08:41 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: + dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: + 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: + 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: + 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: + 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: + 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: + ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: + 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: + e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: + 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: + 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: + 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: + 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: + f1:5e:60:55 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ +97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh +bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn +qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW +wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 +1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A +PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 +ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG +m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 +J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc +jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz +EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:15 2023 GMT + Not After : Apr 28 19:01:15 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: + d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: + 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: + b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: + a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: + 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: + d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: + 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: + 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: + 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: + 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: + 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: + a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: + c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: + 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: + 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: + 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: + 40:27 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: + a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: + d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: + 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: + 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: + c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: + 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: + 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: + 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: + be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: + 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: + 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: + ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: + 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: + 91:5c:43:a9 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC +YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL +3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy +n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu +hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX +Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 ++bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp +LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr +uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN +zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg +zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX +KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 18:57:57 2023 GMT + Not After : Apr 28 18:57:57 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68: + c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b: + 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f: + d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce: + f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b: + 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19: + d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08: + 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c: + 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8: + 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1: + 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05: + 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11: + f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a: + 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01: + fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07: + 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91: + c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65: + 76:d7 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5: + ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60: + d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59: + 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed: + 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a: + f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d: + 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7: + 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28: + 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58: + af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f: + 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95: + 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95: + 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82: + d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8: + dc:29:ac:17 +-----BEGIN CERTIFICATE----- +MIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4 +NTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS +b290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i ++1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi +YM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY +Y0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv +XbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ +7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC +Wz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220 +4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD +AQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3 +LjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5 +Xfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr +UFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I +4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y +VPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb +hpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp +MElNbuoFHtjcKawX +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/misc/trust_config3_bundle.pem b/test/configs/certs/ocsp_peer/mini-ca/misc/trust_config3_bundle.pem new file mode 100644 index 0000000..2ba91b0 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/misc/trust_config3_bundle.pem @@ -0,0 +1,264 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:15 2023 GMT + Not After : Apr 28 19:01:15 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: + d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: + 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: + b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: + a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: + 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: + d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: + 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: + 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: + 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: + 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: + 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: + a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: + c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: + 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: + 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: + 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: + 40:27 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: + a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: + d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: + 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: + 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: + c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: + 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: + 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: + 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: + be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: + 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: + 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: + ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: + 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: + 91:5c:43:a9 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC +YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL +3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy +n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu +hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX +Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 ++bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp +LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr +uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN +zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg +zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX +KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:43 2023 GMT + Not After : Apr 28 19:01:43 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: + 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: + ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: + f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: + ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: + fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: + 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: + c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: + 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: + 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: + d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: + af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: + 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: + 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: + 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: + 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: + d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: + 08:41 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: + dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: + 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: + 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: + 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: + 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: + 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: + ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: + 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: + e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: + 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: + 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: + 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: + 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: + f1:5e:60:55 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ +97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh +bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn +qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW +wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 +1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A +PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 +ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG +m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 +J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc +jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz +EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 18:57:57 2023 GMT + Not After : Apr 28 18:57:57 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68: + c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b: + 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f: + d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce: + f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b: + 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19: + d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08: + 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c: + 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8: + 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1: + 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05: + 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11: + f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a: + 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01: + fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07: + 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91: + c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65: + 76:d7 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5: + ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60: + d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59: + 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed: + 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a: + f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d: + 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7: + 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28: + 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58: + af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f: + 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95: + 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95: + 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82: + d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8: + dc:29:ac:17 +-----BEGIN CERTIFICATE----- +MIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4 +NTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS +b290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i ++1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi +YM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY +Y0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv +XbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ +7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC +Wz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220 +4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD +AQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3 +LjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5 +Xfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr +UFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I +4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y +VPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb +hpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp +MElNbuoFHtjcKawX +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_bundle.pem b/test/configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_bundle.pem new file mode 100644 index 0000000..760eb22 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_bundle.pem @@ -0,0 +1,181 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 29:e1:52:8d:fd:a5:2a:87:eb:1d:e4:1d:47:6c:e1:8a:58:69:73:ab + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Validity + Not Before: May 1 19:28:39 2023 GMT + Not After : Apr 28 19:28:39 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=OCSP Responder + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:a3:0c:ca:eb:80:eb:a1:0e:1e:71:9b:d3:b3:f9: + 65:ce:70:c2:21:06:3c:31:c1:06:7e:a5:a8:4a:e1: + 21:a3:74:54:9f:57:ce:50:d6:c3:29:3c:43:b0:9d: + 3e:54:94:ee:8d:fa:0d:71:6c:df:5e:9e:01:30:79: + 6c:bb:97:5d:af:bb:5b:05:77:72:9f:55:e6:66:45: + f4:e2:c2:cf:7b:0e:58:d6:14:6a:76:29:ac:e3:30: + 28:0d:ee:bd:ca:aa:ae:1f:1e:ef:40:f3:c3:ab:17: + f2:d7:ec:0d:e1:fb:68:9a:09:83:99:11:58:42:94: + f8:0d:d4:9a:6f:9f:3b:e8:56:f0:a9:b7:18:1a:91: + 41:7c:43:e3:db:b1:01:f1:ad:0b:39:d7:65:98:e6: + 15:b0:17:a9:56:6e:fb:84:7a:c0:cc:67:75:fc:f6: + 75:84:31:78:c5:6d:51:8f:d0:19:d3:16:4f:87:ef: + 5b:33:b9:7a:dd:fe:5f:a8:6a:fd:44:54:00:f3:a4: + a6:5b:fd:3b:65:38:4f:82:4f:b9:c4:bd:c9:9a:56: + fc:54:f1:58:2f:cb:ee:f4:08:fd:b7:ec:ad:28:08: + 66:9b:f8:78:98:32:db:b1:56:dd:0e:31:ba:c6:e3: + 56:f5:02:2f:fb:76:28:bb:c4:8b:f3:6b:da:aa:1d: + 38:21 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + CB:5E:50:60:7A:AB:2F:A9:3B:1E:24:AB:02:42:8D:EC:81:60:48:13 + X509v3 Authority Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Key Usage: critical + Digital Signature + X509v3 Extended Key Usage: critical + OCSP Signing + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:18888/intermediate1_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:18888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 48:65:ce:6d:91:46:30:37:b6:f2:76:c0:42:e3:f5:ee:e9:32: + 0e:46:b5:d5:9d:ac:b0:f2:23:f5:35:a8:1c:61:66:81:c0:0d: + bc:a4:bb:b5:be:47:58:8b:f1:d1:5f:73:83:d2:99:da:3e:a3: + 0b:32:81:96:a4:bd:a8:57:8e:fe:3d:c4:93:57:ef:05:77:60: + c9:88:1c:2e:25:7e:ea:c8:95:8d:a6:4a:73:e5:bb:6c:c4:3b: + 01:03:90:8d:12:f5:69:13:c5:79:87:ae:45:cb:49:c8:90:24: + 39:30:cf:27:ba:31:1e:5f:5b:e0:0f:93:82:66:28:33:dc:e3: + a1:a8:fc:ad:40:d0:48:31:63:fb:a0:6a:13:18:b1:8b:59:bb: + ef:96:f8:83:98:6c:4a:18:37:1a:02:ad:c2:42:1d:7e:1c:dc: + 4a:77:b7:f5:ae:97:3e:17:e8:35:96:85:a0:e4:30:c5:03:0b: + 62:55:13:c1:3f:df:15:1b:c3:45:f7:69:d6:5e:f5:77:fc:4f: + e8:28:3b:3e:f0:2c:20:22:81:72:a3:d6:1b:d1:52:63:86:21: + 22:06:7a:5b:f4:2a:c7:e5:b9:97:ac:1b:56:b5:4c:62:e9:f9: + 6f:49:5f:43:3d:9c:e6:85:3a:f8:c9:4c:33:fd:e9:aa:88:8e: + cf:28:5c:69 +-----BEGIN CERTIFICATE----- +MIIELTCCAxWgAwIBAgIUKeFSjf2lKofrHeQdR2zhilhpc6swDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe +Fw0yMzA1MDExOTI4MzlaFw0zMzA0MjgxOTI4MzlaMFcxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEX +MBUGA1UEAwwOT0NTUCBSZXNwb25kZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQCjDMrrgOuhDh5xm9Oz+WXOcMIhBjwxwQZ+pahK4SGjdFSfV85Q1sMp +PEOwnT5UlO6N+g1xbN9engEweWy7l12vu1sFd3KfVeZmRfTiws97DljWFGp2Kazj +MCgN7r3Kqq4fHu9A88OrF/LX7A3h+2iaCYOZEVhClPgN1JpvnzvoVvCptxgakUF8 +Q+PbsQHxrQs512WY5hWwF6lWbvuEesDMZ3X89nWEMXjFbVGP0BnTFk+H71szuXrd +/l+oav1EVADzpKZb/TtlOE+CT7nEvcmaVvxU8Vgvy+70CP237K0oCGab+HiYMtux +Vt0OMbrG41b1Ai/7dii7xIvza9qqHTghAgMBAAGjge0wgeowHQYDVR0OBBYEFMte +UGB6qy+pOx4kqwJCjeyBYEgTMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWY +Go6dMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBYGA1UdJQEB/wQMMAoG +CCsGAQUFBwMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4 +ODgvaW50ZXJtZWRpYXRlMV9jcmwuZGVyMDMGCCsGAQUFBwEBBCcwJTAjBggrBgEF +BQcwAYYXaHR0cDovLzEyNy4wLjAuMToxODg4OC8wDQYJKoZIhvcNAQELBQADggEB +AEhlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFf +c4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S +9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZ +u++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3 +adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9 +nOaFOvjJTDP96aqIjs8oXGk= +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:15 2023 GMT + Not After : Apr 28 19:01:15 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: + d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: + 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: + b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: + a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: + 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: + d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: + 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: + 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: + 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: + 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: + 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: + a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: + c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: + 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: + 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: + 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: + 40:27 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: + a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: + d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: + 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: + 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: + c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: + 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: + 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: + 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: + be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: + 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: + 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: + ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: + 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: + 91:5c:43:a9 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC +YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL +3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy +n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu +hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX +Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 ++bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp +LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr +uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN +zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg +zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX +KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_cert.pem b/test/configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_cert.pem new file mode 100644 index 0000000..218a28e --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_cert.pem @@ -0,0 +1,92 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 29:e1:52:8d:fd:a5:2a:87:eb:1d:e4:1d:47:6c:e1:8a:58:69:73:ab + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Validity + Not Before: May 1 19:28:39 2023 GMT + Not After : Apr 28 19:28:39 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=OCSP Responder + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:a3:0c:ca:eb:80:eb:a1:0e:1e:71:9b:d3:b3:f9: + 65:ce:70:c2:21:06:3c:31:c1:06:7e:a5:a8:4a:e1: + 21:a3:74:54:9f:57:ce:50:d6:c3:29:3c:43:b0:9d: + 3e:54:94:ee:8d:fa:0d:71:6c:df:5e:9e:01:30:79: + 6c:bb:97:5d:af:bb:5b:05:77:72:9f:55:e6:66:45: + f4:e2:c2:cf:7b:0e:58:d6:14:6a:76:29:ac:e3:30: + 28:0d:ee:bd:ca:aa:ae:1f:1e:ef:40:f3:c3:ab:17: + f2:d7:ec:0d:e1:fb:68:9a:09:83:99:11:58:42:94: + f8:0d:d4:9a:6f:9f:3b:e8:56:f0:a9:b7:18:1a:91: + 41:7c:43:e3:db:b1:01:f1:ad:0b:39:d7:65:98:e6: + 15:b0:17:a9:56:6e:fb:84:7a:c0:cc:67:75:fc:f6: + 75:84:31:78:c5:6d:51:8f:d0:19:d3:16:4f:87:ef: + 5b:33:b9:7a:dd:fe:5f:a8:6a:fd:44:54:00:f3:a4: + a6:5b:fd:3b:65:38:4f:82:4f:b9:c4:bd:c9:9a:56: + fc:54:f1:58:2f:cb:ee:f4:08:fd:b7:ec:ad:28:08: + 66:9b:f8:78:98:32:db:b1:56:dd:0e:31:ba:c6:e3: + 56:f5:02:2f:fb:76:28:bb:c4:8b:f3:6b:da:aa:1d: + 38:21 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + CB:5E:50:60:7A:AB:2F:A9:3B:1E:24:AB:02:42:8D:EC:81:60:48:13 + X509v3 Authority Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Key Usage: critical + Digital Signature + X509v3 Extended Key Usage: critical + OCSP Signing + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:18888/intermediate1_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:18888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 48:65:ce:6d:91:46:30:37:b6:f2:76:c0:42:e3:f5:ee:e9:32: + 0e:46:b5:d5:9d:ac:b0:f2:23:f5:35:a8:1c:61:66:81:c0:0d: + bc:a4:bb:b5:be:47:58:8b:f1:d1:5f:73:83:d2:99:da:3e:a3: + 0b:32:81:96:a4:bd:a8:57:8e:fe:3d:c4:93:57:ef:05:77:60: + c9:88:1c:2e:25:7e:ea:c8:95:8d:a6:4a:73:e5:bb:6c:c4:3b: + 01:03:90:8d:12:f5:69:13:c5:79:87:ae:45:cb:49:c8:90:24: + 39:30:cf:27:ba:31:1e:5f:5b:e0:0f:93:82:66:28:33:dc:e3: + a1:a8:fc:ad:40:d0:48:31:63:fb:a0:6a:13:18:b1:8b:59:bb: + ef:96:f8:83:98:6c:4a:18:37:1a:02:ad:c2:42:1d:7e:1c:dc: + 4a:77:b7:f5:ae:97:3e:17:e8:35:96:85:a0:e4:30:c5:03:0b: + 62:55:13:c1:3f:df:15:1b:c3:45:f7:69:d6:5e:f5:77:fc:4f: + e8:28:3b:3e:f0:2c:20:22:81:72:a3:d6:1b:d1:52:63:86:21: + 22:06:7a:5b:f4:2a:c7:e5:b9:97:ac:1b:56:b5:4c:62:e9:f9: + 6f:49:5f:43:3d:9c:e6:85:3a:f8:c9:4c:33:fd:e9:aa:88:8e: + cf:28:5c:69 +-----BEGIN CERTIFICATE----- +MIIELTCCAxWgAwIBAgIUKeFSjf2lKofrHeQdR2zhilhpc6swDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe +Fw0yMzA1MDExOTI4MzlaFw0zMzA0MjgxOTI4MzlaMFcxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEX +MBUGA1UEAwwOT0NTUCBSZXNwb25kZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQCjDMrrgOuhDh5xm9Oz+WXOcMIhBjwxwQZ+pahK4SGjdFSfV85Q1sMp +PEOwnT5UlO6N+g1xbN9engEweWy7l12vu1sFd3KfVeZmRfTiws97DljWFGp2Kazj +MCgN7r3Kqq4fHu9A88OrF/LX7A3h+2iaCYOZEVhClPgN1JpvnzvoVvCptxgakUF8 +Q+PbsQHxrQs512WY5hWwF6lWbvuEesDMZ3X89nWEMXjFbVGP0BnTFk+H71szuXrd +/l+oav1EVADzpKZb/TtlOE+CT7nEvcmaVvxU8Vgvy+70CP237K0oCGab+HiYMtux +Vt0OMbrG41b1Ai/7dii7xIvza9qqHTghAgMBAAGjge0wgeowHQYDVR0OBBYEFMte +UGB6qy+pOx4kqwJCjeyBYEgTMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWY +Go6dMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBYGA1UdJQEB/wQMMAoG +CCsGAQUFBwMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4 +ODgvaW50ZXJtZWRpYXRlMV9jcmwuZGVyMDMGCCsGAQUFBwEBBCcwJTAjBggrBgEF +BQcwAYYXaHR0cDovLzEyNy4wLjAuMToxODg4OC8wDQYJKoZIhvcNAQELBQADggEB +AEhlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFf +c4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S +9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZ +u++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3 +adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9 +nOaFOvjJTDP96aqIjs8oXGk= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/ocsp1/private/ocsp1_keypair.pem b/test/configs/certs/ocsp_peer/mini-ca/ocsp1/private/ocsp1_keypair.pem new file mode 100644 index 0000000..13b6dbe --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/ocsp1/private/ocsp1_keypair.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCjDMrrgOuhDh5x +m9Oz+WXOcMIhBjwxwQZ+pahK4SGjdFSfV85Q1sMpPEOwnT5UlO6N+g1xbN9engEw +eWy7l12vu1sFd3KfVeZmRfTiws97DljWFGp2KazjMCgN7r3Kqq4fHu9A88OrF/LX +7A3h+2iaCYOZEVhClPgN1JpvnzvoVvCptxgakUF8Q+PbsQHxrQs512WY5hWwF6lW +bvuEesDMZ3X89nWEMXjFbVGP0BnTFk+H71szuXrd/l+oav1EVADzpKZb/TtlOE+C +T7nEvcmaVvxU8Vgvy+70CP237K0oCGab+HiYMtuxVt0OMbrG41b1Ai/7dii7xIvz +a9qqHTghAgMBAAECggEAE++sPxPuG6zzhX4hakvYCiAo6GtQAGBi6CjetTsmRwti +DnKoyCMeTUQwXZ+4X5SvP35f1urSPAozSIdMR3qoSqSsqjQy+G8DIyWyHejmgBwe +uhxYcRbC7Ct29k8m9ykb7bO1WtqDZf/hYkvbXbKFFXKM2/IuOcPnuZ8xe+z7IPsQ +ODHnrQs45wQyi2i2/+AbvEJjb3bb3oS8MfoZfvO8F06ejTOmv/ATZSxX0T6ppCPj +HdmKqKDXlYQNA/LQeM4cs2FaQH170R1vGHppDjcs2ezqElB7/HKfKWeEn0Eytu9E +eWw9tZteisnzfqEvDMgOM2eWwAzfIhXSQYMWlVBicQKBgQC6MPaLd4r82BBMj7qx +ChdBxB7LXptvx/q3SrMjZ6GKmrGdXMbsos50XexajktBqkXfUMa8hGqmlciN5xL1 ++w//p7oSzb3VorOyHVXZpc8p79eUeX8ONcwySOYwO+CpqFBBDlvPn1OuPnlUL1pv +IgCMT66flWJxRklDMIJsHr+iWQKBgQDgLq3I2cj4q+3121ECPXKLt+VCHUY0aygc +tl6lvQw61UnmyLQ+k53/MmyPGGCxIFr18DsoKeWYwt3kWTW0MCDrQuO6PZkB268v +gdsmN3nhAKiR0gUwJDrFjpPWr0GAhw9LE7HqpvkQ3fG5YSnXTUibhm6smHg7dzVL +ER+QJ+Y7CQKBgHIDN4WRjy9jEx/+x0BPwIwKDx1TcnURjQoeGPHuLHJWZbrJrBoN +W8TQGsIc7iJopN6pdPjNUQ1vHN8gB3FO6q4PRBbtm3gtaEICSqa7LM8uSeFmQJIw +CTklgKc6k0jwgyxDIZ9SnghNwzf0wzjYJmPFC1Y3QI/CjWwyUTrp3UkJAoGBANHc +IKcS6MWQ/RPYGP+F0kLlBWJc0Smk3knylquES3yPybyXSdQCkDcjVuilo25soXn1 +RwuUHPBiCyIGOPXS0B4r4c6odyF8K4THhQVDjX6KBUNsXZrxb2scy1x/d0wAItrf +NwA5CpM1kWE+idKY8E1XDSfZG0Rfla4N+4QRNb8xAoGAQrVe80TpPpzDH846xaPF +BAhjKz7dRrUQ1n7ZI6yw8I5bU2ky9jSF3PRsDismM23aNbYhIgjcYv0tG0LMlWLV +2eIrU9OoA7aepDPozyhEkENaWqYXX/T8AjD+Kaw7XJnt/NX8eS0RF2qgDA/HEwWw +uf1ecRqpjZ9cxNGLZ+/pOkM= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/ocsp2/ocsp2_bundle.pem b/test/configs/certs/ocsp_peer/mini-ca/ocsp2/ocsp2_bundle.pem new file mode 100644 index 0000000..2d3f2d0 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/ocsp2/ocsp2_bundle.pem @@ -0,0 +1,181 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 7b:97:35:73:2b:2b:5f:74:c6:43:83:8f:ae:65:5b:a0:f5:f4:ff:1f + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Validity + Not Before: May 1 19:29:28 2023 GMT + Not After : Apr 28 19:29:28 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=OCSP Responder 2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b8:98:3d:03:4d:5e:b2:66:5e:51:3b:f9:3d:f2: + 7a:24:6b:70:5c:2f:7a:05:b2:51:77:62:45:e7:33: + 75:77:db:31:6f:2d:13:32:cd:d3:a0:03:84:ee:f9: + 2b:81:9d:e5:c9:ba:e2:25:c9:a7:18:2b:fd:f1:95: + ad:d3:46:90:d9:7b:7f:39:2d:85:b4:70:7c:72:44: + 99:fb:df:9f:22:4c:81:77:35:bb:fe:41:7f:86:f5: + c7:29:53:7c:ee:d4:cc:09:54:fa:cc:b1:4d:4b:c2: + c7:c7:3e:1a:13:59:66:36:31:ae:60:1b:6a:05:b0: + 5b:64:96:77:9d:74:cc:42:6e:13:d1:21:83:94:8e: + 6c:4c:d8:42:57:94:17:ff:26:d4:d1:2f:64:58:b5: + 47:1a:22:38:69:bf:c0:5a:9c:c3:88:01:0a:1d:f7: + d8:68:88:7c:57:5d:44:c4:71:d0:66:8d:1c:39:e0: + af:e8:f7:ce:51:60:7c:1d:b7:d5:e7:b5:3e:6a:a5: + 2b:46:c3:4e:b9:ef:de:bd:a6:be:e2:66:79:a9:6a: + 0d:c1:b2:e7:5e:03:9d:de:dd:41:b9:c9:80:2c:bd: + 6d:1f:09:5f:4e:25:e7:ac:ff:23:47:8f:5f:74:69: + be:81:42:5c:e6:1a:f7:65:1f:eb:a1:d0:69:6f:be: + 7e:89 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + E4:4D:EE:6A:A3:30:91:37:3E:5C:1D:BD:26:96:5F:FF:DB:D3:E2:15 + X509v3 Authority Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Key Usage: critical + Digital Signature + X509v3 Extended Key Usage: critical + OCSP Signing + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:28888/intermediate2_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:28888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 6c:d6:fa:8f:6f:c9:0a:99:0b:ee:6c:27:1f:75:52:b8:82:33: + 41:fe:01:a1:f8:c5:24:4e:9e:3b:e2:89:0f:01:2b:8e:c4:76: + fb:d9:75:5a:b2:9c:e0:36:8d:fd:90:9f:28:92:1b:a3:74:fd: + c5:39:28:51:06:ab:95:f7:64:95:e8:7b:d9:97:35:33:97:05: + 38:87:e6:e6:d7:a5:0b:a1:11:0c:b7:8b:76:b8:a9:46:33:ba: + 50:b3:3b:96:90:65:4b:ea:14:20:c9:f7:0d:8d:5e:89:c6:78: + e3:0b:4f:d2:db:10:46:8a:c4:81:6f:20:13:30:83:a8:45:4d: + 2b:ef:f0:ce:18:a7:96:fc:b9:67:79:e9:a9:f0:2f:b2:33:1c: + 83:cf:a3:4b:df:fd:c5:58:ae:87:83:d9:be:22:85:58:41:f5: + a0:a2:2d:56:98:40:12:78:c5:43:b0:50:34:0f:6c:0b:52:ad: + 68:e1:7a:9e:c1:54:58:bf:b4:f1:c5:3b:bf:97:e4:f9:44:09: + f5:c7:67:7d:dc:3d:ea:a9:9f:0f:3a:aa:9c:4a:c1:ef:a1:52: + 25:e4:57:22:d6:af:c6:c9:c8:02:91:4b:ec:a2:d6:ba:b5:bf: + ed:22:7c:b2:71:6c:78:f4:ba:e4:b9:b7:1f:11:65:d4:4f:77: + 4d:ef:b5:43 +-----BEGIN CERTIFICATE----- +MIIELzCCAxegAwIBAgIUe5c1cysrX3TGQ4OPrmVboPX0/x8wDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe +Fw0yMzA1MDExOTI5MjhaFw0zMzA0MjgxOTI5MjhaMFkxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEZ +MBcGA1UEAwwQT0NTUCBSZXNwb25kZXIgMjCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBALiYPQNNXrJmXlE7+T3yeiRrcFwvegWyUXdiReczdXfbMW8tEzLN +06ADhO75K4Gd5cm64iXJpxgr/fGVrdNGkNl7fzkthbRwfHJEmfvfnyJMgXc1u/5B +f4b1xylTfO7UzAlU+syxTUvCx8c+GhNZZjYxrmAbagWwW2SWd510zEJuE9Ehg5SO +bEzYQleUF/8m1NEvZFi1RxoiOGm/wFqcw4gBCh332GiIfFddRMRx0GaNHDngr+j3 +zlFgfB231ee1PmqlK0bDTrnv3r2mvuJmealqDcGy514Dnd7dQbnJgCy9bR8JX04l +56z/I0ePX3RpvoFCXOYa92Uf66HQaW++fokCAwEAAaOB7TCB6jAdBgNVHQ4EFgQU +5E3uaqMwkTc+XB29JpZf/9vT4hUwHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyi +V3ftFawwDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwFgYDVR0lAQH/BAww +CgYIKwYBBQUHAwkwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDovLzEyNy4wLjAuMToy +ODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYIKwYBBQUHAQEEJzAlMCMGCCsG +AQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4LzANBgkqhkiG9w0BAQsFAAOC +AQEAbNb6j2/JCpkL7mwnH3VSuIIzQf4BofjFJE6eO+KJDwErjsR2+9l1WrKc4DaN +/ZCfKJIbo3T9xTkoUQarlfdkleh72Zc1M5cFOIfm5telC6ERDLeLdripRjO6ULM7 +lpBlS+oUIMn3DY1eicZ44wtP0tsQRorEgW8gEzCDqEVNK+/wzhinlvy5Z3npqfAv +sjMcg8+jS9/9xViuh4PZviKFWEH1oKItVphAEnjFQ7BQNA9sC1KtaOF6nsFUWL+0 +8cU7v5fk+UQJ9cdnfdw96qmfDzqqnErB76FSJeRXItavxsnIApFL7KLWurW/7SJ8 +snFsePS65Lm3HxFl1E93Te+1Qw== +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:43 2023 GMT + Not After : Apr 28 19:01:43 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: + 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: + ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: + f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: + ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: + fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: + 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: + c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: + 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: + 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: + d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: + af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: + 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: + 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: + 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: + 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: + d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: + 08:41 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: + dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: + 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: + 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: + 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: + 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: + 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: + ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: + 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: + e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: + 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: + 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: + 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: + 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: + f1:5e:60:55 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ +97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh +bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn +qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW +wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 +1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A +PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 +ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG +m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 +J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc +jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz +EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/ocsp2/ocsp2_cert.pem b/test/configs/certs/ocsp_peer/mini-ca/ocsp2/ocsp2_cert.pem new file mode 100644 index 0000000..1f26c38 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/ocsp2/ocsp2_cert.pem @@ -0,0 +1,92 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 7b:97:35:73:2b:2b:5f:74:c6:43:83:8f:ae:65:5b:a0:f5:f4:ff:1f + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Validity + Not Before: May 1 19:29:28 2023 GMT + Not After : Apr 28 19:29:28 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=OCSP Responder 2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b8:98:3d:03:4d:5e:b2:66:5e:51:3b:f9:3d:f2: + 7a:24:6b:70:5c:2f:7a:05:b2:51:77:62:45:e7:33: + 75:77:db:31:6f:2d:13:32:cd:d3:a0:03:84:ee:f9: + 2b:81:9d:e5:c9:ba:e2:25:c9:a7:18:2b:fd:f1:95: + ad:d3:46:90:d9:7b:7f:39:2d:85:b4:70:7c:72:44: + 99:fb:df:9f:22:4c:81:77:35:bb:fe:41:7f:86:f5: + c7:29:53:7c:ee:d4:cc:09:54:fa:cc:b1:4d:4b:c2: + c7:c7:3e:1a:13:59:66:36:31:ae:60:1b:6a:05:b0: + 5b:64:96:77:9d:74:cc:42:6e:13:d1:21:83:94:8e: + 6c:4c:d8:42:57:94:17:ff:26:d4:d1:2f:64:58:b5: + 47:1a:22:38:69:bf:c0:5a:9c:c3:88:01:0a:1d:f7: + d8:68:88:7c:57:5d:44:c4:71:d0:66:8d:1c:39:e0: + af:e8:f7:ce:51:60:7c:1d:b7:d5:e7:b5:3e:6a:a5: + 2b:46:c3:4e:b9:ef:de:bd:a6:be:e2:66:79:a9:6a: + 0d:c1:b2:e7:5e:03:9d:de:dd:41:b9:c9:80:2c:bd: + 6d:1f:09:5f:4e:25:e7:ac:ff:23:47:8f:5f:74:69: + be:81:42:5c:e6:1a:f7:65:1f:eb:a1:d0:69:6f:be: + 7e:89 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + E4:4D:EE:6A:A3:30:91:37:3E:5C:1D:BD:26:96:5F:FF:DB:D3:E2:15 + X509v3 Authority Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Key Usage: critical + Digital Signature + X509v3 Extended Key Usage: critical + OCSP Signing + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:28888/intermediate2_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:28888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 6c:d6:fa:8f:6f:c9:0a:99:0b:ee:6c:27:1f:75:52:b8:82:33: + 41:fe:01:a1:f8:c5:24:4e:9e:3b:e2:89:0f:01:2b:8e:c4:76: + fb:d9:75:5a:b2:9c:e0:36:8d:fd:90:9f:28:92:1b:a3:74:fd: + c5:39:28:51:06:ab:95:f7:64:95:e8:7b:d9:97:35:33:97:05: + 38:87:e6:e6:d7:a5:0b:a1:11:0c:b7:8b:76:b8:a9:46:33:ba: + 50:b3:3b:96:90:65:4b:ea:14:20:c9:f7:0d:8d:5e:89:c6:78: + e3:0b:4f:d2:db:10:46:8a:c4:81:6f:20:13:30:83:a8:45:4d: + 2b:ef:f0:ce:18:a7:96:fc:b9:67:79:e9:a9:f0:2f:b2:33:1c: + 83:cf:a3:4b:df:fd:c5:58:ae:87:83:d9:be:22:85:58:41:f5: + a0:a2:2d:56:98:40:12:78:c5:43:b0:50:34:0f:6c:0b:52:ad: + 68:e1:7a:9e:c1:54:58:bf:b4:f1:c5:3b:bf:97:e4:f9:44:09: + f5:c7:67:7d:dc:3d:ea:a9:9f:0f:3a:aa:9c:4a:c1:ef:a1:52: + 25:e4:57:22:d6:af:c6:c9:c8:02:91:4b:ec:a2:d6:ba:b5:bf: + ed:22:7c:b2:71:6c:78:f4:ba:e4:b9:b7:1f:11:65:d4:4f:77: + 4d:ef:b5:43 +-----BEGIN CERTIFICATE----- +MIIELzCCAxegAwIBAgIUe5c1cysrX3TGQ4OPrmVboPX0/x8wDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe +Fw0yMzA1MDExOTI5MjhaFw0zMzA0MjgxOTI5MjhaMFkxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEZ +MBcGA1UEAwwQT0NTUCBSZXNwb25kZXIgMjCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBALiYPQNNXrJmXlE7+T3yeiRrcFwvegWyUXdiReczdXfbMW8tEzLN +06ADhO75K4Gd5cm64iXJpxgr/fGVrdNGkNl7fzkthbRwfHJEmfvfnyJMgXc1u/5B +f4b1xylTfO7UzAlU+syxTUvCx8c+GhNZZjYxrmAbagWwW2SWd510zEJuE9Ehg5SO +bEzYQleUF/8m1NEvZFi1RxoiOGm/wFqcw4gBCh332GiIfFddRMRx0GaNHDngr+j3 +zlFgfB231ee1PmqlK0bDTrnv3r2mvuJmealqDcGy514Dnd7dQbnJgCy9bR8JX04l +56z/I0ePX3RpvoFCXOYa92Uf66HQaW++fokCAwEAAaOB7TCB6jAdBgNVHQ4EFgQU +5E3uaqMwkTc+XB29JpZf/9vT4hUwHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyi +V3ftFawwDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwFgYDVR0lAQH/BAww +CgYIKwYBBQUHAwkwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDovLzEyNy4wLjAuMToy +ODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYIKwYBBQUHAQEEJzAlMCMGCCsG +AQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4LzANBgkqhkiG9w0BAQsFAAOC +AQEAbNb6j2/JCpkL7mwnH3VSuIIzQf4BofjFJE6eO+KJDwErjsR2+9l1WrKc4DaN +/ZCfKJIbo3T9xTkoUQarlfdkleh72Zc1M5cFOIfm5telC6ERDLeLdripRjO6ULM7 +lpBlS+oUIMn3DY1eicZ44wtP0tsQRorEgW8gEzCDqEVNK+/wzhinlvy5Z3npqfAv +sjMcg8+jS9/9xViuh4PZviKFWEH1oKItVphAEnjFQ7BQNA9sC1KtaOF6nsFUWL+0 +8cU7v5fk+UQJ9cdnfdw96qmfDzqqnErB76FSJeRXItavxsnIApFL7KLWurW/7SJ8 +snFsePS65Lm3HxFl1E93Te+1Qw== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/ocsp2/private/ocsp2_keypair.pem b/test/configs/certs/ocsp_peer/mini-ca/ocsp2/private/ocsp2_keypair.pem new file mode 100644 index 0000000..ad13f6f --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/ocsp2/private/ocsp2_keypair.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC4mD0DTV6yZl5R +O/k98noka3BcL3oFslF3YkXnM3V32zFvLRMyzdOgA4Tu+SuBneXJuuIlyacYK/3x +la3TRpDZe385LYW0cHxyRJn7358iTIF3Nbv+QX+G9ccpU3zu1MwJVPrMsU1LwsfH +PhoTWWY2Ma5gG2oFsFtklneddMxCbhPRIYOUjmxM2EJXlBf/JtTRL2RYtUcaIjhp +v8BanMOIAQod99hoiHxXXUTEcdBmjRw54K/o985RYHwdt9XntT5qpStGw0657969 +pr7iZnmpag3BsudeA53e3UG5yYAsvW0fCV9OJees/yNHj190ab6BQlzmGvdlH+uh +0Glvvn6JAgMBAAECggEAIlCyruV4ICPljqZefASSbijG12w/+8UdXdsX8ZXgVWqa +8vbnJb+bgpiE4sPRMaQ/rlOebLXi6RxsdbeEe80XakaJ7QAoZdWvXLKiCW+VrpOY +UafcjbRxV45i+qy5gdBvKaDxipG/M8E+0CwcPtKUrKhpqRYPjIUvSDCshcnLmuF3 +zztB/4VyVEUUaM0pEqSZhxSyraRmGARvF1iOSu1npe3AzWTrrjrSkbk6fi4GyECL +If0EQ1ZD+ZXQ6tcGDyNtmPox7lPMZOgwLJZ5zISXZ6QBjn0JvSzE+e4z0IFinLgx +q5yBz2BhJEN8OBcs3J2N/ivQetWil64YbrbK6WbocQKBgQD/b4uHOuJVVifjIf6/ +kJ0UHhki4Q2Fj164royDigyyzaZmMzrlReZ5rAQLk8wGqw2hI+9gYoYBYqHm71kd +WrwLS1TVZJ6x8TBh0sYOG2CPndqIjWFx9Wjjf1xNknwYdIoEdAAKZ/M1E71V0tZb ++Ampl+lHPnKqYRSCd7gbYBU/TQKBgQC5AKGJusjRRRRWQqQ0RdJuxusZrtAAUd7l +wOGMC0zVQSSvUuegFtWEaZUbByhCARtYp8o4rT6Fw9yOvMaMNcfd8tV5nYVHDsrw +MurPhPitgI0/LdVvkAOO4fgPZHIXV9GbUDGq4uqB61daBSLQg1JjtzG8GvlGiYZl +mKOWEXjWLQKBgQC3nHHaehxZpT20yin5f7U50czVwppaqE05Sdcdcq1gFe2Hx0mN +pypdyaV6wPnGzUxVyaP3T7rt4f1pKCGRtTg4kiTf450jYbEakEzntQw7EAgXYjFq +njKQXWt3I1XqqlLPkqa41DIBtDfEKnMF1wzzCIyaNqxsBq6cffwsSWvcfQKBgF/y +UNUCd0X5Yqu+EjU+BP4I0kNWo2+XBlf36cHc1nM/Psxi3dfsH751l6wV0S4yLsGS ++9DbILL1On0YsIxlFAwq9cYGCOoqZNugPKF1oBcztY2PssMSWJYQ4brx6C3tELtR +IwEygFby/DGmukCT6vXmO7gH8UJA7t/gAu9Ajn/dAoGAI/Ejqb7HborIvCw/p+kB +JkPIhTUuT5XonDm8h6KHWUESPikS7SMeRM/4V+AL/Y5MiiCBfjh3tCOup/16x6GQ +4z6FvcIaYusxKup+afQaDyv1Phv5/mr74liLhC5Qp9EGU2FZrMZwG3EZjSn/0IE+ +dBJeWNtNHiFPcyTzYMMhDBw= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/root/private/root_keypair.pem b/test/configs/certs/ocsp_peer/mini-ca/root/private/root_keypair.pem new file mode 100644 index 0000000..dd6f2fb --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/root/private/root_keypair.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDiIWuf70i53iL7 +WzcJaMe1kldSJO+FAOhxhU0PW4zG508Z9uMLcKNBfnHUD9b98hrKqleRdpqygmJg +zvIALtS8WNNgMEKmKLJQe1gBn/sKZbBA1nzit9qNGdmlUdJGfhRGq/rfzv6ECJhj +Rh1NindXZ9oWizIMfEHipezufSAo6wNf9eYF2IuWeG+uKZpQ99yWMYaBsXjo6+9d +u+1C7JTGVEbsBW8bDDYkxqgGflxWuEM7EfQGCgUVGTsfyGcx6ztbKhUKe/lr5BDu +RL4Z2NtEAfo6VvVsTvNgquTNsq13B0Xv8df1+lKEXANOcuCpkcXZ1gqEM5gx8gJb +PxAVZXbXAgMBAAECggEAW6YC6i+/cIFs+SW3cStT4a29kU/h+axsCPJnUIWg0U6X +WyUaUR0mNZmrRbDjyEmS/Te7xPtmaFn6yFSndVaFpw5zIQV+RbyxxHexK/tscgLT +w/uKYxLz04M6GExIpoRb8Gash3/r3JRlOrsEjlRD2RuAoulob/H+e/8Wv3PcEGio +R8jwCj5DEnWiMxDzgtxsVgR4OeRYqg3zKjWrLALEYoRbFTVncCVA40OnmGJZ3+E5 ++3OOX6p9y/nY36888345yuwiCOTdNwQVaCXnLDZlAIVpB8QmjXVB35RSs+r2H5SF +p/KRbZ/JNKdNrbTKfJyvbnIpyTAtJB9OkhyiR9AegQKBgQDkKAplyZ6ChT3l53nn +4ngFi/nSTfrfJepmA5lVJk1Wxk0a4W++HxJkdKY2sUP7WuQ1xaPdcHxKzfp2HQE5 +L95jObU5dtY64QD4q0xqOw1ISDQi1euqZEmZziupEgPcMtw4sAVhHohzvTWo6a8o +fGMSkLTd+2303xgBCZo2I/hZVwKBgQD9uha6pQmCg4Oi2i/38Vm3ByArnQvgkELC +eGBBJrCE8pSm+ToPtgL+gSuA8PlWzIYlIf0UVOJFyws8GkUFm6n0nUlN0NmK8Rhm +Bg4IvasxdRgtySJzZO7ipAqGIaWJIBi1Vj4/rnAVggkadbQgyw+eCZNc5Pg3D9MV +TJ7d/xHegQKBgQCprGVfITuyUSihKy3rlu4vIdPd5IQnI2lYCGElg+CMIdkBnpmd +SDpDXsSlc9rcuNFyc9LTQW4Nq3USFavtPX4jSK1PWOMk0mQIiku/zL6p/JhZN8GU +7BQYP80UZQNd5K0Fs1Gs0ioj+JhJT9AlSavcCKWZV/yD2M1fKCb5EHMG7QKBgQDV +SvtSeeytp8sgOtU6VMz7fOUBZOsYI43Ll5ArFNAtYxOt7jNuA68urf2ZTnn9Cr/2 +NUVgMx9oVpEiPF8roLlV5mc6IEjQcW72TT69AF0KnYnu63enlADxy78BFQXoaW/7 ++P0pYYXdvsvST4JWUv3U9+3GmMFE4GutKxUeQA+QgQKBgQCauejVixhfKcmkM9nn +MGLSOUuFyd9HpQk3efxylphFNjpohk+k3fVKXBhmE4BDXbSlYUmMemm27tuQ/I6Z +bWOjGl57ZbCgJ7LdXLanJhyJJ6cSmkX8+oD+fwPMrD8yaAfh37MdTnriZKIDMXp2 +7HtfLcz0evmbW06b/dReyvcqyQ== +-----END PRIVATE KEY----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/root/root_cert.pem b/test/configs/certs/ocsp_peer/mini-ca/root/root_cert.pem new file mode 100644 index 0000000..f4658e1 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/root/root_cert.pem @@ -0,0 +1,86 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 18:57:57 2023 GMT + Not After : Apr 28 18:57:57 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68: + c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b: + 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f: + d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce: + f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b: + 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19: + d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08: + 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c: + 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8: + 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1: + 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05: + 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11: + f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a: + 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01: + fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07: + 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91: + c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65: + 76:d7 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5: + ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60: + d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59: + 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed: + 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a: + f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d: + 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7: + 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28: + 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58: + af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f: + 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95: + 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95: + 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82: + d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8: + dc:29:ac:17 +-----BEGIN CERTIFICATE----- +MIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4 +NTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS +b290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i ++1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi +YM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY +Y0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv +XbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ +7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC +Wz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220 +4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD +AQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3 +LjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5 +Xfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr +UFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I +4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y +VPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb +hpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp +MElNbuoFHtjcKawX +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem b/test/configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem new file mode 100644 index 0000000..544e3d4 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem @@ -0,0 +1,186 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3c:c4:82:66:f8:5d:a6:b6:c7:66:e1:b2:01:3f:e0:72:fc:72:61:33 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Validity + Not Before: May 1 19:33:37 2023 GMT + Not After : Apr 28 19:33:37 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:af:26:5c:50:c0:fa:62:b5:fd:3d:c1:9e:26:51: + 58:62:04:37:b0:b5:6a:9b:6a:e3:22:3c:cd:ee:3c: + e7:8b:d3:e2:4c:08:1a:4d:63:c1:81:20:f4:53:a5: + 5d:2f:d2:71:d8:af:e3:26:95:b4:27:14:46:7f:e2: + 0a:73:12:a7:0e:ff:99:5a:29:f5:d0:65:96:b1:d1: + 96:7f:0c:43:b8:71:f2:4b:21:e1:97:6c:1b:01:e5: + 38:1a:39:44:72:d5:19:20:87:fe:90:4f:3b:97:f2: + 7d:bd:57:97:4d:9d:56:50:89:5b:79:29:7a:3a:13: + 97:08:61:c2:0c:a6:02:49:c9:8a:41:ab:8e:9f:25: + c9:33:18:f8:92:64:58:04:cc:a3:9d:cf:d4:d2:bd: + 20:ab:8b:9d:55:df:fb:5b:23:ac:95:12:fa:6f:07: + 93:3f:0e:03:86:c4:9b:25:06:21:9b:03:96:32:b8: + e0:0f:63:e2:1d:34:d1:41:35:19:09:c1:a0:dc:26: + b9:c8:66:fa:87:67:22:6e:0c:a6:e7:0f:24:64:b1: + 4f:84:05:ef:ad:8e:1b:f2:f4:38:87:d3:e3:48:a5: + 82:e0:66:89:1d:92:9a:59:67:a4:1d:03:6f:4d:a5: + fb:3b:c0:0b:73:a7:ab:8f:b4:10:25:8e:69:42:76: + 82:5f + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 43:16:E6:03:AF:37:B2:7B:BD:B3:C8:A2:9C:95:D7:FA:32:F8:9E:6F + X509v3 Authority Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, SSL Server + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:18888/intermediate1_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:18888/ + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + a3:87:9f:05:e4:38:61:f7:c4:5b:17:13:4b:2c:9d:a2:4d:e6: + ad:93:54:c5:a3:00:27:0b:5c:45:c5:bd:f8:b6:a7:5a:2a:ec: + dc:9b:59:8a:c7:59:e7:b9:86:f7:27:be:45:0d:d9:86:76:cf: + 00:71:ad:aa:cc:73:50:8c:68:63:b0:e2:3a:59:dd:85:fa:0d: + f0:82:51:05:79:e6:d5:0e:0b:bb:ed:23:65:8f:d0:8b:01:df: + 86:74:bc:3a:22:90:e4:59:44:91:d5:44:d8:21:4d:4e:10:72: + 0a:12:2e:4a:20:5f:15:e7:16:0b:6f:76:f3:04:1f:da:44:50: + 3b:c3:b3:0f:fa:05:cf:6e:64:9c:65:e2:0d:38:28:31:c3:c3: + b6:66:ef:80:d3:c4:5f:e9:f9:01:e9:ce:e6:99:46:a0:9d:ce: + 90:63:77:d2:85:21:d7:88:32:55:38:fe:10:07:69:cd:c8:06: + b7:6f:49:98:bf:cd:be:4f:ab:44:ea:78:af:ab:01:c8:3e:fa: + d9:54:bc:59:28:db:03:9b:1c:ee:e4:c3:ed:f3:97:30:c6:40: + 33:76:84:40:b2:b8:4d:b4:ca:a9:2d:d1:4d:17:92:ea:c0:c9: + cb:f6:b1:d7:d3:c7:e6:75:15:00:ff:c7:d9:54:63:27:19:5c: + 96:a5:e5:d9 +-----BEGIN CERTIFICATE----- +MIIEYjCCA0qgAwIBAgIUPMSCZvhdprbHZuGyAT/gcvxyYTMwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe +Fw0yMzA1MDExOTMzMzdaFw0zMzA0MjgxOTMzMzdaMFQxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU +MBIGA1UEAwwLVGVzdFNlcnZlcjEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQCvJlxQwPpitf09wZ4mUVhiBDewtWqbauMiPM3uPOeL0+JMCBpNY8GBIPRT +pV0v0nHYr+MmlbQnFEZ/4gpzEqcO/5laKfXQZZax0ZZ/DEO4cfJLIeGXbBsB5Tga +OURy1Rkgh/6QTzuX8n29V5dNnVZQiVt5KXo6E5cIYcIMpgJJyYpBq46fJckzGPiS +ZFgEzKOdz9TSvSCri51V3/tbI6yVEvpvB5M/DgOGxJslBiGbA5YyuOAPY+IdNNFB +NRkJwaDcJrnIZvqHZyJuDKbnDyRksU+EBe+tjhvy9DiH0+NIpYLgZokdkppZZ6Qd +A29Npfs7wAtzp6uPtBAljmlCdoJfAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUQxbm +A683snu9s8iinJXX+jL4nm8wHwYDVR0jBBgwFoAUtZFuT2S3FoR2+bS+mc5glZga +jp0wDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD +AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg +MKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ludGVybWVkaWF0ZTFfY3JsLmRl +cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 +MTg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF +AAOCAQEAo4efBeQ4YffEWxcTSyydok3mrZNUxaMAJwtcRcW9+LanWirs3JtZisdZ +57mG9ye+RQ3ZhnbPAHGtqsxzUIxoY7DiOlndhfoN8IJRBXnm1Q4Lu+0jZY/QiwHf +hnS8OiKQ5FlEkdVE2CFNThByChIuSiBfFecWC2928wQf2kRQO8OzD/oFz25knGXi +DTgoMcPDtmbvgNPEX+n5AenO5plGoJ3OkGN30oUh14gyVTj+EAdpzcgGt29JmL/N +vk+rROp4r6sByD762VS8WSjbA5sc7uTD7fOXMMZAM3aEQLK4TbTKqS3RTReS6sDJ +y/ax19PH5nUVAP/H2VRjJxlclqXl2Q== +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:15 2023 GMT + Not After : Apr 28 19:01:15 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: + d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: + 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: + b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: + a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: + 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: + d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: + 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: + 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: + 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: + 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: + 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: + a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: + c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: + 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: + 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: + 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: + 40:27 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: + a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: + d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: + 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: + 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: + c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: + 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: + 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: + 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: + be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: + 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: + 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: + ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: + 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: + 91:5c:43:a9 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC +YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL +3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy +n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu +hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX +Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 ++bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp +LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr +uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN +zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg +zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX +KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem b/test/configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem new file mode 100644 index 0000000..ef73af8 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem @@ -0,0 +1,97 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3c:c4:82:66:f8:5d:a6:b6:c7:66:e1:b2:01:3f:e0:72:fc:72:61:33 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Validity + Not Before: May 1 19:33:37 2023 GMT + Not After : Apr 28 19:33:37 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:af:26:5c:50:c0:fa:62:b5:fd:3d:c1:9e:26:51: + 58:62:04:37:b0:b5:6a:9b:6a:e3:22:3c:cd:ee:3c: + e7:8b:d3:e2:4c:08:1a:4d:63:c1:81:20:f4:53:a5: + 5d:2f:d2:71:d8:af:e3:26:95:b4:27:14:46:7f:e2: + 0a:73:12:a7:0e:ff:99:5a:29:f5:d0:65:96:b1:d1: + 96:7f:0c:43:b8:71:f2:4b:21:e1:97:6c:1b:01:e5: + 38:1a:39:44:72:d5:19:20:87:fe:90:4f:3b:97:f2: + 7d:bd:57:97:4d:9d:56:50:89:5b:79:29:7a:3a:13: + 97:08:61:c2:0c:a6:02:49:c9:8a:41:ab:8e:9f:25: + c9:33:18:f8:92:64:58:04:cc:a3:9d:cf:d4:d2:bd: + 20:ab:8b:9d:55:df:fb:5b:23:ac:95:12:fa:6f:07: + 93:3f:0e:03:86:c4:9b:25:06:21:9b:03:96:32:b8: + e0:0f:63:e2:1d:34:d1:41:35:19:09:c1:a0:dc:26: + b9:c8:66:fa:87:67:22:6e:0c:a6:e7:0f:24:64:b1: + 4f:84:05:ef:ad:8e:1b:f2:f4:38:87:d3:e3:48:a5: + 82:e0:66:89:1d:92:9a:59:67:a4:1d:03:6f:4d:a5: + fb:3b:c0:0b:73:a7:ab:8f:b4:10:25:8e:69:42:76: + 82:5f + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 43:16:E6:03:AF:37:B2:7B:BD:B3:C8:A2:9C:95:D7:FA:32:F8:9E:6F + X509v3 Authority Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, SSL Server + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:18888/intermediate1_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:18888/ + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + a3:87:9f:05:e4:38:61:f7:c4:5b:17:13:4b:2c:9d:a2:4d:e6: + ad:93:54:c5:a3:00:27:0b:5c:45:c5:bd:f8:b6:a7:5a:2a:ec: + dc:9b:59:8a:c7:59:e7:b9:86:f7:27:be:45:0d:d9:86:76:cf: + 00:71:ad:aa:cc:73:50:8c:68:63:b0:e2:3a:59:dd:85:fa:0d: + f0:82:51:05:79:e6:d5:0e:0b:bb:ed:23:65:8f:d0:8b:01:df: + 86:74:bc:3a:22:90:e4:59:44:91:d5:44:d8:21:4d:4e:10:72: + 0a:12:2e:4a:20:5f:15:e7:16:0b:6f:76:f3:04:1f:da:44:50: + 3b:c3:b3:0f:fa:05:cf:6e:64:9c:65:e2:0d:38:28:31:c3:c3: + b6:66:ef:80:d3:c4:5f:e9:f9:01:e9:ce:e6:99:46:a0:9d:ce: + 90:63:77:d2:85:21:d7:88:32:55:38:fe:10:07:69:cd:c8:06: + b7:6f:49:98:bf:cd:be:4f:ab:44:ea:78:af:ab:01:c8:3e:fa: + d9:54:bc:59:28:db:03:9b:1c:ee:e4:c3:ed:f3:97:30:c6:40: + 33:76:84:40:b2:b8:4d:b4:ca:a9:2d:d1:4d:17:92:ea:c0:c9: + cb:f6:b1:d7:d3:c7:e6:75:15:00:ff:c7:d9:54:63:27:19:5c: + 96:a5:e5:d9 +-----BEGIN CERTIFICATE----- +MIIEYjCCA0qgAwIBAgIUPMSCZvhdprbHZuGyAT/gcvxyYTMwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe +Fw0yMzA1MDExOTMzMzdaFw0zMzA0MjgxOTMzMzdaMFQxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU +MBIGA1UEAwwLVGVzdFNlcnZlcjEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQCvJlxQwPpitf09wZ4mUVhiBDewtWqbauMiPM3uPOeL0+JMCBpNY8GBIPRT +pV0v0nHYr+MmlbQnFEZ/4gpzEqcO/5laKfXQZZax0ZZ/DEO4cfJLIeGXbBsB5Tga +OURy1Rkgh/6QTzuX8n29V5dNnVZQiVt5KXo6E5cIYcIMpgJJyYpBq46fJckzGPiS +ZFgEzKOdz9TSvSCri51V3/tbI6yVEvpvB5M/DgOGxJslBiGbA5YyuOAPY+IdNNFB +NRkJwaDcJrnIZvqHZyJuDKbnDyRksU+EBe+tjhvy9DiH0+NIpYLgZokdkppZZ6Qd +A29Npfs7wAtzp6uPtBAljmlCdoJfAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUQxbm +A683snu9s8iinJXX+jL4nm8wHwYDVR0jBBgwFoAUtZFuT2S3FoR2+bS+mc5glZga +jp0wDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD +AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg +MKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ludGVybWVkaWF0ZTFfY3JsLmRl +cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 +MTg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF +AAOCAQEAo4efBeQ4YffEWxcTSyydok3mrZNUxaMAJwtcRcW9+LanWirs3JtZisdZ +57mG9ye+RQ3ZhnbPAHGtqsxzUIxoY7DiOlndhfoN8IJRBXnm1Q4Lu+0jZY/QiwHf +hnS8OiKQ5FlEkdVE2CFNThByChIuSiBfFecWC2928wQf2kRQO8OzD/oFz25knGXi +DTgoMcPDtmbvgNPEX+n5AenO5plGoJ3OkGN30oUh14gyVTj+EAdpzcgGt29JmL/N +vk+rROp4r6sByD762VS8WSjbA5sc7uTD7fOXMMZAM3aEQLK4TbTKqS3RTReS6sDJ +y/ax19PH5nUVAP/H2VRjJxlclqXl2Q== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem b/test/configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem new file mode 100644 index 0000000..aacb9be --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem @@ -0,0 +1,186 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 2e:91:da:29:59:ff:c4:64:bf:02:bc:27:bb:e3:35:4e:5b:36:f7:91 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Validity + Not Before: May 1 19:33:53 2023 GMT + Not After : Apr 28 19:33:53 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:ac:48:ce:a7:b2:ad:7a:68:01:55:3f:86:20:7e: + bb:26:e6:88:f3:ae:04:15:7d:d9:64:98:85:bc:eb: + bd:d8:0a:c7:26:c4:8e:27:56:8c:a8:9f:51:37:a9: + ec:8a:dc:af:27:05:0c:f5:c0:19:b1:2c:0d:56:66: + 7b:7e:b1:8f:ab:34:61:56:37:a8:ab:51:d6:1d:e6: + a7:56:b2:51:72:57:9b:c5:87:84:6c:ef:e6:18:d4: + 45:b8:ef:52:72:11:02:81:61:f2:36:63:25:18:31: + 7f:c7:91:89:c3:b0:73:13:f0:26:1f:a1:4f:8c:ff: + 94:1c:75:a6:be:38:7d:81:06:33:dd:7b:86:81:c5: + 1f:d2:5d:f6:ea:3f:9f:ab:fb:e7:97:3c:72:ea:b3: + 83:ab:49:88:ac:a9:4b:81:db:fa:e3:bf:79:d9:6e: + 90:bf:8f:68:d8:05:f8:52:ad:98:41:29:e0:2a:18: + 98:b6:b2:61:78:02:02:52:85:02:e0:63:f4:a0:55: + 80:c9:66:8b:ac:4f:8b:36:f4:56:8f:cf:bd:67:86: + 72:92:0b:f9:73:7b:05:cc:3d:91:ed:ed:4f:f0:8f: + 36:99:e5:51:7f:ee:9e:fb:e5:5c:d0:39:a2:f5:51: + 06:92:3c:ad:cc:59:9d:0a:81:50:26:30:01:e9:f4: + b1:e9 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + CD:65:B9:5C:48:35:F7:1E:85:6E:94:50:78:72:BB:3F:F7:BC:22:A6 + X509v3 Authority Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, SSL Server + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:18888/intermediate1_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:18888/ + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 6f:de:f3:92:b2:8b:57:61:7a:b9:06:49:3e:af:e0:1c:3a:d4: + 42:52:fe:d0:7d:97:8a:9b:d0:6d:b9:f3:e6:8b:2a:40:ce:aa: + ed:bb:ce:21:e8:ae:32:9d:eb:5a:00:e0:c1:3a:d7:40:74:1b: + 43:e4:43:f0:61:bf:40:06:75:52:1b:b9:f4:b5:32:55:94:f5: + 84:98:90:cc:27:92:91:b7:3d:8e:f1:12:bf:37:1a:8a:50:41: + 3a:14:0c:cf:93:fe:57:97:7b:fe:af:b9:c0:c2:d6:bb:20:e4: + 0a:6f:12:0b:60:a6:cc:59:46:db:99:db:61:71:d3:a7:f5:a1: + d0:d6:81:87:57:a3:dd:b6:e1:ab:2f:4f:b6:51:21:ec:a6:95: + df:d3:ab:e5:a1:67:a3:ba:b1:b9:71:39:a1:3b:db:5e:c5:6f: + b1:34:27:ae:6d:f6:67:4c:7d:7c:6d:12:37:6f:b5:0b:5a:85: + aa:5d:fd:03:de:59:b5:20:7a:ea:84:a0:a5:75:60:12:12:08: + 77:0e:46:d6:fa:57:fa:b1:43:42:54:38:d7:66:67:cd:fc:b6: + f9:4c:fe:99:71:2b:d5:a6:13:2f:2e:f0:a3:9e:fc:47:03:31: + 79:38:e3:50:8a:de:81:97:80:9e:46:71:5c:9f:e5:de:0c:49: + fc:f5:61:1c +-----BEGIN CERTIFICATE----- +MIIEYjCCA0qgAwIBAgIULpHaKVn/xGS/Arwnu+M1Tls295EwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe +Fw0yMzA1MDExOTMzNTNaFw0zMzA0MjgxOTMzNTNaMFQxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU +MBIGA1UEAwwLVGVzdFNlcnZlcjIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQCsSM6nsq16aAFVP4Ygfrsm5ojzrgQVfdlkmIW8673YCscmxI4nVoyon1E3 +qeyK3K8nBQz1wBmxLA1WZnt+sY+rNGFWN6irUdYd5qdWslFyV5vFh4Rs7+YY1EW4 +71JyEQKBYfI2YyUYMX/HkYnDsHMT8CYfoU+M/5Qcdaa+OH2BBjPde4aBxR/SXfbq +P5+r++eXPHLqs4OrSYisqUuB2/rjv3nZbpC/j2jYBfhSrZhBKeAqGJi2smF4AgJS +hQLgY/SgVYDJZousT4s29FaPz71nhnKSC/lzewXMPZHt7U/wjzaZ5VF/7p775VzQ +OaL1UQaSPK3MWZ0KgVAmMAHp9LHpAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUzWW5 +XEg19x6FbpRQeHK7P/e8IqYwHwYDVR0jBBgwFoAUtZFuT2S3FoR2+bS+mc5glZga +jp0wDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD +AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg +MKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ludGVybWVkaWF0ZTFfY3JsLmRl +cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 +MTg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF +AAOCAQEAb97zkrKLV2F6uQZJPq/gHDrUQlL+0H2XipvQbbnz5osqQM6q7bvOIeiu +Mp3rWgDgwTrXQHQbQ+RD8GG/QAZ1Uhu59LUyVZT1hJiQzCeSkbc9jvESvzcailBB +OhQMz5P+V5d7/q+5wMLWuyDkCm8SC2CmzFlG25nbYXHTp/Wh0NaBh1ej3bbhqy9P +tlEh7KaV39Or5aFno7qxuXE5oTvbXsVvsTQnrm32Z0x9fG0SN2+1C1qFql39A95Z +tSB66oSgpXVgEhIIdw5G1vpX+rFDQlQ412Znzfy2+Uz+mXEr1aYTLy7wo578RwMx +eTjjUIregZeAnkZxXJ/l3gxJ/PVhHA== +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:15 2023 GMT + Not After : Apr 28 19:01:15 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: + d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: + 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: + b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: + a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: + 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: + d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: + 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: + 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: + 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: + 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: + 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: + a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: + c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: + 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: + 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: + 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: + 40:27 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: + a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: + d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: + 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: + 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: + c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: + 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: + 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: + 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: + be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: + 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: + 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: + ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: + 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: + 91:5c:43:a9 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC +YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL +3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy +n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu +hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX +Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 ++bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp +LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr +uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN +zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg +zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX +KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/server1/TestServer2_cert.pem b/test/configs/certs/ocsp_peer/mini-ca/server1/TestServer2_cert.pem new file mode 100644 index 0000000..91ddf56 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/server1/TestServer2_cert.pem @@ -0,0 +1,97 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 2e:91:da:29:59:ff:c4:64:bf:02:bc:27:bb:e3:35:4e:5b:36:f7:91 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 + Validity + Not Before: May 1 19:33:53 2023 GMT + Not After : Apr 28 19:33:53 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:ac:48:ce:a7:b2:ad:7a:68:01:55:3f:86:20:7e: + bb:26:e6:88:f3:ae:04:15:7d:d9:64:98:85:bc:eb: + bd:d8:0a:c7:26:c4:8e:27:56:8c:a8:9f:51:37:a9: + ec:8a:dc:af:27:05:0c:f5:c0:19:b1:2c:0d:56:66: + 7b:7e:b1:8f:ab:34:61:56:37:a8:ab:51:d6:1d:e6: + a7:56:b2:51:72:57:9b:c5:87:84:6c:ef:e6:18:d4: + 45:b8:ef:52:72:11:02:81:61:f2:36:63:25:18:31: + 7f:c7:91:89:c3:b0:73:13:f0:26:1f:a1:4f:8c:ff: + 94:1c:75:a6:be:38:7d:81:06:33:dd:7b:86:81:c5: + 1f:d2:5d:f6:ea:3f:9f:ab:fb:e7:97:3c:72:ea:b3: + 83:ab:49:88:ac:a9:4b:81:db:fa:e3:bf:79:d9:6e: + 90:bf:8f:68:d8:05:f8:52:ad:98:41:29:e0:2a:18: + 98:b6:b2:61:78:02:02:52:85:02:e0:63:f4:a0:55: + 80:c9:66:8b:ac:4f:8b:36:f4:56:8f:cf:bd:67:86: + 72:92:0b:f9:73:7b:05:cc:3d:91:ed:ed:4f:f0:8f: + 36:99:e5:51:7f:ee:9e:fb:e5:5c:d0:39:a2:f5:51: + 06:92:3c:ad:cc:59:9d:0a:81:50:26:30:01:e9:f4: + b1:e9 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + CD:65:B9:5C:48:35:F7:1E:85:6E:94:50:78:72:BB:3F:F7:BC:22:A6 + X509v3 Authority Key Identifier: + B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, SSL Server + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:18888/intermediate1_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:18888/ + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 6f:de:f3:92:b2:8b:57:61:7a:b9:06:49:3e:af:e0:1c:3a:d4: + 42:52:fe:d0:7d:97:8a:9b:d0:6d:b9:f3:e6:8b:2a:40:ce:aa: + ed:bb:ce:21:e8:ae:32:9d:eb:5a:00:e0:c1:3a:d7:40:74:1b: + 43:e4:43:f0:61:bf:40:06:75:52:1b:b9:f4:b5:32:55:94:f5: + 84:98:90:cc:27:92:91:b7:3d:8e:f1:12:bf:37:1a:8a:50:41: + 3a:14:0c:cf:93:fe:57:97:7b:fe:af:b9:c0:c2:d6:bb:20:e4: + 0a:6f:12:0b:60:a6:cc:59:46:db:99:db:61:71:d3:a7:f5:a1: + d0:d6:81:87:57:a3:dd:b6:e1:ab:2f:4f:b6:51:21:ec:a6:95: + df:d3:ab:e5:a1:67:a3:ba:b1:b9:71:39:a1:3b:db:5e:c5:6f: + b1:34:27:ae:6d:f6:67:4c:7d:7c:6d:12:37:6f:b5:0b:5a:85: + aa:5d:fd:03:de:59:b5:20:7a:ea:84:a0:a5:75:60:12:12:08: + 77:0e:46:d6:fa:57:fa:b1:43:42:54:38:d7:66:67:cd:fc:b6: + f9:4c:fe:99:71:2b:d5:a6:13:2f:2e:f0:a3:9e:fc:47:03:31: + 79:38:e3:50:8a:de:81:97:80:9e:46:71:5c:9f:e5:de:0c:49: + fc:f5:61:1c +-----BEGIN CERTIFICATE----- +MIIEYjCCA0qgAwIBAgIULpHaKVn/xGS/Arwnu+M1Tls295EwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe +Fw0yMzA1MDExOTMzNTNaFw0zMzA0MjgxOTMzNTNaMFQxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU +MBIGA1UEAwwLVGVzdFNlcnZlcjIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQCsSM6nsq16aAFVP4Ygfrsm5ojzrgQVfdlkmIW8673YCscmxI4nVoyon1E3 +qeyK3K8nBQz1wBmxLA1WZnt+sY+rNGFWN6irUdYd5qdWslFyV5vFh4Rs7+YY1EW4 +71JyEQKBYfI2YyUYMX/HkYnDsHMT8CYfoU+M/5Qcdaa+OH2BBjPde4aBxR/SXfbq +P5+r++eXPHLqs4OrSYisqUuB2/rjv3nZbpC/j2jYBfhSrZhBKeAqGJi2smF4AgJS +hQLgY/SgVYDJZousT4s29FaPz71nhnKSC/lzewXMPZHt7U/wjzaZ5VF/7p775VzQ +OaL1UQaSPK3MWZ0KgVAmMAHp9LHpAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUzWW5 +XEg19x6FbpRQeHK7P/e8IqYwHwYDVR0jBBgwFoAUtZFuT2S3FoR2+bS+mc5glZga +jp0wDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD +AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg +MKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ludGVybWVkaWF0ZTFfY3JsLmRl +cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 +MTg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF +AAOCAQEAb97zkrKLV2F6uQZJPq/gHDrUQlL+0H2XipvQbbnz5osqQM6q7bvOIeiu +Mp3rWgDgwTrXQHQbQ+RD8GG/QAZ1Uhu59LUyVZT1hJiQzCeSkbc9jvESvzcailBB +OhQMz5P+V5d7/q+5wMLWuyDkCm8SC2CmzFlG25nbYXHTp/Wh0NaBh1ej3bbhqy9P +tlEh7KaV39Or5aFno7qxuXE5oTvbXsVvsTQnrm32Z0x9fG0SN2+1C1qFql39A95Z +tSB66oSgpXVgEhIIdw5G1vpX+rFDQlQ412Znzfy2+Uz+mXEr1aYTLy7wo578RwMx +eTjjUIregZeAnkZxXJ/l3gxJ/PVhHA== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem b/test/configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem new file mode 100644 index 0000000..2ea703d --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCvJlxQwPpitf09 +wZ4mUVhiBDewtWqbauMiPM3uPOeL0+JMCBpNY8GBIPRTpV0v0nHYr+MmlbQnFEZ/ +4gpzEqcO/5laKfXQZZax0ZZ/DEO4cfJLIeGXbBsB5TgaOURy1Rkgh/6QTzuX8n29 +V5dNnVZQiVt5KXo6E5cIYcIMpgJJyYpBq46fJckzGPiSZFgEzKOdz9TSvSCri51V +3/tbI6yVEvpvB5M/DgOGxJslBiGbA5YyuOAPY+IdNNFBNRkJwaDcJrnIZvqHZyJu +DKbnDyRksU+EBe+tjhvy9DiH0+NIpYLgZokdkppZZ6QdA29Npfs7wAtzp6uPtBAl +jmlCdoJfAgMBAAECggEAQLRoOEECfwMCehUUKs20XAl41WQ/7QiQvm4+GXwQgjyV +hkccCGkI7H5TJK+bfHY/LrDTtsZpVmKMJORJvfcvFkBg08lakVFmWWy3L1pFjlcy +DoWGxJzgYVPf5PgxDEcjUDxNU9yhhGHGB/Pa5oZwg7Iqw9kJ2XixPBx5RpjxkXYw +tR8V3IaKq0YRI5lpUfuaofmJnHJnWCMTmawWMxWuTlzlbDDZTHQs8aTDUnwZ26kD +6tYB2Tp3aP3zUE8MQZwOEyhRH1WQeS3kcIWh4UnPyA09g0aTb6YK8qacnTL2CixF +VJpLDtlkQk0TCo06AZkcvWkPTQyFXnVsgkG8rRUlEQKBgQDrTHyf6merJAohUeBV +5IIfoKHWbGc1DXSdmHtCSN9wFGkhCYtfCZ7YaSLjFF7GOvd6mfHJVnIp3aFONqM7 +dk/MZDsAvogO6lU+zgQc+EcKk+e6zyfsUYghy/R3+QKsYtd4SyNDq6cl80MUujjG +pE2b41O57sNCVZgywCCGXvt/ZwKBgQC+jyufgKRIptM+OOhHlKUaxkTDaMHA1KKY +iFPLuLgWmyCYHQq2D6uoCRGnEguEnXtbtOz6SYlMMNfeHtX0SATkdCGae/bh5ibG +uQoWwRMkRkAgl1gyAh7h669pDUiD2gh0q56cS8El7Jgze7NRF4hUyY2mWc5nGhVR +7rHKlOCiSQKBgHBiWevvg5BkaEo91w5vVA9TI7lMkYbvZFGZcNXaBI590TCsZFsC +N1JZ9QXMxu+bXnS6bpehqGmCp/a5dgGCot6WyO+0ETw+hHS45ZIIq7XLqxS4uPLQ +hlrOFXfwAWzg0NVt3ewGYpFnvRR7VX7bHw5j56uY9L4ML+OdjGthlnHlAoGAZAm7 +R/f7xtw1h7POVU22w3CUxtUm6jl2xobDHu7xTYTQvqp4Zg2h+wwPxVqWy171VLaN +tfOG7YWyvbwIbD6mutwwi+5KNFtjve2EW1+u0dtDbRimx1IPrmDRbF/50qZSzBUQ +plKqqmMjn9tvzsGA46oP/+WjksLBsIqTsZsotmkCgYAn8Ap+e6ZNX2uM8Kg7LB+T +hBNGczNOGQX8SpfCeH9eV4VzfpEHn8Fxk+lcI2WpYkandQ8ju2s0mT5OoQ2VjxGT +eql9jMd8MQZTx/aWridt5qG3hsFcx9GILlcXTUqyRH0SFAU7xDO5HzzKP3tiW6BN +YE3GakolPPymOR9q69sT0Q== +-----END PRIVATE KEY----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem b/test/configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem new file mode 100644 index 0000000..7b76a48 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCsSM6nsq16aAFV +P4Ygfrsm5ojzrgQVfdlkmIW8673YCscmxI4nVoyon1E3qeyK3K8nBQz1wBmxLA1W +Znt+sY+rNGFWN6irUdYd5qdWslFyV5vFh4Rs7+YY1EW471JyEQKBYfI2YyUYMX/H +kYnDsHMT8CYfoU+M/5Qcdaa+OH2BBjPde4aBxR/SXfbqP5+r++eXPHLqs4OrSYis +qUuB2/rjv3nZbpC/j2jYBfhSrZhBKeAqGJi2smF4AgJShQLgY/SgVYDJZousT4s2 +9FaPz71nhnKSC/lzewXMPZHt7U/wjzaZ5VF/7p775VzQOaL1UQaSPK3MWZ0KgVAm +MAHp9LHpAgMBAAECggEABbGGbngQX9Un+x8FQesSPgHnGM92wtY6J5Gwn2qhJy6M +VYwwFZ3Nz5pBPbrOY9SRGhPihrdixKOWgWppA8ne0WB4JC26HnGZnFAbAQRVqPbQ +duhd4ILpOpzpkh1K6b+vvU0addXpsUlHJjYZmdy+9tPBkhtwz1xDCFGShrguR0Pa +WTudsee4skdGfw6wMyHEfM4IXXuSfb1hIse1xlnZMPXMMi3ebCqpOy4IzJ4ML7sF +RySdrdAHcWJqOQjPkDTOPCXpthBn3iQ8Fa7Znd0GGLZvdRbq3p10H5LNhMg+LBc7 +oRVQ67qAfQKPHKQMSsR4x2fWo8/hw/QEi3cj6CohYQKBgQDtjDBm7VfbLojZeCMx ++32EZ0bLUTob5qInTKpDbdKcYmxP857LRAglaGu+pkOTnHi6lOjJYSiBDd1+vWm/ +1lgMUjKerI0l5ol5yRHWNDFyeQoh10TqEUbIUqB8E5Vi4gl0DlpnsfEm899rlfhP +dmi1rNpc/C7ZK8Zpt7l4eLbqYQKBgQC5qs+K01WwjtrjoqkEwKqjy7ASrbBuZ56u +wOe+iO7pYVP4/VdAvOsfEYCWfjhoETYGKob9ZZlo3StpQ5Ku5CigpWQVSCvJhO2T +KQe75DfXXxaqoPmlNcqAFpqY383Sm+1r3a815sg83XhQAu7GdCyTrLocBLM9SFWX +fVbojv/EiQKBgBlOpCFzC7cYIBA7ElTS3C5s6kfi4XPzgDb7nfANFTD/81YZOEOj +fdKuazwmbnCdbOdD0gESTsRg+8Xy2/9KEJtPboElFOyCwQauey385X+ykXfFfVwK +dyYEV4CgfXvJZQRuOwdtF6n0tUq68XdVwBYK0kCxxTPxy/ObVTEWezZBAoGAPPX2 +evB0vCnLeN5ZfHP+ExW31AovWbCwC1TPQmIXf40zUxdrZJgi4uqOO9tpjdHI2TFx +bRXEzwd/T2qeaMoFBOoI+Gvf5KS+lIjuPyTpqM9R0+hSz4nf2TqSvAsPu0zzIW2C +L8J8kG9vJ2YvG/3c/QfDe5uXdlGfuMOwm18IX3ECgYAelsVWNSm9YR2H7S6dlgjm +8U1IZO342Ab5qlrIWn9Vr/X9MRsAASlVVESeNeQTToBoub5pakUrpz9nNQy29+TX +xYju72RsCjKywKXWZrCAdHfY+wJJWVo5XkdDZJVl2AYrnP3C07S9aKIjhpGHwz7n +jbbCEkHZREMbQJCQjuKT1w== +-----END PRIVATE KEY----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/server2/TestServer3_bundle.pem b/test/configs/certs/ocsp_peer/mini-ca/server2/TestServer3_bundle.pem new file mode 100644 index 0000000..7a1ee48 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/server2/TestServer3_bundle.pem @@ -0,0 +1,186 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3e:1f:9b:cd:c8:7b:95:f1:64:e6:41:9c:df:6e:03:da:92:9a:90:b7 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Validity + Not Before: Aug 2 22:15:27 2023 GMT + Not After : Jul 30 22:15:27 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer3 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:9a:3c:db:76:c9:19:0f:7b:e6:d3:ed:d1:0b:76: + ae:15:d4:11:1c:66:b8:5d:2a:7d:e3:1f:65:d8:1b: + c4:63:62:f6:5c:8b:18:66:a8:1c:c2:a6:5e:72:f2: + dd:57:42:8a:ab:5d:bd:37:b6:f1:4b:51:f0:b3:6a: + 37:e9:55:78:01:23:ea:53:09:83:2f:7d:59:36:ab: + 33:4f:4c:bc:ef:a9:1c:db:94:79:4c:0d:4a:7c:3f: + 9d:3c:ba:6c:76:82:47:25:eb:79:22:f4:09:6c:78: + 3c:a6:ef:4b:30:90:29:b3:5f:ba:69:b1:1a:95:ed: + 53:e0:c6:24:78:6e:52:af:8e:bc:db:4a:f0:19:d2: + 00:5a:a8:b6:73:4c:17:92:d1:8d:81:9b:4c:b8:35: + 4d:91:dd:df:d3:85:a6:9f:c4:91:19:ec:47:d1:ca: + 4e:0b:c3:06:8c:27:42:95:83:e3:28:6a:3b:74:9c: + 68:b0:55:a5:91:91:cb:37:ad:fa:d8:69:8b:de:2e: + 4a:51:59:32:4b:3d:06:21:04:65:d2:f5:8b:e8:4d: + 45:96:de:63:97:47:81:85:ea:48:f0:9d:23:2d:71: + 87:6f:d2:75:3d:45:bf:de:ad:43:82:db:a5:29:9b: + f9:5e:38:0a:39:a9:38:71:ec:40:40:b5:dc:69:c7: + 0b:73 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 7F:47:8C:9E:F1:73:7E:34:B9:5B:1E:ED:AD:3A:87:42:80:D4:E3:FD + X509v3 Authority Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, SSL Server + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:28888/intermediate2_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:28888/ + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + b9:b4:05:48:a6:ba:6c:99:8b:23:c4:9b:b3:8a:32:3f:ca:62: + 89:81:1e:5d:04:ba:2d:22:a3:0f:5a:5d:a0:ab:40:a4:87:43: + 26:36:0a:09:64:ef:f5:b0:a7:6f:7a:1f:cc:06:6c:f7:8d:9c: + 64:5e:c2:ae:e7:45:39:dc:bc:87:06:e6:d5:aa:6b:32:76:51: + 64:e1:ac:d9:9a:dd:17:47:9b:4e:31:1c:93:f5:c5:ca:d6:b7: + 90:ff:64:97:59:df:2b:7f:ee:2d:7d:73:ef:95:ad:b5:1e:a9: + 0c:48:38:29:0b:39:4f:05:fb:07:cf:ec:94:a3:b3:d5:eb:00: + ed:b2:b9:71:a0:59:b5:3f:7c:f5:20:90:54:a8:ea:36:4c:ae: + 62:5b:2b:6d:05:8d:76:78:87:c9:90:f3:b2:d1:72:fc:87:f5: + 28:4c:ec:19:50:0f:02:32:d4:57:75:d9:c1:b2:dc:0e:d4:9a: + 3a:cd:48:70:1e:c4:2e:fd:4f:b0:89:6a:de:f0:90:91:23:16: + cd:04:fc:61:87:9c:c3:5c:7e:0f:19:ff:26:3e:fb:1b:65:2a: + 49:ae:47:9f:d5:e6:c8:30:bb:13:b9:48:d0:67:57:0f:fb:c6: + df:1c:fc:82:3b:ae:1f:f7:25:c8:df:c0:c5:d1:8d:51:94:74: + 30:be:fb:f7 +-----BEGIN CERTIFICATE----- +MIIEYjCCA0qgAwIBAgIUPh+bzch7lfFk5kGc324D2pKakLcwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe +Fw0yMzA4MDIyMjE1MjdaFw0zMzA3MzAyMjE1MjdaMFQxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU +MBIGA1UEAwwLVGVzdFNlcnZlcjMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQCaPNt2yRkPe+bT7dELdq4V1BEcZrhdKn3jH2XYG8RjYvZcixhmqBzCpl5y +8t1XQoqrXb03tvFLUfCzajfpVXgBI+pTCYMvfVk2qzNPTLzvqRzblHlMDUp8P508 +umx2gkcl63ki9AlseDym70swkCmzX7ppsRqV7VPgxiR4blKvjrzbSvAZ0gBaqLZz +TBeS0Y2Bm0y4NU2R3d/ThaafxJEZ7EfRyk4LwwaMJ0KVg+Moajt0nGiwVaWRkcs3 +rfrYaYveLkpRWTJLPQYhBGXS9YvoTUWW3mOXR4GF6kjwnSMtcYdv0nU9Rb/erUOC +26Upm/leOAo5qThx7EBAtdxpxwtzAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUf0eM +nvFzfjS5Wx7trTqHQoDU4/0wHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyiV3ft +FawwDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD +AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg +MKAuhixodHRwOi8vMTI3LjAuMC4xOjI4ODg4L2ludGVybWVkaWF0ZTJfY3JsLmRl +cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 +Mjg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF +AAOCAQEAubQFSKa6bJmLI8Sbs4oyP8piiYEeXQS6LSKjD1pdoKtApIdDJjYKCWTv +9bCnb3ofzAZs942cZF7CrudFOdy8hwbm1aprMnZRZOGs2ZrdF0ebTjEck/XFyta3 +kP9kl1nfK3/uLX1z75WttR6pDEg4KQs5TwX7B8/slKOz1esA7bK5caBZtT989SCQ +VKjqNkyuYlsrbQWNdniHyZDzstFy/If1KEzsGVAPAjLUV3XZwbLcDtSaOs1IcB7E +Lv1PsIlq3vCQkSMWzQT8YYecw1x+Dxn/Jj77G2UqSa5Hn9XmyDC7E7lI0GdXD/vG +3xz8gjuuH/clyN/AxdGNUZR0ML779w== +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:43 2023 GMT + Not After : Apr 28 19:01:43 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: + 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: + ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: + f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: + ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: + fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: + 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: + c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: + 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: + 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: + d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: + af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: + 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: + 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: + 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: + 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: + d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: + 08:41 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: + dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: + 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: + 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: + 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: + 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: + 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: + ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: + 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: + e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: + 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: + 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: + 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: + 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: + f1:5e:60:55 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ +97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh +bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn +qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW +wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 +1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A +PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 +ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG +m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 +J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc +jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz +EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/server2/TestServer3_cert.pem b/test/configs/certs/ocsp_peer/mini-ca/server2/TestServer3_cert.pem new file mode 100644 index 0000000..b061b3d --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/server2/TestServer3_cert.pem @@ -0,0 +1,97 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3e:1f:9b:cd:c8:7b:95:f1:64:e6:41:9c:df:6e:03:da:92:9a:90:b7 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Validity + Not Before: Aug 2 22:15:27 2023 GMT + Not After : Jul 30 22:15:27 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer3 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:9a:3c:db:76:c9:19:0f:7b:e6:d3:ed:d1:0b:76: + ae:15:d4:11:1c:66:b8:5d:2a:7d:e3:1f:65:d8:1b: + c4:63:62:f6:5c:8b:18:66:a8:1c:c2:a6:5e:72:f2: + dd:57:42:8a:ab:5d:bd:37:b6:f1:4b:51:f0:b3:6a: + 37:e9:55:78:01:23:ea:53:09:83:2f:7d:59:36:ab: + 33:4f:4c:bc:ef:a9:1c:db:94:79:4c:0d:4a:7c:3f: + 9d:3c:ba:6c:76:82:47:25:eb:79:22:f4:09:6c:78: + 3c:a6:ef:4b:30:90:29:b3:5f:ba:69:b1:1a:95:ed: + 53:e0:c6:24:78:6e:52:af:8e:bc:db:4a:f0:19:d2: + 00:5a:a8:b6:73:4c:17:92:d1:8d:81:9b:4c:b8:35: + 4d:91:dd:df:d3:85:a6:9f:c4:91:19:ec:47:d1:ca: + 4e:0b:c3:06:8c:27:42:95:83:e3:28:6a:3b:74:9c: + 68:b0:55:a5:91:91:cb:37:ad:fa:d8:69:8b:de:2e: + 4a:51:59:32:4b:3d:06:21:04:65:d2:f5:8b:e8:4d: + 45:96:de:63:97:47:81:85:ea:48:f0:9d:23:2d:71: + 87:6f:d2:75:3d:45:bf:de:ad:43:82:db:a5:29:9b: + f9:5e:38:0a:39:a9:38:71:ec:40:40:b5:dc:69:c7: + 0b:73 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 7F:47:8C:9E:F1:73:7E:34:B9:5B:1E:ED:AD:3A:87:42:80:D4:E3:FD + X509v3 Authority Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, SSL Server + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:28888/intermediate2_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:28888/ + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + b9:b4:05:48:a6:ba:6c:99:8b:23:c4:9b:b3:8a:32:3f:ca:62: + 89:81:1e:5d:04:ba:2d:22:a3:0f:5a:5d:a0:ab:40:a4:87:43: + 26:36:0a:09:64:ef:f5:b0:a7:6f:7a:1f:cc:06:6c:f7:8d:9c: + 64:5e:c2:ae:e7:45:39:dc:bc:87:06:e6:d5:aa:6b:32:76:51: + 64:e1:ac:d9:9a:dd:17:47:9b:4e:31:1c:93:f5:c5:ca:d6:b7: + 90:ff:64:97:59:df:2b:7f:ee:2d:7d:73:ef:95:ad:b5:1e:a9: + 0c:48:38:29:0b:39:4f:05:fb:07:cf:ec:94:a3:b3:d5:eb:00: + ed:b2:b9:71:a0:59:b5:3f:7c:f5:20:90:54:a8:ea:36:4c:ae: + 62:5b:2b:6d:05:8d:76:78:87:c9:90:f3:b2:d1:72:fc:87:f5: + 28:4c:ec:19:50:0f:02:32:d4:57:75:d9:c1:b2:dc:0e:d4:9a: + 3a:cd:48:70:1e:c4:2e:fd:4f:b0:89:6a:de:f0:90:91:23:16: + cd:04:fc:61:87:9c:c3:5c:7e:0f:19:ff:26:3e:fb:1b:65:2a: + 49:ae:47:9f:d5:e6:c8:30:bb:13:b9:48:d0:67:57:0f:fb:c6: + df:1c:fc:82:3b:ae:1f:f7:25:c8:df:c0:c5:d1:8d:51:94:74: + 30:be:fb:f7 +-----BEGIN CERTIFICATE----- +MIIEYjCCA0qgAwIBAgIUPh+bzch7lfFk5kGc324D2pKakLcwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe +Fw0yMzA4MDIyMjE1MjdaFw0zMzA3MzAyMjE1MjdaMFQxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU +MBIGA1UEAwwLVGVzdFNlcnZlcjMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQCaPNt2yRkPe+bT7dELdq4V1BEcZrhdKn3jH2XYG8RjYvZcixhmqBzCpl5y +8t1XQoqrXb03tvFLUfCzajfpVXgBI+pTCYMvfVk2qzNPTLzvqRzblHlMDUp8P508 +umx2gkcl63ki9AlseDym70swkCmzX7ppsRqV7VPgxiR4blKvjrzbSvAZ0gBaqLZz +TBeS0Y2Bm0y4NU2R3d/ThaafxJEZ7EfRyk4LwwaMJ0KVg+Moajt0nGiwVaWRkcs3 +rfrYaYveLkpRWTJLPQYhBGXS9YvoTUWW3mOXR4GF6kjwnSMtcYdv0nU9Rb/erUOC +26Upm/leOAo5qThx7EBAtdxpxwtzAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUf0eM +nvFzfjS5Wx7trTqHQoDU4/0wHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyiV3ft +FawwDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD +AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg +MKAuhixodHRwOi8vMTI3LjAuMC4xOjI4ODg4L2ludGVybWVkaWF0ZTJfY3JsLmRl +cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 +Mjg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF +AAOCAQEAubQFSKa6bJmLI8Sbs4oyP8piiYEeXQS6LSKjD1pdoKtApIdDJjYKCWTv +9bCnb3ofzAZs942cZF7CrudFOdy8hwbm1aprMnZRZOGs2ZrdF0ebTjEck/XFyta3 +kP9kl1nfK3/uLX1z75WttR6pDEg4KQs5TwX7B8/slKOz1esA7bK5caBZtT989SCQ +VKjqNkyuYlsrbQWNdniHyZDzstFy/If1KEzsGVAPAjLUV3XZwbLcDtSaOs1IcB7E +Lv1PsIlq3vCQkSMWzQT8YYecw1x+Dxn/Jj77G2UqSa5Hn9XmyDC7E7lI0GdXD/vG +3xz8gjuuH/clyN/AxdGNUZR0ML779w== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/server2/TestServer4_bundle.pem b/test/configs/certs/ocsp_peer/mini-ca/server2/TestServer4_bundle.pem new file mode 100644 index 0000000..27f4217 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/server2/TestServer4_bundle.pem @@ -0,0 +1,186 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 16:5e:ab:1c:8b:dc:fc:97:d9:34:9d:fd:cd:7d:b3:3c:51:83:ce:d2 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Validity + Not Before: Aug 2 22:15:38 2023 GMT + Not After : Jul 30 22:15:38 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer4 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:d5:fd:fb:3f:42:c7:ca:02:37:72:6e:78:d5:af: + 8d:b4:4d:f4:4c:0c:8f:8f:67:da:62:c0:2a:0f:f3: + 73:3b:83:c1:3a:df:9e:df:1d:26:12:95:41:ca:52: + 88:4d:8b:38:7f:78:ce:ed:aa:48:b0:dc:57:62:80: + 7a:fc:1f:43:c8:d8:2d:4f:38:c3:22:fc:bb:16:53: + 84:9e:44:0c:f9:51:00:a0:57:97:3f:df:57:08:48: + 3b:2b:55:b3:90:98:98:e6:a6:eb:ca:8f:ec:f8:4f: + dc:4d:7e:71:2e:03:ff:cd:fa:ef:65:7e:6d:8c:35: + be:df:fb:c1:0b:e9:f0:3b:89:24:4d:b4:02:7f:82: + 8e:0a:34:ea:a8:68:9e:f8:4b:39:9a:8f:d5:eb:bc: + 59:68:c9:f0:a5:eb:e9:be:7c:03:49:bd:b5:d9:54: + cf:88:29:b0:2c:a3:e9:08:b6:66:37:57:ef:66:5f: + 6b:0f:34:6d:02:bf:92:2b:cc:e9:9d:c0:a8:92:0d: + 76:8f:ae:f6:3f:24:38:e9:5b:fc:12:a2:ab:fa:42: + 3f:5a:05:e3:5e:bb:08:43:5d:55:18:17:13:0a:27: + 84:5f:05:69:18:a9:45:68:37:a7:35:f9:8c:ef:c5: + 9f:b1:8d:aa:3c:b7:cc:47:b6:e5:85:e2:73:f5:8a: + 5a:71 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + C4:BB:A1:42:EA:15:3E:0E:D1:48:5F:B5:E2:01:42:D0:72:BE:B0:CE + X509v3 Authority Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, SSL Server + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:28888/intermediate2_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:28888/ + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 85:c2:1a:b0:94:8b:a0:f8:2c:85:1e:17:88:4e:ca:2c:d1:f6: + 69:26:e3:a6:94:9f:62:eb:68:54:da:2b:f2:67:23:be:4b:95: + 56:28:08:7a:52:8e:b3:b2:70:2f:c9:db:06:74:b4:8b:8e:84: + 23:0a:74:f7:c1:67:81:69:11:36:2b:0e:4c:0f:2c:76:e6:2d: + 50:f3:e8:59:0d:3a:6c:30:eb:31:16:74:c8:34:d1:62:97:6b: + 1e:2f:5c:56:b0:6e:bc:5e:08:8f:d4:ce:4a:d3:8e:91:70:7d: + 18:d4:3f:40:39:39:67:95:68:f7:16:c6:19:69:41:c2:20:2e: + 45:e3:9d:31:c2:da:67:8d:2c:1f:a2:3f:1e:46:23:19:fd:25: + 16:69:5c:80:09:1b:f7:7f:50:47:1d:d9:6b:aa:7b:0f:20:8d: + 5a:f4:37:f0:c3:a7:31:5f:4d:41:70:c8:c4:aa:2a:69:d0:a8: + 7b:3c:cc:b4:a4:12:54:a3:bf:ce:ea:22:20:58:ae:eb:29:f3: + 15:da:22:05:46:cd:26:ef:63:84:4a:5b:86:47:fe:cb:fa:4a: + 0c:fe:82:e0:db:81:dc:3e:87:8f:93:23:32:de:37:3d:d7:0f: + 6c:f1:74:63:8b:11:b7:f3:69:b7:d6:e0:72:b2:1d:e1:15:10: + 7d:2e:97:de +-----BEGIN CERTIFICATE----- +MIIEYjCCA0qgAwIBAgIUFl6rHIvc/JfZNJ39zX2zPFGDztIwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe +Fw0yMzA4MDIyMjE1MzhaFw0zMzA3MzAyMjE1MzhaMFQxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU +MBIGA1UEAwwLVGVzdFNlcnZlcjQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQDV/fs/QsfKAjdybnjVr420TfRMDI+PZ9piwCoP83M7g8E6357fHSYSlUHK +UohNizh/eM7tqkiw3FdigHr8H0PI2C1POMMi/LsWU4SeRAz5UQCgV5c/31cISDsr +VbOQmJjmpuvKj+z4T9xNfnEuA//N+u9lfm2MNb7f+8EL6fA7iSRNtAJ/go4KNOqo +aJ74Szmaj9XrvFloyfCl6+m+fANJvbXZVM+IKbAso+kItmY3V+9mX2sPNG0Cv5Ir +zOmdwKiSDXaPrvY/JDjpW/wSoqv6Qj9aBeNeuwhDXVUYFxMKJ4RfBWkYqUVoN6c1 ++YzvxZ+xjao8t8xHtuWF4nP1ilpxAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUxLuh +QuoVPg7RSF+14gFC0HK+sM4wHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyiV3ft +FawwDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD +AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg +MKAuhixodHRwOi8vMTI3LjAuMC4xOjI4ODg4L2ludGVybWVkaWF0ZTJfY3JsLmRl +cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 +Mjg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF +AAOCAQEAhcIasJSLoPgshR4XiE7KLNH2aSbjppSfYutoVNor8mcjvkuVVigIelKO +s7JwL8nbBnS0i46EIwp098FngWkRNisOTA8sduYtUPPoWQ06bDDrMRZ0yDTRYpdr +Hi9cVrBuvF4Ij9TOStOOkXB9GNQ/QDk5Z5Vo9xbGGWlBwiAuReOdMcLaZ40sH6I/ +HkYjGf0lFmlcgAkb939QRx3Za6p7DyCNWvQ38MOnMV9NQXDIxKoqadCoezzMtKQS +VKO/zuoiIFiu6ynzFdoiBUbNJu9jhEpbhkf+y/pKDP6C4NuB3D6Hj5MjMt43PdcP +bPF0Y4sRt/Npt9bgcrId4RUQfS6X3g== +-----END CERTIFICATE----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA + Validity + Not Before: May 1 19:01:43 2023 GMT + Not After : Apr 28 19:01:43 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: + 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: + ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: + f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: + ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: + fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: + 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: + c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: + 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: + 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: + d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: + af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: + 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: + 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: + 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: + 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: + d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: + 08:41 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Authority Key Identifier: + C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 + X509v3 Basic Constraints: critical + CA:TRUE, pathlen:0 + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:8888/root_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:8888/ + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: + dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: + 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: + 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: + 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: + 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: + 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: + ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: + 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: + e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: + 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: + 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: + 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: + 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: + f1:5e:60:55 +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 +MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB +MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ +bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ +97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh +bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn +qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW +wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 +1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A +PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag +JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB +AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI +hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 +ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG +m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 +J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc +jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz +EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/server2/TestServer4_cert.pem b/test/configs/certs/ocsp_peer/mini-ca/server2/TestServer4_cert.pem new file mode 100644 index 0000000..7032625 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/server2/TestServer4_cert.pem @@ -0,0 +1,97 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 16:5e:ab:1c:8b:dc:fc:97:d9:34:9d:fd:cd:7d:b3:3c:51:83:ce:d2 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 + Validity + Not Before: Aug 2 22:15:38 2023 GMT + Not After : Jul 30 22:15:38 2033 GMT + Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer4 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:d5:fd:fb:3f:42:c7:ca:02:37:72:6e:78:d5:af: + 8d:b4:4d:f4:4c:0c:8f:8f:67:da:62:c0:2a:0f:f3: + 73:3b:83:c1:3a:df:9e:df:1d:26:12:95:41:ca:52: + 88:4d:8b:38:7f:78:ce:ed:aa:48:b0:dc:57:62:80: + 7a:fc:1f:43:c8:d8:2d:4f:38:c3:22:fc:bb:16:53: + 84:9e:44:0c:f9:51:00:a0:57:97:3f:df:57:08:48: + 3b:2b:55:b3:90:98:98:e6:a6:eb:ca:8f:ec:f8:4f: + dc:4d:7e:71:2e:03:ff:cd:fa:ef:65:7e:6d:8c:35: + be:df:fb:c1:0b:e9:f0:3b:89:24:4d:b4:02:7f:82: + 8e:0a:34:ea:a8:68:9e:f8:4b:39:9a:8f:d5:eb:bc: + 59:68:c9:f0:a5:eb:e9:be:7c:03:49:bd:b5:d9:54: + cf:88:29:b0:2c:a3:e9:08:b6:66:37:57:ef:66:5f: + 6b:0f:34:6d:02:bf:92:2b:cc:e9:9d:c0:a8:92:0d: + 76:8f:ae:f6:3f:24:38:e9:5b:fc:12:a2:ab:fa:42: + 3f:5a:05:e3:5e:bb:08:43:5d:55:18:17:13:0a:27: + 84:5f:05:69:18:a9:45:68:37:a7:35:f9:8c:ef:c5: + 9f:b1:8d:aa:3c:b7:cc:47:b6:e5:85:e2:73:f5:8a: + 5a:71 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + C4:BB:A1:42:EA:15:3E:0E:D1:48:5F:B5:E2:01:42:D0:72:BE:B0:CE + X509v3 Authority Key Identifier: + 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC + X509v3 Basic Constraints: critical + CA:FALSE + Netscape Cert Type: + SSL Client, SSL Server + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 CRL Distribution Points: + Full Name: + URI:http://127.0.0.1:28888/intermediate2_crl.der + Authority Information Access: + OCSP - URI:http://127.0.0.1:28888/ + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 85:c2:1a:b0:94:8b:a0:f8:2c:85:1e:17:88:4e:ca:2c:d1:f6: + 69:26:e3:a6:94:9f:62:eb:68:54:da:2b:f2:67:23:be:4b:95: + 56:28:08:7a:52:8e:b3:b2:70:2f:c9:db:06:74:b4:8b:8e:84: + 23:0a:74:f7:c1:67:81:69:11:36:2b:0e:4c:0f:2c:76:e6:2d: + 50:f3:e8:59:0d:3a:6c:30:eb:31:16:74:c8:34:d1:62:97:6b: + 1e:2f:5c:56:b0:6e:bc:5e:08:8f:d4:ce:4a:d3:8e:91:70:7d: + 18:d4:3f:40:39:39:67:95:68:f7:16:c6:19:69:41:c2:20:2e: + 45:e3:9d:31:c2:da:67:8d:2c:1f:a2:3f:1e:46:23:19:fd:25: + 16:69:5c:80:09:1b:f7:7f:50:47:1d:d9:6b:aa:7b:0f:20:8d: + 5a:f4:37:f0:c3:a7:31:5f:4d:41:70:c8:c4:aa:2a:69:d0:a8: + 7b:3c:cc:b4:a4:12:54:a3:bf:ce:ea:22:20:58:ae:eb:29:f3: + 15:da:22:05:46:cd:26:ef:63:84:4a:5b:86:47:fe:cb:fa:4a: + 0c:fe:82:e0:db:81:dc:3e:87:8f:93:23:32:de:37:3d:d7:0f: + 6c:f1:74:63:8b:11:b7:f3:69:b7:d6:e0:72:b2:1d:e1:15:10: + 7d:2e:97:de +-----BEGIN CERTIFICATE----- +MIIEYjCCA0qgAwIBAgIUFl6rHIvc/JfZNJ39zX2zPFGDztIwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx +ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe +Fw0yMzA4MDIyMjE1MzhaFw0zMzA3MzAyMjE1MzhaMFQxCzAJBgNVBAYTAlVTMQsw +CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU +MBIGA1UEAwwLVGVzdFNlcnZlcjQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQDV/fs/QsfKAjdybnjVr420TfRMDI+PZ9piwCoP83M7g8E6357fHSYSlUHK +UohNizh/eM7tqkiw3FdigHr8H0PI2C1POMMi/LsWU4SeRAz5UQCgV5c/31cISDsr +VbOQmJjmpuvKj+z4T9xNfnEuA//N+u9lfm2MNb7f+8EL6fA7iSRNtAJ/go4KNOqo +aJ74Szmaj9XrvFloyfCl6+m+fANJvbXZVM+IKbAso+kItmY3V+9mX2sPNG0Cv5Ir +zOmdwKiSDXaPrvY/JDjpW/wSoqv6Qj9aBeNeuwhDXVUYFxMKJ4RfBWkYqUVoN6c1 ++YzvxZ+xjao8t8xHtuWF4nP1ilpxAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUxLuh +QuoVPg7RSF+14gFC0HK+sM4wHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyiV3ft +FawwDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD +AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg +MKAuhixodHRwOi8vMTI3LjAuMC4xOjI4ODg4L2ludGVybWVkaWF0ZTJfY3JsLmRl +cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 +Mjg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF +AAOCAQEAhcIasJSLoPgshR4XiE7KLNH2aSbjppSfYutoVNor8mcjvkuVVigIelKO +s7JwL8nbBnS0i46EIwp098FngWkRNisOTA8sduYtUPPoWQ06bDDrMRZ0yDTRYpdr +Hi9cVrBuvF4Ij9TOStOOkXB9GNQ/QDk5Z5Vo9xbGGWlBwiAuReOdMcLaZ40sH6I/ +HkYjGf0lFmlcgAkb939QRx3Za6p7DyCNWvQ38MOnMV9NQXDIxKoqadCoezzMtKQS +VKO/zuoiIFiu6ynzFdoiBUbNJu9jhEpbhkf+y/pKDP6C4NuB3D6Hj5MjMt43PdcP +bPF0Y4sRt/Npt9bgcrId4RUQfS6X3g== +-----END CERTIFICATE----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/server2/private/TestServer3_keypair.pem b/test/configs/certs/ocsp_peer/mini-ca/server2/private/TestServer3_keypair.pem new file mode 100644 index 0000000..bb0d7e4 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/server2/private/TestServer3_keypair.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCaPNt2yRkPe+bT +7dELdq4V1BEcZrhdKn3jH2XYG8RjYvZcixhmqBzCpl5y8t1XQoqrXb03tvFLUfCz +ajfpVXgBI+pTCYMvfVk2qzNPTLzvqRzblHlMDUp8P508umx2gkcl63ki9AlseDym +70swkCmzX7ppsRqV7VPgxiR4blKvjrzbSvAZ0gBaqLZzTBeS0Y2Bm0y4NU2R3d/T +haafxJEZ7EfRyk4LwwaMJ0KVg+Moajt0nGiwVaWRkcs3rfrYaYveLkpRWTJLPQYh +BGXS9YvoTUWW3mOXR4GF6kjwnSMtcYdv0nU9Rb/erUOC26Upm/leOAo5qThx7EBA +tdxpxwtzAgMBAAECggEALjBPYLE0SgjGxWyQj6hI1cyeGy0/xNa2wE9kxmT6WPEH +6grVkdiCVGBSJIZKdpk8wbjes1Kby/yL4o7Kk5u+xkilIZzVpmEZWF/Ii9TlN7gj +Jja+ZGIOjkrWoZsKZCr7d4WezzLZp5wSPcOndrGVa1wdjQ02cvORjNyJi28uX9gd +8uBK5AIXS1lbkt/v+8mrBPgZUttz6gxhlHwxKs6JWWlIpGemNddE39UxuGDGHmVA +aw/gH/G4LNXtbAIPq5zDtFbfCKnQVgU1ppWILehoFqIs8JLtz4LPuvIxeztzKff4 +DU31rs14Zati5ykq9CVqY/d+4nKdstwhRPcPfsvgYQKBgQDBNVPn73A7fRoURpzV +sdJPA4RDbrbiZj0x/cAskuzzx/mmJUuNyuJxGizJU0ebT3VxtdCR2LqpgGEQEaKS +wYmMlSJ4NccugWgRl7/of5d5oY2m6f4W4YaNp4RebdVhNPJ4wSbeW7pH+2OKr2xd +my+m1WJUvRBbPq5kV2BdHNw62QKBgQDMXTqaOjsC9jpOOIjsUHmV55MbMmwK8For +H6e3Dn1ZO0Tpcg33GMLO5wHwzH6dlT2JVJAOdr5HqZgdIqjt30ACZsdf2VkutH94 +OvZmEAbwI9A+TAoxE8QlLYyz/qjJSGopJRU0x+KqEORxBmjO6LVV1GL9VVdoYrlH +Z7mrJ+7RKwKBgQC87LyDS2rfgNEDipjJjPwtLy8iERzb/UVRoONNss3pA15mzIk4 +uW77UbEBnGGkyOn6quKr+tVr8ZD3+YaTIpSx1xLBoTSHkRqGOXD6k+k2knbFBIHl +NdowoeGZxKSmTPPciGLNg7x/rp4Des3oKltKM9XXLpjT4FL+40HjStk+4QKBgQC8 +71AXd9BIy7VZzaCgwUG3GhIBadtDPbRO/AQFFAtE7KuoGz7X+/dWa3F62sQQEgKD +LT/Fb3g5LoyoGvwMdoJp9fVLItj1egAC+pgEAbs4VhPXFFuzxa9oI7VaTwxikmU7 +RsJVOprOWbGo4KES8Ud8Y09lIHof0m2ymy2nE9MRYwKBgDn86ZcbBr6sBXgc6PEM +rq4JXBCX8O17id9rJO37PkhPsOKpNf7YbQwHlHjwkUq5+g7Ec/LbeZ/tssEBY0ab +zUXwgWFMUKJVTEZUFwl2aTBqW8+LSu1TgzGMx2H/sxrvS4ElxC04jpPWUQstcuRH +y3yIz1HsmlMEg7qCiQ4maZE3 +-----END PRIVATE KEY----- diff --git a/test/configs/certs/ocsp_peer/mini-ca/server2/private/TestServer4_keypair.pem b/test/configs/certs/ocsp_peer/mini-ca/server2/private/TestServer4_keypair.pem new file mode 100644 index 0000000..9792728 --- /dev/null +++ b/test/configs/certs/ocsp_peer/mini-ca/server2/private/TestServer4_keypair.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDV/fs/QsfKAjdy +bnjVr420TfRMDI+PZ9piwCoP83M7g8E6357fHSYSlUHKUohNizh/eM7tqkiw3Fdi +gHr8H0PI2C1POMMi/LsWU4SeRAz5UQCgV5c/31cISDsrVbOQmJjmpuvKj+z4T9xN +fnEuA//N+u9lfm2MNb7f+8EL6fA7iSRNtAJ/go4KNOqoaJ74Szmaj9XrvFloyfCl +6+m+fANJvbXZVM+IKbAso+kItmY3V+9mX2sPNG0Cv5IrzOmdwKiSDXaPrvY/JDjp +W/wSoqv6Qj9aBeNeuwhDXVUYFxMKJ4RfBWkYqUVoN6c1+YzvxZ+xjao8t8xHtuWF +4nP1ilpxAgMBAAECggEABmE7dr39Ep3ZDRdz0QwaNY5O6p8Dvy7llQTdZCsaDAPQ +NJsC46w87LgoNVnbUDOGwE8n3TBS2ToCfXBu6joc5V2jkS10LOR7x+0+wpCtEdhL +RFyEKP51u+yaXf8Aut5/zX2bwUbj9d28p89NnMV4AIo7Dau0pKXcDlW1Qk+LztyI +hKFN6hrSFqAurmSt/pu3oo9kI9WJkrCxoj+VjQdVi420uAYOFR22aFaHrzpuHouW +4IzFbLhVF+c33xSbs1OEIpZSFzNucWYEKSwEREcyFgIXfWpDaXjoqWcrvXkeqyo9 +vGytQ3YaEsZPzfzgcViwa30g7WAA7kO9RuwcCPK4wQKBgQDpVmbVnmTlRwFbtdkD +4rjd5vtAB3nfsl0Ex11nU8+Oo0kZWeg8mm+Gba4vjEKfVyojbjFmm0ytQG0OGEK7 +UQ13mE1wueMn5qEVX9nTXIxVwcS7+rQAUrC5a6SSg81WIWzeclkqNc1J1EVC7jtl +zqy3PtC94g4tV68urpD86RRxUQKBgQDqxpWscN1u7GeuYf8rSPhPcoZTupqyrV3L +h+w7jUt5O/vfNPOYIXVfo2u05jiK0mTvLf5tVjYoQDF+x6odA2oBH2yz1ED0DZsf +2AhdtCSrMbxazcl/5fPrIIa1GRBp6y5i0ddX8T19twr/PVoYGRqkU4xoN+KoOKz+ +HLFUUgQPIQKBgG5N9v0DDMVKRL0bAQUSN7xGxf1ly1pRUiHBMUl4WEUgsZy3YM7N +Xu1YiiBWGOSEaxomrFnKDnxUWXlxRJKSZWBk8i7Y4SZqozmcfzeop3qeyCbpBBCn +Bn4RAdJ1VitiT7n0qmwG1Q4St89FGXUuN33Exx8MbxFGQz05LrcwZAaRAoGAVFez +PZfudQMI3GToPqygSCpkh3/qQ3Z008Go5FwGWS9rdOyY9nZOrGURNJPgjD65dBOZ +672lByDIpzsjqfioBG89pf0CuKqKqA38M22cHsRnXle/o+sAjd/JhRXUB7ktmOK5 +8iYAaUFw+fEYhL/ACnjZYDdzfeueekvkiN5OBwECgYB90hQJ2lw5s6GFJd+9T5xS +OMngfLAWDvW8+0hvtWCTLAVpMDWRGhGmvj532jWfkgqnvUemyF541RkV0Hy5K1Xl +0icXtpuZ+REh7NCXFJlEiOd+69OEdu78s5Zy8V1zCkEsgxzl2q6PkBDWfxepgdRC +LbwiAF8h2mxCwvvHbaBiKA== +-----END PRIVATE KEY----- diff --git a/test/configs/certs/rdns/ca.key b/test/configs/certs/rdns/ca.key new file mode 100644 index 0000000..81e8d5e --- /dev/null +++ b/test/configs/certs/rdns/ca.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDd8u+Lbq5qjxj0 +D8YdouHVlsnH+8UH1spKRr53wRqdPX7qC9eyBDMJ8ltUVAfQcKjXqP3EN22rAm+1 +L8vSXcdqxO2pGStXbfrdvDIeiMuJJ2zpz4m3g5T98AcNFUxyUQet4rLHtQGyzK+S +sTSOKCg7hULMvnXMLjUJTBsDgSYPNxLw3jlbppTaEVhna74d0aRNBkHVJvO8BJsT +J39iz2SrhvGzGfhK2Jbov0QPC44hIV28AFrwKbJI8gB7kD9BWe4hNfqoECLP2K/Z +qYw27Ybr9nUVC3ZSnTudAueHwsT+Pa5qbBoNw4QGwyCc7y9nolIOosi7RK1LxxOS +477I9ORrAgMBAAECggEAB2C0csOWm6cvEqnVkZWhASsWgQXO+mA68DYizbNHEbQC +HICRRnMaSTnbrFVwvw6HpkeRS+z1wcn+cZzpnxAL+XIbXlQeWzUmMim31skATwpW +9fy2nLmMgdZxPY+YL3KfGTogbEAJkup36Z3nW9Rc5gGbg1fPbZ6Zl7oGadroq3e+ +0mKdUzKNw46VVUpoHIq5ty/5mGyiew0E6IJEv3tD8MtFZ9r29zyDF6thsj3k1K+l +elRc6hgvP74RI4vJ6yNGWgJRoLK4mHAxhS6Zew3aVr/5P9P0WRnj2WD65geUgaFv +hCcJzqkGQPUl2CGRkspIUErpJkAZRyJyjuw/XU91aQKBgQD9dy6nqqfTpf/+dWzc +VltdhlEuGfXN1Yfg9A4w+o3+goeFwt0dim6plX/iSLbohylXbpPcugx+OMIw0xHs +ZsT8L1NIUSiXIri+VIU+5iU5X+tQUlCXnTy1MhqRSI0BD02Ac/+MMp36RmYXgghd +MIiMN11wDbeGsCzjDecziALi0wKBgQDgKxPmUniPe5eBF3QrpDpBnaxZEz1/e0Hq +xKcN7ClCbfPSvyIGJhB2NJR597vfaHCwsp7ReUK1RrvmRBaBg8bFffCbEfFSyB5c +pZ6r9BXBhPTB9NMsnMn/e3Al8sVcYWbZdgkkxuVE5DdWZS3zXscmTzls/RPvDlOK +yLI+GaiJCQKBgHxjGtBJnabuVhzoP446CwhwvDIlp01mNxKccsJyk3CNWji4ko7G +utwac/H2wVyHyDASIho2a58d957CVo9vN8iS6QoaWhMhhQxSqjld9HKdsftvCgH9 +tict+X/G3PviKKSbSJPi20hReBSdScGB6eD14rL06FX+62haEFZnrxLlAoGAYrK0 +zUjsagg5mY3xCTICsFcyxflRr2peiHZTMy8Sr3vnyZd77Icf4lueL+FiZ6f9Td8n +TAV+2H+vRWAfJKEXiGa95BjPTupra9FD/mO2nIDknu0jeVYDHiiEJUExok4EUaTF +u9qSoXV14+UiiS/msThaiWEYQL3nDIj26Z60dLkCgYBBrZN/M8NwT111KFGXgYzm +CxFCu1WPCRUXIKVJjbj9reJgfHMhj58NFpqtVSS4KNeoc6ZGOfkAT/Nb1/H001TF +AK3CBTGzFQiqNmvmWP+0O6JaqVXPgHXXRCdXz9CnJmyIeJqvQbQEZPlIC4cTOi3U +Yr+e93q7SOS6wbf446ij5A== +-----END PRIVATE KEY----- diff --git a/test/configs/certs/rdns/ca.pem b/test/configs/certs/rdns/ca.pem new file mode 100644 index 0000000..26b6258 --- /dev/null +++ b/test/configs/certs/rdns/ca.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDBzCCAe+gAwIBAgIUC6fLO8KV763IBygl6/0w7EKectswDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx +MzIyMzAyM1owEjEQMA4GA1UEAwwHTkFUUyBDQTCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBAN3y74turmqPGPQPxh2i4dWWycf7xQfWykpGvnfBGp09fuoL +17IEMwnyW1RUB9BwqNeo/cQ3basCb7Uvy9Jdx2rE7akZK1dt+t28Mh6Iy4knbOnP +ibeDlP3wBw0VTHJRB63isse1AbLMr5KxNI4oKDuFQsy+dcwuNQlMGwOBJg83EvDe +OVumlNoRWGdrvh3RpE0GQdUm87wEmxMnf2LPZKuG8bMZ+ErYlui/RA8LjiEhXbwA +WvApskjyAHuQP0FZ7iE1+qgQIs/Yr9mpjDbthuv2dRULdlKdO50C54fCxP49rmps +Gg3DhAbDIJzvL2eiUg6iyLtErUvHE5Ljvsj05GsCAwEAAaNTMFEwHQYDVR0OBBYE +FHMfJfiA8Nb10G8rKcO/am9Ogo6tMB8GA1UdIwQYMBaAFHMfJfiA8Nb10G8rKcO/ +am9Ogo6tMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBABMr45yW +SDHQ+zOW6MXCn2CAqagr9TaRjkqYmaaNh0uDBI+oaFKi8WHCi7jt1iSb2f8wp6dv +jQ97osGsAsU5KgyyliweIaTftd58oegpmADwpQTpVa2RIz6o4z2FKnDm6ZtH5fm+ +RRS+FpcS81s1m6e3gJ3Ie4nIqQRrBcvKpQEgMgiwJH4v2rIB0RvkTztA2EeVyyH/ +IjbrbO6Rc7EpEJNbsVWHcKt0tMNx9F1qicgscrcEgAPG77yz1bP4jKSqhXt6OhM9 +lhXFq+EbzAhbPzkgQuxghHxJuDNZDafKvf1cl/b7jEykLEiKo3s1oto+9yzRKZ2y +uKJ+WESZ0V6XOU4= +-----END CERTIFICATE----- diff --git a/test/configs/certs/rdns/client-a.key b/test/configs/certs/rdns/client-a.key new file mode 100644 index 0000000..c6c6755 --- /dev/null +++ b/test/configs/certs/rdns/client-a.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC2yUu/qqBD+rKF +q2f2ISuw7KZlpne5NN4RXhAK1fe1b6TtZi/hDeDaBo0A5ttFn+lmb99DWwQh5UZO +OrjueeZLjhkMIBqmVY0Z8wu0BudYFWrjbEEFz2oAN4UXxu6PvKdcHsJQHbMjVvxS +j7GauUafZ1MZmcxRizQDW0+bD0NDKERR4JropXC+9Cths9MG8VKXGzYEBX1g9j2r +pRZ064ZDUEM3ImZI5fAdeUPZuutg2J9swgXvppVpXOMQPGR6TUZsQZC8Z39zm3OK +vsyqKE519gpC5ZNTMm9xaD+Dd8tyv3OhMnVZvIEBvtJysKS076cGqxwIeFNZ5zky +iNzZ6A2jAgMBAAECggEATs2pDSiEMNqf5bYYu4ngesYEFRX4Ts5GoSky9GD8Cupn +qfvgz3+6vXSOjOF6jmFbcFDx6emO9ggRT//MI/BFA7TvAoXCx4xhqVrZ5FMHoTeW +nhbZT6Z09kP5at9WkXyNkM8oCf9L8IP7g6K0uNaA5o9WydY/OravhXTz9I9Xeddi +6Cgjr42PVqpdlN/CvQQ+ISYGSYzg6r/o8gaIg1zypJPm3DqZ+TFBxalv7d0FoH54 +eagk+blA6kUGvJ/CX5Fr19hliycDNfxT/uofdVKEUXSWlPQgzvGk6Pb0B/Jco0C5 +BDuvA97LigY4AR5FXZa2wmYhdlBontjixiZNFgeqHQKBgQDn4XriO8ZNkERW4e2V +aLrkRH5nd3IqRX9aUn7Qz1xXV+nE9+984Mwk0xtTkwtmmiII8v+2cfz1J7I+HVB9 +NRFGtcKX2REEM6wp+UPJi8HFx1uOr3Mi/JM8i+1/vq9YPXR4DJ0CBJwydTQo0zIz +jJrqKPUiEAtrAmT4YCz/hdrARQKBgQDJzIdWWCv8LeyFBdJJHikjSy1PETFMoeRj +FEtdi3Oqninn99ovV43fgVXK2C38OaiO0X7N1OwXtJhWaBU5TkmlEbXe7eNlayFc +RFfEmg3RFItFEU348oyNuhqDU7RRG6FapMOX6fAIhaF0Ox+qLlwsW+uNbjxcq/5Q +JIhLji+4xwKBgFk9V9OVeZ4ENajSTmM/6R1MlvEu8Qr/sCGMui2WgE1FEjyxxsNi +qG5LqKqjXrHgA9U539vRRq761gg9s+pb8Agsj3VtHrVlRY7p1YLNfIenT788Gq2O +UwPsPjz5n/XvwNEq2YobSBnLLYXAsgNQy5XuqViwoBRQ2ZcVPHwigauZAoGASzFu +7HmGSj1CeH6m0J/wbDpC88dQO2HHnDOfsYeY3eZ5bvonzqQtNS8YLLFI0Ucay7y8 +Jy5DmsGsUeYOon/NaTikMCEsLkow8BR3L8bHhzTEgEPmbfDS9qx13KF/+wj0orXq +O9zrmAFTG+A7+Em2BsFpkBWXYGCmLm6uapjcp1kCgYA1cGEgpOXGv3HR21ItrpWh +wx/iEqybIdYU5bOr1OxZJE6PgWD3k0DcPAFGbfJX8I5AEQQHhbufv+QMwfdm4irt +/EMWE+5MzmCOPKQzb3jLhvdqAKie2rnDFuFvlhT2woDxQaY5gaS3Tay380WZ3+rW +tHA+wi2+w8REX4KLuaY25A== +-----END PRIVATE KEY----- diff --git a/test/configs/certs/rdns/client-a.pem b/test/configs/certs/rdns/client-a.pem new file mode 100644 index 0000000..aa25f67 --- /dev/null +++ b/test/configs/certs/rdns/client-a.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDpzCCAo+gAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcIwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx +MzIyMzAyM1owgY4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEUMBIGA1UEBwwL +TG9zIEFuZ2VsZXMxDTALBgNVBAoMBE5BVFMxDTALBgNVBAsMBE5BVFMxEjAQBgNV +BAMMCWxvY2FsaG9zdDEUMBIGCgmSJomT8ixkARkWBGZvbzExFDASBgoJkiaJk/Is +ZAEZFgRmb28yMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtslLv6qg +Q/qyhatn9iErsOymZaZ3uTTeEV4QCtX3tW+k7WYv4Q3g2gaNAObbRZ/pZm/fQ1sE +IeVGTjq47nnmS44ZDCAaplWNGfMLtAbnWBVq42xBBc9qADeFF8buj7ynXB7CUB2z +I1b8Uo+xmrlGn2dTGZnMUYs0A1tPmw9DQyhEUeCa6KVwvvQrYbPTBvFSlxs2BAV9 +YPY9q6UWdOuGQ1BDNyJmSOXwHXlD2brrYNifbMIF76aVaVzjEDxkek1GbEGQvGd/ +c5tzir7MqihOdfYKQuWTUzJvcWg/g3fLcr9zoTJ1WbyBAb7ScrCktO+nBqscCHhT +Wec5Mojc2egNowIDAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4YW1w +bGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYDVR0OBBYEFCkA7CbJMs70HE5rQxHm +qHL/UgkOMB8GA1UdIwQYMBaAFHMfJfiA8Nb10G8rKcO/am9Ogo6tMA0GCSqGSIb3 +DQEBCwUAA4IBAQC87KaD2UuvQWIbSPOKy2iPc5bHrsmzxtPzT219NqzI4JZ9QeAz +btvLlMxgACreCb6wzYxaOjbU+O2LqO7mq/M4UyY1+wrAwYn+c3rVcimuPa2bGKin +Le+aBr/4yXAahqH4DW5K4x3x/7c7wyNlj9MHUQDl9A/JHsit68hw1YY/1ALPaOpg +9L5K52gGHVXrxb8In8OkJEoM38G5Zstiuh1umsWNWBp6Nd/FlNr+XbvNuFsWLC/V +lTaHxFt9WRQNINc8RpK4YCHRdaOiEqAGexDkHgNaLHeHEwCvP3Zhj7Zpp+svlmm8 +cLL2A5wl0T0OEehT9wxKaG5cyP7jfYfJmENW +-----END CERTIFICATE----- diff --git a/test/configs/certs/rdns/client-b.key b/test/configs/certs/rdns/client-b.key new file mode 100644 index 0000000..7be1ab0 --- /dev/null +++ b/test/configs/certs/rdns/client-b.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDSHSxxgyq3Yd4H +LpzmpRceCQ8y4qqsysiZ2C/9HhwcvvoUXRJvLB5VzYb1702AZNgbeffrz8RRGcaa +YuPRXwkitRGb/MdmC/AFi2aALk1M4EKzGhw5IjyL6tfk+qahxm0IACJVQW9W7Ts8 +O6odqnqenvXLzvTNn0UWWv7jZonOpqnR36CoxIj9WGNXIlwDtcTwCoLfd9V+kUzX +PybIVrAL13tLPypZ7fpyAkwd/4qYAZ59HbRsQDNvYItkwe4JQiFt4PQ94+eGDXtk +DqttKPi4uUUgSGqRzTNkvkM+KAghRD45XN2J1ugw16GGUt4sQKPViTDh06yQ/OTR +X7CyuvmHAgMBAAECggEAGBhPXe8QSXpYJ1Oxm4OIHKImU4wJzYntqjLQAgNfbu55 +qDcqI4SYL/fDR9W6+gPWgoQgqbKoTTKbA15eyMZQumGwVMaOqUWqL0CWrxFgPPYk +v/w5C1ZBQjxaqMATts6V90popYU8+ud2mQgwkHlZ2XHaNxaTHTdsCU3DGBnMqN1f +tq3glwkHZqUxKsGXwWqSKsDwEUXpcs2ww0fbuzpdZ5IokPX9e9TdFkw10rKXboRn +owcTxHI8kuTVMFg9ht4G39V/LVpqrPsHGz4Zj0tJ8PkwxowcwLMcxg2KVsHRWzC+ +i5UVlzjWVK/Gu1mMMCfySJLBi4GaO88hZzK8acdu2QKBgQDwgN5zkgOELSulfGDQ +x6spyjbEdN9dn/cwCpolMJMg3mdcKP/XjPTTEe1dsWT7mTa9qxjWgcxOzr1A3h4B +ZPJTXgOQyt6rPYw2WffEh6+5H28+629VlhQary7Sp5rZAXiJoHgzHpb9sURwliTL +62GzaCOxJd1AI3qd8atjgI/JmQKBgQDfpwcGmcpAhQMA/owdowCoJhQ7Jc5h3+RD +55WO6sFFxigxa+gSt6NVw3ef5TB/zKyovPGceunaMyVOeKA3RSNYNX3gKKhpugeg +wv465jGw84jPj7b1LMc7ydWHKWQ4M6JLPxlSM9lIV0Zdb+gNu64hCNMGOQljcCuh +a0Fi7XgQHwKBgES1XFgCF/UT/ospjHM5B+JJffoENagGwlS8QqtrRC04vKisoeLP +x0XhskF5I5NpgyUOk2r87AR7tb+pdUMKttwwWK5t9s1PKRaB/3HrHb2yiFKealE2 +LgotkB/oeCmU7P3MlWptS8+wbBAKJPsJBQF/N+stGRdwy2ACIeesW+fJAoGAbG6w +i1S4qDtUz4CaMiw7P0rm+B5GR1GjfACllBrhBZEjH4Lvi1OZTxVrxAv8TZnQBCdH +BNTa2D1/0uHM278becLFeo57yHa3CAxB8hB0+xO3uto0m/3Pwn8ClXtN6amu/8hh +4Gbe66HUrax5116s97sthJDWqm1R2fsEQpBba4UCgYEAitrHGT/Kr/91Ttccc7Ww +Eee/xEJR8iMFU010zkX9MAu1JExXOaMWVJRWlzSIJFhDjEovaq9qxdWPsoDT1C/7 +JIs7RGDVWX2a6uun5Oz9JJJ9KmNBzOAicTBBrWMT67TGWZhVYjEN9PvRyX/kMssP +yqRcbXt4Unw93950LphbVOw= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/rdns/client-b.pem b/test/configs/certs/rdns/client-b.pem new file mode 100644 index 0000000..7ae4f85 --- /dev/null +++ b/test/configs/certs/rdns/client-b.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDejCCAmKgAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcMwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx +MzIyMzAyM1owYjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRQwEgYDVQQHDAtM +b3MgQW5nZWxlczENMAsGA1UECgwETkFUUzENMAsGA1UECwwETkFUUzESMBAGA1UE +AwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0h0s +cYMqt2HeBy6c5qUXHgkPMuKqrMrImdgv/R4cHL76FF0SbyweVc2G9e9NgGTYG3n3 +68/EURnGmmLj0V8JIrURm/zHZgvwBYtmgC5NTOBCsxocOSI8i+rX5PqmocZtCAAi +VUFvVu07PDuqHap6np71y870zZ9FFlr+42aJzqap0d+gqMSI/VhjVyJcA7XE8AqC +33fVfpFM1z8myFawC9d7Sz8qWe36cgJMHf+KmAGefR20bEAzb2CLZMHuCUIhbeD0 +PePnhg17ZA6rbSj4uLlFIEhqkc0zZL5DPigIIUQ+OVzdidboMNehhlLeLECj1Ykw +4dOskPzk0V+wsrr5hwIDAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4 +YW1wbGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYDVR0OBBYEFBN5/1B/G8UTcl5O +tqEHRzrAJ4IHMB8GA1UdIwQYMBaAFHMfJfiA8Nb10G8rKcO/am9Ogo6tMA0GCSqG +SIb3DQEBCwUAA4IBAQAioN2YIK8yKkyn3x/UH+YhgLJj9CZfkCPa2uJTBaXCmZcv +KBPfjs4bQ/c6omLzbGnIVEDHjUEzwxUf40cVQbciPXvrVxB5PIcUY/e8RAgFux7j +YnP4F4fM4DC1yOA5AIqWZGX66GVnw7rslxz5Pko6QKNaHuVgwHHEizN0d8hHexdH +rGHtX3tsFmI7GOwsVgLJNV3VcpT+W8ZdviHtbjL2gR3N/KpXSU3FHmDC56Zi8HyA +iUICVuCo06LCEq5J8M8f5dBEMtLJ31gDX1c/arLJuS6VS/+XrC7lKNT571SXPa9x +gRmY0EtzyYut/yfG2qXWV9Xi0DbrZUKZVIUcqd8y +-----END CERTIFICATE----- diff --git a/test/configs/certs/rdns/client-c.key b/test/configs/certs/rdns/client-c.key new file mode 100644 index 0000000..47f9041 --- /dev/null +++ b/test/configs/certs/rdns/client-c.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDG0eLmkpj4R7o7 +Phx+EUuInMKkHD2yoYVIDc3+ipvj/3bptxWVcRZoWroFLrUroSGnPBI2stXHGFti +5qkCCjmsWLsido8luDsLnr9kBQafBoV+tj+u/ZkJSeTJ+bVXaGS9GR4rGvYPgwRD +dyuebRSrqW/GB5Chp0X4JqxNGrX+SrtU4fz23eBpg1+Tv9Td3xhlw4FdVAphSYS9 +NYLg7fNpIVwXLcvyXgLya2IOvItHGM1TyWq4jsOrgU23z640GzTUEIic34KkgF78 +Be3ZN+0oEiIrojpUb9I0k52NhHL6Jjxu01/IIgan4n7h/tr3O44YOeNfpRtn7RSA +CelkiXUpAgMBAAECggEAAwbQ7cek3Sq3srfYVob237vbLq+2ChEcEU5nKLfSywlp +iwHxfyvl4RgjJXm3jP7P9KwdGYi8E7goM/zk7RYGGLyPug+UpZ4EsBbuPPHipEBv +fyVRGjMxcDXCw4cHSVmv0ZJ0ph1Wn5fT/x7s+BAbCh3zM1A21cR7yTJK2dr36Ei+ +CgUEvVJQh3rult/4JJ5E74oXq6rXRqAjO7i/DXImLiV7YePjixhy4YGpYH8T1Da9 +CdhA53Ec2wqcEfaIZC+fMtz+4XQnCOJJEjJGfGApGadu52sqA6dB0I0D+DWy3H84 +V4MPrJMvllK1GH3pk5zqm+tWlt62R7hc9O4a7OOVgQKBgQD9ydnmPv5Zf03UAzT3 +VLuwvM+5H1EthWV0CsjPf9M5QGfAm5tuxrqWRfZ4mPX8YRJFc0RXBy5WD+UkBwDo +GvmmD96t6FJ+xsXTz0Oq58cUAWiyuqPldbe+NdOIHasam31jBss/jrL61YHU4eec +tCtTZtJQsvQ/X3wjWSF4fOmaOQKBgQDIjWlmxnxC+AhdrtlLYVhg87NoVgAOAZcf +MCnJT9xMbhfmZko75vUdT4OnTXVjtxfqLP8yZE6lgb7GaTum3YLhhfZjnRs3buGk +y+6mKlNNhfrOFCw8tgVzaeGQi+dE0WyBHqBpiKxU6EixY/28oV1+einxu9agZCNy +USU03LdycQKBgQCkSdwGAfdzhkainaTXC4IpCkFKLKzHVBh2A1k1giEBaeEAPXtk +Pb+h8g54yURMKabDULgHwn1LdyS1qtb6aiP8TT+wwrlMmm6MDBtY8ovcNoFJWisp +KohU2NFjxxjxs4B673X5Ye4ZFMfkQI1H0xZM+j3hwcb9k1lwJI3XSr2KeQKBgEUA +AkeN+qq/04EH61L4BwQ0VIGNNS+cdHYSiA3vIAhbyHVItDmv2J1hAhbJm53XHK9B +E/wubrCa1xxEkHV0uNcG1CKppveHerLMRyt8XHTLp+LHJgEtTurKfwTQXZ1bwE0c +UGx+zWvZD0mY8W+4xQYC2fOFgO5mBZwLNxXc6nDBAoGAHJaD/ZSSjtAqQnBYN7Wt +/QOpWx3UrZ4xAKM6KPp0jAKNHgFlH1j7KXIT+fC54TdYw6K9TOzMOG83BFNBs1vf +GTRc7NbMW5fW5NqLEwqBK6lWChqXMwjmz/W7qxRBPqtsmAkubgUO1fBw/XReut7C +XOa0n2b/bfIb5jCCFAH/Gw8= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/rdns/client-c.pem b/test/configs/certs/rdns/client-c.pem new file mode 100644 index 0000000..190c7b9 --- /dev/null +++ b/test/configs/certs/rdns/client-c.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDpzCCAo+gAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcQwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx +MzIyMzAyM1owgY4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEUMBIGA1UEBwwL +TG9zIEFuZ2VsZXMxDTALBgNVBAoMBE5BVFMxDTALBgNVBAsMBE5BVFMxEjAQBgNV +BAMMCWxvY2FsaG9zdDEUMBIGCgmSJomT8ixkARkWBGZvbzMxFDASBgoJkiaJk/Is +ZAEZFgRmb280MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxtHi5pKY ++Ee6Oz4cfhFLiJzCpBw9sqGFSA3N/oqb4/926bcVlXEWaFq6BS61K6EhpzwSNrLV +xxhbYuapAgo5rFi7InaPJbg7C56/ZAUGnwaFfrY/rv2ZCUnkyfm1V2hkvRkeKxr2 +D4MEQ3crnm0Uq6lvxgeQoadF+CasTRq1/kq7VOH89t3gaYNfk7/U3d8YZcOBXVQK +YUmEvTWC4O3zaSFcFy3L8l4C8mtiDryLRxjNU8lquI7Dq4FNt8+uNBs01BCInN+C +pIBe/AXt2TftKBIiK6I6VG/SNJOdjYRy+iY8btNfyCIGp+J+4f7a9zuOGDnjX6Ub +Z+0UgAnpZIl1KQIDAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4YW1w +bGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYDVR0OBBYEFNZnXD/At0QIFoqsEUNn +hd0XqkJWMB8GA1UdIwQYMBaAFHMfJfiA8Nb10G8rKcO/am9Ogo6tMA0GCSqGSIb3 +DQEBCwUAA4IBAQCiIEr4+IiHo4PLK41l6Yuo6QKETZg5tNKluqkT7r5ulYaTss+C +O7nBIZaVAK3h5+uRTzrL2mEUyme4tYBUaIxqVTnjDgIbeKvg3Z0k07Zld7+eeE/B +vnPk3c4aZaKshcbKoA+tZUeHk+BYZ28YkDH70OYsyVhemvTVDDy2EZGdgeaWJAbx +RcQ7iVAIR8SEgJy9PWZMAFChNxa2N7Q1AcEqnU9UV0+XOHVHe+PYgK7SfuNOxM6x +rP2NnkLenDoZ7IXEvl2WYVlnJpIZiNTRz94nkTz5MqjCuWS+YGYlPz76gpGPJ5NT +VEfAZXP0EW8NWJ1XEIM6+u6c1SWnSJfTt6im +-----END CERTIFICATE----- diff --git a/test/configs/certs/rdns/client-d.key b/test/configs/certs/rdns/client-d.key new file mode 100644 index 0000000..421bed9 --- /dev/null +++ b/test/configs/certs/rdns/client-d.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCxkPAQkDuzNvqJ +/iVOuucGi7n2N3VI0CXBoAIUTlyWnf73m2VOjeXUCBL95so3uGBd1Q+R9BwEnPH3 +w8P7zEn007tryzskGywjEqCn2WYW3TKBGuFhuTc5sFE4IesCjglRwebsCn78V7kE +Rnz9/hRgULEIRPjWZkdwnxO03S/4n/lzMxCkZdLjyIWpr99B9l5Z3dAFlMchRqKK +7+IJGorNjEIkVLQIHdWFUAnI2ZBh0Xkuebqxg061dhEnvo8aBi/gqcc/8IVfnUbX +mmQM4nPwRFzOY1/+hfCKOIR1MAQZ7ydXK1C0tuoR3rSvUEPgT7MbAoi8PZIaoLCB +bFDsQ0LjAgMBAAECggEAGDmobvIRRbWPVTVDB2yQvqq1z8V9hFYVg0bzvFZ746FM +RaFsnwd/9BS36+5JPMne/TOSFhJiTNomruwFTdHvbcNXSMZnhq5dsAPyFC72R9Hy +JSgILhTJV98EwwOUSaXG2Ax8F0xP9S3N/wB+PAl749oOamI9OsRQedPSKfTv/wYw +zPNhr4MgnnD34VDnkoYTFIU/n9OKeSgh8okx3PFaPkfYKjaOoIFHA4j1lMMVa7kl +q27MrL/vpWF0MUu88ip3lDmZQC9t6BovoIidwRhNEYo4Qf+igcwcIce2MY77gg3Q +IvWpsG7r/jyr6KFRONvyJWpyIgAq0Y6aRb4cNzkDxQKBgQDWrOdLVg6xw+I8BVrM +9gDJyfeF8zEXu6Mi3zrypyOOd11ZqlVFaeq/sT2s7HCVEZOiqTgWE0ueBry6FGX3 +DkzGpTo1+kd5HiKawViPHnIIVTV/F1Hifrx8bDHRRPVqyh/iTV2c4S8Jdf90zgZJ +vW2wErGTrpTVvgnb5GhZBZIQbQKBgQDTv082UDFD3fpA98t7DptqmYn8hpdAa4Yy +MFMHBNbNntB5G4RDdDut0eWA01MKNRjKrxBw+YjuOgTIRx5oVre73y83932rGSiQ +g1/y4PidwPVEAnVbbLqvvHkkKTbBbHENSh2pWCikJhAGKhVI665bAc84o8o4ikN9 +IQ20TUqujwKBgBt1rHRPgrusYcDpsm6rPPiS1A2XSP1PLBPm2AR36q1riU5R4mxp +mvSAOHJpIBGBWRAicyEnwLFrDTMELvLGKn3yXprO89uDRkvjVW+hJlb4h0pFclz1 +dyi3MjwhhP7u8dshqErL5xdft5h6TgWarHAsQ/ivCSy6DUrKUaqpcsCVAoGASq8A +Nkzkg2ZonL+JYlbNlDShSZMDB/Kku1D2B9S6Gn34U67T4KK/ZdhRVTWz5TbDDsHe +T6qDlFqcUzNaUzy6wyW91sSQ62cNOqNLlTOqHKHxH9KqJ9vaoJ8eLxXmPSSNXz2T +5qW0d+kA39u59CVEMs8ZkoWajoSdtyWQWtakD5sCgYAUgWP7Z0CbUeNyaH811Aef +/P7inGoQEqTYrmnyAFmRyBnmq3C3OjTRiz7+R7cERpgVVmLt4xpNWjmTbcngfvau +c2YViXlsFx9s/xXFQBtppmdH1WeZBIOZbN6EaA5q0oHGOXAW5SmZR3sGKeG5Mz/m +VDNDxg7Fspw4ZdHu6IeOCg== +-----END PRIVATE KEY----- diff --git a/test/configs/certs/rdns/client-d.pem b/test/configs/certs/rdns/client-d.pem new file mode 100644 index 0000000..ea3cb0b --- /dev/null +++ b/test/configs/certs/rdns/client-d.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDrTCCApWgAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcUwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx +MzIyMzAyM1owgZQxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEUMBIGA1UEBwwL +TG9zIEFuZ2VsZXMxDTALBgNVBAsMBE5BVFMxDTALBgNVBAoMBE5BVFMxFjAUBgNV +BAMMDSouZXhhbXBsZS5jb20xFzAVBgoJkiaJk/IsZAEZFgdleGFtcGxlMRMwEQYK +CZImiZPyLGQBGRYDY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +sZDwEJA7szb6if4lTrrnBou59jd1SNAlwaACFE5clp3+95tlTo3l1AgS/ebKN7hg +XdUPkfQcBJzx98PD+8xJ9NO7a8s7JBssIxKgp9lmFt0ygRrhYbk3ObBROCHrAo4J +UcHm7Ap+/Fe5BEZ8/f4UYFCxCET41mZHcJ8TtN0v+J/5czMQpGXS48iFqa/fQfZe +Wd3QBZTHIUaiiu/iCRqKzYxCJFS0CB3VhVAJyNmQYdF5Lnm6sYNOtXYRJ76PGgYv +4KnHP/CFX51G15pkDOJz8ERczmNf/oXwijiEdTAEGe8nVytQtLbqEd60r1BD4E+z +GwKIvD2SGqCwgWxQ7ENC4wIDAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SC +C2V4YW1wbGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYDVR0OBBYEFM4b9lmfPrfM +tHVxnMbxR8oyE0ZtMB8GA1UdIwQYMBaAFHMfJfiA8Nb10G8rKcO/am9Ogo6tMA0G +CSqGSIb3DQEBCwUAA4IBAQB3O3Q+oTfAVitx8K4+ZzXILszfblsGO4kvwvxV6EUB +BUsat6mjawFTGhB1TdBR0CflA/nCjTZpoXNY+nCoJmr/Lxk1V+UNn5zhFTTfhZjU +PA+++5QcYxXz3ukCEqTHLPZKvv2xB24xQUmsdwLlX3VGE1VqBfaz/2x6jFTVT3lz +4W5JPltgFSVOLqQC1T0MY8L0h4eL+JpynHPjMkEwb0U4RPaSBhiRSjHD8dgoZ8Ft +ZbV6WCJeBYey0edKNwujuLl+McVpx6DIy8BPGPEcuo20Sy1BP3JAoN1Wu5J3zLBA +AXwUZ6r7jAjU5v1nx6ijwtv3qavBVdclOlW0aDu3+Y3z +-----END CERTIFICATE----- diff --git a/test/configs/certs/rdns/client-e.key b/test/configs/certs/rdns/client-e.key new file mode 100644 index 0000000..322bf5f --- /dev/null +++ b/test/configs/certs/rdns/client-e.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDRoOr56ToxIMnw +palVuFJwDnPotA0toHmQe2+iGUtsOaDQW5USKynV7YrP2DkBTzai+KyoA6cW6b3M +i8FIivJk9hkxnWp7CuSn0dFZVoyMwSuyelw+ff+8Cn6hNmg9s0BgQklk6pe5LsGh +Rwun6CNPnlJapYycwaH3e7W4fiFrs5njz4yrYQTQZPA6fBybtcT5NatPMWBzCHl9 +HoBzLWEOphN7D+Pa5Cx0zwqHf8h5/NJUWXnNO89/Kca9oGZOm6t2+Dk0OYa+nw1s +d1CWlzSYobilQiEEiYLnPgRLoNiJt7hDZqE4ubVcIDptN/SfiOr5cg6aao2Vdb0P +iNcKn531AgMBAAECggEAJj6R8AMtYy4kuXs3wBRPMfNpESDcMKFQ3Fuwu6WavYVs +AedbdIOcb9OVGiNMDyGZCbWNpl94OfVt86Xhnu7+lw3Q/dzGwujzW2yroX1pSfqq +ErdBIgORxwgxHw9SiEMuMQGW60rjYF0X5WkVRs5StuFfC/udAjJPbtvKhPy58zBu +C2ZtcZFmf4KuwzCkZT8K8zvEFcOOXNIhtKMdPRSsbO7EpWn+J3SA+mG3nKo36swe +6EptIgFo4Z0dSzS1xK50iNOw10dKgxOSJpDjg6vH/cGduUxGvGWW0kiV1vD0ndUU +U1qyQSwlfFfFUk9WrtxoWFCXgZWyhoSONQmG9eRlgQKBgQDtLYVNIKK4n/TEiz9s +BtkFmh+ohQ6vjorjkzrj88Szxy9MivCD5vvU2Q61V15H9255RxgRPZ1lczGP++N2 +IckY+iC4vmeEyKOyFX8XYaJILEXPRkoyj6/PMyS8eWciZ08QW/bhOEDSiIlFrZ+4 +C/0KfMaT3MHRFRNV+jlza2RzNQKBgQDiQ7W+xtCqsvSXbVjPrC4dmxXlo1B/O4P3 +KsdiNoGpzu1I9nfKNKlfwfnzI5vtk7dS85kZPuCQ18fOQ7aO6w3VReg91zoEDSAV +GSJLVHeiRr+XOBCV4Cpy3PloYD+m2f4niT/Ru6CfEmHq6IoouWCqSobZYlkIHtAe +Rfjg1b8XwQKBgQDU8e7iDbArZWok4QTjX86QKg3MhxJlauymYPUZ3y63XtnHtmM1 +ejiWpP+Ar1217aChgMRKDnD3uhJAvR4/XRwpscGZPVLCNEZMyOIfwwgqFp3i+K/7 +b2ig9sE/+xwvFmQ1QuyIz7HblLzy820YmKLrPJYqAaV+rJZ8tjnIuB4rFQKBgF69 +PS9JbfvbfKCfD229SX3p/uwtSrpLgEEQ73VHH1mrpB1F3fiTvkuzG+ZbhaGflUYt +a3BvrHXZc/cA0ULcVulzIQAry0YA/Or4oCxsjL8s4nH97qvitcNslR3IFz7Uzh9d +z5QQ/Ps+JjPG9HqCzF/hXr9M97x3smrKn5/8v8vBAoGBAIRJPSik2ILc8V/1FnPa +aOfcBJwcuKdRJ4yVWiwe6meQCLWBva19rH8qx/D7h+r96a8OlSDZuYltM/UF5rV5 +lEKUs7uH9Xqw8QMMI17fQ//aRx265PJvdLtoO302IUQiC5qZx9YZKSLIzetVEapD +SfKVumS7pEPZgZI6WD5YoahG +-----END PRIVATE KEY----- diff --git a/test/configs/certs/rdns/client-e.pem b/test/configs/certs/rdns/client-e.pem new file mode 100644 index 0000000..61967a7 --- /dev/null +++ b/test/configs/certs/rdns/client-e.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDnzCCAoegAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcYwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx +MzIyMzAyM1owgYYxETAPBgNVBAMMCEpvaG4gRG9lMQ8wDQYDVQQDDAYxMjM0NTYx +DTALBgNVBAMMBGpkb2UxDjAMBgNVBAsMBVVzZXJzMRYwFAYDVQQLDA1Pcmdhbmlj +IFVuaXRzMRQwEgYKCZImiZPyLGQBGRYEYWNtZTETMBEGCgmSJomT8ixkARkWA2Nv +bTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANGg6vnpOjEgyfClqVW4 +UnAOc+i0DS2geZB7b6IZS2w5oNBblRIrKdXtis/YOQFPNqL4rKgDpxbpvcyLwUiK +8mT2GTGdansK5KfR0VlWjIzBK7J6XD59/7wKfqE2aD2zQGBCSWTql7kuwaFHC6fo +I0+eUlqljJzBofd7tbh+IWuzmePPjKthBNBk8Dp8HJu1xPk1q08xYHMIeX0egHMt +YQ6mE3sP49rkLHTPCod/yHn80lRZec07z38pxr2gZk6bq3b4OTQ5hr6fDWx3UJaX +NJihuKVCIQSJguc+BEug2Im3uENmoTi5tVwgOm039J+I6vlyDppqjZV1vQ+I1wqf +nfUCAwEAAaN2MHQwMgYDVR0RBCswKYIJbG9jYWxob3N0ggtleGFtcGxlLmNvbYIP +d3d3LmV4YW1wbGUuY29tMB0GA1UdDgQWBBToYUZkIK4kXJRs9Af7V10od3oJiDAf +BgNVHSMEGDAWgBRzHyX4gPDW9dBvKynDv2pvToKOrTANBgkqhkiG9w0BAQsFAAOC +AQEAKMs8gDYeTAITUMJ2cfQqU2t1I7DGIt4eQ/bJ1LWOhC6eUXpUrwjjNo4pq95P +cct0haleycD4vjZF4/Jv81oSIUsGzQ8r3LZhJlPnzbZzaA4i4Tpxnw8JRE5iiF+z +lm0Cl783Nh8voVKE1uJrSx+pMTH9Ihwu7vBklNI192zLPLPtDFSZ497oLVt1e8Wt +2urYXF4/Wb1pyGL49gp/eOAEpw2j6pyxGSK/QyXyvvPnDLqNXJCOTyx7YQRvZnBy +h9vTtHg4HSKmgrOHA/taW051lh2PQbi3pbi+ik3rk68QY/Y7b3fKjc/6lvXPRoh+ +94valHgOVFJaSG+Qil0A8i6bzQ== +-----END CERTIFICATE----- diff --git a/test/configs/certs/rdns/client-f.key b/test/configs/certs/rdns/client-f.key new file mode 100644 index 0000000..73e026c --- /dev/null +++ b/test/configs/certs/rdns/client-f.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCgF4OXW1j7wJKD +Kq7dKn4MU19yTzR01MkX1rsrC5kkFsihxNYd2pc8MdGB+VaJXaDwT7p/TL1FpGtK +ykgIp01nzceJNrfV6T79aslhvB/nt0BTKlCDDZTRogS6viJUAZGqRSWzbGcXt5GR +LpRGeXO6OpjVdQQqDwTMdbuHRTZ99VDfy1x8TwzWX4NA7xNUuoPfynWmmHKf80/F +kIgxWaRHDwUtKxn4ItVKNfMDAWfArdTMH7ZO4AYfIyM2WS82H587a0DilwR1Pcr2 +q7o6mFB807sdSrjiClUhWwM6A0KVsxnUvQAWkyYh9n01vnDYHrrYK74q1uMUJogc +45EXMhGfAgMBAAECggEAHC+XpQXZuFVS1lIlAF5zW9Mv+k20pB80moRdZro/h6mV +QFoKAq6SMvBofRQ7pqq746N959zIdCL7GutbXF6sU6rYxzFBk6FhzDdJZnI7Qc4z +PpE0JKa2/WOjpodEPvsZvNs4xIrA6d2RfPiw9Wl7tR2M/AQCrC8Bvse5sRq/+dP4 +U24eetvPJDTYLk3PFnZJPShAUEOcKUaL/73nIwCFbC8ez3ae9EIt4CO4ljHORp6h +g2zJFei7ie6VLHAZx4WUeGTVlhgF2bYjsjM1h254ZTIfNgqGs/DaOwzO2kpC5b+m +TE3Tuq1r0b7uMEnsCJb4gj39RYOqThZ9IG3I1oXKiQKBgQDMp67wfZm4jKeNfwr2 +gUxR7Btn0XqB6aMBofAgQiUuq9z8lRr8PTM1wPnw/ZKN6mYunIhXHGP+t8628hJx +FAy0fSy2sy9ucxipcDlyPBalSjtj1i1jqmZJtvmal7zfi0S/wgBn/J0X0lw7927Z +ai1VPVoHOeTouV4PhcJPHkqOuQKBgQDIQa+N4q4kCvmoMJSIHT9/vUV60uOO09QN +4WEtSUdrZVYnSYvO52f11bh1ZLRiZayEfov3vJ9LsLhnil5pNhEvxCZ3v+z5Fqgi +PRxY8+PnWleMC4QhCQOywpUbC5HqvqyEhGVVYt/CKbD0BKJoK/iVSkVUIhQuvh32 +W1x3wcG3FwKBgQCC9iL6fkVqVYe7Ajsvt1nxape2/dPZYnsPRmN1IR7OGOiXMYtN +MpScp1rKHlo0OnUdCsoshFxw2YqMg6fNeLkQnrGIUG3fzgNQGiKIuW78Yt5SavIs +vxQpw89CYCtbGbHqy+iaooqcfd7L+PCUbF+KFnKQATo8urI4WK4ZFxc68QKBgBZs +3Kbr6opYvEpsXkW72L+KR3yQnzEAYa/IPGSg3yGUsIgnwUNDQK4T59slktmt/xq7 +PRtaARCt0oitwxLPHi+WLKSeVoAyXFOxOjpv3WasBD/Hjl1QsBxVk/L6YbXC8njI +hryAHQSWPJ2m4zT9L5IwRgE867usSJis9HbouLOzAoGBAIZtLLPx8KYgjiVx8QPx +4uggZ7iblh1aI5ZO0XbsmWQfVDJ7HKpLFc9vd+nX2++Mv0vm3yv2WslMVliFvjXl +tgZNx10zn2usg+Cqy5728ldTXnXKA9q2KZOfhVPHv3sTao9bji5tywjnvt4ZLp+j +ngI3derKCSC10cXUiDEs3dXI +-----END PRIVATE KEY----- diff --git a/test/configs/certs/rdns/client-f.pem b/test/configs/certs/rdns/client-f.pem new file mode 100644 index 0000000..b80de9e --- /dev/null +++ b/test/configs/certs/rdns/client-f.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDfDCCAmSgAwIBAgIURF3HIP6jjAixTsMisMxARhqRfccwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx +MzIyMzAyM1owZDETMBEGCgmSJomT8ixkARkWA29yZzEXMBUGCgmSJomT8ixkARkW +B09wZW5TU0wxITAMBgNVBAoMBXVzZXJzMBEGCgmSJomT8ixkARkWA0RFVjERMA8G +A1UEAwwISm9obiBEb2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCg +F4OXW1j7wJKDKq7dKn4MU19yTzR01MkX1rsrC5kkFsihxNYd2pc8MdGB+VaJXaDw +T7p/TL1FpGtKykgIp01nzceJNrfV6T79aslhvB/nt0BTKlCDDZTRogS6viJUAZGq +RSWzbGcXt5GRLpRGeXO6OpjVdQQqDwTMdbuHRTZ99VDfy1x8TwzWX4NA7xNUuoPf +ynWmmHKf80/FkIgxWaRHDwUtKxn4ItVKNfMDAWfArdTMH7ZO4AYfIyM2WS82H587 +a0DilwR1Pcr2q7o6mFB807sdSrjiClUhWwM6A0KVsxnUvQAWkyYh9n01vnDYHrrY +K74q1uMUJogc45EXMhGfAgMBAAGjdjB0MDIGA1UdEQQrMCmCCWxvY2FsaG9zdIIL +ZXhhbXBsZS5jb22CD3d3dy5leGFtcGxlLmNvbTAdBgNVHQ4EFgQU1Fzej0nnnMw9 +Aai0AC5g2XsdiDcwHwYDVR0jBBgwFoAUcx8l+IDw1vXQbyspw79qb06Cjq0wDQYJ +KoZIhvcNAQELBQADggEBAKqH6YEyYgW5CnhHNOlbvB7kCNrQN/nAITWJILkJRtch +hMhsODV49fj593Rp3vTJXJ+fRCwJeF5pCfHIyUaEC9FM8IERzK7yZUW3h6X/KfwW +vHdRabjsuNuXk92wvslZK82jjosDBvdLv0pnApVv5OPnSXDUr/kGFnXhe7OQgLAb +OE5o3jRHWGCiSa1CnXrY4fzjDNR/MRgk7N7nQ29cY5fC//3jtBlYlcY7smjH81OF +9kFgQK1Mf83cN3zFLkVejLpquVM1CX210Z0GV7hEg6zr0jS8v1YRnCQ9xp+Euhd+ +TIU2SuidadW4ww/yCWsBfBuoX8ijByvG8sYyQ/jXL3s= +-----END CERTIFICATE----- diff --git a/test/configs/certs/rdns/server.key b/test/configs/certs/rdns/server.key new file mode 100644 index 0000000..c32374e --- /dev/null +++ b/test/configs/certs/rdns/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCjc0LulDTiYpXX +tTTXW8ubZueU3abq16rixbIbpQ+hZlpGhPOMDQbBEJdFuurt6GiwUJYvPonsuLxM +VhwDBGwKAs8Sw0vjwIHoPNBo8oKa7lW6Be2vv1+VBiDOd8ua7xnrLBmpKgADP40a +J+VAgSSCAVBA19tG05CJbxGnGHBgXqNwMamR+DhQYp1JV8JqivGA4UqOAchHOyp9 +ucFItKJ6EeB/qq1O53KnEcB5WqHYGy06/tJPQ/jsdQxakSaVMrKvrhTY4XkSMPAA +sOcrX/BFpWkloNthICYP2ROzV2HYTDfQgGt1SU4wdNseOsRfs0TQRKMa29gx7iQI +pH3D2i/FAgMBAAECggEAArqksjOWy3Xebx7XLkZZ2LMn6DgoNpEZyRYIcoY3w9ff +54AGSf8YrwYOJIupqwW/o7wAufK3HlT6TeL9Lj63aBZiazuIC5o2ARDs4Oe1rw0O +MOAy9wWAvM4Ao7nj1CcsXV0b45NwMx2vRzHF1eUiHSqU8rQ39M+JAQZx+sKBc9Zx +2oiL2Dy2q6YBqgJA24dDk2S2bez32ZWBdyJ+/kMzanhsHZb4a8StuxMjX201vhmG +QzV9PR+oi2dn9Kee/GX4yZUVDZ9A1jOdusk0+SnAPJDMi2JEp5kj898GXW22PZCw +mQ8iEqJCmsB5zWUiYiF0IkmTeSkukZoX3YaTts9jYQKBgQDRBnGrFimcM9QJVbeR +ObGix2TxmgKm8qkLuUb4LX/AgLHCNAy5SdodMOFHdV+zZlrqwY3d54yiFwix5ujD +AaJcAhOKQGX0aTEz3Vd4XIj9Y/uMwhU9XG5zTbbT0QlVWLDy93KJYeVa9ateJsg5 +DBcF025ApLQv/Z6bT2R7VIjdYQKBgQDILtFktkfvQS/uMcJDX35Vw8VDsJWc2Aas +oBQBfHwFM7mzfoofmHnUFh/901uSXvj3YVgNHnSQx7+sJQhViO3RFHSNHwyTs7aV +RErs/vseFpYf9SorASygkOR9bH4z+KIw2GpSMUIWq5c8E1xe7lkZUuHJhQht8Vzv +dVwHBfUo5QKBgQDHwObD30wokIcj3Jyu1nnh27emA85hCSlvoMInziN2LgayVteK +Av/EQcAocAzi/wAHtK0E0ZFeHbEbglYSde9ZCkruJhjI9/YjYHWE+rmXngL5Q0jU ++Q48dpov1maa/0UrDDqS+9EZmgkI0vspOVqPIL0OXdgsvKzkM30NN34MQQKBgEzn +N4giQsZWW6x9Ly0kzWrzV6AmgYOMthuxL55WjWqOMYQUU8nJkNv6V/XyMZasp6aZ +nfMERTGtmyPt4iLCBOKyogfo6rL/cmArqUEcv6oScT/7tmRpAhT5NN4+RMmAdgaf +zVCgHcyJxQLOSUkq9c21uonpcpCSDersQhtsnX9FAoGBAMLm40fsn5hqeOSHfF7W +ZEX0eKSBPuCCWTtUi5c7pPmyj0B8+9o3wiP6YHcs91ysfAWgnJqBlZkPxkwC5zvT +AlS1GNsCCQzZT8c60QiOESTMWa+398AGtFRYVNjbamGcQOW3lprfeXiIQCAyQ2d1 +GNZICja01bMZsLidFoyUIH95 +-----END PRIVATE KEY----- diff --git a/test/configs/certs/rdns/server.pem b/test/configs/certs/rdns/server.pem new file mode 100644 index 0000000..f0f4d72 --- /dev/null +++ b/test/configs/certs/rdns/server.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDLDCCAhSgAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcEwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx +MzIyMzAyM1owFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAo3NC7pQ04mKV17U011vLm2bnlN2m6teq4sWyG6UPoWZa +RoTzjA0GwRCXRbrq7ehosFCWLz6J7Li8TFYcAwRsCgLPEsNL48CB6DzQaPKCmu5V +ugXtr79flQYgznfLmu8Z6ywZqSoAAz+NGiflQIEkggFQQNfbRtOQiW8RpxhwYF6j +cDGpkfg4UGKdSVfCaorxgOFKjgHIRzsqfbnBSLSiehHgf6qtTudypxHAeVqh2Bst +Ov7ST0P47HUMWpEmlTKyr64U2OF5EjDwALDnK1/wRaVpJaDbYSAmD9kTs1dh2Ew3 +0IBrdUlOMHTbHjrEX7NE0ESjGtvYMe4kCKR9w9ovxQIDAQABo3YwdDAyBgNVHREE +KzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYD +VR0OBBYEFBTXnZytLW93IszwKUGgXXh7OdNtMB8GA1UdIwQYMBaAFHMfJfiA8Nb1 +0G8rKcO/am9Ogo6tMA0GCSqGSIb3DQEBCwUAA4IBAQDCrwa/UTnBOoJ/X2FIfaWP +g0llr0OSQx3l2RweWte6O90VB9AWTgstYRVErmYyXV70lYNp+HOPpxak1DEGr+P8 +4REsMjX+odz9+UGOq/n5N+0VAfLTsQ9CG5EnHsfzwamgL/Ax3czrzgvmP4lz0tvp +07le9YUWkuG9UsUhN4/qe65LVweg8AfhihiijlcQe3WnrB7WvyyOZO81lBnLIIar +qq62NSNxPWNa/TGX/og+E5HwTPcMNeMOsok5D5TQtx1zNX+2Zj7i2Py3ScTCCY/G +4U1nO4k+APE/3BKuvnrz8ZJ1UglT0lrVEkVqzW7TNCJpDz37A9QIQVLpw8oOr+Yc +-----END CERTIFICATE----- diff --git a/test/configs/certs/regenerate_rdns_svid.sh b/test/configs/certs/regenerate_rdns_svid.sh new file mode 100755 index 0000000..1fe3586 --- /dev/null +++ b/test/configs/certs/regenerate_rdns_svid.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +set -euo pipefail +# +# regenerate_rnds_svid: just remake the certs in the rdns & svid dirs. +# +# We're getting the hard requirements down in scripts, can integrate all into +# one all-singing all-dancing script later, so that anyone can regenerate +# without having to read test source code. +# + +progname="$(basename "$0" .sh)" +note() { printf >&2 '%s: %s\n' "$progname" "$*"; } +warn() { note "$@"; } +die() { warn "$@"; exit 1; } + +readonly CERT_SUBDIR='test/configs/certs' +readonly RDNS_SUBDIR='rdns' +readonly SVID_SUBDIR='svid' + +# WARNING: +# This data is hard-coded into tests such as TestTLSClientAuthWithRDNSequence +# so do not "fix" it without editing the tests too! +readonly COMMON_SUB_COUNTRY=US +readonly COMMON_SUB_STATE=CA +readonly COMMON_SUB_LOCALITY='Los Angeles' +readonly COMMON_SUB_ORG=NATS +readonly COMMON_SUB_ORGUNIT=NATS +readonly COMMON_SUBJECT_OOU="/C=$COMMON_SUB_COUNTRY/ST=$COMMON_SUB_STATE/L=$COMMON_SUB_LOCALITY/O=$COMMON_SUB_ORG/OU=$COMMON_SUB_ORGUNIT" +readonly COMMON_SUBJECT_OUO="/C=$COMMON_SUB_COUNTRY/ST=$COMMON_SUB_STATE/L=$COMMON_SUB_LOCALITY/OU=$COMMON_SUB_ORGUNIT/O=$COMMON_SUB_ORG" +readonly RDNS_COMMON_SAN='subjectAltName=DNS:localhost,DNS:example.com,DNS:www.example.com' + +readonly CA_KEYFILE=ca.key CA_CERTFILE=ca.pem CA_NAME='NATS CA' CA_SERIAL_FILE='ca.srl' +readonly RSA_SIZE=2048 +readonly DIGEST_ALG=sha256 +readonly CERT_DURATION=$((2 * 365)) + +REPO_TOP="$(git rev-parse --show-toplevel)" +CERT_ABSOLUTE_DIR="$REPO_TOP/$CERT_SUBDIR" +readonly REPO_TOP CERT_ABSOLUTE_DIR + +okay=true +for cmd in openssl ; do + if command -v "$cmd" >/dev/null 2>&1; then + continue + fi + okay=false + warn "missing command: $cmd" +done +$okay || die "missing necessary commands" + +make_keyfile() { + local keyfile="${1:?need a keyfile to create}" + (umask 077; openssl genrsa "$RSA_SIZE" > "$keyfile") +} + +ensure_keyfile() { + local keyfile="${1:?need a keyfile to create}" + local description="${2:?need a description}" + if [ -f "$keyfile" ]; then + note "reusing EXISTING $description file: $keyfile" + return 0 + fi + note "creating NEW $description file: $keyfile" + make_keyfile "$keyfile" +} + +o_req_newkey() { openssl req -newkey "rsa:$RSA_SIZE" -nodes "$@"; } + +o_x509_casign() { + local extfile_contents="$1" + shift + openssl x509 -req -days "$CERT_DURATION" \ + -CA "$CA_CERTFILE" -CAkey "$CA_KEYFILE" -CAcreateserial \ + -sha256 \ + -extfile <(printf '%s\n' "$extfile_contents") \ + "$@" +} + +o_new_cafile() { + local keyfile="$1" cacertfile="$2" + shift 2 + openssl req -x509 -new -key "$CA_KEYFILE" -out "$CA_CERTFILE" -outform pem \ + -days "$CERT_DURATION" -subj "/CN=$CA_NAME" \ + "$@" +# We want these: +# -addext subjectKeyIdentifier=hash \ +# -addext authorityKeyIdentifier=keyid:always,issuer \ +# -addext basicConstraints=critical,CA:true \ +# but even without an extensions section, those seem to have been included anyway, +# resulting in a doubling when I created them, and Go cert parsing did not like +# the doubled X509v3 extensions data, leading to a panic. +# So, removed. +} + +# ######################################################################## +# RDNS + +note "Working on rdns files" +cd "$CERT_ABSOLUTE_DIR/$RDNS_SUBDIR" || die "unable to chdir($CERT_ABSOLUTE_DIR/$RDNS_SUBDIR)" + +note "creating: CA" +# This one is kept in-git, so we don't force-recreate. +# TBD: should we delete, as we do in the parent dir? +ensure_keyfile "$CA_KEYFILE" "rdns CA key" +o_new_cafile "$CA_KEYFILE" "$CA_CERTFILE" + +make_rdns_client_pair() { + local client_id="$1" + local subject="$2" + shift 2 + local prefix="client-$client_id" + note "creating: $prefix" + rm -fv -- "$prefix.key" "$prefix.csr" "$prefix.pem" + # TBD: preserve the .key if it already exists? + # For now, just using the same ultimate command as was documented in the tls_test.go comments, + # so that we minimize moving parts to debug. Key preservation is a future optimization. + # The "$@" goes into the req invocation to let us specify -multivalue-rdn + o_req_newkey -keyout "$prefix.key" -out "$prefix.csr" "$@" -subj "$subject" -addext extendedKeyUsage=clientAuth + o_x509_casign "$RDNS_COMMON_SAN" -in "$prefix.csr" -out "$prefix.pem" + rm -v -- "$prefix.csr" + echo >&2 +} + +make_svid_rsa_pair() { + local prefix="$1" + local subject="$2" + local san_addition="$3" + shift 3 + note "creating: $prefix" + rm -fv -- "$prefix.key" "$prefix.csr" "$prefix.pem" + # TBD: preserve the .key if it already exists? + # For now, just using the same ultimate command as was documented in the tls_test.go comments, + # so that we minimize moving parts to debug. Key preservation is a future optimization. + o_req_newkey -keyout "$prefix.key" -out "$prefix.csr" -subj "$subject" -addext extendedKeyUsage=clientAuth + o_x509_casign "$RDNS_COMMON_SAN${san_addition:+,}${san_addition:-}" -in "$prefix.csr" -out "$prefix.pem" + rm -v -- "$prefix.csr" + echo >&2 +} + +# KEEP DN STRINGS HERE MATCHING THOSE IN tls_test.go SO THAT IT's COPY/PASTE! + +# C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo1, DC = foo2 +make_rdns_client_pair a "$COMMON_SUBJECT_OOU/CN=localhost/DC=foo1/DC=foo2" + +# C = US, ST = California, L = Los Angeles, O = NATS, OU = NATS, CN = localhost +make_rdns_client_pair b "$COMMON_SUBJECT_OOU/CN=localhost" + +# C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo3, DC = foo4 +make_rdns_client_pair c "$COMMON_SUBJECT_OOU/CN=localhost/DC=foo3/DC=foo4" + +# C = US, ST = CA, L = Los Angeles, OU = NATS, O = NATS, CN = *.example.com, DC = example, DC = com +make_rdns_client_pair d "$COMMON_SUBJECT_OUO/CN=*.example.com/DC=example/DC=com" + +# OpenSSL: -subj "/CN=John Doe/CN=123456/CN=jdoe/OU=Users/OU=Organic Units/DC=acme/DC=com" +make_rdns_client_pair e "/CN=John Doe/CN=123456/CN=jdoe/OU=Users/OU=Organic Units/DC=acme/DC=com" + +# OpenSSL: -subj "/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe" +# WITH -multivalue-rdn for the -req +make_rdns_client_pair f "/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe" -multivalue-rdn + +note "creating: server" +rm -fv -- "server.key" "server.csr" "server.pem" +o_req_newkey -keyout "server.key" -out "server.csr" -subj "/CN=localhost" +o_x509_casign "$RDNS_COMMON_SAN" -in "server.csr" -out "server.pem" +rm -v -- "server.csr" +echo >&2 + +rm -rfv "$CA_SERIAL_FILE" + +# ######################################################################## +# SVID + +note "Working on svid files" +cd "$CERT_ABSOLUTE_DIR/$SVID_SUBDIR" || die "unable to chdir($CERT_ABSOLUTE_DIR/$SVID_SUBDIR)" + +note "creating: CA" +ensure_keyfile "$CA_KEYFILE" "svid CA key" +o_new_cafile "$CA_KEYFILE" "$CA_CERTFILE" + +make_svid_rsa_pair client-a "$COMMON_SUBJECT_OOU/CN=localhost/DC=foo1/DC=foo2" 'URI:spiffe://localhost/my-nats-service/user-a' + +make_svid_rsa_pair client-b "/C=US/O=SPIRE" 'URI:spiffe://localhost/my-nats-service/user-b' + +make_svid_rsa_pair server "/CN=localhost" '' + +rm -rfv "$CA_SERIAL_FILE" + +# FIXME: svid-user-a and svid-user-b are ECC certs, but not expiring at the +# same time as the rest and with differing requirements, so not coding that up +# now. diff --git a/test/configs/certs/regenerate_top.sh b/test/configs/certs/regenerate_top.sh new file mode 100755 index 0000000..bd8d761 --- /dev/null +++ b/test/configs/certs/regenerate_top.sh @@ -0,0 +1,228 @@ +#!/bin/sh +set -eu +# +# regenerate_top: just remake the certs in this top-dir +# we don't (currently) handle any sub-dirs +# + +progname="$(basename "$0" .sh)" +note() { printf >&2 '%s: %s\n' "$progname" "$*"; } +warn() { note "$@"; } +die() { warn "$@"; exit 1; } + +readonly COMMON_SUB_COUNTRY=US +readonly COMMON_SUB_STATE=California +readonly COMMON_SUB_ORG=Synadia +readonly COMMON_SUB_ORGUNIT=nats.io +readonly COMMON_SUBJECT="/C=$COMMON_SUB_COUNTRY/ST=$COMMON_SUB_STATE/O=$COMMON_SUB_ORG/OU=$COMMON_SUB_ORGUNIT" + +readonly TEMP_CONFIG=openssl.cnf +readonly TEMP_CA_KEY_REL=ca-key.pem +readonly CA_FILE=ca.pem +CA_NAME="Certificate Authority $(date +%Y-%m-%d)" +readonly CA_NAME +readonly RSA_SIZE=2048 +readonly DIGEST_ALG=sha256 +readonly CERT_DURATION=$((10 * 365)) + +okay=true +for cmd in openssl ; do + if command -v "$cmd" >/dev/null 2>&1; then + continue + fi + okay=false + warn "missing command: $cmd" +done +$okay || die "missing necessary commands" + +delete_list="" +trap 'if test -n "$delete_list"; then rm -rfv $delete_list; fi' EXIT +add_delete() { + delete_list="${delete_list:-}${delete_list:+ }$*" +} + +# Issuer: C = US, ST = CA, O = Synadia, OU = nats.io, CN = localhost, emailAddress = derek@nats.io + +CA_DIR="$(mktemp -d)" +add_delete "$CA_DIR" +mkdir "$CA_DIR/copies" +touch "$CA_DIR/index.txt" + +readonly CA_DIR +readonly CA_KEY="$CA_DIR/$TEMP_CA_KEY_REL" + +COMMON_X509V3=' +basicConstraints = CA:FALSE +nsComment = "nats.io nats-server test-suite certificate" +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer:always +subjectAltName = ${ENV::SUBJECTALTNAME} +' + +cat > "$TEMP_CONFIG" < "$keyfile") +} + +ensure_keyfile() { + local keyfile="${1:?need a keyfile to create}" + local description="${2:?need a description}" + if [ -f "$keyfile" ]; then + note "reusing EXISTING $description file: $keyfile" + return 0 + fi + note "creating NEW $description file: $keyfile" + make_keyfile "$keyfile" +} + +o_req() { openssl req -config "$TEMP_CONFIG" "$@"; } + +sign_csr() { + local san="${1:?need subjectAltName}" + shift + env SUBJECTALTNAME="$san" openssl ca -config "$TEMP_CONFIG" -policy policy_anything -batch "$@" +} + +make_keyfile "$CA_KEY" +o_req -x509 -new -key "$CA_KEY" -out "$CA_FILE" -outform PEM -days "$CERT_DURATION" -subj "$COMMON_SUBJECT/CN=$CA_NAME" -extensions v3_ca + +echo +readonly CLIENT_KEY=client-key.pem +BASE=client-cert +ensure_keyfile "$CLIENT_KEY" "client key" +o_req -new -key "$CLIENT_KEY" -out "$BASE.csr" -subj "$COMMON_SUBJECT/CN=localhost" +add_delete "$BASE.csr" +sign_csr "DNS:localhost, IP:127.0.0.1, IP:::1, email:derek@nats.io" -in "$BASE.csr" -out "$BASE.pem" -extensions nats_client + +echo +readonly CLIENT_ID_AUTH_KEY=client-id-auth-key.pem +BASE=client-id-auth-cert +ensure_keyfile "$CLIENT_ID_AUTH_KEY" "client id auth key" +o_req -new -key "$CLIENT_ID_AUTH_KEY" -out "$BASE.csr" -subj "$COMMON_SUBJECT/CN=localhost" +add_delete "$BASE.csr" +sign_csr "DNS:localhost, IP:127.0.0.1, IP:::1, email:derek@nats.io" -in "$BASE.csr" -out "$BASE.pem" -extensions nats_client + +echo +readonly SERVER_KEY=server-key.pem +BASE=server-cert +ensure_keyfile "$SERVER_KEY" "server key" +o_req -new -key "$SERVER_KEY" -out "$BASE.csr" -subj "$COMMON_SUBJECT/CN=localhost" +add_delete "$BASE.csr" +sign_csr "DNS:localhost, IP:127.0.0.1, IP:::1" -in "$BASE.csr" -out "$BASE.pem" -extensions nats_server + +echo +readonly SK_IPONLY=server-key-iponly.pem +BASE=server-iponly +ensure_keyfile "$SK_IPONLY" "server key, IP-only" +# Be careful not to put something verifiable that's not an IP into the CN field, for verifiers which check CN +o_req -new -key "$SK_IPONLY" -out "$BASE.csr" -subj "$COMMON_SUBJECT/CN=ip-only-localhost" +add_delete "$BASE.csr" +sign_csr "IP:127.0.0.1, IP:::1" -in "$BASE.csr" -out "$BASE.pem" -extensions nats_server + +echo +readonly SK_NOIP=server-key-noip.pem +BASE=server-noip +ensure_keyfile "$SK_NOIP" "server key, no IPs" +o_req -new -key "$SK_NOIP" -out "$BASE.csr" -subj "$COMMON_SUBJECT/CN=localhost" +add_delete "$BASE.csr" +sign_csr "DNS:localhost" -in "$BASE.csr" -out "$BASE.pem" -extensions nats_server + +for SRV in srva srvb; do + echo + KEY="${SRV}-key.pem" + BASE="${SRV}-cert" + ensure_keyfile "$KEY" "server key, variant $SRV" + o_req -new -key "$KEY" -out "$BASE.csr" -subj "$COMMON_SUBJECT/CN=localhost" + add_delete "$BASE.csr" + sign_csr "DNS:localhost, IP:127.0.0.1, IP:::1" -in "$BASE.csr" -out "$BASE.pem" -extensions nats_server +done + diff --git a/test/configs/certs/sans/ca.pem b/test/configs/certs/sans/ca.pem new file mode 100644 index 0000000..5c6389b --- /dev/null +++ b/test/configs/certs/sans/ca.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDBzCCAe+gAwIBAgIUc56AEpW980qCeFPAyYjDMQmgD4MwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU4NTlaGA8yMTI0MDMy +NDA3NTg1OVowEjEQMA4GA1UEAwwHTkFUUyBDQTCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBALcG8Uka9njBYwtVEachD6ZzpiEZV8CVHU4lfXv2e0VD6D/z +Q8DTWc7TRMy+15u/h8ZSNSQm2mNVDUA5SbJPoeka6LqXl47ZMhY9fwIa860KxphQ +ODdwu75v6jjh2IZz4BW3PGjeVAN6pJfejmn07y/lGa+jDvKPl0BEee+Vj2APpDu0 +JRtnNoLvJryuddQq9AaagPXP5wsvS5mRVqjM3w9dvvxGJUZW+nEKDX0cEw3nv1h4 +K57Z/fIHOk9cTH2b960QG/MjadJi/8tvVduBRt6ZZj9M5xQ4dg5+28lJbVq4eoAi +lN4XIbah1tg2q0zWBprYieVLWEgT6680rFMrI2cCAwEAAaNTMFEwHQYDVR0OBBYE +FK442m9BmE189lFzd57Hbz/b95psMB8GA1UdIwQYMBaAFK442m9BmE189lFzd57H +bz/b95psMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAEx7HfRg +QoInmdQwN8IKVB21RU7gdkFT6DxmS/ZEW3SFXctrkPftyToJ3PoFlp7rYooKCI8e +MPOo2GeSt8Ocv2A9qgV+X5euZzKDrOW53kvk6BpvqiadfmOjw/aa4HwjSDh3lUwP +hgQWriNYK7NBXINIxHXT6tRv9NvIvR3UeJ2DBujwAF5rBmTIBI9b5N1oQnE2jeY/ +2LsV8rcnWRcYEoyhb55AJ5aD640saGJ0StmdwRK9nOFzg02xUP4z8QuentlwPVVo +DhCGGbMLY9fneXjrD4jEUjAHn4CWuPLdn/agIrC96qv+PZYUl9hJoDJnshuPczyR +UQFYI1rGa9JuwRw= +-----END CERTIFICATE----- diff --git a/test/configs/certs/sans/client-key.pem b/test/configs/certs/sans/client-key.pem new file mode 100644 index 0000000..afad52f --- /dev/null +++ b/test/configs/certs/sans/client-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDkjw4sZLkZJPTP +/c4gIPXr0IacaCvNJne8fivl1FnVg0mG80zXQcXuPKi1HF3Qg/qZu38p6oZTfeJm +sjkCGVSsNTSY3yqZCM2mLIgQmaSCuKtnvIWjeBUtmMAx6bA0OphD5rYaItflqaVT +kJd2qDALXh6cHI8esCigBAZsnUS90mAgDCQVahMIDkz4jFC/AXwnJUlNzqa7w9PA +SIakuGFvCWgv6uS57BOgn+c3134fCYi2s+A8xhg4o+GanV6+5sbLuwMPo+0DvBfy +vkXIt51WoMm04gNymiVYYZxbNpe9M33ICQuFAoTb7RdN7mUylBVM9MDfdG7ne6TG +8J8Rzl39AgMBAAECggEARvhc0FAeYb5l/kezP6q8chL4GY0HPNabC36qJt1/X5s6 +s5rM23JjAYz4XfSG9P5mTGI6JEvclgRazsS8ivhOoIKM6IMzeP2Qze65+V2cBbNO +FhIl9RKOkPADjfwgaLdhHISBOzBilb/NxFr7jS7AbGc3XgZIMqFEnBtj20oiVEZu +l9BAoiDlAiMKt18AMGxsI6Kr5Px/vuNhQpxjKDw+47wbhHP7EW+6ywVFn+Nb9kV+ +8QC/14GZ9EbDPfKhlU+NqTIAKZbvI+NvAyETB8ezEnEkEcSr3FaWhA4WpHauhijx +qAJSBVsVinoFQVRHwDjZGDJ+M2QkFk24Jm+X3VQySQKBgQD7qFMRYF7+KbXBp8uK +YpW3pyM+28oHaeVjKckCtFuAIJS02yYo2neANPeVIthurBwl4pRSZchNiqW70nSW +aYPw+4JCprYCkfNZFFP6X5FzIlmGRCpgjuWzXHFgkSRnOg+GckcrMu1VcvVYuosC +KqQWkXSbupvrQTK1f8XZQTnR1wKBgQDogLG++I1+wLUFQmPw6c+RrTwj9i1SUEt6 +50pUp+9/KKDXDQTyjV/XZILmaLcn4jcqCXDIvEddkEUngkrazS6quwQtB4wpQgYZ +SSQ/eWsRCvA0VUI+6tHFThPGg1SZREXorhflm7YYa2nsLF118d1pHqKO/gUhBJX+ +Pv5Pzdy8SwKBgQC5E7PzxZJXDcnUIFk67wH4zPzIz8+m4CSJZ0Ojr4zTkCKNV9aa +mQDl0w56KeROkFkrK5W1e5FyJZN5rG995x/X7MCB0CVvgnMbgi24puxLZmm8qwkX +dkBMRqJDSLsjB7o/QHBCvvN4slDp7lcpQr7msha2KOlefNaUUOHqw2OIhQKBgQC5 +4go5kCYv7InNRqL4fTYCVen7JlpdsOxnunrm68zCcQ0GYdZOxVCWuDkfVSD5thY8 +eYe+NSkpWKqxR63o+JoSzaotBhe89JhDpwJf7Qb4fTJF5NQt0Tcc86tDzsPYNYle +2bEpVTBknZv1whKGtXQ7Es8MW3JmT3BL8LkJvKB5uQKBgQCIUUm3zKTXo4fxfR84 +485La1x9ThnOM7aDgLhKJLg0hQTMPSozsKS2wbOIbg45BS9BL/HGkjqHXryY35Ee +wphElPZGLtw4bd7xjSQykXjsCHJIg1lxnEG0sdTaVBD3iQnx4L2pCg8fY0iUC1m4 +KnOwsB6IfaV1EnouUebes3ydjQ== +-----END PRIVATE KEY----- diff --git a/test/configs/certs/sans/client.pem b/test/configs/certs/sans/client.pem new file mode 100644 index 0000000..131d670 --- /dev/null +++ b/test/configs/certs/sans/client.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDLjCCAhagAwIBAgIUTOgYUed1dyp+Gan755liL7b29wEwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU5MDBaGA8yMTI0MDMy +NDA3NTkwMFowFjEUMBIGA1UEAwwLd3d3Lm5hdHMuaW8wggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDkjw4sZLkZJPTP/c4gIPXr0IacaCvNJne8fivl1FnV +g0mG80zXQcXuPKi1HF3Qg/qZu38p6oZTfeJmsjkCGVSsNTSY3yqZCM2mLIgQmaSC +uKtnvIWjeBUtmMAx6bA0OphD5rYaItflqaVTkJd2qDALXh6cHI8esCigBAZsnUS9 +0mAgDCQVahMIDkz4jFC/AXwnJUlNzqa7w9PASIakuGFvCWgv6uS57BOgn+c3134f +CYi2s+A8xhg4o+GanV6+5sbLuwMPo+0DvBfyvkXIt51WoMm04gNymiVYYZxbNpe9 +M33ICQuFAoTb7RdN7mUylBVM9MDfdG7ne6TG8J8Rzl39AgMBAAGjdjB0MDIGA1Ud +EQQrMCmCCWxvY2FsaG9zdIILZXhhbXBsZS5jb22CD3d3dy5leGFtcGxlLmNvbTAd +BgNVHQ4EFgQUiUGJiS99EVH6/iPsQESqZSfXxCIwHwYDVR0jBBgwFoAUrjjab0GY +TXz2UXN3nsdvP9v3mmwwDQYJKoZIhvcNAQELBQADggEBAAugrGg7+6LX7UZ6YhCM +P/CGBnxIMODeuikNf3OAEjwNRJjdcy8OyHKd9pSwEzNZ0jZCN127UDPHQzZGw5JE +KCe3B4a0Lh0G5BQVOOTcUJM9et6aFtBDvNbKhiiwi11d5aAEQX7k3ugPPaEjP9si +SM1b8UBcaO3mGSnveMVjBiInzSkJdc8kPaAsssqvledmQa9RyM1y3ZEcvixbF+6d +pfQG61UXR3rnGEUAoHWmnKzJ+o/kFlThd2jtKT8T4qp2Ws1ga43V5kacFqyWcpfy +6dK1lfr3Mx4qHMlnFp7O9L4ciFLtLiv4ua9fi4zH8zS7EALkwQKxhaDHo6Afi1mI +ASw= +-----END CERTIFICATE----- diff --git a/test/configs/certs/sans/dev-email-key.pem b/test/configs/certs/sans/dev-email-key.pem new file mode 100644 index 0000000..e69fb4f --- /dev/null +++ b/test/configs/certs/sans/dev-email-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCoFArKsSQLasob +BGo4+d2WUxjW3rp7qMZ2M92RvefInqsAq3MBstcsKIV6RrvJYiMeCw4gXkrcnYGM +56/QtNx6fapKltO8FSOs7dSWLQquvOmYbPHmhDzVyb4/Wcied0i/LyeP5HlUtz/9 +F22R+MNLhCYacSytogTz9ZoCLLuqfaAfP7MdqajDeewSn9fO4AAtGbR7ysYU5lkT +7loxW9+VDXN1wKP3U+ziHJVPX2+438OXEVFlM2FCk8qdOja7lmHJnvZcQD7qOywQ +qEsY+mcjh/+JZk2xbhScSB/bsZFFIIfSu5qwUJm2lP9uBksk51FCxWjB1ldAh9mS +yH1rT3x9AgMBAAECggEAGk4GsVm/XyUPq+JJQJEEZ7gqGUjC7UMJCmtMGDjdK2X/ +s8thZKjpaZEO8MfcsGQSRGutAo5XT4c8BQImnzaLEgWUTvejfBpf2rrfDEDQ3O+W +bINSaYYVIk1gX7hMwFZBVaCK48d5YLOMSW8u+Aszf2BXeUhwml0Swt6Tg5ceKMye +fZa9geVZhC61Gtxbk+L4P4cnKLAvlxjFSn3iAS0YNGGsIVIHmpFQ5l0JdMLUF/4h +C6enmgzs9fxnL67Wy54MoICH4qEekW09AHbDI2byKgBNEmmm/xsQlRGoHdwo3QFV +aCDmul7V9LX1DMs3aamzr/+jw0/4bixZvRN9zGgH6QKBgQDraDBtrY2Fk+uFNkPL +zf29H3+W0bibfOKSoyTkvYHYxK44sYAN/ly7BRkH5CaYiTNQWJ26lVwi8nanSnNV +Nl7Hevx1C9Yn++guQtnUNOBvz1CSuRQ0L52US/YF4NKmnVN3Ny4DO99+bHXoWESh +9mMlerG2Axr/FKf8E8bdG81RuQKBgQC2yBBGAJolYaGZyDMgu3CjXuJP1hPuePtC +uXa2hp/uTf0D0bUg2vxDJjoZVSBFmpSoKaybsUip8gI3GFjJXGmu+No3gX8bRDaq +H42+I507I1wbYqaWtVW8HmveAK3qi5ed886p7JGYZW9olTj6sfRlvoWa+P5A/9uz +AUTOlMly5QKBgQDNVNfJEvS6aseoLij8f/SvHeZgWxW3KjtGxF7N1i6IMSX19X1I +t3GS/2NR6sNvkVzc3C7YLKdtJCgyy1HGJeKOBMxoG6b0wVlH4K+31Vder2oMULs6 +ub2tOISjo/KZueivt8W+tF7BG0HNJBDZZNweOOMBa7wEerP7wBRZkIKKoQKBgQCt ++JpUproRHm4b2wue+glp1iP97TsnXgt5JOGzNUwAHEbYXb/St9wnZbki531CArG/ +pXre7czFxM0K96d6cPU+TyoUbrM2lqSZJFNbSLac1TkT77+z7oDd/u6YbXkbpyX2 +d1qbLcoejV2O44lKRBrkxISSTrBh2aWZKXn+Tmu3aQKBgQCu7PIEgAfX78QrQWje +2owrnV/3DQCtBOh1PyQ5a4LgxsSuvO+qRdsK07tYVxKLc5TOq4DJcmbIzrkEptfO +WqMUoRk6BT7EfE7AIm2W3K9ohy6U7Jsej7YxhqYvMZllrksqluKbz51K/q1xDDFa +kkwCvAfMIRx8Z8+2lZkc6yjGug== +-----END PRIVATE KEY----- diff --git a/test/configs/certs/sans/dev-email.pem b/test/configs/certs/sans/dev-email.pem new file mode 100644 index 0000000..81214ab --- /dev/null +++ b/test/configs/certs/sans/dev-email.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDPDCCAiSgAwIBAgIUTOgYUed1dyp+Gan755liL7b29wAwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU5MDBaGA8yMTI0MDMy +NDA3NTkwMFowFjEUMBIGA1UEAwwLd3d3Lm5hdHMuaW8wggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQCoFArKsSQLasobBGo4+d2WUxjW3rp7qMZ2M92RvefI +nqsAq3MBstcsKIV6RrvJYiMeCw4gXkrcnYGM56/QtNx6fapKltO8FSOs7dSWLQqu +vOmYbPHmhDzVyb4/Wcied0i/LyeP5HlUtz/9F22R+MNLhCYacSytogTz9ZoCLLuq +faAfP7MdqajDeewSn9fO4AAtGbR7ysYU5lkT7loxW9+VDXN1wKP3U+ziHJVPX2+4 +38OXEVFlM2FCk8qdOja7lmHJnvZcQD7qOywQqEsY+mcjh/+JZk2xbhScSB/bsZFF +IIfSu5qwUJm2lP9uBksk51FCxWjB1ldAh9mSyH1rT3x9AgMBAAGjgYMwgYAwPgYD +VR0RBDcwNYIMYXBwLm5hdHMuZGV2gRJhZG1pbkBhcHAubmF0cy5kZXaBEXJvb3RA +YXBwLm5hdHMuZGV2MB0GA1UdDgQWBBQfZUwRFVCxJAU4eNYLvovRmH6WezAfBgNV +HSMEGDAWgBSuONpvQZhNfPZRc3eex28/2/eabDANBgkqhkiG9w0BAQsFAAOCAQEA +dVs5iZ1SPtR7fGQoNnMocd2GECOMZQ5cQz27GtLb8jKVbblBjWYNECUpHA44gWhx +YhT4gEBww0w7FeQEnWyNMj8vidEm/Fkz4c01zDqwCd5Lh4UyQlWRwppHnV9+j9Xa +Ye2T89L9txLFzkaHZDhbZJy5/SDs2lRy+6IE1vxmiMZdVIzVzHhLQUIBfC1gh0Yc +qfn2RVbulW11CUgSNXj7G5TPkHw/RG/vs3AsDdgXaCKfwttTirrpFl4OvGKH+xSV +XMykd7Ck3ScD3oPpp3oY/Nq8iHBB8EgB6SZSSVVL9zl/4jo1s5IBr2ekAhSv9HGj +29VtK+VVixHazzkGdvS/OA== +-----END CERTIFICATE----- diff --git a/test/configs/certs/sans/dev-key.pem b/test/configs/certs/sans/dev-key.pem new file mode 100644 index 0000000..81ef41f --- /dev/null +++ b/test/configs/certs/sans/dev-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCdqfE5jKdpe3cJ +1Pd2mwBtfaDnaGsfOxyy7VsIFMGdekkJ7KIN6wX+O0nFT8X8lpC7YnIgTzpZl2/Y +GzlntFaAZ7cZMaeCuce6HsE/vofOmMrNnT2pEp7z0DZn7TzjMdCAyeCnG3N4XGe8 +lRCQLVqHwkuhjlcr8r0A9CWfx0bc2zA1ZkiprnF3Qf0KbN4SfR+oHczsEO48yXD2 +7X42IX3tUU5sljVbDd3TFKSyS4l2EebzhX1bNUj68jvxrDgo5ZTrMSIaiMtLSsoH +ftyCbvG/Rxd8LlAeWqgqkdKXUj8gsL4dCfzRjj1MjWSx1Mjcjp6ujHRyXeFrbvSq +6bvzPXoDAgMBAAECggEAOHkfRQhbDN7jCBwG2a5yjEV0BX9y41hmkraPJUleApzD +HbFraIXW+zXcWjcVSUDbLat5CpamZWHnd2Zk/P7s/whnXrhY8mu+HS4X9U+3UGqN +NfINHIrlcZqSak5hQVXeA8uL0v9zsFiU9ckFCkechkUzlvYnxj+nsHhDI8Sa3s95 +5POvYg1VrxfTM7yq8jo9hNoCKd16ghxPXHLHpimZ32qnmje0GqGHvQD1aL83fcaY +SjSI0OhVf+rkKy8uy63xp20rKj+zCMmLIuV9AiFapiznp5ui7+WAH6oEjMjCzCKH +SzE4arFgYOjHDF8ElMDhzyOrFg50uGA5Cl1kfn9x2QKBgQDXNSqqZQNCFU5Ah7K5 +wLSedQkHmofN8moH44YXLlK2zfEknXsmu56qoGBRlIVPwYqkXTUg9uDiv7IzE6Fh +eADCB+hiwnDPw6vB17aLJc+uE+XtLYWjjPMpOHyR4HAbgq/s3GGzAxqvtPOLrc26 +gzJReRpN/0/7sodj271LitbCuwKBgQC7jH4SiLuRZWobOrFNDtPBpPBvA56/YNWe +0gvHfWYCPwt14aF2mkTo+wvY6bzSgy+KmRjCXSyQD9/5GOUtSCIrM5rQvqI3iYMw +8kkJMDB6b7GEustNydGgpEVl5KqfwUzy+asOnjlqVO3FZSwvJlLNYVtgjRzGWsZX +1pcL/0BlWQKBgQC3u5yBfVwavzXfwfrEu9F4t1LOOxBWs+/ybD2+7L2RdKG8MPdD +SktcQS/6dmKahRJo4WrKifvVmvP4x3mwTVPYVAgCyR6nQtcQ16nxgoaciEB1Dbha +uaugNamkoYkU865e+ogu3Sebe1GynuBVrEz5YfsjPCZ9LR6KEsC6Df7soQKBgQCo +aNrYh8FuKRPjiYumN1c8/oAMH8MP9MM+Dz0WkGrmP3hqDQaw+oxAbRXRXOn6WmR5 +X0pVVddrMWYcRxeb1rcf9gHhyhzeI/QTIq0kvAn8F4nfNuDSZBSB3KCYg4IXXDtd +v4Wz5a6G5eZwp43KdO5LkE2+YFhjYSXTwFT4J+fKWQKBgCyoF/Za1xghz/+dmH/K +vZtARsJgZjWIknsBdTwPEcqZjV7OBe7IKRutPgVnzHGt+3gWLqXQtz9uLxAMw/8B +k1JQuEKEeuAltA2bE6vp8NXP7kC38oR/9Z7aHKru9+8nrrG/dVK28PrSymdd4keB +J+vQW0oEllTH7M7Wgyy3r4O6 +-----END PRIVATE KEY----- diff --git a/test/configs/certs/sans/dev.pem b/test/configs/certs/sans/dev.pem new file mode 100644 index 0000000..9532e17 --- /dev/null +++ b/test/configs/certs/sans/dev.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDIzCCAgugAwIBAgIUTOgYUed1dyp+Gan755liL7b29v8wDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU5MDBaGA8yMTI0MDMy +NDA3NTkwMFowFjEUMBIGA1UEAwwLd3d3Lm5hdHMuaW8wggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQCdqfE5jKdpe3cJ1Pd2mwBtfaDnaGsfOxyy7VsIFMGd +ekkJ7KIN6wX+O0nFT8X8lpC7YnIgTzpZl2/YGzlntFaAZ7cZMaeCuce6HsE/vofO +mMrNnT2pEp7z0DZn7TzjMdCAyeCnG3N4XGe8lRCQLVqHwkuhjlcr8r0A9CWfx0bc +2zA1ZkiprnF3Qf0KbN4SfR+oHczsEO48yXD27X42IX3tUU5sljVbDd3TFKSyS4l2 +EebzhX1bNUj68jvxrDgo5ZTrMSIaiMtLSsoHftyCbvG/Rxd8LlAeWqgqkdKXUj8g +sL4dCfzRjj1MjWSx1Mjcjp6ujHRyXeFrbvSq6bvzPXoDAgMBAAGjazBpMCcGA1Ud +EQQgMB6CDGFwcC5uYXRzLmRldoIOKi5hcHAubmF0cy5kZXYwHQYDVR0OBBYEFDvM +qcu0rwkeE32c0A2VgPLmf4eDMB8GA1UdIwQYMBaAFK442m9BmE189lFzd57Hbz/b +95psMA0GCSqGSIb3DQEBCwUAA4IBAQBxVdfxsnSvL3WTuZroo+inViR8T7RV2Dys +8NiwvO1o+uNATmzV5v2iLux2INEXBgt+71OqzcPPiTvjHcBkh6aTCk2N54mYRK6Q +L3d7wfeYJhHUfeC/idM5c8UQVHkKa7Em+6D/IoWnzen6QTs2CtxJ+UE4Ie56DTHQ +1qc96IqNUKVAq61tZe9u89sCsn9S5r/cyBa8pTMzfjZ6FHJFC/MZHu8InfNJ13MB +q5edoCH761+MbyXz/JyrU2hTcno0z1ZSS6VteehDPdnDPjysoZPCjn1jP7G4rZ3W +eevtdw2p2SP+jdFt5Hy6tvy4sXtj79C4MHVbsAVw6hkE5PNoNRM3 +-----END CERTIFICATE----- diff --git a/test/configs/certs/sans/prod-key.pem b/test/configs/certs/sans/prod-key.pem new file mode 100644 index 0000000..9316296 --- /dev/null +++ b/test/configs/certs/sans/prod-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDQhGgjBMhK3sIJ +IAKwXPUd/QrxbHQ+fGkXsHm9EIMzm9krVfVvsAqjrZyWHqiJ16N66+ysWqVe0Aiu +V+tHoilSzEZhfTRsVz+CPDBotyW8rK25SgLRoQ3AnmG2Co6s1Y05HNMyuUIpoXg6 +Y0+klARJFsOp2kDhCyjW2Vz+M4pJIMY69y4IGjlCWOc4IiQKKIJjD12FxAe0/wtn +U4uypTXqKAYUQX88TS4mCBgxPYh7XZrWfwZzYkXnUAxXy3OIqXtI2ZuQ3MkgaUck +jPJkJNBaLcGgAJq3Msd7t2kXulmIJTnpw0D4jd12v19wDJCxTuaBdiUhhkZX9fyF +Le6T6NopAgMBAAECggEADzlRlJfzazL5Gfs0Mw3waAZ68246jPrenegItV70o1Ka +QB2cKDHDTxi8fWHi5oB/2zhJLPrHxuS7s4XwA8iBSZ4oN9MxPgU/Oeoc0zJm/w/g +9a2jh8xhUYpGwkiiSthjIZO7EEGiJOG5AlNQo+iaKP9wXPD2Sm/7BPe1AErBSo4o +IyXhnx3e32sGAQekR/1AqAl2elXsFeO/nTYOBABd1Tv3RQW7JFtSiDKIIQV50Vhl +2kdFqRlqfEt0JJeRB/hJZloQrKNOR5JUymn497kvmHeLOzNMq7+65naY86BLy+pu +B6uBGgo3AZibfB8R95YpPTj+hn5YsGzYztMKRZ5PUQKBgQDwjXDqTMRdtoWVblvF +89PY2bF4dvyo1WW5FWN7kbjjOmwz+rhsc3ct8IDhxdTh84gxv6+nQfNaXCmPl7LP +Y4Qe6PC1GlhgC/XP0CSb4YhqIlk8Ja1yLOVttT/u77Fd9WRLYHWQqWBMVJvPg+9q +pdXsfIvjQg3cYWrM8l7wzKuJEQKBgQDd6FJ0H9bTFYnMc60BHSujd/xTlxEgkS/2 +c+cFvVJ+Z8+kmNRxLWGeuj/pFh1Lg2o1dP5WH+1oT+gq50MHNyAqLP2+mY/iZTaD +xp6+ChAZpg6FLxKfMr6vv8rfA/6259RjqrGKzVT7ZD9yHAkG86n+ZHvXxBIkVEOF +WslsBhz/mQKBgAKYVAaDC39DT8+b4CuThM6LEqoNBo6/tpg3jdowaUEySSaKZuDs +ofWB6bIF01UYrnhy6+8u+/QWmHAvH8Oy/CLlOmuJvDhVCLLUOQDhfvo+Ip5Ofb+E +knkLgoQOW+h6lDln8fy8hwhrxT6I2tVYsqUZdtzdwo6FEoThfHq2iocxAoGAS0Ri +uzqVbhFvJObTdn5Db/XhoySpTYeRYiGb6Yl2sqNZfbXM0PqYkvMPfGrg0t0nKCyf +m+zGZMw9rle0l3zuLwAFrSF++UZlQDsdWs4k+d4mLKxzv4XBwfDmydAcFEP7+TZL +dA4DViWBk53Ivg/lyACjNOMvWB0wrcnGahqEuXkCgYAh47V2/4fEHLWwbsnvWL8T +6C56/8RUBTvlF2vFtRA/THmYV34pGoM0EWLQ8th4mQQ4mPo6JcQ6L6K1zjw87RJ2 +zvlWhCNZdOUFtrsoVwuRsP3U8PW+qYiDasbNHJoy5UC5jTI5byluM802Vw+9uQ+l +wHleMjtW69bGbRbHWDOVvw== +-----END PRIVATE KEY----- diff --git a/test/configs/certs/sans/prod.pem b/test/configs/certs/sans/prod.pem new file mode 100644 index 0000000..c0b8eb1 --- /dev/null +++ b/test/configs/certs/sans/prod.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDJTCCAg2gAwIBAgIUTOgYUed1dyp+Gan755liL7b29v4wDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU5MDBaGA8yMTI0MDMy +NDA3NTkwMFowFjEUMBIGA1UEAwwLd3d3Lm5hdHMuaW8wggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDQhGgjBMhK3sIJIAKwXPUd/QrxbHQ+fGkXsHm9EIMz +m9krVfVvsAqjrZyWHqiJ16N66+ysWqVe0AiuV+tHoilSzEZhfTRsVz+CPDBotyW8 +rK25SgLRoQ3AnmG2Co6s1Y05HNMyuUIpoXg6Y0+klARJFsOp2kDhCyjW2Vz+M4pJ +IMY69y4IGjlCWOc4IiQKKIJjD12FxAe0/wtnU4uypTXqKAYUQX88TS4mCBgxPYh7 +XZrWfwZzYkXnUAxXy3OIqXtI2ZuQ3MkgaUckjPJkJNBaLcGgAJq3Msd7t2kXulmI +JTnpw0D4jd12v19wDJCxTuaBdiUhhkZX9fyFLe6T6NopAgMBAAGjbTBrMCkGA1Ud +EQQiMCCCDWFwcC5uYXRzLnByb2SCDyouYXBwLm5hdHMucHJvZDAdBgNVHQ4EFgQU +Knk97WhPStd4LcWf52kZxmG4PVowHwYDVR0jBBgwFoAUrjjab0GYTXz2UXN3nsdv +P9v3mmwwDQYJKoZIhvcNAQELBQADggEBACiIPya5Mgy2y4+PrBv6KbYBvpSMA0+g +6OCq3pX4+okIGBoytjL+0DJchEAVGHxf6qnMWSQKnBFgjCHbi805JgJu819PDWFr +lyWsH/YuWP/Df9zl6KiXbrLUBBnGT+bxaqc/FVEsJrhB+29kIrDmxdcL/CcmihwF +pAeYMDs7zjn3W8BQiBG+odapzP7VHsA0L/HpWOZrvJnSIU4vAujnn6SYRyk5y1T1 +7o7nSKo7Fh4YjteTUGzOKFKDS+Mk3SxXA+TIqskV0ZWgZD4KebN69VZWRXgDwTpT +E9jt46Yzmk3RD1NsrfwmgNSGet1+mOzjOLzzXjlxl7qmpYvbyx8U+9c= +-----END CERTIFICATE----- diff --git a/test/configs/certs/sans/server-key.pem b/test/configs/certs/sans/server-key.pem new file mode 100644 index 0000000..23a545f --- /dev/null +++ b/test/configs/certs/sans/server-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDLLgBYDBSeNemh +8b9yPmGiaJkSkJtaXL7ESe1h/aNpdCRrkja7XdHqQ20a2H9X94nXuzvFbpJw8VGT +q8ifGet3+++AJvEP+LfK7HSIvnyewKXZ+Oqf5AJg1YIb3k31VGzGEjjjnW7F/Gw4 +2qKyZ7+CEnIFbo4lLFBoTGvzJo2OwDVKTCby9E428te7rH1+F0OOANSG+SLzl5Vj +bidr4OryTPjO2qJSs22rVnvjZdcG38Z1ams6s4fxwQWvuD9qBBrb0GGtvVP2cYBe +XfOTOydS5IxOerb0EbbijudXVh+xH3dRgvfeJZWTqegCktT6OEa1pe7Q3BoMLtk0 +HJO77RtrAgMBAAECggEAMe91YC5f5t5jNE0A+2f/gPXqIRjvRY8jx4RP1lSLFADa +bqG51+TgEY5Ow1lQOuN4uk+nCgf5784veXav0QKCz7NYkot/lahQK4xfu7ftUusQ +F/lIWNGmD/yJhKXnTRNZxHh4COEJd5lpU7PQZr5+383+vf0E7HfuryORKGpiQiHl +F1ZOYFG2GjFZychMTe+ggQISJqX8U2bE1mIhZNSkBvStt9LDIyWsvbud3j5DHPv/ +nX3EEymKwDiMrkNc+HTodJ3Ci9cSVGJ6igAODeu5wwddn9rpP4obOvJW5B2NuNXR +OirU9o4Hd4cPcUqE8FHloEmIKmALqCQgH3gu/+euUQKBgQDoR4IMFRe4d+G/FJLE +MUnQ1H1ipHOaZcLl0bBFeEtkpuqC5MDoCDJuQCeNrohSrE0dh4z15lq8G79eucYH +E4vbmsyXAm+KgpYZ5S0wvMDoPaXxOSUJe95eUqeGDl3vcexSOXph7/syxXft7MpA +JGFirKH16WiKn4u30siaNK8feQKBgQDf7b1pEdUr+ajhrJ0+LwC+/Q12nBgrULvf +N1rOBXrXX+yI0esk0/i6EWThTZP0iyh/CpxZQyfsqQXBG94spJLGa/C3dcOWohQ4 +tTIYZN9uSozO8OXU+WEHM41kdOByItIpFVzJ2VQ711NizZxchsVUPexUxW99rxuy +JYPe891lAwKBgDX1JoiX/cKkVpSEuvNIX9VbByV6/j/Hk9a/NytsHldrTt7JNOax +wMVLseR/vrs1u1Q4wM3+jOVVrMbccNwD9mE1PIF312FdpHCvhCfmMQnCCGJY2/mw +BJ/0o2XIwJl25WbUY1GM5dWNeaaTcwgja1v4rkbpyZcqKNKy7FamFmhBAoGADTg+ +z14jJJZ3luMW4V3rAFt2GQN8FcqNXM8qyDDgoKhkEWu7IxB0hZ+TQg3PZ0dz25Qn +yMlDKSCNW2omwqHwnFAxBNOp+VGWOgbQb0o9OoiiKXFlUB5s0P0FIuIXggV2PMCr +Gqt+4o2CubmDup6mNaP2OMbzHwtgajL2xwnEjzsCgYEAhEQwa0t+x2+/vZ1NOQW7 +hp7vM2/DyAZnqgAdCwXnlmVayxR4r9OSDZmz5vqJWamQ0UIWYnTkvLN962hYdIC1 +eIQ3yEjxIgZj9Kb+6vnHI81kkjg8LK/XG87HFBnL9juNgdbNQVIyMI84FM2jm+AP +xReBzjjjrY3NrcJ8B6a4AUA= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/sans/server.pem b/test/configs/certs/sans/server.pem new file mode 100644 index 0000000..7ba486f --- /dev/null +++ b/test/configs/certs/sans/server.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDLDCCAhSgAwIBAgIUTOgYUed1dyp+Gan755liL7b29v0wDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU4NTlaGA8yMTI0MDMy +NDA3NTg1OVowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAyy4AWAwUnjXpofG/cj5homiZEpCbWly+xEntYf2jaXQk +a5I2u13R6kNtGth/V/eJ17s7xW6ScPFRk6vInxnrd/vvgCbxD/i3yux0iL58nsCl +2fjqn+QCYNWCG95N9VRsxhI4451uxfxsONqisme/ghJyBW6OJSxQaExr8yaNjsA1 +Skwm8vRONvLXu6x9fhdDjgDUhvki85eVY24na+Dq8kz4ztqiUrNtq1Z742XXBt/G +dWprOrOH8cEFr7g/agQa29Bhrb1T9nGAXl3zkzsnUuSMTnq29BG24o7nV1YfsR93 +UYL33iWVk6noApLU+jhGtaXu0NwaDC7ZNByTu+0bawIDAQABo3YwdDAyBgNVHREE +KzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYD +VR0OBBYEFG5wB9htqpiQze/q1+9rPaq1HlcpMB8GA1UdIwQYMBaAFK442m9BmE18 +9lFzd57Hbz/b95psMA0GCSqGSIb3DQEBCwUAA4IBAQA1tni0pG6LtlStmEUmn9xJ +mcWlWgZtmUbZD7aE5P+SFLt02TY9ea7IoMy3Rv1yRihxLgURNB9Y8aON9IrraVRf +U+EizfG9goQQINs652TNMoxjNEwbF19TzgY7k445gkcl8aZ5fGJJhxP7UGBC5xYL +1Cx0uINiqs0fb3pPrcvhxD+I5tHDwJY7maW0fks/JD1M738hl5zpV1POMvarDFW0 +L4LUDweUuNeylOWyCBv6WFV7avSsMmoQmUvUZoPqp2gk0A17pDeuufI84fHhEghU +UigXcElvZkaoThbE3YA0Hq8UJHSzAe8x0bDSRz4WfNzeqq1RtdZ+H1uPoMIJbZ4D +-----END CERTIFICATE----- diff --git a/test/configs/certs/server-cert.pem b/test/configs/certs/server-cert.pem new file mode 100644 index 0000000..80a9d8f --- /dev/null +++ b/test/configs/certs/server-cert.pem @@ -0,0 +1,99 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 1d:d9:1f:06:dd:fd:90:26:4e:27:ea:2e:01:4b:31:e6:d2:49:31:1f + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27 + Validity + Not Before: Aug 27 20:23:02 2022 GMT + Not After : Aug 24 20:23:02 2032 GMT + Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (2048 bit) + Modulus: + 00:e6:fb:47:65:cd:c9:a2:2d:af:8b:cd:d5:6a:79: + 54:3c:07:5f:eb:5a:71:2b:2b:e5:6f:be:31:fb:16: + 65:68:76:0e:59:e7:e4:57:ca:88:e9:77:d6:41:ad: + 57:7a:42:b2:d2:54:c4:0f:7c:5b:c1:bc:61:97:e3: + 22:3a:3e:1e:4a:5d:47:9f:6b:7d:6f:34:e3:8c:86: + 9d:85:19:29:9a:11:58:44:4c:a1:90:d3:14:61:e1: + 57:da:01:ea:ce:3f:90:ae:9e:5d:13:6d:2c:89:ca: + 39:15:6b:b6:9e:32:d7:2a:4c:48:85:2f:b0:1e:d8: + 4b:62:32:14:eb:32:b6:29:04:34:3c:af:39:b6:8b: + 52:32:4d:bf:43:5f:9b:fb:0d:43:a6:ad:2c:a7:41: + 29:55:c9:70:b3:b5:15:46:34:bf:e4:1e:52:2d:a4: + 49:2e:d5:21:ed:fc:00:f7:a2:0b:bc:12:0a:90:64: + 50:7c:c5:14:70:f5:fb:9b:62:08:78:43:49:31:f3: + 47:b8:93:d4:2d:4c:a9:dc:17:70:76:34:66:ff:65: + c1:39:67:e9:a6:1c:80:6a:f0:9d:b3:28:c8:a3:3a: + b7:5d:de:6e:53:6d:09:b3:0d:b1:13:10:e8:ec:e0: + bd:5e:a1:94:4b:70:bf:dc:bd:8b:b9:82:65:dd:af: + 81:7b + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Comment: + nats.io nats-server test-suite certificate + X509v3 Subject Key Identifier: + 2B:8C:A3:8B:DB:DB:5C:CE:18:DB:F6:A8:31:4E:C2:3E:EE:D3:40:7E + X509v3 Authority Key Identifier: + keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A + DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27 + serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21 + + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1 + Netscape Cert Type: + SSL Client, SSL Server + X509v3 Key Usage: + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, Netscape Server Gated Crypto, Microsoft Server Gated Crypto, TLS Web Client Authentication + Signature Algorithm: sha256WithRSAEncryption + 54:49:34:2b:38:d1:aa:3b:43:60:4c:3f:6a:f8:74:ca:49:53: + a1:af:12:d3:a8:17:90:7b:9d:a3:69:13:6e:da:2c:b7:61:31: + ac:eb:00:93:92:fc:0c:10:d4:18:a0:16:61:94:4b:42:cb:eb: + 7a:f6:80:c6:45:c0:9c:09:aa:a9:48:e8:36:e3:c5:be:36:e0: + e9:78:2a:bb:ab:64:9b:20:eb:e6:0f:63:2b:59:c3:58:0b:3a: + 84:15:04:c1:7e:12:03:1b:09:25:8d:4c:03:e8:18:26:c0:6c: + b7:90:b1:fd:bc:f1:cf:d0:d5:4a:03:15:71:0c:7d:c1:76:87: + 92:f1:3e:bc:75:51:5a:c4:36:a4:ff:91:98:df:33:5d:a7:38: + de:50:29:fd:0f:c8:55:e6:8f:24:c2:2e:98:ab:d9:5d:65:2f: + 50:cc:25:f6:84:f2:21:2e:5e:76:d0:86:1e:69:8b:cb:8a:3a: + 2d:79:21:5e:e7:f7:2d:06:18:a1:13:cb:01:c3:46:91:2a:de: + b4:82:d7:c3:62:6f:08:a1:d5:90:19:30:9d:64:8e:e4:f8:ba: + 4f:2f:ba:13:b4:a3:9f:d1:d5:77:64:8a:3e:eb:53:c5:47:ac: + ab:3e:0e:7a:9b:a6:f4:48:25:66:eb:c7:4c:f9:50:24:eb:71: + e0:75:ae:e6 +-----BEGIN CERTIFICATE----- +MIIE+TCCA+GgAwIBAgIUHdkfBt39kCZOJ+ouAUsx5tJJMR8wDQYJKoZIhvcNAQEL +BQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM +B1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl +IEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw +MjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV +BAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z +dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOb7R2XNyaItr4vN1Wp5 +VDwHX+tacSsr5W++MfsWZWh2Dlnn5FfKiOl31kGtV3pCstJUxA98W8G8YZfjIjo+ +HkpdR59rfW8044yGnYUZKZoRWERMoZDTFGHhV9oB6s4/kK6eXRNtLInKORVrtp4y +1ypMSIUvsB7YS2IyFOsytikENDyvObaLUjJNv0Nfm/sNQ6atLKdBKVXJcLO1FUY0 +v+QeUi2kSS7VIe38APeiC7wSCpBkUHzFFHD1+5tiCHhDSTHzR7iT1C1MqdwXcHY0 +Zv9lwTln6aYcgGrwnbMoyKM6t13eblNtCbMNsRMQ6OzgvV6hlEtwv9y9i7mCZd2v +gXsCAwEAAaOCAZ4wggGaMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu +aW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU +K4yji9vbXM4Y2/aoMU7CPu7TQH4wga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I +RWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh +MRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD +ZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb +M0C2bSEwLAYDVR0RBCUwI4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAA +AAABMBEGCWCGSAGG+EIBAQQEAwIGwDALBgNVHQ8EBAMCBaAwNAYDVR0lBC0wKwYI +KwYBBQUHAwEGCWCGSAGG+EIEAQYKKwYBBAGCNwoDAwYIKwYBBQUHAwIwDQYJKoZI +hvcNAQELBQADggEBAFRJNCs40ao7Q2BMP2r4dMpJU6GvEtOoF5B7naNpE27aLLdh +MazrAJOS/AwQ1BigFmGUS0LL63r2gMZFwJwJqqlI6Dbjxb424Ol4KrurZJsg6+YP +YytZw1gLOoQVBMF+EgMbCSWNTAPoGCbAbLeQsf288c/Q1UoDFXEMfcF2h5LxPrx1 +UVrENqT/kZjfM12nON5QKf0PyFXmjyTCLpir2V1lL1DMJfaE8iEuXnbQhh5pi8uK +Oi15IV7n9y0GGKETywHDRpEq3rSC18Nibwih1ZAZMJ1kjuT4uk8vuhO0o5/R1Xdk +ij7rU8VHrKs+DnqbpvRIJWbrx0z5UCTrceB1ruY= +-----END CERTIFICATE----- diff --git a/test/configs/certs/server-iponly.pem b/test/configs/certs/server-iponly.pem new file mode 100644 index 0000000..a030f86 --- /dev/null +++ b/test/configs/certs/server-iponly.pem @@ -0,0 +1,99 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 21:8e:d4:f9:10:fc:32:5f:46:45:23:53:14:03:f0:b2:dc:a1:d7:e5 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27 + Validity + Not Before: Aug 27 20:23:02 2022 GMT + Not After : Aug 24 20:23:02 2032 GMT + Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=ip-only-localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (2048 bit) + Modulus: + 00:bd:98:dc:14:49:92:43:1a:73:36:75:6d:c1:33: + 73:b3:e7:f6:8a:e2:03:f7:24:93:eb:36:da:8f:64: + c6:56:19:b7:92:d2:f1:df:d3:4f:7e:78:ce:db:dc: + 6e:d4:5c:58:a4:a7:79:6b:49:ff:30:37:d9:71:92: + 21:d7:91:72:8d:1a:21:75:0f:80:23:f0:2f:0e:6a: + 1a:62:9b:eb:1e:74:df:df:61:8a:05:a8:31:b7:d4: + 97:ee:c6:60:1b:6f:b1:85:9b:ac:3d:cd:a1:9e:a8: + b5:56:84:ce:9f:8c:64:a5:57:41:75:81:c6:00:c8: + d8:3d:ef:85:9c:78:04:49:5a:94:29:af:27:0a:be: + 56:60:6b:a0:52:06:53:f5:cf:ce:47:9c:cf:50:8a: + 1d:92:3a:21:d1:24:ed:81:e8:85:97:8c:41:46:96: + da:02:d7:b7:f5:ab:92:4c:ab:0a:e2:19:9f:b8:21: + 07:5c:f7:d1:7e:6e:49:70:a9:c4:45:c3:b2:10:ab: + ca:50:e3:c9:ea:6f:b5:19:64:12:32:0e:93:59:f8: + 8e:fc:a2:bc:33:f3:25:e4:59:98:6d:0e:97:76:b6: + da:9e:50:cf:4d:a1:6b:62:ce:47:84:1d:f7:82:8a: + da:d0:90:e7:a6:1f:43:25:61:57:c3:15:7e:e0:b8: + 5c:17 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Comment: + nats.io nats-server test-suite certificate + X509v3 Subject Key Identifier: + 0A:AC:82:F2:35:73:89:FB:49:07:EB:9C:87:72:D9:91:26:DF:84:8A + X509v3 Authority Key Identifier: + keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A + DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27 + serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21 + + X509v3 Subject Alternative Name: + IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1 + Netscape Cert Type: + SSL Client, SSL Server + X509v3 Key Usage: + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, Netscape Server Gated Crypto, Microsoft Server Gated Crypto, TLS Web Client Authentication + Signature Algorithm: sha256WithRSAEncryption + 87:00:2d:ec:ea:ab:13:3e:21:6c:4a:3e:f1:9b:c3:ab:c5:81: + d3:34:2d:cb:f1:e7:98:96:13:a3:7f:29:00:83:70:20:81:56: + a1:31:b7:d9:fa:35:02:f4:6f:32:8a:58:cf:68:39:e1:f9:fb: + 50:20:60:85:f1:bd:22:73:e6:ac:b6:36:38:75:86:de:8a:cf: + e5:41:df:0f:bd:61:94:3d:3a:59:81:50:c5:44:39:47:d9:e8: + 56:2a:b4:2c:59:b7:89:f5:5d:a7:28:15:6e:98:1e:47:47:7d: + a6:35:65:de:b7:06:9e:04:67:98:04:f9:57:54:82:2a:dc:e2: + 6a:76:df:06:05:b4:61:84:ec:7d:5e:46:34:ea:14:b7:14:16: + fe:51:0e:65:52:c6:74:96:8e:30:3a:b9:c2:d5:05:2d:a8:08: + 11:ba:7e:65:c1:89:c3:61:ec:d6:06:e0:7e:ac:11:ee:18:89: + 67:59:1d:6b:dc:cc:1e:48:c5:79:5b:8f:18:7e:e4:c4:9f:4f: + 3d:a9:11:60:28:77:49:e7:23:4d:fa:49:8d:6b:5a:52:57:9b: + bf:a7:d7:da:13:45:64:dd:d3:2f:4c:14:eb:79:46:8f:fd:41: + ab:88:17:d4:8e:77:45:8c:7f:cb:e5:d8:19:7c:1b:1e:cd:21: + 32:43:a3:c9 +-----BEGIN CERTIFICATE----- +MIIE9jCCA96gAwIBAgIUIY7U+RD8Ml9GRSNTFAPwstyh1+UwDQYJKoZIhvcNAQEL +BQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM +B1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl +IEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw +MjMwMlowYjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV +BAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xGjAYBgNVBAMMEWlwLW9ubHkt +bG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvZjcFEmS +QxpzNnVtwTNzs+f2iuID9yST6zbaj2TGVhm3ktLx39NPfnjO29xu1FxYpKd5a0n/ +MDfZcZIh15FyjRohdQ+AI/AvDmoaYpvrHnTf32GKBagxt9SX7sZgG2+xhZusPc2h +nqi1VoTOn4xkpVdBdYHGAMjYPe+FnHgESVqUKa8nCr5WYGugUgZT9c/OR5zPUIod +kjoh0STtgeiFl4xBRpbaAte39auSTKsK4hmfuCEHXPfRfm5JcKnERcOyEKvKUOPJ +6m+1GWQSMg6TWfiO/KK8M/Ml5FmYbQ6XdrbanlDPTaFrYs5HhB33gora0JDnph9D +JWFXwxV+4LhcFwIDAQABo4IBkzCCAY8wCQYDVR0TBAIwADA5BglghkgBhvhCAQ0E +LBYqbmF0cy5pbyBuYXRzLXNlcnZlciB0ZXN0LXN1aXRlIGNlcnRpZmljYXRlMB0G +A1UdDgQWBBQKrILyNXOJ+0kH65yHctmRJt+EijCBrgYDVR0jBIGmMIGjgBT+z38+ +Lu7Erepbb4hFbMeISBSLOqF1pHMwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNh +bGlmb3JuaWExEDAOBgNVBAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAn +BgNVBAMMIENlcnRpZmljYXRlIEF1dGhvcml0eSAyMDIyLTA4LTI3ghRJnBbtu1z0 +RRusxa2MelszQLZtITAhBgNVHREEGjAYhwR/AAABhxAAAAAAAAAAAAAAAAAAAAAB +MBEGCWCGSAGG+EIBAQQEAwIGwDALBgNVHQ8EBAMCBaAwNAYDVR0lBC0wKwYIKwYB +BQUHAwEGCWCGSAGG+EIEAQYKKwYBBAGCNwoDAwYIKwYBBQUHAwIwDQYJKoZIhvcN +AQELBQADggEBAIcALezqqxM+IWxKPvGbw6vFgdM0Lcvx55iWE6N/KQCDcCCBVqEx +t9n6NQL0bzKKWM9oOeH5+1AgYIXxvSJz5qy2Njh1ht6Kz+VB3w+9YZQ9OlmBUMVE +OUfZ6FYqtCxZt4n1XacoFW6YHkdHfaY1Zd63Bp4EZ5gE+VdUgirc4mp23wYFtGGE +7H1eRjTqFLcUFv5RDmVSxnSWjjA6ucLVBS2oCBG6fmXBicNh7NYG4H6sEe4YiWdZ +HWvczB5IxXlbjxh+5MSfTz2pEWAod0nnI036SY1rWlJXm7+n19oTRWTd0y9MFOt5 +Ro/9QauIF9SOd0WMf8vl2Bl8Gx7NITJDo8k= +-----END CERTIFICATE----- diff --git a/test/configs/certs/server-key-iponly.pem b/test/configs/certs/server-key-iponly.pem new file mode 100644 index 0000000..96748e6 --- /dev/null +++ b/test/configs/certs/server-key-iponly.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC9mNwUSZJDGnM2 +dW3BM3Oz5/aK4gP3JJPrNtqPZMZWGbeS0vHf009+eM7b3G7UXFikp3lrSf8wN9lx +kiHXkXKNGiF1D4Aj8C8Oahpim+sedN/fYYoFqDG31JfuxmAbb7GFm6w9zaGeqLVW +hM6fjGSlV0F1gcYAyNg974WceARJWpQprycKvlZga6BSBlP1z85HnM9Qih2SOiHR +JO2B6IWXjEFGltoC17f1q5JMqwriGZ+4IQdc99F+bklwqcRFw7IQq8pQ48nqb7UZ +ZBIyDpNZ+I78orwz8yXkWZhtDpd2ttqeUM9NoWtizkeEHfeCitrQkOemH0MlYVfD +FX7guFwXAgMBAAECggEAE6qdeYVAJLHDrax0nAvIPqsbCcD0BFjI9ycTeLhNUnUM +c7Bp4nu6zTWez3OIE4MYtsugbp6YV9oTNhKgbAnsRfKl8cyP0CqD1wzue7gMpXYe +Gr+1X2zY62aj8+Kj6XSmh2NkdGy2DQ0W8kiIXkhj0DrC0XuKnF45AAOualKQr0MG +JRFXi/AEEDkyWwfmqteD60HBzdweBqzYvsIyA8cQAgI4LguRKHKmLexqNDL0h0K+ +nPbQl5pg7es3kOaQ7DMICqd+E11btSXBJop95MLVhPXkzAyktIdfc1M7LsBNb7Dw +ja5G+wmmGG0KxCbqMVKCACxs63lWakIxFlrsxi+Q4QKBgQDwrM4f/j289+PBKlED +UCXHsGBgE1vNZMAinfAazWAImTAwWoY0FDrMBBPJGx+Azke5teI82o+Sq4jTL44Y +qvmqblHb9XEQLnRxSdl5WP794o1K9pP7nEvIpL+RngRIC1qt9BEGd5Pmn8FcF4BT +Zo9sdP8Hh6EJLGb33iYxUrTNTwKBgQDJq3GclCnVukbaGbCU2w54EDTJoghIY+W4 +kXeGL1VbSXD/NnyuhH/mua+UwBM7XSFJXKV2vfIhI1Qe9VW+7p4vyiLFp3atSb86 +AirajUPrCJNX1X/sm7fIss9U1ko/JruXSh1DxzEZvBQa52SwYH5bM6LbE6NJRuhS +xMY2ipyiuQKBgQCm0gCl6GH+w4wYbi5tL3agbT7AGWr+eSE8XWD6EvTHwPbH7Vcs +bgE7PHBCawxxCYppzQqdx5jQvxk92K6Tpp8bZRBUeFIAN1L624dkNy236PqqxTNZ +qcJVtuwaEP9CuKwH+y553xSjPISYQqnuJR6wvH+xRm92nlJY6KBse7lavQKBgCEV +NeMIz0AXec4HjtcshFgf2HkHUrKFaMb5XhEuLKN4DchgKN38MHsqFOqjA8SmR3Kg +dyheipzzDbayammS/XI7h67DBQ3yXiNm/Z6ys+SXmIw9IuoutVyAMNDrAm0PrpBo +ARsAT0a4etfbA8KHYdMWSm4D77JypmQFkbqazI1JAoGBAKZTn6KQaBcZ5JrSya17 +sClZPlSsYhIW8EbxqDgNwDippcNrLuJCsWiwMNW+glQoDIcGAo557PwAXrridH89 +5mjFCVggHeso3T6/nsBB86Q8B1hq5E8AszhEKH6IeKd5tFNZkK83eWjAlbIUrEBH +xXZFMGnYPPS7korkEzzuGJ+7 +-----END PRIVATE KEY----- diff --git a/test/configs/certs/server-key-noip.pem b/test/configs/certs/server-key-noip.pem new file mode 100644 index 0000000..aa29bb3 --- /dev/null +++ b/test/configs/certs/server-key-noip.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCt8Ic/MmaHejGb +ylQKrqYayiXVxfxJayEL3qcVyJw8zUEdMiV3aHuD6F0Uei4L6kGRpCDsIBcPy41M +G4ig0ndGZX7RoOZMS8aMOaGzWzRXyKEQDBNUOnSQezu62kFigfXctXNsgzj0oVKr +vcKVPnn/r6Su39YR2SkguLQV4zKTXDbOVrQBAqFFMaOhHuq4xAEEVxFE9FXq4q5o +CHCFwFv/ur/ei7yhxgOiL4rrnrd5OmdqsHDT6AinEiTVu1eIcjfI5i7bh+AqcRos +kJyIKQx1KITWf3UtUAg2K8/zujNyHnoH2yDamDs5hpZM4kpCYRqbC2dNbRPRn0Df +EseNnVBpAgMBAAECggEAcmiqXRwmqmfqZ4Ge4+Pap/ZdCo6OkjAf7XHHTyHD+o47 +jRul3zPfQnU9fDGdRgMQm95sNUQqRx5pUy0tIjMtdyVdVD9UG80fzK4/uPx9olv5 +7Nc0g4trjnkwYYgbx9KZyFGlmTN67BWMjiBj88zDbDW4ybm7UcQYNEipU1g8tQW1 +tUwcZ1oahXfzO75vcMqDVlS2IE0s0AD9sh+AaJIwxV9kSLNjlSwkpsH6PBKKB/3r +WvG2p6Og1whdQ54PGADUVSx1yWFyXQDeygqLmryEWaHJQz1jt7bvaaAMy2PTdwVf +A5LVG3VHkoQOBv8imtpCbU2J7zAk9ypDuRUlpa8h/QKBgQDdCCCbV02BhrqDYchm +ojB95Vx8KtvQdXhvsxShxyuIktuB7W+NnheBmLY0TNcYSQyzithCUBhtmyaC5S4f +dHmT52e7HS0xaL9r9BhAQrtWReMcplKB1IIXtdYXEY3qOjZMxX3seJo0iBWS3hMH +EG6tC6tlr5ZXOKJOrBMGuMgplwKBgQDJdSYkC3AX2p+4BNf3hgQyzotuSVSbx/zu +0ZHhi8Wp7yF49c8+9+ahO9AMrVM0ZSh2buznfF46FNC/C55M7a9Rn60sFQQ16b5L +rJTzlPoUGTnPLt8C3TdMIFg/5cAW6ZgZWNlU3aVU0W34NVh/H2m/M72tGrk250zs +YhZ8/RGV/wKBgQCKlMfs3YXoyhIywaImR1Zj+ORNrYl4X86NKhirffbbgEhEZBvn +DNHsHVVP4UWTImnmQA1rNlC6l+ZDd3G9owd/Jj0xYg+txOEPzFFQKQbQBq1ojxd3 +80dFmmqKuCTkUG8vHzvegZcdjJ0KIlaHvVPHB2QFM1vtf8Kz1MtxEXXeLQKBgDn0 +Bm3WEH/8N3gzhIFDP0/yVO/8DmfmByAYj5PHpqw1C3cFl4HwxJrbXwVWkxn+g75W +OLZ684xX0pky2W4d7hJYEfQdc6GixUh1tD/COpKvkw7D2Am146N1po1zJWgx+LxJ +7/NW86nLuYvupK+lNMF5O/ZhOqjNrzZNHVUFZBq3AoGAPwixh7/ZMX6mmm8foImh +qibytx72gl1jhHWSaX3rwrSOO9dxO2rlI7LOZQrarU632Y9KMkP3HNbBHPRkA4MI +6I9wqawRzGjcpeXIMlPzOHDHYLyrTpEzo8nrSNk/cM8P4RxE12FqySzQIkiN06J7 +AxJ7hVqtX6wZIoqoOa9aK1E= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/server-key.pem b/test/configs/certs/server-key.pem new file mode 100644 index 0000000..f2c2c6c --- /dev/null +++ b/test/configs/certs/server-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEuwIBADANBgkqhkiG9w0BAQEFAASCBKUwggShAgEAAoIBAQDm+0dlzcmiLa+L +zdVqeVQ8B1/rWnErK+VvvjH7FmVodg5Z5+RXyojpd9ZBrVd6QrLSVMQPfFvBvGGX +4yI6Ph5KXUefa31vNOOMhp2FGSmaEVhETKGQ0xRh4VfaAerOP5Cunl0TbSyJyjkV +a7aeMtcqTEiFL7Ae2EtiMhTrMrYpBDQ8rzm2i1IyTb9DX5v7DUOmrSynQSlVyXCz +tRVGNL/kHlItpEku1SHt/AD3ogu8EgqQZFB8xRRw9fubYgh4Q0kx80e4k9QtTKnc +F3B2NGb/ZcE5Z+mmHIBq8J2zKMijOrdd3m5TbQmzDbETEOjs4L1eoZRLcL/cvYu5 +gmXdr4F7AgMBAAECggEBAK4sr3MiEbjcsHJAvXyzjwRRH1Bu+8VtLW7swe2vvrpd +w4aiKXrV/BXpSsRtvPgxkXyvdMSkpuBZeFI7cVTwAJFc86RQPt77x9bwr5ltFwTZ +rXCbRH3b3ZPNhByds3zhS+2Q92itu5cPyanQdn2mor9/lHPyOOGZgobCcynELL6R +wRElkeDyf5ODuWEd7ADC5IFyZuwb3azNVexIK+0yqnMmv+QzEW3hsycFmFGAeB7v +MIMjb2BhLrRr6Y5Nh+k58yM5DCf9h/OJhDpeXwLkxyK4BFg+aZffEbUX0wHDMR7f +/nMv1g6cKvDWiLU8xLzez4t2qNIBNdxw5ZSLyQRRolECgYEA+ySTKrBAqI0Uwn8H +sUFH95WhWUXryeRyGyQsnWAjZGF1+d67sSY2un2W6gfZrxRgiNLWEFq9AaUs0MuH +6syF4Xwx/aZgU/gvsGtkgzuKw1bgvekT9pS/+opmHRCZyQAFEHj0IEpzyB6rW1u/ +LdlR3ShEENnmXilFv/uF/uXP5tMCgYEA63LiT0w46aGPA/E+aLRWU10c1eZ7KdhR +c3En6zfgIxgFs8J38oLdkOR0CF6T53DSuvGR/OprVKdlnUhhDxBgT1oQjK2GlhPx +JV5uMvarJDJxAwsF+7T4H2QtZ00BtEfpyp790+TlypSG1jo/BnSMmX2uEbV722lY +hzINLY49obkCgYBEpN2YyG4T4+PtuXznxRkfogVk+kiVeVx68KtFJLbnw//UGT4i +EHjbBmLOevDT+vTb0QzzkWmh3nzeYRM4aUiatjCPzP79VJPsW54whIDMHZ32KpPr +TQMgPt3kSdpO5zN7KiRIAzGcXE2n/e7GYGUQ1uWr2XMu/4byD5SzdCscQwJ/Ymii +LoKtRvk/zWYHr7uwWSeR5dVvpQ3E/XtONAImrIRd3cRqXfJUqTrTRKxDJXkCmyBc +5FkWg0t0LUkTSDiQCJqcUDA3EINFR1kwthxja72pfpwc5Be/nV9BmuuUysVD8myB +qw8A/KsXsHKn5QrRuVXOa5hvLEXbuqYw29mX6QKBgDGDzIzpR9uPtBCqzWJmc+IJ +z4m/1NFlEz0N0QNwZ/TlhyT60ytJNcmW8qkgOSTHG7RDueEIzjQ8LKJYH7kXjfcF +6AJczUG5PQo9cdJKo9JP3e1037P/58JpLcLe8xxQ4ce03zZpzhsxR2G/tz8DstJs +b8jpnLyqfGrcV2feUtIZ +-----END PRIVATE KEY----- diff --git a/test/configs/certs/server-noip.pem b/test/configs/certs/server-noip.pem new file mode 100644 index 0000000..8592155 --- /dev/null +++ b/test/configs/certs/server-noip.pem @@ -0,0 +1,99 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 1d:5c:7c:59:0c:cd:27:83:dd:97:64:53:b0:44:3c:b4:5b:d4:fc:d1 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27 + Validity + Not Before: Aug 27 20:23:02 2022 GMT + Not After : Aug 24 20:23:02 2032 GMT + Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (2048 bit) + Modulus: + 00:ad:f0:87:3f:32:66:87:7a:31:9b:ca:54:0a:ae: + a6:1a:ca:25:d5:c5:fc:49:6b:21:0b:de:a7:15:c8: + 9c:3c:cd:41:1d:32:25:77:68:7b:83:e8:5d:14:7a: + 2e:0b:ea:41:91:a4:20:ec:20:17:0f:cb:8d:4c:1b: + 88:a0:d2:77:46:65:7e:d1:a0:e6:4c:4b:c6:8c:39: + a1:b3:5b:34:57:c8:a1:10:0c:13:54:3a:74:90:7b: + 3b:ba:da:41:62:81:f5:dc:b5:73:6c:83:38:f4:a1: + 52:ab:bd:c2:95:3e:79:ff:af:a4:ae:df:d6:11:d9: + 29:20:b8:b4:15:e3:32:93:5c:36:ce:56:b4:01:02: + a1:45:31:a3:a1:1e:ea:b8:c4:01:04:57:11:44:f4: + 55:ea:e2:ae:68:08:70:85:c0:5b:ff:ba:bf:de:8b: + bc:a1:c6:03:a2:2f:8a:eb:9e:b7:79:3a:67:6a:b0: + 70:d3:e8:08:a7:12:24:d5:bb:57:88:72:37:c8:e6: + 2e:db:87:e0:2a:71:1a:2c:90:9c:88:29:0c:75:28: + 84:d6:7f:75:2d:50:08:36:2b:cf:f3:ba:33:72:1e: + 7a:07:db:20:da:98:3b:39:86:96:4c:e2:4a:42:61: + 1a:9b:0b:67:4d:6d:13:d1:9f:40:df:12:c7:8d:9d: + 50:69 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Comment: + nats.io nats-server test-suite certificate + X509v3 Subject Key Identifier: + C9:AA:3C:08:39:7E:C1:42:C0:3D:B7:2F:84:21:E7:8A:30:E7:C7:B1 + X509v3 Authority Key Identifier: + keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A + DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27 + serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21 + + X509v3 Subject Alternative Name: + DNS:localhost + Netscape Cert Type: + SSL Client, SSL Server + X509v3 Key Usage: + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, Netscape Server Gated Crypto, Microsoft Server Gated Crypto, TLS Web Client Authentication + Signature Algorithm: sha256WithRSAEncryption + 9b:63:ae:ec:56:ec:0c:7a:d5:88:d1:0a:0a:81:29:37:4f:a6: + 08:b8:78:78:23:af:5b:b7:65:61:d7:64:2a:c9:e7:a6:d2:b1: + cb:36:bf:23:2e:2d:48:85:7f:16:0f:64:af:03:db:5d:0e:a7: + 14:c5:f6:04:b2:6b:92:27:ba:cb:d2:13:25:a2:15:b0:8e:4a: + 2d:eb:41:18:09:b1:68:d5:0f:6b:56:da:86:ed:4a:7a:29:30: + 09:77:63:a4:64:3d:e3:2e:d7:6f:1a:8c:96:c9:cb:81:fe:a3: + 6d:35:e3:09:ea:9b:2e:da:8c:8e:c8:c9:69:b1:83:e7:6f:2d: + 5f:a1:ac:32:ae:29:57:a9:5c:9b:7d:f0:fd:47:3c:f3:6a:d0: + eb:77:8d:70:06:a2:74:3d:d6:37:1e:7b:e7:d9:e4:33:c9:9d: + ad:fa:24:c6:4d:e2:2c:c9:25:cb:75:be:8d:e9:83:7e:ad:db: + 53:9e:97:be:d5:7f:83:90:fc:75:1d:02:29:b7:99:18:a3:39: + 25:a2:54:b7:21:7d:be:0b:4c:ea:ff:80:b9:4b:5e:21:ed:25: + ad:d4:62:52:59:79:83:32:df:30:a1:64:68:05:cc:35:ad:8b: + d3:66:6b:b1:31:b7:b3:b2:d8:0f:5b:96:40:ef:57:1d:7f:b0: + b0:f4:e9:db +-----BEGIN CERTIFICATE----- +MIIE4TCCA8mgAwIBAgIUHVx8WQzNJ4Pdl2RTsEQ8tFvU/NEwDQYJKoZIhvcNAQEL +BQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM +B1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl +IEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw +MjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV +BAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z +dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK3whz8yZod6MZvKVAqu +phrKJdXF/ElrIQvepxXInDzNQR0yJXdoe4PoXRR6LgvqQZGkIOwgFw/LjUwbiKDS +d0ZlftGg5kxLxow5obNbNFfIoRAME1Q6dJB7O7raQWKB9dy1c2yDOPShUqu9wpU+ +ef+vpK7f1hHZKSC4tBXjMpNcNs5WtAECoUUxo6Ee6rjEAQRXEUT0VerirmgIcIXA +W/+6v96LvKHGA6Iviuuet3k6Z2qwcNPoCKcSJNW7V4hyN8jmLtuH4CpxGiyQnIgp +DHUohNZ/dS1QCDYrz/O6M3IeegfbINqYOzmGlkziSkJhGpsLZ01tE9GfQN8Sx42d +UGkCAwEAAaOCAYYwggGCMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu +aW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU +yao8CDl+wULAPbcvhCHnijDnx7Ewga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I +RWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh +MRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD +ZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb +M0C2bSEwFAYDVR0RBA0wC4IJbG9jYWxob3N0MBEGCWCGSAGG+EIBAQQEAwIGwDAL +BgNVHQ8EBAMCBaAwNAYDVR0lBC0wKwYIKwYBBQUHAwEGCWCGSAGG+EIEAQYKKwYB +BAGCNwoDAwYIKwYBBQUHAwIwDQYJKoZIhvcNAQELBQADggEBAJtjruxW7Ax61YjR +CgqBKTdPpgi4eHgjr1u3ZWHXZCrJ56bSscs2vyMuLUiFfxYPZK8D210OpxTF9gSy +a5InusvSEyWiFbCOSi3rQRgJsWjVD2tW2obtSnopMAl3Y6RkPeMu128ajJbJy4H+ +o2014wnqmy7ajI7IyWmxg+dvLV+hrDKuKVepXJt98P1HPPNq0Ot3jXAGonQ91jce +e+fZ5DPJna36JMZN4izJJct1vo3pg36t21Oel77Vf4OQ/HUdAim3mRijOSWiVLch +fb4LTOr/gLlLXiHtJa3UYlJZeYMy3zChZGgFzDWti9Nma7Ext7Oy2A9blkDvVx1/ +sLD06ds= +-----END CERTIFICATE----- diff --git a/test/configs/certs/srva-cert.pem b/test/configs/certs/srva-cert.pem new file mode 100644 index 0000000..ed0489c --- /dev/null +++ b/test/configs/certs/srva-cert.pem @@ -0,0 +1,99 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 0a:7d:45:32:e2:24:e3:81:f9:5a:64:e1:f2:50:92:d8:e2:3e:14:8e + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27 + Validity + Not Before: Aug 27 20:23:02 2022 GMT + Not After : Aug 24 20:23:02 2032 GMT + Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (2048 bit) + Modulus: + 00:9f:b5:d0:95:c7:8c:ee:16:13:fc:d6:90:8c:75: + a9:3b:2c:ab:ff:b6:b6:ae:39:8f:c3:d4:75:7d:01: + ba:0b:f0:db:8a:72:2d:41:49:e1:f1:b5:f6:5c:32: + af:b0:46:89:ee:33:5d:84:47:69:a6:4b:de:3f:8b: + a4:29:4c:98:20:6b:94:29:bc:4b:cb:84:73:d1:6d: + 35:15:63:f8:55:a7:50:b4:2e:3e:e5:61:8a:c4:a5: + fb:7a:88:aa:14:b2:06:36:8f:93:3b:8c:29:d2:ee: + f6:8f:e7:83:68:33:59:17:d6:1f:b4:05:4d:02:ff: + f3:a0:1f:15:2c:ad:8a:33:13:95:95:d7:b6:54:6f: + ac:cb:98:8c:fb:69:c7:a6:cc:77:5f:20:08:e7:8b: + 3b:79:5d:7d:63:2e:6e:cf:ce:b6:04:d0:0a:c5:19: + 6d:ed:aa:45:7d:32:c3:53:ba:5d:d6:5c:e5:30:e0: + c7:3c:2f:88:9e:0d:33:8a:c8:30:40:af:48:a6:aa: + 4d:68:81:55:4f:43:cc:f2:79:3a:c2:23:0d:fe:09: + dc:c0:85:94:cf:c0:f4:e2:7e:c2:64:7a:55:41:f1: + 3f:3d:e3:46:4a:ef:55:33:d2:ea:72:56:25:15:6c: + 47:1b:4d:c5:da:d3:6a:cb:62:db:d4:7c:86:64:81: + 86:61 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Comment: + nats.io nats-server test-suite certificate + X509v3 Subject Key Identifier: + C7:6D:8A:C9:67:13:C9:63:AB:01:87:58:33:DC:FB:24:AC:61:65:4E + X509v3 Authority Key Identifier: + keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A + DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27 + serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21 + + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1 + Netscape Cert Type: + SSL Client, SSL Server + X509v3 Key Usage: + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, Netscape Server Gated Crypto, Microsoft Server Gated Crypto, TLS Web Client Authentication + Signature Algorithm: sha256WithRSAEncryption + a1:3f:db:97:03:ca:e3:34:36:01:8a:d9:34:1e:1e:9e:d5:bf: + 7b:9f:37:cf:61:2a:8e:fa:c3:cb:68:ca:f3:4e:04:78:8d:5a: + 0c:c0:5b:24:fe:56:a5:c0:7a:b1:bd:63:1e:ff:bf:72:9a:fa: + 08:33:4b:e0:2b:35:8b:03:c4:23:7f:54:3a:8a:f2:a9:28:d7: + a8:7f:10:96:93:a3:b5:82:59:f1:b6:05:3b:58:cd:a1:8d:a9: + 77:fa:71:36:f7:68:c9:50:46:4c:0f:c3:f8:f8:90:0c:69:eb: + 53:e6:d0:5d:32:2f:d3:f7:b0:8d:75:68:59:37:d7:de:ee:da: + 42:99:1c:24:03:28:c4:42:8c:17:e4:ed:5a:d8:70:32:01:fb: + f8:57:f8:1a:2d:bd:f6:e1:35:12:28:65:ef:00:05:1b:e2:50: + d3:1e:3c:25:34:56:5a:86:8c:30:9d:e0:f4:79:d8:09:7d:74: + 0d:70:3a:2c:67:2a:d6:d2:6b:ed:d4:bd:1d:f2:d6:02:74:2a: + 75:b9:8b:6b:a4:c1:c7:d8:42:58:ad:85:d1:0c:0f:14:88:12: + 57:34:60:9f:18:32:f2:75:77:69:30:42:65:3a:f7:c1:16:49: + 76:21:ea:cd:95:b1:29:d1:66:e2:90:e9:86:c4:7b:2d:aa:2b: + 00:6e:59:f6 +-----BEGIN CERTIFICATE----- +MIIE+TCCA+GgAwIBAgIUCn1FMuIk44H5WmTh8lCS2OI+FI4wDQYJKoZIhvcNAQEL +BQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM +B1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl +IEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw +MjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV +BAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z +dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJ+10JXHjO4WE/zWkIx1 +qTssq/+2tq45j8PUdX0Bugvw24pyLUFJ4fG19lwyr7BGie4zXYRHaaZL3j+LpClM +mCBrlCm8S8uEc9FtNRVj+FWnULQuPuVhisSl+3qIqhSyBjaPkzuMKdLu9o/ng2gz +WRfWH7QFTQL/86AfFSytijMTlZXXtlRvrMuYjPtpx6bMd18gCOeLO3ldfWMubs/O +tgTQCsUZbe2qRX0yw1O6XdZc5TDgxzwviJ4NM4rIMECvSKaqTWiBVU9DzPJ5OsIj +Df4J3MCFlM/A9OJ+wmR6VUHxPz3jRkrvVTPS6nJWJRVsRxtNxdrTasti29R8hmSB +hmECAwEAAaOCAZ4wggGaMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu +aW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU +x22KyWcTyWOrAYdYM9z7JKxhZU4wga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I +RWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh +MRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD +ZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb +M0C2bSEwLAYDVR0RBCUwI4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAA +AAABMBEGCWCGSAGG+EIBAQQEAwIGwDALBgNVHQ8EBAMCBaAwNAYDVR0lBC0wKwYI +KwYBBQUHAwEGCWCGSAGG+EIEAQYKKwYBBAGCNwoDAwYIKwYBBQUHAwIwDQYJKoZI +hvcNAQELBQADggEBAKE/25cDyuM0NgGK2TQeHp7Vv3ufN89hKo76w8toyvNOBHiN +WgzAWyT+VqXAerG9Yx7/v3Ka+ggzS+ArNYsDxCN/VDqK8qko16h/EJaTo7WCWfG2 +BTtYzaGNqXf6cTb3aMlQRkwPw/j4kAxp61Pm0F0yL9P3sI11aFk3197u2kKZHCQD +KMRCjBfk7VrYcDIB+/hX+BotvfbhNRIoZe8ABRviUNMePCU0VlqGjDCd4PR52Al9 +dA1wOixnKtbSa+3UvR3y1gJ0KnW5i2ukwcfYQlithdEMDxSIElc0YJ8YMvJ1d2kw +QmU698EWSXYh6s2VsSnRZuKQ6YbEey2qKwBuWfY= +-----END CERTIFICATE----- diff --git a/test/configs/certs/srva-key.pem b/test/configs/certs/srva-key.pem new file mode 100644 index 0000000..7084c3e --- /dev/null +++ b/test/configs/certs/srva-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCftdCVx4zuFhP8 +1pCMdak7LKv/trauOY/D1HV9AboL8NuKci1BSeHxtfZcMq+wRonuM12ER2mmS94/ +i6QpTJgga5QpvEvLhHPRbTUVY/hVp1C0Lj7lYYrEpft6iKoUsgY2j5M7jCnS7vaP +54NoM1kX1h+0BU0C//OgHxUsrYozE5WV17ZUb6zLmIz7acemzHdfIAjnizt5XX1j +Lm7PzrYE0ArFGW3tqkV9MsNTul3WXOUw4Mc8L4ieDTOKyDBAr0imqk1ogVVPQ8zy +eTrCIw3+CdzAhZTPwPTifsJkelVB8T8940ZK71Uz0upyViUVbEcbTcXa02rLYtvU +fIZkgYZhAgMBAAECggEAAcqJN1TQmM74lGpXnqunJTnACBMlg1iz6w9T7f359y5R +WAElzsikVxCiXbBX1hNEcr5yuwwN/EZ3jKDlS21He32ZYMpy0yp+Hggqgz3myBcj +SZIHNI1egAwYgOxNdAP3G9+KWC2fPnVdoImJKL8CrcjB/EPe9Dyon9AZbbHYjfAS +wOCy06KTCvwJrNO1mYFSftEOe1S3TVGHp3KX96YWxR/yIgXNC+5bJWzf+X8Ca0i7 +YHsgoF1CqoLg7sq+nMo1xzGEapy2bkx8DRdxigIXBBNnKQdVVO7qjmUOw8EdJZ9/ +luZ+/lb6Oc9RT6+V3B97kpYP7Z+frItGZWuMtifLAQKBgQDP1eI2w/VW6CW0ODjQ +O1ivNzK1CK4L9FNA07tnhfzEOlF/NMqfBz7pYyn+1ObxMcXzFAZAmh5E/Q8b8QEP +4b51IUqGL6FSQlKSlWJlszD7QhUM6nXFmvDo1g3pBZ9w8EW12cL+FM/KBpXKB0aF +S8wcekbvGkgqI6SQEvkcRrM6EQKBgQDEuNXn8B7YTlkw7eXu/F87ZQkWb1bes8VI +aQzVDH/ywu5Cbou8iJYsdfpZFwnJ4ojIl0eBleqXpgb+SiEx8q5zmR/Tt0NDIjfP +lVdlcf3oLxjz6LKwHWPnXmz2WEeYez2/lkBc0XmOYrAF8jy2h1RX4kkvVGEBNNKE +ooiX2Kq3UQKBgFPQQCqvubezZMkZxyeKV0hJrUQ+XNnrTMue1zt3WCjWD9mJs5CQ +PnmmCOCAgSWRWe+eCqtwu3Y8yyQMe84ozkK0Czaz7I2cu2nrnkO/EKiyzIRp5Nas +lix/bhqjrtm/u40LZUxLnkQthah0aKcGmyh80zBHXATKXOnRDO6vQEXRAoGAKmuB +FMaeoB4k9XwXPPWGw5QB5aiUnuk8/WRJtBdB7+NA5WpZD4qUd3npRhTFs5b5z+dv +L+5X6+ONVoA9sDuYiqzWRB0cj/ls94JImkN+cPbW9qGTBo8P3/BopqZaqfLJWerY +bittftqhAmc92E5OTbUp+t7DHDN7wcYalkPObEECgYB0sNTdWVl78AayFqI/5Bg0 +FfZYRTdVyx6qTPsQHqmPGHNn+ZyPIzVJJyfT3Fp8cToJDyYZtp5lUcnsSb2Q3j/C +riF99UpDJfetEFJATlSWjgWkMgQVHm97qIZtr7XLzM2gWJ1/fE0IxIx3ZFGUi19x +l6nGMJ23s64br4XiyNqHIw== +-----END PRIVATE KEY----- diff --git a/test/configs/certs/srvb-cert.pem b/test/configs/certs/srvb-cert.pem new file mode 100644 index 0000000..4f2881b --- /dev/null +++ b/test/configs/certs/srvb-cert.pem @@ -0,0 +1,99 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 20:98:70:f3:55:1c:7e:a3:15:0c:64:c1:a2:8e:56:4d:4a:77:70:e8 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27 + Validity + Not Before: Aug 27 20:23:02 2022 GMT + Not After : Aug 24 20:23:02 2032 GMT + Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (2048 bit) + Modulus: + 00:bc:15:41:85:42:e2:da:58:c4:d4:2b:8a:92:88: + cf:a8:81:5b:03:5d:8b:d5:44:32:07:ae:ef:a2:ff: + 0f:29:14:dd:bf:7e:6b:37:e4:31:e1:94:7e:5e:be: + 74:40:68:ce:3b:d5:bc:00:a6:42:fc:0f:e1:c5:8d: + 4f:09:32:fc:af:6a:17:c8:92:3d:b0:2d:bd:ce:de: + 7f:d8:bd:9e:5b:84:f8:b6:8c:a3:7e:98:37:f1:37: + 1e:aa:cd:87:92:3f:33:34:17:0e:a1:78:3b:1a:88: + d6:73:c2:ce:41:71:81:ea:83:2d:f7:e5:e6:32:f1: + 18:33:6b:ad:9d:ec:0a:3e:8e:74:a6:b7:7b:4b:22: + cb:0e:6b:57:14:cc:4d:c0:27:a1:38:30:df:35:e6: + fe:49:8d:4f:18:4d:be:5d:df:46:d0:4e:3d:00:93: + f8:e7:48:4c:6f:fb:13:c0:fb:6c:8a:9e:53:66:24: + 3f:c3:93:66:57:6a:f6:66:88:94:b7:46:e1:e8:d6: + 8a:04:02:92:e1:c5:c9:b1:71:f0:d2:1e:4a:69:8b: + 0c:ac:a1:01:89:5f:1f:ac:79:da:8d:ad:bf:03:81: + b6:8c:70:5c:1e:74:f5:a4:d8:d1:8a:41:d7:bf:27: + 51:d1:fc:75:b9:34:73:03:ae:10:2e:ab:78:a2:af: + e7:99 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Comment: + nats.io nats-server test-suite certificate + X509v3 Subject Key Identifier: + 5F:CD:62:19:69:22:48:0B:79:B2:8F:53:D7:EA:C6:9B:C2:5E:58:94 + X509v3 Authority Key Identifier: + keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A + DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27 + serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21 + + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1 + Netscape Cert Type: + SSL Client, SSL Server + X509v3 Key Usage: + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, Netscape Server Gated Crypto, Microsoft Server Gated Crypto, TLS Web Client Authentication + Signature Algorithm: sha256WithRSAEncryption + 08:6c:2f:cd:9d:b4:44:42:51:29:c2:55:92:e9:52:69:af:ae: + c6:c4:4b:81:7a:a9:7c:81:f9:dd:27:fe:c1:e2:9b:59:92:38: + 1f:f4:d3:04:b7:ed:59:78:da:09:33:c0:c0:1d:60:79:07:78: + 15:aa:50:c4:f8:7c:47:96:a7:76:3f:ac:5f:eb:7d:8b:05:81: + 83:f7:be:7d:1e:6f:05:d4:bb:7b:ee:14:a8:af:e2:22:82:bf: + 25:7c:f2:30:38:98:68:43:c0:4a:08:80:11:81:be:c1:7e:43: + 22:cb:85:c3:b6:da:37:86:c5:7f:5b:8c:d9:b4:61:3a:08:c0: + 12:a0:a1:ea:2f:cd:58:da:41:8c:1e:10:1f:54:19:c2:6a:2e: + 3a:d9:0e:f6:7c:34:95:72:9a:41:ed:1e:79:84:1f:8e:89:8c: + 43:84:9d:32:42:e9:b5:db:1a:ab:2e:1a:f3:d1:b0:2b:38:fc: + e3:c7:4f:1b:6f:9c:2e:3f:83:7a:fc:22:23:aa:64:6c:6f:c9: + 26:81:81:5b:f1:8c:ca:0a:ba:2d:91:f7:2f:ba:d4:7f:dc:53: + e2:f6:1f:38:50:6e:6b:87:10:b6:02:af:56:f3:83:9f:3e:3d: + 83:e6:59:cf:3b:f8:bd:21:0a:8c:90:d4:26:74:70:1c:b1:cc: + a6:3a:27:e1 +-----BEGIN CERTIFICATE----- +MIIE+TCCA+GgAwIBAgIUIJhw81UcfqMVDGTBoo5WTUp3cOgwDQYJKoZIhvcNAQEL +BQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM +B1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl +IEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw +MjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV +BAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z +dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALwVQYVC4tpYxNQripKI +z6iBWwNdi9VEMgeu76L/DykU3b9+azfkMeGUfl6+dEBozjvVvACmQvwP4cWNTwky +/K9qF8iSPbAtvc7ef9i9nluE+LaMo36YN/E3HqrNh5I/MzQXDqF4OxqI1nPCzkFx +geqDLffl5jLxGDNrrZ3sCj6OdKa3e0siyw5rVxTMTcAnoTgw3zXm/kmNTxhNvl3f +RtBOPQCT+OdITG/7E8D7bIqeU2YkP8OTZldq9maIlLdG4ejWigQCkuHFybFx8NIe +SmmLDKyhAYlfH6x52o2tvwOBtoxwXB509aTY0YpB178nUdH8dbk0cwOuEC6reKKv +55kCAwEAAaOCAZ4wggGaMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu +aW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU +X81iGWkiSAt5so9T1+rGm8JeWJQwga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I +RWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh +MRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD +ZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb +M0C2bSEwLAYDVR0RBCUwI4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAA +AAABMBEGCWCGSAGG+EIBAQQEAwIGwDALBgNVHQ8EBAMCBaAwNAYDVR0lBC0wKwYI +KwYBBQUHAwEGCWCGSAGG+EIEAQYKKwYBBAGCNwoDAwYIKwYBBQUHAwIwDQYJKoZI +hvcNAQELBQADggEBAAhsL82dtERCUSnCVZLpUmmvrsbES4F6qXyB+d0n/sHim1mS +OB/00wS37Vl42gkzwMAdYHkHeBWqUMT4fEeWp3Y/rF/rfYsFgYP3vn0ebwXUu3vu +FKiv4iKCvyV88jA4mGhDwEoIgBGBvsF+QyLLhcO22jeGxX9bjNm0YToIwBKgoeov +zVjaQYweEB9UGcJqLjrZDvZ8NJVymkHtHnmEH46JjEOEnTJC6bXbGqsuGvPRsCs4 +/OPHTxtvnC4/g3r8IiOqZGxvySaBgVvxjMoKui2R9y+61H/cU+L2HzhQbmuHELYC +r1bzg58+PYPmWc87+L0hCoyQ1CZ0cByxzKY6J+E= +-----END CERTIFICATE----- diff --git a/test/configs/certs/srvb-key.pem b/test/configs/certs/srvb-key.pem new file mode 100644 index 0000000..53b2825 --- /dev/null +++ b/test/configs/certs/srvb-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQC8FUGFQuLaWMTU +K4qSiM+ogVsDXYvVRDIHru+i/w8pFN2/fms35DHhlH5evnRAaM471bwApkL8D+HF +jU8JMvyvahfIkj2wLb3O3n/YvZ5bhPi2jKN+mDfxNx6qzYeSPzM0Fw6heDsaiNZz +ws5BcYHqgy335eYy8Rgza62d7Ao+jnSmt3tLIssOa1cUzE3AJ6E4MN815v5JjU8Y +Tb5d30bQTj0Ak/jnSExv+xPA+2yKnlNmJD/Dk2ZXavZmiJS3RuHo1ooEApLhxcmx +cfDSHkppiwysoQGJXx+sedqNrb8DgbaMcFwedPWk2NGKQde/J1HR/HW5NHMDrhAu +q3iir+eZAgMBAAECggEAfFfVJEPO9Clhnx9WEoBOQQmb4QK0Un8uUDQQC4NyQ2ef +Hl12htHfRFJGpV5mPrrLNXWdQBSrDjd1vIQqb1t9AH80UA6Wp+XlqWEhxmm9yqWD +RDSAVk5OFbOqG6ObcE1GDbb8njV9ZZJ3rLpqX4uqJx8ogc5EvQL1S/FxK4mEUEv8 +bq4Iswp590+KwnaYtkXEoieUbE1QC0y6z9BEpEe3WkNVGAmnuJ6wCiE2pW6yzK94 +NxGvnsejgBVDZZprWbLdIztHDJ4+VDDctkxnqsx3cb2o+mK7CCP6C7v9MAvgrMzH +hEDu2WZC6T6GrVysYXSNmlWYPTsPh89F5Dyrp5vq8QKBgQDwTnMq4SBbrq3tsJEV +ChX3pqQ9icyaT/yLI1oWcrdJKpyMdDVxOBPjB04i7pUZFNKqHS/aKjozKmXHVFNy +X9wy5D0der7gqHmMeEV4B7UeO31E2B6nftlntt+V8U3Nu6r9mi3QCbH5x5vBjDH+ +ulx4Qma+CeBL+bP1Z8Vm8a6qNQKBgQDIXbY3jm3Gl2hE2+Ci7JceloRlwp3CYlHS +Kejhxx+secKLbbeXUp2A2uj6lz9DX/Gx+qw8zbhoodh6FkGJVrZchatUxci3P9rs +6c2rVMzcPGSt67tFtGGm04cbZgD83QDj0xPtlolZlrGV55PpUFJkS2AThcitMYkx +oNOZf3hUVQKBgQDK3n/NoFbv74z4D8q7h1M6Sl13ckDfuU0LraGdN+VSCH42+Ngx +J9VqIT2usdn/XZvnb1J5jJKrpUxdNexor1K3SAXsQDaqdLCAjPygs2kNB83ec9GV +N7qUG0ewTxGO8LMO+71XVwgARapRgGiokm4EPJJ+Nto7aaeqMpvtJ3V8TQKBgQCW +tZ5M1Y4OtW4RdU3XXpwgvtihbiyZvLiMm0d3I/7YDdHebE2Ove6IsKBTdRZ5yB4V +YFbTeW2/DhW1Zy4ubpHGsCh6IDbkV1yFnSs9NxDMi4SzKT/4o0BcankSi7wsdtOn +iepQaE2xnF7BQaoI8Yi3tdZzeMrEMyFdufnPWIka7QKBgQC9nCUtLl3FCaeq6PMv +KrNQRu5dPbpy5SJI1SSXKCUU9VaA6XMKrNeQHkgsnsj6KZ9Rl+yNe31L/GHNRiSj +4O/vChYMk1bXgESenvNafSMbl6agwi/Y1GFF6CM6/vDas4tB7bsFQPh+cnb9uIvq +5IFkU1NTCSSydk03li94ZF81Bw== +-----END PRIVATE KEY----- diff --git a/test/configs/certs/svid/ca.key b/test/configs/certs/svid/ca.key new file mode 100644 index 0000000..1e66d40 --- /dev/null +++ b/test/configs/certs/svid/ca.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDXYllrRiGR1WPO +rT07fN7bY5q8Pnsb1zaO13ZlTdQpiNsEa9AkOL7NaAZFZJ9+KdhQYQgbAUb5Cvyw +UIrydTvDrqzghQLaHn9VEHx7xJWJRaCYSqB7MBZCsc7ov/4ntIQO4H/aYhBdcXzj +ELpqeMSSLpnH9+9KmZU4ArxU0dqx+2YzUoCDXszsD8541797ksrxrZnLkuNxDmtZ +ULn55pY/MT7Tq0gkUA+ReDpHeElQuf9L40ODdueSpteR4mDNPQmLoTE5qwTsyhjF +Bght+121Chu3tRwprJqxEjwe/YB6rAFVnlgWwLumfYer/jI8UNoSsiVlj8ToMR11 +kIVF6NGTAgMBAAECggEAVJJoJSCxDsfY/G7207R493FZcWAhS/HKGPWOpkGUtMhp +o4dH/+kONq6Jg9b6b3oz13/6RExQl3qSdRLzNDgHQxOEC1b/IzGFcxvZnWm4A+JE +qHfpIuXjX8XcxnSY8fNGRWcunMwTu/VTgf9wC12nZfUJakU2/zZmKsucMrwCIQ8q +rYBoHlVVs/R+3Hj6FYj5yFeoCsWyPswFd07llu6/h8ADY+P1xnOeilJ8pdNR6vaQ +wZQhBc0m414XRp2OzpKAX0+7sATSUhhGnU5VdlCcP6whHKmva2U3RusIN1Qy7UIA +35sNxMsSzOvozwZXbWMpWaUU69uLtGjeX6qhh+bk9QKBgQD6XyViXijKHN56xFx9 +4inTPlqOrgtuZDm+9pQ/TFaZJfK4GoG/8QyUOXmLAnxbbTAAKWXc/zHXhVr71wsW +4pgsUxypWfND9yQ7dYfJCR7PhkYqdRm3WG+JOss9aL+FULdqjD/4tw4Y5EHc2vpO +plQ0p0/xJtLAlg0aqjAonjDUJwKBgQDcOdrrklov3IvKw7LgKYmvDRjLGEyMm3qs +0fPgWDG7BzZ8nfDQ7o2Pglf/ypuLYTQyZV4DXAXdiQGjkM/SzStsSV0hmEFfaLtX +JPSbk6gjW1+9+sxq1xJfxvu9tlUeLoRwA0Ck0rwGn3y3OjcQCCVL+5jkqmmERihM ++3IY8+/etQKBgG4yZHToNpHGpRFpzb/GRFxqrFbyOavLxzLKurMleVQMxMjNOeBu +SvgOV/WcEXn4E9FuZxwe4iQW2NXRb1sSPqH7rrjHE3ANnc/hyfLs5be3RLi5M6lj +7it7SirsKKeXwDLQKfPhNf40sNxxMQxe3t8R6Pid+tKy+G+NCodGIP2vAoGBALWE +sl8tGE0O2eJc/5koY6X3g5ITCzVyCeFchElMKGFiBpUckZuEpQW+h838L8zoZ9+K +Eb1vIh6BiOpKuCGmnr34klzYZDnB13hyq3N4d0P9UqMCj28YqrprBqBOI0ZnTR4O +cD/qurQyzNcxkqUSMbu5O0Ju+93c/ebF8juDBRlpAoGAM14TPQSUz4oiMVB5VWAj +75OK2rFQLC+mMx2T7u8UqpHVb+A6lUZcj0aN6TgRQxQo63bRXaEc/13BTMKmmqze +rM7bpj4+b08Fr0yhw7fjO2b/q+zn8Gagk6UWIptpY/8WC4OSgaSjMC6ZErPWqRJs +cJXNOa3YaoQcl8SfLW45RNg= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/svid/ca.pem b/test/configs/certs/svid/ca.pem new file mode 100644 index 0000000..a8fdfc4 --- /dev/null +++ b/test/configs/certs/svid/ca.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDBzCCAe+gAwIBAgIUHba56kOIMIRpaKPovfEccmejyJcwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMzZaGA8yMTI0MDgx +MzIyMzAzNlowEjEQMA4GA1UEAwwHTkFUUyBDQTCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBANdiWWtGIZHVY86tPTt83ttjmrw+exvXNo7XdmVN1CmI2wRr +0CQ4vs1oBkVkn34p2FBhCBsBRvkK/LBQivJ1O8OurOCFAtoef1UQfHvElYlFoJhK +oHswFkKxzui//ie0hA7gf9piEF1xfOMQump4xJIumcf370qZlTgCvFTR2rH7ZjNS +gINezOwPznjXv3uSyvGtmcuS43EOa1lQufnmlj8xPtOrSCRQD5F4Okd4SVC5/0vj +Q4N255Km15HiYM09CYuhMTmrBOzKGMUGCG37XbUKG7e1HCmsmrESPB79gHqsAVWe +WBbAu6Z9h6v+MjxQ2hKyJWWPxOgxHXWQhUXo0ZMCAwEAAaNTMFEwHQYDVR0OBBYE +FCcUWYVGF5cPrQMR/VNkapjuXMRbMB8GA1UdIwQYMBaAFCcUWYVGF5cPrQMR/VNk +apjuXMRbMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBALdP3pUX +SkBo3xJ1m7KskL6Xtic9eDxu9aJKYUH4qw16Aoyj+tZm5R1UvyaEsHMG+hFOZRRp +tArbOknL4RZG7Tx/ATTiSQl6ozmkZ8GRpKY6VsN5c6Y1e9sbdXeGgfitWMUkQQhD +Ftqps2TM6IfJBdj4BhVGAjG45tUhOTfMjZliaNZexVyR6XjuhDHcDDpcdaFgDfDA +uAYDDbfhO7MIKxplzrPcI7e9zDdb9mgB8UExNltxrH9w3J1cG4VpKxFtXYs5FO4d +swXWTr2cY4sTT/bhVJET+YnHyyZgg+Ear4n7eujA9klUHEMwPy/QDCn8u/091yew +NnOmSiP2A+dWQB4= +-----END CERTIFICATE----- diff --git a/test/configs/certs/svid/client-a.key b/test/configs/certs/svid/client-a.key new file mode 100644 index 0000000..d86b6e7 --- /dev/null +++ b/test/configs/certs/svid/client-a.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCUuqRV+zUywU/g +8Q5z6i+mJxzHCXjM1XPKu3gmWHtTTfRbQtvGdFg5mWHL8rgvEkeYkK+yn6NmGFPP +z8vJDLyVzz70YQpi1Qv2Awe0bHTgwlCXAKUlHfEapDh3068hEonz2Fp8n652ArNd +oaKHJCUZtKIfm10cs+nX+cuiv+trfJUVYxPyrQhu7lhWhNRwDWedI3pc7382mwze +SwFft0Mi1AyK7JhVMRM6ld91MXuJlLNfHKEejrumkt6dIP3o7cJJGxW4IFjDs4gh +IK9cd6JMeYQXs9Bc03hKazCKCWITrPPiz0BD7beiqFUNcPgOlzqveBnqsXyRLEjE +iQ9a9QwDAgMBAAECggEAEC8994FOU7HyRacrDHVyg/aDoivSB8WT7n7MtKzc7BQd +QC/GshIyZj+JAzmWzhXY0Vhaq5RwTTQz1/cOf61cJEKY4N0NKp4byW9LpQLxtmJO +2hp0M4AjnuOPUDLVPu3BMgDcL+2XKDkOWGULcQCqQjbZjEOgM/cRaQhCHvlwM+er +JqZ0CN6zPrcQTVlwEZLiL+aQ3igZwXaEPBq3VmuEW6rjpq7n7AE19GCmtwzo9PXW +BxZFKMMDxsiiqTY/UyM0Sfci+oGYmDMwVMFh5wDdsyAv6ulF8g0U8YTV4w8uBj0I +BRsSS8P6nwUIWGS9pz/1mgnsESPQtj9cs2dDUp5P0QKBgQDRCdVvlObEeka/rlIM +to9J2iThHPecXHHfD5S5AxC6/fU+n/X2k3Q6GzlYIkFdYQzic8oTtQaODtib9eFL +XHDCNyj4x3Gb7UPBl/f9uqIseUhpAJk6GRiUBgQlCJGp4btMVNx1bhoU+nWgysEo +eGvC/B8WB+04MLGURn1k3TTnawKBgQC2JE/dLa7bYxKH1K7h5HxwDjrdYQGFfWIb +9/uFyZa+1NAWYj0TD3vy/pox7PI/VMMa6oWl2C6Mv7aSLlFTwcZz3YKPv8iB7WHR +F12/s5xcKNID37J4Fo9EFq2751OqKyhW0aU65a7W/mjJY1loLo0fZVmuc1ZGPcDG +v39nwJ1LyQKBgQCY8Ub2Qs2tB4HoENT7dD7WXAMLqbQI9SJjf/TigLmm9tSOo17E +sBIjyXNlZnrIpY9VyD4buSE4ougdBAN4rgPaNZ2Gl/Ypjak7ZcMOI5UwUPSHzuZN +Obtf4h76MZDR6NSu/o/mY0419yJFKNO2lpTRrsHXzqjjqczjncd1GtvwTQKBgCVV +Ug1axMFxECZJQMNKavso34Xq0T4EUiy7apGknUJmKnS8FBqpDNXku8RJ9elnPUpy +EYzJv9jXnttdpQvO5xduqsk+HSIMfwU1jeCyNvVo5IsoudMFhL4HC/s80hhKGk2Z +UQ2+cFTe9ql4tKW3DPQHeRdZyrBBDhduvvWMNHGJAoGAD2x8zNEhBqOuBn5prtsC +WPsjXRIjhXIlQbWwda30an74Bw5Z6OWgZr8nUjBtHn7RkeS8PqTE/UzMTLqD1+TD +PvnA8GZ2WWdxhdqy04qkYS5SnkSSLeU0hUYZkLrjZJv5QdTnhBBVG/kDDI7TO8NA +gKMQVjTCSStdZm02D6EqEfw= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/svid/client-a.pem b/test/configs/certs/svid/client-a.pem new file mode 100644 index 0000000..73fa7ad --- /dev/null +++ b/test/configs/certs/svid/client-a.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIID1DCCArygAwIBAgIUU+CvNNI2Hg/kyKWUkP+fw3bsFe4wDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMzZaGA8yMTI0MDgx +MzIyMzAzNlowgY4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEUMBIGA1UEBwwL +TG9zIEFuZ2VsZXMxDTALBgNVBAoMBE5BVFMxDTALBgNVBAsMBE5BVFMxEjAQBgNV +BAMMCWxvY2FsaG9zdDEUMBIGCgmSJomT8ixkARkWBGZvbzExFDASBgoJkiaJk/Is +ZAEZFgRmb28yMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAlLqkVfs1 +MsFP4PEOc+ovpiccxwl4zNVzyrt4Jlh7U030W0LbxnRYOZlhy/K4LxJHmJCvsp+j +ZhhTz8/LyQy8lc8+9GEKYtUL9gMHtGx04MJQlwClJR3xGqQ4d9OvIRKJ89hafJ+u +dgKzXaGihyQlGbSiH5tdHLPp1/nLor/ra3yVFWMT8q0Ibu5YVoTUcA1nnSN6XO9/ +NpsM3ksBX7dDItQMiuyYVTETOpXfdTF7iZSzXxyhHo67ppLenSD96O3CSRsVuCBY +w7OIISCvXHeiTHmEF7PQXNN4SmswigliE6zz4s9AQ+23oqhVDXD4Dpc6r3gZ6rF8 +kSxIxIkPWvUMAwIDAQABo4GiMIGfMF0GA1UdEQRWMFSCCWxvY2FsaG9zdIILZXhh +bXBsZS5jb22CD3d3dy5leGFtcGxlLmNvbYYpc3BpZmZlOi8vbG9jYWxob3N0L215 +LW5hdHMtc2VydmljZS91c2VyLWEwHQYDVR0OBBYEFDFDpnqTvAQQBUIBMzip3XCT +FdsTMB8GA1UdIwQYMBaAFCcUWYVGF5cPrQMR/VNkapjuXMRbMA0GCSqGSIb3DQEB +CwUAA4IBAQA8lg93kYHKkc4iF+mB8xHtyT1Pmy8CoxbDGy98cac1ny9s9K5kROTZ +rE9p2FaC00jf5T/+si5diXa0EnIjWCk0uRlrW5PL85UbsqHJkWav7zCk7cDa9YrR +nk4LMABf40RP3SOM4yxjjoVU13jFwIZ5M69Hce3CJ6ZkSoWRzWlKf6ECAbX/JIcJ +u8wLBXbJaAvc1CQiZpiSWDvItg+sUn1dU1nmk2o1YmyBPN6uVF874f3Ihazb9wmJ +dkWO48O+FaahjR6GzfjEx7w765ATqvTVYAgGZ55ps65VwfjcQbP1jYO4Sb3Aeeub +yapE1595oBWoWdwtEeOTJrzSIEhkhyiC +-----END CERTIFICATE----- diff --git a/test/configs/certs/svid/client-b.key b/test/configs/certs/svid/client-b.key new file mode 100644 index 0000000..e620ec4 --- /dev/null +++ b/test/configs/certs/svid/client-b.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC7bOmV7/qGy+1L ++cgBawm55lxS2w8QQr/lZrq4k/plQC2UDRvGPDaIihbvbP+pfiAgfSiWd/s+Qhdr +oOL/gkAJo/iD8HYu9FN+Y4vPlJyoEguNJzOyRe7U9YnW+Gsefwr60PoeiKfXK5B8 +J5KahSQSka3hgW+yp4ECzAqpjobGUpzNoFVoQ0o9gBsvBWVOCJ02h5kcwsLFSSbh +h76FPiedKHtA3VAj+vrg7rYT4wHpmSAdJZ7uNvbjbB4Jv35ebTnlkywBcPJi3YGt +UI8O8HTDP1K8mlTitzvqHGMDtiT6OQJpffuJhOJ+PnXd4nOwKN+AAMQ8RxPx7lv5 +n2oYmq0vAgMBAAECggEAQNdYJIBGivXoHZ3HnP37+m2AyUVx+60sGGAm/w5q1TaT +ziGO11EjiK9FagWqmWfCE5tZPF079XkGAz1A6yaTowIwSu0Y5fuvGw2zY9hFLjw3 +9Nh4IRfinZho3I9K99z53nYH4iJi9qTisqE6Le0b4rS1lEukaglbL62Jjsl5RQje +EogelhVsVh1Lv2GxZw2GUZHogTuWK3ZW3objq6HE0oReS9vUDwSL/rwuOcrGFdnb +XkVD9Z+1eD8QD4Jp0rP0Abc8hkotb/alStbHTl3MzBEwVZXyc13HFDB4loOfhJU/ +bgPBgeFh1xXCR15+ouZsbsPkRZ5kwLSsbskUpkXeSQKBgQDe60LcdI0w7edfl3bl +pEOyA4Volh5net4dpQhLZFbjFAoTG1OA0Z9hBoZwoRLi1xkuMdLUzJOH6d9sBni2 +qn2sbHVKXTtb7Ie+RMb3XMpN34tlpqH2+vnrVkxkrPwWluhynVZm4AsefMHRAza2 +K3JBl2M+IAzgsfLUYNj4xWh35QKBgQDXPT5xjdr+NOADocrlf80w2iVnvLah0yVb +FgVenkX1u5ZUiv9calJagfaVQNm4VwxBXGr1/3+PoPdKNWFjnlPO8R1E3Eia1Iu+ +FberWhEpAJuctT+lcPS8sFDh4r5edgyOXE3ikuI4tDRsQErGATfWkIfmuKUThW2u +A8YZWmTXgwKBgQC0VUu0fhEOSRcIzMYhhh5BbvnIntCf09TF1JD+NIDZmIeB53RI +3Mt7OUUe7h8SJQI4zZOKFKkwpTDZ+e5Dj261QtQrwXpYpeM2N/thds4t6y6goAmc +fHIbQKfyNcKFH7WRqtiPxqUYF8LPLDNhcYovp6FgS5s/VpVj26kKruARRQKBgAP6 +Krw4bdKCsZz/kVs6SFVdwpEvKITG/Qq41nKQJCTNSZLYWq2ffVQU0LSjVSKV4TEX +/xX6maXSyJ8Da4BGva+2Pt813tWVldt2VUWCTYzfaQC1TK7G7o7KB0SVRwVB0yuS +QxIruqhnlAxoB7mu7hQ16Xz27n302jj78t4nEHP3AoGBAJMEOropyzzX3sYp4E9c +u+VPyqcEMXfFimh2il0hijgB17dSW2UXeoQ85gevbPfGURL7a549xar1cNgEfQO0 +T82PecgRs83SEcgHT4mntytGY/lsiowAQZ2rXtlyG4ADPwbQGV3UXqV1Bm7yBT38 +YJJKJNKjyMHA8auKGG37WDhu +-----END PRIVATE KEY----- diff --git a/test/configs/certs/svid/client-b.pem b/test/configs/certs/svid/client-b.pem new file mode 100644 index 0000000..3ae380a --- /dev/null +++ b/test/configs/certs/svid/client-b.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDYjCCAkqgAwIBAgIUU+CvNNI2Hg/kyKWUkP+fw3bsFe8wDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMzZaGA8yMTI0MDgx +MzIyMzAzNlowHTELMAkGA1UEBhMCVVMxDjAMBgNVBAoMBVNQSVJFMIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu2zple/6hsvtS/nIAWsJueZcUtsPEEK/ +5Wa6uJP6ZUAtlA0bxjw2iIoW72z/qX4gIH0olnf7PkIXa6Di/4JACaP4g/B2LvRT +fmOLz5ScqBILjSczskXu1PWJ1vhrHn8K+tD6Hoin1yuQfCeSmoUkEpGt4YFvsqeB +AswKqY6GxlKczaBVaENKPYAbLwVlTgidNoeZHMLCxUkm4Ye+hT4nnSh7QN1QI/r6 +4O62E+MB6ZkgHSWe7jb242weCb9+Xm055ZMsAXDyYt2BrVCPDvB0wz9SvJpU4rc7 +6hxjA7Yk+jkCaX37iYTifj513eJzsCjfgADEPEcT8e5b+Z9qGJqtLwIDAQABo4Gi +MIGfMF0GA1UdEQRWMFSCCWxvY2FsaG9zdIILZXhhbXBsZS5jb22CD3d3dy5leGFt +cGxlLmNvbYYpc3BpZmZlOi8vbG9jYWxob3N0L215LW5hdHMtc2VydmljZS91c2Vy +LWIwHQYDVR0OBBYEFAiaPOQsBb3+rAOlSevDy5kp7zCnMB8GA1UdIwQYMBaAFCcU +WYVGF5cPrQMR/VNkapjuXMRbMA0GCSqGSIb3DQEBCwUAA4IBAQBq7BxhZMCTl6Xe +i3GcPlOLp9JPP8bcyYZVB6nAf38eErpwYSPvp4I3IFU+KyZgtDIAb6Oy04BNV9eT +BlpX300ZbylO/TLCrlMIJDYLIt5NciVe8IxsE//uLXFq5wZpcCcL9aQA2g0wW8hi +pkK1dQd3W1ryR/LwKiy0fcZJw/EFskoqq6vPJATIFvH/O0OxdKP9T24YVGgLBzwj +xzUqVlU0CuP0snx6x4F3Oha6kJGwc90RlXszh2ELhO+o4sk4wtfAlEYaM+H3kbAK +lRe8FENyGprOjlxjy6N1noJW9Mgisx9kNcsCzW8auGY6l8Vs3wW3vO3n4ZUnq9qc +t95/tstH +-----END CERTIFICATE----- diff --git a/test/configs/certs/svid/server.key b/test/configs/certs/svid/server.key new file mode 100644 index 0000000..367f4d5 --- /dev/null +++ b/test/configs/certs/svid/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCmmTq1fDtsMKVU +kc9jWb/JlOXU2i7s5uPzETulgrQRhHCWE6gACW3VYhTCecSf6TyLaFeTkwVsqv+r +teOld0gRyA/NdCN0fY04OGfr7eg3dDyK+ovGInP1rh+6UrMl2Y+Iy5n/lymFPFkd +EoqaKjXv5npGQScvrvTYN8hFu6ww3EdPI3rAYwolujGadbXttprLRIZvZQuYI30J +LydCVsFMwfm0lnegEHtr/lx2UbxXeyf9nIOle7UsPAdR7oRXomf5FCoXUuWajv3Q +W9NZEM4E0O2SbH/RO12MmmeXH0mwQuvrZAvgEtc8ZDMzmRZ+AOyCXxp6F7ME9SrB +pQqS7ljPAgMBAAECggEAFHvephXiUIkiquVKtJKCbEysfaFeB67T9Dz4Mn+BmG3K +UgmUrHx8TrJCjy7dqDPLR/DKY9C96ACWcwxoMx4ZdGY+2yoeeeI+DN2SmYWk+byG +/SeVLAVscJa6O/STJmI7rbwq8Swq5MDozBRNimsKOrcB7cVxBgHjBPrG2xwaLd+U +zExqarE89tfvtFVUQobyeOimcSEerMIkOngwtRxbhvF956VpvVEaJDsm1rHqeiZA +rKoky01IKMTXcf6g1bERKKXc/nPbMMOMYDD0FbQM5bK7OqSuN1+XFgLbIQOgShWv +OhlENsa8TttAUq6dst+bv+zxnXTLNhV1INEdJWgSQQKBgQDfUktQoMFRCfwnFCNJ +6r4jR108hLH7e8/2AfEJWAW8iwfvnpf4/fW7kqttcZ0FwLOGDb3TKJbXx0Jlz3Fr +bkD3BIb3+19ludqI+hZanoSUqXBt/iTWRJpOGwpoLSIGNCR01GePMCva5MvETPj5 +nh7la+6AXngvxx8cR7RJMcxAMQKBgQC++hLaJ+cec3YpJGAAIFnXRgtFyzKn+ToE +TvcbJEtHn1HO1MLJhd3E04A0BufjglxJ16Kpc95252O8GyveIKm408vbXjIY3Fxa +y0zlnUB3UNdZ21HdZU3vr8T0C2s1c6tKTStuxUjtzfu1LXElJp/MxfodnyL0UE4L +OiOmza7o/wKBgQC17uY6rmwvaDILaCWDn4D9XrM/jV8uYOTAJz7F/PbXbnCtrwdi +KwtZn5iXFdr1h9L5YBKIh2W0LroSFVjyS6UTWJXYSuqiBjJaK1uwwbmDHzLf+Q7r +nIIVJYRp8HLqmOomX7Bqf7UKpCC5MHgZa88B2tc3rz68tbzLnkLq6m3a0QKBgQCL +8MVVtH2oE6VXGn96ODp9A6eI+g/hhD/eHlr2OY8sKXOR4tflQcGy0+SSuyi6zrB/ +r5JTJ0Oxz3aGxUZM25RiilhMI+cDzp2iLmznqGYvyD41av+/AtesNhlVVvS1U1AF +/yw/XDNHhnPWXDkXpwjbzYqsoArGCf5WPcEYHon30wKBgDhFrftmpc8AgXoCDaAN +Ymb23qURm8m/+FTc+49xX3F/Lk2stW36M1Q2d0Liu30DTj1Yzkeb62ROycCdMN4l +SG87HwV0A2Fs4lAKhaYhd505+3keNg+tAr5puySRaysaT6S9uw9QsoskA8+AZzq4 +mzR28luIuvUmtSP+UBrQ6hyW +-----END PRIVATE KEY----- diff --git a/test/configs/certs/svid/server.pem b/test/configs/certs/svid/server.pem new file mode 100644 index 0000000..35b2bb4 --- /dev/null +++ b/test/configs/certs/svid/server.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDLDCCAhSgAwIBAgIUU+CvNNI2Hg/kyKWUkP+fw3bsFe0wDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMzZaGA8yMTI0MDgx +MzIyMzAzNlowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAppk6tXw7bDClVJHPY1m/yZTl1Nou7Obj8xE7pYK0EYRw +lhOoAAlt1WIUwnnEn+k8i2hXk5MFbKr/q7XjpXdIEcgPzXQjdH2NODhn6+3oN3Q8 +ivqLxiJz9a4fulKzJdmPiMuZ/5cphTxZHRKKmio17+Z6RkEnL6702DfIRbusMNxH +TyN6wGMKJboxmnW17baay0SGb2ULmCN9CS8nQlbBTMH5tJZ3oBB7a/5cdlG8V3sn +/ZyDpXu1LDwHUe6EV6Jn+RQqF1Llmo790FvTWRDOBNDtkmx/0TtdjJpnlx9JsELr +62QL4BLXPGQzM5kWfgDsgl8aehezBPUqwaUKku5YzwIDAQABo3YwdDAyBgNVHREE +KzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYD +VR0OBBYEFO2EJhb54LzOxx6W7lqphByWFT8pMB8GA1UdIwQYMBaAFCcUWYVGF5cP +rQMR/VNkapjuXMRbMA0GCSqGSIb3DQEBCwUAA4IBAQA86URTSVJU6k6VjcnT9Fzh +fnid+OV2NPoKzczw4pTc7aGjkZxtCD1ENlYhHlcni0ZFMIRtLiDARjwhBkVJ5S84 +1NS5l4J86ymazkSFZ27m8y0UeSDuPxZJFA/yBAmt/BoKRNMAAmonepdx73JpbiGE +yMD9RU5qI2E6BGo0B2khRYuY+POPFGPueVbqg3qR+LJPlxp8OIet9HGagEcUK7lG +PeFNKSUCfmuHHD/QO/gmG4ZM9/qB7M3McYh4/+CIihEmhfVK9Odo6Fs0t5MQdcEo +v6++7DlnpwRnmgC8GtEBMK5XJAILb6cI11TearTphFoP7xpvz0VHp3Gy9mA5cdwx +-----END CERTIFICATE----- diff --git a/test/configs/certs/svid/svid-user-a.key b/test/configs/certs/svid/svid-user-a.key new file mode 100644 index 0000000..12bddfc --- /dev/null +++ b/test/configs/certs/svid/svid-user-a.key @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgST6YP9hyfw/Vmoxo +MFp6MJFZu4xaYK3OweYcANEFTkmhRANCAAQCY7xD5sWZDVSRmBu2l4sjJYzpGVqg +d7M8I6LnFjkhkJFc0h9n8jPud8POip9BfXJyLBzmtW+CfZC84zlFSknN +-----END PRIVATE KEY----- diff --git a/test/configs/certs/svid/svid-user-a.pem b/test/configs/certs/svid/svid-user-a.pem new file mode 100644 index 0000000..6f4c57e --- /dev/null +++ b/test/configs/certs/svid/svid-user-a.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIICAzCCAamgAwIBAgIRAJXUSiQv6UVx+RHn17gl6xswCgYIKoZIzj0EAwIwMDEL +MAkGA1UEBhMCVVMxDTALBgNVBAoTBE5BVFMxEjAQBgNVBAMTCWxvY2FsaG9zdDAe +Fw0yMDA1MjcxODI3MTRaFw0yNTA1MDkyMTA1MTFaMB0xCzAJBgNVBAYTAlVTMQ4w +DAYDVQQKEwVTUElSRTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABAJjvEPmxZkN +VJGYG7aXiyMljOkZWqB3szwjoucWOSGQkVzSH2fyM+53w86Kn0F9cnIsHOa1b4J9 +kLzjOUVKSc2jgbYwgbMwDgYDVR0PAQH/BAQDAgOoMB0GA1UdJQQWMBQGCCsGAQUF +BwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBS9gO5XK6fzpkTH +DuhPV7lB2AAbNjAfBgNVHSMEGDAWgBSeUg2uZMN8Eio3bHcxv7zJIzclhzA0BgNV +HREELTArhilzcGlmZmU6Ly9sb2NhbGhvc3QvbXktbmF0cy1zZXJ2aWNlL3VzZXIt +YTAKBggqhkjOPQQDAgNIADBFAiA2TvD3xhOCvn9E2QF42o7gTjqGicTeNInKTEKe +A6AMzgIhAKdpmH5367YqHijKhtfklnM7g8WhdPhn38xWL7jG+5+a +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIC3jCCAcagAwIBAgIRAP6sMTwSA5gCsJWO5fSCokUwDQYJKoZIhvcNAQELBQAw +ajELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFDASBgNVBAcMC0xv +cyBBbmdlbGVzMQ0wCwYDVQQKDAROQVRTMQ0wCwYDVQQLDAROQVRTMRIwEAYDVQQD +DAlsb2NhbGhvc3QwHhcNMjAwNTI3MTgxNzAyWhcNMjUwNTA5MjEwNTExWjAwMQsw +CQYDVQQGEwJVUzENMAsGA1UEChMETkFUUzESMBAGA1UEAxMJbG9jYWxob3N0MFkw +EwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJhF2UV33hUpg53uGmy/GkXEI2ZR8EQmp +EHxG1GWbjHR7FBdVP/HmPyVKu5vfegXZp/hD3H7UYHjiNeKMYyGT4qOBgzCBgDAO +BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUnlINrmTD +fBIqN2x3Mb+8ySM3JYcwHwYDVR0jBBgwFoAUlDSkRxR41mV4yHobEDnU1bjsywgw +HQYDVR0RBBYwFIYSc3BpZmZlOi8vbG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IB +AQA7nxOrNqGZ4U72qkB9YYXSi89HNgYoz1R0sLdRuh0BDpSPLASNymZrzbw1CuZm +pOJ6b6blxDlLKBx+tDBgYejRmVCZq+hD8mIVBT0Vg3uZhmPOo2URQmUcfsas9UXK +dXGh/9FIqq4u3dBA1bCHlKk/bDIu/VkGMkTaHaDXNEcLBSWLdVkMOuuF6YHgKJh5 +UQEsbWt+kfL3MzeMuAQYVuskKWE19+oLfY41jTQUzPY83r9nJkEZaUyVBShj8CAw +K8QHfKrQ1BE6ALrM1zvMS9zMopoalMtNJ1ILL1nYLD2teVv4iSRGyD7JgHUYYax6 +rnloUNEr2o9DlZp8EvK2I4dU +-----END CERTIFICATE----- diff --git a/test/configs/certs/svid/svid-user-b.key b/test/configs/certs/svid/svid-user-b.key new file mode 100644 index 0000000..ff979e2 --- /dev/null +++ b/test/configs/certs/svid/svid-user-b.key @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgEiQo4GXKbViodiF2 +LltOkXLauMoyKJu01c/FUoGpnXahRANCAASiSiVhimnedxcnXY1ffLWV6Ez9XIkq +3pXxtk6q6jvDfn3OPPjIB47OH4KCqNaMoIsKxwK/mtOEETb0/gFqeQWa +-----END PRIVATE KEY----- diff --git a/test/configs/certs/svid/svid-user-b.pem b/test/configs/certs/svid/svid-user-b.pem new file mode 100644 index 0000000..e628f30 --- /dev/null +++ b/test/configs/certs/svid/svid-user-b.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIICAjCCAaigAwIBAgIQUGwbDXAjCmdvfiGGjS/+PzAKBggqhkjOPQQDAjAwMQsw +CQYDVQQGEwJVUzENMAsGA1UEChMETkFUUzESMBAGA1UEAxMJbG9jYWxob3N0MB4X +DTIwMDUyNzE4MjkxMFoXDTI1MDUwOTIxMDUxMVowHTELMAkGA1UEBhMCVVMxDjAM +BgNVBAoTBVNQSVJFMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEokolYYpp3ncX +J12NX3y1lehM/VyJKt6V8bZOquo7w359zjz4yAeOzh+CgqjWjKCLCscCv5rThBE2 +9P4BankFmqOBtjCBszAOBgNVHQ8BAf8EBAMCA6gwHQYDVR0lBBYwFAYIKwYBBQUH +AwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFOZVW2w2T+3afeJU +JMuZg6Q8FXc/MB8GA1UdIwQYMBaAFJ5SDa5kw3wSKjdsdzG/vMkjNyWHMDQGA1Ud +EQQtMCuGKXNwaWZmZTovL2xvY2FsaG9zdC9teS1uYXRzLXNlcnZpY2UvdXNlci1i +MAoGCCqGSM49BAMCA0gAMEUCIQD81ueLXy2MerMclzKoMnP9VDjOLuHVHf7RkLYb +OdqBigIgH0XT2q5pVmDQgCBP2bKaWZndvXlb5kkPw17XcSD2cKs= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIC3jCCAcagAwIBAgIRAP6sMTwSA5gCsJWO5fSCokUwDQYJKoZIhvcNAQELBQAw +ajELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFDASBgNVBAcMC0xv +cyBBbmdlbGVzMQ0wCwYDVQQKDAROQVRTMQ0wCwYDVQQLDAROQVRTMRIwEAYDVQQD +DAlsb2NhbGhvc3QwHhcNMjAwNTI3MTgxNzAyWhcNMjUwNTA5MjEwNTExWjAwMQsw +CQYDVQQGEwJVUzENMAsGA1UEChMETkFUUzESMBAGA1UEAxMJbG9jYWxob3N0MFkw +EwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJhF2UV33hUpg53uGmy/GkXEI2ZR8EQmp +EHxG1GWbjHR7FBdVP/HmPyVKu5vfegXZp/hD3H7UYHjiNeKMYyGT4qOBgzCBgDAO +BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUnlINrmTD +fBIqN2x3Mb+8ySM3JYcwHwYDVR0jBBgwFoAUlDSkRxR41mV4yHobEDnU1bjsywgw +HQYDVR0RBBYwFIYSc3BpZmZlOi8vbG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IB +AQA7nxOrNqGZ4U72qkB9YYXSi89HNgYoz1R0sLdRuh0BDpSPLASNymZrzbw1CuZm +pOJ6b6blxDlLKBx+tDBgYejRmVCZq+hD8mIVBT0Vg3uZhmPOo2URQmUcfsas9UXK +dXGh/9FIqq4u3dBA1bCHlKk/bDIu/VkGMkTaHaDXNEcLBSWLdVkMOuuF6YHgKJh5 +UQEsbWt+kfL3MzeMuAQYVuskKWE19+oLfY41jTQUzPY83r9nJkEZaUyVBShj8CAw +K8QHfKrQ1BE6ALrM1zvMS9zMopoalMtNJ1ILL1nYLD2teVv4iSRGyD7JgHUYYax6 +rnloUNEr2o9DlZp8EvK2I4dU +-----END CERTIFICATE----- diff --git a/test/configs/certs/tlsauth/ca.pem b/test/configs/certs/tlsauth/ca.pem new file mode 100644 index 0000000..e30e699 --- /dev/null +++ b/test/configs/certs/tlsauth/ca.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDBTCCAe2gAwIBAgIUYm7HlYXiXY8u67v2vKjU5LUCHDUwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTQ4MjRaFw0zNDAxMzEy +MTQ4MjRaMBIxEDAOBgNVBAMMB05BVFMgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC9GEoJ84UWZTsnovuz3StEtri3NdKlOqzNCGvurd3rZgop/jZB +RtJXQz9ZxdKx1ARpDV1m3CrQe63UbBRXJxA2XGfmBQ/BPo3IPEXJOsNEs9x5RsSL +RiJ2re7jbWKeQv/ucQPdmLJumAp+TGAzdOM/AnDaVTSLPoARp/Va8Frs7iFfPpuJ +tObvux4qnb/hxS2z39MWjyeM0dVOmjGwx9opxcE0hNI5ZutkoNxpmRayZqJSe85V +BSPGsuBwgncvA2GWTNIGFfN2oxQhSuI8yM7+l/0+BHFWfm2G7/09tWDvFnWTSpTQ +VISM3+6Wh91c6qSd0wsIb8q6jADAD6H8yhT/AgMBAAGjUzBRMB0GA1UdDgQWBBQk +vMZGyNfVHU3oTUASSfYiT4arNTAfBgNVHSMEGDAWgBQkvMZGyNfVHU3oTUASSfYi +T4arNTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBxN8yaacok +1lyg7eTEv7/Sn4VGkfbuWm9ZQ3DPLKJhbe7uVxNChRrR8nKshwlGv1Wa8i0c0lJ3 +O+Uw24gjfhl/8zhFyxh4ijllj6/FVBNqsTvnGQqtKJP4h8kScUIH21mQ6JQAvh4e +RY1sjPwZp+6vvogSrgQQ32jBaa8vfzcL2wECvnT1YqePVZYuRqEBjIvyG0ALlmE9 +DqZ8gH+W8E5IVulLVJxnYArCT1dW5AyM2fBETLB3PAWvSBkaCBl6QR+hLuyeR4vT +m6Qx9EKr8MgIpiH7psnx8C9eF5j5HiwHfhwAdWD9W2tRzTxSZP2LJ9E+qjaohdLf +6NYxXL8AHa17 +-----END CERTIFICATE----- diff --git a/test/configs/certs/tlsauth/certstore/ca.p12 b/test/configs/certs/tlsauth/certstore/ca.p12 new file mode 100644 index 0000000..d9b67ef Binary files /dev/null and b/test/configs/certs/tlsauth/certstore/ca.p12 differ diff --git a/test/configs/certs/tlsauth/certstore/client.p12 b/test/configs/certs/tlsauth/certstore/client.p12 new file mode 100644 index 0000000..3c2e93a Binary files /dev/null and b/test/configs/certs/tlsauth/certstore/client.p12 differ diff --git a/test/configs/certs/tlsauth/certstore/delete-cert-from-store.ps1 b/test/configs/certs/tlsauth/certstore/delete-cert-from-store.ps1 new file mode 100644 index 0000000..3f2eccd --- /dev/null +++ b/test/configs/certs/tlsauth/certstore/delete-cert-from-store.ps1 @@ -0,0 +1,5 @@ +$issuer="NATS CA" +Get-ChildItem Cert:\CurrentUser\My | Where-Object {$_.Issuer -match $issuer} | Remove-Item +Get-ChildItem Cert:\CurrentUser\CA| Where-Object {$_.Issuer -match $issuer} | Remove-Item +Get-ChildItem Cert:\CurrentUser\AuthRoot | Where-Object {$_.Issuer -match $issuer} | Remove-Item +Get-ChildItem Cert:\CurrentUser\Root | Where-Object {$_.Issuer -match $issuer} | Remove-Item diff --git a/test/configs/certs/tlsauth/certstore/ecdsa_server.key b/test/configs/certs/tlsauth/certstore/ecdsa_server.key new file mode 100644 index 0000000..ec46ada --- /dev/null +++ b/test/configs/certs/tlsauth/certstore/ecdsa_server.key @@ -0,0 +1,6 @@ +-----BEGIN PRIVATE KEY----- +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDAfrq5ri+W7sYQp/6xc +lH6YbTy43dnnrKUbbdlzsDn4DPzO1k15LVXx8EPK+7vuh5uhZANiAAR6V4nqBt3k +ZfO9H664fPB8PkuDhphBfzxbSFFcr2DXj11g0ZV56Yjnh3RMC4Lud29ofpTQd8IP +9bspEvjnBvOw60tH9WiquWqxLgSREUZVLEMD1dZ3JSVUfDCI2zzf00s= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/tlsauth/certstore/ecdsa_server.pem b/test/configs/certs/tlsauth/certstore/ecdsa_server.pem new file mode 100644 index 0000000..8f0be17 --- /dev/null +++ b/test/configs/certs/tlsauth/certstore/ecdsa_server.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIB2jCCAWCgAwIBAgIUKRYMoky98mN3mpyL6PMIY8/d2OswCgYIKoZIzj0EAwMw +FjEUMBIGA1UEAwwLbmF0cy1zZXJ2ZXIwHhcNMjUwMjA3MTgxNDM5WhcNMzUwMjA1 +MTgxNDM5WjAWMRQwEgYDVQQDDAtuYXRzLXNlcnZlcjB2MBAGByqGSM49AgEGBSuB +BAAiA2IABHpXieoG3eRl870frrh88Hw+S4OGmEF/PFtIUVyvYNePXWDRlXnpiOeH +dEwLgu53b2h+lNB3wg/1uykS+OcG87DrS0f1aKq5arEuBJERRlUsQwPV1nclJVR8 +MIjbPN/TS6NvMG0wHQYDVR0OBBYEFHbcfCfGs+l2bVg22WLTdV10AnpTMB8GA1Ud +IwQYMBaAFHbcfCfGs+l2bVg22WLTdV10AnpTMA8GA1UdEwEB/wQFMAMBAf8wGgYD +VR0RBBMwEYcEfwAAAYIJbG9jYWxob3N0MAoGCCqGSM49BAMDA2gAMGUCMQDhzRyw +Q+m2fMFyqIgFc890jLIzh2bGqlmdkUpb+/Z/y9zKZQPSG5xhXp7A/FhvM24CMHVW +ZIWBCJJRhw/L3s73QHX1d+M6mNqES16cnnht6j9DF1AddIipcsnBcpo4s7K/Xg== +-----END CERTIFICATE----- diff --git a/test/configs/certs/tlsauth/certstore/ecdsa_server.pfx b/test/configs/certs/tlsauth/certstore/ecdsa_server.pfx new file mode 100644 index 0000000..2dfb2aa Binary files /dev/null and b/test/configs/certs/tlsauth/certstore/ecdsa_server.pfx differ diff --git a/test/configs/certs/tlsauth/certstore/expired.p12 b/test/configs/certs/tlsauth/certstore/expired.p12 new file mode 100644 index 0000000..5870bbc Binary files /dev/null and b/test/configs/certs/tlsauth/certstore/expired.p12 differ diff --git a/test/configs/certs/tlsauth/certstore/generate_ecdsa_test_cert.sh b/test/configs/certs/tlsauth/certstore/generate_ecdsa_test_cert.sh new file mode 100755 index 0000000..0b5e125 --- /dev/null +++ b/test/configs/certs/tlsauth/certstore/generate_ecdsa_test_cert.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -eou pipefail + +SCRIPT_ROOT="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + +cert_file_prefix="${SCRIPT_ROOT}/ecdsa_server" +export_password="s3cr3t" + +openssl req -x509 \ + -days 3650 \ + -newkey ec \ + -pkeyopt ec_paramgen_curve:secp384r1 \ + -sha384 \ + -subj "/CN=nats-server" \ + --addext "subjectAltName=IP:127.0.0.1,DNS:localhost" \ + -nodes \ + -out "${cert_file_prefix}.pem" \ + -keyout "${cert_file_prefix}.key" \ + -outform PEM >/dev/null 2>&1 + +openssl pkcs12 \ + -inkey "${cert_file_prefix}.key" \ + -in "${cert_file_prefix}.pem" \ + -export \ + -password "pass:${export_password}" \ + -out "${cert_file_prefix}.pfx" >/dev/null 2>&1 diff --git a/test/configs/certs/tlsauth/certstore/import-p12-ca.ps1 b/test/configs/certs/tlsauth/certstore/import-p12-ca.ps1 new file mode 100644 index 0000000..888816f --- /dev/null +++ b/test/configs/certs/tlsauth/certstore/import-p12-ca.ps1 @@ -0,0 +1,7 @@ +$fileLocale = $PSScriptRoot + "\ca.p12" +$Pass = ConvertTo-SecureString -String 's3cr3t' -Force -AsPlainText +$User = "whatever" +$Cred = New-Object -TypeName "System.Management.Automation.PSCredential" -ArgumentList $User, $Pass +Import-PfxCertificate -FilePath $filelocale -CertStoreLocation Cert:\CurrentUser\My -Password $Cred.Password +#Import-PfxCertificate -FilePath $filelocale -CertStoreLocation Cert:\LocalMachine\Root -Password $Cred.Password +# TODO? Move to trusted enterprise? Requires some fingerprint parsing. \ No newline at end of file diff --git a/test/configs/certs/tlsauth/certstore/import-p12-client.ps1 b/test/configs/certs/tlsauth/certstore/import-p12-client.ps1 new file mode 100644 index 0000000..f2b4a27 --- /dev/null +++ b/test/configs/certs/tlsauth/certstore/import-p12-client.ps1 @@ -0,0 +1,5 @@ +$fileLocale = $PSScriptRoot + "\client.p12" +$Pass = ConvertTo-SecureString -String 's3cr3t' -Force -AsPlainText +$User = "whatever" +$Cred = New-Object -TypeName "System.Management.Automation.PSCredential" -ArgumentList $User, $Pass +Import-PfxCertificate -FilePath $filelocale -CertStoreLocation Cert:\CurrentUser\My -Password $Cred.Password \ No newline at end of file diff --git a/test/configs/certs/tlsauth/certstore/import-p12-server.ps1 b/test/configs/certs/tlsauth/certstore/import-p12-server.ps1 new file mode 100644 index 0000000..006a1e4 --- /dev/null +++ b/test/configs/certs/tlsauth/certstore/import-p12-server.ps1 @@ -0,0 +1,8 @@ +$file=$args[0] +if (!$file) { $file="server.p12 "} +$fileLocale = $PSScriptRoot + "\" + $file +echo "Installing certificate $fileLocale" +$Pass = ConvertTo-SecureString -String 's3cr3t' -Force -AsPlainText +$User = "whatever" +$Cred = New-Object -TypeName "System.Management.Automation.PSCredential" -ArgumentList $User, $Pass +Import-PfxCertificate -FilePath $filelocale -CertStoreLocation Cert:\CurrentUser\My -Password $Cred.Password \ No newline at end of file diff --git a/test/configs/certs/tlsauth/certstore/not-expired.p12 b/test/configs/certs/tlsauth/certstore/not-expired.p12 new file mode 100644 index 0000000..d99cb55 Binary files /dev/null and b/test/configs/certs/tlsauth/certstore/not-expired.p12 differ diff --git a/test/configs/certs/tlsauth/certstore/pkcs12.md b/test/configs/certs/tlsauth/certstore/pkcs12.md new file mode 100644 index 0000000..b516ab1 --- /dev/null +++ b/test/configs/certs/tlsauth/certstore/pkcs12.md @@ -0,0 +1,26 @@ +# PKCS12 Files + +Refresh PKCS12 files when test certificates and keys (PEM files) are refreshed (e.g. expiry workflow) + +- `client.p12` is a p12/pfx packaging of `client.pem` and `client-key.pem` + +`openssl pkcs12 -export -inkey ./client-key.pem -in ./client.pem -out client.p12` + +To add the CA, use the following: + +`openssl pkcs12 -export -nokeys -in ..\ca.pem -out ca.p12` + +> Note: set the PKCS12 bundle password to `s3cr3t` as required by provisioning scripts + +## Cert Store Provisioning Scripts + +Windows cert store supports p12/pfx bundle for certificate-with-key import. Windows cert store tests will execute +a Powershell script to import relevant PKCS12 bundle into the Windows store before the test. Equivalent to: + +`powershell.exe -command "& '..\test\configs\certs\tlsauth\certstore\import--p12.ps1'"` + +The `delete-cert-from-store.ps1` script deletes imported certificates from the Windows store (if present) that can +cause side-effects and impact the validity of different use tests. + +> Note: Tests are configured for "current user" store context. Execute tests with appropriate Windows permissions +> (e.g. as Admin) if adding tests with "local machine" store context specified. \ No newline at end of file diff --git a/test/configs/certs/tlsauth/certstore/server.p12 b/test/configs/certs/tlsauth/certstore/server.p12 new file mode 100644 index 0000000..f7a1c4d Binary files /dev/null and b/test/configs/certs/tlsauth/certstore/server.p12 differ diff --git a/test/configs/certs/tlsauth/client-key.pem b/test/configs/certs/tlsauth/client-key.pem new file mode 100644 index 0000000..1727e45 --- /dev/null +++ b/test/configs/certs/tlsauth/client-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCjAaejJ5wqwsst +7T71LQCdsbRpctZU2ZbTIOalEvHNikkdB2gIqxlmswxITFVtNp2IvVhcd4KaNGdu +a4uUc7BifG6XwFtmw54SPTlbFtZbr1y4mjF6+Wm3tV0XnMtM18wvkRm7wbYG5fz8 +BfWl4tbCDF8lnuC3YHFwVJst+o1HoZ4YNutDFJqMXkHuxEPsI5PmS3XHHpttPQGr +vKnoYR0y0Cf4y3LCfQCQ+bFe2JS335WmJy2KVm+dpN+L2ASO8VvV/YTltSy8f4HQ +3Dd/VjZtri7ANP61Xrrx4OJS6IRkPo3uxaR306oKyF9a4irX1KY+Vi1gpYOX9foc +mVumCWCrAgMBAAECggEARk+F7CG/QlCQjEhb0ixtqheHPr7KhYHvhTUZV5XC2Aow +fEWAEdEfnUVY5GyMopWewOcPUJ86JeK5xI69/7QhHnIWz/0oT7zMF4jyDwDcSGLt +RzE3a5heid/Aflli9cvVZqUbaPnm1rXoeBrn+PxN7xigB92uh1qhw7ay0tPSkdUL +sIA2fkouwAYIE5TDhKk4lF8Xg6KlYJEesFDGTVsaGL8/RH7VEH/Z7EssiHF60m4l +Oj6Vg7gxMB1UWQFw2/LDMrDw0bLVdAoOC0M7NLXlFdoeKPREvFpIqw8BWMF3NxRV +OpUiLB/tv6A7nqevaa8KoWGUlCVY2DkCkgCmg+K8pQKBgQDVtb3nEJrOR496BaLn +pHKV+pEQjlVfcy36CoTa2PblbjkgSyA6wL9Itpkw4OF/Q8dop0qlbqZM9gUdjEhB +6DfVINNMGx4SkWDtBvPwouCRIWRvZUXdVuZMgDpUY+5n2nEkUHbmSUEtPbhHn5dp +9Tn1dsjNjweDTx2Q6nDuKUlSVwKBgQDDQ1fyyp6qI6opeFkfOsZ78Yk2lM6tRSkI +JqwdOwZoSptZfiaoAQtmwU3zxioFgIv4FjbPCosJyej0Egv7oyMucZHPGhP6kZDJ +LxZ0Jp0kDYYFCmUjOEpDf0mSmFEwmyPDb8052To3EuA4anEQd1KURG/0yKE4efwP +0m3ml9R3zQKBgAh1cxjMPXRgvLsVsgb9KVPqYQeIurRWeMFm3S9UWyFlpXkzwAjT +TD7yi0m1/Pbuldv8kyXNJWPycO1keg+xw1P6QqLGiAAwJOf82Hbz23OjILiQB53l +LKRmhuiENBGEQeowDSS8TYoe4UZkeLfG7w5aL0SDnsaBwSfVP7cNh0ttAoGAEJDO +DVMTUuvjq9EB/pxF6o37Th4hyqFrcb2WLIStbnul4lnJfcdY6EbODjhpqD3XohyA +WeBTG2l90fcV/StB+Na5wBA+Uau31Nmh1gjQnBZpoFPZcLt90WwjGcTCXpVK23HI +v3emcLWxQBgHr5Xv85Q6y1GaG+h9cfowSLfo1qECgYEAmqGyMa9OZ44+zni3nq2A +JM2t4B5QJdG36R/2m4J7odUnmsaRVxw6JxaoF0CctFxGs/zdumwuhYeTfq6Lj/ly +N1kJbvJJpguN1h4sOOLEk646mZzK2h5bMxsmPU2kNZFf1EUlB+Ro8i1TDvGY26Qs +qgjd6RlSZFBmSfVS47ihEQA= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/tlsauth/client.pem b/test/configs/certs/tlsauth/client.pem new file mode 100644 index 0000000..8c873e0 --- /dev/null +++ b/test/configs/certs/tlsauth/client.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDPjCCAiagAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipYwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTQ5MzRaFw0zNDAxMzEy +MTQ5MzRaMCgxEDAOBgNVBAsMB05BVFMuaW8xFDASBgNVBAMMC2V4YW1wbGUuY29t +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAowGnoyecKsLLLe0+9S0A +nbG0aXLWVNmW0yDmpRLxzYpJHQdoCKsZZrMMSExVbTadiL1YXHeCmjRnbmuLlHOw +Ynxul8BbZsOeEj05WxbWW69cuJoxevlpt7VdF5zLTNfML5EZu8G2BuX8/AX1peLW +wgxfJZ7gt2BxcFSbLfqNR6GeGDbrQxSajF5B7sRD7COT5kt1xx6bbT0Bq7yp6GEd +MtAn+Mtywn0AkPmxXtiUt9+VpictilZvnaTfi9gEjvFb1f2E5bUsvH+B0Nw3f1Y2 +ba4uwDT+tV668eDiUuiEZD6N7sWkd9OqCshfWuIq19SmPlYtYKWDl/X6HJlbpglg +qwIDAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93 +d3cuZXhhbXBsZS5jb20wHQYDVR0OBBYEFA5GJsoT3uf5GCyL5KSFoyEuRVltMB8G +A1UdIwQYMBaAFCS8xkbI19UdTehNQBJJ9iJPhqs1MA0GCSqGSIb3DQEBCwUAA4IB +AQAmBOQOp3CR5/P27RUonsgf5MCzblt6tBPJBMiZwB8sJyl9fXCTkzdqD0rJQb7q +5s7p00rUXdeh13oRjFcoFuopMDCk3kBYnKJHXRREJNLHfp6CPUMKlg0GJUZ6v04A +V7gVuhvmynHrmlbibMwbgZtZMnRU3x8JjawEUsEhoj3O2Qfen3sNfaOBlnwVUCBQ +ygSHQ0Pto1kQS+1Pc5DCwnOZ/qh5lORPdO1MNKqeu8HiiSJfuaCrQQM9zm72CHHY +F755qy8OvWjwK0H9rCFBYSrAnYk/pTvXIeBsgNRlURS/qv1rqIEvAbXhRnw7oyvl +P4bYY4pcpk32Ir2mFQFRQnSh +-----END CERTIFICATE----- diff --git a/test/configs/certs/tlsauth/client2-key.pem b/test/configs/certs/tlsauth/client2-key.pem new file mode 100644 index 0000000..6ae5fe0 --- /dev/null +++ b/test/configs/certs/tlsauth/client2-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDMsHsIYdxGt8ws +eoxGQQs4K+AfvznwuW+Qbnx+eGTS+OSaLqL9u81JdXxheeAVsVjRtz5qP3X9Vu75 +gSQU+AVhFp689VNR41vaOsKvX2dzVCaWCEwv66CekAop89vyB6fm14uSr9IusixH +u6vfjuze2FZE5xcw5mSYMysqKTbM4DMjXFJXFg5mw1YC9zkh3zCWUZdmidafwOB6 +MiJkVZsAQjAwc5oaD4I/ubCdmWr8COYbbwUl32k+t0eQfj9cvd4+R7uBYVNWu1BN +g0RqNZZlIM4EVonc1J+eN7oGo57jLrwf4jCOkcg2wd6X62CuuQHhwddQek4KGthM +JUIH7Q9TAgMBAAECggEAE80u2cy9xomZUuQ4FcPNFg4IjImvTT5jMJG/sWxsNIyn +cNL6KZm1blnTQorLxs11TjRv8U9aVrvGOpTnrK+htZa+nIEPImjgRehRVS3hkCKf +6Pu8gxZEX5KHqS9SI8Ph1k8bzYD80E+kQPxC0Em/WH+NOPUyJSTkrmSk1FtQVdle +KFk0+dRm10guI/rcInPUjpY6Cv9nSiAhanRPAtAANUvwYxVEMKzx5ns/JnQfAYAK +gYTopXAT61VqW1jn4opjD+OqparjbK362q/spggHGPhSamXq8y/N9JadSx+5591u +o0oZm/UxefwP1CkE4Q6Pv9n3yiZOwVaOiLLVzmr6wQKBgQD4kC953if336Ga+BdB +Hj83qUk5U2Avp6qWLptd6z/VQXDwlL2oQaaXP83T6XfpXTMwDabUYkpo1JZrFOJl +s9eZfxIDa0eH6evphqp6s/V6EhZkI9lL59ptZlK1hv1glmFDfn7WB1Yo7d8GgUx/ +S7CKoBK2WaX8YNsW9fIunadJeQKBgQDS0EDR46KiQBrjvNK4UxdfERl2af+6Sf+J +BRlfj+YA9/zKEL3jZGWVa5rgrD8BJrTsa86QSSACsxQx5J9vPogKgogmghd9RQVd +tHvIV0C2C6zCmZIvNuzn5+wMV7bWdoABPgbuZGTpc/LCwm//1EbwdNCTom8ssE19 +NTIBb7t4KwKBgQDXOqKheAhLzkz1D1WzgSlkXSWWieeD3D8OBBVsYcPIOP4+k80V +4KML3KexkzvNynIEbg3DYcjktQ/6cP8I6Y0K0MkcRMyPl7I7Z+w+i41Hwlm5JIGI +BJ9Sk4OSw+yqsgxOkT3qvjeRAUhZLaS7pSKdJraNR1s/Ce8sFpM6YjD0oQKBgAUc +gYXVRBs0/LHq4R0Q/q8SZhCl70pgAu8ajYvwnD4HxTxM/Z2m0IO38TBjXL+1ZYuZ +7Y84BquqFeJDzc3PsVK36X8thk5GPyQPfTTVUL9ZNx4cxRuZ9FKHIAUIl2lJxD7D +dz2Od5fldMxeFIMabYHlAy2hMZrex3IyuPyp7dyzAoGAKCqmXStl7QzV6ovrjyJ1 +oMP3BcQhTe1CvOEW7pGce/IGxqB0p7gICIH4UUDEVpitgvGLkOGScsd1rS86Fn+/ +RUlQQdgRIwzrlF1USV1kJ/SUI+7BbhiXUeRUmKuzPLgLSq8K3xEW0z21juSvq64W +gtiiGjt3D3I47v7ecjJ9dIA= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/tlsauth/client2.pem b/test/configs/certs/tlsauth/client2.pem new file mode 100644 index 0000000..edc6651 --- /dev/null +++ b/test/configs/certs/tlsauth/client2.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDOzCCAiOgAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipcwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTUwMzlaFw0zNDAxMzEy +MTUwMzlaMCUxDTALBgNVBAsMBENOQ0YxFDASBgNVBAMMC2V4YW1wbGUuY29tMIIB +IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzLB7CGHcRrfMLHqMRkELOCvg +H7858LlvkG58fnhk0vjkmi6i/bvNSXV8YXngFbFY0bc+aj91/Vbu+YEkFPgFYRae +vPVTUeNb2jrCr19nc1QmlghML+ugnpAKKfPb8gen5teLkq/SLrIsR7ur347s3thW +ROcXMOZkmDMrKik2zOAzI1xSVxYOZsNWAvc5Id8wllGXZonWn8DgejIiZFWbAEIw +MHOaGg+CP7mwnZlq/AjmG28FJd9pPrdHkH4/XL3ePke7gWFTVrtQTYNEajWWZSDO +BFaJ3NSfnje6BqOe4y68H+IwjpHINsHel+tgrrkB4cHXUHpOChrYTCVCB+0PUwID +AQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93d3cu +ZXhhbXBsZS5jb20wHQYDVR0OBBYEFEbKWjY1gwvxJusV+M5wUn+7MKR5MB8GA1Ud +IwQYMBaAFCS8xkbI19UdTehNQBJJ9iJPhqs1MA0GCSqGSIb3DQEBCwUAA4IBAQAo +873zVMG6tfoPRUZ/kEJcPEIaLmaolALyLEx3sZHe/B8hszuuMecBEfa02HwTSlzq +fKrkME95LGE9D+4hxyPEqPTqruESShUvBFQIoTQxePAhhUG9icF4gqUpYvRHXMiR +xIyfH3/KojDlBXRfDOaoXEXshiXfcYqbeh2qFdaoN24Vyh6lkNa2K3SUDAtKVFiF +jjBtNXuH/IJ3EWbs5AOOy98QtBMlT7kmummJVeaRR4QUfnzFrlj5nMSIopoxDm4N +QeoQSC+63fce6ZJLGQqEFQNR6howBcDQ/8fMR/oLsJ1Hr9VshIsu3kGTk4RUqHI9 +ipGt1UTvVf/uMUzA3yoC +-----END CERTIFICATE----- diff --git a/test/configs/certs/tlsauth/server-key.pem b/test/configs/certs/tlsauth/server-key.pem new file mode 100644 index 0000000..a9077a6 --- /dev/null +++ b/test/configs/certs/tlsauth/server-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDAgrLiRyYcP54+ +AlElXoMjhOMg+VZjkVnC4CVXk+3jmKqAzI6bum9QHBnSdN/ULaPFU4Vgoqt1Puv8 +J9snNlSSE4CcgCCmWihFIwSpIaXW/GWCZVvCQDn3QxAZt48vTtEI27tMsRTCk/tV +kuShJ7OxrJyXcntBWbS+TeSOMVIF/v6lqWGjaDauPK0cpesa/qLElaHNlIV0+pM+ +5b3LIcEpuaECVwu5n5c9m/qdX7ZyPF2x8Z2I0zO5nLyFxb3WolPjok/qnZbhC/Gp +OX459F6yCVBmjkakaJocV1Ue7V0dFB7u3aYW0dwa+7Zb0613ZHMWKfAOzJKOiZC5 +xYXBxO4DAgMBAAECggEAA+ILzXOfVneHPHuO4acuRvoIG/yoT1OgHcWoPl16ywBX +BBUPYh9Yqm/lEQrDDwRCpuX54HEEVO+WRY/gpEHv/I6z8k3FTgWu66h6qhZGW/eo +U02yRzdFBz9AV/xzWpvUEvUq3Uk/GkBvRxKyiN7RPMODpXktXxKTo9Kg/XTYnjIb +2RWzKdoeptX6q/RRdPkBXxYV5XIaZLXPy+VaGgI7QcwApmCcin20c5smtLmGE7TV +4hh6wxPwGEiDkfMQGnqWd+VdzbnZjZF2/4ZVCNAbJJaFffTxnoC0uGjE3ImAztmT +TgmFbdfO/5qHTkJmfZaAbmo461k1O2Vd/p6LCN1mhQKBgQDokSJxOMuYurf1STAb +om6E5lGXAYR/2TsBxzfWPvUlZDzaSZ7IU+/zpv9GTATc0mvdIpnT2kZvRoz7nbbR +LFGjc2d1ITRBuDEX+qDkX1puFyhoYPyaz0AuC3AMqg28e3Z1I8nq3Op1TUKAMr9d +ZdOfRUcL65OM+YHPZ197AVwC9wKBgQDT6FgGceqBr3XJDikjU+kJfyrZrcE7sfrE +3RIV6uMj6Myw4WpUpuH7hwqQ0xAir8UKHG6SDScymXtrJCoG1xJmGGDyoLt80VsF +fXVhzL8rhIl48TVWptRUiIeZ2dXvianLkLPlLhVUcTThpjjAAbnNYhod1bHygX12 +w+Bd8lkeVQKBgBYLS7x3qbS8Xht96HV2HAu02R77IdgMey9b9sr0BMCak7oNKGPM +sP3jYmcDZaKYv2iikvolwm9hvJNNC7sf/E0F71SG5TEliGHBe+apsySkRUw/hTIX +WvoCU4ifxdWLzlqkHcuJTR/5RshoBwOPV1PNeUKD/eRq8gb6wW4jXtlZAoGBAK2o +C3MEqcQrUSA53ZaY7jGdKDWJQgC0oyfvbyHNAuVro0sU/3lt5WWmTg9PGDsExjm6 +ARbpdoTt6Ilt8o72c5p9Qf2zoNHyE2CVZruF+egkzi/xo99mCj1YQZ/gN4T80MwE +wpf+wvYXa9m7yWf4QhbA3VwzwodUfMf2T4lN0KCdAoGBAM04ecwBXCNIXzTgJDeK +zxfI95lphcNywdhUNw6KJPDg4ozPu72XrVToAtl7l6+2RPygNhSj117002AAgz30 +l0cfZoogXmIhZt+R39/2e9y31k+GDdEmNtPLsvbLPQ+Ttnab87pUABn4UC1XXGhF +Vt/jfLEw8ZDjB+aO4kFyZPIQ +-----END PRIVATE KEY----- diff --git a/test/configs/certs/tlsauth/server-no-ou-key.pem b/test/configs/certs/tlsauth/server-no-ou-key.pem new file mode 100644 index 0000000..ea0dd78 --- /dev/null +++ b/test/configs/certs/tlsauth/server-no-ou-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDepPh4tWYw66pP +GCEgpYzBTW36/s7cRnsuqhkiUolc5RB//JYaRz/sbIlY7tM8Z21nmjU0IXvlKPUt ++6Winv7VqDU7S1gOssnIPbVhXGCb4fC3JGjQ2qD/pO5WfoUh8yYph9tFg07+zMyQ +XgJiE8JglKCeD7kjna0/lkk63ddQP0npbH5943oMvHDj7UjJyBdE8nWX1LXWAmYJ +Wxym6MDfjWTVnyzwqm8fkl/BHp9fh98Nf9U6/gpq0ckRUHL4prhMw0ChhBVjkBQ8 +CrXrp82TQZbeFPm3uWTV3xf5bnbeO87dgiXP47yvqVFSQI2cFyo2JzEq6LVAeVPT +dKzENXyFAgMBAAECggEAHlLdvKMIPhV65rbknCuwFgvTtOHLjtjSojJspe4T42EX +dDcUwpN9s1e9BS3R+2Ii1n98S5Nb6oQ/kHm7v4BkOPll9qN2ZNoY/XraH16Tkeed +/3OoCvob/3WZOJKW017ojbOBO+B8e9us6OTE8lK6oKjdj2mYz68ED6sKYkggsT8M +ah4KkzrX8xNiQwMRs0VMfXb1M8DUd+nWcamWdUytf3ZPFWxymzYzjjxDuisK4faV +Fuqwk2FRtOuyMeygC7KuPwjBtqXuFuHLYtOlibs5ZFG9Nwd9lWQEaVCnOlgW35zz +CVkdszfCgimCL/K42vcmbdnI3GUO/g3szw9Vy95sAQKBgQDx/PHZks4FdxXSkuxZ +u3Qjh5d/sG9NMK0QUNgF/MnORQjvmVfQOSXObiMxoXBBNW25St00mO0W+ZzXee5z +3ANw6cPgSXla29RKC4yVuAXwOn/kPnuASQxdoGWEKuZJqUKbpK+FDgFbK9cZlSd/ +VNS9eW2j00CCqXK0FQ7FvfCoBQKBgQDriUonm0EwCoNrBlbllOynFYjFwg2yu/ti +H/1SAdmTcD+NZFRwhBmSKxni2rYyMoyqkHN93t8Sz8ELIK4sEsA/XUpEioeGIwMG +HQSRga2XYUVafuI/TQRHeMke703rtbIN7dkjEiWsM1gHoPcTRxZU9hRGq0p89GHq +zOf4tuAqgQKBgA3ceVYHLLnvalaXh+ZT8IEggTMVPirjwOYQW29sXXrtRWfEFt2c +iGfcszNilfWGQ/S7LxSWNe58+dj16QzF64SKP2gXjVYBBZYAN1tUITLzhuPiGFzu +0kCCsY3yjyJlCaW0t0Ed3kIErtuOSabnixAXZopdzXIulp1uH1yAVsqpAoGBAL+7 +ua62FoGp/ULRHUm0SlTVFcqN5iK6Ha/KBKeOM/Ruan2Jz6bsEfjHt0HQ8oG4XoO2 +JR2woHyqvCV3y/C6rt6l7YAQGLRbqel/E6nzG0FggFljcn8/DZ20uFvDR/X5qWDn +XlvLOPmNrjo/kQGTW5172BOS+obvVQoTFT6Ed8SBAoGBAI5802KpwzgN9IjPM1Zf +rBZ6PshZ14PFxeQ2tOhvFcr+N+ceSY19lex+UyQn7F214Kz2U3n35zgNdLhdHLK6 +WokXp21WDqeg8n1cUrQ43yWqVX8cjdNx3P84B5Mdnb0nhbM2smUSuhrO+fOzr37A +JegHloScSH3BcemRFfK31+ba +-----END PRIVATE KEY----- diff --git a/test/configs/certs/tlsauth/server-no-ou.pem b/test/configs/certs/tlsauth/server-no-ou.pem new file mode 100644 index 0000000..2ed37dc --- /dev/null +++ b/test/configs/certs/tlsauth/server-no-ou.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDKjCCAhKgAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipUwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTQ4MzJaFw0zNDAxMzEy +MTQ4MzJaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBAN6k+Hi1ZjDrqk8YISCljMFNbfr+ztxGey6qGSJSiVzlEH/8 +lhpHP+xsiVju0zxnbWeaNTQhe+Uo9S37paKe/tWoNTtLWA6yycg9tWFcYJvh8Lck +aNDaoP+k7lZ+hSHzJimH20WDTv7MzJBeAmITwmCUoJ4PuSOdrT+WSTrd11A/Sels +fn3jegy8cOPtSMnIF0TydZfUtdYCZglbHKbowN+NZNWfLPCqbx+SX8Een1+H3w1/ +1Tr+CmrRyRFQcvimuEzDQKGEFWOQFDwKteunzZNBlt4U+be5ZNXfF/ludt47zt2C +Jc/jvK+pUVJAjZwXKjYnMSrotUB5U9N0rMQ1fIUCAwEAAaN2MHQwMgYDVR0RBCsw +KYIJbG9jYWxob3N0ggtleGFtcGxlLmNvbYIPd3d3LmV4YW1wbGUuY29tMB0GA1Ud +DgQWBBS/WuyC1fvhLL8rnKIXWFdQFnGpozAfBgNVHSMEGDAWgBQkvMZGyNfVHU3o +TUASSfYiT4arNTANBgkqhkiG9w0BAQsFAAOCAQEAVgdcRJgo95MEDgEACekziOHn +n86DdgNin4FDkL7Y2sBDrpej4B0jKPm2H/M4qdB2Z17Ru2TdcXOyk/sMc395GHsN +BAdKuAcysvQ+USR3UXasJmC/CvoKGBOmFf9/Jor8U4Rs01bkXSd6pW8ytT3kyMak +3r5tNugzRxpJvVDgjHlUkfhBoLeeCr+k1cN1OvR4cFhY6vxqS6GBdopFGC3DlnTL +LPetNhQCd+r2mH1RT/56aLLRawy76GkBEZm/+mg+mYjxN3J1hWibouF4ccutvxtt +h2/4PJNsXv5yt4wibazFixJ843KPdfw6pafXbYZsvvgNfvLrpp8beCUwHEYq+w== +-----END CERTIFICATE----- diff --git a/test/configs/certs/tlsauth/server.pem b/test/configs/certs/tlsauth/server.pem new file mode 100644 index 0000000..0e9f92a --- /dev/null +++ b/test/configs/certs/tlsauth/server.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDRjCCAi6gAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipgwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMjA0NTdaFw0zNDAxMzEy +MjA0NTdaMDAxGjAYBgNVBAsMEU5BVFMuaW8gT3BlcmF0b3JzMRIwEAYDVQQDDAls +b2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDAgrLiRyYc +P54+AlElXoMjhOMg+VZjkVnC4CVXk+3jmKqAzI6bum9QHBnSdN/ULaPFU4Vgoqt1 +Puv8J9snNlSSE4CcgCCmWihFIwSpIaXW/GWCZVvCQDn3QxAZt48vTtEI27tMsRTC +k/tVkuShJ7OxrJyXcntBWbS+TeSOMVIF/v6lqWGjaDauPK0cpesa/qLElaHNlIV0 ++pM+5b3LIcEpuaECVwu5n5c9m/qdX7ZyPF2x8Z2I0zO5nLyFxb3WolPjok/qnZbh +C/GpOX459F6yCVBmjkakaJocV1Ue7V0dFB7u3aYW0dwa+7Zb0613ZHMWKfAOzJKO +iZC5xYXBxO4DAgMBAAGjdjB0MDIGA1UdEQQrMCmCCWxvY2FsaG9zdIILZXhhbXBs +ZS5jb22CD3d3dy5leGFtcGxlLmNvbTAdBgNVHQ4EFgQUcpTmScu/KZHhs0KbdxtN +GXndXBEwHwYDVR0jBBgwFoAUJLzGRsjX1R1N6E1AEkn2Ik+GqzUwDQYJKoZIhvcN +AQELBQADggEBAKdqG82z3JBim/hiGf4LZT90ZHhw7ngPT9HUV4jYRIk0ngJ37ogK +KCYW0UBCkugdf0elxcggjAsJZGlz+hW2j8MynEqJ9UU7jPPp4AKJqZHy5x49Y1iL +kFlJE5a3LFJUaVG4JeYMqTL2zDtoj+hk7QPPoz88moDUbOHg3HccObHlISelVPON +K/kvnJ2NfXImYkh7MusRxVuB4LcRRi5rwT0pOdtSPBCeSH96BOeCHTriPHGecgc4 +71tgSaELXPM1YnaM2WmXoGU1MZ7Dx6c2q97FI+SWgKfm7B1GQGyAghgKxlRyhfNj +UvCrbaZDInrMWpMo3+upIBWpHzfmJVvUcYI= +-----END CERTIFICATE----- diff --git a/test/configs/jetstream/restore_bad_stream/backup.json b/test/configs/jetstream/restore_bad_stream/backup.json new file mode 100644 index 0000000..86b7391 --- /dev/null +++ b/test/configs/jetstream/restore_bad_stream/backup.json @@ -0,0 +1,33 @@ +{ + "config": { + "name": "TEST", + "subjects": [ + "foo" + ], + "retention": "limits", + "max_consumers": -1, + "max_msgs_per_subject": -1, + "max_msgs": -1, + "max_bytes": -1, + "max_age": 0, + "max_msg_size": -1, + "storage": "file", + "discard": "old", + "num_replicas": 1, + "duplicate_window": 120000000000, + "sealed": false, + "deny_delete": false, + "deny_purge": false, + "allow_rollup_hdrs": false + }, + "state": { + "messages": 10, + "bytes": 381, + "first_seq": 1, + "first_ts": "2022-03-07T23:59:01.710801Z", + "last_seq": 10, + "last_ts": "2022-03-07T23:59:01.712378Z", + "num_subjects": 1, + "consumer_count": 1 + } +} \ No newline at end of file diff --git a/test/configs/jetstream/restore_bad_stream/stream.tar.s2 b/test/configs/jetstream/restore_bad_stream/stream.tar.s2 new file mode 100755 index 0000000..f83c8d7 Binary files /dev/null and b/test/configs/jetstream/restore_bad_stream/stream.tar.s2 differ diff --git a/test/configs/jetstream/restore_empty_R1F_stream/backup.json b/test/configs/jetstream/restore_empty_R1F_stream/backup.json new file mode 100644 index 0000000..ebc48e7 --- /dev/null +++ b/test/configs/jetstream/restore_empty_R1F_stream/backup.json @@ -0,0 +1,35 @@ +{ + "config": { + "name": "STREAM", + "subjects": [ + "stream" + ], + "retention": "limits", + "max_consumers": -1, + "max_msgs_per_subject": -1, + "max_msgs": -1, + "max_bytes": -1, + "max_age": 0, + "max_msg_size": -1, + "storage": "file", + "discard": "old", + "num_replicas": 1, + "duplicate_window": 120000000000, + "sealed": false, + "deny_delete": false, + "deny_purge": false, + "allow_rollup_hdrs": false, + "allow_direct": true, + "mirror_direct": false, + "consumer_limits": {} + }, + "state": { + "messages": 0, + "bytes": 0, + "first_seq": 0, + "first_ts": "0001-01-01T00:00:00Z", + "last_seq": 0, + "last_ts": "0001-01-01T00:00:00Z", + "consumer_count": 0 + } +} \ No newline at end of file diff --git a/test/configs/jetstream/restore_empty_R1F_stream/stream.tar.s2 b/test/configs/jetstream/restore_empty_R1F_stream/stream.tar.s2 new file mode 100644 index 0000000..a0e9234 Binary files /dev/null and b/test/configs/jetstream/restore_empty_R1F_stream/stream.tar.s2 differ diff --git a/test/configs/jetstream/restore_empty_R3F_stream/backup.json b/test/configs/jetstream/restore_empty_R3F_stream/backup.json new file mode 100644 index 0000000..f9bdb2d --- /dev/null +++ b/test/configs/jetstream/restore_empty_R3F_stream/backup.json @@ -0,0 +1,35 @@ +{ + "config": { + "name": "STREAM", + "subjects": [ + "stream" + ], + "retention": "limits", + "max_consumers": -1, + "max_msgs_per_subject": -1, + "max_msgs": -1, + "max_bytes": -1, + "max_age": 0, + "max_msg_size": -1, + "storage": "file", + "discard": "old", + "num_replicas": 3, + "duplicate_window": 120000000000, + "sealed": false, + "deny_delete": false, + "deny_purge": false, + "allow_rollup_hdrs": false, + "allow_direct": true, + "mirror_direct": false, + "consumer_limits": {} + }, + "state": { + "messages": 0, + "bytes": 0, + "first_seq": 0, + "first_ts": "0001-01-01T00:00:00Z", + "last_seq": 0, + "last_ts": "0001-01-01T00:00:00Z", + "consumer_count": 0 + } +} \ No newline at end of file diff --git a/test/configs/jetstream/restore_empty_R3F_stream/stream.tar.s2 b/test/configs/jetstream/restore_empty_R3F_stream/stream.tar.s2 new file mode 100644 index 0000000..2775b74 Binary files /dev/null and b/test/configs/jetstream/restore_empty_R3F_stream/stream.tar.s2 differ diff --git a/test/configs/nkeys/op.jwt b/test/configs/nkeys/op.jwt new file mode 100644 index 0000000..9335689 --- /dev/null +++ b/test/configs/nkeys/op.jwt @@ -0,0 +1,3 @@ +-----BEGIN TEST OPERATOR JWT----- +eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJhdWQiOiJURVNUUyIsImV4cCI6MTg1OTEyMTI3NSwianRpIjoiWE5MWjZYWVBIVE1ESlFSTlFPSFVPSlFHV0NVN01JNVc1SlhDWk5YQllVS0VRVzY3STI1USIsImlhdCI6MTU0Mzc2MTI3NSwiaXNzIjoiT0NBVDMzTVRWVTJWVU9JTUdOR1VOWEo2NkFIMlJMU0RBRjNNVUJDWUFZNVFNSUw2NU5RTTZYUUciLCJuYW1lIjoiU3luYWRpYSBDb21tdW5pY2F0aW9ucyBJbmMuIiwibmJmIjoxNTQzNzYxMjc1LCJzdWIiOiJPQ0FUMzNNVFZVMlZVT0lNR05HVU5YSjY2QUgyUkxTREFGM01VQkNZQVk1UU1JTDY1TlFNNlhRRyIsInR5cGUiOiJvcGVyYXRvciIsIm5hdHMiOnsic2lnbmluZ19rZXlzIjpbIk9EU0tSN01ZRlFaNU1NQUo2RlBNRUVUQ1RFM1JJSE9GTFRZUEpSTUFWVk40T0xWMllZQU1IQ0FDIiwiT0RTS0FDU1JCV1A1MzdEWkRSVko2NTdKT0lHT1BPUTZLRzdUNEhONk9LNEY2SUVDR1hEQUhOUDIiLCJPRFNLSTM2TFpCNDRPWTVJVkNSNlA1MkZaSlpZTVlXWlZXTlVEVExFWjVUSzJQTjNPRU1SVEFCUiJdfX0.hyfz6E39BMUh0GLzovFfk3wT4OfualftjdJ_eYkLfPvu5tZubYQ_Pn9oFYGCV_6yKy3KMGhWGUCyCdHaPhalBw +------END TEST OPERATOR JWT------ \ No newline at end of file diff --git a/test/configs/nkeys/sigkeys.txt b/test/configs/nkeys/sigkeys.txt new file mode 100644 index 0000000..590aadc --- /dev/null +++ b/test/configs/nkeys/sigkeys.txt @@ -0,0 +1,20 @@ + +######################################################## +# TESTS ONLY # +######################################################## + +# These are the public signing keys + +-----BEGIN SIGNING KEYS----- +ODSKR7MYFQZ5MMAJ6FPMEETCTE3RIHOFLTYPJRMAVVN4OLV2YYAMHCAC +ODSKACSRBWP537DZDRVJ657JOIGOPOQ6KG7T4HN6OK4F6IECGXDAHNP2 +ODSKI36LZB44OY5IVCR6P52FZJZYMYWZVWNUDTLEZ5TK2PN3OEMRTABR +------END SIGNING KEYS------ + +# These are the seeds. + +----BEGIN SIGNING SEEDS----- +SOAO7RDW6CLJORHHBS4DPYYIIIAASEIUJ5WWS5FMWLNTFHUCKQ5CAC45AA +SOAEL3NFOTU6YK3DBTEKQYZ2C5IWSVZWWZCQDASBUOHJKBFLVANK27JMMQ +SOACSMP662P2BZDKVF6WCB6FIQYORADDWWWEAI55QY24CQRTY4METUING4 +------END SIGING SEEDS------ diff --git a/test/configs/nkeys/test.seed b/test/configs/nkeys/test.seed new file mode 100644 index 0000000..8a60e9c --- /dev/null +++ b/test/configs/nkeys/test.seed @@ -0,0 +1,7 @@ +######################################################## +# TESTS ONLY # +######################################################## + +-----BEGIN TEST OPERATOR SEED----- +SOAFYNORQLQFJYBYNUGC5D7SH2MXMUX5BFEWWGHN3EK4VGG5TPT5DZP7QU +------END TEST OPERATOR SEED------ diff --git a/test/fanout_test.go b/test/fanout_test.go new file mode 100644 index 0000000..a94a832 --- /dev/null +++ b/test/fanout_test.go @@ -0,0 +1,129 @@ +// Copyright 2018-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !race && !skipnoracetests + +package test + +import ( + "fmt" + "strconv" + "sync" + "testing" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" +) + +// IMPORTANT: Tests in this file are not executed when running with the -race flag. +// The test name should be prefixed with TestNoRace so we can run only +// those tests: go test -run=TestNoRace ... + +// As we look to improve high fanout situations make sure we +// have a test that checks ordering for all subscriptions from a single subscriber. +func TestNoRaceHighFanoutOrdering(t *testing.T) { + opts := &server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT} + + s := RunServer(opts) + defer s.Shutdown() + + url := fmt.Sprintf("nats://%s", s.Addr()) + + const ( + nconns = 100 + nsubs = 100 + npubs = 500 + ) + + // make unique + subj := nats.NewInbox() + + var wg sync.WaitGroup + wg.Add(nconns * nsubs) + + for i := 0; i < nconns; i++ { + nc, err := nats.Connect(url) + if err != nil { + t.Fatalf("Expected a successful connect on %d, got %v\n", i, err) + } + + nc.SetErrorHandler(func(c *nats.Conn, s *nats.Subscription, e error) { + t.Fatalf("Got an error %v for %+v\n", s, err) + }) + + for y := 0; y < nsubs; y++ { + expected := 0 + nc.Subscribe(subj, func(msg *nats.Msg) { + n, _ := strconv.Atoi(string(msg.Data)) + if n != expected { + t.Fatalf("Expected %d but received %d\n", expected, n) + } + expected++ + if expected >= npubs { + wg.Done() + } + }) + } + nc.Flush() + defer nc.Close() + } + + nc, _ := nats.Connect(url) + + for i := 0; i < npubs; i++ { + nc.Publish(subj, []byte(strconv.Itoa(i))) + } + defer nc.Close() + + wg.Wait() +} + +func TestNoRaceRouteFormTimeWithHighSubscriptions(t *testing.T) { + srvA, optsA := RunServerWithConfig("./configs/srv_a.conf") + defer srvA.Shutdown() + + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConn(t, clientA) + + // Now add lots of subscriptions. These will need to be forwarded + // to new routes when they are added. + subsTotal := 100000 + for i := 0; i < subsTotal; i++ { + subject := fmt.Sprintf("FOO.BAR.BAZ.%d", i) + sendA(fmt.Sprintf("SUB %s %d\r\n", subject, i)) + } + sendA("PING\r\n") + expectA(pongRe) + + srvB, _ := RunServerWithConfig("./configs/srv_b.conf") + defer srvB.Shutdown() + + checkClusterFormed(t, srvA, srvB) + + // Now wait for all subscriptions to be processed. + if err := checkExpectedSubs(subsTotal, srvB); err != nil { + // Make sure we are not a slow consumer + // Check for slow consumer status + if srvA.NumSlowConsumers() > 0 { + t.Fatal("Did not receive all subscriptions due to slow consumer") + } else { + t.Fatalf("%v", err) + } + } + // Just double check the slow consumer status. + if srvA.NumSlowConsumers() > 0 { + t.Fatalf("Received a slow consumer notification: %d", srvA.NumSlowConsumers()) + } +} diff --git a/test/gateway_test.go b/test/gateway_test.go new file mode 100644 index 0000000..fcca866 --- /dev/null +++ b/test/gateway_test.go @@ -0,0 +1,839 @@ +// Copyright 2018-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "bufio" + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "net" + "net/url" + "regexp" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" +) + +func testDefaultOptionsForGateway(name string) *server.Options { + o := DefaultTestOptions + o.Port = -1 + o.Cluster.Name = name + o.Gateway.Name = name + o.Gateway.Host = "127.0.0.1" + o.Gateway.Port = -1 + return &o +} + +func runGatewayServer(o *server.Options) *server.Server { + s := RunServer(o) + return s +} + +func createGatewayConn(t testing.TB, host string, port int) net.Conn { + t.Helper() + return createClientConn(t, host, port) +} + +func setupGatewayConn(t testing.TB, c net.Conn, org, dst string) (sendFun, expectFun) { + t.Helper() + dstInfo := checkInfoMsg(t, c) + if dstInfo.Gateway != dst { + t.Fatalf("Expected to connect to %q, got %q", dst, dstInfo.Gateway) + } + cs := fmt.Sprintf("CONNECT {\"verbose\":%v,\"pedantic\":%v,\"tls_required\":%v,\"gateway\":%q}\r\n", + false, false, false, org) + sendProto(t, c, cs) + sendProto(t, c, fmt.Sprintf("INFO {\"gateway\":%q}\r\n", org)) + return sendCommand(t, c), expectCommand(t, c) +} + +func expectNumberOfProtos(t *testing.T, expFn expectFun, proto *regexp.Regexp, expected int, ignore ...*regexp.Regexp) { + t.Helper() + buf := []byte(nil) + for count := 0; count != expected; { + buf = append(buf, expFn(anyRe)...) + for _, skip := range ignore { + buf = skip.ReplaceAll(buf, []byte(``)) + } + count += len(proto.FindAllSubmatch(buf, -1)) + if count > expected { + t.Fatalf("Expected %v matches, got %v", expected, count) + } + buf = proto.ReplaceAll(buf, []byte(``)) + } + if len(buf) != 0 { + t.Fatalf("did not consume everything, left with: %q", buf) + } +} + +func TestGatewayAccountInterest(t *testing.T) { + server.GatewayDoNotForceInterestOnlyMode(true) + defer server.GatewayDoNotForceInterestOnlyMode(false) + + ob := testDefaultOptionsForGateway("B") + sb := runGatewayServer(ob) + defer sb.Shutdown() + + gA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) + defer gA.Close() + + gASend, gAExpect := setupGatewayConn(t, gA, "A", "B") + gASend("PING\r\n") + gAExpect(pongRe) + + // Sending a bunch of messages. On the first, "B" will send an A- + // protocol. + for i := 0; i < 100; i++ { + gASend("RMSG $foo foo 2\r\nok\r\n") + } + // We expect single A- followed by PONG. If "B" was sending more + // this expect call would fail. + gAExpect(aunsubRe) + gASend("PING\r\n") + gAExpect(pongRe) + + // Start gateway C that connects to B + gC := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) + defer gC.Close() + + gCSend, gCExpect := setupGatewayConn(t, gC, "C", "B") + gCSend("PING\r\n") + gCExpect(pongRe) + // Send more messages, C should get A-, but A should not (already + // got it). + for i := 0; i < 100; i++ { + gCSend("RMSG $foo foo 2\r\nok\r\n") + } + gCExpect(aunsubRe) + gCSend("PING\r\n") + gCExpect(pongRe) + expectNothing(t, gA) + + // Restart one of the gateway, and resend a message, verify + // that it receives A- (things get cleared on reconnect) + gC.Close() + gC = createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) + defer gC.Close() + + gCSend, gCExpect = setupGatewayConn(t, gC, "C", "B") + gCSend("PING\r\n") + gCExpect(pongRe) + gCSend("RMSG $foo foo 2\r\nok\r\n") + gCExpect(aunsubRe) + gCSend("PING\r\n") + gCExpect(pongRe) + expectNothing(t, gA) + + // Close again and re-create, but this time don't send anything. + gC.Close() + gC = createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) + defer gC.Close() + + gCSend, gCExpect = setupGatewayConn(t, gC, "C", "B") + gCSend("PING\r\n") + gCExpect(pongRe) + + // Now register the $foo account on B and create a subscription, + // A should receive an A+ because B knows that it previously sent + // an A-, but since it did not send one to C, C should not receive + // the A+. + client := createClientConn(t, ob.Host, ob.Port) + defer client.Close() + clientSend, clientExpect := setupConnWithAccount(t, sb, client, "$foo") + clientSend("SUB not.used 1234567\r\nPING\r\n") + clientExpect(pongRe) + gAExpect(asubRe) + expectNothing(t, gC) +} + +func TestGatewaySubjectInterest(t *testing.T) { + server.GatewayDoNotForceInterestOnlyMode(true) + defer server.GatewayDoNotForceInterestOnlyMode(false) + + ob := testDefaultOptionsForGateway("B") + fooAcc := server.NewAccount("$foo") + ob.Accounts = []*server.Account{fooAcc} + ob.Users = []*server.User{{Username: "ivan", Password: "password", Account: fooAcc}} + sb := runGatewayServer(ob) + defer sb.Shutdown() + + // Create a client on B + client := createClientConn(t, ob.Host, ob.Port) + defer client.Close() + clientSend, clientExpect := setupConnWithUserPass(t, client, "ivan", "password") + // Since we want to test RS+/-, we need to have at + // least a subscription on B so that sending from A does + // not result in A- + clientSend("SUB not.used 1234567\r\nPING\r\n") + clientExpect(pongRe) + + gA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) + defer gA.Close() + + gASend, gAExpect := setupGatewayConn(t, gA, "A", "B") + gASend("PING\r\n") + gAExpect(pongRe) + + for i := 0; i < 100; i++ { + gASend("RMSG $foo foo 2\r\nok\r\n") + } + // We expect single RS- followed by PONG. If "B" was sending more + // this expect call would fail. + gAExpect(runsubRe) + gASend("PING\r\n") + gAExpect(pongRe) + + // Start gateway C that connects to B + gC := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) + defer gC.Close() + + gCSend, gCExpect := setupGatewayConn(t, gC, "C", "B") + gCSend("PING\r\n") + gCExpect(pongRe) + // Send more messages, C should get RS-, but A should not (already + // got it). + for i := 0; i < 100; i++ { + gCSend("RMSG $foo foo 2\r\nok\r\n") + } + gCExpect(runsubRe) + gCSend("PING\r\n") + gCExpect(pongRe) + expectNothing(t, gA) + + // Restart one of the gateway, and resend a message, verify + // that it receives RS- (things get cleared on reconnect) + gC.Close() + gC = createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) + defer gC.Close() + + gCSend, gCExpect = setupGatewayConn(t, gC, "C", "B") + gCSend("PING\r\n") + gCExpect(pongRe) + gCSend("RMSG $foo foo 2\r\nok\r\n") + gCExpect(runsubRe) + gCSend("PING\r\n") + gCExpect(pongRe) + expectNothing(t, gA) + + // Close again and re-create, but this time don't send anything. + gC.Close() + gC = createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) + defer gC.Close() + + gCSend, gCExpect = setupGatewayConn(t, gC, "C", "B") + gCSend("PING\r\n") + gCExpect(pongRe) + + // Now register a subscription on foo for account $foo on B. + // A should receive a RS+ because B knows that it previously + // sent a RS-, but since it did not send one to C, C should + // not receive the RS+. + clientSend("SUB foo 1\r\nSUB foo 2\r\n") + // Also subscribe to subject that was not used before, + // so there should be no RS+ for this one. + clientSend("SUB bar 3\r\nPING\r\n") + clientExpect(pongRe) + + gAExpect(rsubRe) + expectNothing(t, gC) + // Check that we get only one protocol + expectNothing(t, gA) + + // Unsubscribe the 2 subs on foo, expect to receive nothing. + clientSend("UNSUB 1\r\nUNSUB 2\r\nPING\r\n") + clientExpect(pongRe) + + expectNothing(t, gC) + expectNothing(t, gA) + + gC.Close() + + // Send on foo, should get an RS- + gASend("RMSG $foo foo 2\r\nok\r\n") + gAExpect(runsubRe) + // Subscribe on foo, should get an RS+ that removes the no-interest + clientSend("SUB foo 4\r\nPING\r\n") + clientExpect(pongRe) + gAExpect(rsubRe) + // Send on bar, message should be received. + gASend("RMSG $foo bar 2\r\nok\r\n") + clientExpect(msgRe) + // Unsub foo and bar + clientSend("UNSUB 3\r\nUNSUB 4\r\nPING\r\n") + clientExpect(pongRe) + expectNothing(t, gA) + // Send on both foo and bar expect RS- + gASend("RMSG $foo foo 2\r\nok\r\n") + gAExpect(runsubRe) + gASend("RMSG $foo bar 2\r\nok\r\n") + gAExpect(runsubRe) + // Now have client create sub on "*", this should cause RS+ on * + // The remote will have cleared its no-interest on foo and bar + // and this receiving side is supposed to be doing the same. + clientSend("SUB * 5\r\nPING\r\n") + clientExpect(pongRe) + buf := gAExpect(rsubRe) + if !bytes.Contains(buf, []byte("$foo *")) { + t.Fatalf("Expected RS+ on %q, got %q", "*", buf) + } + // Check that the remote has cleared by sending from the client + // on foo and bar + clientSend("PUB foo 2\r\nok\r\n") + clientExpect(msgRe) + clientSend("PUB bar 2\r\nok\r\n") + clientExpect(msgRe) + // Check that A can send too and does not receive an RS- + gASend("RMSG $foo foo 2\r\nok\r\n") + expectNothing(t, gA) + clientExpect(msgRe) + gASend("RMSG $foo bar 2\r\nok\r\n") + expectNothing(t, gA) + clientExpect(msgRe) +} + +func TestGatewayQueue(t *testing.T) { + server.GatewayDoNotForceInterestOnlyMode(true) + defer server.GatewayDoNotForceInterestOnlyMode(false) + + ob := testDefaultOptionsForGateway("B") + fooAcc := server.NewAccount("$foo") + ob.Accounts = []*server.Account{fooAcc} + ob.Users = []*server.User{{Username: "ivan", Password: "password", Account: fooAcc}} + sb := runGatewayServer(ob) + defer sb.Shutdown() + + gA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) + defer gA.Close() + + gASend, gAExpect := setupGatewayConn(t, gA, "A", "B") + gASend("PING\r\n") + gAExpect(pongRe) + + client := createClientConn(t, ob.Host, ob.Port) + defer client.Close() + clientSend, clientExpect := setupConnWithUserPass(t, client, "ivan", "password") + + // Create one queue sub on foo.* for group bar. + clientSend("SUB foo.* bar 1\r\nPING\r\n") + clientExpect(pongRe) + // Expect RS+ + gAExpect(rsubRe) + // Add another queue sub on same group + clientSend("SUB foo.* bar 2\r\nPING\r\n") + clientExpect(pongRe) + // Should not receive another RS+ for that one + expectNothing(t, gA) + // However, if subject is different, we can expect to receive another RS+ + clientSend("SUB foo.> bar 3\r\nPING\r\n") + clientExpect(pongRe) + gAExpect(rsubRe) + + // Unsub one of the foo.* qsub, no RS- should be received + clientSend("UNSUB 1\r\nPING\r\n") + clientExpect(pongRe) + expectNothing(t, gA) + // Remove the other one, now we should get the RS- + clientSend("UNSUB 2\r\nPING\r\n") + clientExpect(pongRe) + gAExpect(runsubRe) + // Remove last one + clientSend("UNSUB 3\r\nPING\r\n") + clientExpect(pongRe) + gAExpect(runsubRe) + + // Create some queues and check that interest is sent + // when GW reconnects. + clientSend("SUB foo bar 4\r\n") + gAExpect(rsubRe) + clientSend("SUB foo baz 5\r\n") + gAExpect(rsubRe) + clientSend("SUB foo bat 6\r\n") + gAExpect(rsubRe) + // There is already one on foo/bar, so nothing sent + clientSend("SUB foo bar 7\r\n") + expectNothing(t, gA) + // Add regular sub that should not cause RS+ + clientSend("SUB foo 8\r\n") + expectNothing(t, gA) + + // Recreate gA + gA.Close() + gA = createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) + defer gA.Close() + gASend, gAExpect = setupGatewayConn(t, gA, "A", "B") + // A should receive 3 RS+ + expectNumberOfProtos(t, gAExpect, rsubRe, 3) + // Nothing more + expectNothing(t, gA) + gASend("PING\r\n") + gAExpect(pongRe) + + // Have A send a message on subject that has no sub + gASend("RMSG $foo new.subject 2\r\nok\r\n") + gAExpect(runsubRe) + // Now create a queue sub and check that we do not receive + // an RS+ without the queue name. + clientSend("SUB new.* queue 9\r\nPING\r\n") + clientExpect(pongRe) + buf := gAExpect(rsubRe) + if !bytes.Contains(buf, []byte("new.* queue")) { + t.Fatalf("Should have receives RS+ for new.* for queue, did not: %v", buf) + } + // Check for no other RS+. A should still keep an RS- for plain + // sub on new.subject + expectNothing(t, gA) + // Send message, expected to be received by client + gASend("RMSG $foo new.subject | queue 2\r\nok\r\n") + clientExpect(msgRe) + // Unsubscribe the queue sub + clientSend("UNSUB 9\r\nPING\r\n") + clientExpect(pongRe) + // A should receive RS- for this queue sub + buf = gAExpect(runsubRe) + if !bytes.Contains(buf, []byte("new.* queue")) { + t.Fatalf("Should have receives RS- for new.* for queue, did not: %v", buf) + } + expectNothing(t, gA) +} + +func TestGatewaySendAllSubs(t *testing.T) { + server.GatewayDoNotForceInterestOnlyMode(true) + defer server.GatewayDoNotForceInterestOnlyMode(false) + + ob := testDefaultOptionsForGateway("B") + sb := runGatewayServer(ob) + defer sb.Shutdown() + + // Create a client on B + client := createClientConn(t, ob.Host, ob.Port) + defer client.Close() + clientSend, clientExpect := setupConn(t, client) + // Since we want to test RS+/-, we need to have at + // least a subscription on B so that sending from A does + // not result in A- + clientSend("SUB not.used 1234567\r\nPING\r\n") + clientExpect(pongRe) + + gA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) + defer gA.Close() + + gASend, gAExpect := setupGatewayConn(t, gA, "A", "B") + gASend("PING\r\n") + gAExpect(pongRe) + + // Bombard B with messages on different subjects. + // TODO(ik): Adapt if/when we change the conditions for the + // switch. + for i := 0; i < 1010; i++ { + gASend(fmt.Sprintf("RMSG $G foo.%d 2\r\nok\r\n", i)) + if i < 1000 { + gAExpect(runsubRe) + } + } + // Expect an INFO + RS+ $G not.used + INFO + buf := bufio.NewReader(gA) + for i := 0; i < 3; i++ { + line, _, err := buf.ReadLine() + if err != nil { + t.Fatalf("Error reading: %v", err) + } + switch i { + case 0: + case 2: + if !bytes.HasPrefix(line, []byte("INFO {")) { + t.Fatalf("Expected INFO, got: %s", line) + } + case 1: + if !bytes.HasPrefix(line, []byte("RS+ ")) { + t.Fatalf("Expected RS+, got: %s", line) + } + } + } + // After this point, any new sub or unsub on B should be + // sent to A. + clientSend("SUB foo 1\r\n") + gAExpect(rsubRe) + clientSend("UNSUB 1\r\n") + gAExpect(runsubRe) +} + +func TestGatewayNoPanicOnBadProtocol(t *testing.T) { + ob := testDefaultOptionsForGateway("B") + sb := runGatewayServer(ob) + defer sb.Shutdown() + + for _, test := range []struct { + name string + proto string + }{ + {"sub", "SUB > 1\r\n"}, + {"unsub", "UNSUB 1\r\n"}, + {"rsub", "RS+ $foo foo 2\r\n"}, + {"runsub", "RS- $foo foo 2\r\n"}, + {"pub", "PUB foo 2\r\nok\r\n"}, + {"msg", "MSG foo 2\r\nok\r\n"}, + {"rmsg", "RMSG $foo foo 2\r\nok\r\n"}, + {"anything", "xxxx\r\n"}, + } { + t.Run(test.name, func(t *testing.T) { + // Create raw tcp connection to gateway port + client := createClientConn(t, ob.Gateway.Host, ob.Gateway.Port) + defer client.Close() + clientSend := sendCommand(t, client) + clientSend(test.proto) + }) + } + + // Server should not have crashed. + client := createClientConn(t, ob.Host, ob.Port) + defer client.Close() + clientSend, clientExpect := setupConn(t, client) + clientSend("PING\r\n") + clientExpect(pongRe) +} + +func TestGatewayNoAccUnsubAfterQSub(t *testing.T) { + server.GatewayDoNotForceInterestOnlyMode(true) + defer server.GatewayDoNotForceInterestOnlyMode(false) + + ob := testDefaultOptionsForGateway("B") + sb := runGatewayServer(ob) + defer sb.Shutdown() + + gA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) + defer gA.Close() + + gASend, gAExpect := setupGatewayConn(t, gA, "A", "B") + gASend("PING\r\n") + gAExpect(pongRe) + + // Simulate a client connecting to A and publishing a message + // so we get an A- from B since there is no interest. + gASend("RMSG $G foo 2\r\nok\r\n") + gAExpect(runsubRe) + + // Now create client on B and create queue sub. + client := createClientConn(t, ob.Host, ob.Port) + defer client.Close() + clientSend, clientExpect := setupConn(t, client) + + clientSend("SUB bar queue 1\r\nPING\r\n") + clientExpect(pongRe) + + // A should receive an RS+ for this queue sub. + gAExpect(rsubRe) + + // On B, create a plain sub now. We should get nothing. + clientSend("SUB baz 2\r\nPING\r\n") + clientExpect(pongRe) + + expectNothing(t, gA) +} + +func TestGatewayErrorOnRSentFromOutbound(t *testing.T) { + server.GatewayDoNotForceInterestOnlyMode(true) + defer server.GatewayDoNotForceInterestOnlyMode(false) + + ob := testDefaultOptionsForGateway("B") + sb := runGatewayServer(ob) + defer sb.Shutdown() + + for _, test := range []struct { + name string + proto string + }{ + {"RS+", "RS+"}, + {"RS-", "RS-"}, + } { + t.Run(test.name, func(t *testing.T) { + gA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) + defer gA.Close() + + gASend, gAExpect := setupGatewayConn(t, gA, "A", "B") + gASend("PING\r\n") + gAExpect(pongRe) + + gASend(fmt.Sprintf("%s foo bar\r\n", test.proto)) + expectDisconnect(t, gA) + }) + } +} + +func TestGatewaySystemConnectionAllowedToPublishOnGWPrefix(t *testing.T) { + sc := createSuperCluster(t, 2, 2) + defer sc.shutdown() + + o := sc.clusters[1].opts[1] + url := fmt.Sprintf("nats://sys:pass@%s:%d", o.Host, o.Port) + nc, err := nats.Connect(url) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + reply := nats.NewInbox() + sub, err := nc.SubscribeSync(reply) + if err != nil { + t.Fatalf("Error on subscribe: %v", err) + } + if err := nc.PublishRequest("$SYS.REQ.SERVER.PING", reply, nil); err != nil { + t.Fatalf("Failed to send request: %v", err) + } + for i := 0; i < 4; i++ { + if _, err := sub.NextMsg(time.Second); err != nil { + t.Fatalf("Expected to get a response, got %v", err) + } + } +} + +func TestGatewayTLSMixedIPAndDNS(t *testing.T) { + server.SetGatewaysSolicitDelay(5 * time.Millisecond) + defer server.ResetGatewaysSolicitDelay() + + // Run this test extra times to make sure not flaky since it + // on solicit time. + for i := 0; i < 10; i++ { + t.Run("", func(t *testing.T) { + confA1 := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + server_name: A1 + gateway { + name: "A" + listen: "127.0.0.1:-1" + tls { + cert_file: "./configs/certs/server-iponly.pem" + key_file: "./configs/certs/server-key-iponly.pem" + ca_file: "./configs/certs/ca.pem" + timeout: 2 + } + connect_retries: 3 + } + cluster { + listen: "127.0.0.1:-1" + }`)) + srvA1, optsA1 := RunServerWithConfig(confA1) + defer srvA1.Shutdown() + + confA2Template := ` + listen: 127.0.0.1:-1 + server_name: A2 + gateway { + name: "A" + listen: "localhost:-1" + tls { + cert_file: "./configs/certs/server-cert.pem" + key_file: "./configs/certs/server-key.pem" + ca_file: "./configs/certs/ca.pem" + timeout: 2 + } + connect_retries: 3 + } + cluster { + listen: "127.0.0.1:-1" + routes [ + "nats://%s:%d" + ] + }` + confA2 := createConfFile(t, []byte(fmt.Sprintf(confA2Template, + optsA1.Cluster.Host, optsA1.Cluster.Port))) + srvA2, optsA2 := RunServerWithConfig(confA2) + defer srvA2.Shutdown() + + checkClusterFormed(t, srvA1, srvA2) + + // Create a GW connection to cluster "A". Don't use the helper since we need verification etc. + o := DefaultTestOptions + o.Port = -1 + o.ServerName = "B1" + o.Gateway.Name = "B" + o.Gateway.Host = "127.0.0.1" + o.Gateway.Port = -1 + + tc := &server.TLSConfigOpts{} + tc.CertFile = "./configs/certs/server-cert.pem" + tc.KeyFile = "./configs/certs/server-key.pem" + tc.CaFile = "./configs/certs/ca.pem" + tc.Timeout = 2.0 + tlsConfig, err := server.GenTLSConfig(tc) + if err != nil { + t.Fatalf("Error generating TLS config: %v", err) + } + tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert + tlsConfig.RootCAs = tlsConfig.ClientCAs + + o.Gateway.TLSConfig = tlsConfig.Clone() + + rurl, _ := url.Parse(fmt.Sprintf("nats://%s:%d", optsA2.Gateway.Host, optsA2.Gateway.Port)) + remote := &server.RemoteGatewayOpts{Name: "A", URLs: []*url.URL{rurl}} + remote.TLSConfig = tlsConfig.Clone() + o.Gateway.Gateways = []*server.RemoteGatewayOpts{remote} + + srvB := RunServer(&o) + defer srvB.Shutdown() + + waitForOutboundGateways(t, srvB, 1, 10*time.Second) + waitForOutboundGateways(t, srvA1, 1, 10*time.Second) + waitForOutboundGateways(t, srvA2, 1, 10*time.Second) + + // Now kill off srvA2 and force serverB to connect to srvA1. + srvA2.Shutdown() + + // Make sure this works. + waitForOutboundGateways(t, srvB, 1, 30*time.Second) + }) + } +} + +func TestGatewayAdvertiseInCluster(t *testing.T) { + server.GatewayDoNotForceInterestOnlyMode(true) + defer server.GatewayDoNotForceInterestOnlyMode(false) + + ob1 := testDefaultOptionsForGateway("B") + ob1.Cluster.Name = "B" + ob1.Cluster.Host = "127.0.0.1" + ob1.Cluster.Port = -1 + sb1 := runGatewayServer(ob1) + defer sb1.Shutdown() + + gA := createGatewayConn(t, ob1.Gateway.Host, ob1.Gateway.Port) + defer gA.Close() + + gASend, gAExpect := setupGatewayConn(t, gA, "A", "B") + gASend("PING\r\n") + gAExpect(pongRe) + + ob2 := testDefaultOptionsForGateway("B") + ob2.Cluster.Name = "B" + ob2.Cluster.Host = "127.0.0.1" + ob2.Cluster.Port = -1 + ob2.Routes = server.RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", ob1.Cluster.Port)) + ob2.Gateway.Advertise = "srvB:7222" + sb2 := runGatewayServer(ob2) + defer sb2.Shutdown() + + checkClusterFormed(t, sb1, sb2) + + buf := gAExpect(infoRe) + si := &server.Info{} + json.Unmarshal(buf[5:], si) + var ok bool + for _, u := range si.GatewayURLs { + if u == "srvB:7222" { + ok = true + break + } + } + if !ok { + t.Fatalf("Url srvB:7222 was not found: %q", si.GatewayURLs) + } + + ob3 := testDefaultOptionsForGateway("B") + ob3.Cluster.Name = "B" + ob3.Cluster.Host = "127.0.0.1" + ob3.Cluster.Port = -1 + ob3.Routes = server.RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", ob1.Cluster.Port)) + ob3.Gateway.Advertise = "srvB:7222" + sb3 := runGatewayServer(ob3) + defer sb3.Shutdown() + + checkClusterFormed(t, sb1, sb2, sb3) + + // Since it is the save srvB:7222 url, we should not get an update. + expectNothing(t, gA) + + // Now shutdown sb2 and make sure that we are not getting an update + // with srvB:7222 missing. + sb2.Shutdown() + expectNothing(t, gA) +} + +func TestGatewayAuthTimeout(t *testing.T) { + for _, test := range []struct { + name string + setAuth bool // + wait time.Duration + }{ + {"auth not explicitly set", false, 2500 * time.Millisecond}, + {"auth set", true, 500 * time.Millisecond}, + } { + t.Run(test.name, func(t *testing.T) { + ob := testDefaultOptionsForGateway("B") + if test.setAuth { + ob.Gateway.AuthTimeout = 0.25 + } + sb := RunServer(ob) + defer sb.Shutdown() + + sa := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) + defer sa.Close() + + gAExpect := expectCommand(t, sa) + + dstInfo := checkInfoMsg(t, sa) + if dstInfo.Gateway != "B" { + t.Fatalf("Expected to connect to %q, got %q", "B", dstInfo.Gateway) + } + + // Don't send our CONNECT and we should be disconnected due to auth timeout. + time.Sleep(test.wait) + gAExpect(errRe) + expectDisconnect(t, sa) + }) + } +} + +func TestGatewayFirstPingGoesAfterConnect(t *testing.T) { + server.GatewayDoNotForceInterestOnlyMode(true) + defer server.GatewayDoNotForceInterestOnlyMode(false) + + ob := testDefaultOptionsForGateway("B") + // For this test, we want the first ping to NOT be disabled. + ob.DisableShortFirstPing = false + // Also, for this test increase auth_timeout so that it does not disconnect + // while checking... + ob.Gateway.AuthTimeout = 10.0 + sb := RunServer(ob) + defer sb.Shutdown() + + sa := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) + defer sa.Close() + + gASend, gAExpect := sendCommand(t, sa), expectCommand(t, sa) + dstInfo := checkInfoMsg(t, sa) + if dstInfo.Gateway != "B" { + t.Fatalf("Expected to connect to %q, got %q", "B", dstInfo.Gateway) + } + + // Wait and we should not be receiving a PING from server B until we send + // a CONNECT. We need to wait for more than the initial PING, so cannot + // use expectNothing() helper here. + buf := make([]byte, 256) + sa.SetReadDeadline(time.Now().Add(2 * time.Second)) + if n, err := sa.Read(buf); err == nil { + t.Fatalf("Expected nothing, got %s", buf[:n]) + } + + // Now send connect and INFO + cs := fmt.Sprintf("CONNECT {\"verbose\":%v,\"pedantic\":%v,\"tls_required\":%v,\"gateway\":%q}\r\n", + false, false, false, "A") + gASend(cs) + gASend(fmt.Sprintf("INFO {\"gateway\":%q}\r\n", "A")) + + // We should get the first PING + gAExpect(pingRe) +} diff --git a/test/gosrv_test.go b/test/gosrv_test.go new file mode 100644 index 0000000..c06aaee --- /dev/null +++ b/test/gosrv_test.go @@ -0,0 +1,65 @@ +// Copyright 2012-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "net" + "runtime" + "testing" + "time" +) + +func TestSimpleGoServerShutdown(t *testing.T) { + base := runtime.NumGoroutine() + opts := DefaultTestOptions + opts.Port = -1 + s := RunServer(&opts) + s.Shutdown() + checkFor(t, time.Second, 100*time.Millisecond, func() error { + delta := (runtime.NumGoroutine() - base) + if delta > 1 { + return fmt.Errorf("%d go routines still exist post Shutdown()", delta) + } + return nil + }) +} + +func TestGoServerShutdownWithClients(t *testing.T) { + base := runtime.NumGoroutine() + opts := DefaultTestOptions + opts.Port = -1 + s := RunServer(&opts) + addr := s.Addr().(*net.TCPAddr) + for i := 0; i < 50; i++ { + createClientConn(t, "127.0.0.1", addr.Port) + } + s.Shutdown() + // Wait longer for client connections + time.Sleep(1 * time.Second) + delta := (runtime.NumGoroutine() - base) + // There may be some finalizers or IO, but in general more than + // 2 as a delta represents a problem. + if delta > 2 { + t.Fatalf("%d Go routines still exist post Shutdown()", delta) + } +} + +func TestGoServerMultiShutdown(t *testing.T) { + opts := DefaultTestOptions + opts.Port = -1 + s := RunServer(&opts) + s.Shutdown() + s.Shutdown() +} diff --git a/test/leafnode_test.go b/test/leafnode_test.go new file mode 100644 index 0000000..c5428d4 --- /dev/null +++ b/test/leafnode_test.go @@ -0,0 +1,4509 @@ +// Copyright 2019-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "math/rand/v2" + "net" + "net/url" + "os" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/jwt/v2" + "github.com/nats-io/nats-server/v2/logger" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nkeys" + "github.com/nats-io/nuid" +) + +func createLeafConn(t tLogger, host string, port int) net.Conn { + return createClientConn(t, host, port) +} + +func testDefaultOptionsForLeafNodes() *server.Options { + o := DefaultTestOptions + o.Port = -1 + o.LeafNode.Host = o.Host + o.LeafNode.Port = -1 + o.NoSystemAccount = true + return &o +} + +func runLeafServer() (*server.Server, *server.Options) { + o := testDefaultOptionsForLeafNodes() + return RunServer(o), o +} + +func runLeafServerOnPort(port int) (*server.Server, *server.Options) { + o := testDefaultOptionsForLeafNodes() + o.LeafNode.Port = port + return RunServer(o), o +} + +func runSolicitLeafServer(lso *server.Options) (*server.Server, *server.Options) { + return runSolicitLeafServerToURL(fmt.Sprintf("nats-leaf://%s:%d", lso.LeafNode.Host, lso.LeafNode.Port)) +} + +func runSolicitLeafServerToURL(surl string) (*server.Server, *server.Options) { + o := DefaultTestOptions + o.Port = -1 + o.NoSystemAccount = true + rurl, _ := url.Parse(surl) + o.LeafNode.Remotes = []*server.RemoteLeafOpts{{URLs: []*url.URL{rurl}}} + o.LeafNode.ReconnectInterval = 100 * time.Millisecond + return RunServer(&o), &o +} + +func TestLeafNodeInfo(t *testing.T) { + s, opts := runLeafServer() + defer s.Shutdown() + + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + info := checkInfoMsg(t, lc) + if !info.AuthRequired { + t.Fatalf("AuthRequired should always be true for leaf nodes") + } + // By default headers should be true. + if !info.Headers { + t.Fatalf("Expected to have headers on by default") + } + + sendProto(t, lc, "CONNECT {}\r\n") + checkLeafNodeConnected(t, s) + + // Now close connection, make sure we are doing the right accounting in the server. + lc.Close() + + checkLeafNodeConnections(t, s, 0) +} + +func TestLeafNodeSplitBuffer(t *testing.T) { + s, opts := runLeafServer() + defer s.Shutdown() + + nc, err := nats.Connect(s.ClientURL()) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + nc.QueueSubscribe("foo", "bar", func(m *nats.Msg) { + m.Respond([]byte("ok")) + }) + nc.Flush() + + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + sendProto(t, lc, "CONNECT {}\r\n") + checkLeafNodeConnected(t, s) + + leafSend, leafExpect := setupLeaf(t, lc, 2) + + leafSend("LS+ reply\r\nPING\r\n") + leafExpect(pongRe) + + leafSend("LMSG foo ") + time.Sleep(time.Millisecond) + leafSend("+ reply bar 2\r\n") + time.Sleep(time.Millisecond) + leafSend("OK\r") + time.Sleep(time.Millisecond) + leafSend("\n") + leafExpect(lmsgRe) +} + +func TestNumLeafNodes(t *testing.T) { + s, opts := runLeafServer() + defer s.Shutdown() + + createNewLeafNode := func() net.Conn { + t.Helper() + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + checkInfoMsg(t, lc) + sendProto(t, lc, "CONNECT {}\r\n") + return lc + } + checkLeafNodeConnections(t, s, 0) + + lc1 := createNewLeafNode() + defer lc1.Close() + checkLeafNodeConnections(t, s, 1) + + lc2 := createNewLeafNode() + defer lc2.Close() + checkLeafNodeConnections(t, s, 2) + + // Now test remove works. + lc1.Close() + checkLeafNodeConnections(t, s, 1) + + lc2.Close() + checkLeafNodeConnections(t, s, 0) +} + +func TestLeafNodeRequiresConnect(t *testing.T) { + opts := testDefaultOptionsForLeafNodes() + opts.LeafNode.AuthTimeout = 0.001 + s := RunServer(opts) + defer s.Shutdown() + + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + info := checkInfoMsg(t, lc) + if !info.AuthRequired { + t.Fatalf("Expected AuthRequired to force CONNECT") + } + if info.TLSRequired { + t.Fatalf("Expected TLSRequired to be false") + } + if info.TLSVerify { + t.Fatalf("Expected TLSVerify to be false") + } + + // Now wait and make sure we get disconnected. + errBuf := expectResult(t, lc, errRe) + + if !strings.Contains(string(errBuf), "Authentication Timeout") { + t.Fatalf("Authentication Timeout response incorrect: %q", errBuf) + } + expectDisconnect(t, lc) +} + +func setupLeaf(t *testing.T, lc net.Conn, expectedSubs int) (sendFun, expectFun) { + t.Helper() + send, expect := setupConn(t, lc) + // A loop detection subscription is sent, so consume this here, along + // with the ones that caller expect on setup. + expectNumberOfProtos(t, expect, lsubRe, expectedSubs, infoStartRe, pingRe) + return send, expect +} + +func TestLeafNodeSendsSubsAfterConnect(t *testing.T) { + s, opts := runLeafServer() + defer s.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + send("SUB foo 1\r\n") + send("SUB bar 2\r\n") + send("SUB foo baz 3\r\n") + send("SUB foo baz 4\r\n") + send("SUB bar 5\r\n") + send("PING\r\n") + expect(pongRe) + + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + // This should compress down to 1 for foo, 1 for bar, and 1 for foo [baz] + // and one for the loop detection subject. + setupLeaf(t, lc, 4) +} + +func TestLeafNodeSendsSubsOngoing(t *testing.T) { + s, opts := runLeafServer() + defer s.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + send("PING\r\n") + expect(pongRe) + + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + leafSend, leafExpect := setupLeaf(t, lc, 1) + leafSend("PING\r\n") + leafExpect(pongRe) + + send("SUB foo 1\r\n") + leafExpect(lsubRe) + + // Check queues send updates each time. + // TODO(dlc) - If we decide to suppress this with a timer approach this test will break. + send("SUB foo bar 2\r\n") + leafExpect(lsubRe) + send("SUB foo bar 3\r\n") + leafExpect(lsubRe) + send("SUB foo bar 4\r\n") + leafExpect(lsubRe) + + // Now check more normal subs do nothing. + send("SUB foo 5\r\n") + expectNothing(t, lc) + + // Check going back down does nothing til we hit 0. + send("UNSUB 5\r\n") + expectNothing(t, lc) + send("UNSUB 1\r\n") + leafExpect(lunsubRe) + + // Queues going down should always send updates. + send("UNSUB 2\r\n") + leafExpect(lsubRe) + send("UNSUB 3\r\n") + leafExpect(lsubRe) + send("UNSUB 4\r\n") + leafExpect(lunsubRe) +} + +func TestLeafNodeSubs(t *testing.T) { + s, opts := runLeafServer() + defer s.Shutdown() + + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + leafSend, leafExpect := setupLeaf(t, lc, 1) + + leafSend("PING\r\n") + leafExpect(pongRe) + + leafSend("LS+ foo\r\n") + expectNothing(t, lc) + leafSend("PING\r\n") + leafExpect(pongRe) + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + send("PING\r\n") + expect(pongRe) + + send("PUB foo 2\r\nOK\r\n") + matches := lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkLmsg(t, matches[0], "foo", "", "2", "OK") + + // Second sub should not change delivery + leafSend("LS+ foo\r\n") + expectNothing(t, lc) + leafSend("PING\r\n") + leafExpect(pongRe) + + send("PUB foo 3\r\nOK!\r\n") + matches = lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkLmsg(t, matches[0], "foo", "", "3", "OK!") + + // Now add in a queue sub with weight 4. + leafSend("LS+ foo bar 4\r\n") + expectNothing(t, lc) + leafSend("PING\r\n") + leafExpect(pongRe) + + send("PUB foo 4\r\nOKOK\r\n") + matches = lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkLmsg(t, matches[0], "foo", "| bar", "4", "OKOK") + + // Now add in a queue sub with weight 4. + leafSend("LS+ foo baz 2\r\n") + expectNothing(t, lc) + leafSend("PING\r\n") + leafExpect(pongRe) + + send("PUB foo 5\r\nHELLO\r\n") + matches = lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkLmsg(t, matches[0], "foo", "| bar baz", "5", "HELLO") + + // Test Unsub + leafSend("LS- foo\r\n") + leafSend("LS- foo bar\r\n") + leafSend("LS- foo baz\r\n") + expectNothing(t, lc) + leafSend("PING\r\n") + leafExpect(pongRe) + + send("PUB foo 5\r\nHELLO\r\n") + expectNothing(t, lc) +} + +func TestLeafNodeMsgDelivery(t *testing.T) { + s, opts := runLeafServer() + defer s.Shutdown() + + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + leafSend, leafExpect := setupLeaf(t, lc, 1) + + leafSend("PING\r\n") + leafExpect(pongRe) + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + send("PING\r\n") + expect(pongRe) + + send("SUB foo 1\r\nPING\r\n") + expect(pongRe) + leafExpect(lsubRe) + + // Now send from leaf side. + leafSend("LMSG foo 2\r\nOK\r\n") + expectNothing(t, lc) + + matches := msgRe.FindAllSubmatch(expect(msgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkMsg(t, matches[0], "foo", "1", "", "2", "OK") + + send("UNSUB 1\r\nPING\r\n") + expect(pongRe) + leafExpect(lunsubRe) + send("SUB foo bar 2\r\nPING\r\n") + expect(pongRe) + leafExpect(lsubRe) + + // Now send again from leaf side. This is targeted so this should + // not be delivered. + leafSend("LMSG foo 2\r\nOK\r\n") + expectNothing(t, lc) + expectNothing(t, c) + + // Now send targeted, and we should receive it. + leafSend("LMSG foo | bar 2\r\nOK\r\n") + expectNothing(t, lc) + + matches = msgRe.FindAllSubmatch(expect(msgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkMsg(t, matches[0], "foo", "2", "", "2", "OK") + + // Check reply + queues + leafSend("LMSG foo + myreply bar 2\r\nOK\r\n") + expectNothing(t, lc) + + matches = msgRe.FindAllSubmatch(expect(msgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkMsg(t, matches[0], "foo", "2", "myreply", "2", "OK") +} + +func TestLeafNodeAndRoutes(t *testing.T) { + optsA := LoadConfig("./configs/srv_a_leaf.conf") + optsA.DisableShortFirstPing = true + optsB := LoadConfig("./configs/srv_b.conf") + optsB.DisableShortFirstPing = true + srvA := RunServer(optsA) + defer srvA.Shutdown() + srvB := RunServer(optsB) + defer srvB.Shutdown() + checkClusterFormed(t, srvA, srvB) + + lc := createLeafConn(t, optsA.LeafNode.Host, optsA.LeafNode.Port) + defer lc.Close() + + leafSend, leafExpect := setupLeaf(t, lc, 6) + leafSend("PING\r\n") + leafExpect(pongRe) + + c := createClientConn(t, optsB.Host, optsB.Port) + defer c.Close() + + send, expect := setupConn(t, c) + send("PING\r\n") + expect(pongRe) + + send("SUB foo 1\r\nPING\r\n") + expect(pongRe) + leafExpect(lsubRe) + + send("SUB foo 2\r\nPING\r\n") + expect(pongRe) + expectNothing(t, lc) + + send("UNSUB 2\r\nPING\r\n") + expect(pongRe) + expectNothing(t, lc) + send("UNSUB 1\r\nPING\r\n") + expect(pongRe) + leafExpect(lunsubRe) + + // Now put it back and test msg flow. + send("SUB foo 1\r\nPING\r\n") + expect(pongRe) + leafExpect(lsubRe) + + leafSend("LMSG foo + myreply bar 2\r\nOK\r\n") + expectNothing(t, lc) + + matches := msgRe.FindAllSubmatch(expect(msgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkMsg(t, matches[0], "foo", "1", "myreply", "2", "OK") + + // Now check reverse. + leafSend("LS+ bar\r\n") + expectNothing(t, lc) + leafSend("PING\r\n") + leafExpect(pongRe) + + send("PUB bar 2\r\nOK\r\n") + matches = lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkLmsg(t, matches[0], "bar", "", "2", "OK") +} + +// Helper function to check that a leaf node has connected to our server. +func checkLeafNodeConnected(t *testing.T, s *server.Server) { + t.Helper() + checkLeafNodeConnections(t, s, 1) +} + +func checkLeafNodeConnections(t *testing.T, s *server.Server, expected int) { + t.Helper() + checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { + if nln := s.NumLeafNodes(); nln != expected { + return fmt.Errorf("Expected a connected leafnode for server %q, got %d", s.ID(), nln) + } + return nil + }) +} + +func TestLeafNodeSolicit(t *testing.T) { + s, opts := runLeafServer() + defer s.Shutdown() + + sl, _ := runSolicitLeafServer(opts) + defer sl.Shutdown() + + checkLeafNodeConnected(t, s) + + // Now test reconnect. + s.Shutdown() + // Need to restart it on the same port. + s, _ = runLeafServerOnPort(opts.LeafNode.Port) + defer s.Shutdown() + checkLeafNodeConnected(t, s) +} + +func TestLeafNodeNoEcho(t *testing.T) { + s, opts := runLeafServer() + defer s.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + send("PING\r\n") + expect(pongRe) + + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + leafSend, leafExpect := setupLeaf(t, lc, 1) + leafSend("PING\r\n") + leafExpect(pongRe) + + // We should not echo back to ourselves. Set up 'foo' subscriptions + // on both sides and send message across the leafnode connection. It + // should not come back. + + send("SUB foo 1\r\n") + leafExpect(lsubRe) + + leafSend("LS+ foo\r\n") + expectNothing(t, lc) + leafSend("PING\r\n") + leafExpect(pongRe) + + leafSend("LMSG foo 2\r\nOK\r\n") + expectNothing(t, lc) +} + +func TestLeafNodeHeaderSupport(t *testing.T) { + srvA, optsA := runLeafServer() + defer srvA.Shutdown() + + srvB, optsB := runSolicitLeafServer(optsA) + defer srvB.Shutdown() + + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + sendA, expectA := setupHeaderConn(t, clientA) + sendA("SUB foo bar 22\r\n") + sendA("SUB bar 11\r\n") + sendA("PING\r\n") + expectA(pongRe) + + if err := checkExpectedSubs(3, srvB); err != nil { + t.Fatalf("%v", err) + } + + sendB, expectB := setupHeaderConn(t, clientB) + // Can not have \r\n in payload fyi for regex. + // With reply + sendB("HPUB foo reply 12 14\r\nK1:V1,K2:V2 ok\r\n") + sendB("PING\r\n") + expectB(pongRe) + + expectHeaderMsgs := expectHeaderMsgsCommand(t, expectA) + matches := expectHeaderMsgs(1) + checkHmsg(t, matches[0], "foo", "22", "reply", "12", "14", "K1:V1,K2:V2 ", "ok") + + // Without reply + sendB("HPUB foo 12 14\r\nK1:V1,K2:V2 ok\r\n") + sendB("PING\r\n") + expectB(pongRe) + + matches = expectHeaderMsgs(1) + checkHmsg(t, matches[0], "foo", "22", "", "12", "14", "K1:V1,K2:V2 ", "ok") + + // Without queues or reply + sendB("HPUB bar 12 14\r\nK1:V1,K2:V2 ok\r\n") + sendB("PING\r\n") + expectB(pongRe) + + matches = expectHeaderMsgs(1) + checkHmsg(t, matches[0], "bar", "11", "", "12", "14", "K1:V1,K2:V2 ", "ok") + + // Without queues but with reply + sendB("HPUB bar reply 12 14\r\nK1:V1,K2:V2 ok\r\n") + sendB("PING\r\n") + expectB(pongRe) + + matches = expectHeaderMsgs(1) + checkHmsg(t, matches[0], "bar", "11", "reply", "12", "14", "K1:V1,K2:V2 ", "ok") +} + +// Used to setup clusters of clusters for tests. +type cluster struct { + servers []*server.Server + opts []*server.Options + name string + t *testing.T +} + +func testDefaultClusterOptionsForLeafNodes() *server.Options { + o := DefaultTestOptions + o.Port = -1 + o.Cluster.Host = o.Host + o.Cluster.Port = -1 + o.Gateway.Host = o.Host + o.Gateway.Port = -1 + o.LeafNode.Host = o.Host + o.LeafNode.Port = -1 + return &o +} + +func (c *cluster) shutdown() { + if c == nil { + return + } + for i, s := range c.servers { + if cf := c.opts[i].ConfigFile; cf != "" { + os.RemoveAll(cf) + } + if sd := s.StoreDir(); sd != "" { + os.RemoveAll(sd) + } + s.Shutdown() + } +} + +func shutdownCluster(c *cluster) { + c.shutdown() +} + +func (c *cluster) totalSubs() int { + totalSubs := 0 + for _, s := range c.servers { + totalSubs += int(s.NumSubscriptions()) + } + return totalSubs +} + +// Wait for the expected number of outbound gateways, or fails. +func waitForOutboundGateways(t *testing.T, s *server.Server, expected int, timeout time.Duration) { + t.Helper() + if timeout < 2*time.Second { + timeout = 2 * time.Second + } + checkFor(t, timeout, 15*time.Millisecond, func() error { + if n := s.NumOutboundGateways(); n != expected { + return fmt.Errorf("Expected %v outbound gateway(s), got %v (ulimit -n too low?)", + expected, n) + } + return nil + }) +} + +// Creates a full cluster with numServers and given name and makes sure its well formed. +// Will have Gateways and Leaf Node connections active. +func createClusterWithName(t *testing.T, clusterName string, numServers int, connectTo ...*cluster) *cluster { + t.Helper() + return createClusterEx(t, false, 5*time.Millisecond, true, clusterName, numServers, connectTo...) +} + +// Creates a cluster and optionally additional accounts and users. +// Will have Gateways and Leaf Node connections active. +func createClusterEx(t *testing.T, doAccounts bool, gwSolicit time.Duration, waitOnGWs bool, clusterName string, numServers int, connectTo ...*cluster) *cluster { + t.Helper() + + if clusterName == "" || numServers < 1 { + t.Fatalf("Bad params") + } + + // Setup some accounts and users. + // $SYS is always the system account. And we have default FOO and BAR accounts, as well + // as DLC and NGS which do a service import. + createAccountsAndUsers := func() ([]*server.Account, []*server.User) { + if !doAccounts { + return []*server.Account{server.NewAccount("$SYS")}, nil + } + + sys := server.NewAccount("$SYS") + ngs := server.NewAccount("NGS") + dlc := server.NewAccount("DLC") + foo := server.NewAccount("FOO") + bar := server.NewAccount("BAR") + + accounts := []*server.Account{sys, foo, bar, ngs, dlc} + + ngs.AddServiceExport("ngs.usage.*", nil) + dlc.AddServiceImport(ngs, "ngs.usage", "ngs.usage.dlc") + + // Setup users + users := []*server.User{ + {Username: "dlc", Password: "pass", Permissions: nil, Account: dlc}, + {Username: "ngs", Password: "pass", Permissions: nil, Account: ngs}, + {Username: "foo", Password: "pass", Permissions: nil, Account: foo}, + {Username: "bar", Password: "pass", Permissions: nil, Account: bar}, + {Username: "sys", Password: "pass", Permissions: nil, Account: sys}, + } + return accounts, users + } + + bindGlobal := func(s *server.Server) { + ngs, err := s.LookupAccount("NGS") + if err != nil { + return + } + // Bind global to service import + gacc, _ := s.LookupAccount("$G") + gacc.AddServiceImport(ngs, "ngs.usage", "ngs.usage.$G") + } + + // If we are going to connect to another cluster set that up now for options. + var gws []*server.RemoteGatewayOpts + for _, c := range connectTo { + // Gateways autodiscover here too, so just need one address from the set. + gwAddr := fmt.Sprintf("nats-gw://%s:%d", c.opts[0].Gateway.Host, c.opts[0].Gateway.Port) + gwurl, _ := url.Parse(gwAddr) + gws = append(gws, &server.RemoteGatewayOpts{Name: c.name, URLs: []*url.URL{gwurl}}) + } + + // Make the GWs form faster for the tests. + server.SetGatewaysSolicitDelay(gwSolicit) + defer server.ResetGatewaysSolicitDelay() + + // Create seed first. + o := testDefaultClusterOptionsForLeafNodes() + o.Gateway.Name = clusterName + o.Gateway.Gateways = gws + // All of these need system accounts. + o.Accounts, o.Users = createAccountsAndUsers() + o.SystemAccount = "$SYS" + o.ServerName = fmt.Sprintf("%s1", clusterName) + // Run the server + s := RunServer(o) + bindGlobal(s) + + c := &cluster{servers: make([]*server.Server, 0, numServers), opts: make([]*server.Options, 0, numServers), name: clusterName} + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + + // For connecting to seed server above. + routeAddr := fmt.Sprintf("nats-route://%s:%d", o.Cluster.Host, o.Cluster.Port) + rurl, _ := url.Parse(routeAddr) + routes := []*url.URL{rurl} + + for i := 1; i < numServers; i++ { + o := testDefaultClusterOptionsForLeafNodes() + o.Gateway.Name = clusterName + o.Gateway.Gateways = gws + o.Routes = routes + // All of these need system accounts. + o.Accounts, o.Users = createAccountsAndUsers() + o.SystemAccount = "$SYS" + o.ServerName = fmt.Sprintf("%s%d", clusterName, i+1) + s := RunServer(o) + bindGlobal(s) + + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + } + checkClusterFormed(t, c.servers...) + + if waitOnGWs { + // Wait on gateway connections if we were asked to connect to other gateways. + if numGWs := len(connectTo); numGWs > 0 { + for _, s := range c.servers { + waitForOutboundGateways(t, s, numGWs, 2*time.Second) + } + } + } + c.t = t + return c +} + +func TestLeafNodeGatewayRequiresSystemAccount(t *testing.T) { + o := testDefaultClusterOptionsForLeafNodes() + o.Gateway.Name = "CLUSTER-A" + _, err := server.NewServer(o) + if err == nil { + t.Fatalf("Expected an error with no system account defined") + } +} + +func TestLeafNodeGatewaySendsSystemEvent(t *testing.T) { + server.SetGatewaysSolicitDelay(50 * time.Millisecond) + defer server.ResetGatewaysSolicitDelay() + + ca := createClusterWithName(t, "A", 1) + defer shutdownCluster(ca) + cb := createClusterWithName(t, "B", 1, ca) + defer shutdownCluster(cb) + + // Create client on a server in cluster A + opts := ca.opts[0] + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + // Listen for the leaf node event. + send, expect := setupConnWithAccount(t, ca.servers[0], c, "$SYS") + send("SUB $SYS.ACCOUNT.$G.LEAFNODE.CONNECT 1\r\nPING\r\n") + expect(pongRe) + + opts = cb.opts[0] + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + // This is for our global responses since we are setting up GWs above. + leafSend, leafExpect := setupLeaf(t, lc, 8) + leafSend("PING\r\n") + leafExpect(pongRe) + + matches := rawMsgRe.FindAllSubmatch(expect(rawMsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + m := matches[0] + if string(m[subIndex]) != "$SYS.ACCOUNT.$G.LEAFNODE.CONNECT" { + t.Fatalf("Got wrong subject for leaf node event, got %q, wanted %q", + m[subIndex], "$SYS.ACCOUNT.$G.LEAFNODE.CONNECT") + } +} + +func TestLeafNodeGatewayInterestPropagation(t *testing.T) { + server.SetGatewaysSolicitDelay(10 * time.Millisecond) + defer server.ResetGatewaysSolicitDelay() + + ca := createClusterWithName(t, "A", 3) + defer shutdownCluster(ca) + cb := createClusterWithName(t, "B", 3, ca) + defer shutdownCluster(cb) + + sl1, sl1Opts := runSolicitLeafServer(ca.opts[1]) + defer sl1.Shutdown() + + c := createClientConn(t, sl1Opts.Host, sl1Opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + send("SUB foo 1\r\n") + send("PING\r\n") + expect(pongRe) + + // Now we will create a new leaf node on cluster B, expect to get the + // interest for "foo". + opts := cb.opts[0] + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + _, leafExpect := setupConn(t, lc) + buf := leafExpect(infoStartRe) + buf = infoStartRe.ReplaceAll(buf, []byte(nil)) + + foundFoo := false + for count := 0; count < 10; { + // skip first time if we still have data (buf from above may already have some left) + if count != 0 || len(buf) == 0 { + buf = append(buf, leafExpect(anyRe)...) + } + count += len(lsubRe.FindAllSubmatch(buf, -1)) + if count > 10 { + t.Fatalf("Expected %v matches, got %v (buf=%s)", 10, count, buf) + } + if strings.Contains(string(buf), "foo") { + foundFoo = true + } + buf = lsubRe.ReplaceAll(buf, []byte(nil)) + } + if len(buf) != 0 { + t.Fatalf("did not consume everything, left with: %q", buf) + } + if !foundFoo { + t.Fatalf("Expected interest for 'foo' as 'LS+ foo\\r\\n', got %q", buf) + } +} + +func TestLeafNodeAuthSystemEventNoCrash(t *testing.T) { + ca := createClusterWithName(t, "A", 1) + defer shutdownCluster(ca) + + opts := ca.opts[0] + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + leafSend := sendCommand(t, lc) + leafSend("LS+ foo\r\n") + checkInfoMsg(t, lc) +} + +func TestLeafNodeWithRouteAndGateway(t *testing.T) { + server.SetGatewaysSolicitDelay(50 * time.Millisecond) + defer server.ResetGatewaysSolicitDelay() + + ca := createClusterWithName(t, "A", 3) + defer shutdownCluster(ca) + cb := createClusterWithName(t, "B", 3, ca) + defer shutdownCluster(cb) + + // Create client on a server in cluster A + opts := ca.opts[0] + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + send("PING\r\n") + expect(pongRe) + + // Create a leaf node connection on a server in cluster B + opts = cb.opts[0] + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + // This is for our global responses since we are setting up GWs above. + leafSend, leafExpect := setupLeaf(t, lc, 8) + leafSend("PING\r\n") + leafExpect(pongRe) + + // Make sure we see interest graph propagation on the leaf node + // connection. This is required since leaf nodes only send data + // in the presence of interest. + send("SUB foo 1\r\nPING\r\n") + expect(pongRe) + leafExpect(lsubRe) + + send("SUB foo 2\r\nPING\r\n") + expect(pongRe) + expectNothing(t, lc) + + send("UNSUB 2\r\n") + expectNothing(t, lc) + send("UNSUB 1\r\n") + leafExpect(lunsubRe) + + // Now put it back and test msg flow. + send("SUB foo 1\r\nPING\r\n") + expect(pongRe) + leafExpect(lsubRe) + + leafSend("LMSG foo 2\r\nOK\r\n") + expectNothing(t, lc) + + matches := msgRe.FindAllSubmatch(expect(msgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkMsg(t, matches[0], "foo", "1", "", "2", "OK") + + // Now check reverse. + leafSend("LS+ bar\r\n") + expectNothing(t, lc) + leafSend("PING\r\n") + leafExpect(pongRe) + + send("PUB bar 2\r\nOK\r\n") + matches = lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkLmsg(t, matches[0], "bar", "", "2", "OK") +} + +// This will test that we propagate interest only mode after a leafnode +// has been established and a new server joins a remote cluster. +func TestLeafNodeWithGatewaysAndStaggeredStart(t *testing.T) { + ca := createClusterWithName(t, "A", 3) + defer shutdownCluster(ca) + + // Create the leafnode on a server in cluster A. + opts := ca.opts[0] + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + leafSend, leafExpect := setupLeaf(t, lc, 8) + leafSend("PING\r\n") + leafExpect(pongRe) + + // Now setup the cluster B. + cb := createClusterWithName(t, "B", 3, ca) + defer shutdownCluster(cb) + + // Create client on a server in cluster B + opts = cb.opts[0] + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + send("PING\r\n") + expect(pongRe) + + // Make sure we see interest graph propagation on the leaf node + // connection. This is required since leaf nodes only send data + // in the presence of interest. + send("SUB foo 1\r\nPING\r\n") + expect(pongRe) + leafExpect(lsubRe) +} + +// This will test that we propagate interest only mode after a leafnode +// has been established and a server is restarted.. +func TestLeafNodeWithGatewaysServerRestart(t *testing.T) { + ca := createClusterWithName(t, "A", 3) + defer shutdownCluster(ca) + + // Now setup the cluster B. + cb := createClusterWithName(t, "B", 3, ca) + defer shutdownCluster(cb) + + // Create the leafnode on a server in cluster B. + opts := cb.opts[1] + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + leafSend, leafExpect := setupLeaf(t, lc, 8) + leafSend("PING\r\n") + leafExpect(pongRe) + + // Create client on a server in cluster A + opts = ca.opts[1] + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + send("PING\r\n") + expect(pongRe) + + // Make sure we see interest graph propagation on the leaf node + // connection. This is required since leaf nodes only send data + // in the presence of interest. + send("SUB foo 1\r\nPING\r\n") + expect(pongRe) + leafExpect(lsubRe) + + // Close old leaf connection and simulate a reconnect. + lc.Close() + + // Shutdown and recreate B and the leafnode connection to it. + shutdownCluster(cb) + + // Create new cluster with longer solicit and don't wait for GW connect. + cb = createClusterEx(t, false, 500*time.Millisecond, false, "B", 1, ca) + defer shutdownCluster(cb) + + opts = cb.opts[0] + lc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + _, leafExpect = setupLeaf(t, lc, 8) + + // Now wait on GW solicit to fire + time.Sleep(500 * time.Millisecond) + + // We should see the interest for 'foo' here. + leafExpect(lsubRe) +} + +func TestLeafNodeLocalizedDQ(t *testing.T) { + s, opts := runLeafServer() + defer s.Shutdown() + + sl, slOpts := runSolicitLeafServer(opts) + defer sl.Shutdown() + + checkLeafNodeConnected(t, s) + + c := createClientConn(t, slOpts.Host, slOpts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + send("SUB foo bar 1\r\n") + send("SUB foo bar 2\r\n") + send("SUB foo bar 3\r\n") + send("SUB foo bar 4\r\n") + send("PING\r\n") + expect(pongRe) + + // Now create another client on the main leaf server. + sc := createClientConn(t, opts.Host, opts.Port) + defer sc.Close() + + sendL, expectL := setupConn(t, sc) + sendL("SUB foo bar 11\r\n") + sendL("SUB foo bar 12\r\n") + sendL("SUB foo bar 13\r\n") + sendL("SUB foo bar 14\r\n") + sendL("PING\r\n") + expectL(pongRe) + + for i := 0; i < 10; i++ { + send("PUB foo 2\r\nOK\r\n") + } + expectNothing(t, sc) + + matches := msgRe.FindAllSubmatch(expect(msgRe), -1) + if len(matches) != 10 { + t.Fatalf("Expected 10 msgs, got %d", len(matches)) + } + for i := 0; i < 10; i++ { + checkMsg(t, matches[i], "foo", "", "", "2", "OK") + } +} + +func TestLeafNodeBasicAuth(t *testing.T) { + content := ` + listen: "127.0.0.1:-1" + + leafnodes { + listen: "127.0.0.1:-1" + authorization { + user: "derek" + password: "s3cr3t!" + timeout: 2.2 + } + } + ` + conf := createConfFile(t, []byte(content)) + + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + // This should fail since we want u/p + setupConn(t, lc) + errBuf := expectResult(t, lc, errRe) + if !strings.Contains(string(errBuf), "Authorization Violation") { + t.Fatalf("Authentication Timeout response incorrect: %q", errBuf) + } + expectDisconnect(t, lc) + + // Try bad password as well. + lc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + // This should fail since we want u/p + setupConnWithUserPass(t, lc, "derek", "badpassword") + errBuf = expectResult(t, lc, errRe) + if !strings.Contains(string(errBuf), "Authorization Violation") { + t.Fatalf("Authentication Timeout response incorrect: %q", errBuf) + } + expectDisconnect(t, lc) + + // This one should work. + lc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + leafSend, leafExpect := setupConnWithUserPass(t, lc, "derek", "s3cr3t!") + leafExpect(infoRe) + leafExpect(lsubRe) + leafSend("PING\r\n") + expectResult(t, lc, pongRe) + + checkLeafNodeConnected(t, s) +} + +func runTLSSolicitLeafServer(lso *server.Options) (*server.Server, *server.Options) { + o := DefaultTestOptions + o.Port = -1 + rurl, _ := url.Parse(fmt.Sprintf("nats-leaf://%s:%d", lso.LeafNode.Host, lso.LeafNode.Port)) + remote := &server.RemoteLeafOpts{URLs: []*url.URL{rurl}} + remote.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12} + host, _, _ := net.SplitHostPort(lso.LeafNode.Host) + remote.TLSConfig.ServerName = host + remote.TLSConfig.InsecureSkipVerify = true + remote.Compression.Mode = server.CompressionOff + o.LeafNode.Remotes = []*server.RemoteLeafOpts{remote} + o.LeafNode.Compression.Mode = server.CompressionOff + return RunServer(&o), &o +} + +func TestLeafNodeTLS(t *testing.T) { + content := ` + listen: "127.0.0.1:-1" + + leafnodes { + listen: "127.0.0.1:-1" + tls { + cert_file: "./configs/certs/server-cert.pem" + key_file: "./configs/certs/server-key.pem" + timeout: 0.1 + } + } + ` + conf := createConfFile(t, []byte(content)) + + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + info := checkInfoMsg(t, lc) + if !info.TLSRequired { + t.Fatalf("Expected TLSRequired to be true") + } + if info.TLSVerify { + t.Fatalf("Expected TLSVerify to be false") + } + // We should get a disconnect here since we have not upgraded to TLS. + expectDisconnect(t, lc) + + // This should work ok. + sl, _ := runTLSSolicitLeafServer(opts) + defer sl.Shutdown() + + checkLeafNodeConnected(t, s) +} + +func TestLeafNodeTLSConnCloseEarly(t *testing.T) { + content := ` + listen: "127.0.0.1:-1" + + leafnodes { + listen: "127.0.0.1:-1" + tls { + cert_file: "./configs/certs/server-cert.pem" + key_file: "./configs/certs/server-key.pem" + timeout: 2.0 + } + } + ` + conf := createConfFile(t, []byte(content)) + + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + lc, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", opts.LeafNode.Port)) + if err != nil { + t.Fatalf("Unable to connect: %v", err) + } + // Then close right away + lc.Close() + + // Check server does not crash... + time.Sleep(250 * time.Millisecond) + if s.ID() == "" { + t.Fatalf("should not happen") + } +} + +type captureLeafNodeErrLogger struct { + dummyLogger + ch chan string +} + +func (c *captureLeafNodeErrLogger) Errorf(format string, v ...any) { + msg := fmt.Sprintf(format, v...) + select { + case c.ch <- msg: + default: + } +} + +func TestLeafNodeTLSMixIP(t *testing.T) { + content := ` + listen: "127.0.0.1:-1" + leafnodes { + listen: "127.0.0.1:-1" + authorization { + user: dlc + pass: monkey + } + tls { + cert_file: "./configs/certs/server-noip.pem" + key_file: "./configs/certs/server-key-noip.pem" + ca_file: "./configs/certs/ca.pem" + } + } + ` + conf := createConfFile(t, []byte(content)) + + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + slContent := ` + listen: "127.0.0.1:-1" + leafnodes { + reconnect: 1 + remotes: [ + { + url: [tls://127.0.0.1:%d, "tls://localhost:%d"] + tls { ca_file: "./configs/certs/ca.pem" } + } + ] + } + ` + slconf := createConfFile(t, []byte(fmt.Sprintf(slContent, opts.LeafNode.Port, opts.LeafNode.Port))) + + // This will fail but we want to make sure in the correct way, not with + // TLS issue because we used an IP for serverName. + sl, _ := RunServerWithConfig(slconf) + defer sl.Shutdown() + + ll := &captureLeafNodeErrLogger{ch: make(chan string, 2)} + sl.SetLogger(ll, false, false) + + // We may or may not get an error depending on timing. For the handshake bug + // we would always get it, so make sure if we have anything it is not that. + select { + case msg := <-ll.ch: + if strings.Contains(msg, "TLS handshake error") && strings.Contains(msg, "doesn't contain any IP SANs") { + t.Fatalf("Got bad error about TLS handshake") + } + default: + } +} + +func runLeafNodeOperatorServer(t *testing.T) (*server.Server, *server.Options, string) { + t.Helper() + content := ` + port: -1 + server_name: OP + operator = "./configs/nkeys/op.jwt" + resolver = MEMORY + listen: "127.0.0.1:-1" + leafnodes { + listen: "127.0.0.1:-1" + } + ` + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + return s, opts, conf +} + +func genCredsFile(t *testing.T, jwt string, seed []byte) string { + creds := ` + -----BEGIN NATS USER JWT----- + %s + ------END NATS USER JWT------ + + ************************* IMPORTANT ************************* + NKEY Seed printed below can be used to sign and prove identity. + NKEYs are sensitive and should be treated as secrets. + + -----BEGIN USER NKEY SEED----- + %s + ------END USER NKEY SEED------ + + ************************************************************* + ` + return createConfFile(t, []byte(strings.Replace(fmt.Sprintf(creds, jwt, seed), "\t\t", "", -1))) +} + +func runSolicitWithCredentials(t *testing.T, opts *server.Options, creds string) (*server.Server, *server.Options, string) { + content := ` + port: -1 + leafnodes { + remotes = [ + { + url: nats-leaf://127.0.0.1:%d + credentials: '%s' + } + ] + } + ` + config := fmt.Sprintf(content, opts.LeafNode.Port, creds) + conf := createConfFile(t, []byte(config)) + s, opts := RunServerWithConfig(conf) + return s, opts, conf +} + +func TestLeafNodeOperatorModel(t *testing.T) { + s, opts, _ := runLeafNodeOperatorServer(t) + defer s.Shutdown() + + // Make sure we get disconnected without proper credentials etc. + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + // This should fail since we want user jwt, signed nonce etc. + setupConn(t, lc) + errBuf := expectResult(t, lc, errRe) + if !strings.Contains(string(errBuf), "Authorization Violation") { + t.Fatalf("Authentication Timeout response incorrect: %q", errBuf) + } + expectDisconnect(t, lc) + + // Setup account and a user that will be used by the remote leaf node server. + // createAccount automatically registers with resolver etc.. + _, akp := createAccount(t, s) + kp, _ := nkeys.CreateUser() + pub, _ := kp.PublicKey() + nuc := jwt.NewUserClaims(pub) + ujwt, err := nuc.Encode(akp) + if err != nil { + t.Fatalf("Error generating user JWT: %v", err) + } + seed, _ := kp.Seed() + mycreds := genCredsFile(t, ujwt, seed) + + sl, _, _ := runSolicitWithCredentials(t, opts, mycreds) + defer sl.Shutdown() + + checkLeafNodeConnected(t, s) +} + +func TestLeafNodeUserPermsForConnection(t *testing.T) { + s, opts, _ := runLeafNodeOperatorServer(t) + defer s.Shutdown() + + // Setup account and a user that will be used by the remote leaf node server. + // createAccount automatically registers with resolver etc.. + acc, akp := createAccount(t, s) + kp, _ := nkeys.CreateUser() + pub, _ := kp.PublicKey() + nuc := jwt.NewUserClaims(pub) + nuc.Permissions.Pub.Allow.Add("foo.>") + nuc.Permissions.Pub.Allow.Add("baz.>") + nuc.Permissions.Sub.Allow.Add("foo.>") + // we would be immediately disconnected if that would not work + nuc.Permissions.Sub.Deny.Add("$SYS.>") + ujwt, err := nuc.Encode(akp) + if err != nil { + t.Fatalf("Error generating user JWT: %v", err) + } + seed, _ := kp.Seed() + mycreds := genCredsFile(t, ujwt, seed) + + content := ` + port: -1 + leafnodes { + remotes = [ + { + url: nats-leaf://127.0.0.1:%d + credentials: '%s' + deny_import: "foo.33" + deny_export: "foo.33" + } + ] + } + ` + config := fmt.Sprintf(content, opts.LeafNode.Port, mycreds) + lnconf := createConfFile(t, []byte(config)) + sl, _ := RunServerWithConfig(lnconf) + defer sl.Shutdown() + + checkLeafNodeConnected(t, s) + + // Create credentials for a normal unrestricted user that we will connect to the op server. + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(url, createUserCreds(t, s, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // Create a user on the leafnode server that solicited. + nc2, err := nats.Connect(sl.ClientURL()) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + // Make sure subscriptions properly do or do not make it to the hub. + nc2.SubscribeSync("bar") + checkNoSubInterest(t, s, acc.GetName(), "bar", 20*time.Millisecond) + // This one should. + nc2.SubscribeSync("foo.22") + checkSubInterest(t, s, acc.GetName(), "foo.22", 100*time.Millisecond) + + // Capture everything. + sub, _ := nc.SubscribeSync(">") + nc.Flush() + + checkSubInterest(t, sl, "$G", ">", 100*time.Millisecond) + // Now check local pubs are not forwarded. + nc2.Publish("baz.22", nil) + m, err := sub.NextMsg(1 * time.Second) + if err != nil || m.Subject != "baz.22" { + t.Fatalf("Expected to received this message") + } + nc2.Publish("bar.22", nil) + if _, err := sub.NextMsg(100 * time.Millisecond); err == nil { + t.Fatalf("Did not expect to receive this message") + } + + // Check local overrides work. + nc2.Publish("foo.33", nil) + if _, err := sub.NextMsg(100 * time.Millisecond); err == nil { + t.Fatalf("Did not expect to receive this message") + } + + // This would trigger the sub interest below. + sub.Unsubscribe() + nc.Flush() + + nc2.SubscribeSync("foo.33") + checkNoSubInterest(t, s, acc.GetName(), "foo.33", 20*time.Millisecond) +} + +func TestLeafNodeMultipleAccounts(t *testing.T) { + // So we will create a main server with two accounts. The remote server, acting as a leaf node, will simply have + // the $G global account and no auth. Make sure things work properly here. + s, opts, _ := runLeafNodeOperatorServer(t) + defer s.Shutdown() + + // Setup the two accounts for this server. + a, akp1 := createAccount(t, s) + kp1, _ := nkeys.CreateUser() + pub1, _ := kp1.PublicKey() + nuc1 := jwt.NewUserClaims(pub1) + ujwt1, err := nuc1.Encode(akp1) + if err != nil { + t.Fatalf("Error generating user JWT: %v", err) + } + + // Create second account. + createAccount(t, s) + + // Create the leaf node server using the first account. + seed, _ := kp1.Seed() + mycreds := genCredsFile(t, ujwt1, seed) + + sl, lopts, _ := runSolicitWithCredentials(t, opts, mycreds) + defer sl.Shutdown() + + checkLeafNodeConnected(t, s) + + // To connect to main server. + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + + nc1, err := nats.Connect(url, createUserCreds(t, s, akp1)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc1.Close() + + // This is a client connected to the leaf node with no auth, + // binding to account1 via leafnode connection. + // To connect to leafnode server. + lurl := fmt.Sprintf("nats://%s:%d", lopts.Host, lopts.Port) + ncl, err := nats.Connect(lurl) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncl.Close() + + lsub, _ := ncl.SubscribeSync("foo.test") + + // Wait for the subs to propagate. LDS + foo.test + checkSubInterest(t, s, a.GetName(), "foo.test", 2*time.Second) + + // Now send from nc1 with account 1, should be received by our leafnode subscriber. + nc1.Publish("foo.test", nil) + + _, err = lsub.NextMsg(1 * time.Second) + if err != nil { + t.Fatalf("Error during wait for next message: %s", err) + } +} + +func TestLeafNodeOperatorAndPermissions(t *testing.T) { + s, opts, conf := runLeafNodeOperatorServer(t) + defer os.Remove(conf) + defer s.Shutdown() + + acc, akp := createAccount(t, s) + kp, _ := nkeys.CreateUser() + pub, _ := kp.PublicKey() + + // Create SRV user, with no limitations + srvnuc := jwt.NewUserClaims(pub) + srvujwt, err := srvnuc.Encode(akp) + if err != nil { + t.Fatalf("Error generating user JWT: %v", err) + } + seed, _ := kp.Seed() + srvcreds := genCredsFile(t, srvujwt, seed) + defer os.Remove(srvcreds) + + // Create connection for SRV + srvnc, err := nats.Connect(s.ClientURL(), nats.UserCredentials(srvcreds)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer srvnc.Close() + + // Create on the server "s" a subscription on "*" and on "foo". + // We check that the subscription on "*" will be able to receive + // messages since LEAF has publish permissions on "foo", so msg + // should be received. + srvsubStar, err := srvnc.SubscribeSync("*") + if err != nil { + t.Fatalf("Error on subscribe: %v", err) + } + srvsubFoo, err := srvnc.SubscribeSync("foo") + if err != nil { + t.Fatalf("Error on subscribe: %v", err) + } + srvnc.Flush() + + // Create LEAF user, with pub perms on "foo" and sub perms on "bar" + leafnuc := jwt.NewUserClaims(pub) + leafnuc.Permissions.Pub.Allow.Add("foo") + leafnuc.Permissions.Sub.Allow.Add("bar") + leafnuc.Permissions.Sub.Allow.Add("baz") + leafujwt, err := leafnuc.Encode(akp) + if err != nil { + t.Fatalf("Error generating user JWT: %v", err) + } + leafcreds := genCredsFile(t, leafujwt, seed) + defer os.Remove(leafcreds) + + content := ` + port: -1 + server_name: LN + leafnodes { + remotes = [ + { + url: nats-leaf://127.0.0.1:%d + credentials: '%s' + } + ] + } + ` + config := fmt.Sprintf(content, opts.LeafNode.Port, leafcreds) + lnconf := createConfFile(t, []byte(config)) + defer os.Remove(lnconf) + sl, _ := RunServerWithConfig(lnconf) + defer sl.Shutdown() + + checkLeafNodeConnected(t, s) + + // Check that interest makes it to "sl" server. + // This helper does not check for wildcard interest... + checkSubInterest(t, sl, "$G", "foo", time.Second) + + // Create connection for LEAF and subscribe on "bar" + leafnc, err := nats.Connect(sl.ClientURL(), nats.UserCredentials(leafcreds)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer leafnc.Close() + + leafsub, err := leafnc.SubscribeSync("bar") + if err != nil { + t.Fatalf("Error on subscribe: %v", err) + } + + // To check that we can pull in 'baz'. + leafsubpwc, err := leafnc.SubscribeSync("*") + if err != nil { + t.Fatalf("Error on subscribe: %v", err) + } + leafnc.Flush() + + // Make sure the interest on "bar" from "sl" server makes it to the "s" server. + checkSubInterest(t, s, acc.GetName(), "bar", time.Second) + // Check for local interest too. + checkSubInterest(t, sl, "$G", "bar", time.Second) + + // Now that we know that "s" has received interest on "bar", create + // the sub on "bar" locally on "s" + srvsub, err := srvnc.SubscribeSync("bar") + if err != nil { + t.Fatalf("Error on subscribe: %v", err) + } + + srvnc.Publish("bar", []byte("hello")) + if _, err := srvsub.NextMsg(time.Second); err != nil { + t.Fatalf("SRV did not get message: %v", err) + } + if _, err := leafsub.NextMsg(time.Second); err != nil { + t.Fatalf("LEAF did not get message: %v", err) + } + if _, err := leafsubpwc.NextMsg(time.Second); err != nil { + t.Fatalf("LEAF did not get message: %v", err) + } + + // The leafnode has a sub on '*', that should pull in a publish to 'baz'. + srvnc.Publish("baz", []byte("hello")) + if _, err := leafsubpwc.NextMsg(time.Second); err != nil { + t.Fatalf("LEAF did not get message: %v", err) + } + + // User LEAF user on "sl" server, publish on "foo" + leafnc.Publish("foo", []byte("hello")) + // The user SRV on "s" receives it because the LN connection + // is allowed to publish on "foo". + if _, err := srvsubFoo.NextMsg(time.Second); err != nil { + t.Fatalf("SRV did not get message: %v", err) + } + // The wildcard subscription should get it too. + if _, err := srvsubStar.NextMsg(time.Second); err != nil { + t.Fatalf("SRV did not get message: %v", err) + } + + // However, even when using an unrestricted user connects to "sl" and + // publishes on "bar", the user SRV on "s" should not receive it because + // the LN connection is not allowed to publish (send msg over) on "bar". + nc, err := nats.Connect(sl.ClientURL(), nats.UserCredentials(srvcreds)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + nc.Publish("bar", []byte("should not received")) + if _, err := srvsub.NextMsg(250 * time.Millisecond); err == nil { + t.Fatal("Should not have received message") + } +} + +func TestLeafNodeSignerUser(t *testing.T) { + s, opts, _ := runLeafNodeOperatorServer(t) + defer s.Shutdown() + + // Setup the two accounts for this server. + _, akp1 := createAccount(t, s) + apk1, _ := akp1.PublicKey() + + // add a signing key to the account + akp2, err := nkeys.CreateAccount() + if err != nil { + t.Fatal(err) + } + apk2, _ := akp2.PublicKey() + + token, err := s.AccountResolver().Fetch(apk1) + if err != nil { + t.Fatal(err) + } + ac, err := jwt.DecodeAccountClaims(token) + if err != nil { + t.Fatal(err) + } + + ac.SigningKeys.Add(apk2) + okp, _ := nkeys.FromSeed(oSeed) + token, err = ac.Encode(okp) + if err != nil { + t.Fatal(err) + } + // update the resolver + account, _ := s.LookupAccount(apk1) + err = s.AccountResolver().Store(apk1, token) + if err != nil { + t.Fatal(err) + } + s.UpdateAccountClaims(account, ac) + + tt, err := s.AccountResolver().Fetch(apk1) + if err != nil { + t.Fatal(err) + } + ac2, err := jwt.DecodeAccountClaims(tt) + if err != nil { + t.Fatal(err) + } + if len(ac2.SigningKeys) != 1 { + t.Fatal("signing key is not added") + } + if _, ok := ac2.SigningKeys[apk2]; !ok { + t.Fatal("signing key is not added") + } + + // create an user signed by the signing key + kp1, _ := nkeys.CreateUser() + pub1, _ := kp1.PublicKey() + nuc1 := jwt.NewUserClaims(pub1) + nuc1.IssuerAccount = apk1 + ujwt1, err := nuc1.Encode(akp2) + if err != nil { + t.Fatalf("Error generating user JWT: %v", err) + } + + // Create the leaf node server using the first account. + seed, _ := kp1.Seed() + mycreds := genCredsFile(t, ujwt1, seed) + + sl, _, _ := runSolicitWithCredentials(t, opts, mycreds) + defer sl.Shutdown() + + checkLeafNodeConnected(t, s) +} + +func TestLeafNodeExportsImports(t *testing.T) { + // So we will create a main server with two accounts. The remote server, acting as a leaf node, will simply have + // the $G global account and no auth. Make sure things work properly here. + s, opts, _ := runLeafNodeOperatorServer(t) + defer s.Shutdown() + + // Setup the two accounts for this server. + okp, _ := nkeys.FromSeed(oSeed) + + // Create second account with exports + acc2, akp2 := createAccount(t, s) + akp2Pub, _ := akp2.PublicKey() + akp2AC := jwt.NewAccountClaims(akp2Pub) + streamExport := &jwt.Export{Subject: "foo.stream", Type: jwt.Stream} + serviceExport := &jwt.Export{Subject: "req.echo", Type: jwt.Service} + akp2AC.Exports.Add(streamExport, serviceExport) + akp2ACJWT, err := akp2AC.Encode(okp) + if err != nil { + t.Fatalf("Error generating account JWT: %v", err) + } + + if err := s.AccountResolver().Store(akp2Pub, akp2ACJWT); err != nil { + t.Fatalf("Account Resolver returned an error: %v", err) + } + s.UpdateAccountClaims(acc2, akp2AC) + + // Now create the first account and add on the imports. This will be what is used in the leafnode. + acc1, akp1 := createAccount(t, s) + akp1Pub, _ := akp1.PublicKey() + akp1AC := jwt.NewAccountClaims(akp1Pub) + streamImport := &jwt.Import{Account: akp2Pub, Subject: "foo.stream", To: "import", Type: jwt.Stream} + serviceImport := &jwt.Import{Account: akp2Pub, Subject: "import.request", To: "req.echo", Type: jwt.Service} + akp1AC.Imports.Add(streamImport, serviceImport) + akp1ACJWT, err := akp1AC.Encode(okp) + if err != nil { + t.Fatalf("Error generating account JWT: %v", err) + } + if err := s.AccountResolver().Store(akp1Pub, akp1ACJWT); err != nil { + t.Fatalf("Account Resolver returned an error: %v", err) + } + s.UpdateAccountClaims(acc1, akp1AC) + + // Create the user will we use to connect the leafnode. + kp1, _ := nkeys.CreateUser() + pub1, _ := kp1.PublicKey() + nuc1 := jwt.NewUserClaims(pub1) + ujwt1, err := nuc1.Encode(akp1) + if err != nil { + t.Fatalf("Error generating user JWT: %v", err) + } + + // Create the leaf node server using the first account. + seed, _ := kp1.Seed() + mycreds := genCredsFile(t, ujwt1, seed) + + sl, lopts, _ := runSolicitWithCredentials(t, opts, mycreds) + defer sl.Shutdown() + + checkLeafNodeConnected(t, s) + + // To connect to main server. + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + + // Imported + nc1, err := nats.Connect(url, createUserCreds(t, s, akp1)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc1.Close() + + // Exported + nc2, err := nats.Connect(url, createUserCreds(t, s, akp2)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + // Leaf node connection. + lurl := fmt.Sprintf("nats://%s:%d", lopts.Host, lopts.Port) + ncl, err := nats.Connect(lurl) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncl.Close() + + // So everything should be setup here. So let's test streams first. + lsub, _ := ncl.SubscribeSync("import.foo.stream") + + // Wait for all subs to propagate. + checkSubInterest(t, s, acc1.GetName(), "import.foo.stream", time.Second) + + // Pub to other account with export on original subject. + nc2.Publish("foo.stream", nil) + + _, err = lsub.NextMsg(1 * time.Second) + if err != nil { + t.Fatalf("Error during wait for next message: %s", err) + } + + // Services + // Create listener on nc2 + nc2.Subscribe("req.echo", func(msg *nats.Msg) { + nc2.Publish(msg.Reply, []byte("WORKED")) + }) + nc2.Flush() + + // Now send the request on the leaf node client. + if _, err := ncl.Request("import.request", []byte("fingers crossed"), 500*time.Millisecond); err != nil { + t.Fatalf("Did not receive response: %v", err) + } +} + +func TestLeafNodeExportImportComplexSetup(t *testing.T) { + content := ` + port: -1 + operator = "./configs/nkeys/op.jwt" + resolver = MEMORY + cluster { + port: -1 + name: xyz + } + leafnodes { + listen: "127.0.0.1:-1" + } + ` + conf := createConfFile(t, []byte(content)) + s1, s1Opts := RunServerWithConfig(conf) + defer s1.Shutdown() + + content = fmt.Sprintf(` + port: -1 + operator = "./configs/nkeys/op.jwt" + resolver = MEMORY + cluster { + port: -1 + name: xyz + routes: ["nats://%s:%d"] + } + leafnodes { + listen: "127.0.0.1:-1" + } + `, s1Opts.Cluster.Host, s1Opts.Cluster.Port) + conf = createConfFile(t, []byte(content)) + s2, s2Opts := RunServerWithConfig(conf) + defer s2.Shutdown() + + // Setup the two accounts for this server. + okp, _ := nkeys.FromSeed(oSeed) + + // Create second account with exports + acc2, akp2 := createAccount(t, s1) + akp2Pub, _ := akp2.PublicKey() + akp2AC := jwt.NewAccountClaims(akp2Pub) + streamExport := &jwt.Export{Subject: "foo.stream", Type: jwt.Stream} + serviceExport := &jwt.Export{Subject: "req.echo", Type: jwt.Service} + akp2AC.Exports.Add(streamExport, serviceExport) + akp2ACJWT, err := akp2AC.Encode(okp) + if err != nil { + t.Fatalf("Error generating account JWT: %v", err) + } + + if err := s1.AccountResolver().Store(akp2Pub, akp2ACJWT); err != nil { + t.Fatalf("Account Resolver returned an error: %v", err) + } + s1.UpdateAccountClaims(acc2, akp2AC) + + // Now create the first account and add on the imports. This will be what is used in the leafnode. + acc1, akp1 := createAccount(t, s1) + akp1Pub, _ := akp1.PublicKey() + akp1AC := jwt.NewAccountClaims(akp1Pub) + streamImport := &jwt.Import{Account: akp2Pub, Subject: "foo.stream", To: "import", Type: jwt.Stream} + serviceImport := &jwt.Import{Account: akp2Pub, Subject: "import.request", To: "req.echo", Type: jwt.Service} + akp1AC.Imports.Add(streamImport, serviceImport) + akp1ACJWT, err := akp1AC.Encode(okp) + if err != nil { + t.Fatalf("Error generating account JWT: %v", err) + } + if err := s1.AccountResolver().Store(akp1Pub, akp1ACJWT); err != nil { + t.Fatalf("Account Resolver returned an error: %v", err) + } + s1.UpdateAccountClaims(acc1, akp1AC) + + if err := s2.AccountResolver().Store(akp2Pub, akp2ACJWT); err != nil { + t.Fatalf("Account Resolver returned an error: %v", err) + } + // Just make sure that account object registered in S2 is not acc2 + if a, err := s2.LookupAccount(acc2.Name); err != nil || a == acc2 { + t.Fatalf("Lookup account error: %v - accounts are same: %v", err, a == acc2) + } + + if err := s2.AccountResolver().Store(akp1Pub, akp1ACJWT); err != nil { + t.Fatalf("Account Resolver returned an error: %v", err) + } + // Just make sure that account object registered in S2 is not acc1 + if a, err := s2.LookupAccount(acc1.Name); err != nil || a == acc1 { + t.Fatalf("Lookup account error: %v - accounts are same: %v", err, a == acc1) + } + + // Create the user will we use to connect the leafnode. + kp1, _ := nkeys.CreateUser() + pub1, _ := kp1.PublicKey() + nuc1 := jwt.NewUserClaims(pub1) + ujwt1, err := nuc1.Encode(akp1) + if err != nil { + t.Fatalf("Error generating user JWT: %v", err) + } + + // Create the leaf node server using the first account. + seed, _ := kp1.Seed() + mycreds := genCredsFile(t, ujwt1, seed) + + sl, lopts, _ := runSolicitWithCredentials(t, s1Opts, mycreds) + defer sl.Shutdown() + + checkLeafNodeConnected(t, s1) + + // Url to server s2 + s2URL := fmt.Sprintf("nats://%s:%d", s2Opts.Host, s2Opts.Port) + + // Imported + nc1, err := nats.Connect(s2URL, createUserCreds(t, s2, akp1)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc1.Close() + + // Exported + nc2, err := nats.Connect(s2URL, createUserCreds(t, s2, akp2)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + // Leaf node connection. + lurl := fmt.Sprintf("nats://%s:%d", lopts.Host, lopts.Port) + ncl, err := nats.Connect(lurl) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncl.Close() + + // So everything should be setup here. So let's test streams first. + lsub, _ := ncl.SubscribeSync("import.foo.stream") + + // Wait for the sub to propagate to s2. LDS + subject above. + checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { + if acc1.RoutedSubs() != 6 { + return fmt.Errorf("Still no routed subscription: %d", acc1.RoutedSubs()) + } + return nil + }) + + // Pub to other account with export on original subject. + nc2.Publish("foo.stream", nil) + + if _, err = lsub.NextMsg(1 * time.Second); err != nil { + t.Fatalf("Did not receive stream message: %s", err) + } + + // Services + // Create listener on nc2 (which connects to s2) + gotIt := int32(0) + nc2.Subscribe("req.echo", func(msg *nats.Msg) { + atomic.AddInt32(&gotIt, 1) + nc2.Publish(msg.Reply, []byte("WORKED")) + }) + nc2.Flush() + + // Wait for it to make it across. + time.Sleep(250 * time.Millisecond) + + // Now send the request on the leaf node client. + if _, err := ncl.Request("import.request", []byte("fingers crossed"), 5500*time.Millisecond); err != nil { + if atomic.LoadInt32(&gotIt) == 0 { + t.Fatalf("Request was not received") + } + t.Fatalf("Did not receive response: %v", err) + } +} + +func TestLeafNodeInfoURLs(t *testing.T) { + for _, test := range []struct { + name string + useAdvertise bool + }{ + { + "without advertise", + false, + }, + { + "with advertise", + true, + }, + } { + t.Run(test.name, func(t *testing.T) { + opts := testDefaultOptionsForLeafNodes() + opts.Cluster.Name = "A" + opts.Cluster.Port = -1 + opts.LeafNode.Host = "127.0.0.1" + if test.useAdvertise { + opts.LeafNode.Advertise = "me:1" + } + s1 := RunServer(opts) + defer s1.Shutdown() + + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + info := checkInfoMsg(t, lc) + if sz := len(info.LeafNodeURLs); sz != 1 { + t.Fatalf("Expected LeafNodeURLs array to be size 1, got %v", sz) + } + var s1LNURL string + if test.useAdvertise { + s1LNURL = "me:1" + } else { + s1LNURL = net.JoinHostPort(opts.LeafNode.Host, strconv.Itoa(opts.LeafNode.Port)) + } + if url := info.LeafNodeURLs[0]; url != s1LNURL { + t.Fatalf("Expected URL to be %s, got %s", s1LNURL, url) + } + lc.Close() + + opts2 := testDefaultOptionsForLeafNodes() + opts2.Cluster.Name = "A" + opts2.Cluster.Port = -1 + opts2.Routes = server.RoutesFromStr(fmt.Sprintf("nats://%s:%d", opts.Cluster.Host, opts.Cluster.Port)) + opts2.LeafNode.Host = "127.0.0.1" + if test.useAdvertise { + opts2.LeafNode.Advertise = "me:2" + } + s2 := RunServer(opts2) + defer s2.Shutdown() + + checkClusterFormed(t, s1, s2) + + lc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + info = checkInfoMsg(t, lc) + if sz := len(info.LeafNodeURLs); sz != 2 { + t.Fatalf("Expected LeafNodeURLs array to be size 2, got %v", sz) + } + var s2LNURL string + if test.useAdvertise { + s2LNURL = "me:2" + } else { + s2LNURL = net.JoinHostPort(opts2.LeafNode.Host, strconv.Itoa(opts2.LeafNode.Port)) + } + var ok [2]int + for _, url := range info.LeafNodeURLs { + if url == s1LNURL { + ok[0]++ + } else if url == s2LNURL { + ok[1]++ + } + } + for i, res := range ok { + if res != 1 { + t.Fatalf("URL from server %v was found %v times", i+1, res) + } + } + lc.Close() + + // Remove s2, and wait for route to be lost on s1. + s2.Shutdown() + checkNumRoutes(t, s1, 0) + + // Now check that s1 returns only itself in the URLs array. + lc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + info = checkInfoMsg(t, lc) + if sz := len(info.LeafNodeURLs); sz != 1 { + t.Fatalf("Expected LeafNodeURLs array to be size 1, got %v", sz) + } + if url := info.LeafNodeURLs[0]; url != s1LNURL { + t.Fatalf("Expected URL to be %s, got %s", s1LNURL, url) + } + lc.Close() + + s1.Shutdown() + + // Now we need a configuration where both s1 and s2 have a route + // to each other, so we need explicit configuration. We are trying + // to get S1->S2 and S2->S1 so one of the route is dropped. This + // should not affect the number of URLs reported in INFO. + + opts.Cluster.Port = 5223 + opts.Routes = server.RoutesFromStr(fmt.Sprintf("nats://%s:5224", opts2.Host)) + s1, _ = server.NewServer(opts) + defer s1.Shutdown() + + opts2.Cluster.Port = 5224 + opts2.Routes = server.RoutesFromStr(fmt.Sprintf("nats://%s:5223", opts.Host)) + s2, _ = server.NewServer(opts2) + defer s2.Shutdown() + + // Start this way to increase chance of having the two connect + // to each other at the same time. This will cause one of the + // route to be dropped. + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + s1.Start() + wg.Done() + }() + go func() { + s2.Start() + wg.Done() + }() + + checkClusterFormed(t, s1, s2) + + lc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + info = checkInfoMsg(t, lc) + if sz := len(info.LeafNodeURLs); sz != 2 { + t.Fatalf("Expected LeafNodeURLs array to be size 2, got %v", sz) + } + ok[0], ok[1] = 0, 0 + for _, url := range info.LeafNodeURLs { + if url == s1LNURL { + ok[0]++ + } else if url == s2LNURL { + ok[1]++ + } + } + for i, res := range ok { + if res != 1 { + t.Fatalf("URL from server %v was found %v times", i+1, res) + } + } + }) + } +} + +func TestLeafNodeFailover(t *testing.T) { + server.SetGatewaysSolicitDelay(50 * time.Millisecond) + defer server.ResetGatewaysSolicitDelay() + + ca := createClusterWithName(t, "A", 2) + defer shutdownCluster(ca) + + cb := createClusterWithName(t, "B", 1, ca) + defer shutdownCluster(cb) + + // Start a server that creates LeafNode connection to first + // server in cluster A. + s, opts := runSolicitLeafServer(ca.opts[0]) + defer s.Shutdown() + + // Shutdown that server on A. + ca.servers[0].Shutdown() + + // Make sure that s reconnects its LN connection + checkLeafNodeConnected(t, ca.servers[1]) + + // Verify that LeafNode info protocol is sent to the server `s` + // with list of new servers. To do that, we will restart + // ca[0] but with a different LN listen port. + ca.opts[0].Port = -1 + ca.opts[0].Cluster.Port = -1 + ca.opts[0].Routes = server.RoutesFromStr(fmt.Sprintf("nats://%s:%d", ca.opts[1].Cluster.Host, ca.opts[1].Cluster.Port)) + ca.opts[0].LeafNode.Port = -1 + newa0 := RunServer(ca.opts[0]) + defer newa0.Shutdown() + + checkClusterFormed(t, newa0, ca.servers[1]) + + // Shutdown the server the LN is currently connected to. It should + // reconnect to newa0. + ca.servers[1].Shutdown() + checkLeafNodeConnected(t, newa0) + + // Now shutdown newa0 and make sure `s` does not reconnect + // to server in gateway. + newa0.Shutdown() + + // Wait for more than the reconnect attempts. + time.Sleep(opts.LeafNode.ReconnectInterval + 50*time.Millisecond) + + checkLeafNodeConnections(t, cb.servers[0], 0) +} + +func TestLeafNodeAdvertise(t *testing.T) { + // Create a dummy listener which will we use for the advertise address. + // We will then stop the server and the test will be a success if + // this listener accepts a connection. + ch := make(chan struct{}, 1) + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Error starting listener: %v", err) + } + defer l.Close() + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + c, _ := l.Accept() + if c != nil { + c.Close() + } + l.Close() + ch <- struct{}{} + }() + + port := l.Addr().(*net.TCPAddr).Port + + o2 := testDefaultOptionsForLeafNodes() + o2.LeafNode.Advertise = fmt.Sprintf("127.0.0.1:%d", port) + o2.Cluster.Name = "A" + o2.Cluster.Port = -1 + s2 := RunServer(o2) + defer s2.Shutdown() + + o1 := testDefaultOptionsForLeafNodes() + o1.Cluster.Name = "A" + o1.Cluster.Port = -1 + o1.Routes = server.RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o2.Cluster.Port)) + s1 := RunServer(o1) + defer s1.Shutdown() + + checkClusterFormed(t, s1, s2) + + // Start a server that connects to s1. It should be made aware + // of s2 (and its advertise address). + s, _ := runSolicitLeafServer(o1) + defer s.Shutdown() + + // Wait for leaf node connection to be established on s1. + checkLeafNodeConnected(t, s1) + + // Shutdown s1. The listener that we created should be the one + // receiving the connection from s. + s1.Shutdown() + + select { + case <-ch: + case <-time.After(5 * time.Second): + t.Fatalf("Server did not reconnect to advertised address") + } + wg.Wait() +} + +func TestLeafNodeConnectionLimitsSingleServer(t *testing.T) { + s, opts, _ := runLeafNodeOperatorServer(t) + defer s.Shutdown() + + // Setup account and a user that will be used by the remote leaf node server. + // createAccount automatically registers with resolver etc.. + acc, akp := createAccount(t, s) + + // Now update with limits for lead node connections. + const maxleafs = 2 + + apub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(apub) + nac.Limits.LeafNodeConn = maxleafs + s.UpdateAccountClaims(acc, nac) + + // Make sure we have the limits updated in acc. + if mleafs := acc.MaxActiveLeafNodes(); mleafs != maxleafs { + t.Fatalf("Expected to have max leafnodes of %d, got %d", maxleafs, mleafs) + } + + // Create the user credentials for the leadnode connection. + kp, _ := nkeys.CreateUser() + pub, _ := kp.PublicKey() + nuc := jwt.NewUserClaims(pub) + ujwt, err := nuc.Encode(akp) + if err != nil { + t.Fatalf("Error generating user JWT: %v", err) + } + seed, _ := kp.Seed() + mycreds := genCredsFile(t, ujwt, seed) + + checkAccConnectionCounts := func(t *testing.T, expected int) { + t.Helper() + checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { + // Make sure we are accounting properly here. + if nln := acc.NumLeafNodes(); nln != expected { + return fmt.Errorf("expected %v leaf node, got %d", expected, nln) + } + // clients and leafnodes counted together. + if nc := acc.NumConnections(); nc != expected { + return fmt.Errorf("expected %v for total connections, got %d", expected, nc) + } + return nil + }) + } + + checkAccNLF := func(n int) { + t.Helper() + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if nln := acc.NumLeafNodes(); nln != n { + return fmt.Errorf("Expected %d leaf node, got %d", n, nln) + } + return nil + }) + } + + sl, _, _ := runSolicitWithCredentials(t, opts, mycreds) + defer sl.Shutdown() + + checkLeafNodeConnections(t, s, 1) + + // Make sure we are accounting properly here. + checkAccNLF(1) + // clients and leafnodes counted together. + if nc := acc.NumConnections(); nc != 1 { + t.Fatalf("Expected 1 for total connections, got %d", nc) + } + + s2, _, _ := runSolicitWithCredentials(t, opts, mycreds) + defer s2.Shutdown() + checkLeafNodeConnections(t, s, 2) + checkAccConnectionCounts(t, 2) + + s2.Shutdown() + checkLeafNodeConnections(t, s, 1) + checkAccConnectionCounts(t, 1) + + // Make sure we are accounting properly here. + checkAccNLF(1) + // clients and leafnodes counted together. + if nc := acc.NumConnections(); nc != 1 { + t.Fatalf("Expected 1 for total connections, got %d", nc) + } + + // Now add back the second one as #3. + s3, _, _ := runSolicitWithCredentials(t, opts, mycreds) + defer s3.Shutdown() + checkLeafNodeConnections(t, s, 2) + + checkAccConnectionCounts(t, 2) + + // Once we are here we should not be able to create anymore. Limit == 2. + s4, _, _ := runSolicitWithCredentials(t, opts, mycreds) + defer s4.Shutdown() + + checkAccConnectionCounts(t, 2) + + // Make sure s4 has 0 still. We need checkFor because it is possible + // that when we check we have actually the connection registered for + // a short period before it is closed due to limit. + checkFor(t, time.Second, 15*time.Millisecond, func() error { + if nln := s4.NumLeafNodes(); nln != 0 { + return fmt.Errorf("Expected no leafnodes accounted for in s4, got %d", nln) + } + return nil + }) + + // Make sure this is still 2. + checkLeafNodeConnections(t, s, 2) +} + +func TestLeafNodeConnectionLimitsCluster(t *testing.T) { + content := ` + port: -1 + operator = "./configs/nkeys/op.jwt" + system_account = "AD2VB6C25DQPEUUQ7KJBUFX2J4ZNVBPOHSCBISC7VFZXVWXZA7VASQZG" + resolver = MEMORY + cluster { + port: -1 + name: xyz + } + leafnodes { + listen: "127.0.0.1:-1" + } + resolver_preload = { + AD2VB6C25DQPEUUQ7KJBUFX2J4ZNVBPOHSCBISC7VFZXVWXZA7VASQZG : "eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJqdGkiOiJDSzU1UERKSUlTWU5QWkhLSUpMVURVVTdJT1dINlM3UkE0RUc2TTVGVUQzUEdGQ1RWWlJRIiwiaWF0IjoxNTQzOTU4NjU4LCJpc3MiOiJPQ0FUMzNNVFZVMlZVT0lNR05HVU5YSjY2QUgyUkxTREFGM01VQkNZQVk1UU1JTDY1TlFNNlhRRyIsInN1YiI6IkFEMlZCNkMyNURRUEVVVVE3S0pCVUZYMko0Wk5WQlBPSFNDQklTQzdWRlpYVldYWkE3VkFTUVpHIiwidHlwZSI6ImFjY291bnQiLCJuYXRzIjp7ImxpbWl0cyI6e319fQ.7m1fysYUsBw15Lj88YmYoHxOI4HlOzu6qgP8Zg-1q9mQXUURijuDGVZrtb7gFYRlo-nG9xZyd2ZTRpMA-b0xCQ" + } + ` + conf := createConfFile(t, []byte(content)) + s1, s1Opts := RunServerWithConfig(conf) + defer s1.Shutdown() + + content = fmt.Sprintf(` + port: -1 + operator = "./configs/nkeys/op.jwt" + system_account = "AD2VB6C25DQPEUUQ7KJBUFX2J4ZNVBPOHSCBISC7VFZXVWXZA7VASQZG" + resolver = MEMORY + cluster { + port: -1 + name: xyz + routes: ["nats://%s:%d"] + } + leafnodes { + listen: "127.0.0.1:-1" + } + resolver_preload = { + AD2VB6C25DQPEUUQ7KJBUFX2J4ZNVBPOHSCBISC7VFZXVWXZA7VASQZG : "eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJqdGkiOiJDSzU1UERKSUlTWU5QWkhLSUpMVURVVTdJT1dINlM3UkE0RUc2TTVGVUQzUEdGQ1RWWlJRIiwiaWF0IjoxNTQzOTU4NjU4LCJpc3MiOiJPQ0FUMzNNVFZVMlZVT0lNR05HVU5YSjY2QUgyUkxTREFGM01VQkNZQVk1UU1JTDY1TlFNNlhRRyIsInN1YiI6IkFEMlZCNkMyNURRUEVVVVE3S0pCVUZYMko0Wk5WQlBPSFNDQklTQzdWRlpYVldYWkE3VkFTUVpHIiwidHlwZSI6ImFjY291bnQiLCJuYXRzIjp7ImxpbWl0cyI6e319fQ.7m1fysYUsBw15Lj88YmYoHxOI4HlOzu6qgP8Zg-1q9mQXUURijuDGVZrtb7gFYRlo-nG9xZyd2ZTRpMA-b0xCQ" + } + `, s1Opts.Cluster.Host, s1Opts.Cluster.Port) + conf = createConfFile(t, []byte(content)) + s2, s2Opts := RunServerWithConfig(conf) + defer s2.Shutdown() + + // Setup the two accounts for this server. + okp, _ := nkeys.FromSeed(oSeed) + + // Setup account and a user that will be used by the remote leaf node server. + // createAccount automatically registers with resolver etc.. + acc, akp := createAccount(t, s1) + + // Now update with limits for lead node connections. + const maxleafs = 10 + + apub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(apub) + nac.Limits.LeafNodeConn = maxleafs + + ajwt, err := nac.Encode(okp) + if err != nil { + t.Fatalf("Error generating account JWT: %v", err) + } + if err := s1.AccountResolver().Store(apub, ajwt); err != nil { + t.Fatalf("Account Resolver returned an error: %v", err) + } + s1.UpdateAccountClaims(acc, nac) + + if err := s2.AccountResolver().Store(apub, ajwt); err != nil { + t.Fatalf("Account Resolver returned an error: %v", err) + } + // Make sure that account object registered in S2 is not acc2 + acc2, err := s2.LookupAccount(acc.Name) + if err != nil || acc == acc2 { + t.Fatalf("Lookup account error: %v - accounts are same: %v", err, acc == acc2) + } + + // Create the user credentials for the leadnode connection. + kp, _ := nkeys.CreateUser() + pub, _ := kp.PublicKey() + nuc := jwt.NewUserClaims(pub) + ujwt, err := nuc.Encode(akp) + if err != nil { + t.Fatalf("Error generating user JWT: %v", err) + } + seed, _ := kp.Seed() + mycreds := genCredsFile(t, ujwt, seed) + + loop := maxleafs / 2 + + // Now create maxleafs/2 leaf node servers on each operator server. + for i := 0; i < loop; i++ { + sl1, _, _ := runSolicitWithCredentials(t, s1Opts, mycreds) + defer sl1.Shutdown() + + sl2, _, _ := runSolicitWithCredentials(t, s2Opts, mycreds) + defer sl2.Shutdown() + } + + checkLeafNodeConnections(t, s1, loop) + checkLeafNodeConnections(t, s2, loop) + + // Now check that we have the remotes registered. This will prove we are sending + // and processing the leaf node connect events properly etc. + checkAccLFCount := func(acc *server.Account, remote bool, n int) { + t.Helper() + checkFor(t, time.Second, 10*time.Millisecond, func() error { + str := "" + var nln int + if remote { + nln = acc.NumRemoteLeafNodes() + str = "remote" + } else { + nln = acc.NumLeafNodes() + } + if nln != n { + return fmt.Errorf("number of expected %sleaf nodes is %v, got %v", str, n, nln) + } + return nil + }) + } + checkAccLFCount(acc, true, loop) + checkAccLFCount(acc2, true, loop) + + // Now that we are here we should not be allowed anymore leaf nodes. + l, _, _ := runSolicitWithCredentials(t, s1Opts, mycreds) + defer l.Shutdown() + + checkAccLFCount(acc, false, maxleafs) + // Should still be at loop size. + checkLeafNodeConnections(t, s1, loop) + + l, _, _ = runSolicitWithCredentials(t, s2Opts, mycreds) + defer l.Shutdown() + checkAccLFCount(acc2, false, maxleafs) + // Should still be at loop size. + checkLeafNodeConnections(t, s2, loop) +} + +func TestLeafNodeSwitchGatewayToInterestModeOnly(t *testing.T) { + server.SetGatewaysSolicitDelay(50 * time.Millisecond) + defer server.ResetGatewaysSolicitDelay() + + ca := createClusterWithName(t, "A", 3) + defer shutdownCluster(ca) + cb := createClusterWithName(t, "B", 3, ca) + defer shutdownCluster(cb) + + // Create client on a server in cluster A + opts := ca.opts[0] + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + send("PING\r\n") + expect(pongRe) + + // Send a message from this client on "foo" so that B + // registers a no-interest for account "$G" + send("PUB foo 2\r\nok\r\nPING\r\n") + expect(pongRe) + + // Create a leaf node connection on a server in cluster B + opts = cb.opts[0] + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + // This is for our global responses since we are setting up GWs above. + leafSend, leafExpect := setupLeaf(t, lc, 8) + leafSend("PING\r\n") + leafExpect(pongRe) +} + +// The MSG proto for routes and gateways is RMSG, and we have an +// optimization that a scratch buffer has RMSG and when doing a +// client we just start at scratch[1]. For leaf nodes its LMSG and we +// rewrite scratch[0], but never reset it which causes protocol +// errors when used with routes or gateways after use to send +// to a leafnode. +// We will create a server with a leafnode connection and a route +// and a gateway connection. + +// route connections to simulate. +func TestLeafNodeResetsMSGProto(t *testing.T) { + server.GatewayDoNotForceInterestOnlyMode(true) + defer server.GatewayDoNotForceInterestOnlyMode(false) + + opts := testDefaultOptionsForLeafNodes() + opts.Cluster.Name = "xyz" + opts.Cluster.Host = opts.Host + opts.Cluster.Port = -1 + opts.Gateway.Name = "xyz" + opts.Gateway.Host = opts.Host + opts.Gateway.Port = -1 + opts.Accounts = []*server.Account{server.NewAccount("$SYS")} + opts.SystemAccount = "$SYS" + + s := RunServer(opts) + defer s.Shutdown() + + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + leafSend, leafExpect := setupConn(t, lc) + + // To avoid possible INFO when switching to interest mode only, + // delay start of gateway. + time.Sleep(500 * time.Millisecond) + + gw := createGatewayConn(t, opts.Gateway.Host, opts.Gateway.Port) + defer gw.Close() + + gwSend, gwExpect := setupGatewayConn(t, gw, "A", "xyz") + gwSend("PING\r\n") + gwExpect(pongRe) + + // This is for our global responses since we are setting up GWs above. + leafExpect(lsubRe) + + // Now setup interest in the leaf node for 'foo'. + leafSend("LS+ foo\r\nPING\r\n") + leafExpect(pongRe) + + // Send msg from the gateway. + gwSend("RMSG $G foo 2\r\nok\r\nPING\r\n") + gwExpect(pongRe) + + leafExpect(lmsgRe) + + // At this point the gw inside our main server's scratch buffer is LMSG. When we do + // same with a connected route with interest it should fail. + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + checkInfoMsg(t, rc) + routeSend, routeExpect := setupRouteEx(t, rc, opts, "RC") + + routeSend("RS+ $G foo\r\nPING\r\n") + routeExpect(pongRe) + + // This is for the route interest we just created. + leafExpect(lsubRe) + + // Send msg from the gateway. + gwSend("RMSG $G foo 2\r\nok\r\nPING\r\n") + gwExpect(pongRe) + + leafExpect(lmsgRe) + + // Now make sure we get it on route. This will fail with the proto bug. + routeExpect(rmsgRe) +} + +// We need to make sure that as a remote server we also send our local subs on connect. +func TestLeafNodeSendsRemoteSubsOnConnect(t *testing.T) { + s, opts := runLeafServer() + defer s.Shutdown() + + sl, slOpts := runSolicitLeafServer(opts) + defer sl.Shutdown() + + checkLeafNodeConnected(t, s) + s.Shutdown() + + c := createClientConn(t, slOpts.Host, slOpts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + send("SUB foo 1\r\n") + send("PING\r\n") + expect(pongRe) + + // Need to restart it on the same port. + s, _ = runLeafServerOnPort(opts.LeafNode.Port) + defer s.Shutdown() + checkLeafNodeConnected(t, s) + + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + setupLeaf(t, lc, 3) +} + +func TestLeafNodeServiceImportLikeNGS(t *testing.T) { + gwSolicit := 10 * time.Millisecond + ca := createClusterEx(t, true, gwSolicit, true, "A", 3) + defer shutdownCluster(ca) + cb := createClusterEx(t, true, gwSolicit, true, "B", 3, ca) + defer shutdownCluster(cb) + + // Hang a responder off of cluster A. + opts := ca.opts[0] + url := fmt.Sprintf("nats://ngs:pass@%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(url) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // Create a queue subscriber to send results + nc.QueueSubscribe("ngs.usage.*", "ngs", func(m *nats.Msg) { + m.Respond([]byte("22")) + }) + nc.Flush() + + // Now create a leafnode server on B. + opts = cb.opts[1] + sl, slOpts := runSolicitLeafServerToURL(fmt.Sprintf("nats-leaf://dlc:pass@%s:%d", opts.LeafNode.Host, opts.LeafNode.Port)) + defer sl.Shutdown() + + checkLeafNodeConnected(t, sl) + + // Create a normal direct connect client on B. + url = fmt.Sprintf("nats://dlc:pass@%s:%d", opts.Host, opts.Port) + nc2, err := nats.Connect(url) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + if _, err := nc2.Request("ngs.usage", []byte("fingers crossed"), 500*time.Millisecond); err != nil { + t.Fatalf("Did not receive response: %v", err) + } + + // Now create a client on the leafnode. + url = fmt.Sprintf("nats://%s:%d", slOpts.Host, slOpts.Port) + ncl, err := nats.Connect(url) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncl.Close() + + if _, err := ncl.Request("ngs.usage", []byte("fingers crossed"), 500*time.Millisecond); err != nil { + t.Fatalf("Did not receive response: %v", err) + } +} + +func TestLeafNodeServiceImportResponderOnLeaf(t *testing.T) { + gwSolicit := 10 * time.Millisecond + ca := createClusterEx(t, true, gwSolicit, true, "A", 3) + defer shutdownCluster(ca) + + // Now create a leafnode server on A that will bind to the NGS account. + opts := ca.opts[1] + sl, slOpts := runSolicitLeafServerToURL(fmt.Sprintf("nats-leaf://ngs:pass@%s:%d", opts.LeafNode.Host, opts.LeafNode.Port)) + defer sl.Shutdown() + + checkLeafNodeConnected(t, sl) + + // Now create a client on the leafnode. + ncl, err := nats.Connect(fmt.Sprintf("nats://%s:%d", slOpts.Host, slOpts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncl.Close() + + // Create a queue subscriber to send results + ncl.QueueSubscribe("ngs.usage.*", "ngs", func(m *nats.Msg) { + m.Respond([]byte("22")) + }) + ncl.Flush() + + // Create a normal direct connect client on A. Needs to be same server as leafnode. + opts = ca.opts[1] + nc, err := nats.Connect(fmt.Sprintf("nats://dlc:pass@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + if _, err := nc.Request("ngs.usage", []byte("fingers crossed"), 500*time.Millisecond); err != nil { + t.Fatalf("Did not receive response: %v", err) + } +} + +func TestLeafNodeSendsAccountingEvents(t *testing.T) { + s, opts, _ := runLeafNodeOperatorServer(t) + defer s.Shutdown() + + // System account + acc, akp := createAccount(t, s) + if err := s.SetSystemAccount(acc.Name); err != nil { + t.Fatalf("Expected this succeed, got %v", err) + } + + // Leafnode Account + lacc, lakp := createAccount(t, s) + + // Create a system account user and connect a client to listen for the events. + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(url, createUserCreds(t, s, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // Watch only for our leaf node account. + cSub, _ := nc.SubscribeSync(fmt.Sprintf("$SYS.ACCOUNT.%s.CONNECT", lacc.Name)) + dSub, _ := nc.SubscribeSync(fmt.Sprintf("$SYS.ACCOUNT.%s.DISCONNECT", lacc.Name)) + nc.Flush() + + // Now create creds for the leafnode and connect the server. + kp, _ := nkeys.CreateUser() + pub, _ := kp.PublicKey() + nuc := jwt.NewUserClaims(pub) + ujwt, err := nuc.Encode(lakp) + if err != nil { + t.Fatalf("Error generating user JWT: %v", err) + } + seed, _ := kp.Seed() + mycreds := genCredsFile(t, ujwt, seed) + + sl, _, _ := runSolicitWithCredentials(t, opts, mycreds) + defer sl.Shutdown() + + // Wait for connect event + msg, err := cSub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Error waiting for account connect event: %v", err) + } + m := server.ConnectEventMsg{} + if err := json.Unmarshal(msg.Data, &m); err != nil { + t.Fatal("Did not get correctly formatted event") + } + + // Shutdown leafnode to generate disconnect event. + sl.Shutdown() + + msg, err = dSub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Error waiting for account disconnect event: %v", err) + } + dm := server.DisconnectEventMsg{} + if err := json.Unmarshal(msg.Data, &dm); err != nil { + t.Fatal("Did not get correctly formatted event") + } +} + +func TestLeafNodeDistributedQueueAcrossGWs(t *testing.T) { + gwSolicit := 10 * time.Millisecond + ca := createClusterEx(t, true, gwSolicit, true, "A", 3) + defer shutdownCluster(ca) + cb := createClusterEx(t, true, gwSolicit, true, "B", 3, ca) + defer shutdownCluster(cb) + cc := createClusterEx(t, true, gwSolicit, true, "C", 3, ca, cb) + defer shutdownCluster(cc) + + // Create queue subscribers + createQS := func(c *cluster) *nats.Conn { + t.Helper() + opts := c.opts[rand.IntN(len(c.opts))] + url := fmt.Sprintf("nats://ngs:pass@%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(url) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + nc.QueueSubscribe("ngs.usage.*", "dq", func(m *nats.Msg) { + m.Respond([]byte(c.name)) + }) + nc.Flush() + return nc + } + + ncA := createQS(ca) + defer ncA.Close() + ncB := createQS(cb) + defer ncB.Close() + ncC := createQS(cc) + defer ncC.Close() + + connectAndRequest := func(url, clusterName string, nreqs int) { + t.Helper() + nc, err := nats.Connect(url) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + for i := 0; i < nreqs; i++ { + m, err := nc.Request("ngs.usage", nil, 500*time.Millisecond) + if err != nil { + t.Fatalf("Did not receive a response: %v", err) + } + if string(m.Data) != clusterName { + t.Fatalf("Expected to prefer %q, but got response from %q", clusterName, m.Data) + } + } + } + + checkClientDQ := func(c *cluster, nreqs int) { + t.Helper() + // Pick one at random. + opts := c.opts[rand.IntN(len(c.opts))] + url := fmt.Sprintf("nats://dlc:pass@%s:%d", opts.Host, opts.Port) + connectAndRequest(url, c.name, nreqs) + } + + // First check that this works with direct connected clients. + checkClientDQ(ca, 100) + checkClientDQ(cb, 100) + checkClientDQ(cc, 100) + + createLNS := func(c *cluster) (*server.Server, *server.Options) { + t.Helper() + // Pick one at random. + copts := c.opts[rand.IntN(len(c.servers))] + s, opts := runSolicitLeafServerToURL(fmt.Sprintf("nats-leaf://dlc:pass@%s:%d", copts.LeafNode.Host, copts.LeafNode.Port)) + checkLeafNodeConnected(t, s) + return s, opts + } + + checkLeafDQ := func(opts *server.Options, clusterName string, nreqs int) { + t.Helper() + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + connectAndRequest(url, clusterName, nreqs) + } + + // Test leafnodes to all clusters. + for _, c := range []*cluster{ca, cb, cc} { + // Now create a leafnode on cluster. + sl, slOpts := createLNS(c) + defer sl.Shutdown() + // Now connect to the leafnode server and run test. + checkLeafDQ(slOpts, c.name, 100) + } +} + +func TestLeafNodeDistributedQueueEvenly(t *testing.T) { + gwSolicit := 10 * time.Millisecond + ca := createClusterEx(t, true, gwSolicit, true, "A", 3) + defer shutdownCluster(ca) + cb := createClusterEx(t, true, gwSolicit, true, "B", 3, ca) + defer shutdownCluster(cb) + + // Create queue subscribers + createQS := func(c *cluster) *nats.Conn { + t.Helper() + opts := c.opts[rand.IntN(len(c.opts))] + url := fmt.Sprintf("nats://ngs:pass@%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(url) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + cid, _ := nc.GetClientID() + response := []byte(fmt.Sprintf("%s:%d:%d", c.name, opts.Port, cid)) + nc.QueueSubscribe("ngs.usage.*", "dq", func(m *nats.Msg) { + m.Respond(response) + }) + nc.Flush() + return nc + } + + ncA1 := createQS(ca) + defer ncA1.Close() + + ncA2 := createQS(ca) + defer ncA2.Close() + + ncA3 := createQS(ca) + defer ncA3.Close() + + resp := make(map[string]int) + + connectAndRequest := func(url, clusterName string, nreqs int) { + t.Helper() + nc, err := nats.Connect(url) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + for i := 0; i < nreqs; i++ { + m, err := nc.Request("ngs.usage", nil, 500*time.Millisecond) + if err != nil { + t.Fatalf("Did not receive a response: %v", err) + } + if string(m.Data[0]) != clusterName { + t.Fatalf("Expected to prefer %q, but got response from %q", clusterName, m.Data[0]) + } + resp[string(m.Data)]++ + } + } + + createLNS := func(c *cluster) (*server.Server, *server.Options) { + t.Helper() + // Pick one at random. + copts := c.opts[rand.IntN(len(c.servers))] + s, opts := runSolicitLeafServerToURL(fmt.Sprintf("nats-leaf://dlc:pass@%s:%d", copts.LeafNode.Host, copts.LeafNode.Port)) + checkLeafNodeConnected(t, s) + return s, opts + } + + checkLeafDQ := func(opts *server.Options, clusterName string, nreqs int) { + t.Helper() + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + connectAndRequest(url, clusterName, nreqs) + } + + // Now create a leafnode on cluster A. + sl, slOpts := createLNS(ca) + defer sl.Shutdown() + + // Now connect to the leafnode server and run test. + checkLeafDQ(slOpts, ca.name, 100) + + // Should have some for all 3 QS [ncA1, ncA2, ncA3] + if lr := len(resp); lr != 3 { + t.Fatalf("Expected all 3 queue subscribers to have received some messages, only got %d", lr) + } + // Now check that we have at least 10% for each subscriber. + for _, r := range resp { + if r < 10 { + t.Fatalf("Got a subscriber with less than 10 responses: %d", r) + } + } +} + +func TestLeafNodeDefaultPort(t *testing.T) { + o := testDefaultOptionsForLeafNodes() + o.LeafNode.Port = server.DEFAULT_LEAFNODE_PORT + s := RunServer(o) + defer s.Shutdown() + + conf := createConfFile(t, []byte(` + port: -1 + leaf { + remotes = [ + { + url: "leafnode://127.0.0.1" + } + ] + } + `)) + + sl, _ := RunServerWithConfig(conf) + defer sl.Shutdown() + + checkLeafNodeConnected(t, s) +} + +// Since leafnode's are true interest only we need to make sure that we +// register the proper interest with global routing $GR.xxxxxx._INBOX.> +func TestLeafNodeAndGatewayGlobalRouting(t *testing.T) { + server.SetGatewaysSolicitDelay(50 * time.Millisecond) + defer server.ResetGatewaysSolicitDelay() + + ca := createClusterWithName(t, "A", 3) + defer shutdownCluster(ca) + cb := createClusterWithName(t, "B", 3, ca) + defer shutdownCluster(cb) + + sl, slOpts := runSolicitLeafServer(ca.opts[1]) + defer sl.Shutdown() + + checkLeafNodeConnected(t, ca.servers[1]) + + // Create a client on the leafnode. This will listen for requests. + ncl, err := nats.Connect(fmt.Sprintf("nats://%s:%d", slOpts.Host, slOpts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncl.Close() + + ncl.Subscribe("foo", func(m *nats.Msg) { + m.Respond([]byte("World")) + }) + ncl.Flush() + + // Since for leafnodes the account becomes interest-only mode, + // let's make sure that the interest on "foo" has time to propagate + // to cluster B. + time.Sleep(250 * time.Millisecond) + + // Create a direct connect requestor. Try with all possible + // servers in cluster B to make sure that we also receive the + // reply when the accepting leafnode server does not have + // its outbound GW connection to the requestor's server. + for i := 0; i < 3; i++ { + opts := cb.opts[i] + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(url) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // We don't use an INBOX here because we had a bug where + // leafnode would subscribe to _GR_.*.*.*.> instead of + // _GR_.>, and inbox masked that because of their number + // of tokens. + reply := nuid.Next() + sub, err := nc.SubscribeSync(reply) + if err != nil { + t.Fatalf("Error on subscribe: %v", err) + } + if err := nc.PublishRequest("foo", reply, []byte("Hello")); err != nil { + t.Fatalf("Failed to send request: %v", err) + } + if _, err := sub.NextMsg(250 * time.Millisecond); err != nil { + t.Fatalf("Did not get reply from server %d: %v", i, err) + } + nc.Close() + } +} + +func checkLeafNode2Connected(t *testing.T, s *server.Server) { + t.Helper() + checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { + if nln := s.NumLeafNodes(); nln != 2 { + return fmt.Errorf("Expected a connected leafnode for server %q, got none", s.ID()) + } + return nil + }) +} + +func TestLeafNodesStaggeredSubPub(t *testing.T) { + s, opts := runLeafServer() + defer s.Shutdown() + + sl1, sl1Opts := runSolicitLeafServer(opts) + defer sl1.Shutdown() + + checkLeafNodeConnected(t, s) + + // Create a client on the leafnode and a subscription. + ncl, err := nats.Connect(fmt.Sprintf("nats://%s:%d", sl1Opts.Host, sl1Opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncl.Close() + + sub, err := ncl.SubscribeSync("foo") + if err != nil { + t.Fatalf("Failed to subscribe: %v", err) + } + + sl2, sl2Opts := runSolicitLeafServer(opts) + defer sl2.Shutdown() + + checkLeafNode2Connected(t, s) + + // Create a client on the second leafnode and publish a message. + ncl2, err := nats.Connect(fmt.Sprintf("nats://%s:%d", sl2Opts.Host, sl2Opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncl2.Close() + + ncl2.Publish("foo", nil) + + checkFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error { + if nmsgs, _, err := sub.Pending(); err != nil || nmsgs != 1 { + return fmt.Errorf("Did not receive the message: %v", err) + } + return nil + }) +} + +func TestLeafNodeMultipleRemoteURLs(t *testing.T) { + s, opts := runLeafServer() + defer s.Shutdown() + + content := ` + port: -1 + leafnodes { + remotes = [ + { + urls: [nats-leaf://127.0.0.1:%d,nats-leaf://localhost:%d] + } + ] + } + ` + + config := fmt.Sprintf(content, opts.LeafNode.Port, opts.LeafNode.Port) + conf := createConfFile(t, []byte(config)) + sl, _ := RunServerWithConfig(conf) + defer sl.Shutdown() + + checkLeafNodeConnected(t, s) +} + +func runSolicitLeafCluster(t *testing.T, clusterName string, d1, d2 *cluster) *cluster { + c := &cluster{servers: make([]*server.Server, 0, 2), opts: make([]*server.Options, 0, 2), name: clusterName} + + // Who we will solicit for server 1 + ci := rand.IntN(len(d1.opts)) + opts := d1.opts[ci] + surl := fmt.Sprintf("nats-leaf://%s:%d", opts.LeafNode.Host, opts.LeafNode.Port) + + o := DefaultTestOptions + o.Port = -1 + rurl, _ := url.Parse(surl) + o.LeafNode.Remotes = []*server.RemoteLeafOpts{{URLs: []*url.URL{rurl}}} + o.LeafNode.ReconnectInterval = 100 * time.Millisecond + o.Cluster.Name = clusterName + o.Cluster.Host = o.Host + o.Cluster.Port = -1 + s := RunServer(&o) + checkLeafNodeConnected(t, d1.servers[ci]) + + c.servers = append(c.servers, s) + c.opts = append(c.opts, &o) + + // Grab route info + routeAddr := fmt.Sprintf("nats-route://%s:%d", o.Cluster.Host, o.Cluster.Port) + curl, _ := url.Parse(routeAddr) + + // Who we will solicit for server 2 + ci = rand.IntN(len(d2.opts)) + opts = d2.opts[ci] + surl = fmt.Sprintf("nats-leaf://%s:%d", opts.LeafNode.Host, opts.LeafNode.Port) + + // This is for the case were d1 == d2 and we select the same server. + plfn := d2.servers[ci].NumLeafNodes() + + o2 := DefaultTestOptions + o2.Port = -1 + rurl, _ = url.Parse(surl) + o2.LeafNode.Remotes = []*server.RemoteLeafOpts{{URLs: []*url.URL{rurl}}} + o2.LeafNode.ReconnectInterval = 100 * time.Millisecond + o2.Cluster.Name = clusterName + o2.Cluster.Host = o.Host + o2.Cluster.Port = -1 + o2.Routes = []*url.URL{curl} + s = RunServer(&o2) + + if plfn == 0 { + checkLeafNodeConnected(t, d2.servers[ci]) + } else { + checkLeafNode2Connected(t, d2.servers[ci]) + } + + c.servers = append(c.servers, s) + c.opts = append(c.opts, &o2) + + checkClusterFormed(t, c.servers...) + + return c +} + +func clientForCluster(t *testing.T, c *cluster) *nats.Conn { + t.Helper() + opts := c.opts[rand.IntN(len(c.opts))] + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(url) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + return nc +} + +func TestLeafNodeCycleWithSolicited(t *testing.T) { + server.SetGatewaysSolicitDelay(10 * time.Millisecond) + defer server.ResetGatewaysSolicitDelay() + + // Accepting leafnode cluster, e.g. NGS + ca := createClusterWithName(t, "A", 3) + defer shutdownCluster(ca) + cb := createClusterWithName(t, "B", 3, ca) + defer shutdownCluster(cb) + + // Create the responders. + requestsReceived := int32(0) + + nc := clientForCluster(t, ca) + defer nc.Close() + nc.QueueSubscribe("request", "cycles", func(m *nats.Msg) { + atomic.AddInt32(&requestsReceived, 1) + m.Respond([]byte("22")) + }) + + nc = clientForCluster(t, cb) + defer nc.Close() + nc.QueueSubscribe("request", "cycles", func(m *nats.Msg) { + atomic.AddInt32(&requestsReceived, 1) + m.Respond([]byte("33")) + }) + + // Soliciting cluster, both solicited connected to the "A" cluster + sc := runSolicitLeafCluster(t, "SC", ca, ca) + defer shutdownCluster(sc) + + checkInterest := func(s *server.Server, subject string) bool { + t.Helper() + acc, _ := s.LookupAccount("$G") + return acc.SubscriptionInterest(subject) + } + + waitForInterest := func(subject string, servers ...*server.Server) { + t.Helper() + checkFor(t, time.Second, 10*time.Millisecond, func() error { + for _, s := range servers { + if !checkInterest(s, subject) { + return fmt.Errorf("No interest") + } + } + return nil + }) + } + + waitForInterest("request", + sc.servers[0], sc.servers[1], + ca.servers[0], ca.servers[1], ca.servers[2], + cb.servers[0], cb.servers[1], cb.servers[2], + ) + + // Connect a client to a random server in sc + createClientAndRequest := func(c *cluster) (*nats.Conn, *nats.Subscription) { + nc := clientForCluster(t, c) + reply := nats.NewInbox() + sub, err := nc.SubscribeSync(reply) + if err != nil { + t.Fatalf("Could not subscribe: %v", err) + } + if err := nc.PublishRequest("request", reply, []byte("fingers crossed")); err != nil { + t.Fatalf("Error sending request: %v", err) + } + return nc, sub + } + + verifyOneResponse := func(sub *nats.Subscription) { + time.Sleep(250 * time.Millisecond) + m, _, err := sub.Pending() + if err != nil { + t.Fatalf("Error calling Pending(): %v", err) + } + if m > 1 { + t.Fatalf("Received more then one response, cycle indicated: %d", m) + } + } + + verifyRequestTotal := func(nre int32) { + if nr := atomic.LoadInt32(&requestsReceived); nr != nre { + t.Fatalf("Expected %d requests received, got %d", nre, nr) + } + } + + // This should pass to here, but if we have a cycle things will be spinning and we will receive + // too many responses when it should only be 1. + nc, rsub := createClientAndRequest(sc) + defer nc.Close() + verifyOneResponse(rsub) + verifyRequestTotal(1) + + // Do a solicit across GW, so shut this one down. + nc.Close() + shutdownCluster(sc) + + // Soliciting cluster, connect to different clusters across a GW. + sc = runSolicitLeafCluster(t, "SC", ca, cb) + defer shutdownCluster(sc) + + nc, rsub = createClientAndRequest(sc) + defer nc.Close() + verifyOneResponse(rsub) + verifyRequestTotal(2) // This is total since use same responders. +} + +func TestLeafNodeNoRaceGeneratingNonce(t *testing.T) { + opts := testDefaultOptionsForLeafNodes() + opts.Cluster.Port = -1 + opts.Cluster.Name = "xyz" + s := RunServer(opts) + defer s.Shutdown() + + quitCh := make(chan struct{}) + wg := sync.WaitGroup{} + wg.Add(2) + + go func() { + defer wg.Done() + for { + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + checkInfoMsg(t, lc) + lc.Close() + select { + case <-quitCh: + return + case <-time.After(time.Millisecond): + } + } + }() + + go func() { + defer wg.Done() + for { + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + checkInfoMsg(t, rc) + rc.Close() + select { + case <-quitCh: + return + case <-time.After(time.Millisecond): + } + } + }() + + // Let this run for a bit to see if we get data race + time.Sleep(100 * time.Millisecond) + close(quitCh) + wg.Wait() +} + +func runSolicitAndAcceptLeafServer(lso *server.Options) (*server.Server, *server.Options) { + surl := fmt.Sprintf("nats-leaf://%s:%d", lso.LeafNode.Host, lso.LeafNode.Port) + o := testDefaultOptionsForLeafNodes() + o.Port = -1 + rurl, _ := url.Parse(surl) + o.LeafNode.Remotes = []*server.RemoteLeafOpts{{URLs: []*url.URL{rurl}}} + o.LeafNode.ReconnectInterval = 100 * time.Millisecond + return RunServer(o), o +} + +func TestLeafNodeDaisyChain(t *testing.T) { + // To quickly enable trace and debug logging + // doLog, doTrace, doDebug = true, true, true + s1, opts1 := runLeafServer() + defer s1.Shutdown() + + s2, opts2 := runSolicitAndAcceptLeafServer(opts1) + defer s2.Shutdown() + checkLeafNodeConnected(t, s1) + + s3, _ := runSolicitLeafServer(opts2) + defer s3.Shutdown() + checkLeafNodeConnections(t, s2, 2) + + // Make so we can tell the two apart since in same PID. + if doLog { + s1.SetLogger(logger.NewTestLogger("[S-1] - ", false), true, true) + s2.SetLogger(logger.NewTestLogger("[S-2] - ", false), true, true) + s3.SetLogger(logger.NewTestLogger("[S-3] - ", false), true, true) + } + + nc1, err := nats.Connect(s1.ClientURL()) + if err != nil { + t.Fatalf("Error creating client: %v", err) + } + defer nc1.Close() + + nc1.Subscribe("ngs.usage", func(msg *nats.Msg) { + msg.Respond([]byte("22 msgs")) + }) + nc1.Flush() + + checkSubInterest(t, s3, "$G", "ngs.usage", time.Second) + + nc2, err := nats.Connect(s3.ClientURL()) + if err != nil { + t.Fatalf("Error creating client: %v", err) + } + defer nc2.Close() + + if _, err = nc2.Request("ngs.usage", []byte("1h"), time.Second); err != nil { + t.Fatalf("Expected a response") + } +} + +// This will test failover to a server with a cert with only an IP after successfully connecting +// to a server with a cert with both. +func TestClusterTLSMixedIPAndDNS(t *testing.T) { + confA := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + leafnodes { + listen: "127.0.0.1:-1" + tls { + cert_file: "./configs/certs/server-iponly.pem" + key_file: "./configs/certs/server-key-iponly.pem" + ca_file: "./configs/certs/ca.pem" + timeout: 2 + } + } + cluster { + listen: "127.0.0.1:-1" + name: xyz + } + `)) + srvA, optsA := RunServerWithConfig(confA) + defer srvA.Shutdown() + + bConfigTemplate := ` + listen: 127.0.0.1:-1 + leafnodes { + listen: "127.0.0.1:-1" + tls { + cert_file: "./configs/certs/server-cert.pem" + key_file: "./configs/certs/server-key.pem" + ca_file: "./configs/certs/ca.pem" + timeout: 2 + } + } + cluster { + listen: "127.0.0.1:-1" + name: xyz + routes [ + "nats://%s:%d" + ] + } + ` + confB := createConfFile(t, []byte(fmt.Sprintf(bConfigTemplate, + optsA.Cluster.Host, optsA.Cluster.Port))) + srvB, optsB := RunServerWithConfig(confB) + defer srvB.Shutdown() + + checkClusterFormed(t, srvA, srvB) + + // Solicit a leafnode server here. Don't use the helper since we need verification etc. + o := DefaultTestOptions + o.Port = -1 + rurl, _ := url.Parse(fmt.Sprintf("nats-leaf://%s:%d", optsB.LeafNode.Host, optsB.LeafNode.Port)) + o.LeafNode.ReconnectInterval = 10 * time.Millisecond + remote := &server.RemoteLeafOpts{URLs: []*url.URL{rurl}} + remote.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12} + pool := x509.NewCertPool() + rootPEM, err := os.ReadFile("./configs/certs/ca.pem") + if err != nil || rootPEM == nil { + t.Fatalf("Error loading or parsing rootCA file: %v", err) + } + ok := pool.AppendCertsFromPEM(rootPEM) + if !ok { + t.Fatalf("Failed to parse root certificate from %q", "./configs/certs/ca.pem") + } + remote.TLSConfig.RootCAs = pool + o.LeafNode.Remotes = []*server.RemoteLeafOpts{remote} + sl, _ := RunServer(&o), &o + defer sl.Shutdown() + + checkLeafNodeConnected(t, srvB) + + // Now kill off srvB and force client to connect to srvA. + srvB.Shutdown() + + // Make sure this works. + checkLeafNodeConnected(t, srvA) +} + +// This will test for a bug in stream export/import with leafnodes. +// https://github.com/nats-io/nats-server/issues/1332 +func TestStreamExportWithMultipleAccounts(t *testing.T) { + confA := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + leafnodes { + listen: "127.0.0.1:-1" + } + `)) + srvA, optsA := RunServerWithConfig(confA) + defer srvA.Shutdown() + + bConfigTemplate := ` + listen: 127.0.0.1:-1 + leafnodes { + listen: "127.0.0.1:-1" + remotes = [ + { + url:"nats://127.0.0.1:%d" + account:"EXTERNAL" + } + ] + } + accounts: { + INTERNAL: { + users: [ + {user: good, password: pwd} + ] + imports: [ + { + stream: { account: EXTERNAL, subject: "foo"}, prefix: "bar" + } + ] + }, + EXTERNAL: { + users: [ + {user: bad, password: pwd} + ] + exports: [{stream: "foo"}] + }, + } + ` + + confB := createConfFile(t, []byte(fmt.Sprintf(bConfigTemplate, optsA.LeafNode.Port))) + srvB, optsB := RunServerWithConfig(confB) + defer srvB.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("nats://good:pwd@%s:%d", optsB.Host, optsB.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + wcsub, err := nc.SubscribeSync(">") + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + defer wcsub.Unsubscribe() + nc.Flush() + + nc2, err := nats.Connect(fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + nc2.Publish("foo", nil) + + msg, err := wcsub.NextMsg(1 * time.Second) + if err != nil { + t.Fatalf("Did not receive the message: %v", err) + } + if msg.Subject != "bar.foo" { + t.Fatalf("Received on wrong subject: %q", msg.Subject) + } +} + +// This will test for a bug in service export/import with leafnodes. +// https://github.com/nats-io/nats-server/issues/1336 +func TestServiceExportWithMultipleAccounts(t *testing.T) { + confA := createConfFile(t, []byte(` + server_name: A + listen: 127.0.0.1:-1 + leafnodes { + listen: "127.0.0.1:-1" + } + `)) + + srvA, optsA := RunServerWithConfig(confA) + defer srvA.Shutdown() + + bConfigTemplate := ` + server_name: B + listen: 127.0.0.1:-1 + leafnodes { + listen: "127.0.0.1:-1" + remotes = [ + { + url:"nats://127.0.0.1:%d" + account:"EXTERNAL" + } + ] + } + accounts: { + INTERNAL: { + users: [ + {user: good, password: pwd} + ] + imports: [ + { + service: { + account: EXTERNAL + subject: "foo" + } + } + ] + }, + EXTERNAL: { + users: [ + {user: evil, password: pwd} + ] + exports: [{service: "foo"}] + }, + } + ` + + confB := createConfFile(t, []byte(fmt.Sprintf(bConfigTemplate, optsA.LeafNode.Port))) + + srvB, optsB := RunServerWithConfig(confB) + defer srvB.Shutdown() + + checkLeafNodeConnected(t, srvB) + + // connect to confA, and offer a service + nc2, err := nats.Connect(fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + nc2.Subscribe("foo", func(msg *nats.Msg) { + if err := msg.Respond([]byte("world")); err != nil { + t.Fatalf("Error on respond: %v", err) + } + }) + nc2.Flush() + + checkSubInterest(t, srvB, "INTERNAL", "foo", time.Second) + + nc, err := nats.Connect(fmt.Sprintf("nats://good:pwd@%s:%d", optsB.Host, optsB.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + resp, err := nc.Request("foo", []byte("hello"), 2*time.Second) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if resp == nil || strings.Compare("world", string(resp.Data)) != 0 { + t.Fatal("Did not receive the correct message") + } +} + +// This will test for a bug in service export/import with leafnode restart. +// https://github.com/nats-io/nats-server/issues/1344 +func TestServiceExportWithLeafnodeRestart(t *testing.T) { + confG := createConfFile(t, []byte(` + server_name: G + listen: 127.0.0.1:-1 + leafnodes { + listen: "127.0.0.1:-1" + authorization { + account:"EXTERNAL" + user: ln + password: pass + } + } + + accounts: { + INTERNAL: { + users: [ + {user: good, password: pwd} + ] + exports: [{service: "foo", response: singleton}] + imports: [ + { + service: { + account: EXTERNAL + subject: "evilfoo" + }, to: from_evilfoo + } + ] + }, + EXTERNAL: { + users: [ + {user: evil, password: pwd} + ] + exports: [{service: "evilfoo", response: singleton}] + imports: [ + { + service: { + account: INTERNAL + subject: "foo" + }, to: goodfoo + } + ] + } + } + `)) + + srvG, optsG := RunServerWithConfig(confG) + defer srvG.Shutdown() + + eConfigTemplate := ` + server_name: E + listen: 127.0.0.1:-1 + leafnodes { + listen: "127.0.0.1:-1" + remotes = [ + { + url:"nats://ln:pass@127.0.0.1:%d" + account:"EXTERNAL_GOOD" + } + ] + } + + accounts: { + INTERNAL_EVILINC: { + users: [ + {user: evil, password: pwd} + ] + exports: [{service: "foo", response: singleton}] + imports: [ + { + service: { + account: EXTERNAL_GOOD + subject: "goodfoo" + }, to: from_goodfoo + } + ] + }, + EXTERNAL_GOOD: { + users: [ + {user: good, password: pwd} + ] + exports: [{service: "goodfoo", response: singleton}] + imports: [ + { + service: { + account: INTERNAL_EVILINC + subject: "foo" + }, to: evilfoo + } + ] + }, + } + ` + + confE := createConfFile(t, []byte(fmt.Sprintf(eConfigTemplate, optsG.LeafNode.Port))) + + srvE, optsE := RunServerWithConfig(confE) + defer srvE.Shutdown() + + // connect to confE, and offer a service + nc2, err := nats.Connect(fmt.Sprintf("nats://evil:pwd@%s:%d", optsE.Host, optsE.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + nc2.Subscribe("foo", func(msg *nats.Msg) { + if err := msg.Respond([]byte("world")); err != nil { + t.Fatalf("Error on respond: %v", err) + } + }) + nc2.Flush() + + nc, err := nats.Connect(fmt.Sprintf("nats://good:pwd@%s:%d", optsG.Host, optsG.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + resp, err := nc.Request("from_evilfoo", []byte("hello"), 2*time.Second) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if resp == nil || strings.Compare("world", string(resp.Data)) != 0 { + t.Fatal("Did not receive the correct message") + } + + // Now restart server E and requestor and replier. + nc.Close() + nc2.Close() + srvE.Shutdown() + + srvE, optsE = RunServerWithConfig(confE) + defer srvE.Shutdown() + + nc2, err = nats.Connect(fmt.Sprintf("nats://evil:pwd@%s:%d", optsE.Host, optsE.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + nc2.Subscribe("foo", func(msg *nats.Msg) { + if err := msg.Respond([]byte("world")); err != nil { + t.Fatalf("Error on respond: %v", err) + } + }) + nc2.Flush() + + nc, err = nats.Connect(fmt.Sprintf("nats://good:pwd@%s:%d", optsG.Host, optsG.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + resp, err = nc.Request("from_evilfoo", []byte("hello"), 2*time.Second) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if resp == nil || strings.Compare("world", string(resp.Data)) != 0 { + t.Fatal("Did not receive the correct message") + } +} + +func TestLeafNodeQueueSubscriberUnsubscribe(t *testing.T) { + s, opts := runLeafServer() + defer s.Shutdown() + + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + leafSend, leafExpect := setupLeaf(t, lc, 1) + leafSend("PING\r\n") + leafExpect(pongRe) + + // Create a client on leaf server and create a queue sub + c1 := createClientConn(t, opts.Host, opts.Port) + defer c1.Close() + + send1, expect1 := setupConn(t, c1) + send1("SUB foo bar 1\r\nPING\r\n") + expect1(pongRe) + + // Leaf should receive an LS+ foo bar 1 + leafExpect(lsubRe) + + // Create a second client on leaf server and create queue sub on same group. + c2 := createClientConn(t, opts.Host, opts.Port) + defer c2.Close() + + send2, expect2 := setupConn(t, c2) + send2("SUB foo bar 1\r\nPING\r\n") + expect2(pongRe) + + // Leaf should receive an LS+ foo bar 2 + leafExpect(lsubRe) + + // Now close c1 + c1.Close() + + // Leaf should receive an indication that the queue group went to 1. + // Which means LS+ foo bar 1. + buf := leafExpect(lsubRe) + if matches := lsubRe.FindAllSubmatch(buf, -1); len(matches) != 1 { + t.Fatalf("Expected only 1 LS+, got %v", len(matches)) + } + // Make sure that we did not get a LS- at the same time. + if bytes.Contains(buf, []byte("LS-")) { + t.Fatalf("Unexpected LS- in response: %q", buf) + } + // Make sure we receive nothing... + expectNothing(t, lc) +} + +func TestLeafNodeOriginClusterSingleHub(t *testing.T) { + s, opts := runLeafServer() + defer s.Shutdown() + + c1 := ` + listen: 127.0.0.1:-1 + cluster { name: ln22, listen: 127.0.0.1:-1 } + leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } + ` + lconf1 := createConfFile(t, []byte(fmt.Sprintf(c1, opts.LeafNode.Port))) + + ln1, lopts1 := RunServerWithConfig(lconf1) + defer ln1.Shutdown() + + c2 := ` + listen: 127.0.0.1:-1 + cluster { name: ln22, listen: 127.0.0.1:-1, routes = [ nats-route://127.0.0.1:%d] } + leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } + ` + lconf2 := createConfFile(t, []byte(fmt.Sprintf(c2, lopts1.Cluster.Port, opts.LeafNode.Port))) + + ln2, _ := RunServerWithConfig(lconf2) + defer ln2.Shutdown() + + ln3, _ := RunServerWithConfig(lconf2) + defer ln3.Shutdown() + + checkClusterFormed(t, ln1, ln2, ln3) + checkLeafNodeConnections(t, s, 3) + + // So now we are setup with 3 solicited leafnodes all connected to a hub. + // We will create two clients, one on each leafnode server. + nc1, err := nats.Connect(ln1.ClientURL()) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc1.Close() + + nc2, err := nats.Connect(ln2.ClientURL()) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + checkInterest := func(s *server.Server, subject string) bool { + t.Helper() + acc, _ := s.LookupAccount("$G") + return acc.SubscriptionInterest(subject) + } + + waitForInterest := func(subject string, servers ...*server.Server) { + t.Helper() + checkFor(t, time.Second, 10*time.Millisecond, func() error { + for _, s := range servers { + if !checkInterest(s, subject) { + return fmt.Errorf("No interest") + } + } + return nil + }) + } + + subj := "foo.bar" + + sub, _ := nc2.SubscribeSync(subj) + waitForInterest(subj, ln1, ln2, ln3, s) + + // Make sure we truncated the subscription bouncing through the hub and back to other leafnodes. + for _, s := range []*server.Server{ln1, ln3} { + acc, _ := s.LookupAccount("$G") + if nms := acc.Interest(subj); nms != 1 { + t.Fatalf("Expected only one active subscription, got %d", nms) + } + } + + // Send a message. + nc1.Publish(subj, nil) + nc1.Flush() + // Wait to propagate + time.Sleep(25 * time.Millisecond) + + // Make sure we only get it once. + if n, _, _ := sub.Pending(); n != 1 { + t.Fatalf("Expected only one message, got %d", n) + } +} + +func TestLeafNodeOriginCluster(t *testing.T) { + ca := createClusterWithName(t, "A", 3) + defer shutdownCluster(ca) + + c1 := ` + server_name: L1 + listen: 127.0.0.1:-1 + cluster { name: ln22, listen: 127.0.0.1:-1 } + leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } + ` + lconf1 := createConfFile(t, []byte(fmt.Sprintf(c1, ca.opts[0].LeafNode.Port))) + + ln1, lopts1 := RunServerWithConfig(lconf1) + defer ln1.Shutdown() + + c2 := ` + server_name: L2 + listen: 127.0.0.1:-1 + cluster { name: ln22, listen: 127.0.0.1:-1, routes = [ nats-route://127.0.0.1:%d] } + leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } + ` + lconf2 := createConfFile(t, []byte(fmt.Sprintf(c2, lopts1.Cluster.Port, ca.opts[1].LeafNode.Port))) + + ln2, _ := RunServerWithConfig(lconf2) + defer ln2.Shutdown() + + c3 := ` + server_name: L3 + listen: 127.0.0.1:-1 + cluster { name: ln22, listen: 127.0.0.1:-1, routes = [ nats-route://127.0.0.1:%d] } + leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } + ` + lconf3 := createConfFile(t, []byte(fmt.Sprintf(c3, lopts1.Cluster.Port, ca.opts[2].LeafNode.Port))) + + ln3, _ := RunServerWithConfig(lconf3) + defer ln3.Shutdown() + + checkClusterFormed(t, ln1, ln2, ln3) + checkLeafNodeConnections(t, ca.servers[0], 1) + checkLeafNodeConnections(t, ca.servers[1], 1) + checkLeafNodeConnections(t, ca.servers[2], 1) + + // So now we are setup with 3 solicited leafnodes connected to different servers in the hub cluster. + // We will create two clients, one on each leafnode server. + nc1, err := nats.Connect(ln1.ClientURL()) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc1.Close() + + nc2, err := nats.Connect(ln2.ClientURL()) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + checkInterest := func(s *server.Server, subject string) bool { + t.Helper() + acc, _ := s.LookupAccount("$G") + return acc.SubscriptionInterest(subject) + } + + waitForInterest := func(subject string, servers ...*server.Server) { + t.Helper() + checkFor(t, time.Second, 10*time.Millisecond, func() error { + for _, s := range servers { + if !checkInterest(s, subject) { + return fmt.Errorf("No interest") + } + } + return nil + }) + } + + subj := "foo.bar" + + sub, _ := nc2.SubscribeSync(subj) + waitForInterest(subj, ln1, ln2, ln3, ca.servers[0], ca.servers[1], ca.servers[2]) + + // Make sure we truncated the subscription bouncing through the hub and back to other leafnodes. + for _, s := range []*server.Server{ln1, ln3} { + acc, _ := s.LookupAccount("$G") + if nms := acc.Interest(subj); nms != 1 { + t.Fatalf("Expected only one active subscription, got %d", nms) + } + } + + // Send a message. + nc1.Publish(subj, nil) + nc1.Flush() + // Wait to propagate + time.Sleep(25 * time.Millisecond) + + // Make sure we only get it once. + if n, _, _ := sub.Pending(); n != 1 { + t.Fatalf("Expected only one message, got %d", n) + } + // eat the msg + sub.NextMsg(time.Second) + + // Now create interest on the hub side. This will draw the message from a leafnode + // to the hub. We want to make sure that message does not bounce back to other leafnodes. + nc3, err := nats.Connect(ca.servers[0].ClientURL()) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc3.Close() + + wcSubj := "foo.*" + wcsub, _ := nc3.SubscribeSync(wcSubj) + // This is a placeholder that we can use to check all interest has propagated. + nc3.SubscribeSync("bar") + waitForInterest("bar", ln1, ln2, ln3, ca.servers[0], ca.servers[1], ca.servers[2]) + + // Send another message. + m := nats.NewMsg(subj) + m.Header.Add("Accept-Encoding", "json") + m.Header.Add("Authorization", "s3cr3t") + m.Data = []byte("Hello Headers!") + + nc1.PublishMsg(m) + nc1.Flush() + // Wait to propagate + time.Sleep(25 * time.Millisecond) + + // Make sure we only get it once. + if n, _, _ := sub.Pending(); n != 1 { + t.Fatalf("Expected only one message, got %d", n) + } + // Also for wc + if n, _, _ := wcsub.Pending(); n != 1 { + t.Fatalf("Expected only one message, got %d", n) + } + + // grab the msg + msg, _ := sub.NextMsg(time.Second) + if !bytes.Equal(m.Data, msg.Data) { + t.Fatalf("Expected the payloads to match, wanted %q, got %q", m.Data, msg.Data) + } + if len(msg.Header) != 2 { + t.Fatalf("Expected 2 header entries, got %d", len(msg.Header)) + } + if msg.Header.Get("Authorization") != "s3cr3t" { + t.Fatalf("Expected auth header to match, wanted %q, got %q", "s3cr3t", msg.Header.Get("Authorization")) + } +} + +func TestLeafNodeAdvertiseInCluster(t *testing.T) { + o1 := testDefaultOptionsForLeafNodes() + o1.Cluster.Name = "abc" + o1.Cluster.Host = "127.0.0.1" + o1.Cluster.Port = -1 + s1 := runGatewayServer(o1) + defer s1.Shutdown() + + lc := createLeafConn(t, o1.LeafNode.Host, o1.LeafNode.Port) + defer lc.Close() + + leafSend, leafExpect := setupLeaf(t, lc, 1) + leafSend("PING\r\n") + leafExpect(pongRe) + + o2 := testDefaultOptionsForLeafNodes() + o2.Cluster.Name = "abc" + o2.Cluster.Host = "127.0.0.1" + o2.Cluster.Port = -1 + o2.Routes = server.RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port)) + o2.LeafNode.Advertise = "srvB:7222" + s2 := runGatewayServer(o2) + defer s2.Shutdown() + + checkClusterFormed(t, s1, s2) + + buf := leafExpect(infoRe) + si := &server.Info{} + json.Unmarshal(buf[5:], si) + var ok bool + for _, u := range si.LeafNodeURLs { + if u == "srvB:7222" { + ok = true + break + } + } + if !ok { + t.Fatalf("Url srvB:7222 was not found: %q", si.GatewayURLs) + } + + o3 := testDefaultOptionsForLeafNodes() + o3.Cluster.Name = "abc" + o3.Cluster.Host = "127.0.0.1" + o3.Cluster.Port = -1 + o3.Routes = server.RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port)) + o3.LeafNode.Advertise = "srvB:7222" + s3 := runGatewayServer(o3) + defer s3.Shutdown() + + checkClusterFormed(t, s1, s2, s3) + + // Since it is the save srvB:7222 url, we should not get an update. + expectNothing(t, lc) + + // Now shutdown s2 and make sure that we are not getting an update + // with srvB:7222 missing. + s2.Shutdown() + expectNothing(t, lc) +} + +func TestLeafNodeAndGatewaysStreamAndShadowSubs(t *testing.T) { + server.SetGatewaysSolicitDelay(10 * time.Millisecond) + defer server.ResetGatewaysSolicitDelay() + conf1 := createConfFile(t, []byte(` + port: -1 + system_account: SYS + accounts { + SYS {} + A: { + users: [{ user: a, password: pwd, permissions: {publish: [A.b.>]} }] + exports: [{ stream: A.b.>, accounts: [B] }] + }, + B: { + users: [{ user: b, password: pwd, permissions: {subscribe: [ A.b.> ]}}] + imports: [{ stream: { account: A, subject: A.b.> } }] + } + } + gateway { + name: "A" + port: -1 + } + leafnodes { + port: -1 + authorization: { + users: [ + {user: a, password: pwd, account: A} + ] + } + } + `)) + s1, o1 := RunServerWithConfig(conf1) + defer s1.Shutdown() + + conf2 := createConfFile(t, []byte(fmt.Sprintf(` + port: -1 + system_account: SYS + accounts { + SYS {} + A: { + users: [{ user: a, password: pwd, permissions: {publish: [A.b.>]} }] + exports: [{ stream: A.b.>, accounts: [B] }] + }, + B: { + users: [{ user: b, password: pwd, permissions: {subscribe: [ A.b.> ]}}] + imports: [{ stream: { account: A, subject: A.b.> } }] + } + } + gateway { + name: "B" + port: -1 + gateways [ + { + name: "A" + urls: ["nats://127.0.0.1:%d"] + } + ] + } + `, o1.Gateway.Port))) + s2, o2 := RunServerWithConfig(conf2) + defer s2.Shutdown() + + waitForOutboundGateways(t, s1, 1, 2*time.Second) + waitForOutboundGateways(t, s2, 1, 2*time.Second) + + nc, err := nats.Connect(fmt.Sprintf("nats://b:pwd@127.0.0.1:%d", o2.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + sub, err := nc.SubscribeSync("A.b.>") + if err != nil { + t.Fatalf("Error on subscibe: %v", err) + } + defer sub.Unsubscribe() + + conf3 := createConfFile(t, []byte(fmt.Sprintf(` + port: -1 + system_account: SYS + accounts: { + SYS {} + C: { + imports: [{ stream: { account: D, subject: b.> }, prefix: A }] + } + D: { + users: [{ user: d, password: pwd, permissions: {publish: [ b.> ]} }] + exports: [{ stream: b.>, accounts: [C] }] + } + } + leafnodes { + remotes [ + { + url: "nats://a:pwd@127.0.0.1:%d" + account: C + } + ] + } + `, o1.LeafNode.Port))) + s3, o3 := RunServerWithConfig(conf3) + defer s3.Shutdown() + + checkLeafNodeConnected(t, s1) + checkLeafNodeConnected(t, s3) + + ncl, err := nats.Connect(fmt.Sprintf("nats://d:pwd@127.0.0.1:%d", o3.Port)) + if err != nil { + t.Fatalf("Error connecting: %v", err) + } + defer ncl.Close() + + ncl.Publish("b.c", []byte("test")) + if _, err := sub.NextMsg(time.Second); err != nil { + t.Fatalf("Did not receive message: %v", err) + } +} + +func TestLeafnodeHeaders(t *testing.T) { + srv, opts := runLeafServer() + defer srv.Shutdown() + leaf, _ := runSolicitLeafServer(opts) + defer leaf.Shutdown() + + checkLeafNodeConnected(t, srv) + checkLeafNodeConnected(t, leaf) + + snc, err := nats.Connect(srv.ClientURL()) + if err != nil { + t.Fatal(err.Error()) + } + defer snc.Close() + + lnc, err := nats.Connect(leaf.ClientURL()) + if err != nil { + t.Fatal(err.Error()) + } + defer lnc.Close() + + // Start with subscription on leaf so that we check that srv has the interest + // (since we are going to publish from srv) + lsub, err := lnc.SubscribeSync("test") + if err != nil { + t.Fatalf("subscribe failed: %s", err) + } + lnc.Flush() + checkSubInterest(t, srv, "$G", "test", time.Second) + + ssub, err := snc.SubscribeSync("test") + if err != nil { + t.Fatalf("subscribe failed: %s", err) + } + + msg := nats.NewMsg("test") + msg.Header.Add("Test", "Header") + if len(msg.Header) == 0 { + t.Fatalf("msg header is empty") + } + err = snc.PublishMsg(msg) + if err != nil { + t.Fatal(err.Error()) + } + + smsg, err := ssub.NextMsg(time.Second) + if err != nil { + t.Fatalf("next failed: %s", err) + } + if len(smsg.Header) == 0 { + t.Fatalf("server msgs header is empty") + } + + lmsg, err := lsub.NextMsg(time.Second) + if err != nil { + t.Fatalf("next failed: %s", err) + } + if len(lmsg.Header) == 0 { + t.Fatalf("leaf msg header is empty") + } +} + +func TestLeafNodeClusterNameWithSpacesRejected(t *testing.T) { + s, opts := runLeafServer() + defer s.Shutdown() + + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + checkInfoMsg(t, lc) + sendProto(t, lc, "CONNECT {\"cluster\":\"my cluster\"}\r\n") + expect := expectCommand(t, lc) + expect(errRe) + expectDisconnect(t, lc) + + lc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + leafSend, leafExpect := setupLeaf(t, lc, 1) + // The accept side does expect an INFO from an handshake, + // so it will "ignore" the first one. + leafSend("INFO {\"server\":\"server\"}\r\n") + leafSend("INFO {\"cluster\":\"my cluster\"}\r\n") + leafExpect(errRe) + expectDisconnect(t, lc) +} + +func TestLeafNodeConnectInfo(t *testing.T) { + for _, test := range []struct { + name string + sys string + hasSys bool + }{ + {"with explicit system account", "system_account: SYS", true}, + {"without explicit system account", "", false}, + } { + t.Run(test.name, func(t *testing.T) { + conf := createConfFile(t, []byte(fmt.Sprintf(` + port: -1 + %s + accounts { + SYS: { users: [{ user: sys, password: pwd}] } + A: { users: [{ user: a, password: pwd}] } + B: { users: [{ user: b, password: pwd}] } + } + leafnodes { + port: -1 + } + `, test.sys))) + hub, oHub := RunServerWithConfig(conf) + defer hub.Shutdown() + + checkInfoOnConnect := func(user, acc string, isSys bool) { + t.Helper() + lc := createLeafConn(t, oHub.LeafNode.Host, oHub.LeafNode.Port) + defer lc.Close() + + checkInfoMsg(t, lc) + + sendProto(t, lc, fmt.Sprintf("CONNECT {\"user\":%q,\"pass\":\"pwd\"}\r\n", user)) + info := checkInfoMsg(t, lc) + if !info.ConnectInfo { + t.Fatal("Expected ConnectInfo to be true") + } + if an := info.RemoteAccount; an != acc { + t.Fatalf("Expected account %q, got %q", acc, info.RemoteAccount) + } + if ais := info.IsSystemAccount; ais != isSys { + t.Fatalf("Expected IsSystemAccount to be %v, got %v", isSys, ais) + } + checkLeafNodeConnected(t, hub) + } + checkInfoOnConnect("a", "A", false) + checkInfoOnConnect("sys", "SYS", test.hasSys) + checkInfoOnConnect("b", "B", false) + }) + } +} diff --git a/test/log_test.go b/test/log_test.go new file mode 100644 index 0000000..d5d29c4 --- /dev/null +++ b/test/log_test.go @@ -0,0 +1,132 @@ +package test + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" +) + +func RunServerWithLogging(opts *server.Options) *server.Server { + if opts == nil { + dto := DefaultTestOptions + opts = &dto + } + opts.NoLog = false + opts.Cluster.PoolSize = -1 + opts.Cluster.Compression.Mode = server.CompressionOff + opts.LeafNode.Compression.Mode = server.CompressionOff + s, err := server.NewServer(opts) + if err != nil || s == nil { + panic(fmt.Sprintf("No NATS Server object returned: %v", err)) + } + s.ConfigureLogger() + go s.Start() + if !s.ReadyForConnections(10 * time.Second) { + panic("Unable to start NATS Server in Go Routine") + } + return s +} + +func TestLogMaxArchives(t *testing.T) { + // With logfile_size_limit set to small 100 characters, plain startup rotates 8 times + for _, test := range []struct { + name string + config string + totEntriesExpected int + }{ + { + "Default implicit, no max logs, expect 0 purged logs", + ` + port: -1 + log_file: %s + logfile_size_limit: 100 + `, + 9, + }, + { + "Default explicit, no max logs, expect 0 purged logs", + ` + port: -1 + log_file: %s + logfile_size_limit: 100 + logfile_max_num: 0 + `, + 9, + }, + { + "Default explicit - negative val, no max logs, expect 0 purged logs", + ` + port: -1 + log_file: %s + logfile_size_limit: 100 + logfile_max_num: -42 + `, + 9, + }, + { + "1-max num, expect 8 purged logs", + ` + port: -1 + log_file: %s + logfile_size_limit: 100 + logfile_max_num: 1 + `, + 1, + }, + { + "5-max num, expect 4 purged logs; use opt alias", + ` + port: -1 + log_file: %s + log_size_limit: 100 + log_max_num: 5 + `, + 5, + }, + { + "100-max num, expect 0 purged logs", + ` + port: -1 + log_file: %s + logfile_size_limit: 100 + logfile_max_num: 100 + `, + 9, + }, + } { + t.Run(test.name, func(t *testing.T) { + d, err := os.MkdirTemp("", "logtest") + if err != nil { + t.Fatalf("Error creating temp dir: %v", err) + } + content := fmt.Sprintf(test.config, filepath.Join(d, "nats-server.log")) + // server config does not like plain windows backslash + if runtime.GOOS == "windows" { + content = filepath.ToSlash(content) + } + opts, err := server.ProcessConfigFile(createConfFile(t, []byte(content))) + if err != nil { + t.Fatalf("Error processing config file: %v", err) + } + s := RunServerWithLogging(opts) + if s == nil { + t.Fatalf("No NATS Server object returned") + } + s.Shutdown() + // Windows filesystem can be a little pokey on the flush, so wait a bit after shutdown... + time.Sleep(500 * time.Millisecond) + entries, err := os.ReadDir(d) + if err != nil { + t.Fatalf("Error reading dir: %v", err) + } + if len(entries) != test.totEntriesExpected { + t.Fatalf("Expected %d log files, got %d", test.totEntriesExpected, len(entries)) + } + }) + } +} diff --git a/test/maxpayload_test.go b/test/maxpayload_test.go new file mode 100644 index 0000000..f125e03 --- /dev/null +++ b/test/maxpayload_test.go @@ -0,0 +1,158 @@ +// Copyright 2015-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "net" + "runtime" + "strings" + "testing" + "time" + + "github.com/nats-io/jwt/v2" + "github.com/nats-io/nats.go" + "github.com/nats-io/nkeys" +) + +func TestMaxPayload(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/override.conf") + defer srv.Shutdown() + + endpoint := net.JoinHostPort(opts.Host, fmt.Sprintf("%d", opts.Port)) + nc, err := nats.Connect(fmt.Sprintf("nats://%s/", endpoint)) + if err != nil { + t.Fatalf("Could not connect to server: %v", err) + } + defer nc.Close() + + size := 4 * 1024 * 1024 + big := sizedBytes(size) + err = nc.Publish("foo", big) + + if err != nats.ErrMaxPayload { + t.Fatalf("Expected a Max Payload error") + } + + conn, err := net.DialTimeout("tcp", endpoint, nc.Opts.Timeout) + if err != nil { + t.Fatalf("Could not make a raw connection to the server: %v", err) + } + defer conn.Close() + info := make([]byte, 512) + _, err = conn.Read(info) + if err != nil { + t.Fatalf("Expected an info message to be sent by the server: %s", err) + } + pub := fmt.Sprintf("PUB bar %d\r\n", size) + _, err = conn.Write([]byte(pub)) + if err != nil { + t.Fatalf("Could not publish event to the server: %s", err) + } + + errMsg := make([]byte, 35) + _, err = conn.Read(errMsg) + if err != nil { + t.Fatalf("Expected an error message to be sent by the server: %s", err) + } + + if !strings.Contains(string(errMsg), "Maximum Payload Violation") { + t.Errorf("Received wrong error message (%v)\n", string(errMsg)) + } + + // Client proactively omits sending the message so server + // does not close the connection. + if nc.IsClosed() { + t.Errorf("Expected connection to not be closed.") + } + + // On the other hand client which did not proactively omitted + // publishing the bytes following what is suggested by server + // in the info message has its connection closed. + _, err = conn.Write(big) + if err == nil && runtime.GOOS != "windows" { + t.Errorf("Expected error due to maximum payload transgression.") + } + + // On windows, the previous write will not fail because the connection + // is not fully closed at this stage. + if runtime.GOOS == "windows" { + // Issuing a PING and not expecting the PONG. + _, err = conn.Write([]byte("PING\r\n")) + if err == nil { + conn.SetReadDeadline(time.Now().Add(500 * time.Millisecond)) + _, err = conn.Read(big) + if err == nil { + t.Errorf("Expected closed connection due to maximum payload transgression.") + } + } + } +} + +func TestMaxPayloadOverrun(t *testing.T) { + opts := DefaultTestOptions + opts.Port = -1 + opts.MaxPayload = 10000 + s := RunServer(&opts) + defer s.Shutdown() + + // Overrun a int32 + c := createClientConn(t, "127.0.0.1", opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + send("PUB foo 199380988\r\n") + expect(errRe) + + // Now overrun an int64, parseSize will have returned -1, + // so we get disconnected. + c = createClientConn(t, "127.0.0.1", opts.Port) + defer c.Close() + + send, _ = setupConn(t, c) + send("PUB foo 18446744073709551615123\r\n") + expectDisconnect(t, c) +} + +func TestAsyncInfoWithSmallerMaxPayload(t *testing.T) { + s, opts := runOperatorServer(t) + defer s.Shutdown() + + const testMaxPayload = 522 + + okp, _ := nkeys.FromSeed(oSeed) + akp, _ := nkeys.CreateAccount() + apub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(apub) + nac.Limits.Payload = testMaxPayload + ajwt, err := nac.Encode(okp) + if err != nil { + t.Fatalf("Error generating account JWT: %v", err) + } + if err := s.AccountResolver().Store(apub, ajwt); err != nil { + t.Fatalf("Account Resolver returned an error: %v", err) + } + + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(url, createUserCreds(t, s, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + nc.Flush() + defer nc.Close() + + if mp := nc.MaxPayload(); mp != testMaxPayload { + t.Fatalf("Expected MaxPayload of %d, got %d", testMaxPayload, mp) + } +} diff --git a/test/monitor_test.go b/test/monitor_test.go new file mode 100644 index 0000000..5d67f16 --- /dev/null +++ b/test/monitor_test.go @@ -0,0 +1,884 @@ +// Copyright 2012-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" +) + +const CLIENT_PORT = 11422 +const MONITOR_PORT = 11522 + +func runMonitorServer() *server.Server { + resetPreviousHTTPConnections() + opts := DefaultTestOptions + opts.Port = CLIENT_PORT + opts.HTTPPort = MONITOR_PORT + opts.HTTPHost = "127.0.0.1" + opts.NoSystemAccount = true + return RunServer(&opts) +} + +// Runs a clustered pair of monitor servers for testing the /routez endpoint +func runMonitorServerClusteredPair(t *testing.T) (*server.Server, *server.Server) { + resetPreviousHTTPConnections() + opts := DefaultTestOptions + opts.Port = CLIENT_PORT + opts.HTTPPort = MONITOR_PORT + opts.HTTPHost = "127.0.0.1" + opts.Cluster = server.ClusterOpts{Name: "M", Host: "127.0.0.1", Port: 10223} + opts.Routes = server.RoutesFromStr("nats-route://127.0.0.1:10222") + opts.NoSystemAccount = true + + s1 := RunServer(&opts) + + opts2 := DefaultTestOptions + opts2.Port = CLIENT_PORT + 1 + opts2.HTTPPort = MONITOR_PORT + 1 + opts2.HTTPHost = "127.0.0.1" + opts2.Cluster = server.ClusterOpts{Name: "M", Host: "127.0.0.1", Port: 10222} + opts2.Routes = server.RoutesFromStr("nats-route://127.0.0.1:10223") + opts2.NoSystemAccount = true + + s2 := RunServer(&opts2) + + checkClusterFormed(t, s1, s2) + + return s1, s2 +} + +func runMonitorServerNoHTTPPort() *server.Server { + resetPreviousHTTPConnections() + opts := DefaultTestOptions + opts.Port = CLIENT_PORT + opts.HTTPPort = 0 + opts.NoSystemAccount = true + + return RunServer(&opts) +} + +func resetPreviousHTTPConnections() { + http.DefaultTransport.(*http.Transport).CloseIdleConnections() +} + +// Make sure that we do not run the http server for monitoring unless asked. +func TestNoMonitorPort(t *testing.T) { + s := runMonitorServerNoHTTPPort() + defer s.Shutdown() + + url := fmt.Sprintf("http://127.0.0.1:%d/", MONITOR_PORT) + if resp, err := http.Get(url + "varz"); err == nil { + t.Fatalf("Expected error: Got %+v\n", resp) + } + if resp, err := http.Get(url + "healthz"); err == nil { + t.Fatalf("Expected error: Got %+v\n", resp) + } + if resp, err := http.Get(url + "connz"); err == nil { + t.Fatalf("Expected error: Got %+v\n", resp) + } +} + +// testEndpointDataRace tests a monitoring endpoint for data races by polling +// while client code acts to ensure statistics are updated. It is designed to +// run under the -race flag to catch violations. The caller must start the +// NATS server. +func testEndpointDataRace(endpoint string, t *testing.T) { + var doneWg sync.WaitGroup + + url := fmt.Sprintf("http://127.0.0.1:%d/", MONITOR_PORT) + + // Poll as fast as we can, while creating connections, publishing, + // and subscribing. + clientDone := int64(0) + doneWg.Add(1) + go func() { + for atomic.LoadInt64(&clientDone) == 0 { + resp, err := http.Get(url + endpoint) + if err != nil { + t.Errorf("Expected no error: Got %v\n", err) + } else { + resp.Body.Close() + } + } + doneWg.Done() + }() + + // create connections, subscriptions, and publish messages to + // update the monitor variables. + var conns []net.Conn + for i := 0; i < 50; i++ { + cl := createClientConnSubscribeAndPublish(t) + // keep a few connections around to test monitor variables. + if i%10 == 0 { + conns = append(conns, cl) + } else { + cl.Close() + } + } + atomic.AddInt64(&clientDone, 1) + + // wait for the endpoint polling goroutine to exit + doneWg.Wait() + + // cleanup the conns + for _, cl := range conns { + cl.Close() + } +} + +func TestEndpointDataRaces(t *testing.T) { + // setup a small cluster to test /routez + s1, s2 := runMonitorServerClusteredPair(t) + defer s1.Shutdown() + defer s2.Shutdown() + + // test all of our endpoints + testEndpointDataRace("varz", t) + testEndpointDataRace("connz", t) + testEndpointDataRace("routez", t) + testEndpointDataRace("subsz", t) + testEndpointDataRace("stacksz", t) +} + +func TestVarz(t *testing.T) { + s := runMonitorServer() + defer s.Shutdown() + + url := fmt.Sprintf("http://127.0.0.1:%d/", MONITOR_PORT) + resp, err := http.Get(url + "varz") + if err != nil { + t.Fatalf("Expected no error: Got %v\n", err) + } + if resp.StatusCode != 200 { + t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Got an error reading the body: %v\n", err) + } + + v := server.Varz{} + if err := json.Unmarshal(body, &v); err != nil { + t.Fatalf("Got an error unmarshalling the body: %v\n", err) + } + + // Do some sanity checks on values + if time.Since(v.Start) > 10*time.Second { + t.Fatal("Expected start time to be within 10 seconds.") + } + + cl := createClientConnSubscribeAndPublish(t) + defer cl.Close() + + resp, err = http.Get(url + "varz") + if err != nil { + t.Fatalf("Expected no error: Got %v\n", err) + } + if resp.StatusCode != 200 { + t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) + } + defer resp.Body.Close() + body, err = io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Got an error reading the body: %v\n", err) + } + + if strings.Contains(string(body), "cluster_port") { + t.Fatal("Varz body contains cluster information when no cluster is defined.") + } + + v = server.Varz{} + if err := json.Unmarshal(body, &v); err != nil { + t.Fatalf("Got an error unmarshalling the body: %v\n", err) + } + + if v.Connections != 1 { + t.Fatalf("Expected Connections of 1, got %v\n", v.Connections) + } + if v.InMsgs != 1 { + t.Fatalf("Expected InMsgs of 1, got %v\n", v.InMsgs) + } + if v.OutMsgs != 1 { + t.Fatalf("Expected OutMsgs of 1, got %v\n", v.OutMsgs) + } + if v.InBytes != 5 { + t.Fatalf("Expected InBytes of 5, got %v\n", v.InBytes) + } + if v.OutBytes != 5 { + t.Fatalf("Expected OutBytes of 5, got %v\n", v.OutBytes) + } + if v.MaxPending != server.MAX_PENDING_SIZE { + t.Fatalf("Expected MaxPending of %d, got %v\n", + server.MAX_PENDING_SIZE, v.MaxPending) + } + if v.WriteDeadline != server.DEFAULT_FLUSH_DEADLINE { + t.Fatalf("Expected WriteDeadline of %d, got %v\n", + server.DEFAULT_FLUSH_DEADLINE, v.WriteDeadline) + } +} + +func TestConnz(t *testing.T) { + s := runMonitorServer() + defer s.Shutdown() + + url := fmt.Sprintf("http://127.0.0.1:%d/", MONITOR_PORT) + resp, err := http.Get(url + "connz") + if err != nil { + t.Fatalf("Expected no error: Got %v\n", err) + } + if resp.StatusCode != 200 { + t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Got an error reading the body: %v\n", err) + } + + c := server.Connz{} + if err := json.Unmarshal(body, &c); err != nil { + t.Fatalf("Got an error unmarshalling the body: %v\n", err) + } + + // Test contents.. + if c.NumConns != 0 { + t.Fatalf("Expected 0 connections, got %d\n", c.NumConns) + } + if c.Total != 0 { + t.Fatalf("Expected 0 live connections, got %d\n", c.Total) + } + if c.Conns == nil || len(c.Conns) != 0 { + t.Fatalf("Expected 0 connections in array, got %p\n", c.Conns) + } + + cl := createClientConnSubscribeAndPublish(t) + defer cl.Close() + + resp, err = http.Get(url + "connz") + if err != nil { + t.Fatalf("Expected no error: Got %v\n", err) + } + if resp.StatusCode != 200 { + t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) + } + defer resp.Body.Close() + body, err = io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Got an error reading the body: %v\n", err) + } + if err := json.Unmarshal(body, &c); err != nil { + t.Fatalf("Got an error unmarshalling the body: %v\n", err) + } + + if c.NumConns != 1 { + t.Fatalf("Expected 1 connection, got %d\n", c.NumConns) + } + if c.Total != 1 { + t.Fatalf("Expected 1 live connection, got %d\n", c.Total) + } + if c.Conns == nil || len(c.Conns) != 1 { + t.Fatalf("Expected 1 connection in array, got %p\n", c.Conns) + } + + if c.Limit != server.DefaultConnListSize { + t.Fatalf("Expected limit of %d, got %v\n", server.DefaultConnListSize, c.Limit) + } + + if c.Offset != 0 { + t.Fatalf("Expected offset of 0, got %v\n", c.Offset) + } + + // Test inside details of each connection + ci := c.Conns[0] + + if ci.Cid == 0 { + t.Fatalf("Expected non-zero cid, got %v\n", ci.Cid) + } + if ci.IP != "127.0.0.1" { + t.Fatalf("Expected \"127.0.0.1\" for IP, got %v\n", ci.IP) + } + if ci.Port == 0 { + t.Fatalf("Expected non-zero port, got %v\n", ci.Port) + } + if ci.NumSubs != 1 { + t.Fatalf("Expected num_subs of 1, got %v\n", ci.NumSubs) + } + if len(ci.Subs) != 0 { + t.Fatalf("Expected subs of 0, got %v\n", ci.Subs) + } + if ci.InMsgs != 1 { + t.Fatalf("Expected InMsgs of 1, got %v\n", ci.InMsgs) + } + if ci.OutMsgs != 1 { + t.Fatalf("Expected OutMsgs of 1, got %v\n", ci.OutMsgs) + } + if ci.InBytes != 5 { + t.Fatalf("Expected InBytes of 1, got %v\n", ci.InBytes) + } + if ci.OutBytes != 5 { + t.Fatalf("Expected OutBytes of 1, got %v\n", ci.OutBytes) + } +} + +func TestTLSConnz(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/tls.conf") + defer srv.Shutdown() + rootCAFile := "./configs/certs/ca.pem" + clientCertFile := "./configs/certs/client-cert.pem" + clientKeyFile := "./configs/certs/client-key.pem" + + // Test with secure connection + endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) + nurl := fmt.Sprintf("tls://%s:%s@%s/", opts.Username, opts.Password, endpoint) + nc, err := nats.Connect(nurl, nats.RootCAs(rootCAFile)) + if err != nil { + t.Fatalf("Got an error on Connect with Secure Options: %+v\n", err) + } + defer nc.Close() + ch := make(chan struct{}) + nc.Subscribe("foo", func(m *nats.Msg) { ch <- struct{}{} }) + nc.Publish("foo", []byte("Hello")) + + // Wait for message + <-ch + + url := fmt.Sprintf("https://127.0.0.1:%d/", opts.HTTPSPort) + tlsConfig := &tls.Config{} + caCert, err := os.ReadFile(rootCAFile) + if err != nil { + t.Fatalf("Got error reading RootCA file: %s", err) + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + tlsConfig.RootCAs = caCertPool + + cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile) + if err != nil { + t.Fatalf("Got error reading client certificates: %s", err) + } + tlsConfig.Certificates = []tls.Certificate{cert} + transport := &http.Transport{TLSClientConfig: tlsConfig} + httpClient := &http.Client{Transport: transport} + + resp, err := httpClient.Get(url + "connz") + if err != nil { + t.Fatalf("Expected no error: Got %v\n", err) + } + if resp.StatusCode != 200 { + t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + + if err != nil { + t.Fatalf("Got an error reading the body: %v\n", err) + } + c := server.Connz{} + if err := json.Unmarshal(body, &c); err != nil { + t.Fatalf("Got an error unmarshalling the body: %v\n", err) + } + + if c.NumConns != 1 { + t.Fatalf("Expected 1 connection, got %d\n", c.NumConns) + } + if c.Total != 1 { + t.Fatalf("Expected 1 live connection, got %d\n", c.Total) + } + if c.Conns == nil || len(c.Conns) != 1 { + t.Fatalf("Expected 1 connection in array, got %d\n", len(c.Conns)) + } + + // Test inside details of each connection + ci := c.Conns[0] + + if ci.Cid == 0 { + t.Fatalf("Expected non-zero cid, got %v\n", ci.Cid) + } + if ci.IP != "127.0.0.1" { + t.Fatalf("Expected \"127.0.0.1\" for IP, got %v\n", ci.IP) + } + if ci.Port == 0 { + t.Fatalf("Expected non-zero port, got %v\n", ci.Port) + } + if ci.NumSubs != 1 { + t.Fatalf("Expected num_subs of 1, got %v\n", ci.NumSubs) + } + if len(ci.Subs) != 0 { + t.Fatalf("Expected subs of 0, got %v\n", ci.Subs) + } + if ci.InMsgs != 1 { + t.Fatalf("Expected InMsgs of 1, got %v\n", ci.InMsgs) + } + if ci.OutMsgs != 1 { + t.Fatalf("Expected OutMsgs of 1, got %v\n", ci.OutMsgs) + } + if ci.InBytes != 5 { + t.Fatalf("Expected InBytes of 1, got %v\n", ci.InBytes) + } + if ci.OutBytes != 5 { + t.Fatalf("Expected OutBytes of 1, got %v\n", ci.OutBytes) + } + if ci.Start.IsZero() { + t.Fatalf("Expected Start to be valid\n") + } + if ci.Uptime == "" { + t.Fatalf("Expected Uptime to be valid\n") + } + if ci.LastActivity.IsZero() { + t.Fatalf("Expected LastActivity to be valid\n") + } + if ci.LastActivity.UnixNano() < ci.Start.UnixNano() { + t.Fatalf("Expected LastActivity [%v] to be > Start [%v]\n", ci.LastActivity, ci.Start) + } + if ci.Idle == "" { + t.Fatalf("Expected Idle to be valid\n") + } +} + +func TestConnzWithSubs(t *testing.T) { + s := runMonitorServer() + defer s.Shutdown() + + cl := createClientConnSubscribeAndPublish(t) + defer cl.Close() + + url := fmt.Sprintf("http://127.0.0.1:%d/", MONITOR_PORT) + resp, err := http.Get(url + "connz?subs=1") + if err != nil { + t.Fatalf("Expected no error: Got %v\n", err) + } + if resp.StatusCode != 200 { + t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Got an error reading the body: %v\n", err) + } + + c := server.Connz{} + if err := json.Unmarshal(body, &c); err != nil { + t.Fatalf("Got an error unmarshalling the body: %v\n", err) + } + + // Test inside details of each connection + ci := c.Conns[0] + if len(ci.Subs) != 1 || ci.Subs[0] != "foo" { + t.Fatalf("Expected subs of 1, got %v\n", ci.Subs) + } +} + +func TestConnzWithAuth(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/multi_user.conf") + defer srv.Shutdown() + + endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) + curl := fmt.Sprintf("nats://%s:%s@%s/", opts.Users[0].Username, opts.Users[0].Password, endpoint) + nc, err := nats.Connect(curl) + if err != nil { + t.Fatalf("Got an error on Connect: %+v\n", err) + } + defer nc.Close() + + ch := make(chan struct{}) + nc.Subscribe("foo", func(m *nats.Msg) { ch <- struct{}{} }) + nc.Publish("foo", []byte("Hello")) + + // Wait for message + <-ch + + url := fmt.Sprintf("http://127.0.0.1:%d/", opts.HTTPPort) + + resp, err := http.Get(url + "connz?auth=1") + if err != nil { + t.Fatalf("Expected no error: Got %v\n", err) + } + if resp.StatusCode != 200 { + t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Got an error reading the body: %v\n", err) + } + + c := server.Connz{} + if err := json.Unmarshal(body, &c); err != nil { + t.Fatalf("Got an error unmarshalling the body: %v\n", err) + } + + // Test that we have authorized_user and its Alice. + ci := c.Conns[0] + if ci.AuthorizedUser != opts.Users[0].Username { + t.Fatalf("Expected authorized_user to be %q, got %q\n", + opts.Users[0].Username, ci.AuthorizedUser) + } +} + +func TestConnzWithAccounts(t *testing.T) { + resetPreviousHTTPConnections() + s, opts := RunServerWithConfig("./configs/multi_accounts.conf") + defer s.Shutdown() + + endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) + + // Connect all the users. Tests depend on knowing users, accounts. + if len(opts.Users) != 6 { + t.Fatalf("Expected 6 total users, got %d", len(opts.Users)) + } + if len(opts.Accounts) != 3 { + t.Fatalf("Expected 3 total accounts, got %d", len(opts.Accounts)) + } + + // Map from user to account name. + utoa := make(map[string]string) + conns := make([]*nats.Conn, len(opts.Users)) + for _, u := range opts.Users { + nc, err := nats.Connect(fmt.Sprintf("nats://%s:%s@%s/", u.Username, u.Password, endpoint)) + if err != nil { + t.Fatalf("Got an error on Connect: %+v\n", err) + } + defer nc.Close() + utoa[u.Username] = u.Account.Name + conns = append(conns, nc) + } + + url := fmt.Sprintf("http://127.0.0.1:%d/", opts.HTTPPort) + + grabConnz := func(args string) *server.Connz { + t.Helper() + if args != "" { + args = "&" + args + } + resp, err := http.Get(url + "connz?auth=1" + args) + if err != nil { + t.Fatalf("Expected no error: Got %v\n", err) + } + if resp.StatusCode != 200 { + t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Got an error reading the body: %v\n", err) + } + c := server.Connz{} + if err := json.Unmarshal(body, &c); err != nil { + t.Fatalf("Got an error unmarshalling the body: %v\n", err) + } + return &c + } + + c := grabConnz("") + if c.NumConns != 6 { + t.Fatalf("Expected 6 connection entries, got %d", c.NumConns) + } + + checkConn := func(ci *server.ConnInfo) { + t.Helper() + user := ci.AuthorizedUser + account := utoa[user] + if user == "" || account == "" { + t.Fatalf("Empty user or account: %q - %q", user, account) + } + if ci.Account != account { + t.Fatalf("Expected account of %q, got %q", account, ci.Account) + } + } + + for i, ci := range c.Conns { + if ci.Cid != uint64(i+1) { + t.Fatalf("Expected CID of %d, got %d", i+1, ci.Cid) + } + checkConn(ci) + } + + // Now make sure we can pull connections by account and user + pullByAccount := func(accName, state string) { + t.Helper() + c = grabConnz("acc=" + accName + "&state=" + state) + if c.NumConns != 2 { + t.Fatalf("Expected 2 connection entries, got %d", c.NumConns) + } + for _, ci := range c.Conns { + if ci.Account != accName { + t.Fatalf("Expected %q account, go %q", accName, ci.Account) + } + } + } + + pullByUser := func(user, state string) { + t.Helper() + c = grabConnz("user=" + user + "&state=" + state) + if c.NumConns != 1 { + t.Fatalf("Expected 1 connection, got %d", c.NumConns) + } + if c.Conns[0].AuthorizedUser != user { + t.Fatalf("Expected user %q, got %q", user, c.Conns[0].AuthorizedUser) + } + } + + pullByAccount("engineering", "open") + pullByAccount("finance", "open") + pullByAccount("legal", "open") + + pullByUser("alice", "open") + pullByUser("bob", "open") + + pullByUser("john", "open") + pullByUser("mary", "open") + + pullByUser("peter", "open") + pullByUser("paul", "open") + + // Now closed and make sure these work on closed as well. + for _, nc := range conns { + nc.Close() + } + + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if numClients := s.NumClients(); numClients != 0 { + return fmt.Errorf("Number of client is %d", numClients) + } + return nil + }) + + pullByAccount("engineering", "closed") + pullByAccount("finance", "closed") + pullByAccount("legal", "closed") + + pullByUser("alice", "closed") + pullByUser("bob", "closed") + + pullByUser("john", "closed") + pullByUser("mary", "closed") + + pullByUser("peter", "closed") + pullByUser("paul", "closed") +} + +func TestConnzWithOffsetAndLimit(t *testing.T) { + s := runMonitorServer() + defer s.Shutdown() + + cl1 := createClientConnSubscribeAndPublish(t) + defer cl1.Close() + + cl2 := createClientConnSubscribeAndPublish(t) + defer cl2.Close() + + url := fmt.Sprintf("http://127.0.0.1:%d/", MONITOR_PORT) + resp, err := http.Get(url + "connz?offset=1&limit=1") + if err != nil { + t.Fatalf("Expected no error: Got %v\n", err) + } + if resp.StatusCode != 200 { + t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Got an error reading the body: %v\n", err) + } + + c := server.Connz{} + if err := json.Unmarshal(body, &c); err != nil { + t.Fatalf("Got an error unmarshalling the body: %v\n", err) + } + + if c.Limit != 1 { + t.Fatalf("Expected limit of 1, got %v\n", c.Limit) + } + + if c.Offset != 1 { + t.Fatalf("Expected offset of 1, got %v\n", c.Offset) + } + + if len(c.Conns) != 1 { + t.Fatalf("Expected conns of 1, got %v\n", len(c.Conns)) + } +} + +func TestSubsz(t *testing.T) { + s := runMonitorServer() + defer s.Shutdown() + + cl := createClientConnSubscribeAndPublish(t) + defer cl.Close() + + url := fmt.Sprintf("http://127.0.0.1:%d/", MONITOR_PORT) + resp, err := http.Get(url + "subscriptionsz") + if err != nil { + t.Fatalf("Expected no error: Got %v\n", err) + } + if resp.StatusCode != 200 { + t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Got an error reading the body: %v\n", err) + } + + su := server.Subsz{} + if err := json.Unmarshal(body, &su); err != nil { + t.Fatalf("Got an error unmarshalling the body: %v\n", err) + } + + // Do some sanity checks on values + if su.NumSubs != 1 { + t.Fatalf("Expected num_subs of 1, got %v\n", su.NumSubs) + } +} + +func TestHTTPHost(t *testing.T) { + s := runMonitorServer() + defer s.Shutdown() + + // Grab non-127.0.0.1 address and try to use that to connect. + // Should fail. + var ip net.IP + ifaces, _ := net.Interfaces() + for _, i := range ifaces { + addrs, _ := i.Addrs() + for _, addr := range addrs { + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + // Skip loopback/127.0.0.1 or any ipv6 for now. + if ip.IsLoopback() || ip.To4() == nil { + ip = nil + continue + } + break + } + if ip != nil { + break + } + } + if ip == nil { + t.Fatalf("Could not find non-loopback IPV4 address") + } + url := fmt.Sprintf("http://%v:%d/", ip, MONITOR_PORT) + if resp, err := http.Get(url + "varz"); err == nil { + t.Fatalf("Expected error: Got %+v\n", resp) + } +} + +// Create a connection to test ConnInfo +func createClientConnSubscribeAndPublish(t *testing.T) net.Conn { + cl := createClientConn(t, "127.0.0.1", CLIENT_PORT) + send, expect := setupConn(t, cl) + expectMsgs := expectMsgsCommand(t, expect) + + send("SUB foo 1\r\nPUB foo 5\r\nhello\r\n") + expectMsgs(1) + + return cl +} + +func TestMonitorNoTLSConfig(t *testing.T) { + opts := DefaultTestOptions + opts.Port = CLIENT_PORT + opts.HTTPHost = "127.0.0.1" + opts.HTTPSPort = MONITOR_PORT + s := server.New(&opts) + defer s.Shutdown() + // Check with manually starting the monitoring, which should return an error + if err := s.StartMonitoring(); err == nil || !strings.Contains(err.Error(), "TLS") { + t.Fatalf("Expected error about missing TLS config, got %v", err) + } + // Also check by calling Start(), which should produce a fatal error + dl := &dummyLogger{} + s.SetLogger(dl, false, false) + defer s.SetLogger(nil, false, false) + s.Start() + if !strings.Contains(dl.msg, "TLS") { + t.Fatalf("Expected error about missing TLS config, got %v", dl.msg) + } +} + +func TestMonitorErrorOnListen(t *testing.T) { + s := runMonitorServer() + defer s.Shutdown() + + opts := DefaultTestOptions + opts.Port = CLIENT_PORT + 1 + opts.HTTPHost = "127.0.0.1" + opts.HTTPPort = MONITOR_PORT + s2 := server.New(&opts) + defer s2.Shutdown() + if err := s2.StartMonitoring(); err == nil || !strings.Contains(err.Error(), "listen") { + t.Fatalf("Expected error about not able to start listener, got %v", err) + } +} + +func TestMonitorBothPortsConfigured(t *testing.T) { + opts := DefaultTestOptions + opts.Port = CLIENT_PORT + opts.HTTPHost = "127.0.0.1" + opts.HTTPPort = MONITOR_PORT + opts.HTTPSPort = MONITOR_PORT + 1 + s := server.New(&opts) + defer s.Shutdown() + if err := s.StartMonitoring(); err == nil || !strings.Contains(err.Error(), "specify both") { + t.Fatalf("Expected error about ports configured, got %v", err) + } +} + +func TestMonitorStop(t *testing.T) { + resetPreviousHTTPConnections() + opts := DefaultTestOptions + opts.Port = CLIENT_PORT + opts.HTTPHost = "127.0.0.1" + opts.HTTPPort = MONITOR_PORT + url := fmt.Sprintf("http://%v:%d/", opts.HTTPHost, MONITOR_PORT) + // Create a server instance and start only the monitoring http server. + s := server.New(&opts) + if err := s.StartMonitoring(); err != nil { + t.Fatalf("Error starting monitoring: %v", err) + } + // Make sure http server is started + resp, err := http.Get(url + "varz") + if err != nil { + t.Fatalf("Error on http request: %v", err) + } + resp.Body.Close() + // Although the server itself was not started (we did not call s.Start()), + // Shutdown() should stop the http server. + s.Shutdown() + // HTTP request should now fail + if resp, err := http.Get(url + "varz"); err == nil { + t.Fatalf("Expected error: Got %+v\n", resp) + } +} diff --git a/test/new_routes_test.go b/test/new_routes_test.go new file mode 100644 index 0000000..03f24c1 --- /dev/null +++ b/test/new_routes_test.go @@ -0,0 +1,2128 @@ +// Copyright 2018-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "encoding/json" + "fmt" + "net" + "regexp" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/logger" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" +) + +func runNewRouteServer(t *testing.T) (*server.Server, *server.Options) { + return RunServerWithConfig("./configs/new_cluster.conf") +} + +func TestNewRouteInfoOnConnect(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + info := checkInfoMsg(t, rc) + if info.Port != opts.Cluster.Port { + t.Fatalf("Received wrong information for port, expected %d, got %d", + info.Port, opts.Cluster.Port) + } + + // Make sure we advertise new proto. + if info.Proto < server.RouteProtoV2 { + t.Fatalf("Expected routeProtoV2 or above, got %d", info.Proto) + } + // New proto should always send nonce too. + if info.Nonce == "" { + t.Fatalf("Expected a non empty nonce in new route INFO") + } + // By default headers should be true. + if !info.Headers { + t.Fatalf("Expected to have headers on by default") + } + // Leafnode origin cluster support. + if !info.LNOC { + t.Fatalf("Expected to have leafnode origin cluster support") + } + if !info.LNOCU { + t.Fatalf("Expected to have leafnode origin cluster in unsub protocol support") + } +} + +func TestNewRouteLNImpliesLeafNodeOriginSupport(t *testing.T) { + t.Run("CONNECT", func(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + checkInfoMsg(t, rc) + send, expect := sendCommand(t, rc), expectCommand(t, rc) + send(`CONNECT {"name":"LN-CONNECT","cluster":"xyz","ln":true}` + "\r\n") + send("LS+ leaf $G foo\r\nLS- leaf $G foo\r\nPING\r\n") + expect(pongRe) + }) + + t.Run("INFO", func(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + info := checkInfoMsg(t, rc) + routeSend, routeExpect := setupRouteEx(t, rc, opts, "LN-INFO") + info.ID, info.Name = "LN-INFO", "" + info.LN, info.LNOC, info.LNOCU = true, false, false + b, err := json.Marshal(info) + if err != nil { + t.Fatalf("Could not marshal test route info: %v", err) + } + routeSend(fmt.Sprintf("INFO %s\r\n", b)) + routeSend("LS+ leaf $G foo\r\nLS- leaf $G foo\r\nPING\r\n") + routeExpect(pongRe) + }) +} + +func TestNewRouteLNLeafInterestProtocol(t *testing.T) { + hubConf := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + cluster { name: xyz, listen: 127.0.0.1:-1 } + leafnodes { listen: 127.0.0.1:-1, no_advertise: true } + no_sys_acc: true + `)) + hub, opts := RunServerWithConfig(hubConf) + defer hub.Shutdown() + + leafConf := createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:-1 + leafnodes { remotes: [{url: "nats-leaf://127.0.0.1:%d"}] } + no_sys_acc: true + `, opts.LeafNode.Port))) + leaf, _ := RunServerWithConfig(leafConf) + defer leaf.Shutdown() + checkLeafNodeConnected(t, hub) + + lnSubRe := regexp.MustCompile(`LS\+\s+_\s+([^\s]+)\s+([^\s]+)\s*([^\s]+)?\s*(\d+)?\r\n`) + lnUnsubRe := regexp.MustCompile(`LS\-\s+_\s+([^\s]+)\s+([^\s]+)\s*([^\s]+)?\r\n`) + + for _, test := range []struct { + name string + ln bool + subRe *regexp.Regexp + unsubRe *regexp.Regexp + }{ + {"LN", true, lnSubRe, lnUnsubRe}, + {"pre-LN", false, rsubRe, runsubRe}, + } { + t.Run(test.name, func(t *testing.T) { + nc, err := nats.Connect(leaf.ClientURL()) + if err != nil { + t.Fatalf("Unexpected client connect error: %v", err) + } + defer nc.Close() + sub, err := nc.SubscribeSync("foo") + if err != nil { + t.Fatalf("Unexpected subscribe error: %v", err) + } + if err := nc.Flush(); err != nil { + t.Fatalf("Unexpected flush error: %v", err) + } + checkSubInterest(t, hub, "$G", "foo", time.Second) + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + routeID := "LN-PROTO-" + test.name + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + info := checkInfoMsg(t, rc) + info.ID, info.Name, info.LN = routeID, "", test.ln + b, err := json.Marshal(info) + if err != nil { + t.Fatalf("Could not marshal test route info: %v", err) + } + routeSend(fmt.Sprintf("INFO %s\r\n", b)) + routeExpect(test.subRe) + + if err := sub.Unsubscribe(); err != nil { + t.Fatalf("Unexpected unsubscribe error: %v", err) + } + routeExpect(test.unsubRe) + }) + } +} + +func TestNewRouteLNIsolatedLeafInterest(t *testing.T) { + hub1Conf := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + cluster { name: xyz, listen: 127.0.0.1:-1 } + leafnodes { listen: 127.0.0.1:-1, no_advertise: true } + no_sys_acc: true + `)) + hub1, hub1Opts := RunServerWithConfig(hub1Conf) + defer hub1.Shutdown() + + hub2Conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:-1 + cluster { + name: xyz + listen: 127.0.0.1:-1 + routes: ["nats-route://127.0.0.1:%d"] + } + leafnodes { listen: 127.0.0.1:-1, no_advertise: true } + no_sys_acc: true + `, hub1Opts.Cluster.Port))) + hub2, hub2Opts := RunServerWithConfig(hub2Conf) + defer hub2.Shutdown() + checkClusterFormed(t, hub1, hub2) + + startLeaf := func(name string, isolated bool) *server.Server { + t.Helper() + conf := createConfFile(t, []byte(fmt.Sprintf(` + server_name: %s + listen: 127.0.0.1:-1 + leafnodes { + remotes: [{ + url: "nats-leaf://127.0.0.1:%d" + request_isolation: %v + }] + } + no_sys_acc: true + `, name, hub2Opts.LeafNode.Port, isolated))) + leaf, _ := RunServerWithConfig(conf) + return leaf + } + + normalLeaf := startLeaf("NORMAL", false) + defer normalLeaf.Shutdown() + isolatedLeaf := startLeaf("ISOLATED", true) + defer isolatedLeaf.Shutdown() + checkLeafNodeConnected(t, normalLeaf) + checkLeafNodeConnected(t, isolatedLeaf) + + // Use a raw leaf connection with no cluster in CONNECT. Its interest must + // cross the hub route as LS+ with the no-origin sentinel so the second hub retains leaf provenance. + source := createLeafConn(t, hub1Opts.LeafNode.Host, hub1Opts.LeafNode.Port) + defer source.Close() + checkInfoMsg(t, source) + sourceSend := sendCommand(t, source) + sourceSend("CONNECT {}\r\nLS+ eastwest\r\n") + + checkSubInterest(t, hub2, "$G", "eastwest", time.Second) + checkSubInterest(t, normalLeaf, "$G", "eastwest", time.Second) + isolatedAcc, err := isolatedLeaf.LookupAccount("$G") + if err != nil { + t.Fatalf("Could not lookup isolated leaf account: %v", err) + } + if isolatedAcc.SubscriptionInterest("eastwest") { + t.Fatal("Isolated leaf unexpectedly received routed leaf interest") + } + + sourceSend("LS- eastwest\r\n") + checkFor(t, time.Second, 10*time.Millisecond, func() error { + acc, err := normalLeaf.LookupAccount("$G") + if err != nil { + return err + } + if acc.SubscriptionInterest("eastwest") { + return fmt.Errorf("normal leaf still has east-west interest") + } + return nil + }) +} + +func TestNewRouteHeaderSupport(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + sendA, expectA := setupHeaderConn(t, clientA) + sendA("SUB foo bar 22\r\n") + sendA("PING\r\n") + expectA(pongRe) + + if err := checkExpectedSubs(1, srvA, srvB); err != nil { + t.Fatalf("%v", err) + } + + sendB, expectB := setupHeaderConn(t, clientB) + // Can not have \r\n in payload fyi for regex. + sendB("HPUB foo reply 12 14\r\nK1:V1,K2:V2 ok\r\n") + sendB("PING\r\n") + expectB(pongRe) + + expectHeaderMsgs := expectHeaderMsgsCommand(t, expectA) + matches := expectHeaderMsgs(1) + checkHmsg(t, matches[0], "foo", "22", "reply", "12", "14", "K1:V1,K2:V2 ", "ok") +} + +func TestNewRouteHeaderSupportOldAndNew(t *testing.T) { + optsA := LoadConfig("./configs/srv_a.conf") + optsA.NoHeaderSupport = true + srvA := RunServer(optsA) + defer srvA.Shutdown() + + srvB, optsB := RunServerWithConfig("./configs/srv_b.conf") + defer srvB.Shutdown() + + checkClusterFormed(t, srvA, srvB) + + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + sendA, expectA := setupHeaderConn(t, clientA) + sendA("SUB foo bar 22\r\n") + sendA("PING\r\n") + expectA(pongRe) + + if err := checkExpectedSubs(1, srvA, srvB); err != nil { + t.Fatalf("%v", err) + } + + sendB, expectB := setupHeaderConn(t, clientB) + // Can not have \r\n in payload fyi for regex. + sendB("HPUB foo reply 12 14\r\nK1:V1,K2:V2 ok\r\n") + sendB("PING\r\n") + expectB(pongRe) + + expectMsgs := expectMsgsCommand(t, expectA) + matches := expectMsgs(1) + checkMsg(t, matches[0], "foo", "22", "reply", "2", "ok") +} + +func sendRouteInfo(t *testing.T, rc net.Conn, routeSend sendFun, routeID string) { + info := checkInfoMsg(t, rc) + info.ID = routeID + info.Name = "" + b, err := json.Marshal(info) + if err != nil { + t.Fatalf("Could not marshal test route info: %v", err) + } + routeSend(fmt.Sprintf("INFO %s\r\n", b)) +} + +func TestNewRouteConnectSubs(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + + // Create 10 normal subs and 10 queue subscribers. + for i := 0; i < 10; i++ { + send(fmt.Sprintf("SUB foo %d\r\n", i)) + send(fmt.Sprintf("SUB foo bar %d\r\n", 100+i)) + } + send("PING\r\n") + expect(pongRe) + + // This client should not be considered active since no subscriptions or + // messages have been published. + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:22" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + sendRouteInfo(t, rc, routeSend, routeID) + buf := routeExpect(rsubRe) + + matches := rsubRe.FindAllSubmatch(buf, -1) + if len(matches) != 2 { + t.Fatalf("Expected 2 results, got %d", len(matches)) + } + for _, m := range matches { + if string(m[1]) != "$G" { + t.Fatalf("Expected global account name of '$G', got %q", m[1]) + } + if string(m[2]) != "foo" { + t.Fatalf("Expected subject of 'foo', got %q", m[2]) + } + if m[3] != nil { + if string(m[3]) != "bar" { + t.Fatalf("Expected group of 'bar', got %q", m[3]) + } + // Expect a weighted count for the queue group + if len(m) != 5 { + t.Fatalf("Expected a weight for the queue group") + } + if m[4] == nil || string(m[4]) != "10" { + t.Fatalf("Expected Weight of '10', got %q", m[4]) + } + } + } + + // Close the client connection, check the results. + c.Close() + + // Expect 2 + for numUnSubs := 0; numUnSubs != 2; { + buf := routeExpect(runsubRe) + numUnSubs += len(runsubRe.FindAllSubmatch(buf, -1)) + } +} + +func TestNewRouteConnectSubsWithAccount(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + accName := "$FOO" + s.RegisterAccount(accName) + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConnWithAccount(t, s, c, accName) + + // Create 10 normal subs and 10 queue subscribers. + for i := 0; i < 10; i++ { + send(fmt.Sprintf("SUB foo %d\r\n", i)) + send(fmt.Sprintf("SUB foo bar %d\r\n", 100+i)) + } + send("PING\r\n") + expect(pongRe) + + // This client should not be considered active since no subscriptions or + // messages have been published. + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:22" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + sendRouteInfo(t, rc, routeSend, routeID) + buf := routeExpect(rsubRe) + + matches := rsubRe.FindAllSubmatch(buf, -1) + if len(matches) != 2 { + t.Fatalf("Expected 2 results, got %d", len(matches)) + } + for _, m := range matches { + if string(m[1]) != accName { + t.Fatalf("Expected global account name of %q, got %q", accName, m[1]) + } + if string(m[2]) != "foo" { + t.Fatalf("Expected subject of 'foo', got %q", m[2]) + } + if m[3] != nil { + if string(m[3]) != "bar" { + t.Fatalf("Expected group of 'bar', got %q", m[3]) + } + // Expect the SID to be the total weighted count for the queue group + if len(m) != 5 { + t.Fatalf("Expected a weight for the queue group") + } + if m[4] == nil || string(m[4]) != "10" { + t.Fatalf("Expected Weight of '10', got %q", m[4]) + } + } + } + + // Close the client connection, check the results. + c.Close() + + // Expect 2 + for numUnSubs := 0; numUnSubs != 2; { + buf := routeExpect(runsubRe) + numUnSubs += len(runsubRe.FindAllSubmatch(buf, -1)) + } +} + +func TestNewRouteRSubs(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + foo, err := s.RegisterAccount("$foo") + if err != nil { + t.Fatalf("Error creating account '$foo': %v", err) + } + bar, err := s.RegisterAccount("$bar") + if err != nil { + t.Fatalf("Error creating account '$bar': %v", err) + } + + // Create a client an account foo. + clientA := createClientConn(t, opts.Host, opts.Port) + sendA, expectA := setupConnWithAccount(t, s, clientA, "$foo") + defer clientA.Close() + sendA("PING\r\n") + expectA(pongRe) + + if foonc := foo.NumConnections(); foonc != 1 { + t.Fatalf("Expected foo account to have 1 client, got %d", foonc) + } + if barnc := bar.NumConnections(); barnc != 0 { + t.Fatalf("Expected bar account to have 0 clients, got %d", barnc) + } + + // Create a routeConn + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:33" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + sendRouteInfo(t, rc, routeSend, routeID) + routeSend("PING\r\n") + routeExpect(pongRe) + + // Have the client listen on foo. + sendA("SUB foo 1\r\nPING\r\n") + expectA(pongRe) + + // Now create a new client for account $bar and have them subscribe. + clientB := createClientConn(t, opts.Host, opts.Port) + sendB, expectB := setupConnWithAccount(t, s, clientB, "$bar") + defer clientB.Close() + + sendB("PING\r\n") + expectB(pongRe) + + if foonc := foo.NumConnections(); foonc != 1 { + t.Fatalf("Expected foo account to have 1 client, got %d", foonc) + } + if barnc := bar.NumConnections(); barnc != 1 { + t.Fatalf("Expected bar account to have 1 client, got %d", barnc) + } + + // Have the client listen on foo. + sendB("SUB foo 1\r\nPING\r\n") + expectB(pongRe) + + routeExpect(rsubRe) + + // Unsubscribe on clientA from foo subject. + sendA("UNSUB 1\r\nPING\r\n") + expectA(pongRe) + + // We should get an RUSUB here. + routeExpect(runsubRe) + + // Now unsubscribe clientB, which should trigger an RS-. + sendB("UNSUB 1\r\nPING\r\n") + expectB(pongRe) + // We should get an RUSUB here. + routeExpect(runsubRe) + + // Now close down the clients. + clientA.Close() + + sendB("SUB foo 2\r\nPING\r\n") + expectB(pongRe) + + routeExpect(rsubRe) + + // Now close down client B. + clientB.Close() + + // This should trigger an RS- + routeExpect(runsubRe) +} + +func TestNewRouteProgressiveNormalSubs(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:33" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + sendRouteInfo(t, rc, routeSend, routeID) + routeSend("PING\r\n") + routeExpect(pongRe) + + // For progressive we will expect to receive first normal sub but + // not subsequent ones. + send("SUB foo 1\r\nPING\r\n") + expect(pongRe) + + routeExpect(rsubRe) + + send("SUB foo 2\r\nPING\r\n") + expect(pongRe) + expectNothing(t, rc) + + var buf []byte + + // Check that sid is showing us total number of subscriptions. + checkQueueSub := func(n string) { + matches := rsubRe.FindAllSubmatch(buf, -1) + if len(matches) != 1 { + t.Fatalf("Expected 1 result, got %d", len(matches)) + } + m := matches[0] + if len(m) != 5 { + t.Fatalf("Expected a SID for the queue group, only got %d elements", len(m)) + } + if string(m[4]) != n { + t.Fatalf("Expected %q, got %q", n, m[4]) + } + } + + // We should always get the SUB info for QUEUES. + send("SUB foo bar 3\r\nPING\r\n") + expect(pongRe) + buf = routeExpect(rsubRe) + checkQueueSub("1") + + send("SUB foo bar 4\r\nPING\r\n") + expect(pongRe) + buf = routeExpect(rsubRe) + checkQueueSub("2") + + send("SUB foo bar 5\r\nPING\r\n") + expect(pongRe) + buf = routeExpect(rsubRe) + checkQueueSub("3") + + // Now walk them back down. + // Again we should always get updates for queue subscribers. + // And these will be RS+ protos walking the weighted count back down. + send("UNSUB 5\r\nPING\r\n") + expect(pongRe) + buf = routeExpect(rsubRe) + checkQueueSub("2") + + send("UNSUB 4\r\nPING\r\n") + expect(pongRe) + buf = routeExpect(rsubRe) + checkQueueSub("1") + + // This one should send UNSUB + send("UNSUB 3\r\nPING\r\n") + expect(pongRe) + routeExpect(runsubRe) + + // Now normal ones. + send("UNSUB 1\r\nPING\r\n") + expect(pongRe) + expectNothing(t, rc) + + send("UNSUB 2\r\nPING\r\n") + expect(pongRe) + routeExpect(runsubRe) +} + +func TestNewRouteClientClosedWithNormalSubscriptions(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:44" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + sendRouteInfo(t, rc, routeSend, routeID) + routeSend("PING\r\n") + routeExpect(pongRe) + + send("SUB foo 1\r\nPING\r\n") + expect(pongRe) + routeExpect(rsubRe) + + for i := 2; i < 100; i++ { + send(fmt.Sprintf("SUB foo %d\r\n", i)) + } + send("PING\r\n") + expect(pongRe) + + // Expect nothing from the route. + expectNothing(t, rc) + + // Now close connection. + c.Close() + expectNothing(t, c) + + buf := routeExpect(runsubRe) + matches := runsubRe.FindAllSubmatch(buf, -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 unsub response when closing client connection, got %d", len(matches)) + } +} + +func TestNewRouteClientClosedWithQueueSubscriptions(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:44" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + sendRouteInfo(t, rc, routeSend, routeID) + routeSend("PING\r\n") + routeExpect(pongRe) + + for i := 0; i < 100; i++ { + send(fmt.Sprintf("SUB foo bar %d\r\n", i)) + } + send("PING\r\n") + expect(pongRe) + + // Queue subscribers will send all updates. + for numRSubs := 0; numRSubs != 100; { + buf := routeExpect(rsubRe) + numRSubs += len(rsubRe.FindAllSubmatch(buf, -1)) + } + + // Now close connection. + c.Close() + expectNothing(t, c) + + // We should only get one unsub for the queue subscription. + matches := runsubRe.FindAllSubmatch(routeExpect(runsubRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 unsub response when closing client connection, got %d", len(matches)) + } +} + +func TestNewRouteRUnsubAccountSpecific(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + // Create a routeConn + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:77" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + sendRouteInfo(t, rc, routeSend, routeID) + + // Now create 500 subs on same subject but all different accounts. + for i := 0; i < 500; i++ { + account := fmt.Sprintf("$foo.account.%d", i) + s.RegisterAccount(account) + routeSend(fmt.Sprintf("RS+ %s foo\r\n", account)) + } + routeSend("PING\r\n") + routeExpect(pongRe) + + routeSend("RS- $foo.account.22 foo\r\nPING\r\n") + routeExpect(pongRe) + + // Do not expect a message on that account. + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConnWithAccount(t, s, c, "$foo.account.22") + send("PUB foo 2\r\nok\r\nPING\r\n") + expect(pongRe) + c.Close() + + // But make sure we still receive on others + c = createClientConn(t, opts.Host, opts.Port) + defer c.Close() + send, expect = setupConnWithAccount(t, s, c, "$foo.account.33") + send("PUB foo 2\r\nok\r\nPING\r\n") + expect(pongRe) + + matches := rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkRmsg(t, matches[0], "$foo.account.33", "foo", "", "2", "ok") +} + +func TestNewRouteRSubCleanupOnDisconnect(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + // Create a routeConn + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:77" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + sendRouteInfo(t, rc, routeSend, routeID) + + // Now create 100 subs on 3 different accounts. + for i := 0; i < 100; i++ { + subject := fmt.Sprintf("foo.%d", i) + routeSend(fmt.Sprintf("RS+ $foo %s\r\n", subject)) + routeSend(fmt.Sprintf("RS+ $bar %s\r\n", subject)) + routeSend(fmt.Sprintf("RS+ $baz %s bar %d\r\n", subject, i+1)) + } + routeSend("PING\r\n") + routeExpect(pongRe) + + rc.Close() + + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if ns := s.NumSubscriptions(); ns != 0 { + return fmt.Errorf("Number of subscriptions is %d", ns) + } + return nil + }) +} + +func TestNewRouteSendSubsAndMsgs(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:44" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + sendRouteInfo(t, rc, routeSend, routeID) + routeSend("PING\r\n") + routeExpect(pongRe) + + // Now let's send in interest from the new protocol. + // Normal Subscription + routeSend("RS+ $G foo\r\n") + // Make sure things were processed. + routeSend("PING\r\n") + routeExpect(pongRe) + + // Now create a client and send a message, make sure we receive it + // over the route connection. + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + send("PUB foo 2\r\nok\r\nPING\r\n") + expect(pongRe) + + buf := routeExpect(rmsgRe) + matches := rmsgRe.FindAllSubmatch(buf, -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkRmsg(t, matches[0], "$G", "foo", "", "2", "ok") + + // Queue Subscription + routeSend("RS+ $G foo bar 1\r\n") + // Make sure things were processed. + routeSend("PING\r\n") + routeExpect(pongRe) + + send("PUB foo reply 2\r\nok\r\nPING\r\n") + expect(pongRe) + + matches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkRmsg(t, matches[0], "$G", "foo", "+ reply bar", "2", "ok") + + // Another Queue Subscription + routeSend("RS+ $G foo baz 1\r\n") + // Make sure things were processed. + routeSend("PING\r\n") + routeExpect(pongRe) + + send("PUB foo reply 2\r\nok\r\nPING\r\n") + expect(pongRe) + + matches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkRmsg(t, matches[0], "$G", "foo", "+ reply bar baz", "2", "ok") + + // Matching wildcard + routeSend("RS+ $G *\r\n") + // Make sure things were processed. + routeSend("PING\r\n") + routeExpect(pongRe) + + send("PUB foo reply 2\r\nok\r\nPING\r\n") + expect(pongRe) + + matches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkRmsg(t, matches[0], "$G", "foo", "+ reply bar baz", "2", "ok") + + // No reply + send("PUB foo 2\r\nok\r\nPING\r\n") + expect(pongRe) + + matches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkRmsg(t, matches[0], "$G", "foo", "| bar baz", "2", "ok") + + // Now unsubscribe from the queue group. + routeSend("RS- $G foo baz\r\n") + routeSend("RS- $G foo bar\r\n") + + routeSend("PING\r\n") + routeExpect(pongRe) + + // Now send and make sure they are removed. We should still get the message. + send("PUB foo 2\r\nok\r\nPING\r\n") + expect(pongRe) + + matches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkRmsg(t, matches[0], "$G", "foo", "", "2", "ok") + + routeSend("RS- $G foo\r\n") + routeSend("RS- $G *\r\n") + + routeSend("PING\r\n") + routeExpect(pongRe) + + // Now we should not receive messages anymore. + send("PUB foo 2\r\nok\r\nPING\r\n") + expect(pongRe) + + expectNothing(t, rc) +} + +func TestNewRouteProcessRoutedMsgs(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:55" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + sendRouteInfo(t, rc, routeSend, routeID) + routeSend("PING\r\n") + routeExpect(pongRe) + + // Create a client + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + + // Normal sub to start + send("SUB foo 1\r\nPING\r\n") + expect(pongRe) + routeExpect(rsubRe) + + expectMsgs := expectMsgsCommand(t, expect) + + // Now send in a RMSG to the route and make sure its delivered to the client. + routeSend("RMSG $G foo 2\r\nok\r\nPING\r\n") + routeExpect(pongRe) + matches := expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "", "2", "ok") + + // Now send in a RMSG to the route with a reply and make sure its delivered to the client. + routeSend("RMSG $G foo reply 2\r\nok\r\nPING\r\n") + routeExpect(pongRe) + + matches = expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "reply", "2", "ok") + + // Now add in a queue subscriber for the client. + send("SUB foo bar 11\r\nPING\r\n") + expect(pongRe) + routeExpect(rsubRe) + + // Now add in another queue subscriber for the client. + send("SUB foo baz 22\r\nPING\r\n") + expect(pongRe) + routeExpect(rsubRe) + + // If we send from a route with no queues. Should only get one message. + routeSend("RMSG $G foo 2\r\nok\r\nPING\r\n") + routeExpect(pongRe) + matches = expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "", "2", "ok") + + // Now send to a specific queue group. We should get multiple messages now. + routeSend("RMSG $G foo | bar 2\r\nok\r\nPING\r\n") + routeExpect(pongRe) + matches = expectMsgs(2) + checkMsg(t, matches[0], "foo", "1", "", "2", "ok") + + // Now send to both queue groups. We should get all messages now. + routeSend("RMSG $G foo | bar baz 2\r\nok\r\nPING\r\n") + routeExpect(pongRe) + matches = expectMsgs(3) + checkMsg(t, matches[0], "foo", "1", "", "2", "ok") + + // Make sure we do the right thing with reply. + routeSend("RMSG $G foo + reply bar baz 2\r\nok\r\nPING\r\n") + routeExpect(pongRe) + matches = expectMsgs(3) + checkMsg(t, matches[0], "foo", "1", "reply", "2", "ok") +} + +func TestNewRouteQueueSubsDistribution(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + sendA, expectA := setupConn(t, clientA) + sendB, expectB := setupConn(t, clientB) + + // Create 100 subscribers on each server. + for i := 0; i < 100; i++ { + sproto := fmt.Sprintf("SUB foo bar %d\r\n", i) + sendA(sproto) + sendB(sproto) + } + sendA("PING\r\n") + expectA(pongRe) + sendB("PING\r\n") + expectB(pongRe) + + // Each server should have its 100 local subscriptions, plus 1 for the route. + if err := checkExpectedSubs(101, srvA, srvB); err != nil { + t.Fatal(err.Error()) + } + + sender := createClientConn(t, optsA.Host, optsA.Port) + defer sender.Close() + send, expect := setupConn(t, sender) + + // Send 100 messages from Sender + for i := 0; i < 100; i++ { + send("PUB foo 2\r\nok\r\n") + } + send("PING\r\n") + expect(pongRe) + + numAReceived := len(msgRe.FindAllSubmatch(expectA(msgRe), -1)) + numBReceived := len(msgRe.FindAllSubmatch(expectB(msgRe), -1)) + + // We may not be able to properly time all messages being ready. + for numAReceived+numBReceived != 100 { + if buf := peek(clientB); buf != nil { + numBReceived += len(msgRe.FindAllSubmatch(buf, -1)) + } + if buf := peek(clientA); buf != nil { + numAReceived += len(msgRe.FindAllSubmatch(buf, -1)) + } + } + // These should be close to 50/50 + if numAReceived < 30 || numBReceived < 30 { + t.Fatalf("Expected numbers to be close to 50/50, got %d/%d", numAReceived, numBReceived) + } +} + +// Since we trade interest in accounts now, we have a potential issue with a new client +// connecting via a brand new account, publishing and properly doing a flush, then exiting. +// If existing subscribers were present but on a remote server they may not get the message. +func TestNewRouteSinglePublishOnNewAccount(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + srvA.RegisterAccount("$TEST22") + srvB.RegisterAccount("$TEST22") + + // Create and establish a listener on foo for $TEST22 account. + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConnWithAccount(t, srvA, clientA, "$TEST22") + sendA("SUB foo 1\r\nPING\r\n") + expectA(pongRe) + + if err := checkExpectedSubs(1, srvB); err != nil { + t.Fatal(err.Error()) + } + + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + // Send a message, flush to make sure server processed and close connection. + sendB, expectB := setupConnWithAccount(t, srvB, clientB, "$TEST22") + sendB("PUB foo 2\r\nok\r\nPING\r\n") + expectB(pongRe) + clientB.Close() + + expectMsgs := expectMsgsCommand(t, expectA) + matches := expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "", "2", "ok") +} + +// Same as above but make sure it works for queue subscribers as well. +func TestNewRouteSinglePublishToQueueSubscriberOnNewAccount(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + srvA.RegisterAccount("$TEST22") + srvB.RegisterAccount("$TEST22") + + // Create and establish a listener on foo for $TEST22 account. + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConnWithAccount(t, srvA, clientA, "$TEST22") + sendA("SUB foo bar 1\r\nPING\r\n") + expectA(pongRe) + + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + // Send a message, flush to make sure server processed and close connection. + sendB, expectB := setupConnWithAccount(t, srvB, clientB, "$TEST22") + sendB("PUB foo bar 2\r\nok\r\nPING\r\n") + expectB(pongRe) + defer clientB.Close() + + expectMsgs := expectMsgsCommand(t, expectA) + matches := expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "bar", "2", "ok") + + sendB("PUB foo bar 2\r\nok\r\nPING\r\n") + expectB(pongRe) + matches = expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "bar", "2", "ok") +} + +// Same as above but make sure it works for queue subscribers over multiple routes as well. +func TestNewRouteSinglePublishToMultipleQueueSubscriberOnNewAccount(t *testing.T) { + srvA, srvB, srvC, optsA, optsB, optsC := runThreeServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + defer srvC.Shutdown() + + srvA.RegisterAccount("$TEST22") + srvB.RegisterAccount("$TEST22") + srvC.RegisterAccount("$TEST22") + + // Create and establish a listener on foo/bar for $TEST22 account. Do this on ClientA and ClientC. + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConnWithAccount(t, srvA, clientA, "$TEST22") + sendA("SUB foo bar 11\r\nPING\r\n") + expectA(pongRe) + + clientC := createClientConn(t, optsC.Host, optsC.Port) + defer clientC.Close() + + sendC, expectC := setupConnWithAccount(t, srvC, clientC, "$TEST22") + sendC("SUB foo bar 33\r\nPING\r\n") + expectC(pongRe) + + sendA("PING\r\n") + expectA(pongRe) + sendC("PING\r\n") + expectC(pongRe) + + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + time.Sleep(100 * time.Millisecond) + + // Send a message, flush to make sure server processed and close connection. + sendB, expectB := setupConnWithAccount(t, srvB, clientB, "$TEST22") + sendB("PUB foo 2\r\nok\r\nPING\r\n") + expectB(pongRe) + defer clientB.Close() + + // This should trigger either clientA or clientC, but not both.. + bufA := peek(clientA) + bufC := peek(clientC) + + if bufA != nil && bufC != nil { + t.Fatalf("Expected one or the other, but got something on both") + } + numReceived := len(msgRe.FindAllSubmatch(bufA, -1)) + numReceived += len(msgRe.FindAllSubmatch(bufC, -1)) + if numReceived != 1 { + t.Fatalf("Expected only 1 msg, got %d", numReceived) + } + + // Now make sure that we are distributing correctly between A and C + // Send 100 messages from Sender + for i := 0; i < 100; i++ { + sendB("PUB foo 2\r\nok\r\n") + } + sendB("PING\r\n") + expectB(pongRe) + + numAReceived := len(msgRe.FindAllSubmatch(expectA(msgRe), -1)) + numCReceived := len(msgRe.FindAllSubmatch(expectC(msgRe), -1)) + + // We may not be able to properly time all messages being ready. + + for numAReceived+numCReceived != 100 { + if buf := peek(clientC); buf != nil { + numCReceived += len(msgRe.FindAllSubmatch(buf, -1)) + } + if buf := peek(clientA); buf != nil { + numAReceived += len(msgRe.FindAllSubmatch(buf, -1)) + } + } + + // These should be close to 50/50 + if numAReceived < 30 || numCReceived < 30 { + t.Fatalf("Expected numbers to be close to 50/50, got %d/%d", numAReceived, numCReceived) + } +} + +func registerAccounts(t *testing.T, s *server.Server) (*server.Account, *server.Account) { + // Now create two accounts. + f, err := s.RegisterAccount("$foo") + if err != nil { + t.Fatalf("Error creating account '$foo': %v", err) + } + b, err := s.RegisterAccount("$bar") + if err != nil { + t.Fatalf("Error creating account '$bar': %v", err) + } + return f, b +} + +func addStreamExport(subject string, authorized []*server.Account, targets ...*server.Account) { + for _, acc := range targets { + acc.AddStreamExport(subject, authorized) + } +} + +func addServiceExport(subject string, authorized []*server.Account, targets ...*server.Account) { + for _, acc := range targets { + acc.AddServiceExport(subject, authorized) + } +} + +var isPublic = []*server.Account(nil) + +func TestNewRouteStreamImport(t *testing.T) { + testNewRouteStreamImport(t, false) +} + +func testNewRouteStreamImport(t *testing.T, duplicateSub bool) { + t.Helper() + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + // Do Accounts for the servers. + fooA, _ := registerAccounts(t, srvA) + fooB, barB := registerAccounts(t, srvB) + + // Add export to both. + addStreamExport("foo", isPublic, fooA, fooB) + // Add import abilities to server B's bar account from foo. + barB.AddStreamImport(fooB, "foo", "") + + // clientA will be connected to srvA and be the stream producer. + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConnWithAccount(t, srvA, clientA, "$foo") + + // Now setup client B on srvB who will do a sub from account $bar + // that should map account $foo's foo subject. + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + sendB, expectB := setupConnWithAccount(t, srvB, clientB, "$bar") + sendB("SUB foo 1\r\n") + if duplicateSub { + sendB("SUB foo 1\r\n") + } + sendB("PING\r\n") + expectB(pongRe) + + // The subscription on "foo" for account $bar will also become + // a subscription on "foo" for account $foo due to import. + // So total of 2 subs. + if err := checkExpectedSubs(2, srvA); err != nil { + t.Fatal(err.Error()) + } + + // Send on clientA + sendA("PING\r\n") + expectA(pongRe) + + sendA("PUB foo 2\r\nok\r\nPING\r\n") + expectA(pongRe) + + expectMsgs := expectMsgsCommand(t, expectB) + matches := expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "", "2", "ok") + + // Send Again on clientA + sendA("PUB foo 2\r\nok\r\nPING\r\n") + expectA(pongRe) + + matches = expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "", "2", "ok") + + sendB("UNSUB 1\r\nPING\r\n") + expectB(pongRe) + + if err := checkExpectedSubs(0, srvA); err != nil { + t.Fatal(err.Error()) + } + + sendA("PUB foo 2\r\nok\r\nPING\r\n") + expectA(pongRe) + expectNothing(t, clientA) +} + +func TestNewRouteStreamImportLargeFanout(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + // Do Accounts for the servers. + // This account will export a stream. + fooA, err := srvA.RegisterAccount("$foo") + if err != nil { + t.Fatalf("Error creating account '$foo': %v", err) + } + fooB, err := srvB.RegisterAccount("$foo") + if err != nil { + t.Fatalf("Error creating account '$foo': %v", err) + } + + // Add export to both. + addStreamExport("foo", isPublic, fooA, fooB) + + // Now we will create 100 accounts who will all import from foo. + fanout := 100 + barA := make([]*server.Account, fanout) + for i := 0; i < fanout; i++ { + acc := fmt.Sprintf("$bar-%d", i) + barA[i], err = srvB.RegisterAccount(acc) + if err != nil { + t.Fatalf("Error creating account %q: %v", acc, err) + } + // Add import abilities to server B's bar account from foo. + barA[i].AddStreamImport(fooB, "foo", "") + } + + // clientA will be connected to srvA and be the stream producer. + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + // Now setup fanout clients on srvB who will do a sub from account $bar + // that should map account $foo's foo subject. + clientB := make([]net.Conn, fanout) + sendB := make([]sendFun, fanout) + expectB := make([]expectFun, fanout) + + for i := 0; i < fanout; i++ { + clientB[i] = createClientConn(t, optsB.Host, optsB.Port) + defer clientB[i].Close() + sendB[i], expectB[i] = setupConnWithAccount(t, srvB, clientB[i], barA[i].Name) + sendB[i]("SUB foo 1\r\nPING\r\n") + expectB[i](pongRe) + } + + // Since we do not shadow all the bar acounts on srvA they will be dropped + // when they hit the other side, which means we could only have one sub for + // all the imports on srvA, and srvB will have 2*fanout, one normal and one + // that represents the import. + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if ns := srvA.NumSubscriptions(); ns != uint32(1) { + return fmt.Errorf("Number of subscriptions is %d", ns) + } + if ns := srvB.NumSubscriptions(); ns != uint32(2*fanout) { + return fmt.Errorf("Number of subscriptions is %d", ns) + } + return nil + }) +} + +func TestNewRouteReservedReply(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + + // Test that clients can't send to reserved service import replies. + send("PUB foo _R_.foo 2\r\nok\r\nPING\r\n") + expect(errRe) +} + +func TestNewRouteServiceImport(t *testing.T) { + // To quickly enable trace and debug logging + //doLog, doTrace, doDebug = true, true, true + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + // Make so we can tell the two apart since in same PID. + if doLog { + srvA.SetLogger(logger.NewTestLogger("[SRV-A] - ", false), true, true) + srvB.SetLogger(logger.NewTestLogger("[SRV-B] - ", false), true, true) + } + + // Do Accounts for the servers. + fooA, barA := registerAccounts(t, srvA) + fooB, barB := registerAccounts(t, srvB) + + // Add export to both. + addServiceExport("test.request", isPublic, fooA, fooB) + + // Add import abilities to server B's bar account from foo. + // Meaning that when a user sends a request on foo.request from account bar, + // the request will be mapped to be received by the responder on account foo. + if err := barB.AddServiceImport(fooB, "foo.request", "test.request"); err != nil { + t.Fatalf("Error adding service import: %v", err) + } + // Do same on A. + if err := barA.AddServiceImport(fooA, "foo.request", "test.request"); err != nil { + t.Fatalf("Error adding service import: %v", err) + } + + // clientA will be connected to srvA and be the service endpoint and responder. + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConnWithAccount(t, srvA, clientA, "$foo") + sendA("SUB test.request 1\r\nPING\r\n") + expectA(pongRe) + + // Now setup client B on srvB who will do a sub from account $bar + // that should map account $foo's foo subject. + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + sendB, expectB := setupConnWithAccount(t, srvB, clientB, "$bar") + sendB("SUB reply 1\r\nPING\r\n") + expectB(pongRe) + + // Wait for all subs to be propagated. (1 on foo, 2 on bar) + if err := checkExpectedSubs(3, srvA, srvB); err != nil { + t.Fatal(err.Error()) + } + + // Send the request from clientB on foo.request, + sendB("PUB foo.request reply 2\r\nhi\r\nPING\r\n") + expectB(pongRe) + + expectMsgsA := expectMsgsCommand(t, expectA) + expectMsgsB := expectMsgsCommand(t, expectB) + + // Expect the request on A + matches := expectMsgsA(1) + reply := string(matches[0][replyIndex]) + checkMsg(t, matches[0], "test.request", "1", reply, "2", "hi") + if reply == "reply" { + t.Fatalf("Expected randomized reply, but got original") + } + + sendA(fmt.Sprintf("PUB %s 2\r\nok\r\nPING\r\n", reply)) + expectA(pongRe) + + matches = expectMsgsB(1) + checkMsg(t, matches[0], "reply", "1", "", "2", "ok") + + // This will be the responder and the wildcard for all service replies. + if ts := fooA.TotalSubs(); ts != 2 { + t.Fatalf("Expected two subs to be left on fooA, but got %d", ts) + } + + routez, _ := srvA.Routez(&server.RoutezOptions{Subscriptions: true}) + r := routez.Routes[0] + if r == nil { + t.Fatalf("Expected 1 route, got none") + } + if r.NumSubs != 2 { + t.Fatalf("Expected 2 subs in the route connection, got %v", r.NumSubs) + } +} + +func TestNewRouteServiceExportWithWildcards(t *testing.T) { + for _, test := range []struct { + name string + public bool + }{ + { + name: "public", + public: true, + }, + { + name: "private", + public: false, + }, + } { + t.Run(test.name, func(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + // Do Accounts for the servers. + fooA, barA := registerAccounts(t, srvA) + fooB, barB := registerAccounts(t, srvB) + + var accs []*server.Account + // Add export to both. + if !test.public { + accs = []*server.Account{barA} + } + addServiceExport("ngs.update.*", accs, fooA) + if !test.public { + accs = []*server.Account{barB} + } + addServiceExport("ngs.update.*", accs, fooB) + + // Add import abilities to server B's bar account from foo. + if err := barB.AddServiceImport(fooB, "ngs.update", "ngs.update.$bar"); err != nil { + t.Fatalf("Error adding service import: %v", err) + } + // Do same on A. + if err := barA.AddServiceImport(fooA, "ngs.update", "ngs.update.$bar"); err != nil { + t.Fatalf("Error adding service import: %v", err) + } + + // clientA will be connected to srvA and be the service endpoint and responder. + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConnWithAccount(t, srvA, clientA, "$foo") + sendA("SUB ngs.update.* 1\r\nPING\r\n") + expectA(pongRe) + + // Now setup client B on srvB who will do a sub from account $bar + // that should map account $foo's foo subject. + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + sendB, expectB := setupConnWithAccount(t, srvB, clientB, "$bar") + sendB("SUB reply 1\r\nPING\r\n") + expectB(pongRe) + + // Wait for all subs to be propagated. (1 on foo, 2 on bar) + if err := checkExpectedSubs(3, srvA, srvB); err != nil { + t.Fatal(err.Error()) + } + + // Send the request from clientB on foo.request, + sendB("PUB ngs.update reply 2\r\nhi\r\nPING\r\n") + expectB(pongRe) + + expectMsgsA := expectMsgsCommand(t, expectA) + expectMsgsB := expectMsgsCommand(t, expectB) + + // Expect the request on A + matches := expectMsgsA(1) + reply := string(matches[0][replyIndex]) + checkMsg(t, matches[0], "ngs.update.$bar", "1", reply, "2", "hi") + if reply == "reply" { + t.Fatalf("Expected randomized reply, but got original") + } + + sendA(fmt.Sprintf("PUB %s 2\r\nok\r\nPING\r\n", reply)) + expectA(pongRe) + + matches = expectMsgsB(1) + checkMsg(t, matches[0], "reply", "1", "", "2", "ok") + + if ts := fooA.TotalSubs(); ts != 2 { + t.Fatalf("Expected two subs to be left on fooA, but got %d", ts) + } + + routez, _ := srvA.Routez(&server.RoutezOptions{Subscriptions: true}) + r := routez.Routes[0] + if r == nil { + t.Fatalf("Expected 1 route, got none") + } + if r.NumSubs != 2 { + t.Fatalf("Expected 2 subs in the route connection, got %v", r.NumSubs) + } + }) + } +} + +func TestNewRouteServiceImportQueueGroups(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + // Do Accounts for the servers. + fooA, barA := registerAccounts(t, srvA) + fooB, barB := registerAccounts(t, srvB) + + // Add export to both. + addServiceExport("test.request", isPublic, fooA, fooB) + + // Add import abilities to server B's bar account from foo. + if err := barB.AddServiceImport(fooB, "foo.request", "test.request"); err != nil { + t.Fatalf("Error adding service import: %v", err) + } + // Do same on A. + if err := barA.AddServiceImport(fooA, "foo.request", "test.request"); err != nil { + t.Fatalf("Error adding service import: %v", err) + } + + // clientA will be connected to srvA and be the service endpoint and responder. + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConnWithAccount(t, srvA, clientA, "$foo") + sendA("SUB test.request QGROUP 1\r\nPING\r\n") + expectA(pongRe) + + // Now setup client B on srvB who will do a sub from account $bar + // that should map account $foo's foo subject. + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + sendB, expectB := setupConnWithAccount(t, srvB, clientB, "$bar") + sendB("SUB reply QGROUP_TOO 1\r\nPING\r\n") + expectB(pongRe) + + // Wait for all subs to be propagated. (1 on foo, 2 on bar) + if err := checkExpectedSubs(3, srvA, srvB); err != nil { + t.Fatal(err.Error()) + } + + // Send the request from clientB on foo.request, + sendB("PUB foo.request reply 2\r\nhi\r\nPING\r\n") + expectB(pongRe) + + expectMsgsA := expectMsgsCommand(t, expectA) + expectMsgsB := expectMsgsCommand(t, expectB) + + // Expect the request on A + matches := expectMsgsA(1) + reply := string(matches[0][replyIndex]) + checkMsg(t, matches[0], "test.request", "1", reply, "2", "hi") + if reply == "reply" { + t.Fatalf("Expected randomized reply, but got original") + } + + sendA(fmt.Sprintf("PUB %s 2\r\nok\r\nPING\r\n", reply)) + expectA(pongRe) + + matches = expectMsgsB(1) + checkMsg(t, matches[0], "reply", "1", "", "2", "ok") + + if ts := fooA.TotalSubs(); ts != 2 { + t.Fatalf("Expected two subs to be left on fooA, but got %d", ts) + } + + routez, _ := srvA.Routez(&server.RoutezOptions{Subscriptions: true}) + r := routez.Routes[0] + if r == nil { + t.Fatalf("Expected 1 route, got none") + } + if r.NumSubs != 2 { + t.Fatalf("Expected 2 subs in the route connection, got %v", r.NumSubs) + } +} + +func TestNewRouteServiceImportDanglingRemoteSubs(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + // Do Accounts for the servers. + fooA, _ := registerAccounts(t, srvA) + fooB, barB := registerAccounts(t, srvB) + + // Add in the service export for the requests. Make it public. + if err := fooA.AddServiceExport("test.request", nil); err != nil { + t.Fatalf("Error adding account service export to client foo: %v", err) + } + + // Add export to both. + addServiceExport("test.request", isPublic, fooA, fooB) + + // Add import abilities to server B's bar account from foo. + if err := barB.AddServiceImport(fooB, "foo.request", "test.request"); err != nil { + t.Fatalf("Error adding service import: %v", err) + } + + // clientA will be connected to srvA and be the service endpoint, but will not send responses. + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConnWithAccount(t, srvA, clientA, "$foo") + // Express interest. + sendA("SUB test.request 1\r\nPING\r\n") + expectA(pongRe) + + // Now setup client B on srvB who will do a sub from account $bar + // that should map account $foo's foo subject. + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + sendB, expectB := setupConnWithAccount(t, srvB, clientB, "$bar") + sendB("SUB reply 1\r\nPING\r\n") + expectB(pongRe) + + // Wait for all subs to be propagated (1 on foo and 1 on bar on srvA) + // (note that srvA is not importing) + if err := checkExpectedSubs(2, srvA); err != nil { + t.Fatal(err.Error()) + } + // Wait for all subs to be propagated (1 on foo and 2 on bar) + if err := checkExpectedSubs(3, srvB); err != nil { + t.Fatal(err.Error()) + } + + // Send 100 requests from clientB on foo.request, + for i := 0; i < 100; i++ { + sendB("PUB foo.request reply 2\r\nhi\r\n") + } + sendB("PING\r\n") + expectB(pongRe) + + numRequests := 0 + // Expect the request on A + checkFor(t, time.Second, 10*time.Millisecond, func() error { + buf := expectA(msgRe) + matches := msgRe.FindAllSubmatch(buf, -1) + numRequests += len(matches) + if numRequests != 100 { + return fmt.Errorf("Number of requests is %d", numRequests) + } + return nil + }) + + expectNothing(t, clientB) + + // These reply subjects will be dangling off of $foo account on serverA. + // Remove our service endpoint and wait for the dangling replies to go to zero. + sendA("UNSUB 1\r\nPING\r\n") + expectA(pongRe) + + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if ts := fooA.TotalSubs(); ts != 1 { + return fmt.Errorf("Number of subs is %d, should be only 1", ts) + } + return nil + }) +} + +func TestNewRouteNoQueueSubscribersBounce(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, optsA.Port) + urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, optsB.Port) + + ncA, err := nats.Connect(urlA) + if err != nil { + t.Fatalf("Failed to create connection for ncA: %v\n", err) + } + defer ncA.Close() + + ncB, err := nats.Connect(urlB) + if err != nil { + t.Fatalf("Failed to create connection for ncB: %v\n", err) + } + defer ncB.Close() + + response := []byte("I will help you") + + // Create a lot of queue subscribers on A, and have one on B. + ncB.QueueSubscribe("foo.request", "workers", func(m *nats.Msg) { + ncB.Publish(m.Reply, response) + }) + + for i := 0; i < 100; i++ { + ncA.QueueSubscribe("foo.request", "workers", func(m *nats.Msg) { + ncA.Publish(m.Reply, response) + }) + } + ncB.Flush() + ncA.Flush() + + // Send all requests from B + numAnswers := 0 + for i := 0; i < 500; i++ { + if _, err := ncB.Request("foo.request", []byte("Help Me"), time.Second); err != nil { + t.Fatalf("Received an error on Request test [%d]: %s", i, err) + } + numAnswers++ + // After we have sent 20 close the ncA client. + if i == 20 { + ncA.Close() + } + } + + if numAnswers != 500 { + t.Fatalf("Expect to get all 500 responses, got %d", numAnswers) + } +} + +func TestNewRouteLargeDistinctQueueSubscribers(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, optsA.Port) + urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, optsB.Port) + + ncA, err := nats.Connect(urlA) + if err != nil { + t.Fatalf("Failed to create connection for ncA: %v\n", err) + } + defer ncA.Close() + + ncB, err := nats.Connect(urlB) + if err != nil { + t.Fatalf("Failed to create connection for ncB: %v\n", err) + } + defer ncB.Close() + + const nqsubs = 100 + + qsubs := make([]*nats.Subscription, 100) + + // Create 100 queue subscribers on B all with different queue groups. + for i := 0; i < nqsubs; i++ { + qg := fmt.Sprintf("worker-%d", i) + qsubs[i], _ = ncB.QueueSubscribeSync("foo", qg) + } + ncB.Flush() + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if ns := srvA.NumSubscriptions(); ns != 100 { + return fmt.Errorf("Number of subscriptions is %d", ns) + } + return nil + }) + + // Send 10 messages. We should receive 1000 responses. + for i := 0; i < 10; i++ { + ncA.Publish("foo", nil) + } + ncA.Flush() + + checkFor(t, 2*time.Second, 10*time.Millisecond, func() error { + for i := 0; i < nqsubs; i++ { + if n, _, _ := qsubs[i].Pending(); n != 10 { + return fmt.Errorf("Number of messages is %d", n) + } + } + return nil + }) +} + +func TestNewRouteLeafNodeOriginSupport(t *testing.T) { + content := ` + listen: 127.0.0.1:-1 + cluster { name: xyz, listen: 127.0.0.1:-1 } + leafnodes { listen: 127.0.0.1:-1 } + no_sys_acc: true + ` + conf := createConfFile(t, []byte(content)) + + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + gacc, _ := s.LookupAccount("$G") + + lcontent := ` + listen: 127.0.0.1:-1 + cluster { name: ln1, listen: 127.0.0.1:-1 } + leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } + no_sys_acc: true + ` + lconf := createConfFile(t, []byte(fmt.Sprintf(lcontent, opts.LeafNode.Port))) + + ln, _ := RunServerWithConfig(lconf) + defer ln.Shutdown() + + checkLeafNodeConnected(t, s) + + lgacc, _ := ln.LookupAccount("$G") + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "LNOC:22" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + pingPong := func() { + t.Helper() + routeSend("PING\r\n") + routeExpect(pongRe) + } + + info := checkInfoMsg(t, rc) + info.ID = routeID + info.Name = "" + info.LNOC = true + // Overwrite to false to check that we are getting LS- without origin + // if we are an old server. LN implies LNOCU, so clear it too. + info.LNOCU = false + info.LN = false + b, err := json.Marshal(info) + if err != nil { + t.Fatalf("Could not marshal test route info: %v", err) + } + + routeSend(fmt.Sprintf("INFO %s\r\n", b)) + routeExpect(rlsubRe) + pingPong() + + sendLSProtosFromRoute := func(lnocu bool) { + t.Helper() + + // Make sure it can process and LS+ + routeSend("LS+ ln1 $G foo\r\n") + pingPong() + + // Check interest is registered on remote server. + if !gacc.SubscriptionInterest("foo") { + t.Fatalf("Expected interest on \"foo\"") + } + + // This should not have been sent to the leafnode since same origin cluster. + time.Sleep(10 * time.Millisecond) + if lgacc.SubscriptionInterest("foo") { + t.Fatalf("Did not expect interest on \"foo\"") + } + + // Now unsub. Either act as an old server that does not support origin + // in the LS- or as a new server. + if lnocu { + routeSend("LS- ln1 $G foo\r\n") + } else { + routeSend("LS- $G foo\r\n") + } + pingPong() + + // Interest should be gone. + if gacc.SubscriptionInterest("foo") { + t.Fatalf("Expected no interest on \"foo\"") + } + + // Make sure we did not incorrectly send an interest to the leaf. + time.Sleep(10 * time.Millisecond) + if lgacc.SubscriptionInterest("foo") { + t.Fatalf("Did not expect interest on \"foo\"") + } + + // Repeat with a queue. + routeSend("LS+ ln1 $G foo bar 1\r\n") + pingPong() + + if !gacc.SubscriptionInterest("foo") { + t.Fatalf("Expected interest on \"foo\"") + } + + // This should not have been sent to the leafnode since same origin cluster. + time.Sleep(10 * time.Millisecond) + if lgacc.SubscriptionInterest("foo") { + t.Fatalf("Did not expect interest on \"foo\"") + } + + // Now unsub. + if lnocu { + routeSend("LS- ln1 $G foo bar\r\n") + } else { + routeSend("LS- $G foo bar\r\n") + } + pingPong() + + // Subscription should be gone. + if gacc.SubscriptionInterest("foo") { + t.Fatalf("Expected no interest on \"foo\"") + } + + // Make sure we did not incorrectly send an interest to the leaf. + time.Sleep(10 * time.Millisecond) + if lgacc.SubscriptionInterest("foo") { + t.Fatalf("Did not expect interest on \"foo\"") + } + } + + // Check the LS+/- when not supporting origin in LS- + sendLSProtosFromRoute(false) + + // Create a connection on the leafnode server. + nc, err := nats.Connect(ln.ClientURL()) + if err != nil { + t.Fatalf("Unexpected error connecting %v", err) + } + defer nc.Close() + + sub, _ := nc.SubscribeSync("bar") + // Let it propagate to the main server + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if !gacc.SubscriptionInterest("bar") { + return fmt.Errorf("No interest") + } + return nil + }) + // For "bar" + routeExpect(rlsubRe) + + // Now pretend like we send a message to the main server over the + // route but from the same origin cluster, should not be delivered + // to the leafnode. + + // Make sure it can process and LMSG. + // LMSG for routes is like HMSG with an origin cluster before the account. + routeSend("LMSG ln1 $G bar 0 2\r\nok\r\n") + pingPong() + + // Let it propagate if not properly truncated. + time.Sleep(10 * time.Millisecond) + if n, _, _ := sub.Pending(); n != 0 { + t.Fatalf("Should not have received the message on bar") + } + + // Try one with all the bells and whistles. + routeSend("LMSG ln1 $G foo + reply bar baz 0 2\r\nok\r\n") + pingPong() + + // Let it propagate if not properly truncated. + time.Sleep(10 * time.Millisecond) + if n, _, _ := sub.Pending(); n != 0 { + t.Fatalf("Should not have received the message on bar") + } + + // Now unsubscribe, we should receive an LS- without origin. + sub.Unsubscribe() + routeExpect(lunsubRe) + + // Quick check for queues + sub, _ = nc.QueueSubscribeSync("baz", "bat") + // Let it propagate to the main server + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if !gacc.SubscriptionInterest("baz") { + return fmt.Errorf("No interest") + } + return nil + }) + // For "baz" + routeExpect(rlsubRe) + sub.Unsubscribe() + routeExpect(lunsubRe) + + // Restart our routed server, but this time indicate support + // for LS- with origin cluster. + rc.Close() + rc = createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeSend, routeExpect = setupRouteEx(t, rc, opts, routeID) + + info = checkInfoMsg(t, rc) + info.ID = routeID + info.Name = "" + // These should be already set to true since the server that sends the + // INFO has them enabled, but just be explicit. + info.LNOC = true + info.LNOCU = true + b, err = json.Marshal(info) + if err != nil { + t.Fatalf("Could not marshal test route info: %v", err) + } + + routeSend(fmt.Sprintf("INFO %s\r\n", b)) + routeExpect(rlsubRe) + pingPong() + + // Check the LS+/LS- + sendLSProtosFromRoute(true) + + sub, _ = nc.SubscribeSync("bar") + routeExpect(rlsubRe) + sub.Unsubscribe() + routeExpect(rlunsubRe) + + sub, _ = nc.QueueSubscribeSync("baz", "bat") + routeExpect(rlsubRe) + sub.Unsubscribe() + routeExpect(rlunsubRe) +} + +// Check that real duplicate subscription (that is, sent by client with same sid) +// are ignored and do not register multiple shadow subscriptions. +func TestNewRouteDuplicateSubscription(t *testing.T) { + // This is same test than TestNewRouteStreamImport but calling "SUB foo 1" twice. + testNewRouteStreamImport(t, true) + + opts := LoadConfig("./configs/new_cluster.conf") + opts.DisableShortFirstPing = true + s := RunServer(opts) + defer s.Shutdown() + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_DUPLICATE:22" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + sendRouteInfo(t, rc, routeSend, routeID) + routeSend("PING\r\n") + routeExpect(pongRe) + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + send, expect := setupConn(t, c) + + // Create a real duplicate subscriptions (same sid) + send("SUB foo 1\r\nSUB foo 1\r\nPING\r\n") + expect(pongRe) + + // Route should receive single RS+ + routeExpect(rsubRe) + + // Unsubscribe. + send("UNSUB 1\r\nPING\r\n") + expect(pongRe) + + // Route should receive RS-. + // With defect, only 1 subscription would be found during the unsubscribe, + // however route map would have been updated twice when processing the + // duplicate SUB, which means that the RS- would not be received because + // the count would still be 1. + routeExpect(runsubRe) +} diff --git a/test/norace_test.go b/test/norace_test.go new file mode 100644 index 0000000..2b2c712 --- /dev/null +++ b/test/norace_test.go @@ -0,0 +1,706 @@ +// Copyright 2019-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !race && !skip_no_race_tests && !skip_no_race_1_tests + +package test + +import ( + "context" + crand "crypto/rand" + "encoding/json" + "fmt" + "net" + "net/url" + "os" + "regexp" + "runtime" + "strconv" + "sync" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nuid" +) + +// IMPORTANT: Tests in this file are not executed when running with the -race flag. +// The test name should be prefixed with TestNoRace so we can run only +// those tests: go test -run=TestNoRace ... + +func TestNoRaceRouteSendSubs(t *testing.T) { + template := ` + port: -1 + write_deadline: "2s" + cluster { + port: -1 + pool_size: -1 + compression: disabled + %s + } + no_sys_acc: true + ` + cfa := createConfFile(t, []byte(fmt.Sprintf(template, ""))) + srvA, optsA := RunServerWithConfig(cfa) + srvA.Shutdown() + optsA.DisableShortFirstPing = true + srvA = RunServer(optsA) + defer srvA.Shutdown() + + cfb := createConfFile(t, []byte(fmt.Sprintf(template, ""))) + srvB, optsB := RunServerWithConfig(cfb) + srvB.Shutdown() + optsB.DisableShortFirstPing = true + srvB = RunServer(optsB) + defer srvB.Shutdown() + + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + clientASend, clientAExpect := setupConn(t, clientA) + clientASend("PING\r\n") + clientAExpect(pongRe) + + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + clientBSend, clientBExpect := setupConn(t, clientB) + clientBSend("PING\r\n") + clientBExpect(pongRe) + + // total number of subscriptions per server + totalPerServer := 100000 + for i := 0; i < totalPerServer/2; i++ { + proto := fmt.Sprintf("SUB foo.%d %d\r\n", i, i*2+1) + clientASend(proto) + clientBSend(proto) + proto = fmt.Sprintf("SUB bar.%d queue.%d %d\r\n", i, i, i*2+2) + clientASend(proto) + clientBSend(proto) + } + clientASend("PING\r\n") + clientAExpect(pongRe) + clientBSend("PING\r\n") + clientBExpect(pongRe) + + if err := checkExpectedSubs(totalPerServer, srvA, srvB); err != nil { + t.Fatal(err.Error()) + } + + routes := fmt.Sprintf(` + routes: [ + "nats://%s:%d" + ] + `, optsA.Cluster.Host, optsA.Cluster.Port) + if err := os.WriteFile(cfb, []byte(fmt.Sprintf(template, routes)), 0600); err != nil { + t.Fatalf("Error rewriting B's config file: %v", err) + } + if err := srvB.Reload(); err != nil { + t.Fatalf("Error on reload: %v", err) + } + + checkClusterFormed(t, srvA, srvB) + if err := checkExpectedSubs(2*totalPerServer, srvA, srvB); err != nil { + t.Fatal(err.Error()) + } + + checkSlowConsumers := func(t *testing.T) { + t.Helper() + if srvB.NumSlowConsumers() != 0 || srvA.NumSlowConsumers() != 0 { + t.Fatalf("Expected no slow consumers, got %d for srvA and %df for srvB", + srvA.NumSlowConsumers(), srvB.NumSlowConsumers()) + } + } + checkSlowConsumers(t) + + type sender struct { + c net.Conn + sf sendFun + ef expectFun + } + var senders []*sender + createSenders := func(t *testing.T, host string, port int) { + t.Helper() + for i := 0; i < 25; i++ { + s := &sender{} + s.c = createClientConn(t, host, port) + s.sf, s.ef = setupConn(t, s.c) + s.sf("PING\r\n") + s.ef(pongRe) + senders = append(senders, s) + } + } + createSenders(t, optsA.Host, optsA.Port) + createSenders(t, optsB.Host, optsB.Port) + for _, s := range senders { + defer s.c.Close() + } + + // Now create SUBs on A and B for "ping.replies" and simulate + // that there are thousands of replies being sent on + // both sides. + createSubOnReplies := func(t *testing.T, host string, port int) net.Conn { + t.Helper() + c := createClientConn(t, host, port) + send, expect := setupConn(t, c) + send("SUB ping.replies 123456789\r\nPING\r\n") + expect(pongRe) + return c + } + requestorOnA := createSubOnReplies(t, optsA.Host, optsA.Port) + defer requestorOnA.Close() + + requestorOnB := createSubOnReplies(t, optsB.Host, optsB.Port) + defer requestorOnB.Close() + + if err := checkExpectedSubs(2*totalPerServer+2, srvA, srvB); err != nil { + t.Fatal(err.Error()) + } + + totalReplies := 120000 + payload := sizedBytes(400) + expectedBytes := (len(fmt.Sprintf("MSG ping.replies 123456789 %d\r\n\r\n", len(payload))) + len(payload)) * totalReplies + ch := make(chan error, 2) + recvReplies := func(c net.Conn) { + var buf [32 * 1024]byte + + for total := 0; total < expectedBytes; { + n, err := c.Read(buf[:]) + if err != nil { + ch <- fmt.Errorf("read error: %v", err) + return + } + total += n + } + ch <- nil + } + go recvReplies(requestorOnA) + go recvReplies(requestorOnB) + + wg := sync.WaitGroup{} + wg.Add(len(senders)) + replyMsg := fmt.Sprintf("PUB ping.replies %d\r\n%s\r\n", len(payload), payload) + for _, s := range senders { + go func(s *sender, count int) { + defer wg.Done() + for i := 0; i < count; i++ { + s.sf(replyMsg) + } + s.sf("PING\r\n") + s.ef(pongRe) + }(s, totalReplies/len(senders)) + } + + for i := 0; i < 2; i++ { + select { + case e := <-ch: + if e != nil { + t.Fatalf("Error: %v", e) + } + case <-time.After(10 * time.Second): + t.Fatalf("Did not receive all %v replies", totalReplies) + } + } + checkSlowConsumers(t) + wg.Wait() + checkSlowConsumers(t) + + // Let's remove the route and do a config reload. + // Otherwise, on test shutdown the client close + // will cause the server to try to send unsubs and + // this can delay the test. + if err := os.WriteFile(cfb, []byte(fmt.Sprintf(template, "")), 0600); err != nil { + t.Fatalf("Error rewriting B's config file: %v", err) + } + if err := srvB.Reload(); err != nil { + t.Fatalf("Error on reload: %v", err) + } +} + +func TestNoRaceDynamicResponsePermsMemory(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/authorization.conf") + defer srv.Shutdown() + + // We will test the timeout to make sure that we are not showing excessive growth + // when a reply subject is not utilized by the responder. + + // Alice can do anything, so she will be our requestor + rc := createClientConn(t, opts.Host, opts.Port) + defer rc.Close() + expectAuthRequired(t, rc) + doAuthConnect(t, rc, "", "alice", DefaultPass) + expectResult(t, rc, okRe) + + // MY_STREAM_SERVICE has an expiration of 10ms for the response permissions. + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, "", "svcb", DefaultPass) + expectResult(t, c, okRe) + + sendProto(t, c, "SUB my.service.req 1\r\n") + expectResult(t, c, okRe) + + var m runtime.MemStats + + runtime.GC() + runtime.ReadMemStats(&m) + pta := m.TotalAlloc + + // Need this so we do not blow the allocs on expectResult which makes 32k each time. + expBuf := make([]byte, 32768) + expect := func(c net.Conn, re *regexp.Regexp) { + t.Helper() + c.SetReadDeadline(time.Now().Add(2 * time.Second)) + n, _ := c.Read(expBuf) + c.SetReadDeadline(time.Time{}) + buf := expBuf[:n] + if !re.Match(buf) { + t.Fatalf("Response did not match expected: \n\tReceived:'%q'\n\tExpected:'%s'", buf, re) + } + } + + // Now send off some requests. We will not answer them and this will build up reply + // permissions in the server. + for i := 0; i < 10000; i++ { + pub := fmt.Sprintf("PUB my.service.req resp.%d 2\r\nok\r\n", i) + sendProto(t, rc, pub) + expect(rc, okRe) + expect(c, msgRe) + } + + const max = 20 * 1024 * 1024 // 20MB + checkFor(t, time.Second, 25*time.Millisecond, func() error { + runtime.GC() + runtime.ReadMemStats(&m) + used := m.TotalAlloc - pta + if used > max { + return fmt.Errorf("Using too much memory, expect < 20MB, got %dMB", used/(1024*1024)) + } + return nil + }) +} + +func TestNoRaceLargeClusterMem(t *testing.T) { + // Try to clean up. + runtime.GC() + var m runtime.MemStats + runtime.ReadMemStats(&m) + pta := m.TotalAlloc + + opts := func() *server.Options { + o := DefaultTestOptions + o.Host = "127.0.0.1" + o.Port = -1 + o.Cluster.Host = o.Host + o.Cluster.Port = -1 + return &o + } + + var servers []*server.Server + + // Create seed first. + o := opts() + s := RunServer(o) + servers = append(servers, s) + + // For connecting to seed server above. + routeAddr := fmt.Sprintf("nats-route://%s:%d", o.Cluster.Host, o.Cluster.Port) + rurl, _ := url.Parse(routeAddr) + routes := []*url.URL{rurl} + + numServers := 15 + + for i := 1; i < numServers; i++ { + o := opts() + o.Routes = routes + s := RunServer(o) + servers = append(servers, s) + } + checkClusterFormed(t, servers...) + + // Calculate in MB what we are using now. + const max = 80 * 1024 * 1024 // 80MB + runtime.ReadMemStats(&m) + used := m.TotalAlloc - pta + if used > max { + t.Fatalf("Cluster using too much memory, expect < 80MB, got %dMB", used/(1024*1024)) + } + + for _, s := range servers { + s.Shutdown() + } +} + +// Make sure we have the correct remote state when dealing with queue subscribers +// across many client connections. +func TestNoRaceQueueSubWeightOrderMultipleConnections(t *testing.T) { + opts, err := server.ProcessConfigFile("./configs/new_cluster.conf") + if err != nil { + t.Fatalf("Error processing config file: %v", err) + } + opts.DisableShortFirstPing = true + s := RunServer(opts) + defer s.Shutdown() + + // Create 100 connections to s + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + clients := make([]*nats.Conn, 0, 100) + for i := 0; i < 100; i++ { + nc, err := nats.Connect(url, nats.NoReconnect()) + if err != nil { + t.Fatalf("Error connecting: %v", err) + } + defer nc.Close() + clients = append(clients, nc) + } + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:22" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + info := checkInfoMsg(t, rc) + + info.ID = routeID + info.Name = routeID + + b, err := json.Marshal(info) + if err != nil { + t.Fatalf("Could not marshal test route info: %v", err) + } + // Send our INFO and wait for a PONG. This will prevent a race + // where server will have started processing queue subscriptions + // and then processes the route's INFO (sending its current subs) + // followed by updates. + routeSend(fmt.Sprintf("INFO %s\r\nPING\r\n", b)) + routeExpect(pongRe) + + start := make(chan bool) + for _, nc := range clients { + go func(nc *nats.Conn) { + <-start + // Now create 100 identical queue subscribers on each connection. + for i := 0; i < 100; i++ { + if _, err := nc.QueueSubscribe("foo", "bar", func(_ *nats.Msg) {}); err != nil { + return + } + } + nc.Flush() + }(nc) + } + close(start) + + // We did have this where we wanted to get every update, but now with optimizations + // we just want to make sure we always are increasing and that a previous update to + // a lesser queue weight is never delivered for this test. + maxExpected := 10000 + updates := 0 + for qw := 0; qw < maxExpected; { + buf := routeExpect(rsubRe) + matches := rsubRe.FindAllSubmatch(buf, -1) + for _, m := range matches { + if len(m) != 5 { + t.Fatalf("Expected a weight for the queue group") + } + nqw, err := strconv.Atoi(string(m[4])) + if err != nil { + t.Fatalf("Got an error converting queue weight: %v", err) + } + // Make sure the new value only increases, ok to skip since we will + // optimize this now, but needs to always be increasing. + if nqw <= qw { + t.Fatalf("Was expecting increasing queue weight after %d, got %d", qw, nqw) + } + qw = nqw + updates++ + } + } + if updates >= maxExpected { + t.Fatalf("Was not expecting all %v updates to be received", maxExpected) + } +} + +func TestNoRaceClusterLeaksSubscriptions(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + checkClusterFormed(t, srvA, srvB) + + urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, optsA.Port) + urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, optsB.Port) + + numResponses := 100 + repliers := make([]*nats.Conn, 0, numResponses) + + var noOpErrHandler = func(_ *nats.Conn, _ *nats.Subscription, _ error) {} + + // Create 100 repliers + for i := 0; i < 50; i++ { + nc1, _ := nats.Connect(urlA) + defer nc1.Close() + nc1.SetErrorHandler(noOpErrHandler) + nc2, _ := nats.Connect(urlB) + defer nc2.Close() + nc2.SetErrorHandler(noOpErrHandler) + repliers = append(repliers, nc1, nc2) + nc1.Subscribe("test.reply", func(m *nats.Msg) { + m.Respond([]byte("{\"sender\": 22 }")) + }) + nc2.Subscribe("test.reply", func(m *nats.Msg) { + m.Respond([]byte("{\"sender\": 33 }")) + }) + nc1.Flush() + nc2.Flush() + } + + servers := fmt.Sprintf("%s, %s", urlA, urlB) + req := sizedBytes(8 * 1024) + + // Now run a requestor in a loop, creating and tearing down each time to + // simulate running a modified nats-req. + doReq := func() { + msgs := make(chan *nats.Msg, 1) + inbox := nats.NewInbox() + grp := nuid.Next() + // Create 8 queue Subscribers for responses. + for i := 0; i < 8; i++ { + nc, _ := nats.Connect(servers) + nc.SetErrorHandler(noOpErrHandler) + nc.ChanQueueSubscribe(inbox, grp, msgs) + nc.Flush() + defer nc.Close() + } + nc, _ := nats.Connect(servers) + nc.SetErrorHandler(noOpErrHandler) + nc.PublishRequest("test.reply", inbox, req) + defer nc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + + var received int + for { + select { + case <-msgs: + received++ + if received >= numResponses { + return + } + case <-ctx.Done(): + return + } + } + } + + var wg sync.WaitGroup + + doRequests := func(n int) { + for i := 0; i < n; i++ { + doReq() + } + wg.Done() + } + + concurrent := 10 + wg.Add(concurrent) + for i := 0; i < concurrent; i++ { + go doRequests(10) + } + wg.Wait() + + // Close responders too, should have zero(0) subs attached to routes. + for _, nc := range repliers { + nc.Close() + } + + // Make sure no clients remain. This is to make sure the test is correct and that + // we have closed all the client connections. + checkFor(t, time.Second, 10*time.Millisecond, func() error { + v1, _ := srvA.Varz(nil) + v2, _ := srvB.Varz(nil) + if v1.Connections != 0 || v2.Connections != 0 { + return fmt.Errorf("We have lingering client connections %d:%d", v1.Connections, v2.Connections) + } + return nil + }) + + loadRoutez := func() (*server.Routez, *server.Routez) { + v1, err := srvA.Routez(&server.RoutezOptions{Subscriptions: true}) + if err != nil { + t.Fatalf("Error getting Routez: %v", err) + } + v2, err := srvB.Routez(&server.RoutezOptions{Subscriptions: true}) + if err != nil { + t.Fatalf("Error getting Routez: %v", err) + } + return v1, v2 + } + + checkFor(t, time.Second, 10*time.Millisecond, func() error { + r1, r2 := loadRoutez() + if r1.Routes[0].NumSubs != 0 { + return fmt.Errorf("Leaked %d subs: %+v", r1.Routes[0].NumSubs, r1.Routes[0].Subs) + } + if r2.Routes[0].NumSubs != 0 { + return fmt.Errorf("Leaked %d subs: %+v", r2.Routes[0].NumSubs, r2.Routes[0].Subs) + } + return nil + }) +} + +func TestNoRaceLeafNodeSmapUpdate(t *testing.T) { + s, opts := runLeafServer() + defer s.Shutdown() + + // Create a client on leaf server + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + csend, cexpect := setupConn(t, c) + + numSubs := make(chan int, 1) + doneCh := make(chan struct{}, 1) + wg := sync.WaitGroup{} + wg.Add(1) + + go func() { + defer wg.Done() + for i := 1; ; i++ { + csend(fmt.Sprintf("SUB foo.%d %d\r\n", i, i)) + select { + case <-doneCh: + numSubs <- i + return + default: + } + } + }() + + time.Sleep(5 * time.Millisecond) + // Create leaf node + lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) + defer lc.Close() + + setupConn(t, lc) + checkLeafNodeConnected(t, s) + + close(doneCh) + ns := <-numSubs + csend("PING\r\n") + cexpect(pongRe) + wg.Wait() + + // Make sure we receive as many LS+ protocols (since all subs are unique). + // But we also have to count for LDS subject. + // There may be so many protocols and partials, that expectNumberOfProtos may + // not work. Do a manual search here. + checkLS := func(proto string, expected int) { + t.Helper() + p := []byte(proto) + cur := 0 + buf := make([]byte, 32768) + for ls := 0; ls < expected; { + lc.SetReadDeadline(time.Now().Add(2 * time.Second)) + n, err := lc.Read(buf) + lc.SetReadDeadline(time.Time{}) + if err == nil && n > 0 { + for i := 0; i < n; i++ { + if buf[i] == p[cur] { + cur++ + if cur == len(p) { + ls++ + cur = 0 + } + } else { + cur = 0 + } + } + } + if err != nil || ls > expected { + t.Fatalf("Expected %v %sgot %v, err: %v", expected, proto, ls, err) + } + } + } + checkLS("LS+ ", ns+1) + + // Now unsub all those subs... + for i := 1; i <= ns; i++ { + csend(fmt.Sprintf("UNSUB %d\r\n", i)) + } + csend("PING\r\n") + cexpect(pongRe) + + // Expect that many LS- + checkLS("LS- ", ns) +} + +func TestNoRaceSlowProxy(t *testing.T) { + t.Skip() + + opts := DefaultTestOptions + opts.Port = -1 + s := RunServer(&opts) + defer s.Shutdown() + + rttTarget := 22 * time.Millisecond + bwTarget := 10 * 1024 * 1024 / 8 // 10mbit + + sp := newSlowProxy(rttTarget, bwTarget, bwTarget, &opts) + defer sp.stop() + + nc, err := nats.Connect(sp.clientURL()) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + defer nc.Close() + + doRTT := func() time.Duration { + t.Helper() + const samples = 5 + var total time.Duration + for i := 0; i < samples; i++ { + rtt, _ := nc.RTT() + total += rtt + } + return total / samples + } + + rtt := doRTT() + if rtt < rttTarget || rtt > (rttTarget*3/2) { + t.Fatalf("rtt is out of range, target of %v, actual %v", rttTarget, rtt) + } + + // Now test send BW. + const payloadSize = 64 * 1024 + var payload [payloadSize]byte + crand.Read(payload[:]) + + // 5MB total. + bytesSent := (5 * 1024 * 1024) + toSend := bytesSent / payloadSize + + start := time.Now() + for i := 0; i < toSend; i++ { + nc.Publish("z", payload[:]) + } + nc.Flush() + tt := time.Since(start) + bps := float64(bytesSent) / tt.Seconds() + min, max := float64(bwTarget)*0.8, float64(bwTarget)*1.25 + if bps < min || bps > max { + t.Fatalf("bps is off, target is %v, actual is %v", bwTarget, bps) + } +} diff --git a/test/ocsp_peer_test.go b/test/ocsp_peer_test.go new file mode 100644 index 0000000..7f89928 --- /dev/null +++ b/test/ocsp_peer_test.go @@ -0,0 +1,3101 @@ +// Copyright 2023-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/nats-io/nats-server/v2/internal/ocsp" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "golang.org/x/crypto/ocsp" +) + +func NewOCSPResponderRootCA(t *testing.T) *http.Server { + t.Helper() + respCertPEM := "configs/certs/ocsp_peer/mini-ca/caocsp/caocsp_cert.pem" + respKeyPEM := "configs/certs/ocsp_peer/mini-ca/caocsp/private/caocsp_keypair.pem" + issuerCertPEM := "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + return NewOCSPResponderDesignatedCustomAddress(t, issuerCertPEM, respCertPEM, respKeyPEM, "127.0.0.1:8888") +} + +func NewOCSPResponderIntermediateCA1(t *testing.T) *http.Server { + t.Helper() + respCertPEM := "configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_bundle.pem" + respKeyPEM := "configs/certs/ocsp_peer/mini-ca/ocsp1/private/ocsp1_keypair.pem" + issuerCertPEM := "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem" + return NewOCSPResponderDesignatedCustomAddress(t, issuerCertPEM, respCertPEM, respKeyPEM, "127.0.0.1:18888") +} + +func NewOCSPResponderIntermediateCA1Undelegated(t *testing.T) *http.Server { + t.Helper() + issuerCertPEM := "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem" + issuerCertKey := "configs/certs/ocsp_peer/mini-ca/intermediate1/private/intermediate1_keypair.pem" + return NewOCSPResponderCustomAddress(t, issuerCertPEM, issuerCertKey, "127.0.0.1:18888") +} + +func NewOCSPResponderBadDelegateIntermediateCA1(t *testing.T) *http.Server { + t.Helper() + // UserA2 is a cert issued by intermediate1, but intermediate1 did not add OCSP signing extension + respCertPEM := "configs/certs/ocsp_peer/mini-ca/client1/UserA2_bundle.pem" + respKeyPEM := "configs/certs/ocsp_peer/mini-ca/client1/private/UserA2_keypair.pem" + issuerCertPEM := "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem" + return NewOCSPResponderDesignatedCustomAddress(t, issuerCertPEM, respCertPEM, respKeyPEM, "127.0.0.1:18888") +} + +func NewOCSPResponderIntermediateCA2(t *testing.T) *http.Server { + t.Helper() + respCertPEM := "configs/certs/ocsp_peer/mini-ca/ocsp2/ocsp2_bundle.pem" + respKeyPEM := "configs/certs/ocsp_peer/mini-ca/ocsp2/private/ocsp2_keypair.pem" + issuerCertPEM := "configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem" + return NewOCSPResponderDesignatedCustomAddress(t, issuerCertPEM, respCertPEM, respKeyPEM, "127.0.0.1:28888") +} + +// TestOCSPPeerGoodClients is test of two NATS client (AIA enabled at leaf and cert) under good path (different intermediates) +// and default ocsp_cache implementation and oscp_cache=false configuration +func TestOCSPPeerGoodClients(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem", ocsp.Good) + + intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) + + intermediateCA2Responder := NewOCSPResponderIntermediateCA2(t) + intermediateCA2ResponderURL := fmt.Sprintf("http://%s", intermediateCA2Responder.Addr) + defer intermediateCA2Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA2ResponderURL, "configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem", ocsp.Good) + + for _, test := range []struct { + name string + config string + opts []nats.Option + err error + rerr error + configure func() + }{ + { + "Default cache: mTLS OCSP peer check on inbound client connection, client of intermediate CA 1", + ` + port: -1 + # default ocsp_cache since omitted + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration, non-default ca_timeout + ocsp_peer: { + verify: true + ca_timeout: 5 + allowed_clockskew: 30 + } + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() {}, + }, + { + "Default cache: mTLS OCSP peer check on inbound client connection, client of intermediate CA 2", + ` + port: -1 + # default ocsp_cache since omitted + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Short form configuration + ocsp_peer: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client2/UserB1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client2/private/UserB1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() {}, + }, + { + "Explicit true cache: mTLS OCSP peer check on inbound client connection, client of intermediate CA 1", + ` + port: -1 + # Short form configuration + ocsp_cache: true + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + ca_timeout: 5 + allowed_clockskew: 30 + } + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() {}, + }, + } { + t.Run(test.name, func(t *testing.T) { + deleteLocalStore(t, "") + test.configure() + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + nc.Subscribe("ping", func(m *nats.Msg) { + m.Respond([]byte("pong")) + }) + nc.Flush() + _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) + if test.rerr != nil && err == nil { + t.Errorf("Expected error getting response") + } else if test.rerr == nil && err != nil { + t.Errorf("Expected response") + } + }) + } +} + +// TestOCSPPeerUnknownClient is test of NATS client that is OCSP status Unknown from its OCSP Responder +func TestOCSPPeerUnknownClient(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + + intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) + defer intermediateCA1Responder.Shutdown(ctx) + + for _, test := range []struct { + name string + config string + opts []nats.Option + err error + rerr error + configure func() + }{ + { + "Default cache, mTLS OCSP peer check on inbound client connection, client unknown to intermediate CA 1", + ` + port: -1 + # Cache configuration is default + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Short form configuration + ocsp_peer: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + errors.New("remote error: tls: bad certificate"), + errors.New("expect error"), + func() {}, + }, + } { + t.Run(test.name, func(t *testing.T) { + deleteLocalStore(t, "") + test.configure() + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + + t.Errorf("Expected connection error, fell through") + }) + } +} + +// TestOCSPPeerRevokedClient is test of NATS client that is OCSP status Revoked from its OCSP Responder +func TestOCSPPeerRevokedClient(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + + intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Revoked) + + for _, test := range []struct { + name string + config string + opts []nats.Option + err error + rerr error + configure func() + }{ + { + "mTLS OCSP peer check on inbound client connection, client revoked by intermediate CA 1", + ` + port: -1 + # Cache configuration is default + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Turn on CA OCSP check so this revoked client should NOT be able to connect + ocsp_peer: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + errors.New("remote error: tls: bad certificate"), + errors.New("expect error"), + func() {}, + }, + { + "Explicit disable, mTLS OCSP peer check on inbound client connection, client revoked by intermediate CA 1 but no OCSP check", + ` + port: -1 + # Cache configuration is default + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Explicit disable of OCSP peer check + ocsp_peer: false + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() {}, + }, + { + "Implicit disable, mTLS OCSP peer check on inbound client connection, client revoked by intermediate CA 1 but no OCSP check", + ` + port: -1 + # Cache configuration is default + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Implicit disable of OCSP peer check (i.e. not configured) + # ocsp_peer: false + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() {}, + }, + { + "Explicit disable (long form), mTLS OCSP peer check on inbound client connection, client revoked by intermediate CA 1 but no OCSP check", + ` + port: -1 + # Cache configuration is default + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Explicit disable of OCSP peer check, long form + ocsp_peer: { verify: false } + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() {}, + }, + } { + t.Run(test.name, func(t *testing.T) { + deleteLocalStore(t, "") + test.configure() + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + }) + } +} + +// TestOCSPPeerUnknownAndRevokedIntermediate test of NATS client that is OCSP good but either its intermediate is unknown or revoked +func TestOCSPPeerUnknownAndRevokedIntermediate(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Revoked) + // No test OCSP status set on intermediate2, so unknown + + intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) + + intermediateCA2Responder := NewOCSPResponderIntermediateCA2(t) + intermediateCA2ResponderURL := fmt.Sprintf("http://%s", intermediateCA2Responder.Addr) + defer intermediateCA2Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA2ResponderURL, "configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem", ocsp.Good) + + for _, test := range []struct { + name string + config string + opts []nats.Option + err error + rerr error + configure func() + }{ + { + "mTLS OCSP peer check on inbound client connection, client's intermediate is revoked", + ` + port: -1 + # Cache configuration is default + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Short form configuration + ocsp_peer: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + errors.New("remote error: tls: bad certificate"), + errors.New("expect error"), + func() {}, + }, + { + "mTLS OCSP peer check on inbound client connection, client's intermediate is unknown'", + ` + port: -1 + # Cache configuration is default + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Short form configuration + ocsp_peer: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client2/UserB1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client2/private/UserB1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + errors.New("remote error: tls: bad certificate"), + errors.New("expect error"), + func() {}, + }, + } { + t.Run(test.name, func(t *testing.T) { + deleteLocalStore(t, "") + test.configure() + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + + t.Errorf("Expected connection error, fell through") + }) + } +} + +// TestOCSPPeerLeafGood tests Leaf Spoke peer checking Leaf Hub, Leaf Hub peer checking Leaf Spoke, and both peer checking +func TestOCSPPeerLeafGood(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + + intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem", ocsp.Good) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer2_cert.pem", ocsp.Good) + + for _, test := range []struct { + name string + hubconfig string + spokeconfig string + expected int + }{ + { + "OCSP peer check on Leaf Hub by Leaf Spoke (TLS client OCSP verification of TLS server)", + ` + port: -1 + # Cache configuration is default + leaf: { + listen: 127.0.0.1:7444 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + } + } + `, + ` + port: -1 + leaf: { + remotes: [ + { + url: "nats://127.0.0.1:7444", + tls: { + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + # Short form configuration + ocsp_peer: true + } + } + ] + } + `, + 1, + }, + { + "OCSP peer check on Leaf Spoke by Leaf Hub (TLS server OCSP verification of TLS client)", + ` + port: -1 + # Cache configuration is default + leaf: { + listen: 127.0.0.1:7444 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Short form configuration + ocsp_peer: true + } + } + `, + ` + port: -1 + leaf: { + remotes: [ + { + url: "nats://127.0.0.1:7444", + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + } + } + ] + } + `, + 1, + }, + { + "OCSP peer check bi-directionally", + ` + port: -1 + # Cache configuration is default + leaf: { + listen: 127.0.0.1:7444 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Short form configuration + ocsp_peer: true + } + } + `, + ` + port: -1 + leaf: { + remotes: [ + { + url: "nats://127.0.0.1:7444", + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + # Short form configuration + ocsp_peer: true + } + } + ] + } + `, + 1, + }, + } { + t.Run(test.name, func(t *testing.T) { + deleteLocalStore(t, "") + hubcontent := test.hubconfig + hubconf := createConfFile(t, []byte(hubcontent)) + hub, _ := RunServerWithConfig(hubconf) + defer hub.Shutdown() + + spokecontent := test.spokeconfig + spokeconf := createConfFile(t, []byte(spokecontent)) + spoke, _ := RunServerWithConfig(spokeconf) + defer spoke.Shutdown() + + checkLeafNodeConnectedCount(t, hub, test.expected) + }) + } +} + +// TestOCSPPeerLeafReject tests rejected Leaf Hub, rejected Leaf Spoke, and both rejecting each other +func TestOCSPPeerLeafReject(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + + intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem", ocsp.Revoked) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer2_cert.pem", ocsp.Revoked) + + for _, test := range []struct { + name string + hubconfig string + spokeconfig string + expected int + }{ + { + "OCSP peer check on Leaf Hub by Leaf Spoke (TLS client OCSP verification of TLS server)", + ` + port: -1 + # Cache configuration is default + leaf: { + listen: 127.0.0.1:7444 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + } + } + `, + ` + port: -1 + leaf: { + remotes: [ + { + url: "nats://127.0.0.1:7444", + tls: { + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + # Short form configuration + ocsp_peer: true + } + } + ] + } + `, + 0, + }, + { + "OCSP peer check on Leaf Spoke by Leaf Hub (TLS server OCSP verification of TLS client)", + ` + port: -1 + leaf: { + listen: 127.0.0.1:7444 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Short form configuration + ocsp_peer: true + } + } + `, + ` + port: -1 + leaf: { + remotes: [ + { + url: "nats://127.0.0.1:7444", + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + } + } + ] + } + `, + 0, + }, + { + "OCSP peer check bi-directionally", + ` + port: -1 + leaf: { + listen: 127.0.0.1:7444 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Short form configuration + ocsp_peer: true + } + } + `, + ` + port: -1 + leaf: { + remotes: [ + { + url: "nats://127.0.0.1:7444", + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + # Short form configuration + ocsp_peer: true + } + } + ] + } + `, + 0, + }, + } { + t.Run(test.name, func(t *testing.T) { + deleteLocalStore(t, "") + hubcontent := test.hubconfig + hubconf := createConfFile(t, []byte(hubcontent)) + hub, _ := RunServerWithConfig(hubconf) + defer hub.Shutdown() + spokecontent := test.spokeconfig + spokeconf := createConfFile(t, []byte(spokecontent)) + spoke, _ := RunServerWithConfig(spokeconf) + defer spoke.Shutdown() + // Need to inject some time for leaf connection attempts to complete, could refine this to better + // negative test + time.Sleep(2000 * time.Millisecond) + checkLeafNodeConnectedCount(t, hub, test.expected) + }) + } +} + +func checkLeafNodeConnectedCount(t testing.TB, s *server.Server, lnCons int) { + t.Helper() + checkFor(t, 5*time.Second, 15*time.Millisecond, func() error { + if nln := s.NumLeafNodes(); nln != lnCons { + return fmt.Errorf("expected %d connected leafnode(s) for server %q, got %d", + lnCons, s.ID(), nln) + } + return nil + }) +} + +// TestOCSPPeerGoodClientsNoneCache is test of two NATS client (AIA enabled at leaf and cert) under good path (different intermediates) +// and ocsp cache type of none (no-op) +func TestOCSPPeerGoodClientsNoneCache(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem", ocsp.Good) + + intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) + + intermediateCA2Responder := NewOCSPResponderIntermediateCA2(t) + intermediateCA2ResponderURL := fmt.Sprintf("http://%s", intermediateCA2Responder.Addr) + defer intermediateCA2Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA2ResponderURL, "configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem", ocsp.Good) + + deleteLocalStore(t, "") + + for _, test := range []struct { + name string + config string + opts []nats.Option + err error + rerr error + configure func() + }{ + { + "None cache explicit long form: mTLS OCSP peer check on inbound client connection, client of intermediate CA 1", + ` + port: -1 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + ca_timeout: 5 + allowed_clockskew: 30 + } + } + # Long form configuration + ocsp_cache: { + type: none + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() {}, + }, + { + "None cache explicit short form: mTLS OCSP peer check on inbound client connection, client of intermediate CA 1", + ` + port: -1 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + ca_timeout: 5 + allowed_clockskew: 30 + } + } + # Short form configuration + ocsp_cache: false + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() {}, + }, + } { + t.Run(test.name, func(t *testing.T) { + test.configure() + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + nc.Subscribe("ping", func(m *nats.Msg) { + m.Respond([]byte("pong")) + }) + nc.Flush() + _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) + if test.rerr != nil && err == nil { + t.Errorf("Expected error getting response") + } else if test.rerr == nil && err != nil { + t.Errorf("Expected response") + } + }) + } +} + +// TestOCSPPeerGoodClientsLocalCache is test of two NATS client (AIA enabled at leaf and cert) under good path (different intermediates) +// and leveraging the local ocsp cache type +func TestOCSPPeerGoodClientsLocalCache(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem", ocsp.Good) + + intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) + + intermediateCA2Responder := NewOCSPResponderIntermediateCA2(t) + intermediateCA2ResponderURL := fmt.Sprintf("http://%s", intermediateCA2Responder.Addr) + defer intermediateCA2Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA2ResponderURL, "configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem", ocsp.Good) + + for _, test := range []struct { + name string + config string + opts []nats.Option + err error + rerr error + configure func() + }{ + { + "Default cache, short form: mTLS OCSP peer check on inbound client connection, UserA1 client of intermediate CA 1", + ` + port: -1 + http_port: 8222 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + ca_timeout: 5 + allowed_clockskew: 30 + } + } + # Short form configuration, local as default + ocsp_cache: true + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() {}, + }, + { + "Local cache long form: mTLS OCSP peer check on inbound client connection, UserB1 client of intermediate CA 2", + ` + port: -1 + http_port: 8222 + + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Short form configuration + ocsp_peer: true + } + # Long form configuration + ocsp_cache: { + type: local + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client2/UserB1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client2/private/UserB1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() {}, + }, + } { + t.Run(test.name, func(t *testing.T) { + // Cleanup any previous test that saved a local cache + deleteLocalStore(t, "") + test.configure() + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + nc.Close() + + v := monitorGetVarzHelper(t, 8222) + if v.OCSPResponseCache == nil { + t.Fatalf("Expected OCSP statistics to be in varz") + } + if v.OCSPResponseCache.Misses != 2 || v.OCSPResponseCache.Responses != 2 { + t.Errorf("Expected cache misses and cache items to be 2, got %d and %d", v.OCSPResponseCache.Misses, v.OCSPResponseCache.Responses) + } + + // Should get a cache hit now + nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + nc.Subscribe("ping", func(m *nats.Msg) { + m.Respond([]byte("pong")) + }) + nc.Flush() + _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) + if test.rerr != nil && err == nil { + t.Errorf("Expected error getting response") + } else if test.rerr == nil && err != nil { + t.Errorf("Expected response") + } + + v = monitorGetVarzHelper(t, 8222) + if v.OCSPResponseCache == nil { + t.Fatalf("Expected OCSP statistics to be in varz") + } + if v.OCSPResponseCache.Misses != 2 || v.OCSPResponseCache.Hits != 2 || v.OCSPResponseCache.Responses != 2 { + t.Errorf("Expected cache misses, hits and cache items to be 2, got %d and %d and %d", v.OCSPResponseCache.Misses, v.OCSPResponseCache.Hits, v.OCSPResponseCache.Responses) + } + }) + } +} + +func TestOCSPPeerMonitor(t *testing.T) { + for _, test := range []struct { + name string + config string + NATSClient bool + WSClient bool + MQTTClient bool + LeafClient bool + LeafRemotes bool + NumTrueLeafRemotes int + }{ + { + "Monitor peer config setting on NATS client", + ` + port: -1 + http_port: 8222 + # Default cache configuration + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + } + } + `, + true, + false, + false, + false, + false, + 0, + }, + { + "Monitor peer config setting on Websockets client", + ` + port: -1 + http_port: 8222 + # Default cache configuration + websocket: { + port: 8443 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + } + } + } + `, + false, + true, + false, + false, + false, + 0, + }, + { + "Monitor peer config setting on MQTT client", + ` + port: -1 + http_port: 8222 + # Default cache configuration + # Required for MQTT + server_name: "my_mqtt_server" + jetstream: { + enabled: true + } + mqtt: { + port: 1883 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + } + } + } + `, + false, + false, + true, + false, + false, + 0, + }, + { + "Monitor peer config setting on Leaf client", + ` + port: -1 + http_port: 8222 + # Default cache configuration + leaf: { + port: 7422 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + } + } + } + `, + false, + false, + false, + true, + false, + 0, + }, + { + "Monitor peer config on some Leaf Remotes as well as Leaf client", + ` + port: -1 + http_port: 8222 + # Default cache configuration + leaf: { + port: 7422 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + } + } + remotes: [ + { + url: "nats-leaf://bogus:7422" + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + # Long form configuration + ocsp_peer: { + verify: true + } + } + }, + { + url: "nats-leaf://anotherbogus:7422" + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + # Short form configuration + ocsp_peer: true + } + }, + { + url: "nats-leaf://yetanotherbogus:7422" + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + # Peer not configured (default false) + } + } + ] + } + `, + false, + false, + false, + true, + true, + 2, + }, + } { + t.Run(test.name, func(t *testing.T) { + content := test.config + conf := createConfFile(t, []byte(content)) + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + v := monitorGetVarzHelper(t, 8222) + if test.NATSClient { + if !v.TLSOCSPPeerVerify { + t.Fatalf("Expected NATS Client TLSOCSPPeerVerify to be true, got false") + } + } + if test.WSClient { + if !v.Websocket.TLSOCSPPeerVerify { + t.Fatalf("Expected WS Client TLSOCSPPeerVerify to be true, got false") + } + } + if test.LeafClient { + if !v.LeafNode.TLSOCSPPeerVerify { + t.Fatalf("Expected Leaf Client TLSOCSPPeerVerify to be true, got false") + } + } + if test.LeafRemotes { + cnt := 0 + for _, r := range v.LeafNode.Remotes { + if r.TLSOCSPPeerVerify { + cnt++ + } + } + if cnt != test.NumTrueLeafRemotes { + t.Fatalf("Expected %d Leaf Remotes with TLSOCSPPeerVerify true, got %d", test.NumTrueLeafRemotes, cnt) + } + } + }) + } +} + +func TestOCSPResponseCacheMonitor(t *testing.T) { + for _, test := range []struct { + name string + config string + expect string + }{ + { + "Monitor local cache enabled, explicit cache true", + ` + port: -1 + http_port: 8222 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + } + } + # Short form configuration + ocsp_cache: true + `, + "local", + }, + { + "Monitor local cache enabled, explicit cache type local", + ` + port: -1 + http_port: 8222 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + } + } + # Long form configuration + ocsp_cache: { + type: local + } + `, + "local", + }, + { + "Monitor local cache enabled, implicit default", + ` + port: -1 + http_port: 8222 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + } + } + # Short form configuration + # ocsp_cache: true + `, + "local", + }, + { + "Monitor none cache enabled, explicit cache false (short)", + ` + port: -1 + http_port: 8222 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + } + } + # Short form configuration + ocsp_cache: false + `, + "", + }, + { + "Monitor none cache enabled, explicit cache false (long)", + ` + port: -1 + http_port: 8222 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + } + } + # Long form configuration + ocsp_cache: { + type: none + } + `, + "", + }, + } { + t.Run(test.name, func(t *testing.T) { + deleteLocalStore(t, "") + content := test.config + conf := createConfFile(t, []byte(content)) + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + v := monitorGetVarzHelper(t, 8222) + var ct string + if v.OCSPResponseCache != nil { + ct = v.OCSPResponseCache.Type + } + if ct != test.expect { + t.Fatalf("Expected OCSP Response Cache to be %s, got %s", test.expect, ct) + } + }) + } +} + +func TestOCSPResponseCacheChangeAndReload(t *testing.T) { + deleteLocalStore(t, "") + + // Start with ocsp cache set to none + content := ` + port: -1 + http_port: 8222 + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Short form configuration + ocsp_peer: true + } + # Long form configuration + ocsp_cache: { + type: none + } + ` + conf := createConfFile(t, []byte(content)) + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + v := monitorGetVarzHelper(t, 8222) + var ct string + if v.OCSPResponseCache != nil { + ct = v.OCSPResponseCache.Type + } + if ct != "" { + t.Fatalf("Expected OCSP Response Cache to have empty type in varz indicating none") + } + + // Change to local cache + content = ` + port: -1 + http_port: 8222 + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Short form configuration + ocsp_peer: true + } + # Long form configuration + ocsp_cache: { + type: local + } + ` + if err := os.WriteFile(conf, []byte(content), 0666); err != nil { + t.Fatalf("Error writing config: %v", err) + } + if err := s.Reload(); err != nil { + t.Fatal(err) + } + time.Sleep(2 * time.Second) + v = monitorGetVarzHelper(t, 8222) + ct = "" + if v.OCSPResponseCache != nil { + ct = v.OCSPResponseCache.Type + } + if ct != "local" { + t.Fatalf("Expected OCSP Response Cache type to be local, got %q", ct) + } +} + +func deleteLocalStore(t *testing.T, dir string) { + t.Helper() + if dir == "" { + // default + dir = "_rc_" + } + if err := os.RemoveAll(dir); err != nil { + t.Fatalf("Error cleaning up local store: %v", err) + } +} + +func monitorGetVarzHelper(t *testing.T, httpPort int) *server.Varz { + t.Helper() + url := fmt.Sprintf("http://127.0.0.1:%d/", httpPort) + resp, err := http.Get(url + "varz") + if err != nil { + t.Fatalf("Expected no error: Got %v\n", err) + } + if resp.StatusCode != 200 { + t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Got an error reading the body: %v\n", err) + } + v := server.Varz{} + if err := json.Unmarshal(body, &v); err != nil { + t.Fatalf("Got an error unmarshalling the body: %v\n", err) + } + return &v +} + +func writeCacheFile(dir string, content []byte) error { + if dir == "" { + dir = "_rc_" + } + err := os.MkdirAll(filepath.Join(dir), os.ModePerm) + if err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, "cache.json"), content, os.ModePerm) +} + +// TestOCSPPeerPreserveRevokedCacheItem is test of the preserve_revoked cache policy +func TestOCSPPeerPreserveRevokedCacheItem(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + + for _, test := range []struct { + name string + config string + opts []nats.Option + responses int64 + revokes int64 + goods int64 + unknowns int64 + err error + rerr error + clean bool + }{ + { + "Test expired revoked cert not actually deleted", + ` + port: -1 + http_port: 8222 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Turn on CA OCSP check so this revoked client should NOT be able to connect + ocsp_peer: { + verify: true + ca_timeout: 0.5 + } + } + # preserve revoked true + ocsp_cache: { + type: local + preserve_revoked: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + 1, + 1, + 0, + 0, + errors.New("remote error: tls: bad certificate"), + errors.New("expect error"), + true, + }, + { + "Test expired revoked cert replaced by current good cert", + ` + port: -1 + http_port: 8222 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Turn on CA OCSP check so this revoked client should NOT be able to connect + ocsp_peer: true + } + # preserve revoked true + ocsp_cache: { + type: local + preserve_revoked: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + 2, + 0, + 2, + 0, + nil, + nil, + false, + }, + } { + t.Run(test.name, func(t *testing.T) { + var intermediateCA1Responder *http.Server + // clean slate starting the test and start the leaf CA responder for first run + if test.clean { + deleteLocalStore(t, "") + // establish the revoked item (expired) in cache + c := []byte(` + { + "5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=": { + "subject": "CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US", + "cached_at": "2023-05-29T17:56:45Z", + "resp_status": "revoked", + "resp_expires": "2023-05-29T17:56:49Z", + "resp": "/wYAAFMyc1R3TwBSBQBao1Qr1QzUMIIGUQoBAKCCBkowggZGBgkrBgEFBQcwAQEEggY3MIIGMzCB46FZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HQDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA1MjkxNzU2MDBaMHUwczBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBc2ZAAQNDVaoBE2dwD0QQE0OVowDQYJKoZIhvcNAQELBQADggEBAGAax/vkv3SBFNbxp2utc/N6Rje4E0ceC972sWgqYjzYrH0oc/acg+OAXaxUjwqoQWaT+dHaI4D5qoTkMx7XlWATjI2L72IUTf6Luo92jPzyDFwb10CdeFHtRtEYD54Qbi/nD4oxQ8cSoLKC3wft2l3E/mK/1I4Mxwq15CioK4MhfzTISoeGZbjDXPKgloJOG3rn9v64vFGV6dosbLgaXEs+MPcCsPQYkwhOOyazuewRmIDOBp5QSsKPhqsT8Rs20t8LGTMkvjZniFWJs90l9QL9F1m3obq5nyuxrGt+7Rf5zoj4T+0XCOGtE+b7cRCLg43tFuTbaAQG8Z+qkPzpza+gggQ1MIIEMTCCBC0wggMVoAMCAQICFCnhUo39pSqH6x3kHUds4YpYaXOrOj8BBDBaUSLaLwIIGjAYSS+oEUludGVybWVkaWF0ZSBDQSAxMB4XDTIzMDUwMTE5MjgzOVoXDTMzMDQyOA0PUasVAEkMMIIBIi4nAgABBQD0QAEPADCCAQoCggEBAKMMyuuA66EOHnGb07P5Zc5wwiEGPDHBBn6lqErhIaN0VJ9XzlDWwyk8Q7CdPlSU7o36DXFs316eATB5bLuXXa+7WwV3cp9V5mZF9OLCz3sOWNYUanYprOMwKA3uvcqqrh8e70Dzw6sX8tfsDeH7aJoJg5kRWEKU+A3Umm+fO+hW8Km3GBqRQXxD49uxAfGtCznXZZjmFbAXqVZu+4R6wMxndfz2dYQxeMVtUY/QGdMWT4fvWzO5et3+X6hq/URUAPOkplv9O2U4T4JPucS9yZpW/FTxWC/L7vQI/bfsrSgIZpv4eJgy27FW3Q4xusbjVvUCL/t2KLvEi/Nr2qodOCECAwEAAaOB7TCB6jAdBgNVHQ4EFgQUy15QYHqrL6k7HiSrAkKN7IFgSBMwHwYDVR0jBBgwFoBSyQNQMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQ4YBAMCB4AwFgEeACUBEBAMMAoGCIm0sAMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4ODgvaV1WKDFfY3JsLmRlcjAzEUscAQEEJzAlMCMRWwwwAYYXWkoALhICkTnw/0hlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=" + } + }`) + err := writeCacheFile("", c) + if err != nil { + t.Fatal(err) + } + } else { + intermediateCA1Responder = NewOCSPResponderIntermediateCA1(t) + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) + defer intermediateCA1Responder.Shutdown(ctx) + } + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + v := monitorGetVarzHelper(t, 8222) + if v.OCSPResponseCache == nil { + t.Fatalf("Expected OCSP statistics to be in varz") + } + responses := v.OCSPResponseCache.Responses + revokes := v.OCSPResponseCache.Revokes + goods := v.OCSPResponseCache.Goods + unknowns := v.OCSPResponseCache.Unknowns + if !(responses == test.responses && revokes == test.revokes && goods == test.goods && unknowns == test.unknowns) { + t.Fatalf("Expected %d response, %d revoked, %d good, %d unknown; got [%d] and [%d] and [%d] and [%d]", test.responses, test.revokes, test.goods, test.unknowns, responses, revokes, goods, unknowns) + } + }) + } +} + +// TestOCSPStapleFeatureInterop is a test of a NATS client (AIA enabled at leaf and cert) connecting to a NATS Server +// in which both ocsp_peer is enabled on NATS client connections (verify client) and the ocsp staple is enabled such +// that the NATS Server will staple its own OCSP response and make available to the NATS client during handshake. +func TestOCSPStapleFeatureInterop(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + + intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem", ocsp.Good) + + for _, test := range []struct { + name string + config string + opts []nats.Option + err error + rerr error + configure func() + }{ + { + "Interop: Both Good: mTLS OCSP peer check on inbound client connection and server's OCSP staple validated at client", + ` + port: -1 + ocsp_cache: true + ocsp: { + mode: always + } + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration, non-default ca_timeout + ocsp_peer: { + verify: true + ca_timeout: 5 + allowed_clockskew: 30 + } + } + `, + []nats.Option{ + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("expected OCSP staple to be present") + } + resp, err := ocsp.ParseResponse(s.OCSPResponse, s.VerifiedChains[0][1]) + if err != nil || resp.Status != ocsp.Good { + return fmt.Errorf("expected a valid GOOD stapled response") + } + return nil + }, + }), + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() { + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) + }, + }, + { + "Interop: Bad Client: mTLS OCSP peer check on inbound client connection and server's OCSP staple validated at client", + ` + port: -1 + ocsp_cache: true + ocsp: { + mode: always + } + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration, non-default ca_timeout + ocsp_peer: { + verify: true + ca_timeout: 5 + allowed_clockskew: 30 + } + } + `, + []nats.Option{ + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("expected OCSP staple to be present") + } + resp, err := ocsp.ParseResponse(s.OCSPResponse, s.VerifiedChains[0][1]) + if err != nil || resp.Status != ocsp.Good { + return fmt.Errorf("expected a valid GOOD stapled response") + } + return nil + }, + }), + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + fmt.Errorf("remote error: tls: bad certificate"), + nil, + func() { + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Revoked) + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + deleteLocalStore(t, "") + test.configure() + content := test.config + conf := createConfFile(t, []byte(content)) + + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + nc.Subscribe("ping", func(m *nats.Msg) { + m.Respond([]byte("pong")) + }) + nc.Flush() + _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) + if test.rerr != nil && err == nil { + t.Errorf("Expected error getting response") + } else if test.rerr == nil && err != nil { + t.Errorf("Expected response") + } + }) + } +} + +// TestOCSPPeerWarnOnlyOption is test of NATS client that is OCSP Revoked status but allowed to pass with warn_only option +func TestOCSPPeerWarnOnlyOption(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + + intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Revoked) + + for _, test := range []struct { + name string + config string + opts []nats.Option + err error + rerr error + configure func() + }{ + { + "Revoked NATS client with warn_only explicitly set to false", + ` + port: -1 + # Cache configuration is default + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Enable OCSP peer but with warn_only option set to false + ocsp_peer: { + verify: true + warn_only: false + } + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + errors.New("remote error: tls: bad certificate"), + errors.New("expect error"), + func() {}, + }, + { + "Revoked NATS client with warn_only explicitly set to true", + ` + port: -1 + # Cache configuration is default + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Enable OCSP peer but with warn_only option set to true + ocsp_peer: { + verify: true + warn_only: true + } + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() {}, + }, + } { + t.Run(test.name, func(t *testing.T) { + deleteLocalStore(t, "") + test.configure() + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + }) + } +} + +// TestOCSPPeerUnknownIsGoodOption is test of NATS client that is OCSP status Unknown from its OCSP Responder but we treat +// status Unknown as "Good" +func TestOCSPPeerUnknownIsGoodOption(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + + intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) + defer intermediateCA1Responder.Shutdown(ctx) + + for _, test := range []struct { + name string + config string + opts []nats.Option + err error + rerr error + configure func() + }{ + { + "Unknown NATS client with no unknown_is_good option set (default false)", + ` + port: -1 + # Cache configuration is default + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Short form configuration + ocsp_peer: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + errors.New("remote error: tls: bad certificate"), + errors.New("expect error"), + func() {}, + }, + { + "Unknown NATS client with unknown_is_good set to true", + ` + port: -1 + # Cache configuration is default + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + unknown_is_good: true + } + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() {}, + }, + } { + t.Run(test.name, func(t *testing.T) { + deleteLocalStore(t, "") + test.configure() + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + }) + } +} + +// TestOCSPPeerAllowWhenCAUnreachableOption is test of the allow_when_ca_unreachable peer option +func TestOCSPPeerAllowWhenCAUnreachableOption(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + + for _, test := range []struct { + name string + config string + opts []nats.Option + cachedResponse string + err error + rerr error + }{ + { + "Expired Revoked response in cache for UserA1 -- should be rejected connection (expired revoke honored)", + ` + port: -1 + http_port: 8222 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Turn on CA OCSP check but allow when CA is unreachable + ocsp_peer: { + verify: true + ca_timeout: 0.5 + allow_when_ca_unreachable: true + } + } + # preserve revoked true + ocsp_cache: { + type: local + preserve_revoked: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + ` + { + "5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=": { + "subject": "CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US", + "cached_at": "2023-05-29T17:56:45Z", + "resp_status": "revoked", + "resp_expires": "2023-05-29T17:56:49Z", + "resp": "/wYAAFMyc1R3TwBSBQBao1Qr1QzUMIIGUQoBAKCCBkowggZGBgkrBgEFBQcwAQEEggY3MIIGMzCB46FZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HQDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA1MjkxNzU2MDBaMHUwczBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBc2ZAAQNDVaoBE2dwD0QQE0OVowDQYJKoZIhvcNAQELBQADggEBAGAax/vkv3SBFNbxp2utc/N6Rje4E0ceC972sWgqYjzYrH0oc/acg+OAXaxUjwqoQWaT+dHaI4D5qoTkMx7XlWATjI2L72IUTf6Luo92jPzyDFwb10CdeFHtRtEYD54Qbi/nD4oxQ8cSoLKC3wft2l3E/mK/1I4Mxwq15CioK4MhfzTISoeGZbjDXPKgloJOG3rn9v64vFGV6dosbLgaXEs+MPcCsPQYkwhOOyazuewRmIDOBp5QSsKPhqsT8Rs20t8LGTMkvjZniFWJs90l9QL9F1m3obq5nyuxrGt+7Rf5zoj4T+0XCOGtE+b7cRCLg43tFuTbaAQG8Z+qkPzpza+gggQ1MIIEMTCCBC0wggMVoAMCAQICFCnhUo39pSqH6x3kHUds4YpYaXOrOj8BBDBaUSLaLwIIGjAYSS+oEUludGVybWVkaWF0ZSBDQSAxMB4XDTIzMDUwMTE5MjgzOVoXDTMzMDQyOA0PUasVAEkMMIIBIi4nAgABBQD0QAEPADCCAQoCggEBAKMMyuuA66EOHnGb07P5Zc5wwiEGPDHBBn6lqErhIaN0VJ9XzlDWwyk8Q7CdPlSU7o36DXFs316eATB5bLuXXa+7WwV3cp9V5mZF9OLCz3sOWNYUanYprOMwKA3uvcqqrh8e70Dzw6sX8tfsDeH7aJoJg5kRWEKU+A3Umm+fO+hW8Km3GBqRQXxD49uxAfGtCznXZZjmFbAXqVZu+4R6wMxndfz2dYQxeMVtUY/QGdMWT4fvWzO5et3+X6hq/URUAPOkplv9O2U4T4JPucS9yZpW/FTxWC/L7vQI/bfsrSgIZpv4eJgy27FW3Q4xusbjVvUCL/t2KLvEi/Nr2qodOCECAwEAAaOB7TCB6jAdBgNVHQ4EFgQUy15QYHqrL6k7HiSrAkKN7IFgSBMwHwYDVR0jBBgwFoBSyQNQMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQ4YBAMCB4AwFgEeACUBEBAMMAoGCIm0sAMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4ODgvaV1WKDFfY3JsLmRlcjAzEUscAQEEJzAlMCMRWwwwAYYXWkoALhICkTnw/0hlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=" + } + }`, + errors.New("remote error: tls: bad certificate"), + errors.New("expect error"), + }, + { + "Expired Good response in cache for UserA1 -- should be allowed connection (cached item irrelevant)", + ` + port: -1 + http_port: 8222 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Turn on CA OCSP check but allow when CA is unreachable + ocsp_peer: { + verify: true + ca_timeout: 0.5 + allow_when_ca_unreachable: true + } + } + # preserve revoked true + ocsp_cache: { + type: local + preserve_revoked: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + ` + { + "5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=": { + "subject": "CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US", + "cached_at": "2023-06-05T16:33:52Z", + "resp_status": "good", + "resp_expires": "2023-06-05T16:33:55Z", + "resp": "/wYAAFMyc1R3TwBYBQBgpzMn1wzUMIIGUwoBAKCCBkwwggZIBgkrBgEFBQcwAQEEggY5MIIGNTCB5aFZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HYDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA2MDUxNjMzMDBaMHcwdTBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBeAADZmABA1MVqgEToTAPRAATVaMA0GCSqGSIb3DQEBCwUAA4IBAQB9csJxA2VcpQYmC5lzI0dJJnEC1zcaP1oFbBm5VsZAbWh4mIGTqyEjMkucBqANTypnhWFRYcZE5nJE8tN8Aen0EBYkhk1wfEIhincaF3zs6x5RzigxQOqTPAanO55keKH5exYnfBcyCwAQiBbQaTXLJJdjerfqiRjqgsnLW8yl2IC8+kxTCfR4GHTK34mvqVWYYhP/xbaxTLJTp35t1vf1o78ct9R+z8W5sCSO4TsMNnExZlu4Ejeon/KivMR22j7nelTEaDCuaOl03WxKh9yhw2ix8V7lvR74wh5f4fjLZria6Y0+lUjwvvrZyBRwA62W/ihe3778q8KhLECFPQSaoIIENTCCBDEwggQtMIIDFaADAgECAhQp4VKN/aUqh+sd5B1HbOGKWGlzqzo/AQQwWlEk2jECCBowGEkxqBFJbnRlcm1lZGlhdGUgQ0EgMTAeFw0yMzA1MDExOTI4MzlaFw0zMzA0MjgND1GtFQBJDDCCASIuJwIAAQUA9EABDwAwggEKAoIBAQCjDMrrgOuhDh5xm9Oz+WXOcMIhBjwxwQZ+pahK4SGjdFSfV85Q1sMpPEOwnT5UlO6N+g1xbN9engEweWy7l12vu1sFd3KfVeZmRfTiws97DljWFGp2KazjMCgN7r3Kqq4fHu9A88OrF/LX7A3h+2iaCYOZEVhClPgN1JpvnzvoVvCptxgakUF8Q+PbsQHxrQs512WY5hWwF6lWbvuEesDMZ3X89nWEMXjFbVGP0BnTFk+H71szuXrd/l+oav1EVADzpKZb/TtlOE+CT7nEvcmaVvxU8Vgvy+70CP237K0oCGab+HiYMtuxVt0OMbrG41b1Ai/7dii7xIvza9qqHTghAgMBAAGjge0wgeowHQYDVR0OBBYEFMteUGB6qy+pOx4kqwJCjeyBYEgTMB8GA1UdIwQYMBaAUssDUDAMBgNVHRMBAf8EAjAAMA4GA1UdDwEOGAQDAgeAMBYBHgAlARAQDDAKBgiJtrADCTA9BgNVHR8ENjA0MDKgMKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ldVigxX2NybC5kZXIwMxFLHAEBBCcwJTAjEVsMMAGGF1pKAC4SAgALBQD0AQEBAEhlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=" + } + }`, + nil, + nil, + }, + { + "Expired Unknown response in cache for UserA1 -- should be allowed connection (cached item irrelevant)", + ` + port: -1 + http_port: 8222 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Turn on CA OCSP check but allow when CA is unreachable + ocsp_peer: { + verify: true + ca_timeout: 0.5 + allow_when_ca_unreachable: true + } + } + # preserve revoked true + ocsp_cache: { + type: local + preserve_revoked: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + ` + { + "5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=": { + "subject": "CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US", + "cached_at": "2023-06-05T16:45:01Z", + "resp_status": "unknown", + "resp_expires": "2023-06-05T16:45:05Z", + "resp": "/wYAAFMyc1R3TwBSBQBH1aW01wzUMIIGUwoBAKCCBkwwggZIBgkrBgEFBQcwAQEEggY5MIIGNTCB5aFZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HYDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA2MDUxNjQ1MDBaMHcwdTBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBeCADpmAAwxWqAROhMA9EABNVowDQYJKoZIhvcNAQELBQADggEBAAzwED88ngiFj3hLzIejZ8DE/ppticX0vgAjUM8oXjDjwNXpSQ5xdXHsDk4RAYAVpNyiAfL2fapwz91g3JuZot6npp8smtZC5D0YMKK8iMjrx2aMqVyv+ai/33WG8PRWpBNzSTYaLhlBFjhUrx8HDu97ozNbmfgDWzRS1LqkJRa5YXvkyppqYTFSX73DV9R9tOVOwZ0x5WEKst9IJ+88mXsOBGyuye2Gh9RK6KsLgaOwiD9FBf18WRKIixeVM1Y/xHc/iwFPi8k3Z6hZ6gHX8NboQ/djCyzYVSWsUedTo/62uuagHPRWoYci4HQl4bSFXfcEO/EkWGnqkWBHfYZ4soigggQ1MIIEMTCCBC0wggMVoAMCAQICFCnhUo39pSqH6x3kHUds4YpYaXOrOj8BBDBaUSTaMQIIGjAYSTGoEUludGVybWVkaWF0ZSBDQSAxMB4XDTIzMDUwMTE5MjgzOVoXDTMzMDQyOA0PUa0VAEkMMIIBIi4nAgABBQD0QAEPADCCAQoCggEBAKMMyuuA66EOHnGb07P5Zc5wwiEGPDHBBn6lqErhIaN0VJ9XzlDWwyk8Q7CdPlSU7o36DXFs316eATB5bLuXXa+7WwV3cp9V5mZF9OLCz3sOWNYUanYprOMwKA3uvcqqrh8e70Dzw6sX8tfsDeH7aJoJg5kRWEKU+A3Umm+fO+hW8Km3GBqRQXxD49uxAfGtCznXZZjmFbAXqVZu+4R6wMxndfz2dYQxeMVtUY/QGdMWT4fvWzO5et3+X6hq/URUAPOkplv9O2U4T4JPucS9yZpW/FTxWC/L7vQI/bfsrSgIZpv4eJgy27FW3Q4xusbjVvUCL/t2KLvEi/Nr2qodOCECAwEAAaOB7TCB6jAdBgNVHQ4EFgQUy15QYHqrL6k7HiSrAkKN7IFgSBMwHwYDVR0jBBgwFoBSywNQMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQ4YBAMCB4AwFgEeACUBEBAMMAoGCIm2sAMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4ODgvaV1WKDFfY3JsLmRlcjAzEUscAQEEJzAlMCMRWwwwAYYXWkoALhICkTnw/0hlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=" + } + }`, + nil, + nil, + }, + { + "No response in cache for UserA1 -- should be allowed connection", + ` + port: -1 + http_port: 8222 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Turn on CA OCSP check but allow when CA is unreachable + ocsp_peer: { + verify: true + ca_timeout: 0.5 + allow_when_ca_unreachable: true + } + } + # preserve revoked true + ocsp_cache: { + type: local + preserve_revoked: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + "", + nil, + nil, + }, + } { + t.Run(test.name, func(t *testing.T) { + deleteLocalStore(t, "") + c := []byte(test.cachedResponse) + err := writeCacheFile("", c) + if err != nil { + t.Fatal(err) + } + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + }) + } +} + +// TestOCSPResponseCacheLocalStoreOpt is test of default and non-default local_store option +func TestOCSPResponseCacheLocalStoreOpt(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + + for _, test := range []struct { + name string + config string + opts []nats.Option + cachedResponse string + err error + rerr error + storeLocation string + }{ + { + "Test load from non-default local store _custom_; connect will reject only if cache file found and loaded", + ` + port: -1 + http_port: 8222 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Turn on CA OCSP check but allow when CA is unreachable + ocsp_peer: { + verify: true + ca_timeout: 0.5 + allow_when_ca_unreachable: true + } + } + # preserve revoked true + ocsp_cache: { + type: local + local_store: "_custom_" + preserve_revoked: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + ` + { + "5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=": { + "subject": "CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US", + "cached_at": "2023-05-29T17:56:45Z", + "resp_status": "revoked", + "resp_expires": "2023-05-29T17:56:49Z", + "resp": "/wYAAFMyc1R3TwBSBQBao1Qr1QzUMIIGUQoBAKCCBkowggZGBgkrBgEFBQcwAQEEggY3MIIGMzCB46FZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HQDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA1MjkxNzU2MDBaMHUwczBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBc2ZAAQNDVaoBE2dwD0QQE0OVowDQYJKoZIhvcNAQELBQADggEBAGAax/vkv3SBFNbxp2utc/N6Rje4E0ceC972sWgqYjzYrH0oc/acg+OAXaxUjwqoQWaT+dHaI4D5qoTkMx7XlWATjI2L72IUTf6Luo92jPzyDFwb10CdeFHtRtEYD54Qbi/nD4oxQ8cSoLKC3wft2l3E/mK/1I4Mxwq15CioK4MhfzTISoeGZbjDXPKgloJOG3rn9v64vFGV6dosbLgaXEs+MPcCsPQYkwhOOyazuewRmIDOBp5QSsKPhqsT8Rs20t8LGTMkvjZniFWJs90l9QL9F1m3obq5nyuxrGt+7Rf5zoj4T+0XCOGtE+b7cRCLg43tFuTbaAQG8Z+qkPzpza+gggQ1MIIEMTCCBC0wggMVoAMCAQICFCnhUo39pSqH6x3kHUds4YpYaXOrOj8BBDBaUSLaLwIIGjAYSS+oEUludGVybWVkaWF0ZSBDQSAxMB4XDTIzMDUwMTE5MjgzOVoXDTMzMDQyOA0PUasVAEkMMIIBIi4nAgABBQD0QAEPADCCAQoCggEBAKMMyuuA66EOHnGb07P5Zc5wwiEGPDHBBn6lqErhIaN0VJ9XzlDWwyk8Q7CdPlSU7o36DXFs316eATB5bLuXXa+7WwV3cp9V5mZF9OLCz3sOWNYUanYprOMwKA3uvcqqrh8e70Dzw6sX8tfsDeH7aJoJg5kRWEKU+A3Umm+fO+hW8Km3GBqRQXxD49uxAfGtCznXZZjmFbAXqVZu+4R6wMxndfz2dYQxeMVtUY/QGdMWT4fvWzO5et3+X6hq/URUAPOkplv9O2U4T4JPucS9yZpW/FTxWC/L7vQI/bfsrSgIZpv4eJgy27FW3Q4xusbjVvUCL/t2KLvEi/Nr2qodOCECAwEAAaOB7TCB6jAdBgNVHQ4EFgQUy15QYHqrL6k7HiSrAkKN7IFgSBMwHwYDVR0jBBgwFoBSyQNQMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQ4YBAMCB4AwFgEeACUBEBAMMAoGCIm0sAMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4ODgvaV1WKDFfY3JsLmRlcjAzEUscAQEEJzAlMCMRWwwwAYYXWkoALhICkTnw/0hlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=" + } + }`, + errors.New("remote error: tls: bad certificate"), + errors.New("expect error"), + "_custom_", + }, + { + "Test load from default local store when \"\" set; connect will reject only if cache file found and loaded", + ` + port: -1 + http_port: 8222 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Turn on CA OCSP check but allow when CA is unreachable + ocsp_peer: { + verify: true + ca_timeout: 0.5 + allow_when_ca_unreachable: true + } + } + # preserve revoked true + ocsp_cache: { + type: local + local_store: "" + preserve_revoked: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + ` + { + "5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=": { + "subject": "CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US", + "cached_at": "2023-05-29T17:56:45Z", + "resp_status": "revoked", + "resp_expires": "2023-05-29T17:56:49Z", + "resp": "/wYAAFMyc1R3TwBSBQBao1Qr1QzUMIIGUQoBAKCCBkowggZGBgkrBgEFBQcwAQEEggY3MIIGMzCB46FZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HQDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA1MjkxNzU2MDBaMHUwczBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBc2ZAAQNDVaoBE2dwD0QQE0OVowDQYJKoZIhvcNAQELBQADggEBAGAax/vkv3SBFNbxp2utc/N6Rje4E0ceC972sWgqYjzYrH0oc/acg+OAXaxUjwqoQWaT+dHaI4D5qoTkMx7XlWATjI2L72IUTf6Luo92jPzyDFwb10CdeFHtRtEYD54Qbi/nD4oxQ8cSoLKC3wft2l3E/mK/1I4Mxwq15CioK4MhfzTISoeGZbjDXPKgloJOG3rn9v64vFGV6dosbLgaXEs+MPcCsPQYkwhOOyazuewRmIDOBp5QSsKPhqsT8Rs20t8LGTMkvjZniFWJs90l9QL9F1m3obq5nyuxrGt+7Rf5zoj4T+0XCOGtE+b7cRCLg43tFuTbaAQG8Z+qkPzpza+gggQ1MIIEMTCCBC0wggMVoAMCAQICFCnhUo39pSqH6x3kHUds4YpYaXOrOj8BBDBaUSLaLwIIGjAYSS+oEUludGVybWVkaWF0ZSBDQSAxMB4XDTIzMDUwMTE5MjgzOVoXDTMzMDQyOA0PUasVAEkMMIIBIi4nAgABBQD0QAEPADCCAQoCggEBAKMMyuuA66EOHnGb07P5Zc5wwiEGPDHBBn6lqErhIaN0VJ9XzlDWwyk8Q7CdPlSU7o36DXFs316eATB5bLuXXa+7WwV3cp9V5mZF9OLCz3sOWNYUanYprOMwKA3uvcqqrh8e70Dzw6sX8tfsDeH7aJoJg5kRWEKU+A3Umm+fO+hW8Km3GBqRQXxD49uxAfGtCznXZZjmFbAXqVZu+4R6wMxndfz2dYQxeMVtUY/QGdMWT4fvWzO5et3+X6hq/URUAPOkplv9O2U4T4JPucS9yZpW/FTxWC/L7vQI/bfsrSgIZpv4eJgy27FW3Q4xusbjVvUCL/t2KLvEi/Nr2qodOCECAwEAAaOB7TCB6jAdBgNVHQ4EFgQUy15QYHqrL6k7HiSrAkKN7IFgSBMwHwYDVR0jBBgwFoBSyQNQMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQ4YBAMCB4AwFgEeACUBEBAMMAoGCIm0sAMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4ODgvaV1WKDFfY3JsLmRlcjAzEUscAQEEJzAlMCMRWwwwAYYXWkoALhICkTnw/0hlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=" + } + }`, + errors.New("remote error: tls: bad certificate"), + errors.New("expect error"), + "_rc_", + }, + } { + t.Run(test.name, func(t *testing.T) { + deleteLocalStore(t, test.storeLocation) + c := []byte(test.cachedResponse) + err := writeCacheFile(test.storeLocation, c) + if err != nil { + t.Fatal(err) + } + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + }) + } +} + +// TestOCSPPeerIncrementalSaveLocalCache is test of timer-based response cache save as new entries added +func TestOCSPPeerIncrementalSaveLocalCache(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem", ocsp.Good) + + intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) + + intermediateCA2Responder := NewOCSPResponderIntermediateCA2(t) + intermediateCA2ResponderURL := fmt.Sprintf("http://%s", intermediateCA2Responder.Addr) + defer intermediateCA2Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA2ResponderURL, "configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem", ocsp.Good) + + var fi os.FileInfo + var err error + + for _, test := range []struct { + name string + config string + opts [][]nats.Option + err error + rerr error + configure func() + }{ + { + "Default cache, short form: mTLS OCSP peer check on inbound client connection, UserA1 client of intermediate CA 1", + ` + port: -1 + http_port: 8222 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + ca_timeout: 5 + allowed_clockskew: 30 + } + } + # Local cache with custom save_interval for testability + ocsp_cache: { + type: local + # Save if dirty ever 1 second + save_interval: 1 + } + `, + [][]nats.Option{ + { + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + { + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client2/UserB1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client2/private/UserB1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + }, + nil, + nil, + func() {}, + }, + } { + t.Run(test.name, func(t *testing.T) { + // Cleanup any previous test that saved a local cache + deleteLocalStore(t, "") + fi, err = statCacheFile("") + if err != nil && fi != nil && fi.Size() != 0 { + t.Fatalf("Expected no local cache file, got a FileInfo with size %d", fi.Size()) + } + test.configure() + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + // Connect with UserA1 client and get a CA Response + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts[0]...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + nc.Close() + time.Sleep(2 * time.Second) + fi, err = statCacheFile("") + if err == nil && fi != nil && fi.Size() > 0 { + // good + } else { + if err != nil { + t.Fatalf("Expected an extant local cache file, got error: %v", err) + } + if fi != nil { + t.Fatalf("Expected non-zero size local cache file, got a FileInfo with size %d", fi.Size()) + } + } + firstFi := fi + // Connect with UserB1 client and get another CA Response + nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts[1]...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + nc.Close() + time.Sleep(2 * time.Second) + fi, err = statCacheFile("") + if err == nil && fi != nil && fi.Size() > firstFi.Size() { + // good + } else { + if err != nil { + t.Fatalf("Expected an extant local cache file, got error: %v", err) + } + if fi != nil { + t.Fatalf("Expected non-zero size local cache file with more bytes, got a FileInfo with size %d", fi.Size()) + } + } + }) + } +} + +func statCacheFile(dir string) (os.FileInfo, error) { + if dir == "" { + dir = "_rc_" + } + return os.Stat(filepath.Join(dir, "cache.json")) +} + +// TestOCSPPeerUndelegatedCAResponseSigner +func TestOCSPPeerUndelegatedCAResponseSigner(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + + intermediateCA1Responder := NewOCSPResponderIntermediateCA1Undelegated(t) + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) + + for _, test := range []struct { + name string + config string + opts []nats.Option + err error + rerr error + configure func() + }{ + { + "mTLS OCSP peer check on inbound client connection, responder is CA (undelegated)", + ` + port: -1 + # Cache configuration is default + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Turn on CA OCSP check so unvalidated clients can't connect + ocsp_peer: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() {}, + }, + } { + t.Run(test.name, func(t *testing.T) { + deleteLocalStore(t, "") + test.configure() + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + }) + } +} + +// TestOCSPPeerDelegatedCAResponseSigner +func TestOCSPPeerDelegatedCAResponseSigner(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + + intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) + + for _, test := range []struct { + name string + config string + opts []nats.Option + err error + rerr error + configure func() + }{ + { + "mTLS OCSP peer check on inbound client connection, responder is CA (undelegated)", + ` + port: -1 + # Cache configuration is default + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Turn on CA OCSP check so unvalidated clients can't connect + ocsp_peer: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() {}, + }, + } { + t.Run(test.name, func(t *testing.T) { + deleteLocalStore(t, "") + test.configure() + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + }) + } +} + +// TestOCSPPeerBadDelegatedCAResponseSigner +func TestOCSPPeerBadDelegatedCAResponseSigner(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + + intermediateCA1Responder := NewOCSPResponderBadDelegateIntermediateCA1(t) + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) + + for _, test := range []struct { + name string + config string + opts []nats.Option + err error + rerr error + configure func() + }{ + { + "mTLS OCSP peer check on inbound client connection, responder is not a legal delegate", + ` + port: -1 + # Cache configuration is default + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Turn on CA OCSP check so unvalidated clients can't connect + ocsp_peer: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + errors.New("remote error: tls: bad certificate"), + errors.New("expect error"), + func() {}, + }, + } { + t.Run(test.name, func(t *testing.T) { + deleteLocalStore(t, "") + test.configure() + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + }) + } +} + +// TestOCSPPeerNextUpdateUnset is test of scenario when responder does not set NextUpdate and cache TTL option is used +func TestOCSPPeerNextUpdateUnset(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + + respCertPEM := "configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_bundle.pem" + respKeyPEM := "configs/certs/ocsp_peer/mini-ca/ocsp1/private/ocsp1_keypair.pem" + issuerCertPEM := "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem" + intermediateCA1Responder := NewOCSPResponderBase(t, issuerCertPEM, respCertPEM, respKeyPEM, true, "127.0.0.1:18888", 0, "") + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) + + for _, test := range []struct { + name string + config string + opts []nats.Option + err error + rerr error + expectedMisses int64 + configure func() + }{ + { + "TTL set to 4 seconds with second client connection leveraging cache from first client connect", + ` + port: -1 + http_port: 8222 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + ca_timeout: 5 + allowed_clockskew: 0 + cache_ttl_when_next_update_unset: 4 + } + } + # Short form configuration, local as default + ocsp_cache: true + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + 2, + func() {}, + }, + { + "TTL set to 1 seconds with second client connection not leveraging cache items from first client connect", + ` + port: -1 + http_port: 8222 + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + ca_timeout: 5 + allowed_clockskew: 0 + cache_ttl_when_next_update_unset: 1 + } + } + # Short form configuration, local as default + ocsp_cache: true + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + 3, + func() {}, + }, + } { + t.Run(test.name, func(t *testing.T) { + // Cleanup any previous test that saved a local cache + deleteLocalStore(t, "") + test.configure() + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + nc.Close() + + // Wait interval shorter than first test, and longer than second test + time.Sleep(2 * time.Second) + + nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + + v := monitorGetVarzHelper(t, 8222) + if v.OCSPResponseCache == nil { + t.Fatalf("Expected OCSP statistics to be in varz") + } + if v.OCSPResponseCache.Misses != test.expectedMisses || v.OCSPResponseCache.Responses != 2 { + t.Errorf("Expected cache misses to be %d and cache items to be 2, got %d and %d", test.expectedMisses, v.OCSPResponseCache.Misses, v.OCSPResponseCache.Responses) + } + }) + } +} + +func TestOCSPMonitoringPort(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rootCAResponder := NewOCSPResponderRootCA(t) + rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) + defer rootCAResponder.Shutdown(ctx) + SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) + + respCertPEM := "configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_bundle.pem" + respKeyPEM := "configs/certs/ocsp_peer/mini-ca/ocsp1/private/ocsp1_keypair.pem" + issuerCertPEM := "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem" + intermediateCA1Responder := NewOCSPResponderBase(t, issuerCertPEM, respCertPEM, respKeyPEM, true, "127.0.0.1:18888", 0, "") + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem", ocsp.Good) + + for _, test := range []struct { + name string + config string + opts []nats.Option + err error + rerr error + }{ + { + "https with ocsp_peer", + ` + net: 127.0.0.1 + port: -1 + https: -1 + # Short form configuration + ocsp_cache: true + store_dir = %s + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + # Long form configuration + ocsp_peer: { + verify: true + ca_timeout: 5 + allowed_clockskew: 30 + } + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + }, + { + "https with just ocsp", + ` + net: 127.0.0.1 + port: -1 + https: -1 + ocsp { + mode = always + url = http://127.0.0.1:18888 + } + store_dir = %s + + tls: { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), + nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + }, + } { + t.Run(test.name, func(t *testing.T) { + deleteLocalStore(t, "") + content := test.config + conf := createConfFile(t, []byte(fmt.Sprintf(content, t.TempDir()))) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + nc.Subscribe("ping", func(m *nats.Msg) { + m.Respond([]byte("pong")) + }) + nc.Flush() + _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) + if test.rerr != nil && err == nil { + t.Errorf("Expected error getting response") + } else if test.rerr == nil && err != nil { + t.Errorf("Expected response") + } + + // Make request to the HTTPS port using the client cert. + tlsConfig := &tls.Config{} + clientCertFile := "./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem" + clientKeyFile := "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem" + cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile) + if err != nil { + t.Fatal(err) + } + tlsConfig.Certificates = []tls.Certificate{cert} + caCertFile := "./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + caCert, err := os.ReadFile(caCertFile) + if err != nil { + t.Fatal(err) + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + tlsConfig.RootCAs = caCertPool + + hc := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + }, + } + resp, err := hc.Get("https://" + s.MonitorAddr().String()) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Errorf("Unexpected status: %v", resp.Status) + } + }) + } +} diff --git a/test/ocsp_test.go b/test/ocsp_test.go new file mode 100644 index 0000000..5384337 --- /dev/null +++ b/test/ocsp_test.go @@ -0,0 +1,4131 @@ +// Copyright 2021-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "bytes" + "context" + "crypto/tls" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + . "github.com/nats-io/nats-server/v2/internal/ocsp" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "golang.org/x/crypto/ocsp" +) + +func TestOCSPAlwaysMustStapleAndShutdown(t *testing.T) { + // Certs that have must staple will auto shutdown the server. + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + serverCert = "configs/certs/ocsp/server-cert.pem" + serverKey = "configs/certs/ocsp/server-key.pem" + ) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, serverCert, ocsp.Good) + + opts := server.Options{} + opts.Host = "127.0.0.1" + opts.NoLog = true + opts.NoSigs = true + opts.MaxControlLine = 4096 + opts.Port = -1 + opts.TLSCert = serverCert + opts.TLSKey = serverKey + opts.TLSCaCert = caCert + opts.TLSTimeout = 5 + tcOpts := &server.TLSConfigOpts{ + CertFile: opts.TLSCert, + KeyFile: opts.TLSKey, + CaFile: opts.TLSCaCert, + Timeout: opts.TLSTimeout, + } + + tlsConf, err := server.GenTLSConfig(tcOpts) + if err != nil { + t.Fatal(err) + } + opts.TLSConfig = tlsConf + + opts.OCSPConfig = &server.OCSPConfig{ + Mode: server.OCSPModeAlways, + OverrideURLs: []string{addr}, + } + srv := RunServer(&opts) + defer srv.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + resp, err := GetOCSPStatus(s) + if err != nil { + return err + } + if resp.Status != ocsp.Good { + return fmt.Errorf("invalid staple") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + sub, err := nc.SubscribeSync("foo") + if err != nil { + t.Fatal(err) + } + nc.Publish("foo", []byte("hello world")) + nc.Flush() + + _, err = sub.NextMsg(1 * time.Second) + if err != nil { + t.Fatal(err) + } + nc.Close() + + // The server will shutdown because the server becomes revoked + // and the policy is to always must-staple. The OCSP Responder + // instructs the NATS Server to fetch OCSP Staples every 2 seconds. + time.Sleep(2 * time.Second) + SetOCSPStatus(t, addr, serverCert, ocsp.Revoked) + time.Sleep(2 * time.Second) + + // Should be connection refused since server will abort now. + _, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nats.ErrNoServers { + t.Errorf("Expected connection refused") + } + // Verify that the server finishes shutdown + srv.WaitForShutdown() +} + +func TestOCSPMustStapleShutdown(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + serverCert = "configs/certs/ocsp/server-status-request-cert.pem" + serverKey = "configs/certs/ocsp/server-status-request-key.pem" + ) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, serverCert, ocsp.Good) + + opts := server.Options{} + opts.Host = "127.0.0.1" + opts.NoLog = true + opts.NoSigs = true + opts.MaxControlLine = 4096 + opts.Port = -1 + opts.TLSCert = serverCert + opts.TLSKey = serverKey + opts.TLSCaCert = caCert + opts.TLSTimeout = 5 + tlsConfigOpts := &server.TLSConfigOpts{ + CertFile: opts.TLSCert, + KeyFile: opts.TLSKey, + CaFile: opts.TLSCaCert, + Timeout: opts.TLSTimeout, + } + + tlsConf, err := server.GenTLSConfig(tlsConfigOpts) + if err != nil { + t.Fatal(err) + } + opts.TLSConfig = tlsConf + + opts.OCSPConfig = &server.OCSPConfig{ + Mode: server.OCSPModeMust, + OverrideURLs: []string{addr}, + } + + srv := RunServer(&opts) + defer srv.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + resp, err := GetOCSPStatus(s) + if err != nil { + return err + } + if resp.Status != ocsp.Good { + return fmt.Errorf("invalid staple") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + sub, err := nc.SubscribeSync("foo") + if err != nil { + t.Fatal(err) + } + nc.Publish("foo", []byte("hello world")) + nc.Flush() + + _, err = sub.NextMsg(1 * time.Second) + if err != nil { + t.Fatal(err) + } + nc.Close() + + // The server will shutdown because the server becomes revoked + // and the policy is to always must-staple. The OCSP Responder + // instructs the NATS Server to fetch OCSP Staples every 2 seconds. + time.Sleep(2 * time.Second) + SetOCSPStatus(t, addr, serverCert, ocsp.Revoked) + time.Sleep(2 * time.Second) + + // Should be connection refused since server will abort now. + _, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nats.ErrNoServers { + t.Errorf("Expected connection refused") + } +} + +func TestOCSPMustStapleAutoDoesNotShutdown(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + serverCert = "configs/certs/ocsp/server-status-request-url-01-cert.pem" + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, serverCert, ocsp.Good) + + content := ` + port: -1 + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + ` + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + resp, err := GetOCSPStatus(s) + if err != nil { + return err + } + if resp.Status != ocsp.Good { + t.Errorf("Expected valid OCSP staple status") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + sub, err := nc.SubscribeSync("foo") + if err != nil { + t.Fatal(err) + } + nc.Publish("foo", []byte("hello world")) + nc.Flush() + + _, err = sub.NextMsg(1 * time.Second) + if err != nil { + t.Fatal(err) + } + nc.Close() + + // The server will shutdown because the server becomes revoked + // and the policy is to always must-staple. The OCSP Responder + // instructs the NATS Server to fetch OCSP Staples every 2 seconds. + time.Sleep(2 * time.Second) + SetOCSPStatus(t, addr, serverCert, ocsp.Revoked) + time.Sleep(2 * time.Second) + + // Should not be connection refused, the client will continue running and + // be served the stale OCSP staple instead. + nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + resp, err := GetOCSPStatus(s) + if err != nil { + return err + } + if resp.Status != ocsp.Revoked { + t.Errorf("Expected revoked status") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + nc.Close() +} + +func TestOCSPAutoWithoutMustStapleDoesNotShutdownOnRevoke(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + serverCert = "configs/certs/ocsp/server-cert.pem" + serverKey = "configs/certs/ocsp/server-key.pem" + ) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, serverCert, ocsp.Good) + + opts := server.Options{} + opts.Host = "127.0.0.1" + opts.NoLog = true + opts.NoSigs = true + opts.MaxControlLine = 4096 + opts.Port = -1 + opts.TLSCert = serverCert + opts.TLSKey = serverKey + opts.TLSCaCert = caCert + opts.TLSTimeout = 5 + tlsConfigOpts := &server.TLSConfigOpts{ + CertFile: opts.TLSCert, + KeyFile: opts.TLSKey, + CaFile: opts.TLSCaCert, + Timeout: opts.TLSTimeout, + } + tlsConf, err := server.GenTLSConfig(tlsConfigOpts) + if err != nil { + t.Fatal(err) + } + opts.TLSConfig = tlsConf + + opts.OCSPConfig = &server.OCSPConfig{ + Mode: server.OCSPModeAuto, + OverrideURLs: []string{addr}, + } + + srv := RunServer(&opts) + defer srv.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse != nil { + return fmt.Errorf("Unexpected OCSP staple for certificate") + } + return nil + }, + }), + + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + sub, err := nc.SubscribeSync("foo") + if err != nil { + t.Fatal(err) + } + nc.Publish("foo", []byte("hello world")) + nc.Flush() + + _, err = sub.NextMsg(1 * time.Second) + if err != nil { + t.Fatal(err) + } + nc.Close() + + // Revoke the client certificate, nothing will happens since does + // not have MustStaple. + time.Sleep(2 * time.Second) + SetOCSPStatus(t, addr, serverCert, ocsp.Revoked) + time.Sleep(2 * time.Second) + + // Should not be connection refused since server will continue running. + nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Errorf("Unexpected error: %s", err) + } + nc.Close() +} + +func TestOCSPClient(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + serverCert = "configs/certs/ocsp/server-cert.pem" + serverKey = "configs/certs/ocsp/server-key.pem" + ) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + ocspURL := fmt.Sprintf("http://%s", ocspr.Addr) + defer ocspr.Shutdown(ctx) + + for _, test := range []struct { + name string + config string + opts []nats.Option + err error + rerr error + configure func() + }{ + { + "OCSP Stapling makes server fail to boot if status is unknown", + ` + port: -1 + + # Enable OCSP stapling with policy to honor must staple if present. + ocsp: true + + tls { + cert_file: "configs/certs/ocsp/server-cert.pem" + key_file: "configs/certs/ocsp/server-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() {}, + }, + { + "OCSP Stapling ignored by default if server without must staple status", + ` + port: -1 + + ocsp: true + + tls { + cert_file: "configs/certs/ocsp/server-cert.pem" + key_file: "configs/certs/ocsp/server-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() { SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) }, + }, + { + "OCSP Stapling honored by default if server has must staple status", + ` + port: -1 + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() { + SetOCSPStatus(t, ocspURL, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) + }, + }, + { + "OCSP Stapling can be disabled even if server has must staple status", + ` + port: -1 + + ocsp: false + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + `, + []nats.Option{ + nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() { + SetOCSPStatus(t, ocspURL, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Revoked) + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + test.configure() + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + + nc.Subscribe("ping", func(m *nats.Msg) { + m.Respond([]byte("pong")) + }) + nc.Flush() + + _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) + if test.rerr != nil && err == nil { + t.Errorf("Expected error getting response") + } else if test.rerr == nil && err != nil { + t.Errorf("Expected response") + } + }) + } +} + +func TestOCSPReloadRotateTLSCertWithNoURL(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + serverCert = "configs/certs/ocsp/server-status-request-url-01-cert.pem" + updatedServerCert = "configs/certs/ocsp/server-status-request-cert.pem" + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, serverCert, ocsp.Good) + + content := ` + port: -1 + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + ` + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + resp, err := GetOCSPStatus(s) + if err != nil { + return err + } + if resp.Status != ocsp.Good { + t.Errorf("Expected valid OCSP staple status") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + sub, err := nc.SubscribeSync("foo") + if err != nil { + t.Fatal(err) + } + nc.Publish("foo", []byte("hello world")) + nc.Flush() + + _, err = sub.NextMsg(1 * time.Second) + if err != nil { + t.Fatal(err) + } + nc.Close() + + // Change the contents with another that will fail to get a staple + // since it does not have an URL. + content = ` + port: -1 + + tls { + cert_file: "configs/certs/ocsp/server-status-request-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + ` + if err := os.WriteFile(conf, []byte(content), 0666); err != nil { + t.Fatalf("Error writing config: %v", err) + } + // Reload show warning because of cert missing OCSP Url so cannot be used + // with OCSP stapling. + if err := s.Reload(); err != nil { + t.Fatal(err) + } + expectedErr := fmt.Errorf("missing OCSP response") + // The server will not shutdown because the reload will fail. + _, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + // The new certificate does not have OCSP Staples since + // it could not fetch one from a OCSP server. + if s.OCSPResponse == nil { + return expectedErr + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err == nil || !strings.Contains(err.Error(), expectedErr.Error()) { + t.Fatalf("Unexpected error: %s, got %s", expectedErr, err) + } +} + +func TestOCSPReloadRotateTLSCertDisableMustStaple(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + serverCert = "configs/certs/ocsp/server-status-request-url-01-cert.pem" + updatedServerCert = "configs/certs/ocsp/server-status-request-cert.pem" + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, serverCert, ocsp.Good) + + storeDir := t.TempDir() + + originalContent := ` + port: -1 + + store_dir: '%s' + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + ` + + content := fmt.Sprintf(originalContent, storeDir) + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + var staple []byte + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + staple = s.OCSPResponse + resp, err := GetOCSPStatus(s) + if err != nil { + return err + } + if resp.Status != ocsp.Good { + t.Errorf("Expected valid OCSP staple status") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + sub, err := nc.SubscribeSync("foo") + if err != nil { + t.Fatal(err) + } + nc.Publish("foo", []byte("hello world")) + nc.Flush() + + _, err = sub.NextMsg(1 * time.Second) + if err != nil { + t.Fatal(err) + } + nc.Close() + + files := []string{} + err = filepath.Walk(storeDir+"/ocsp/", func(path string, info os.FileInfo, err error) error { + if info.IsDir() { + return nil + } + files = append(files, path) + return nil + }) + if err != nil { + t.Fatal(err) + } + found := false + for _, file := range files { + data, err := os.ReadFile(file) + if err != nil { + t.Error(err) + } + if bytes.Equal(staple, data) { + found = true + } + } + if !found { + t.Error("Could not find OCSP Staple") + } + + // Change the contents with another that has OCSP Stapling disabled. + updatedContent := ` + port: -1 + + store_dir: '%s' + + tls { + cert_file: "configs/certs/ocsp/server-cert.pem" + key_file: "configs/certs/ocsp/server-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + ` + content = fmt.Sprintf(updatedContent, storeDir) + if err := os.WriteFile(conf, []byte(content), 0666); err != nil { + t.Fatalf("Error writing config: %v", err) + } + if err := s.Reload(); err != nil { + t.Fatal(err) + } + + // The new certificate does not have must staple so they will be missing. + time.Sleep(4 * time.Second) + + nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse != nil { + return fmt.Errorf("unexpected OCSP Staple!") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + nc.Close() + + // Re-enable OCSP Stapling + content = fmt.Sprintf(originalContent, storeDir) + if err := os.WriteFile(conf, []byte(content), 0666); err != nil { + t.Fatalf("Error writing config: %v", err) + } + if err := s.Reload(); err != nil { + t.Fatal(err) + } + + var newStaple []byte + nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + newStaple = s.OCSPResponse + resp, err := GetOCSPStatus(s) + if err != nil { + return err + } + if resp.Status != ocsp.Good { + t.Errorf("Expected valid OCSP staple status") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + nc.Close() + + // Confirm that it got a new staple. + files = []string{} + err = filepath.Walk(storeDir+"/ocsp/", func(path string, info os.FileInfo, err error) error { + if info.IsDir() { + return nil + } + files = append(files, path) + return nil + }) + if err != nil { + t.Fatal(err) + } + found = false + for _, file := range files { + data, err := os.ReadFile(file) + if err != nil { + t.Error(err) + } + if bytes.Equal(newStaple, data) { + found = true + } + } + if !found { + t.Error("Could not find OCSP Staple") + } + if bytes.Equal(staple, newStaple) { + t.Error("Expected new OCSP Staple") + } +} + +func TestOCSPReloadRotateTLSCertEnableMustStaple(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + serverCert = "configs/certs/ocsp/server-cert.pem" + updatedServerCert = "configs/certs/ocsp/server-status-request-url-01-cert.pem" + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, serverCert, ocsp.Good) + SetOCSPStatus(t, addr, updatedServerCert, ocsp.Good) + + // Start without OCSP Stapling MustStaple + content := ` + port: -1 + + tls { + cert_file: "configs/certs/ocsp/server-cert.pem" + key_file: "configs/certs/ocsp/server-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + ` + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse != nil { + return fmt.Errorf("unexpected OCSP Staple!") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + sub, err := nc.SubscribeSync("foo") + if err != nil { + t.Fatal(err) + } + nc.Publish("foo", []byte("hello world")) + nc.Flush() + + _, err = sub.NextMsg(1 * time.Second) + if err != nil { + t.Fatal(err) + } + nc.Close() + + // Change the contents with another that has OCSP Stapling enabled. + content = ` + port: -1 + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + ` + if err := os.WriteFile(conf, []byte(content), 0666); err != nil { + t.Fatalf("Error writing config: %v", err) + } + if err := s.Reload(); err != nil { + t.Fatal(err) + } + + // The new certificate does not have must staple so they will be missing. + time.Sleep(2 * time.Second) + + nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + resp, err := GetOCSPStatus(s) + if err != nil { + return err + } + if resp.Status != ocsp.Good { + t.Errorf("Expected valid OCSP staple status") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + nc.Close() +} + +func TestOCSPCluster(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-03-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-04-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-05-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-06-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-07-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-08-cert.pem", ocsp.Good) + + // Store Dirs + storeDirA := t.TempDir() + storeDirB := t.TempDir() + storeDirC := t.TempDir() + + // Seed server configuration + srvConfA := ` + host: "127.0.0.1" + port: -1 + + server_name: "AAA" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + cluster { + name: AB + host: "127.0.0.1" + advertise: 127.0.0.1 + port: -1 + pool_size: -1 + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } + ` + srvConfA = fmt.Sprintf(srvConfA, storeDirA) + sconfA := createConfFile(t, []byte(srvConfA)) + srvA, optsA := RunServerWithConfig(sconfA) + defer srvA.Shutdown() + + // The rest + srvConfB := ` + host: "127.0.0.1" + port: -1 + + server_name: "BBB" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-03-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-03-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + cluster { + name: AB + host: "127.0.0.1" + advertise: 127.0.0.1 + port: -1 + pool_size: -1 + + routes: [ nats://127.0.0.1:%d ] + connect_retries: 30 + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-04-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-04-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } + ` + srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.Cluster.Port) + conf := createConfFile(t, []byte(srvConfB)) + srvB, optsB := RunServerWithConfig(conf) + defer srvB.Shutdown() + + // Client connects to server A. + cA, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", optsA.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cA.Close() + checkClusterFormed(t, srvA, srvB) + + // Revoke the seed server cluster certificate, following servers will not be able to verify connection. + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Revoked) + + // Original set of servers still can communicate to each other, even though the cert has been revoked. + // NOTE: Should we unplug from the cluster in case our server is revoke and OCSP policy is always or must? + checkClusterFormed(t, srvA, srvB) + + // Wait for seed server to notice that its certificate has been revoked, + // so that new routes can't connect to it. + time.Sleep(6 * time.Second) + + // Start another server against the seed server that has an invalid OCSP Staple + srvConfC := ` + host: "127.0.0.1" + port: -1 + + server_name: "CCC" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-05-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-05-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + cluster { + name: AB + host: "127.0.0.1" + advertise: 127.0.0.1 + port: -1 + pool_size: -1 + + routes: [ nats://127.0.0.1:%d ] + connect_retries: 30 + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-06-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-06-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } + ` + srvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.Cluster.Port) + conf = createConfFile(t, []byte(srvConfC)) + srvC, optsC := RunServerWithConfig(conf) + defer srvC.Shutdown() + + cB, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", optsB.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cB.Close() + cC, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", optsC.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cC.Close() + + // There should be no connectivity between the clients due to the revoked staple. + _, err = cA.Subscribe("foo", func(m *nats.Msg) { + m.Respond(nil) + }) + if err != nil { + t.Errorf("%v", err) + } + cA.Flush() + _, err = cB.Subscribe("bar", func(m *nats.Msg) { + m.Respond(nil) + }) + if err != nil { + t.Fatal(err) + } + cB.Flush() + resp, err := cC.Request("foo", nil, 2*time.Second) + if err == nil { + t.Errorf("Unexpected success, response: %+v", resp) + } + resp, err = cC.Request("bar", nil, 2*time.Second) + if err == nil { + t.Errorf("Unexpected success, response: %+v", resp) + } + + // Switch the certs from the seed server to new ones that are not revoked, + // this should restart OCSP Stapling for the cluster routes. + srvConfA = ` + host: "127.0.0.1" + port: -1 + + server_name: "AAA" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-07-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-07-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + cluster { + port: -1 + pool_size: -1 + compression: "disabled" + name: AB + host: "127.0.0.1" + advertise: 127.0.0.1 + connect_retries: 30 + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-08-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-08-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } + ` + srvConfA = fmt.Sprintf(srvConfA, storeDirA) + if err := os.WriteFile(sconfA, []byte(srvConfA), 0666); err != nil { + t.Fatalf("Error writing config: %v", err) + } + if err := srvA.Reload(); err != nil { + t.Fatal(err) + } + // Wait to get a new OCSP Staple. + time.Sleep(10 * time.Second) + checkClusterFormed(t, srvA, srvB, srvC) + + // Now clients connect to C can communicate with B and A. + _, err = cC.Request("foo", nil, 2*time.Second) + if err != nil { + t.Errorf("%v", err) + } + _, err = cC.Request("bar", nil, 2*time.Second) + if err != nil { + t.Errorf("%v", err) + } +} + +func TestOCSPLeaf(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-03-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-04-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-05-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-06-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-07-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-08-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/client-cert.pem", ocsp.Good) + + // Store Dirs + storeDirA := t.TempDir() + storeDirB := t.TempDir() + storeDirC := t.TempDir() + + // LeafNode server configuration + srvConfA := ` + host: "127.0.0.1" + port: -1 + + server_name: "AAA" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + + leafnodes { + host: "127.0.0.1" + port: -1 + advertise: "127.0.0.1" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + # Leaf connection must present certs. + verify: true + } + } + ` + srvConfA = fmt.Sprintf(srvConfA, storeDirA) + sconfA := createConfFile(t, []byte(srvConfA)) + srvA, optsA := RunServerWithConfig(sconfA) + defer srvA.Shutdown() + + // LeafNode that has the original as a remote and running + // without OCSP Stapling for the leaf remote. + srvConfB := ` + host: "127.0.0.1" + port: -1 + + server_name: "BBB" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-03-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-03-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + + leafnodes { + remotes: [ { + url: "tls://127.0.0.1:%d" + tls { + # Cert without OCSP Stapling enabled is able to connect. + cert_file: "configs/certs/ocsp/client-cert.pem" + key_file: "configs/certs/ocsp/client-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } ] + } + ` + srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.LeafNode.Port) + conf := createConfFile(t, []byte(srvConfB)) + srvB, optsB := RunServerWithConfig(conf) + defer srvB.Shutdown() + + // Client connects to server A. + cA, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsA.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cA.Close() + checkLeafNodeConnected(t, srvA) + + // Revoke the seed server cluster certificate, following servers will not be able to verify connection. + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Revoked) + + // Original set of servers still can communicate to each other via leafnode, even though the staple + // for the leaf server has been revoked. + checkLeafNodeConnected(t, srvA) + + // Wait for seed server to notice that its certificate has been revoked. + time.Sleep(6 * time.Second) + + // Start another server against the seed server that has an revoked OCSP Staple. + srvConfC := ` + host: "127.0.0.1" + port: -1 + + server_name: "CCC" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-05-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-05-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + leafnodes { + remotes: [ { + url: "tls://127.0.0.1:%d" + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-06-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-06-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } ] + } + ` + srvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.LeafNode.Port) + conf = createConfFile(t, []byte(srvConfC)) + srvC, optsC := RunServerWithConfig(conf) + defer srvC.Shutdown() + + cB, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsB.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cB.Close() + cC, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsC.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cC.Close() + + // There should be connectivity between the clients even if there is a revoked staple + // from a leafnode connection. + _, err = cA.Subscribe("foo", func(m *nats.Msg) { + m.Respond(nil) + }) + if err != nil { + t.Errorf("%v", err) + } + cA.Flush() + _, err = cB.Subscribe("bar", func(m *nats.Msg) { + m.Respond(nil) + }) + if err != nil { + t.Fatal(err) + } + cB.Flush() + + // Ensure the subscriptions are known by the server we're connected to. + time.Sleep(100 * time.Millisecond) + + _, err = cC.Request("foo", nil, 2*time.Second) + if err != nil { + t.Errorf("Expected success, got: %+v", err) + } + _, err = cC.Request("bar", nil, 2*time.Second) + if err != nil { + t.Errorf("Expected success, got: %+v", err) + } + + // Switch the certs from the leafnode server to new ones that are not revoked, + // this should restart OCSP Stapling for the leafnode server. + srvConfA = ` + host: "127.0.0.1" + port: -1 + + server_name: "AAA" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-07-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-07-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + leafnodes { + host: "127.0.0.1" + port: -1 + advertise: "127.0.0.1" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-08-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-08-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + verify: true + } + } + ` + srvConfA = fmt.Sprintf(srvConfA, storeDirA) + if err := os.WriteFile(sconfA, []byte(srvConfA), 0666); err != nil { + t.Fatalf("Error writing config: %v", err) + } + if err := srvA.Reload(); err != nil { + t.Fatal(err) + } + time.Sleep(4 * time.Second) + + // A <-> A + _, err = cA.Request("foo", nil, 2*time.Second) + if err != nil { + t.Errorf("%v", err) + } + + // B <-> A + _, err = cB.Request("foo", nil, 2*time.Second) + if err != nil { + t.Errorf("%v", err) + } + + // C <-> A + _, err = cC.Request("foo", nil, 2*time.Second) + if err != nil { + t.Errorf("%v", err) + } + // C <-> B via leafnode A + _, err = cC.Request("bar", nil, 2*time.Second) + if err != nil { + t.Errorf("%v", err) + } +} + +func TestOCSPLeafNoVerify(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-03-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-04-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-05-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-06-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-07-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-08-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/client-cert.pem", ocsp.Good) + + // Store Dirs + storeDirA := t.TempDir() + storeDirB := t.TempDir() + storeDirC := t.TempDir() + + // LeafNode server configuration + srvConfA := ` + host: "127.0.0.1" + port: -1 + + server_name: "AAA" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + + leafnodes { + host: "127.0.0.1" + port: -1 + advertise: "127.0.0.1" + # for this test, explicitly disable compression because we do it + # in RunServer but here we do a config reload... + compression: off + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + # Leaf server does not require certs for clients. + verify: false + } + } + ` + srvConfA = fmt.Sprintf(srvConfA, storeDirA) + sconfA := createConfFile(t, []byte(srvConfA)) + srvA, optsA := RunServerWithConfig(sconfA) + defer srvA.Shutdown() + + // LeafNode remote that will connect to A and will not present certs. + srvConfB := ` + host: "127.0.0.1" + port: -1 + + server_name: "BBB" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-03-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-03-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + + leafnodes { + remotes: [ { + url: "tls://127.0.0.1:%d" + tls { + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } ] + } + ` + srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.LeafNode.Port) + conf := createConfFile(t, []byte(srvConfB)) + srvB, optsB := RunServerWithConfig(conf) + defer srvB.Shutdown() + + // Client connects to server A. + cA, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsA.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cA.Close() + checkLeafNodeConnected(t, srvA) + + // Revoke the seed server cluster certificate, following servers will not be able to verify connection. + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Revoked) + + // Original set of servers still can communicate to each other, even though the cert has been revoked. + checkLeafNodeConnected(t, srvA) + + // Wait for seed server to notice that its certificate has been revoked. + time.Sleep(6 * time.Second) + + // Start another server against the seed server that has an revoked OCSP Staple. + srvConfC := ` + host: "127.0.0.1" + port: -1 + + server_name: "CCC" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-05-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-05-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + leafnodes { + remotes: [ { + url: "tls://127.0.0.1:%d" + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-06-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-06-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + timeout: 5 + } + } ] + } + ` + srvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.LeafNode.Port) + conf = createConfFile(t, []byte(srvConfC)) + srvC, optsC := RunServerWithConfig(conf) + defer srvC.Shutdown() + + cB, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsB.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cB.Close() + cC, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsC.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cC.Close() + + // There should be connectivity between the clients even if there is a revoked staple + // from a leafnode connection. + _, err = cA.Subscribe("foo", func(m *nats.Msg) { + m.Respond(nil) + }) + if err != nil { + t.Errorf("%v", err) + } + cA.Flush() + _, err = cB.Subscribe("bar", func(m *nats.Msg) { + m.Respond(nil) + }) + if err != nil { + t.Fatal(err) + } + cB.Flush() + + // Ensure the subscriptions are known by the server we're connected to. + time.Sleep(100 * time.Millisecond) + + _, err = cC.Request("foo", nil, 2*time.Second) + if err != nil { + t.Errorf("Expected success, got: %+v", err) + } + _, err = cC.Request("bar", nil, 2*time.Second) + if err != nil { + t.Errorf("Expected success, got: %+v", err) + } + + // Switch the certs from the leafnode server to new ones that are not revoked, + // this should restart OCSP Stapling for the leafnode server. + srvConfA = ` + host: "127.0.0.1" + port: -1 + + server_name: "AAA" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-07-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-07-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + leafnodes { + host: "127.0.0.1" + port: -1 + advertise: "127.0.0.1" + compression: off + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-08-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-08-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + verify: true + } + } + ` + srvConfA = fmt.Sprintf(srvConfA, storeDirA) + if err := os.WriteFile(sconfA, []byte(srvConfA), 0666); err != nil { + t.Fatalf("Error writing config: %v", err) + } + if err := srvA.Reload(); err != nil { + t.Fatal(err) + } + time.Sleep(4 * time.Second) + + // A <-> A + _, err = cA.Request("foo", nil, 2*time.Second) + if err != nil { + t.Errorf("%v", err) + } + + // B <-> A + _, err = cB.Request("foo", nil, 2*time.Second) + if err != nil { + t.Errorf("%v", err) + } + + // C <-> A + _, err = cC.Request("foo", nil, 2*time.Second) + if err != nil { + t.Errorf("%v", err) + } + // C <-> B via leafnode A + _, err = cC.Request("bar", nil, 2*time.Second) + if err != nil { + t.Errorf("%v", err) + } +} + +func TestOCSPLeafVerifyLeafRemote(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-03-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-04-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-05-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-06-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-07-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-08-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/client-cert.pem", ocsp.Good) + + // Store Dirs + storeDirA := t.TempDir() + storeDirB := t.TempDir() + + // LeafNode server configuration + srvConfA := ` + host: "127.0.0.1" + port: -1 + + server_name: "AAA" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + + leafnodes { + host: "127.0.0.1" + port: -1 + advertise: "127.0.0.1" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + verify: true + } + } + ` + srvConfA = fmt.Sprintf(srvConfA, storeDirA) + sconfA := createConfFile(t, []byte(srvConfA)) + srvA, optsA := RunServerWithConfig(sconfA) + defer srvA.Shutdown() + + // LeafNode remote that will connect to A and will not present certs. + srvConfB := ` + host: "127.0.0.1" + port: -1 + + server_name: "BBB" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-03-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-03-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + + leafnodes { + remotes: [ { + url: "tls://127.0.0.1:%d" + tls { + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } ] + } + ` + srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.LeafNode.Port) + conf := createConfFile(t, []byte(srvConfB)) + srvB, _ := RunServerWithConfig(conf) + defer srvB.Shutdown() + + // Client connects to server A. + cA, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsA.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cA.Close() + + // Should not have been able to connect. + checkLeafNodeConnections(t, srvA, 0) +} + +func TestOCSPLeafVerifyAndMapLeafRemote(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-03-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-04-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-05-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-06-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-07-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-08-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/client-cert.pem", ocsp.Good) + + // Store Dirs + storeDirA := t.TempDir() + storeDirB := t.TempDir() + + // LeafNode server configuration + srvConfA := ` + host: "127.0.0.1" + port: -1 + + server_name: "AAA" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + verify_and_map: true + } + store_dir: '%s' + + leafnodes { + host: "127.0.0.1" + port: -1 + advertise: "127.0.0.1" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + verify_and_map: true + } + } + + accounts: { + leaf: { + users: [ {user: "C=US, ST=CA, L=San Francisco, O=Synadia, OU=nats.io, CN=localhost server-status-request-url-04"} ] + } + client: { + users: [ {user: "C=US, ST=CA, L=San Francisco, O=Synadia, OU=nats.io, CN=localhost client"} ] + } + } + + ` + srvConfA = fmt.Sprintf(srvConfA, storeDirA) + sconfA := createConfFile(t, []byte(srvConfA)) + srvA, optsA := RunServerWithConfig(sconfA) + defer srvA.Shutdown() + + // LeafNode remote that will connect to A and will not present certs. + srvConfB := ` + host: "127.0.0.1" + port: -1 + + server_name: "BBB" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-03-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-03-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + + leafnodes { + remotes: [ { + url: "tls://127.0.0.1:%d" + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-04-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-04-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } ] + } + ` + srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.LeafNode.Port) + conf := createConfFile(t, []byte(srvConfB)) + srvB, _ := RunServerWithConfig(conf) + defer srvB.Shutdown() + + // Client connects to server A. + cA, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsA.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cA.Close() + checkLeafNodeConnections(t, srvA, 1) +} + +func TestOCSPGateway(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-03-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-04-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-05-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-06-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-07-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-08-cert.pem", ocsp.Good) + + // Store Dirs + storeDirA := t.TempDir() + storeDirB := t.TempDir() + storeDirC := t.TempDir() + + // Gateway server configuration + srvConfA := ` + host: "127.0.0.1" + port: -1 + + server_name: "AAA" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + gateway { + name: A + host: "127.0.0.1" + port: -1 + advertise: "127.0.0.1" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } + ` + srvConfA = fmt.Sprintf(srvConfA, storeDirA) + sconfA := createConfFile(t, []byte(srvConfA)) + srvA, optsA := RunServerWithConfig(sconfA) + defer srvA.Shutdown() + + // LeafNode that has the original as a remote. + srvConfB := ` + host: "127.0.0.1" + port: -1 + + server_name: "BBB" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-03-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-03-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + gateway { + name: B + host: "127.0.0.1" + advertise: "127.0.0.1" + port: -1 + gateways: [{ + name: "A" + url: "nats://127.0.0.1:%d" + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-04-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-04-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + }] + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-04-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-04-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } + ` + srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.Gateway.Port) + conf := createConfFile(t, []byte(srvConfB)) + srvB, optsB := RunServerWithConfig(conf) + defer srvB.Shutdown() + + // Client connects to server A. + cA, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsA.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cA.Close() + waitForOutboundGateways(t, srvB, 1, 5*time.Second) + + // Revoke the seed server cluster certificate, following servers will not be able to verify connection. + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Revoked) + + // Original set of servers still can communicate to each other, even though the cert has been revoked. + waitForOutboundGateways(t, srvA, 1, 5*time.Second) + waitForOutboundGateways(t, srvB, 1, 5*time.Second) + + // Wait for gateway A to notice that its certificate has been revoked, + // so that new gateways can't connect to it. + time.Sleep(6 * time.Second) + + // Start another server against the seed server that has an invalid OCSP Staple + srvConfC := ` + host: "127.0.0.1" + port: -1 + + server_name: "CCC" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-05-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-05-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + gateway { + name: C + host: "127.0.0.1" + advertise: "127.0.0.1" + port: -1 + gateways: [{name: "A", url: "nats://127.0.0.1:%d" }] + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-06-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-06-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } + ` + srvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.Gateway.Port) + conf = createConfFile(t, []byte(srvConfC)) + srvC, optsC := RunServerWithConfig(conf) + defer srvC.Shutdown() + + cB, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsB.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cB.Close() + cC, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsC.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cC.Close() + + // There should be no connectivity between the clients due to the revoked staple. + _, err = cA.Subscribe("foo", func(m *nats.Msg) { + m.Respond(nil) + }) + if err != nil { + t.Errorf("%v", err) + } + cA.Flush() + _, err = cB.Subscribe("bar", func(m *nats.Msg) { + m.Respond(nil) + }) + if err != nil { + t.Fatal(err) + } + cB.Flush() + + // Gateway C was not able to mesh with Gateway A because of the revoked OCSP staple + // so these requests to A and B should fail. + resp, err := cC.Request("foo", nil, 2*time.Second) + if err == nil { + t.Errorf("Unexpected success, response: %+v", resp) + } + // Make request to B + resp, err = cC.Request("bar", nil, 2*time.Second) + if err == nil { + t.Errorf("Unexpected success, response: %+v", resp) + } + + // Switch the certs from the seed server to new ones that are not revoked, + // this should restart OCSP Stapling for the cluster routes. + srvConfA = ` + host: "127.0.0.1" + port: -1 + + server_name: "AAA" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-07-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-07-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + gateway { + name: A + host: "127.0.0.1" + port: -1 + advertise: "127.0.0.1" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-08-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-08-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } + ` + + srvConfA = fmt.Sprintf(srvConfA, storeDirA) + if err := os.WriteFile(sconfA, []byte(srvConfA), 0666); err != nil { + t.Fatalf("Error writing config: %v", err) + } + if err := srvA.Reload(); err != nil { + t.Fatal(err) + } + time.Sleep(4 * time.Second) + waitForOutboundGateways(t, srvA, 2, 5*time.Second) + waitForOutboundGateways(t, srvB, 2, 5*time.Second) + waitForOutboundGateways(t, srvC, 2, 5*time.Second) + + // Now clients connect to C can communicate with B and A. + _, err = cC.Request("foo", nil, 2*time.Second) + if err != nil { + t.Errorf("%v", err) + } + _, err = cC.Request("bar", nil, 2*time.Second) + if err != nil { + t.Errorf("%v", err) + } +} + +func TestOCSPGatewayIntermediate(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem", ocsp.Good) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer2_cert.pem", ocsp.Good) + + // Gateway server configuration + srvConfA := ` + host: "127.0.0.1" + port: -1 + + server_name: "AAA" + + ocsp: { + mode: always + url: %s + } + + gateway { + name: A + host: "127.0.0.1" + port: -1 + advertise: "127.0.0.1" + + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + } + } + ` + srvConfA = fmt.Sprintf(srvConfA, intermediateCA1ResponderURL) + sconfA := createConfFile(t, []byte(srvConfA)) + srvA, optsA := RunServerWithConfig(sconfA) + defer srvA.Shutdown() + + srvConfB := ` + host: "127.0.0.1" + port: -1 + + server_name: "BBB" + + ocsp: { + mode: always + url: %s + } + + gateway { + name: B + host: "127.0.0.1" + advertise: "127.0.0.1" + port: -1 + gateways: [{ + name: "A" + url: "nats://127.0.0.1:%d" + }] + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + } + } + ` + srvConfB = fmt.Sprintf(srvConfB, intermediateCA1ResponderURL, optsA.Gateway.Port) + conf := createConfFile(t, []byte(srvConfB)) + srvB, optsB := RunServerWithConfig(conf) + defer srvB.Shutdown() + + // Client connects to server A. + cA, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsA.Port), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cA.Close() + waitForOutboundGateways(t, srvB, 1, 5*time.Second) + + cB, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsB.Port), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cB.Close() +} + +func TestOCSPGatewayReload(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + + // Node A + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Good) + + // Node B + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-03-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-04-cert.pem", ocsp.Good) + + // Node C + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-05-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-06-cert.pem", ocsp.Good) + + // Node A rotated certs + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-07-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-08-cert.pem", ocsp.Good) + + // Store Dirs + storeDirA := t.TempDir() + storeDirB := t.TempDir() + storeDirC := t.TempDir() + + // Gateway server configuration + srvConfA := ` + host: "127.0.0.1" + port: -1 + + server_name: "AAA" + + ocsp { mode = always } + + store_dir: '%s' + gateway { + name: A + host: "127.0.0.1" + port: -1 + advertise: "127.0.0.1" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } + ` + srvConfA = fmt.Sprintf(srvConfA, storeDirA) + sconfA := createConfFile(t, []byte(srvConfA)) + srvA, optsA := RunServerWithConfig(sconfA) + defer srvA.Shutdown() + + // Gateway B connects to Gateway A. + srvConfB := ` + host: "127.0.0.1" + port: -1 + + server_name: "BBB" + + ocsp { mode = always } + + store_dir: '%s' + gateway { + name: B + host: "127.0.0.1" + advertise: "127.0.0.1" + port: -1 + gateways: [{ + name: "A" + url: "nats://127.0.0.1:%d" + }] + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-04-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-04-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } + ` + srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.Gateway.Port) + conf := createConfFile(t, []byte(srvConfB)) + srvB, optsB := RunServerWithConfig(conf) + defer srvB.Shutdown() + + // Client connects to server A. + cA, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsA.Port), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cA.Close() + + // Wait for connectivity between A and B. + waitForOutboundGateways(t, srvB, 1, 5*time.Second) + + // Gateway C also connects to Gateway A. + srvConfC := ` + host: "127.0.0.1" + port: -1 + + server_name: "CCC" + + ocsp { mode = always } + + store_dir: '%s' + gateway { + name: C + host: "127.0.0.1" + advertise: "127.0.0.1" + port: -1 + gateways: [{name: "A", url: "nats://127.0.0.1:%d" }] + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-06-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-06-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } + ` + srvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.Gateway.Port) + conf = createConfFile(t, []byte(srvConfC)) + srvC, optsC := RunServerWithConfig(conf) + defer srvC.Shutdown() + + //////////////////////////////////////////////////////////////////////////// + // // + // A and B are connected at this point and C is starting with certs that // + // will be rotated, in v2.10.8 on reload now all OCSP monitors are also // + // always restarted. // + // // + //////////////////////////////////////////////////////////////////////////// + cB, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsB.Port), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cB.Close() + cC, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsC.Port), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cC.Close() + + _, err = cA.Subscribe("foo", func(m *nats.Msg) { + m.Respond(nil) + }) + if err != nil { + t.Errorf("%v", err) + } + cA.Flush() + _, err = cB.Subscribe("bar", func(m *nats.Msg) { + m.Respond(nil) + }) + if err != nil { + t.Fatal(err) + } + cB.Flush() + + ///////////////////////////////////////////////////////////////////////////////// + // // + // Switch all the certs from server A, all OCSP monitors should be restarted // + // so it should have new staples. // + // // + ///////////////////////////////////////////////////////////////////////////////// + srvConfA = ` + host: "127.0.0.1" + port: -1 + + server_name: "AAA" + + ocsp { mode = always } + + store_dir: '%s' + gateway { + name: A + host: "127.0.0.1" + port: -1 + advertise: "127.0.0.1" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-08-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-08-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } + ` + + srvConfA = fmt.Sprintf(srvConfA, storeDirA) + if err := os.WriteFile(sconfA, []byte(srvConfA), 0666); err != nil { + t.Fatalf("Error writing config: %v", err) + } + if err := srvA.Reload(); err != nil { + t.Fatal(err) + } + waitForOutboundGateways(t, srvA, 2, 5*time.Second) + waitForOutboundGateways(t, srvB, 2, 5*time.Second) + waitForOutboundGateways(t, srvC, 2, 5*time.Second) + + // Now clients connect to C can communicate with B and A. + _, err = cC.Request("foo", nil, 2*time.Second) + if err != nil { + t.Errorf("%v", err) + } + _, err = cC.Request("bar", nil, 2*time.Second) + if err != nil { + t.Errorf("%v", err) + } +} + +func TestOCSPCustomConfig(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + serverCert = "configs/certs/ocsp/server-cert.pem" + serverKey = "configs/certs/ocsp/server-key.pem" + ) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + ocspURL := fmt.Sprintf("http://%s", ocspr.Addr) + defer ocspr.Shutdown(ctx) + + var ( + errExpectedNoStaple = fmt.Errorf("expected no staple") + errMissingStaple = fmt.Errorf("missing OCSP Staple from server") + ) + + for _, test := range []struct { + name string + config string + opts []nats.Option + err error + rerr error + configure func() + }{ + { + "OCSP Stapling in auto mode makes server fail to boot if status is revoked", + ` + port: -1 + + ocsp { + mode: auto + } + + tls { + cert_file: "configs/certs/ocsp/server-cert.pem" + key_file: "configs/certs/ocsp/server-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + `, + []nats.Option{ + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse != nil { + return errExpectedNoStaple + } + return nil + }, + }), + nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() { SetOCSPStatus(t, ocspURL, serverCert, ocsp.Revoked) }, + }, + { + "OCSP Stapling must staple ignored if disabled with ocsp: false", + ` + port: -1 + + ocsp: false + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + `, + []nats.Option{ + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse != nil { + return errExpectedNoStaple + } + return nil + }, + }), + nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() { + SetOCSPStatus(t, ocspURL, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) + }, + }, + { + "OCSP Stapling must staple ignored if disabled with ocsp mode never", + ` + port: -1 + + ocsp: { mode: never } + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + `, + []nats.Option{ + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse != nil { + return errExpectedNoStaple + } + return nil + }, + }), + nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() { + SetOCSPStatus(t, ocspURL, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) + }, + }, + { + "OCSP Stapling in always mode fetches a staple even if cert does not have one", + ` + port: -1 + + ocsp { + mode: always + url: "http://127.0.0.1:8888" + } + + tls { + cert_file: "configs/certs/ocsp/server-cert.pem" + key_file: "configs/certs/ocsp/server-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + `, + []nats.Option{ + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return errMissingStaple + } + return nil + }, + }), + nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() { SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) }, + }, + { + "OCSP Stapling in must staple mode does not fetch staple if there is no must staple flag", + ` + port: -1 + + ocsp { + mode: must + url: "http://127.0.0.1:8888" + } + + tls { + cert_file: "configs/certs/ocsp/server-cert.pem" + key_file: "configs/certs/ocsp/server-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + `, + []nats.Option{ + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse != nil { + return errExpectedNoStaple + } + return nil + }, + }), + nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() { SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) }, + }, + { + "OCSP Stapling in must staple mode fetches staple if there is a must staple flag", + ` + port: -1 + + ocsp { + mode: must + url: "http://127.0.0.1:8888" + } + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + `, + []nats.Option{ + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return errMissingStaple + } + return nil + }, + }), + nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + func() { + SetOCSPStatus(t, ocspURL, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + test.configure() + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + + nc.Subscribe("ping", func(m *nats.Msg) { + m.Respond([]byte("pong")) + }) + nc.Flush() + + _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) + if test.rerr != nil && err == nil { + t.Errorf("Expected error getting response") + } else if test.rerr == nil && err != nil { + t.Errorf("Expected response") + } + }) + } +} + +func TestOCSPCustomConfigReloadDisable(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + serverCert = "configs/certs/ocsp/server-cert.pem" + updatedServerCert = "configs/certs/ocsp/server-status-request-url-01-cert.pem" + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, serverCert, ocsp.Good) + SetOCSPStatus(t, addr, updatedServerCert, ocsp.Good) + + // Start with server without OCSP Stapling MustStaple + content := ` + port: -1 + + ocsp: { mode: always, url: "http://127.0.0.1:8888" } + + tls { + cert_file: "configs/certs/ocsp/server-cert.pem" + key_file: "configs/certs/ocsp/server-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + ` + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple!") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + sub, err := nc.SubscribeSync("foo") + if err != nil { + t.Fatal(err) + } + nc.Publish("foo", []byte("hello world")) + nc.Flush() + + _, err = sub.NextMsg(1 * time.Second) + if err != nil { + t.Fatal(err) + } + nc.Close() + + // Change and disable OCSP Stapling. + content = ` + port: -1 + + ocsp: { mode: never } + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + ` + if err := os.WriteFile(conf, []byte(content), 0666); err != nil { + t.Fatalf("Error writing config: %v", err) + } + if err := s.Reload(); err != nil { + t.Fatal(err) + } + + // The new certificate has must staple but OCSP Stapling is disabled. + time.Sleep(2 * time.Second) + + nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse != nil { + return fmt.Errorf("unexpected OCSP Staple!") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + nc.Close() +} + +func TestOCSPCustomConfigReloadEnable(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + serverCert = "configs/certs/ocsp/server-cert.pem" + updatedServerCert = "configs/certs/ocsp/server-status-request-url-01-cert.pem" + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, serverCert, ocsp.Good) + SetOCSPStatus(t, addr, updatedServerCert, ocsp.Good) + + // Start with server without OCSP Stapling MustStaple + content := ` + port: -1 + + ocsp: { mode: never, url: "http://127.0.0.1:8888" } + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + ` + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse != nil { + return fmt.Errorf("unexpected OCSP Staple!") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + sub, err := nc.SubscribeSync("foo") + if err != nil { + t.Fatal(err) + } + nc.Publish("foo", []byte("hello world")) + nc.Flush() + + _, err = sub.NextMsg(1 * time.Second) + if err != nil { + t.Fatal(err) + } + nc.Close() + + // Change and disable OCSP Stapling. + content = ` + port: -1 + + ocsp: { mode: always, url: "http://127.0.0.1:8888" } + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + ` + if err := os.WriteFile(conf, []byte(content), 0666); err != nil { + t.Fatalf("Error writing config: %v", err) + } + if err := s.Reload(); err != nil { + t.Fatal(err) + } + time.Sleep(2 * time.Second) + + nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple!") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + nc.Close() +} + +func TestOCSPTLSConfigNoLeafSet(t *testing.T) { + o := DefaultTestOptions + o.HTTPHost = "127.0.0.1" + o.HTTPSPort = -1 + o.TLSConfig = &tls.Config{ServerName: "localhost"} + cert, err := tls.LoadX509KeyPair("configs/certs/server-cert.pem", "configs/certs/server-key.pem") + if err != nil { + t.Fatalf("Got error reading certificates: %s", err) + } + o.TLSConfig.Certificates = []tls.Certificate{cert} + s := RunServer(&o) + s.Shutdown() +} + +func TestOCSPSuperCluster(t *testing.T) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponder(t, caCert, caKey) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-03-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-04-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-05-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-06-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-07-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-08-cert.pem", ocsp.Good) + SetOCSPStatus(t, addr, "configs/certs/ocsp/server-cert.pem", ocsp.Good) + + // Store Dirs + storeDirA := t.TempDir() + storeDirB := t.TempDir() + storeDirC := t.TempDir() + storeDirD := t.TempDir() + + // Gateway server configuration + srvConfA := ` + host: "127.0.0.1" + port: -1 + + server_name: "A" + + ocsp { mode: "always" } + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + + cluster { + name: A + host: "127.0.0.1" + advertise: 127.0.0.1 + port: -1 + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } + + gateway { + name: A + host: "127.0.0.1" + port: -1 + advertise: "127.0.0.1" + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-03-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-03-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + verify: true + } + } + ` + srvConfA = fmt.Sprintf(srvConfA, storeDirA) + sconfA := createConfFile(t, []byte(srvConfA)) + srvA, optsA := RunServerWithConfig(sconfA) + defer srvA.Shutdown() + + // Server that has the original as a cluster. + srvConfB := ` + host: "127.0.0.1" + port: -1 + + server_name: "B" + + ocsp { mode: "always" } + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + + cluster { + name: A + host: "127.0.0.1" + advertise: 127.0.0.1 + port: -1 + + routes: [ nats://127.0.0.1:%d ] + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + } + + gateway { + name: A + host: "127.0.0.1" + advertise: "127.0.0.1" + port: -1 + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-03-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-03-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + verify: true + } + } + ` + srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.Cluster.Port) + conf := createConfFile(t, []byte(srvConfB)) + srvB, optsB := RunServerWithConfig(conf) + defer srvB.Shutdown() + + // Client connects to server A. + cA, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsA.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + + } + defer cA.Close() + + // Start another server that will make connect as a gateway to cluster A. + srvConfC := ` + host: "127.0.0.1" + port: -1 + + server_name: "C" + + ocsp { mode: "always" } + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-05-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-05-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + gateway { + name: C + host: "127.0.0.1" + advertise: "127.0.0.1" + port: -1 + gateways: [{ + name: "A", + urls: ["nats://127.0.0.1:%d"] + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-06-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-06-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + }] + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-06-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-06-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + verify: true + } + } + ` + srvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.Gateway.Port) + conf = createConfFile(t, []byte(srvConfC)) + srvC, optsC := RunServerWithConfig(conf) + defer srvC.Shutdown() + + // Check that server is connected to any server from the other cluster. + checkClusterFormed(t, srvA, srvB) + waitForOutboundGateways(t, srvC, 1, 5*time.Second) + + // Start one more server that will become another gateway. + srvConfD := ` + host: "127.0.0.1" + port: -1 + + server_name: "D" + + ocsp { mode: "auto", url: "%s" } + + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-07-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-07-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + } + store_dir: '%s' + gateway { + name: D + host: "127.0.0.1" + advertise: "127.0.0.1" + port: -1 + gateways: [{ + name: "A", + urls: ["nats://127.0.0.1:%d"] + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-08-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-08-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + }}, + { + name: "C", + urls: ["nats://127.0.0.1:%d"] + + #################################################################### + ## TEST NOTE: This cert does not have an OCSP Staple intentionally## + #################################################################### + tls { + ca_file: "configs/certs/ocsp/ca-cert.pem" + cert_file: "configs/certs/ocsp/server-cert.pem" + key_file: "configs/certs/ocsp/server-key.pem" + timeout: 5 + }} + ] + tls { + cert_file: "configs/certs/ocsp/server-status-request-url-08-cert.pem" + key_file: "configs/certs/ocsp/server-status-request-url-08-key.pem" + ca_file: "configs/certs/ocsp/ca-cert.pem" + timeout: 5 + verify: true + } + } + ` + srvConfD = fmt.Sprintf(srvConfD, addr, storeDirD, optsA.Gateway.Port, optsC.Gateway.Port) + conf = createConfFile(t, []byte(srvConfD)) + srvD, _ := RunServerWithConfig(conf) + defer srvD.Shutdown() + + // There should be a single gateway here because one of the gateway connections does not have a OCSP staple. + waitForOutboundGateways(t, srvD, 1, 10*time.Second) + + // Connect to cluster A using server B. + cB, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsB.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cB.Close() + + // Connects to cluster C using server C. + cC, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsC.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cC.Close() + + _, err = cA.Subscribe("foo", func(m *nats.Msg) { + m.Respond([]byte("From Server A")) + }) + if err != nil { + t.Errorf("%v", err) + } + cA.Flush() + + _, err = cB.Subscribe("bar", func(m *nats.Msg) { + m.Respond([]byte("From Server B")) + }) + if err != nil { + t.Fatal(err) + } + cB.Flush() + + // Confirm that a message from server C can flow back to server A via gateway.. + var ( + resp *nats.Msg + lerr error + ) + for i := 0; i < 10; i++ { + resp, lerr = cC.Request("foo", nil, 500*time.Millisecond) + if lerr != nil { + continue + } + got := string(resp.Data) + expected := "From Server A" + if got != expected { + t.Fatalf("Expected %v, got: %v", expected, got) + } + + // Make request to B + resp, lerr = cC.Request("bar", nil, 500*time.Millisecond) + if lerr != nil { + continue + } + got = string(resp.Data) + expected = "From Server B" + if got != expected { + t.Errorf("Expected %v, got: %v", expected, got) + } + lerr = nil + break + } + if lerr != nil { + t.Errorf("Unexpected error: %v", lerr) + } + if n := srvD.NumOutboundGateways(); n > 1 { + t.Errorf("Expected single gateway, got: %v", n) + } +} + +func TestOCSPLocalIssuerDetermination(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + + // Test constants + ocspURL := intermediateCA1ResponderURL + clientTrustBundle := "configs/certs/ocsp_peer/mini-ca/misc/trust_config1_bundle.pem" + serverCert := "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem" + + var ( + errMissingStaple = fmt.Errorf("missing OCSP Staple from server") + ) + + for _, test := range []struct { + name string + config string + opts []nats.Option + err error + rerr error + serverStart bool + configure func() + }{ + { + "Correct issuer configured in cert bundle", + ` + port: -1 + + ocsp { + mode: always + } + + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + } + `, + []nats.Option{ + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return errMissingStaple + } + return nil + }, + }), + nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), + nats.RootCAs(clientTrustBundle), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + true, + func() { + SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) + }, + }, + { + "Wrong issuer configured in cert bundle, server no start", + ` + port: -1 + + ocsp { + mode: always + } + + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/misc/misconfig_TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + } + `, + []nats.Option{ + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return errMissingStaple + } + return nil + }, + }), + nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), + nats.RootCAs(clientTrustBundle), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + false, + func() { + SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) + }, + }, + { + "Issuer configured in CA bundle only, configuration 1", + ` + port: -1 + + ocsp { + mode: always + } + + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/misc/trust_config1_bundle.pem" + timeout: 5 + } + `, + []nats.Option{ + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return errMissingStaple + } + return nil + }, + }), + nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), + nats.RootCAs(clientTrustBundle), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + true, + func() { + SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) + }, + }, + { + "Issuer configured in CA bundle only, configuration 2", + ` + port: -1 + + ocsp { + mode: always + } + + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/misc/trust_config2_bundle.pem" + timeout: 5 + } + `, + []nats.Option{ + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return errMissingStaple + } + return nil + }, + }), + nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), + nats.RootCAs(clientTrustBundle), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + true, + func() { + SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) + }, + }, + { + "Issuer configured in CA bundle only, configuration 3", + ` + port: -1 + + ocsp { + mode: always + } + + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/misc/trust_config3_bundle.pem" + timeout: 5 + } + `, + []nats.Option{ + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return errMissingStaple + } + return nil + }, + }), + nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), + nats.RootCAs(clientTrustBundle), + nats.ErrorHandler(noOpErrHandler), + }, + nil, + nil, + true, + func() { + SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + defer func() { + r := recover() + if r != nil && test.serverStart { + t.Fatalf("Expected server start, unexpected panic: %v", r) + } + if r == nil && !test.serverStart { + t.Fatalf("Expected server to not start and panic thrown") + } + }() + test.configure() + content := test.config + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + // server may not start for some tests + if s != nil { + defer s.Shutdown() + } + + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + + nc.Subscribe("ping", func(m *nats.Msg) { + m.Respond([]byte("pong")) + }) + nc.Flush() + + _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) + if test.rerr != nil && err == nil { + t.Errorf("Expected error getting response") + } else if test.rerr == nil && err != nil { + t.Errorf("Expected response") + } + }) + } +} + +func TestMixedCAOCSPSuperCluster(t *testing.T) { + const ( + caCert = "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) + intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) + defer intermediateCA1Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem", ocsp.Good) + + intermediateCA2Responder := NewOCSPResponderIntermediateCA2(t) + intermediateCA2ResponderURL := fmt.Sprintf("http://%s", intermediateCA2Responder.Addr) + defer intermediateCA2Responder.Shutdown(ctx) + SetOCSPStatus(t, intermediateCA2ResponderURL, "configs/certs/ocsp_peer/mini-ca/server2/TestServer3_cert.pem", ocsp.Good) + + // Store Dirs + storeDirA := t.TempDir() + storeDirB := t.TempDir() + storeDirC := t.TempDir() + + // Gateway server configuration + srvConfA := ` + host: "127.0.0.1" + port: -1 + + server_name: "A" + + ocsp { mode: "always" } + + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + } + store_dir: '%s' + + cluster { + name: A + host: "127.0.0.1" + advertise: 127.0.0.1 + port: -1 + + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + } + } + + gateway { + name: A + host: "127.0.0.1" + port: -1 + advertise: "127.0.0.1" + + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + } + } + ` + srvConfA = fmt.Sprintf(srvConfA, storeDirA) + sconfA := createConfFile(t, []byte(srvConfA)) + srvA, optsA := RunServerWithConfig(sconfA) + defer srvA.Shutdown() + + // Server that has the original as a cluster. + srvConfB := ` + host: "127.0.0.1" + port: -1 + + server_name: "B" + + ocsp { mode: "always" } + + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + } + store_dir: '%s' + + cluster { + name: A + host: "127.0.0.1" + advertise: 127.0.0.1 + port: -1 + + routes: [ nats://127.0.0.1:%d ] + + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + } + } + + gateway { + name: A + host: "127.0.0.1" + advertise: "127.0.0.1" + port: -1 + + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + } + } + ` + srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.Cluster.Port) + conf := createConfFile(t, []byte(srvConfB)) + srvB, optsB := RunServerWithConfig(conf) + defer srvB.Shutdown() + + // Client connects to server A. + cA, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsA.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + + } + defer cA.Close() + + // Start another server that will make connect as a gateway to cluster A but with different CA issuer. + srvConfC := ` + host: "127.0.0.1" + port: -1 + + server_name: "C" + + ocsp { mode: "always" } + + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/server2/TestServer3_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server2/private/TestServer3_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + } + store_dir: '%s' + gateway { + name: C + host: "127.0.0.1" + advertise: "127.0.0.1" + port: -1 + gateways: [{ + name: "A", + urls: ["nats://127.0.0.1:%d"] + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/server2/TestServer3_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server2/private/TestServer3_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + } + }] + tls { + cert_file: "configs/certs/ocsp_peer/mini-ca/server2/TestServer3_bundle.pem" + key_file: "configs/certs/ocsp_peer/mini-ca/server2/private/TestServer3_keypair.pem" + ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" + timeout: 5 + verify: true + } + } + ` + srvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.Gateway.Port) + conf = createConfFile(t, []byte(srvConfC)) + srvC, optsC := RunServerWithConfig(conf) + defer srvC.Shutdown() + + // Check that server is connected to any server from the other cluster. + checkClusterFormed(t, srvA, srvB) + waitForOutboundGateways(t, srvC, 1, 5*time.Second) + + // Connect to cluster A using server B. + cB, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsB.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cB.Close() + + // Connects to cluster C using server C. + cC, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsC.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + if s.OCSPResponse == nil { + return fmt.Errorf("missing OCSP Staple from server") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer cC.Close() + + _, err = cA.Subscribe("foo", func(m *nats.Msg) { + m.Respond([]byte("From Server A")) + }) + if err != nil { + t.Errorf("%v", err) + } + cA.Flush() + + _, err = cB.Subscribe("bar", func(m *nats.Msg) { + m.Respond([]byte("From Server B")) + }) + if err != nil { + t.Fatal(err) + } + cB.Flush() + + // Confirm that a message from server C can flow back to server A via gateway.. + var ( + resp *nats.Msg + lerr error + ) + for i := 0; i < 10; i++ { + resp, lerr = cC.Request("foo", nil, 500*time.Millisecond) + if lerr != nil { + continue + } + got := string(resp.Data) + expected := "From Server A" + if got != expected { + t.Fatalf("Expected %v, got: %v", expected, got) + } + + // Make request to B + resp, lerr = cC.Request("bar", nil, 500*time.Millisecond) + if lerr != nil { + continue + } + got = string(resp.Data) + expected = "From Server B" + if got != expected { + t.Errorf("Expected %v, got: %v", expected, got) + } + lerr = nil + break + } + if lerr != nil { + t.Errorf("Unexpected error: %v", lerr) + } +} + +func TestOCSPResponderHTTPMethods(t *testing.T) { + t.Run("prefer get", func(t *testing.T) { + testOCSPResponderHTTPMethods(t, "GET") + }) + t.Run("prefer post", func(t *testing.T) { + testOCSPResponderHTTPMethods(t, "POST") + }) + t.Run("all methods failing", func(t *testing.T) { + testOCSPResponderFailing(t, "TEST") + }) +} + +func testOCSPResponderHTTPMethods(t *testing.T, method string) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + serverCert = "configs/certs/ocsp/server-cert.pem" + serverKey = "configs/certs/ocsp/server-key.pem" + ) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponderPreferringHTTPMethod(t, caCert, caKey, method) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, serverCert, ocsp.Good) + + // Add another responder that fails. + badaddr := "http://127.0.0.1:8889" + badocsp := NewOCSPResponderCustomAddress(t, caCert, caKey, badaddr) + defer badocsp.Shutdown(ctx) + + opts := server.Options{} + opts.Host = "127.0.0.1" + opts.NoLog = true + opts.NoSigs = true + opts.MaxControlLine = 4096 + opts.Port = -1 + opts.TLSCert = serverCert + opts.TLSKey = serverKey + opts.TLSCaCert = caCert + opts.TLSTimeout = 5 + tcOpts := &server.TLSConfigOpts{ + CertFile: opts.TLSCert, + KeyFile: opts.TLSKey, + CaFile: opts.TLSCaCert, + Timeout: opts.TLSTimeout, + } + + tlsConf, err := server.GenTLSConfig(tcOpts) + if err != nil { + t.Fatal(err) + } + opts.TLSConfig = tlsConf + + opts.OCSPConfig = &server.OCSPConfig{ + Mode: server.OCSPModeAlways, + OverrideURLs: []string{badaddr, addr}, + } + srv := RunServer(&opts) + defer srv.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + nats.Secure(&tls.Config{ + VerifyConnection: func(s tls.ConnectionState) error { + resp, err := GetOCSPStatus(s) + if err != nil { + return err + } + if resp.Status != ocsp.Good { + return fmt.Errorf("invalid staple") + } + return nil + }, + }), + nats.RootCAs(caCert), + nats.ErrorHandler(noOpErrHandler), + ) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + sub, err := nc.SubscribeSync("foo") + if err != nil { + t.Fatal(err) + } + nc.Publish("foo", []byte("hello world")) + nc.Flush() + + _, err = sub.NextMsg(1 * time.Second) + if err != nil { + t.Fatal(err) + } + nc.Close() +} + +func testOCSPResponderFailing(t *testing.T, method string) { + const ( + caCert = "configs/certs/ocsp/ca-cert.pem" + caKey = "configs/certs/ocsp/ca-key.pem" + serverCert = "configs/certs/ocsp/server-cert.pem" + serverKey = "configs/certs/ocsp/server-key.pem" + ) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ocspr := NewOCSPResponderPreferringHTTPMethod(t, caCert, caKey, method) + defer ocspr.Shutdown(ctx) + addr := fmt.Sprintf("http://%s", ocspr.Addr) + SetOCSPStatus(t, addr, serverCert, ocsp.Good) + + opts := server.Options{} + opts.Host = "127.0.0.1" + opts.NoLog = true + opts.NoSigs = true + opts.MaxControlLine = 4096 + opts.Port = -1 + opts.TLSCert = serverCert + opts.TLSKey = serverKey + opts.TLSCaCert = caCert + opts.TLSTimeout = 5 + tcOpts := &server.TLSConfigOpts{ + CertFile: opts.TLSCert, + KeyFile: opts.TLSKey, + CaFile: opts.TLSCaCert, + Timeout: opts.TLSTimeout, + } + + tlsConf, err := server.GenTLSConfig(tcOpts) + if err != nil { + t.Fatal(err) + } + opts.TLSConfig = tlsConf + + opts.OCSPConfig = &server.OCSPConfig{ + Mode: server.OCSPModeAlways, + OverrideURLs: []string{addr}, + } + expected := "bad OCSP status update for certificate at 'configs/certs/ocsp/server-cert.pem': " + expected += "exhausted ocsp servers: non-ok http status on POST request (reqlen=68): 400\nnon-ok http status on GET request (reqlen=92): 400" + _, err = server.NewServer(&opts) + if err == nil { + t.Error("Unexpected success setting up server") + } else if err.Error() != expected { + t.Errorf("Expected %q, got: %q", expected, err.Error()) + } +} diff --git a/test/operator_test.go b/test/operator_test.go new file mode 100644 index 0000000..b621b5a --- /dev/null +++ b/test/operator_test.go @@ -0,0 +1,684 @@ +// Copyright 2018-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "bytes" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/nats-io/jwt/v2" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nkeys" +) + +const ( + testOpConfig = "./configs/operator.conf" + testOpInlineConfig = "./configs/operator_inline.conf" +) + +// This matches ./configs/nkeys_jwts/test.seed +// Test operator seed. +var oSeed = []byte("SOAFYNORQLQFJYBYNUGC5D7SH2MXMUX5BFEWWGHN3EK4VGG5TPT5DZP7QU") + +// This is a signing key seed. +var skSeed = []byte("SOAEL3NFOTU6YK3DBTEKQYZ2C5IWSVZWWZCQDASBUOHJKBFLVANK27JMMQ") + +func checkKeys(t *testing.T, opts *server.Options, opc *jwt.OperatorClaims, expected int) { + // We should have filled in the TrustedKeys here. + if len(opts.TrustedKeys) != expected { + t.Fatalf("Should have %d trusted keys, got %d", expected, len(opts.TrustedKeys)) + } + // Check that we properly placed all keys from the opc into TrustedKeys + chkMember := func(s string) { + for _, c := range opts.TrustedKeys { + if s == c { + return + } + } + t.Fatalf("Expected %q to be in TrustedKeys", s) + } + chkMember(opc.Issuer) + for _, sk := range opc.SigningKeys { + chkMember(sk) + } +} + +// This will test that we enforce certain restrictions when you use trusted operators. +// Like auth is always true, can't define accounts or users, required to define an account resolver, etc. +func TestOperatorRestrictions(t *testing.T) { + opts, err := server.ProcessConfigFile(testOpConfig) + if err != nil { + t.Fatalf("Error processing config file: %v", err) + } + opts.NoSigs = true + if _, err := server.NewServer(opts); err != nil { + t.Fatalf("Expected to create a server successfully") + } + // TrustedKeys get defined when processing from above, trying again with + // same opts should not work. + if _, err := server.NewServer(opts); err == nil { + t.Fatalf("Expected an error with TrustedKeys defined") + } + // Must wipe and rebuild to succeed. + wipeOpts := func() { + opts.TrustedKeys = nil + opts.Accounts = nil + opts.Users = nil + opts.Nkeys = nil + } + + wipeOpts() + opts.Accounts = []*server.Account{{Name: "TEST"}} + if _, err := server.NewServer(opts); err == nil { + t.Fatalf("Expected an error with Accounts defined") + } + wipeOpts() + opts.Users = []*server.User{{Username: "TEST"}} + if _, err := server.NewServer(opts); err == nil { + t.Fatalf("Expected an error with Users defined") + } + wipeOpts() + opts.Nkeys = []*server.NkeyUser{{Nkey: "TEST"}} + if _, err := server.NewServer(opts); err == nil { + t.Fatalf("Expected an error with Nkey Users defined") + } + wipeOpts() + + opts.AccountResolver = nil + if _, err := server.NewServer(opts); err == nil { + t.Fatalf("Expected an error without an AccountResolver defined") + } +} + +func TestOperatorConfig(t *testing.T) { + opts, err := server.ProcessConfigFile(testOpConfig) + if err != nil { + t.Fatalf("Error processing config file: %v", err) + } + opts.NoSigs = true + // Check we have the TrustedOperators + if len(opts.TrustedOperators) != 1 { + t.Fatalf("Expected to load the operator") + } + _, err = server.NewServer(opts) + if err != nil { + t.Fatalf("Expected to create a server: %v", err) + } + // We should have filled in the public TrustedKeys here. + // Our master public key (issuer) plus the signing keys (3). + checkKeys(t, opts, opts.TrustedOperators[0], 4) +} + +func TestOperatorConfigInline(t *testing.T) { + opts, err := server.ProcessConfigFile(testOpInlineConfig) + if err != nil { + t.Fatalf("Error processing config file: %v", err) + } + opts.NoSigs = true + // Check we have the TrustedOperators + if len(opts.TrustedOperators) != 1 { + t.Fatalf("Expected to load the operator") + } + _, err = server.NewServer(opts) + if err != nil { + t.Fatalf("Expected to create a server: %v", err) + } + // We should have filled in the public TrustedKeys here. + // Our master public key (issuer) plus the signing keys (3). + checkKeys(t, opts, opts.TrustedOperators[0], 4) +} + +func runOperatorServer(t *testing.T) (*server.Server, *server.Options) { + return RunServerWithConfig(testOpConfig) +} + +func publicKeyFromKeyPair(t *testing.T, pair nkeys.KeyPair) (pkey string) { + var err error + if pkey, err = pair.PublicKey(); err != nil { + t.Fatalf("Expected no error %v", err) + } + return +} + +func createAccountForOperatorKey(t *testing.T, s *server.Server, seed []byte) nkeys.KeyPair { + t.Helper() + okp, _ := nkeys.FromSeed(seed) + akp, _ := nkeys.CreateAccount() + pub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(pub) + jwt, _ := nac.Encode(okp) + if err := s.AccountResolver().Store(pub, jwt); err != nil { + t.Fatalf("Account Resolver returned an error: %v", err) + } + return akp +} + +func createAccount(t *testing.T, s *server.Server) (acc *server.Account, akp nkeys.KeyPair) { + t.Helper() + akp = createAccountForOperatorKey(t, s, oSeed) + if pub, err := akp.PublicKey(); err != nil { + t.Fatalf("Expected this to pass, got %v", err) + } else if acc, err = s.LookupAccount(pub); err != nil { + t.Fatalf("Error looking up account: %v", err) + } + return acc, akp +} + +func createUserCreds(t *testing.T, s *server.Server, akp nkeys.KeyPair) nats.Option { + t.Helper() + opt, _ := createUserCredsOption(t, s, akp) + return opt +} + +func createUserCredsOption(t *testing.T, s *server.Server, akp nkeys.KeyPair) (nats.Option, string) { + t.Helper() + kp, _ := nkeys.CreateUser() + pub, _ := kp.PublicKey() + nuc := jwt.NewUserClaims(pub) + ujwt, err := nuc.Encode(akp) + if err != nil { + t.Fatalf("Error generating user JWT: %v", err) + } + userCB := func() (string, error) { + return ujwt, nil + } + sigCB := func(nonce []byte) ([]byte, error) { + sig, _ := kp.Sign(nonce) + return sig, nil + } + return nats.UserJWT(userCB, sigCB), pub +} + +func TestOperatorServer(t *testing.T) { + s, opts := runOperatorServer(t) + defer s.Shutdown() + + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + if _, err := nats.Connect(url); err == nil { + t.Fatalf("Expected to fail with no credentials") + } + + _, akp := createAccount(t, s) + nc, err := nats.Connect(url, createUserCreds(t, s, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + nc.Close() + + // Now create an account from another operator, this should fail. + okp, _ := nkeys.CreateOperator() + seed, _ := okp.Seed() + akp = createAccountForOperatorKey(t, s, seed) + _, err = nats.Connect(url, createUserCreds(t, s, akp)) + if err == nil { + t.Fatalf("Expected error on connect") + } + // The account should not be in memory either + if v, err := s.LookupAccount(publicKeyFromKeyPair(t, akp)); err == nil { + t.Fatalf("Expected account to NOT be in memory: %v", v) + } +} + +func TestOperatorSystemAccount(t *testing.T) { + s, _ := runOperatorServer(t) + defer s.Shutdown() + + // Create an account from another operator, this should fail if used as a system account. + okp, _ := nkeys.CreateOperator() + seed, _ := okp.Seed() + akp := createAccountForOperatorKey(t, s, seed) + if err := s.SetSystemAccount(publicKeyFromKeyPair(t, akp)); err == nil { + t.Fatalf("Expected this to fail") + } + if acc := s.SystemAccount(); acc != nil { + t.Fatalf("Expected no account to be set for system account") + } + + acc, _ := createAccount(t, s) + if err := s.SetSystemAccount(acc.Name); err != nil { + t.Fatalf("Expected this succeed, got %v", err) + } + if sysAcc := s.SystemAccount(); sysAcc != acc { + t.Fatalf("Did not get matching account for system account") + } +} + +func TestOperatorSigningKeys(t *testing.T) { + s, opts := runOperatorServer(t) + defer s.Shutdown() + + // Create an account with a signing key, not the master key. + akp := createAccountForOperatorKey(t, s, skSeed) + // Make sure we can set system account. + if err := s.SetSystemAccount(publicKeyFromKeyPair(t, akp)); err != nil { + t.Fatalf("Expected this succeed, got %v", err) + } + + // Make sure we can create users with it too. + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(url, createUserCreds(t, s, akp)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + nc.Close() +} + +func TestOperatorMemResolverPreload(t *testing.T) { + s, opts := RunServerWithConfig("./configs/resolver_preload.conf") + defer s.Shutdown() + + // Make sure we can look up the account. + acc, _ := s.LookupAccount("ADM2CIIL3RWXBA6T2HW3FODNCQQOUJEHHQD6FKCPVAMHDNTTSMO73ROX") + if acc == nil { + t.Fatalf("Expected to properly lookup account") + } + sacc := s.SystemAccount() + if sacc == nil { + t.Fatalf("Expected to have system account registered") + } + if sacc.Name != opts.SystemAccount { + t.Fatalf("System account does not match, wanted %q, got %q", opts.SystemAccount, sacc.Name) + } +} + +func TestOperatorConfigReloadDoesntKillNonce(t *testing.T) { + s, _ := runOperatorServer(t) + defer s.Shutdown() + + if !s.NonceRequired() { + t.Fatalf("Error nonce should be required") + } + + if err := s.Reload(); err != nil { + t.Fatalf("Error on reload: %v", err) + } + + if !s.NonceRequired() { + t.Fatalf("Error nonce should still be required after reload") + } +} + +func createAccountForConfig(t *testing.T) (string, nkeys.KeyPair) { + t.Helper() + okp, _ := nkeys.FromSeed(oSeed) + akp, _ := nkeys.CreateAccount() + pub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(pub) + jwt, _ := nac.Encode(okp) + return jwt, akp +} + +func TestReloadDoesNotWipeAccountsWithOperatorMode(t *testing.T) { + // We will run an operator mode server that forms a cluster. We will + // make sure that a reload does not wipe account information. + // We will force reload of auth by changing cluster auth timeout. + + // Create two accounts, system and normal account. + sysJWT, sysKP := createAccountForConfig(t) + sysPub, _ := sysKP.PublicKey() + + accJWT, accKP := createAccountForConfig(t) + accPub, _ := accKP.PublicKey() + + cf := ` + listen: 127.0.0.1:-1 + cluster { + name: "A" + listen: 127.0.0.1:-1 + pool_size: -1 + compression: "disabled" + authorization { + timeout: 2.2 + } %s + } + + operator = "./configs/nkeys/op.jwt" + system_account = "%s" + + resolver = MEMORY + resolver_preload = { + %s : "%s" + %s : "%s" + } + ` + contents := strings.Replace(fmt.Sprintf(cf, "", sysPub, sysPub, sysJWT, accPub, accJWT), "\n\t", "\n", -1) + conf := createConfFile(t, []byte(contents)) + + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + // Create a new server and route to main one. + routeStr := fmt.Sprintf("\n\t\troutes = [nats-route://%s:%d]", opts.Cluster.Host, opts.Cluster.Port) + contents2 := strings.Replace(fmt.Sprintf(cf, routeStr, sysPub, sysPub, sysJWT, accPub, accJWT), "\n\t", "\n", -1) + + conf2 := createConfFile(t, []byte(contents2)) + + s2, opts2 := RunServerWithConfig(conf2) + defer s2.Shutdown() + + checkClusterFormed(t, s, s2) + + // Create a client on the first server and subscribe. + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(url, createUserCreds(t, s, accKP)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + ch := make(chan bool) + nc.Subscribe("foo", func(m *nats.Msg) { ch <- true }) + nc.Flush() + + // Use this to check for message. + checkForMsg := func() { + t.Helper() + select { + case <-ch: + case <-time.After(2 * time.Second): + t.Fatal("Timeout waiting for message across route") + } + } + + // Wait for "foo" interest to be propagated to s2's account `accPub` + checkSubInterest(t, s2, accPub, "foo", 2*time.Second) + + // Create second client and send message from this one. Interest should be here. + url2 := fmt.Sprintf("nats://%s:%d/", opts2.Host, opts2.Port) + nc2, err := nats.Connect(url2, createUserCreds(t, s2, accKP)) + if err != nil { + t.Fatalf("Error creating client: %v\n", err) + } + defer nc2.Close() + + // Check that we can send messages. + nc2.Publish("foo", nil) + checkForMsg() + + // Now shutdown nc2 and srvA. + nc2.Close() + s2.Shutdown() + + // Now change config and do reload which will do an auth change. + b, err := os.ReadFile(conf) + if err != nil { + t.Fatal(err) + } + newConf := bytes.Replace(b, []byte("2.2"), []byte("3.3"), 1) + err = os.WriteFile(conf, newConf, 0644) + if err != nil { + t.Fatal(err) + } + + // This will cause reloadAuthorization to kick in and reprocess accounts. + s.Reload() + + s2, opts2 = RunServerWithConfig(conf2) + defer s2.Shutdown() + + checkClusterFormed(t, s, s2) + + checkSubInterest(t, s2, accPub, "foo", 2*time.Second) + + // Reconnect and make sure this works. If accounts blown away this will fail. + url2 = fmt.Sprintf("nats://%s:%d/", opts2.Host, opts2.Port) + nc2, err = nats.Connect(url2, createUserCreds(t, s2, accKP)) + if err != nil { + t.Fatalf("Error creating client: %v\n", err) + } + defer nc2.Close() + + // Check that we can send messages. + nc2.Publish("foo", nil) + checkForMsg() +} + +func TestReloadDoesUpdateAccountsWithMemoryResolver(t *testing.T) { + // We will run an operator mode server with a memory resolver. + // Reloading should behave similar to configured accounts. + + // Create two accounts, system and normal account. + sysJWT, sysKP := createAccountForConfig(t) + sysPub, _ := sysKP.PublicKey() + + accJWT, accKP := createAccountForConfig(t) + accPub, _ := accKP.PublicKey() + + cf := ` + listen: 127.0.0.1:-1 + cluster { + name: "A" + listen: 127.0.0.1:-1 + pool_size: -1 + authorization { + timeout: 2.2 + } %s + } + + operator = "./configs/nkeys/op.jwt" + system_account = "%s" + + resolver = MEMORY + resolver_preload = { + %s : "%s" + %s : "%s" + } + ` + contents := strings.Replace(fmt.Sprintf(cf, "", sysPub, sysPub, sysJWT, accPub, accJWT), "\n\t", "\n", -1) + conf := createConfFile(t, []byte(contents)) + + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + // Create a client on the first server and subscribe. + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(url, createUserCreds(t, s, accKP)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + asyncErr := make(chan error, 1) + nc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { + asyncErr <- err + }) + defer nc.Close() + + nc.Subscribe("foo", func(m *nats.Msg) {}) + nc.Flush() + + // Now update and remove normal account and make sure we get disconnected. + accJWT2, accKP2 := createAccountForConfig(t) + accPub2, _ := accKP2.PublicKey() + contents = strings.Replace(fmt.Sprintf(cf, "", sysPub, sysPub, sysJWT, accPub2, accJWT2), "\n\t", "\n", -1) + err = os.WriteFile(conf, []byte(contents), 0644) + if err != nil { + t.Fatal(err) + } + + // This will cause reloadAuthorization to kick in and reprocess accounts. + s.Reload() + + select { + case err := <-asyncErr: + if err != nats.ErrAuthorization { + t.Fatalf("Expected ErrAuthorization, got %v", err) + } + case <-time.After(2 * time.Second): + // Give it up to 2 sec. + t.Fatal("Expected connection to be disconnected") + } + + // Make sure we can lool up new account and not old one. + if _, err := s.LookupAccount(accPub2); err != nil { + t.Fatalf("Error looking up account: %v", err) + } + + if _, err := s.LookupAccount(accPub); err == nil { + t.Fatalf("Expected error looking up old account") + } +} + +func TestReloadFailsWithBadAccountsWithMemoryResolver(t *testing.T) { + // Create two accounts, system and normal account. + sysJWT, sysKP := createAccountForConfig(t) + sysPub, _ := sysKP.PublicKey() + + // Create an expired account by hand here. We want to make sure we start up correctly + // with expired or otherwise accounts with validation issues. + okp, _ := nkeys.FromSeed(oSeed) + + akp, _ := nkeys.CreateAccount() + apub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(apub) + nac.IssuedAt = time.Now().Add(-10 * time.Second).Unix() + nac.Expires = time.Now().Add(-2 * time.Second).Unix() + ajwt, err := nac.Encode(okp) + if err != nil { + t.Fatalf("Error generating account JWT: %v", err) + } + + cf := ` + listen: 127.0.0.1:-1 + cluster { + name: "A" + listen: 127.0.0.1:-1 + pool_size: -1 + authorization { + timeout: 2.2 + } %s + } + + operator = "./configs/nkeys/op.jwt" + system_account = "%s" + + resolver = MEMORY + resolver_preload = { + %s : "%s" + %s : "%s" + } + ` + contents := strings.Replace(fmt.Sprintf(cf, "", sysPub, sysPub, sysJWT, apub, ajwt), "\n\t", "\n", -1) + conf := createConfFile(t, []byte(contents)) + + s, _ := RunServerWithConfig(conf) + defer s.Shutdown() + + // Now add in bogus account for second item and make sure reload fails. + contents = strings.Replace(fmt.Sprintf(cf, "", sysPub, sysPub, sysJWT, "foo", "bar"), "\n\t", "\n", -1) + err = os.WriteFile(conf, []byte(contents), 0644) + if err != nil { + t.Fatal(err) + } + + if err := s.Reload(); err == nil { + t.Fatalf("Expected fatal error with bad account on reload") + } + + // Put it back with a normal account and reload should succeed. + accJWT, accKP := createAccountForConfig(t) + accPub, _ := accKP.PublicKey() + + contents = strings.Replace(fmt.Sprintf(cf, "", sysPub, sysPub, sysJWT, accPub, accJWT), "\n\t", "\n", -1) + err = os.WriteFile(conf, []byte(contents), 0644) + if err != nil { + t.Fatal(err) + } + if err := s.Reload(); err != nil { + t.Fatalf("Got unexpected error on reload: %v", err) + } +} + +func TestConnsRequestDoesNotLoadAccountCheckingConnLimits(t *testing.T) { + // Create two accounts, system and normal account. + sysJWT, sysKP := createAccountForConfig(t) + sysPub, _ := sysKP.PublicKey() + + // Do this account by hand to add in connection limits + okp, _ := nkeys.FromSeed(oSeed) + accKP, _ := nkeys.CreateAccount() + accPub, _ := accKP.PublicKey() + nac := jwt.NewAccountClaims(accPub) + nac.Limits.Conn = 10 + accJWT, _ := nac.Encode(okp) + + cf := ` + listen: 127.0.0.1:-1 + cluster { + name: "A" + listen: 127.0.0.1:-1 + authorization { + timeout: 2.2 + } %s + } + + operator = "./configs/nkeys/op.jwt" + system_account = "%s" + + resolver = MEMORY + resolver_preload = { + %s : "%s" + %s : "%s" + } + ` + contents := strings.Replace(fmt.Sprintf(cf, "", sysPub, sysPub, sysJWT, accPub, accJWT), "\n\t", "\n", -1) + conf := createConfFile(t, []byte(contents)) + + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + // Create a new server and route to main one. + routeStr := fmt.Sprintf("\n\t\troutes = [nats-route://%s:%d]", opts.Cluster.Host, opts.Cluster.Port) + contents2 := strings.Replace(fmt.Sprintf(cf, routeStr, sysPub, sysPub, sysJWT, accPub, accJWT), "\n\t", "\n", -1) + + conf2 := createConfFile(t, []byte(contents2)) + + s2, _ := RunServerWithConfig(conf2) + defer s2.Shutdown() + + checkClusterFormed(t, s, s2) + + // Make sure that we do not have the account loaded. + // Just SYS and $G + if nla := s.NumLoadedAccounts(); nla != 2 { + t.Fatalf("Expected only 2 loaded accounts, got %d", nla) + } + if nla := s2.NumLoadedAccounts(); nla != 2 { + t.Fatalf("Expected only 2 loaded accounts, got %d", nla) + } + + // Now connect to first server on accPub. + nc, err := nats.Connect(s.ClientURL(), createUserCreds(t, s, accKP)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // Just wait for the request for connections to move to S2 and cause a fetch. + // This is what we want to fix. + time.Sleep(100 * time.Millisecond) + + // We should have 3 here for sure. + if nla := s.NumLoadedAccounts(); nla != 3 { + t.Fatalf("Expected 3 loaded accounts, got %d", nla) + } + + // Now make sure that we still only have 2 loaded accounts on server 2. + if nla := s2.NumLoadedAccounts(); nla != 2 { + t.Fatalf("Expected only 2 loaded accounts, got %d", nla) + } +} diff --git a/test/opts_test.go b/test/opts_test.go new file mode 100644 index 0000000..111610d --- /dev/null +++ b/test/opts_test.go @@ -0,0 +1,45 @@ +// Copyright 2015-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "testing" +) + +func TestServerConfig(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/override.conf") + defer srv.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + sinfo := checkInfoMsg(t, c) + if sinfo.MaxPayload != opts.MaxPayload { + t.Fatalf("Expected max_payload from server, got %d vs %d", + opts.MaxPayload, sinfo.MaxPayload) + } +} + +func TestTLSConfig(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/tls.conf") + defer srv.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + sinfo := checkInfoMsg(t, c) + if !sinfo.TLSRequired { + t.Fatal("Expected TLSRequired to be true when configured") + } +} diff --git a/test/pedantic_test.go b/test/pedantic_test.go new file mode 100644 index 0000000..a289b62 --- /dev/null +++ b/test/pedantic_test.go @@ -0,0 +1,109 @@ +// Copyright 2012-2019 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "testing" + + "github.com/nats-io/nats-server/v2/server" +) + +func runPedanticServer() *server.Server { + opts := DefaultTestOptions + + opts.NoLog = false + opts.Trace = true + + opts.Port = PROTO_TEST_PORT + return RunServer(&opts) +} + +func TestPedanticSub(t *testing.T) { + s := runPedanticServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) + defer c.Close() + + send := sendCommand(t, c) + expect := expectCommand(t, c) + doConnect(t, c, false, true, false) + + // Ping should still be same + send("PING\r\n") + expect(pongRe) + + // Test malformed subjects for SUB + // Sub can contain wildcards, but + // subject must still be legit. + + // Empty terminal token + send("SUB foo. 1\r\n") + expect(errRe) + + // Empty beginning token + send("SUB .foo. 1\r\n") + expect(errRe) + + // Empty middle token + send("SUB foo..bar 1\r\n") + expect(errRe) + + // Bad non-terminal FWC + send("SUB foo.>.bar 1\r\n") + buf := expect(errRe) + + // Check that itr is 'Invalid Subject' + matches := errRe.FindAllSubmatch(buf, -1) + if len(matches) != 1 { + t.Fatal("Wanted one overall match") + } + if string(matches[0][1]) != "'Invalid Subject'" { + t.Fatalf("Expected 'Invalid Subject', got %s", string(matches[0][1])) + } +} + +func TestPedanticPub(t *testing.T) { + s := runPedanticServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) + defer c.Close() + + send := sendCommand(t, c) + expect := expectCommand(t, c) + doConnect(t, c, false, true, false) + + // Ping should still be same + send("PING\r\n") + expect(pongRe) + + // Test malformed subjects for PUB + // PUB subjects can not have wildcards + // This will error in pedantic mode + send("PUB foo.* 2\r\nok\r\n") + expect(errRe) + + send("PUB foo.> 2\r\nok\r\n") + expect(errRe) + + send("PUB foo. 2\r\nok\r\n") + expect(errRe) + + send("PUB .foo 2\r\nok\r\n") + expect(errRe) + + send("PUB foo..* 2\r\nok\r\n") + expect(errRe) +} diff --git a/test/pid_test.go b/test/pid_test.go new file mode 100644 index 0000000..4c53095 --- /dev/null +++ b/test/pid_test.go @@ -0,0 +1,45 @@ +// Copyright 2012-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "os" + "testing" +) + +func TestPidFile(t *testing.T) { + opts := DefaultTestOptions + + file := createTempFile(t, "nats-server:pid_") + file.Close() + opts.PidFile = file.Name() + + s := RunServer(&opts) + s.Shutdown() + + buf, err := os.ReadFile(opts.PidFile) + if err != nil { + t.Fatalf("Could not read pid_file: %v", err) + } + if len(buf) <= 0 { + t.Fatal("Expected a non-zero length pid_file") + } + + pid := 0 + fmt.Sscanf(string(buf), "%d", &pid) + if pid != os.Getpid() { + t.Fatalf("Expected pid to be %d, got %d\n", os.Getpid(), pid) + } +} diff --git a/test/ping_test.go b/test/ping_test.go new file mode 100644 index 0000000..ca19af5 --- /dev/null +++ b/test/ping_test.go @@ -0,0 +1,231 @@ +// Copyright 2012-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "crypto/tls" + "fmt" + "net" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" +) + +const ( + PING_TEST_PORT = 9972 + PING_INTERVAL = 50 * time.Millisecond + PING_MAX = 2 +) + +func runPingServer() *server.Server { + opts := DefaultTestOptions + opts.Port = PING_TEST_PORT + opts.PingInterval = PING_INTERVAL + opts.MaxPingsOut = PING_MAX + return RunServer(&opts) +} + +func TestPingSentToTLSConnection(t *testing.T) { + opts := DefaultTestOptions + opts.Port = PING_TEST_PORT + opts.PingInterval = PING_INTERVAL + opts.MaxPingsOut = PING_MAX + opts.TLSCert = "configs/certs/server-cert.pem" + opts.TLSKey = "configs/certs/server-key.pem" + opts.TLSCaCert = "configs/certs/ca.pem" + + tc := server.TLSConfigOpts{} + tc.CertFile = opts.TLSCert + tc.KeyFile = opts.TLSKey + tc.CaFile = opts.TLSCaCert + + opts.TLSConfig, _ = server.GenTLSConfig(&tc) + opts.TLSTimeout = 5 + s := RunServer(&opts) + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PING_TEST_PORT) + defer c.Close() + + checkInfoMsg(t, c) + c = tls.Client(c, &tls.Config{InsecureSkipVerify: true}) + tlsConn := c.(*tls.Conn) + tlsConn.Handshake() + + cs := fmt.Sprintf("CONNECT {\"verbose\":%v,\"pedantic\":%v,\"tls_required\":%v}\r\n", false, false, true) + sendProto(t, c, cs) + + expect := expectCommand(t, c) + + // Expect the max to be delivered correctly.. + for i := 0; i < PING_MAX; i++ { + time.Sleep(PING_INTERVAL / 2) + expect(pingRe) + } + + // We should get an error from the server + time.Sleep(PING_INTERVAL) + expect(errRe) + + // Server should close the connection at this point.. + time.Sleep(PING_INTERVAL) + c.SetWriteDeadline(time.Now().Add(PING_INTERVAL)) + + var err error + for { + _, err = c.Write([]byte("PING\r\n")) + if err != nil { + break + } + } + c.SetWriteDeadline(time.Time{}) + + if err == nil { + t.Fatal("No error: Expected to have connection closed") + } + if ne, ok := err.(net.Error); ok && ne.Timeout() { + t.Fatal("timeout: Expected to have connection closed") + } +} + +func TestPingInterval(t *testing.T) { + s := runPingServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PING_TEST_PORT) + defer c.Close() + + doConnect(t, c, false, false, false) + + expect := expectCommand(t, c) + + // Expect the max to be delivered correctly.. + for i := 0; i < PING_MAX; i++ { + time.Sleep(PING_INTERVAL / 2) + expect(pingRe) + } + + // We should get an error from the server + time.Sleep(PING_INTERVAL) + expect(errRe) + + // Server should close the connection at this point.. + time.Sleep(PING_INTERVAL) + c.SetWriteDeadline(time.Now().Add(PING_INTERVAL)) + + var err error + for { + _, err = c.Write([]byte("PING\r\n")) + if err != nil { + break + } + } + c.SetWriteDeadline(time.Time{}) + + if err == nil { + t.Fatal("No error: Expected to have connection closed") + } + if ne, ok := err.(net.Error); ok && ne.Timeout() { + t.Fatal("timeout: Expected to have connection closed") + } +} + +func TestUnpromptedPong(t *testing.T) { + s := runPingServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PING_TEST_PORT) + defer c.Close() + + doConnect(t, c, false, false, false) + + expect := expectCommand(t, c) + + // Send lots of PONGs in a row... + for i := 0; i < 100; i++ { + c.Write([]byte("PONG\r\n")) + } + + // The server should still send the max number of PINGs and then + // close the connection. + for i := 0; i < PING_MAX; i++ { + time.Sleep(PING_INTERVAL / 2) + expect(pingRe) + } + + // We should get an error from the server + time.Sleep(PING_INTERVAL) + expect(errRe) + + // Server should close the connection at this point.. + c.SetWriteDeadline(time.Now().Add(PING_INTERVAL)) + var err error + for { + _, err = c.Write([]byte("PING\r\n")) + if err != nil { + break + } + } + c.SetWriteDeadline(time.Time{}) + + if err == nil { + t.Fatal("No error: Expected to have connection closed") + } + if ne, ok := err.(net.Error); ok && ne.Timeout() { + t.Fatal("timeout: Expected to have connection closed") + } +} + +func TestPingSuppresion(t *testing.T) { + pingInterval := 100 * time.Millisecond + highWater := 130 * time.Millisecond + opts := DefaultTestOptions + opts.Port = PING_TEST_PORT + opts.PingInterval = pingInterval + opts.DisableShortFirstPing = true + + s := RunServer(&opts) + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PING_TEST_PORT) + defer c.Close() + + connectTime := time.Now() + + send, expect := setupConn(t, c) + + expect(pingRe) + pingTime := time.Since(connectTime) + send("PONG\r\n") + + // Should be > 100 but less then 120(ish) + if pingTime < pingInterval { + t.Fatalf("pingTime too low: %v", pingTime) + } + // +5 is just for fudging in case things are slow in the testing system. + if pingTime > highWater { + t.Fatalf("pingTime too high: %v", pingTime) + } + + time.Sleep(pingInterval / 2) + + // Sending a PING should suppress. + send("PING\r\n") + expect(pongRe) + + // This will wait for the time period where a PING should have fired + // and been delivered. We expect nothing here since it should be suppressed. + expectNothingTimeout(t, c, time.Now().Add(100*time.Millisecond)) +} diff --git a/test/port_test.go b/test/port_test.go new file mode 100644 index 0000000..32df849 --- /dev/null +++ b/test/port_test.go @@ -0,0 +1,52 @@ +// Copyright 2014-2019 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "net" + "strconv" + "testing" + + "github.com/nats-io/nats-server/v2/server" +) + +func TestResolveRandomPort(t *testing.T) { + opts := &server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, NoSigs: true} + s := RunServer(opts) + defer s.Shutdown() + + addr := s.Addr() + _, port, err := net.SplitHostPort(addr.String()) + if err != nil { + t.Fatalf("Expected no error: Got %v\n", err) + } + + portNum, err := strconv.Atoi(port) + if err != nil { + t.Fatalf("Expected no error: Got %v\n", err) + } + + if portNum == server.DEFAULT_PORT { + t.Fatalf("Expected server to choose a random port\nGot: %d", server.DEFAULT_PORT) + } + + if portNum == server.RANDOM_PORT { + t.Fatalf("Expected server to choose a random port\nGot: %d", server.RANDOM_PORT) + } + + if opts.Port != portNum { + t.Fatalf("Options port (%d) should have been overridden by chosen random port (%d)", + opts.Port, portNum) + } +} diff --git a/test/ports_test.go b/test/ports_test.go new file mode 100644 index 0000000..ee4ca4a --- /dev/null +++ b/test/ports_test.go @@ -0,0 +1,207 @@ +// Copyright 2018-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" +) + +// waits until a calculated list of listeners is resolved or a timeout +func waitForFile(path string, dur time.Duration) ([]byte, error) { + end := time.Now().Add(dur) + for time.Now().Before(end) { + if _, err := os.Stat(path); os.IsNotExist(err) { + time.Sleep(25 * time.Millisecond) + continue + } else { + return os.ReadFile(path) + } + } + return nil, errors.New("Timeout") +} + +func portFile(dirname string) string { + return filepath.Join(dirname, fmt.Sprintf("%s_%d.ports", filepath.Base(os.Args[0]), os.Getpid())) +} + +func TestPortsFile(t *testing.T) { + portFileDir := t.TempDir() + + opts := DefaultTestOptions + opts.PortsFileDir = portFileDir + opts.Port = -1 + opts.HTTPPort = -1 + opts.ProfPort = -1 + opts.Cluster.Port = -1 + opts.Websocket.Port = -1 + tc := &server.TLSConfigOpts{ + CertFile: "./configs/certs/server-cert.pem", + KeyFile: "./configs/certs/server-key.pem", + } + opts.Websocket.TLSConfig, _ = server.GenTLSConfig(tc) + + s := RunServer(&opts) + // this for test cleanup in case we fail - will be ignored if server already shutdown + defer s.Shutdown() + + ports := s.PortsInfo(5 * time.Second) + + if ports == nil { + t.Fatal("services failed to start in 5 seconds") + } + + // the pid file should be + portsFile := portFile(portFileDir) + + if portsFile == "" { + t.Fatal("Expected a ports file") + } + + // try to read a file here - the file should be a json + buf, err := waitForFile(portsFile, 5*time.Second) + if err != nil { + t.Fatalf("Could not read ports file: %v", err) + } + if len(buf) <= 0 { + t.Fatal("Expected a non-zero length ports file") + } + + readPorts := server.Ports{} + json.Unmarshal(buf, &readPorts) + + if len(readPorts.Nats) == 0 || !strings.HasPrefix(readPorts.Nats[0], "nats://") { + t.Fatal("Expected at least one nats url") + } + + if len(readPorts.Monitoring) == 0 || !strings.HasPrefix(readPorts.Monitoring[0], "http://") { + t.Fatal("Expected at least one monitoring url") + } + + if len(readPorts.Cluster) == 0 || !strings.HasPrefix(readPorts.Cluster[0], "nats://") { + t.Fatal("Expected at least one cluster listen url") + } + + if len(readPorts.Profile) == 0 || !strings.HasPrefix(readPorts.Profile[0], "http://") { + t.Fatal("Expected at least one profile listen url") + } + + if len(readPorts.WebSocket) == 0 || !strings.HasPrefix(readPorts.WebSocket[0], "wss://") { + t.Fatal("Expected at least one ws listen url") + } + + // testing cleanup + s.Shutdown() + // if we called shutdown, the cleanup code should have kicked + if _, err := os.Stat(portsFile); os.IsNotExist(err) { + // good + } else { + t.Fatalf("the port file %s was not deleted", portsFile) + } +} + +// makes a temp directory with two directories 'A' and 'B' +// the location of the ports file is changed from dir A to dir B. +func TestPortsFileReload(t *testing.T) { + tempDir := t.TempDir() + + // make child temp dir A + dirA := filepath.Join(tempDir, "A") + os.MkdirAll(dirA, 0777) + + // write the config file with a reference to A + config := fmt.Sprintf(` + ports_file_dir: "%s" + port: -1 + `, dirA) + config = strings.Replace(config, "\\", "/", -1) + confPath := filepath.Join(tempDir, fmt.Sprintf("%d.conf", os.Getpid())) + if err := os.WriteFile(confPath, []byte(config), 0666); err != nil { + t.Fatalf("Error writing ports file (%s): %v", confPath, err) + } + + opts, err := server.ProcessConfigFile(confPath) + if err != nil { + t.Fatalf("Error processing the configuration: %v", err) + } + + s := RunServer(opts) + defer s.Shutdown() + + ports := s.PortsInfo(5 * time.Second) + if ports == nil { + t.Fatal("services failed to start in 5 seconds") + } + + // get the ports file path name + portsFileInA := portFile(dirA) + // the file should be in dirA + if !strings.HasPrefix(portsFileInA, dirA) { + t.Fatalf("expected ports file to be in [%s] but was in [%s]", dirA, portsFileInA) + } + // wait for it + buf, err := waitForFile(portsFileInA, 5*time.Second) + if err != nil { + t.Fatalf("Could not read ports file: %v", err) + } + if len(buf) <= 0 { + t.Fatal("Expected a non-zero length ports file") + } + + // change the configuration for the ports file to dirB + dirB := filepath.Join(tempDir, "B") + os.MkdirAll(dirB, 0777) + + config = fmt.Sprintf(` + ports_file_dir: "%s" + port: -1 + `, dirB) + config = strings.Replace(config, "\\", "/", -1) + if err := os.WriteFile(confPath, []byte(config), 0666); err != nil { + t.Fatalf("Error writing ports file (%s): %v", confPath, err) + } + + // reload the server + if err := s.Reload(); err != nil { + t.Fatalf("error reloading server: %v", err) + } + + // wait for the new file to show up + portsFileInB := portFile(dirB) + buf, err = waitForFile(portsFileInB, 5*time.Second) + if !strings.HasPrefix(portsFileInB, dirB) { + t.Fatalf("expected ports file to be in [%s] but was in [%s]", dirB, portsFileInB) + } + if err != nil { + t.Fatalf("Could not read ports file: %v", err) + } + if len(buf) <= 0 { + t.Fatal("Expected a non-zero length ports file") + } + + // the file in dirA should have deleted + if _, err := os.Stat(portsFileInA); os.IsNotExist(err) { + // good + } else { + t.Fatalf("the port file %s was not deleted", portsFileInA) + } +} diff --git a/test/proto_test.go b/test/proto_test.go new file mode 100644 index 0000000..4c0c4dd --- /dev/null +++ b/test/proto_test.go @@ -0,0 +1,319 @@ +// Copyright 2012-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "encoding/json" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" +) + +const PROTO_TEST_PORT = 9922 + +func runProtoServer() *server.Server { + opts := DefaultTestOptions + opts.Port = PROTO_TEST_PORT + opts.MaxControlLine = 256 + opts.NoSystemAccount = true + return RunServer(&opts) +} + +func TestProtoBasics(t *testing.T) { + s := runProtoServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) + defer c.Close() + + send, expect := setupConn(t, c) + expectMsgs := expectMsgsCommand(t, expect) + + // Ping + send("PING\r\n") + expect(pongRe) + + // Single Msg + send("SUB foo 1\r\nPUB foo 5\r\nhello\r\n") + matches := expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "", "5", "hello") + + // 2 Messages + send("SUB * 2\r\nPUB foo 2\r\nok\r\n") + matches = expectMsgs(2) + // Could arrive in any order + checkMsg(t, matches[0], "foo", "", "", "2", "ok") + checkMsg(t, matches[1], "foo", "", "", "2", "ok") +} + +func TestProtoErr(t *testing.T) { + s := runProtoServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) + defer c.Close() + + send, expect := setupConn(t, c) + + // Make sure we get an error on bad proto + send("ZZZ") + expect(errRe) +} + +func TestUnsubMax(t *testing.T) { + s := runProtoServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) + defer c.Close() + + send, expect := setupConn(t, c) + expectMsgs := expectMsgsCommand(t, expect) + + send("SUB foo 22\r\n") + send("UNSUB 22 2\r\n") + for i := 0; i < 100; i++ { + send("PUB foo 2\r\nok\r\n") + } + + time.Sleep(50 * time.Millisecond) + + matches := expectMsgs(2) + checkMsg(t, matches[0], "foo", "22", "", "2", "ok") + checkMsg(t, matches[1], "foo", "22", "", "2", "ok") +} + +func TestQueueSub(t *testing.T) { + s := runProtoServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) + defer c.Close() + + send, expect := setupConn(t, c) + expectMsgs := expectMsgsCommand(t, expect) + + sent := 100 + send("SUB foo qgroup1 22\r\n") + send("SUB foo qgroup1 32\r\n") + for i := 0; i < sent; i++ { + send("PUB foo 2\r\nok\r\n") + } + // Wait for responses + time.Sleep(250 * time.Millisecond) + + matches := expectMsgs(sent) + sids := make(map[string]int) + for _, m := range matches { + sids[string(m[sidIndex])]++ + } + if len(sids) != 2 { + t.Fatalf("Expected only 2 sids, got %d\n", len(sids)) + } + for k, c := range sids { + if c < 35 { + t.Fatalf("Expected ~50 (+-15) msgs for sid:'%s', got %d\n", k, c) + } + } +} + +func TestMultipleQueueSub(t *testing.T) { + s := runProtoServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) + defer c.Close() + + send, expect := setupConn(t, c) + expectMsgs := expectMsgsCommand(t, expect) + + sent := 100 + send("SUB foo g1 1\r\n") + send("SUB foo g1 2\r\n") + send("SUB foo g2 3\r\n") + send("SUB foo g2 4\r\n") + + for i := 0; i < sent; i++ { + send("PUB foo 2\r\nok\r\n") + } + // Wait for responses + time.Sleep(250 * time.Millisecond) + + matches := expectMsgs(sent * 2) + sids := make(map[string]int) + for _, m := range matches { + sids[string(m[sidIndex])]++ + } + if len(sids) != 4 { + t.Fatalf("Expected 4 sids, got %d\n", len(sids)) + } + for k, c := range sids { + if c < 35 { + t.Fatalf("Expected ~50 (+-15) msgs for '%s', got %d\n", k, c) + } + } +} + +func TestPubToArgState(t *testing.T) { + s := runProtoServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) + defer c.Close() + + send, expect := setupConn(t, c) + + send("PUBS foo 2\r\nok\r\n") + expect(errRe) +} + +func TestSubToArgState(t *testing.T) { + s := runProtoServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) + defer c.Close() + + send, expect := setupConn(t, c) + + send("SUBZZZ foo 1\r\n") + expect(errRe) +} + +// Issue #63 +func TestProtoCrash(t *testing.T) { + s := runProtoServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) + defer c.Close() + + send, expect := sendCommand(t, c), expectCommand(t, c) + + checkInfoMsg(t, c) + + send("CONNECT {\"verbose\":true,\"tls_required\":false,\"user\":\"test\",\"pedantic\":true,\"pass\":\"password\"}") + + time.Sleep(100 * time.Millisecond) + + send("\r\n") + expect(okRe) +} + +// Issue #136 +func TestDuplicateProtoSub(t *testing.T) { + s := runProtoServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) + defer c.Close() + + send, expect := setupConn(t, c) + + send("PING\r\n") + expect(pongRe) + + send("SUB foo 1\r\n") + + send("SUB foo 1\r\n") + + ns := 0 + + for i := 0; i < 5; i++ { + ns = int(s.NumSubscriptions()) + if ns == 0 { + time.Sleep(50 * time.Millisecond) + } else { + break + } + } + + if ns != 1 { + t.Fatalf("Expected 1 subscription, got %d\n", ns) + } +} + +func TestIncompletePubArg(t *testing.T) { + s := runProtoServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) + defer c.Close() + send, expect := setupConn(t, c) + + size := 10000 + goodBuf := "" + for i := 0; i < size; i++ { + goodBuf += "A" + } + goodBuf += "\r\n" + + badSize := 3371 + badBuf := "" + for i := 0; i < badSize; i++ { + badBuf += "B" + } + // Message is corrupted and since we are still reading from client, + // next PUB accidentally becomes part of the payload of the + // incomplete message thus breaking the protocol. + badBuf2 := "" + for i := 0; i < size; i++ { + badBuf2 += "C" + } + badBuf2 += "\r\n" + + pub := "PUB example 10000\r\n" + send(pub + goodBuf + pub + goodBuf + pub + badBuf + pub + badBuf2) + expect(errRe) +} + +func TestControlLineMaximums(t *testing.T) { + s := runProtoServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) + defer c.Close() + + send, expect := setupConn(t, c) + + pubTooLong := "PUB foo " + for i := 0; i < 32; i++ { + pubTooLong += "2222222222" + } + send(pubTooLong) + expect(errRe) +} + +func TestServerInfoWithClientAdvertise(t *testing.T) { + opts := DefaultTestOptions + opts.Port = PROTO_TEST_PORT + opts.ClientAdvertise = "me:1" + s := RunServer(&opts) + defer s.Shutdown() + + c := createClientConn(t, opts.Host, PROTO_TEST_PORT) + defer c.Close() + + buf := expectResult(t, c, infoRe) + js := infoRe.FindAllSubmatch(buf, 1)[0][1] + var sinfo server.Info + err := json.Unmarshal(js, &sinfo) + if err != nil { + t.Fatalf("Could not unmarshal INFO json: %v\n", err) + } + if sinfo.Host != "me" || sinfo.Port != 1 { + t.Fatalf("Expected INFO Host:Port to be me:1, got %s:%d", sinfo.Host, sinfo.Port) + } +} diff --git a/test/route_discovery_test.go b/test/route_discovery_test.go new file mode 100644 index 0000000..9cf69a0 --- /dev/null +++ b/test/route_discovery_test.go @@ -0,0 +1,682 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" +) + +func runSeedServer(t *testing.T) (*server.Server, *server.Options) { + return RunServerWithConfig("./configs/seed.conf") +} + +func runAuthSeedServer(t *testing.T) (*server.Server, *server.Options) { + return RunServerWithConfig("./configs/auth_seed.conf") +} + +func TestSeedFirstRouteInfo(t *testing.T) { + s, opts := runSeedServer(t) + defer s.Shutdown() + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + _, routeExpect := setupRoute(t, rc, opts) + buf := routeExpect(infoRe) + + info := server.Info{} + if err := json.Unmarshal(buf[4:], &info); err != nil { + t.Fatalf("Could not unmarshal route info: %v", err) + } + + if info.ID != s.ID() { + t.Fatalf("Expected seed's ID %q, got %q", s.ID(), info.ID) + } +} + +func TestSeedMultipleRouteInfo(t *testing.T) { + s, opts := runSeedServer(t) + defer s.Shutdown() + + rc1 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc1.Close() + + rc1ID := "2222" + rc1Port := 22 + rc1Host := "127.0.0.1" + + routeSend1, route1Expect := setupRouteEx(t, rc1, opts, rc1ID) + route1Expect(infoRe) + + // register ourselves via INFO + r1Info := server.Info{ID: rc1ID, Host: rc1Host, Port: rc1Port} + b, _ := json.Marshal(r1Info) + infoJSON := fmt.Sprintf(server.InfoProto, b) + routeSend1(infoJSON) + routeSend1("PING\r\n") + route1Expect(pongRe) + + rc2 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc2.Close() + + rc2ID := "2224" + rc2Port := 24 + rc2Host := "127.0.0.1" + + routeSend2, route2Expect := setupRouteEx(t, rc2, opts, rc2ID) + + hp2 := fmt.Sprintf("nats-route://%s/", net.JoinHostPort(rc2Host, strconv.Itoa(rc2Port))) + + // register ourselves via INFO + r2Info := server.Info{ID: rc2ID, Host: rc2Host, Port: rc2Port} + b, _ = json.Marshal(r2Info) + infoJSON = fmt.Sprintf(server.InfoProto, b) + routeSend2(infoJSON) + + // Now read back the second INFO route1 should receive letting + // it know about route2 + buf := route1Expect(infoRe) + + info := server.Info{} + if err := json.Unmarshal(buf[4:], &info); err != nil { + t.Fatalf("Could not unmarshal route info: %v", err) + } + + if info.ID != rc2ID { + t.Fatalf("Expected info.ID to be %q, got %q", rc2ID, info.ID) + } + if info.IP == "" { + t.Fatalf("Expected a IP for the implicit route") + } + if info.IP != hp2 { + t.Fatalf("Expected IP Host of %s, got %s\n", hp2, info.IP) + } + + route2Expect(infoRe) + routeSend2("PING\r\n") + route2Expect(pongRe) + + // Now let's do a third. + rc3 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc3.Close() + + rc3ID := "2226" + rc3Port := 26 + rc3Host := "127.0.0.1" + + routeSend3, _ := setupRouteEx(t, rc3, opts, rc3ID) + + // register ourselves via INFO + r3Info := server.Info{ID: rc3ID, Host: rc3Host, Port: rc3Port} + b, _ = json.Marshal(r3Info) + infoJSON = fmt.Sprintf(server.InfoProto, b) + routeSend3(infoJSON) + + // Now read back out the info from the seed route + buf = route1Expect(infoRe) + + info = server.Info{} + if err := json.Unmarshal(buf[4:], &info); err != nil { + t.Fatalf("Could not unmarshal route info: %v", err) + } + + if info.ID != rc3ID { + t.Fatalf("Expected info.ID to be %q, got %q", rc3ID, info.ID) + } + + // Now read back out the info from the seed route + buf = route2Expect(infoRe) + + info = server.Info{} + if err := json.Unmarshal(buf[4:], &info); err != nil { + t.Fatalf("Could not unmarshal route info: %v", err) + } + + if info.ID != rc3ID { + t.Fatalf("Expected info.ID to be %q, got %q", rc3ID, info.ID) + } +} + +func TestSeedSolicitWorks(t *testing.T) { + s1, opts := runSeedServer(t) + defer s1.Shutdown() + + // Create the routes string for others to connect to the seed. + routesStr := fmt.Sprintf("nats-route://%s:%d/", opts.Cluster.Host, opts.Cluster.Port) + + // Run Server #2 + s2Opts := nextServerOpts(opts) + s2Opts.Routes = server.RoutesFromStr(routesStr) + + s2 := RunServer(s2Opts) + defer s2.Shutdown() + + // Run Server #3 + s3Opts := nextServerOpts(s2Opts) + + s3 := RunServer(s3Opts) + defer s3.Shutdown() + + // Wait for a bit for graph to connect + time.Sleep(500 * time.Millisecond) + + // Grab Routez from monitor ports, make sure we are fully connected. + url := fmt.Sprintf("http://%s:%d/", opts.Host, opts.HTTPPort) + rz := readHTTPRoutez(t, url) + for _, route := range rz.Routes { + if route.LastActivity.IsZero() { + t.Error("Expected LastActivity to be valid\n") + } + } + ris := expectRids(t, rz, []string{s2.ID(), s3.ID()}) + if ris[s2.ID()].IsConfigured { + t.Fatalf("Expected server not to be configured\n") + } + if ris[s3.ID()].IsConfigured { + t.Fatalf("Expected server not to be configured\n") + } + + // Server 2 did solicit routes to Server 1. + url = fmt.Sprintf("http://%s:%d/", s2Opts.Host, s2Opts.HTTPPort) + rz = readHTTPRoutez(t, url) + for _, route := range rz.Routes { + if route.LastActivity.IsZero() { + t.Error("Expected LastActivity to be valid") + } + } + ris = expectRids(t, rz, []string{s1.ID(), s3.ID()}) + if !ris[s1.ID()].IsConfigured { + t.Fatalf("Expected seed server to be configured\n") + } + if ris[s3.ID()].IsConfigured { + t.Fatalf("Expected server not to be configured\n") + } + + url = fmt.Sprintf("http://%s:%d/", s3Opts.Host, s3Opts.HTTPPort) + rz = readHTTPRoutez(t, url) + for _, route := range rz.Routes { + if route.LastActivity.IsZero() { + t.Error("Expected LastActivity to be valid") + } + } + ris = expectRids(t, rz, []string{s1.ID(), s2.ID()}) + if !ris[s1.ID()].IsConfigured { + t.Fatalf("Expected seed server to be configured\n") + } + if ris[s2.ID()].IsConfigured { + t.Fatalf("Expected server not to be configured\n") + } +} + +type serverInfo struct { + server *server.Server + opts *server.Options +} + +func checkConnected(t *testing.T, servers []serverInfo, current int, oneSeed bool) error { + s := servers[current] + + // Grab Routez from monitor ports, make sure we are fully connected + url := fmt.Sprintf("http://%s:%d/", s.opts.Host, s.opts.HTTPPort) + rz := readHTTPRoutez(t, url) + total := len(servers) + var ids []string + for i := 0; i < total; i++ { + if i == current { + continue + } + ids = append(ids, servers[i].server.ID()) + } + ris, err := expectRidsNoFatal(t, true, rz, ids) + if err != nil { + return err + } + for i := 0; i < total; i++ { + if i == current { + continue + } + s := servers[i] + if current == 0 || ((oneSeed && i > 0) || (!oneSeed && (i != current-1))) { + if ris[s.server.ID()].IsConfigured { + return fmt.Errorf("Expected server %s:%d not to be configured", s.opts.Host, s.opts.Port) + } + } else if oneSeed || (i == current-1) { + if !ris[s.server.ID()].IsConfigured { + return fmt.Errorf("Expected server %s:%d to be configured", s.opts.Host, s.opts.Port) + } + } + } + return nil +} + +func TestStressSeedSolicitWorks(t *testing.T) { + s1, opts := runSeedServer(t) + defer s1.Shutdown() + + // Create the routes string for others to connect to the seed. + routesStr := fmt.Sprintf("nats-route://%s:%d/", opts.Cluster.Host, opts.Cluster.Port) + + s2Opts := nextServerOpts(opts) + s2Opts.Routes = server.RoutesFromStr(routesStr) + + s3Opts := nextServerOpts(s2Opts) + s4Opts := nextServerOpts(s3Opts) + + for i := 0; i < 10; i++ { + func() { + // Run these servers manually, because we want them to start and + // connect to s1 as fast as possible. + + s2 := server.New(s2Opts) + if s2 == nil { + panic("No NATS Server object returned.") + } + defer s2.Shutdown() + go s2.Start() + + s3 := server.New(s3Opts) + if s3 == nil { + panic("No NATS Server object returned.") + } + defer s3.Shutdown() + go s3.Start() + + s4 := server.New(s4Opts) + if s4 == nil { + panic("No NATS Server object returned.") + } + defer s4.Shutdown() + go s4.Start() + + serversInfo := []serverInfo{{s1, opts}, {s2, s2Opts}, {s3, s3Opts}, {s4, s4Opts}} + + checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { + for j := 0; j < len(serversInfo); j++ { + if err := checkConnected(t, serversInfo, j, true); err != nil { + return err + } + } + return nil + }) + }() + checkNumRoutes(t, s1, 0) + } +} + +func TestChainedSolicitWorks(t *testing.T) { + s1, opts := runSeedServer(t) + defer s1.Shutdown() + + // Create the routes string for others to connect to the seed. + routesStr := fmt.Sprintf("nats-route://%s:%d/", opts.Cluster.Host, opts.Cluster.Port) + + // Run Server #2 + s2Opts := nextServerOpts(opts) + s2Opts.Routes = server.RoutesFromStr(routesStr) + + s2 := RunServer(s2Opts) + defer s2.Shutdown() + + // Run Server #3 + s3Opts := nextServerOpts(s2Opts) + // We will have s3 connect to s2, not the seed. + routesStr = fmt.Sprintf("nats-route://%s:%d/", s2Opts.Cluster.Host, s2Opts.Cluster.Port) + s3Opts.Routes = server.RoutesFromStr(routesStr) + + s3 := RunServer(s3Opts) + defer s3.Shutdown() + + // Wait for a bit for graph to connect + time.Sleep(500 * time.Millisecond) + + // Grab Routez from monitor ports, make sure we are fully connected + url := fmt.Sprintf("http://%s:%d/", opts.Host, opts.HTTPPort) + rz := readHTTPRoutez(t, url) + ris := expectRids(t, rz, []string{s2.ID(), s3.ID()}) + if ris[s2.ID()].IsConfigured { + t.Fatalf("Expected server not to be configured\n") + } + if ris[s3.ID()].IsConfigured { + t.Fatalf("Expected server not to be configured\n") + } + + url = fmt.Sprintf("http://%s:%d/", s2Opts.Host, s2Opts.HTTPPort) + rz = readHTTPRoutez(t, url) + ris = expectRids(t, rz, []string{s1.ID(), s3.ID()}) + if !ris[s1.ID()].IsConfigured { + t.Fatalf("Expected seed server to be configured\n") + } + if ris[s3.ID()].IsConfigured { + t.Fatalf("Expected server not to be configured\n") + } + + url = fmt.Sprintf("http://%s:%d/", s3Opts.Host, s3Opts.HTTPPort) + rz = readHTTPRoutez(t, url) + ris = expectRids(t, rz, []string{s1.ID(), s2.ID()}) + if !ris[s2.ID()].IsConfigured { + t.Fatalf("Expected s2 server to be configured\n") + } + if ris[s1.ID()].IsConfigured { + t.Fatalf("Expected seed server not to be configured\n") + } +} + +func TestStressChainedSolicitWorks(t *testing.T) { + s1, opts := runSeedServer(t) + defer s1.Shutdown() + + // Create the routes string for s2 to connect to the seed + routesStr := fmt.Sprintf("nats-route://%s:%d/", opts.Cluster.Host, opts.Cluster.Port) + s2Opts := nextServerOpts(opts) + s2Opts.Routes = server.RoutesFromStr(routesStr) + + s3Opts := nextServerOpts(s2Opts) + // Create the routes string for s3 to connect to s2 + routesStr = fmt.Sprintf("nats-route://%s:%d/", s2Opts.Cluster.Host, s2Opts.Cluster.Port) + s3Opts.Routes = server.RoutesFromStr(routesStr) + + s4Opts := nextServerOpts(s3Opts) + // Create the routes string for s4 to connect to s3 + routesStr = fmt.Sprintf("nats-route://%s:%d/", s3Opts.Cluster.Host, s3Opts.Cluster.Port) + s4Opts.Routes = server.RoutesFromStr(routesStr) + + for i := 0; i < 10; i++ { + func() { + // Run these servers manually, because we want them to start and + // connect to s1 as fast as possible. + + s2 := server.New(s2Opts) + if s2 == nil { + panic("No NATS Server object returned.") + } + defer s2.Shutdown() + go s2.Start() + + s3 := server.New(s3Opts) + if s3 == nil { + panic("No NATS Server object returned.") + } + defer s3.Shutdown() + go s3.Start() + + s4 := server.New(s4Opts) + if s4 == nil { + panic("No NATS Server object returned.") + } + defer s4.Shutdown() + go s4.Start() + + serversInfo := []serverInfo{{s1, opts}, {s2, s2Opts}, {s3, s3Opts}, {s4, s4Opts}} + + checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { + for j := 0; j < len(serversInfo); j++ { + if err := checkConnected(t, serversInfo, j, false); err != nil { + return err + } + } + return nil + }) + }() + checkNumRoutes(t, s1, 0) + } +} + +func TestAuthSeedSolicitWorks(t *testing.T) { + s1, opts := runAuthSeedServer(t) + defer s1.Shutdown() + + // Create the routes string for others to connect to the seed. + routesStr := fmt.Sprintf("nats-route://%s:%s@%s:%d/", opts.Cluster.Username, opts.Cluster.Password, opts.Cluster.Host, opts.Cluster.Port) + + // Run Server #2 + s2Opts := nextServerOpts(opts) + s2Opts.Routes = server.RoutesFromStr(routesStr) + + s2 := RunServer(s2Opts) + defer s2.Shutdown() + + // Run Server #3 + s3Opts := nextServerOpts(s2Opts) + + s3 := RunServer(s3Opts) + defer s3.Shutdown() + + // Wait for a bit for graph to connect + time.Sleep(500 * time.Millisecond) + + // Grab Routez from monitor ports, make sure we are fully connected + url := fmt.Sprintf("http://%s:%d/", opts.Host, opts.HTTPPort) + rz := readHTTPRoutez(t, url) + ris := expectRids(t, rz, []string{s2.ID(), s3.ID()}) + if ris[s2.ID()].IsConfigured { + t.Fatalf("Expected server not to be configured\n") + } + if ris[s3.ID()].IsConfigured { + t.Fatalf("Expected server not to be configured\n") + } + + url = fmt.Sprintf("http://%s:%d/", s2Opts.Host, s2Opts.HTTPPort) + rz = readHTTPRoutez(t, url) + ris = expectRids(t, rz, []string{s1.ID(), s3.ID()}) + if !ris[s1.ID()].IsConfigured { + t.Fatalf("Expected seed server to be configured\n") + } + if ris[s3.ID()].IsConfigured { + t.Fatalf("Expected server not to be configured\n") + } + + url = fmt.Sprintf("http://%s:%d/", s3Opts.Host, s3Opts.HTTPPort) + rz = readHTTPRoutez(t, url) + ris = expectRids(t, rz, []string{s1.ID(), s2.ID()}) + if !ris[s1.ID()].IsConfigured { + t.Fatalf("Expected seed server to be configured\n") + } + if ris[s2.ID()].IsConfigured { + t.Fatalf("Expected server not to be configured\n") + } +} + +// Helper to check for correct route memberships +func expectRids(t *testing.T, rz *server.Routez, rids []string) map[string]*server.RouteInfo { + ri, err := expectRidsNoFatal(t, false, rz, rids) + if err != nil { + t.Fatalf("%v", err) + } + return ri +} + +func expectRidsNoFatal(t *testing.T, direct bool, rz *server.Routez, rids []string) (map[string]*server.RouteInfo, error) { + caller := 1 + if !direct { + caller++ + } + if len(rids) != rz.NumRoutes { + _, fn, line, _ := runtime.Caller(caller) + return nil, fmt.Errorf("[%s:%d] Expecting %d routes, got %d\n", fn, line, len(rids), rz.NumRoutes) + } + set := make(map[string]bool) + for _, v := range rids { + set[v] = true + } + // Make result map for additional checking + ri := make(map[string]*server.RouteInfo) + for _, r := range rz.Routes { + if !set[r.RemoteID] { + _, fn, line, _ := runtime.Caller(caller) + return nil, fmt.Errorf("[%s:%d] Route with rid %s unexpected, expected %+v\n", fn, line, r.RemoteID, rids) + } + ri[r.RemoteID] = r + } + return ri, nil +} + +// Helper to easily grab routez info. +func readHTTPRoutez(t *testing.T, url string) *server.Routez { + resetPreviousHTTPConnections() + resp, err := http.Get(url + "routez") + if err != nil { + stackFatalf(t, "Expected no error: Got %v\n", err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + stackFatalf(t, "Expected a 200 response, got %d\n", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + stackFatalf(t, "Got an error reading the body: %v\n", err) + } + r := server.Routez{} + if err := json.Unmarshal(body, &r); err != nil { + stackFatalf(t, "Got an error unmarshalling the body: %v\n", err) + } + return &r +} + +func TestSeedReturnIPInInfo(t *testing.T) { + s, opts := runSeedServer(t) + defer s.Shutdown() + + rc1 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc1.Close() + + rc1ID := "2222" + rc1Port := 22 + rc1Host := "127.0.0.1" + + routeSend1, route1Expect := setupRouteEx(t, rc1, opts, rc1ID) + route1Expect(infoRe) + + // register ourselves via INFO + r1Info := server.Info{ID: rc1ID, Host: rc1Host, Port: rc1Port} + b, _ := json.Marshal(r1Info) + infoJSON := fmt.Sprintf(server.InfoProto, b) + routeSend1(infoJSON) + routeSend1("PING\r\n") + route1Expect(pongRe) + + rc2 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc2.Close() + + rc2ID := "2224" + rc2Port := 24 + rc2Host := "127.0.0.1" + + routeSend2, _ := setupRouteEx(t, rc2, opts, rc2ID) + + // register ourselves via INFO + r2Info := server.Info{ID: rc2ID, Host: rc2Host, Port: rc2Port} + b, _ = json.Marshal(r2Info) + infoJSON = fmt.Sprintf(server.InfoProto, b) + routeSend2(infoJSON) + + // Now read info that route1 should have received from the seed + buf := route1Expect(infoRe) + + info := server.Info{} + if err := json.Unmarshal(buf[4:], &info); err != nil { + t.Fatalf("Could not unmarshal route info: %v", err) + } + + if info.IP == "" { + t.Fatal("Expected to have IP in INFO") + } + rip, _, err := net.SplitHostPort(strings.TrimPrefix(info.IP, "nats-route://")) + if err != nil { + t.Fatalf("Error parsing url: %v", err) + } + addr, ok := rc1.RemoteAddr().(*net.TCPAddr) + if !ok { + t.Fatal("Unable to get IP address from route") + } + s1 := strings.ToLower(addr.IP.String()) + s2 := strings.ToLower(rip) + if s1 != s2 { + t.Fatalf("Expected IP %s, got %s", s1, s2) + } +} + +func TestImplicitRouteRetry(t *testing.T) { + srvSeed, optsSeed := runSeedServer(t) + defer srvSeed.Shutdown() + + optsA := nextServerOpts(optsSeed) + optsA.Routes = server.RoutesFromStr(fmt.Sprintf("nats://%s:%d", optsSeed.Cluster.Host, optsSeed.Cluster.Port)) + optsA.Cluster.ConnectRetries = 5 + srvA := RunServer(optsA) + defer srvA.Shutdown() + + optsB := nextServerOpts(optsA) + rcb := createRouteConn(t, optsSeed.Cluster.Host, optsSeed.Cluster.Port) + defer rcb.Close() + rcbID := "ServerB" + routeBSend, routeBExpect := setupRouteEx(t, rcb, optsB, rcbID) + routeBExpect(infoRe) + // register ourselves via INFO + rbInfo := server.Info{ID: rcbID, Host: optsB.Cluster.Host, Port: optsB.Cluster.Port} + b, _ := json.Marshal(rbInfo) + infoJSON := fmt.Sprintf(server.InfoProto, b) + routeBSend(infoJSON) + routeBSend("PING\r\n") + routeBExpect(pongRe) + + // srvA should try to connect. Wait to make sure that it fails. + time.Sleep(1200 * time.Millisecond) + + // Setup a fake route listen for routeB + rbListen, err := net.Listen("tcp", fmt.Sprintf("%s:%d", optsB.Cluster.Host, optsB.Cluster.Port)) + if err != nil { + t.Fatalf("Error during listen: %v", err) + } + defer rbListen.Close() + c, err := rbListen.Accept() + if err != nil { + t.Fatalf("Error during accept: %v", err) + } + defer c.Close() + + br := bufio.NewReaderSize(c, 32768) + // Consume CONNECT and INFO + for i := 0; i < 2; i++ { + c.SetReadDeadline(time.Now().Add(2 * time.Second)) + buf, _, err := br.ReadLine() + c.SetReadDeadline(time.Time{}) + if err != nil { + t.Fatalf("Error reading: %v", err) + } + if i == 0 { + continue + } + buf = buf[len("INFO "):] + info := &server.Info{} + if err := json.Unmarshal(buf, info); err != nil { + t.Fatalf("Error during unmarshal: %v", err) + } + // Check INFO is from server A. + if info.ID != srvA.ID() { + t.Fatalf("Expected CONNECT from %v, got CONNECT from %v", srvA.ID(), info.ID) + } + } +} diff --git a/test/routes_test.go b/test/routes_test.go new file mode 100644 index 0000000..4cd1d83 --- /dev/null +++ b/test/routes_test.go @@ -0,0 +1,1174 @@ +// Copyright 2012-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "encoding/json" + "fmt" + "io" + "net" + "os" + "runtime" + "strconv" + "sync" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/internal/testhelper" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" +) + +const clientProtoInfo = 1 + +func runRouteServer(t *testing.T) (*server.Server, *server.Options) { + return RunServerWithConfig("./configs/cluster.conf") +} + +func runRouteServerOverrides(t *testing.T, cbo func(*server.Options), cbs func(*server.Server)) (*server.Server, *server.Options) { + return RunServerWithConfigOverrides("./configs/cluster.conf", cbo, cbs) +} + +func TestRouterListeningSocket(t *testing.T) { + s, opts := runRouteServer(t) + defer s.Shutdown() + + // Check that the cluster socket is able to be connected. + addr := fmt.Sprintf("%s:%d", opts.Cluster.Host, opts.Cluster.Port) + checkSocket(t, addr, 2*time.Second) +} + +func TestRouteGoServerShutdown(t *testing.T) { + base := runtime.NumGoroutine() + s, _ := runRouteServer(t) + s.Shutdown() + time.Sleep(50 * time.Millisecond) + delta := (runtime.NumGoroutine() - base) + if delta > 1 { + t.Fatalf("%d Go routines still exist post Shutdown()", delta) + } +} + +func TestSendRouteInfoOnConnect(t *testing.T) { + s, opts := runRouteServer(t) + defer s.Shutdown() + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RouteID" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + buf := routeExpect(infoRe) + + info := server.Info{} + if err := json.Unmarshal(buf[4:], &info); err != nil { + t.Fatalf("Could not unmarshal route info: %v", err) + } + + if !info.AuthRequired { + t.Fatal("Expected to see AuthRequired") + } + if info.Port != opts.Cluster.Port { + t.Fatalf("Received wrong information for port, expected %d, got %d", + info.Port, opts.Cluster.Port) + } + + // Need to send a different INFO than the one received, otherwise the server + // will detect as a "cycle" and close the connection. + info.ID = routeID + info.Name = "" + b, err := json.Marshal(info) + if err != nil { + t.Fatalf("Could not marshal test route info: %v", err) + } + infoJSON := fmt.Sprintf("INFO %s\r\n", b) + routeSend(infoJSON) + routeSend("PING\r\n") + routeExpect(pongRe) +} + +func TestRouteToSelf(t *testing.T) { + l := testhelper.NewDummyLogger(100) + s, opts := runRouteServerOverrides(t, nil, + func(s *server.Server) { + s.SetLogger(l, true, true) + }) + defer s.Shutdown() + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeSend, routeExpect := setupRouteEx(t, rc, opts, s.ID()) + buf := routeExpect(infoRe) + + info := server.Info{} + if err := json.Unmarshal(buf[4:], &info); err != nil { + t.Fatalf("Could not unmarshal route info: %v", err) + } + + if !info.AuthRequired { + t.Fatal("Expected to see AuthRequired") + } + if info.Port != opts.Cluster.Port { + t.Fatalf("Received wrong information for port, expected %d, got %d", + info.Port, opts.Cluster.Port) + } + + // Now send it back and that should be detected as a route to self and the + // connection closed. + routeSend(string(buf)) + routeSend("PING\r\n") + rc.SetReadDeadline(time.Now().Add(2 * time.Second)) + if _, err := rc.Read(buf); err == nil { + t.Fatal("Expected route connection to be closed") + } + // This should have been removed by removePassFromTrace(), but we also check debug logs here + l.CheckForProhibited(t, "route authorization password found", "top_secret") +} + +func TestSendRouteSubAndUnsub(t *testing.T) { + s, opts := runRouteServer(t) + defer s.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, _ := setupConn(t, c) + + // We connect to the route. + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + expectAuthRequired(t, rc) + routeSend, routeExpect := setupRouteEx(t, rc, opts, "ROUTER:xyz") + routeSend("INFO {\"server_id\":\"ROUTER:xyz\"}\r\n") + + routeSend("PING\r\n") + routeExpect(pongRe) + + // Send SUB via client connection + send("SUB foo 22\r\n") + + // Make sure the RS+ is broadcast via the route + expectResult(t, rc, rsubRe) + + // Send UNSUB via client connection + send("UNSUB 22\r\n") + + // Make sure the RS- is broadcast via the route + expectResult(t, rc, runsubRe) + + // Explicitly shutdown the server, otherwise this test would + // cause following test to fail. + s.Shutdown() +} + +func TestSendRouteSolicit(t *testing.T) { + s, opts := runRouteServer(t) + defer s.Shutdown() + + // Listen for a connection from the server on the first route. + if len(opts.Routes) <= 0 { + t.Fatalf("Need an outbound solicted route for this test") + } + rURL := opts.Routes[0] + + conn := acceptRouteConn(t, rURL.Host, server.DEFAULT_ROUTE_CONNECT) + defer conn.Close() + + // We should receive a connect message right away due to auth. + buf := expectResult(t, conn, connectRe) + + // Check INFO follows. Could be inline, with first result, if not + // check follow-on buffer. + if !infoRe.Match(buf) { + expectResult(t, conn, infoRe) + } +} + +func TestRouteForwardsMsgFromClients(t *testing.T) { + s, opts := runRouteServer(t) + defer s.Shutdown() + + client := createClientConn(t, opts.Host, opts.Port) + defer client.Close() + + clientSend, clientExpect := setupConn(t, client) + + route := acceptRouteConn(t, opts.Routes[0].Host, server.DEFAULT_ROUTE_CONNECT) + defer route.Close() + + routeSend, routeExpect := setupRoute(t, route, opts) + expectMsgs := expectMsgsCommand(t, routeExpect) + + // Eat the CONNECT and INFO protos + buf := routeExpect(connectRe) + if !infoRe.Match(buf) { + routeExpect(infoRe) + } + + // Send SUB via route connection, RS+ + routeSend("RS+ $G foo\r\nPING\r\n") + routeExpect(pongRe) + + // Send PUB via client connection + clientSend("PUB foo 2\r\nok\r\nPING\r\n") + clientExpect(pongRe) + + matches := expectMsgs(1) + checkRmsg(t, matches[0], "$G", "foo", "", "2", "ok") +} + +func TestRouteForwardsMsgToClients(t *testing.T) { + s, opts := runRouteServer(t) + defer s.Shutdown() + + client := createClientConn(t, opts.Host, opts.Port) + defer client.Close() + + clientSend, clientExpect := setupConn(t, client) + expectMsgs := expectMsgsCommand(t, clientExpect) + + route := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer route.Close() + expectAuthRequired(t, route) + routeSend, _ := setupRoute(t, route, opts) + + // Subscribe to foo + clientSend("SUB foo 1\r\nPING\r\n") + // Use ping roundtrip to make sure its processed. + clientExpect(pongRe) + + // Send RMSG proto via route connection + routeSend("RMSG $G foo 2\r\nok\r\n") + + matches := expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "", "2", "ok") +} + +func TestRouteOneHopSemantics(t *testing.T) { + s, opts := runRouteServer(t) + defer s.Shutdown() + + route := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer route.Close() + + expectAuthRequired(t, route) + routeSend, _ := setupRoute(t, route, opts) + + // Express interest on this route for foo. + routeSend("RS+ $G foo\r\n") + + // Send MSG proto via route connection + routeSend("RMSG foo 2\r\nok\r\n") + + // Make sure it does not come back! + expectNothing(t, route) +} + +func TestRouteOnlySendOnce(t *testing.T) { + s, opts := runRouteServer(t) + defer s.Shutdown() + + client := createClientConn(t, opts.Host, opts.Port) + defer client.Close() + + clientSend, clientExpect := setupConn(t, client) + + route := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer route.Close() + + expectAuthRequired(t, route) + routeSend, routeExpect := setupRoute(t, route, opts) + expectMsgs := expectMsgsCommand(t, routeExpect) + + // Express multiple interest on this route for foo. + routeSend("RS+ $G foo\r\n") + routeSend("RS+ $G foo\r\n") + routeSend("PING\r\n") + routeExpect(pongRe) + + // Send PUB via client connection + clientSend("PUB foo 2\r\nok\r\nPING\r\n") + clientExpect(pongRe) + + expectMsgs(1) + routeSend("PING\r\n") + routeExpect(pongRe) +} + +func TestRouteQueueSemantics(t *testing.T) { + s, opts := runRouteServer(t) + defer s.Shutdown() + + client := createClientConn(t, opts.Host, opts.Port) + clientSend, clientExpect := setupConn(t, client) + clientExpectMsgs := expectMsgsCommand(t, clientExpect) + + defer client.Close() + + route := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer route.Close() + + expectAuthRequired(t, route) + routeSend, routeExpect := setupRouteEx(t, route, opts, "ROUTER:xyz") + routeSend("INFO {\"server_id\":\"ROUTER:xyz\"}\r\n") + expectMsgs := expectRmsgsCommand(t, routeExpect) + + // Express multiple interest on this route for foo, queue group bar. + routeSend("RS+ $G foo bar 1\r\n") + routeSend("RS+ $G foo bar 2\r\n") + + // Use ping roundtrip to make sure its processed. + routeSend("PING\r\n") + routeExpect(pongRe) + + // Send PUB via client connection + clientSend("PUB foo 2\r\nok\r\n") + // Use ping roundtrip to make sure its processed. + clientSend("PING\r\n") + clientExpect(pongRe) + + // Only 1 + matches := expectMsgs(1) + checkRmsg(t, matches[0], "$G", "foo", "| bar", "2", "ok") + + // Add normal Interest as well to route interest. + routeSend("RS+ $G foo\r\n") + + // Use ping roundtrip to make sure its processed. + routeSend("PING\r\n") + routeExpect(pongRe) + + // Send PUB via client connection + clientSend("PUB foo 2\r\nok\r\nPING\r\n") + // Use ping roundtrip to make sure its processed. + clientExpect(pongRe) + + // Should be 1 now for everything. Always receive 1 message. + matches = expectMsgs(1) + checkRmsg(t, matches[0], "$G", "foo", "| bar", "2", "ok") + + // Now create a queue subscription for the client as well as a normal one. + clientSend("SUB foo 1\r\n") + // Use ping roundtrip to make sure its processed. + clientSend("PING\r\n") + clientExpect(pongRe) + routeExpect(rsubRe) + + clientSend("SUB foo bar 2\r\nPING\r\n") + // Use ping roundtrip to make sure its processed. + clientExpect(pongRe) + routeExpect(rsubRe) + + // Deliver a MSG from the route itself, make sure the client receives both. + routeSend("RMSG $G foo | bar 2\r\nok\r\n") + + // Use ping roundtrip to make sure its processed. + routeSend("PING\r\n") + routeExpect(pongRe) + + // Should get 2 msgs. + matches = clientExpectMsgs(2) + + // Expect first to be the normal subscriber, next will be the queue one. + checkMsg(t, matches[0], "foo", "", "", "2", "ok") +} + +func TestSolicitRouteReconnect(t *testing.T) { + l := testhelper.NewDummyLogger(100) + s, opts := runRouteServerOverrides(t, nil, + func(s *server.Server) { + s.SetLogger(l, true, true) + }) + defer s.Shutdown() + + rURL := opts.Routes[0] + + route := acceptRouteConn(t, rURL.Host, 2*server.DEFAULT_ROUTE_CONNECT) + + // Go ahead and close the Route. + route.Close() + + // We expect to get called back.. + route = acceptRouteConn(t, rURL.Host, 2*server.DEFAULT_ROUTE_CONNECT) + route.Close() + + // Now we want to check for the debug logs when it tries to reconnect + l.CheckForProhibited(t, "route authorization password found", ":bar") +} + +func TestMultipleRoutesSameId(t *testing.T) { + s, opts := runRouteServer(t) + defer s.Shutdown() + + route1 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer route1.Close() + + expectAuthRequired(t, route1) + route1Send, route1Expect := setupRouteEx(t, route1, opts, "ROUTE:2222") + + route2 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer route2.Close() + + expectAuthRequired(t, route2) + route2Send, route2Expect := setupRouteEx(t, route2, opts, "ROUTE:2222") + + // Send SUB via route connections + sub := "RS+ $G foo\r\nPING\r\n" + route1Send(sub) + route2Send(sub) + route1Expect(pongRe) + route2Expect(pongRe) + + // Make sure we do not get anything on a RMSG send to a router. + // Send RMSG proto via route connection + route1Send("RMSG $G foo 2\r\nok\r\n") + + expectNothing(t, route1) + expectNothing(t, route2) + + // Setup a client + client := createClientConn(t, opts.Host, opts.Port) + clientSend, clientExpect := setupConn(t, client) + defer client.Close() + + // Send PUB via client connection + clientSend("PUB foo 2\r\nok\r\nPING\r\n") + clientExpect(pongRe) + + // We should only receive on one route, not both. + // Check both manually. + route1.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) + buf, _ := io.ReadAll(route1) + route1.SetReadDeadline(time.Time{}) + if len(buf) <= 0 { + route2.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) + buf, _ = io.ReadAll(route2) + route2.SetReadDeadline(time.Time{}) + if len(buf) <= 0 { + t.Fatal("Expected to get one message on a route, received none.") + } + } + + matches := rmsgRe.FindAllSubmatch(buf, -1) + if len(matches) != 1 { + t.Fatalf("Expected 1 msg, got %d\n", len(matches)) + } + checkRmsg(t, matches[0], "$G", "foo", "", "2", "ok") +} + +func TestRouteResendsLocalSubsOnReconnect(t *testing.T) { + s, opts := runRouteServer(t) + defer s.Shutdown() + + client := createClientConn(t, opts.Host, opts.Port) + defer client.Close() + + clientSend, clientExpect := setupConn(t, client) + + // Setup a local subscription, make sure it reaches. + clientSend("SUB foo 1\r\n") + clientSend("PING\r\n") + clientExpect(pongRe) + + route := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer route.Close() + routeSend, routeExpect := setupRouteEx(t, route, opts, "ROUTE:1234") + + // Expect to see the local sub echoed through after we send our INFO. + time.Sleep(50 * time.Millisecond) + buf := routeExpect(infoRe) + + // Generate our own INFO so we can send one to trigger the local subs. + info := server.Info{} + if err := json.Unmarshal(buf[4:], &info); err != nil { + t.Fatalf("Could not unmarshal route info: %v", err) + } + info.ID = "ROUTE:1234" + info.Name = "" + b, err := json.Marshal(info) + if err != nil { + t.Fatalf("Could not marshal test route info: %v", err) + } + infoJSON := fmt.Sprintf("INFO %s\r\n", b) + + // Trigger the send of local subs. + routeSend(infoJSON) + + routeExpect(rsubRe) + + // Close and then re-open + route.Close() + + // Give some time for the route close to be processed before trying to recreate. + checkNumRoutes(t, s, 0) + + route = createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer route.Close() + + routeSend, routeExpect = setupRouteEx(t, route, opts, "ROUTE:1234") + + routeExpect(infoRe) + + routeSend(infoJSON) + routeExpect(rsubRe) +} + +type ignoreLogger struct{} + +func (l *ignoreLogger) Fatalf(f string, args ...any) {} +func (l *ignoreLogger) Errorf(f string, args ...any) {} + +func TestRouteConnectOnShutdownRace(t *testing.T) { + s, opts := runRouteServer(t) + defer s.Shutdown() + + l := &ignoreLogger{} + + var wg sync.WaitGroup + + cQuit := make(chan bool, 1) + + wg.Add(1) + + go func() { + defer wg.Done() + for { + route := createRouteConn(l, opts.Cluster.Host, opts.Cluster.Port) + if route != nil { + setupRouteEx(l, route, opts, "ROUTE:1234") + route.Close() + } + select { + case <-cQuit: + return + default: + } + } + }() + + time.Sleep(5 * time.Millisecond) + s.Shutdown() + + cQuit <- true + + wg.Wait() +} + +func TestRouteSendAsyncINFOToClients(t *testing.T) { + f := func(opts *server.Options) { + s := RunServer(opts) + defer s.Shutdown() + + clientURL := net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port)) + + oldClient := createClientConn(t, opts.Host, opts.Port) + defer oldClient.Close() + + oldClientSend, oldClientExpect := setupConn(t, oldClient) + oldClientSend("PING\r\n") + oldClientExpect(pongRe) + + newClient := createClientConn(t, opts.Host, opts.Port) + defer newClient.Close() + + newClientSend, newClientExpect := setupConnWithProto(t, newClient, clientProtoInfo) + newClientSend("PING\r\n") + + // For new clients, the initial PING (after a CONNECT) will result in a + // PONG followed by an INFO, so make sure we receive them separately. + getPongAlone := func(c net.Conn) { + t.Helper() + pongBuf := make([]byte, len("PONG\r\n")) + c.SetReadDeadline(time.Now().Add(2 * time.Second)) + n, err := c.Read(pongBuf) + c.SetReadDeadline(time.Time{}) + if n <= 0 && err != nil { + t.Fatalf("Error reading from conn: %v\n", err) + } + if !pongRe.Match(pongBuf) { + t.Fatalf("Response did not match expected: \n\tReceived:'%q'\n\tExpected:'%s'\n", pongBuf, pongRe) + } + } + getPongAlone(newClient) + // The new client should receive an INFO with the ConnectInfo boolean set to true. + newClientExpect(infoRe) + + routeID := "Server-B" + + createRoute := func() (net.Conn, sendFun, expectFun) { + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + buf := routeExpect(infoRe) + + info := server.Info{} + if err := json.Unmarshal(buf[4:], &info); err != nil { + stackFatalf(t, "Could not unmarshal route info: %v", err) + } + if opts.Cluster.NoAdvertise { + if len(info.ClientConnectURLs) != 0 { + stackFatalf(t, "Expected ClientConnectURLs to be empty, got %v", info.ClientConnectURLs) + } + } else { + if len(info.ClientConnectURLs) == 0 { + stackFatalf(t, "Expected a list of URLs, got none") + } + if info.ClientConnectURLs[0] != clientURL { + stackFatalf(t, "Expected ClientConnectURLs to be %q, got %q", clientURL, info.ClientConnectURLs[0]) + } + } + + return rc, routeSend, routeExpect + } + + sendRouteINFO := func(routeSend sendFun, routeExpect expectFun, urls []string) { + routeInfo := server.Info{} + routeInfo.ID = routeID + routeInfo.Cluster = "xyz" + routeInfo.Host = "127.0.0.1" + routeInfo.Port = 5222 + routeInfo.ClientConnectURLs = urls + b, err := json.Marshal(routeInfo) + if err != nil { + stackFatalf(t, "Could not marshal test route info: %v", err) + } + infoJSON := fmt.Sprintf("INFO %s\r\n", b) + routeSend(infoJSON) + routeSend("PING\r\n") + routeExpect(pongRe) + } + + checkClientConnectURLS := func(urls, expected []string) { + // Order of array is not guaranteed. + ok := false + if len(urls) == len(expected) { + m := make(map[string]struct{}, len(expected)) + for _, url := range expected { + m[url] = struct{}{} + } + ok = true + for _, url := range urls { + if _, present := m[url]; !present { + ok = false + break + } + } + } + if !ok { + stackFatalf(t, "Expected ClientConnectURLs to be %v, got %v", expected, urls) + } + } + + var checkConnectInfo bool + checkINFOReceived := func(client net.Conn, clientExpect expectFun, expectedURLs []string) { + if opts.Cluster.NoAdvertise { + if !checkConnectInfo { + expectNothing(t, client) + return + } + // Since it is no advertise, but we still get an INFO, the URLs + // should be empty. + expectedURLs = nil + } + buf := clientExpect(infoRe) + info := server.Info{} + if err := json.Unmarshal(buf[4:], &info); err != nil { + stackFatalf(t, "Could not unmarshal route info: %v", err) + } + checkClientConnectURLS(info.ClientConnectURLs, expectedURLs) + if checkConnectInfo { + if !info.ConnectInfo { + stackFatalf(t, "ConnectInfo should have been true") + } + if info.RemoteAccount != "$G" { + stackFatalf(t, "RemoteAccount should be %q, got %q", "$G", info.RemoteAccount) + } + if info.IsSystemAccount { + stackFatalf(t, "IsSystemAccount should have been false") + } + } + } + + // Create a route + rc, routeSend, routeExpect := createRoute() + defer rc.Close() + + // Send an INFO with single URL + routeClientConnectURLs := []string{"127.0.0.1:5222"} + sendRouteINFO(routeSend, routeExpect, routeClientConnectURLs) + + // Expect nothing for old clients + expectNothing(t, oldClient) + + // We expect to get the one from the server we connect to and the other route. + expectedURLs := []string{clientURL, routeClientConnectURLs[0]} + + // Expect new client to receive an INFO (unless disabled) + checkINFOReceived(newClient, newClientExpect, expectedURLs) + + // Disconnect the route + rc.Close() + + // Expect nothing for old clients + expectNothing(t, oldClient) + + // Expect new client to receive an INFO (unless disabled). + // The content will now have the disconnected route ClientConnectURLs + // removed from the INFO. So it should be the one from the server the + // client is connected to. + checkINFOReceived(newClient, newClientExpect, []string{clientURL}) + + // Reconnect the route. + rc, routeSend, routeExpect = createRoute() + defer rc.Close() + + // Resend the same route INFO json. The server will now send + // the INFO since the disconnected route ClientConnectURLs was + // removed in previous step. + sendRouteINFO(routeSend, routeExpect, routeClientConnectURLs) + + // Expect nothing for old clients + expectNothing(t, oldClient) + + // Expect new client to receive an INFO (unless disabled) + checkINFOReceived(newClient, newClientExpect, expectedURLs) + + // Now stop the route and restart with an additional URL + rc.Close() + + // On route disconnect, clients will receive an updated INFO + expectNothing(t, oldClient) + checkINFOReceived(newClient, newClientExpect, []string{clientURL}) + + rc, routeSend, routeExpect = createRoute() + defer rc.Close() + + // Create a client not sending the CONNECT until after route is added + clientNoConnect := createClientConn(t, opts.Host, opts.Port) + defer clientNoConnect.Close() + + // Create a client that does not send the first PING yet + clientNoPing := createClientConn(t, opts.Host, opts.Port) + defer clientNoPing.Close() + clientNoPingSend, clientNoPingExpect := setupConnWithProto(t, clientNoPing, clientProtoInfo) + + // The route now has an additional URL + routeClientConnectURLs = append(routeClientConnectURLs, "127.0.0.1:7777") + expectedURLs = append(expectedURLs, "127.0.0.1:7777") + // This causes the server to add the route and send INFO to clients + sendRouteINFO(routeSend, routeExpect, routeClientConnectURLs) + + // Expect nothing for old clients + expectNothing(t, oldClient) + + // Expect new client to receive an INFO, and verify content as expected. + checkINFOReceived(newClient, newClientExpect, expectedURLs) + + // Expect nothing yet for client that did not send the PING + expectNothing(t, clientNoPing) + + // Now send the first PING + clientNoPingSend("PING\r\n") + getPongAlone(clientNoPing) + // We should receive an INFO but with ConnectInfo==true, etc.. + checkConnectInfo = true + checkINFOReceived(clientNoPing, clientNoPingExpect, expectedURLs) + checkConnectInfo = false + + // Have the client that did not send the connect do it now + clientNoConnectSend, clientNoConnectExpect := setupConnWithProto(t, clientNoConnect, clientProtoInfo) + // Send the PING + clientNoConnectSend("PING\r\n") + getPongAlone(clientNoConnect) + checkConnectInfo = true + checkINFOReceived(clientNoConnect, clientNoConnectExpect, expectedURLs) + checkConnectInfo = false + + // Create a client connection and verify content of initial INFO contains array + // (but empty if no advertise option is set) + cli := createClientConn(t, opts.Host, opts.Port) + defer cli.Close() + buf := expectResult(t, cli, infoRe) + js := infoRe.FindAllSubmatch(buf, 1)[0][1] + var sinfo server.Info + err := json.Unmarshal(js, &sinfo) + if err != nil { + t.Fatalf("Could not unmarshal INFO json: %v\n", err) + } + if opts.Cluster.NoAdvertise { + if len(sinfo.ClientConnectURLs) != 0 { + t.Fatalf("Expected ClientConnectURLs to be empty, got %v", sinfo.ClientConnectURLs) + } + } else { + checkClientConnectURLS(sinfo.ClientConnectURLs, expectedURLs) + } + + // Add a new route + routeID = "Server-C" + rc2, route2Send, route2Expect := createRoute() + defer rc2.Close() + + // Send an INFO with single URL + rc2ConnectURLs := []string{"127.0.0.1:8888"} + sendRouteINFO(route2Send, route2Expect, rc2ConnectURLs) + + // This is the combined client connect URLs array + totalConnectURLs := expectedURLs + totalConnectURLs = append(totalConnectURLs, rc2ConnectURLs...) + + // Expect nothing for old clients + expectNothing(t, oldClient) + + // Expect new client to receive an INFO (unless disabled) + checkINFOReceived(newClient, newClientExpect, totalConnectURLs) + + // Make first route disconnect + rc.Close() + + // Expect nothing for old clients + expectNothing(t, oldClient) + + // Expect new client to receive an INFO (unless disabled) + // The content should be the server client is connected to and the last route + checkINFOReceived(newClient, newClientExpect, []string{"127.0.0.1:5242", "127.0.0.1:8888"}) + } + + opts := LoadConfig("./configs/cluster.conf") + // For this test, be explicit about listen spec. + opts.Host = "127.0.0.1" + opts.Port = 5242 + opts.DisableShortFirstPing = true + + f(opts) + opts.Cluster.NoAdvertise = true + f(opts) +} + +func TestRouteBasicPermissions(t *testing.T) { + srvA, optsA := RunServerWithConfig("./configs/srv_a_perms.conf") + defer srvA.Shutdown() + + srvB, optsB := RunServerWithConfig("./configs/srv_b.conf") + defer srvB.Shutdown() + + checkClusterFormed(t, srvA, srvB) + + // Create a connection to server B + ncb, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsB.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncb.Close() + ch := make(chan bool, 1) + cb := func(_ *nats.Msg) { + ch <- true + } + // Subscribe on Server B on "bar" and "baz", which should be accepted by server A across the route + // Due to allowing "*" + subBbar, err := ncb.Subscribe("bar", cb) + if err != nil { + t.Fatalf("Error on subscribe: %v", err) + } + defer subBbar.Unsubscribe() + subBbaz, err := ncb.Subscribe("baz", cb) + if err != nil { + t.Fatalf("Error on subscribe: %v", err) + } + defer subBbaz.Unsubscribe() + ncb.Flush() + if err := checkExpectedSubs(2, srvA, srvB); err != nil { + t.Fatal(err.Error()) + } + + // Create a connection to server A + nca, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsA.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nca.Close() + // Publish on bar and baz, messages should be received. + if err := nca.Publish("bar", []byte("hello")); err != nil { + t.Fatalf("Error on publish: %v", err) + } + if err := nca.Publish("baz", []byte("hello")); err != nil { + t.Fatalf("Error on publish: %v", err) + } + for i := 0; i < 2; i++ { + select { + case <-ch: + case <-time.After(time.Second): + t.Fatal("Did not get the messages") + } + } + + // From B, start a subscription on "foo", which server A should drop since + // it only exports on "bar" and "baz" + subBfoo, err := ncb.Subscribe("foo", cb) + if err != nil { + t.Fatalf("Error on subscribe: %v", err) + } + defer subBfoo.Unsubscribe() + ncb.Flush() + // B should have now 3 subs + if err := checkExpectedSubs(3, srvB); err != nil { + t.Fatal(err.Error()) + } + // and A still 2. + if err := checkExpectedSubs(2, srvA); err != nil { + t.Fatal(err.Error()) + } + // So producing on "foo" from A should not be forwarded to B. + if err := nca.Publish("foo", []byte("hello")); err != nil { + t.Fatalf("Error on publish: %v", err) + } + select { + case <-ch: + t.Fatal("Message should not have been received") + case <-time.After(100 * time.Millisecond): + } + + // Now on A, create a subscription on something that A does not import, + // like "bat". + subAbat, err := nca.Subscribe("bat", cb) + if err != nil { + t.Fatalf("Error on subscribe: %v", err) + } + defer subAbat.Unsubscribe() + nca.Flush() + // A should have 3 subs + if err := checkExpectedSubs(3, srvA); err != nil { + t.Fatal(err.Error()) + } + // And from B, send a message on that subject and make sure it is not received. + if err := ncb.Publish("bat", []byte("hello")); err != nil { + t.Fatalf("Error on publish: %v", err) + } + select { + case <-ch: + t.Fatal("Message should not have been received") + case <-time.After(100 * time.Millisecond): + } + + // Stop subscription on foo from B + subBfoo.Unsubscribe() + ncb.Flush() + // Back to 2 subs on B + if err := checkExpectedSubs(2, srvB); err != nil { + t.Fatal(err.Error()) + } + + // Create subscription on foo from A, this should be forwared to B. + subAfoo, err := nca.Subscribe("foo", cb) + if err != nil { + t.Fatalf("Error on subscribe: %v", err) + } + defer subAfoo.Unsubscribe() + // Create another one so that test the import permissions cache + subAfoo2, err := nca.Subscribe("foo", cb) + if err != nil { + t.Fatalf("Error on subscribe: %v", err) + } + defer subAfoo2.Unsubscribe() + nca.Flush() + // A should have 5 subs + if err := checkExpectedSubs(5, srvA); err != nil { + t.Fatal(err.Error()) + } + // B should have 3 since we coalesce te two for 'foo' + if err := checkExpectedSubs(3, srvB); err != nil { + t.Fatal(err.Error()) + } + // Send a message from B and check that it is received. + if err := ncb.Publish("foo", []byte("hello")); err != nil { + t.Fatalf("Error on publish: %v", err) + } + for i := 0; i < 2; i++ { + select { + case <-ch: + case <-time.After(time.Second): + t.Fatal("Did not get the message") + } + } + + // Close connection from B, and restart server B too. + // We want to make sure that + ncb.Close() + srvB.Shutdown() + + // Since B had 2 local subs, A should still only go from 4 to 3 + if err := checkExpectedSubs(3, srvA); err != nil { + t.Fatal(err.Error()) + } + + // Restart server B + srvB, optsB = RunServerWithConfig("./configs/srv_b.conf") + defer srvB.Shutdown() + // Check that subs from A that can be sent to B are sent. + // That would be 2 (the 2 subscriptions on foo) as one. + if err := checkExpectedSubs(1, srvB); err != nil { + t.Fatal(err.Error()) + } + + // Connect to B and send on "foo" and make sure we receive + ncb, err = nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsB.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncb.Close() + if err := ncb.Publish("foo", []byte("hello")); err != nil { + t.Fatalf("Error on publish: %v", err) + } + for i := 0; i < 2; i++ { + select { + case <-ch: + case <-time.After(time.Second): + t.Fatal("Did not get the message") + } + } + + // Send on "bat" and make sure that this is not received. + if err := ncb.Publish("bat", []byte("hello")); err != nil { + t.Fatalf("Error on publish: %v", err) + } + select { + case <-ch: + t.Fatal("Message should not have been received") + case <-time.After(100 * time.Millisecond): + } + + nca.Close() + ncb.Close() + + srvA.Shutdown() + srvB.Shutdown() + + optsA.Cluster.Permissions.Export = nil + srvA = RunServer(optsA) + defer srvA.Shutdown() + + srvB = RunServer(optsB) + defer srvB.Shutdown() + + checkClusterFormed(t, srvA, srvB) + + nca, err = nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsA.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nca.Close() + // Subscribe on "bar" which is not imported + if _, err := nca.Subscribe("bar", cb); err != nil { + t.Fatalf("Error on subscribe: %v", err) + } + if err := checkExpectedSubs(1, srvA); err != nil { + t.Fatal(err.Error()) + } + + // Publish from B, should not be received + ncb, err = nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsB.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncb.Close() + if err := ncb.Publish("bar", []byte("hello")); err != nil { + t.Fatalf("Error on publish: %v", err) + } + select { + case <-ch: + t.Fatal("Message should not have been received") + case <-time.After(100 * time.Millisecond): + //ok + } + // Subscribe on "baz" on B + if _, err := ncb.Subscribe("baz", cb); err != nil { + t.Fatalf("Error on subscribe: %v", err) + } + if err := checkExpectedSubs(1, srvB); err != nil { + t.Fatal(err.Error()) + } + if err := checkExpectedSubs(2, srvA); err != nil { + t.Fatal(err.Error()) + } + // Publish from A, since there is no export restriction, message should be received. + if err := nca.Publish("baz", []byte("hello")); err != nil { + t.Fatalf("Error on publish: %v", err) + } + select { + case <-ch: + // ok + case <-time.After(250 * time.Millisecond): + t.Fatal("Message should have been received") + } +} + +func createConfFile(t testing.TB, content []byte) string { + t.Helper() + conf := createTempFile(t, "") + fName := conf.Name() + conf.Close() + if err := os.WriteFile(fName, content, 0666); err != nil { + t.Fatalf("Error writing conf file: %v", err) + } + return fName +} + +func TestRoutesOnlyImportOrExport(t *testing.T) { + contents := []string{ + `import: "foo"`, + `import: { + allow: "foo" + }`, + `import: { + deny: "foo" + }`, + `import: { + allow: "foo" + deny: "foo" + }`, + `export: "foo"`, + `export: { + allow: "foo" + }`, + `export: { + deny: "foo" + }`, + `export: { + allow: "foo" + deny: "foo" + }`, + } + f := func(c string) { + cf := createConfFile(t, []byte(fmt.Sprintf(` + port: -1 + cluster { + name: "Z" + port: -1 + authorization { + user: ivan + password: pwd + permissions { + %s + } + } + } + `, c))) + s, _ := RunServerWithConfig(cf) + s.Shutdown() + } + for _, c := range contents { + f(c) + } +} diff --git a/test/service_latency_test.go b/test/service_latency_test.go new file mode 100644 index 0000000..e35f7a9 --- /dev/null +++ b/test/service_latency_test.go @@ -0,0 +1,1923 @@ +// Copyright 2019-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "encoding/json" + "fmt" + "math/rand/v2" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/jwt/v2" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nkeys" +) + +// Used to setup superclusters for tests. +type supercluster struct { + t *testing.T + clusters []*cluster +} + +func (sc *supercluster) shutdown() { + if sc == nil { + return + } + for _, c := range sc.clusters { + shutdownCluster(c) + } +} + +const digits = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" +const base = 36 +const cnlen = 8 + +func randClusterName() string { + var name []byte + rn := rand.Int64() + for i := 0; i < cnlen; i++ { + name = append(name, digits[rn%base]) + rn /= base + } + return string(name[:cnlen]) +} + +func createSuperCluster(t *testing.T, numServersPer, numClusters int) *supercluster { + clusters := []*cluster{} + + for i := 0; i < numClusters; i++ { + // Pick cluster name and setup default accounts. + c := createClusterEx(t, true, 5*time.Millisecond, true, randClusterName(), numServersPer, clusters...) + clusters = append(clusters, c) + } + return &supercluster{t, clusters} +} + +func (sc *supercluster) setResponseThreshold(t *testing.T, maxTime time.Duration) { + t.Helper() + for _, c := range sc.clusters { + for _, s := range c.servers { + foo, err := s.LookupAccount("FOO") + if err != nil { + t.Fatalf("Error looking up account 'FOO': %v", err) + } + if err := foo.SetServiceExportResponseThreshold("ngs.usage.*", maxTime); err != nil { + t.Fatalf("Error setting response threshold") + } + } + } +} + +func (sc *supercluster) setImportShare(t *testing.T) { + t.Helper() + for _, c := range sc.clusters { + for _, s := range c.servers { + foo, err := s.LookupAccount("FOO") + if err != nil { + t.Fatalf("Error looking up account 'FOO': %v", err) + } + bar, err := s.LookupAccount("BAR") + if err != nil { + t.Fatalf("Error looking up account 'BAR': %v", err) + } + if err := bar.SetServiceImportSharing(foo, "ngs.usage.bar", true); err != nil { + t.Fatalf("Error setting import sharing: %v", err) + } + } + } +} + +func (sc *supercluster) setupLatencyTracking(t *testing.T, p int) { + t.Helper() + for _, c := range sc.clusters { + for _, s := range c.servers { + foo, err := s.LookupAccount("FOO") + if err != nil { + t.Fatalf("Error looking up account 'FOO': %v", err) + } + bar, err := s.LookupAccount("BAR") + if err != nil { + t.Fatalf("Error looking up account 'BAR': %v", err) + } + if err := foo.AddServiceExport("ngs.usage.*", nil); err != nil { + t.Fatalf("Error adding service export to 'FOO': %v", err) + } + if err := foo.TrackServiceExportWithSampling("ngs.usage.*", "results", p); err != nil { + t.Fatalf("Error adding latency tracking to 'FOO': %v", err) + } + if err := bar.AddServiceImport(foo, "ngs.usage", "ngs.usage.bar"); err != nil { + t.Fatalf("Error adding service import to 'ngs.usage': %v", err) + } + } + } +} + +func (sc *supercluster) removeLatencyTracking(t *testing.T) { + t.Helper() + for _, c := range sc.clusters { + for _, s := range c.servers { + foo, err := s.LookupAccount("FOO") + if err != nil { + t.Fatalf("Error looking up account 'FOO': %v", err) + } + foo.UnTrackServiceExport("ngs.usage.*") + } + } +} + +func (sc *supercluster) totalSubs() int { + totalSubs := 0 + for _, c := range sc.clusters { + totalSubs += c.totalSubs() + } + return totalSubs +} + +func clientConnectWithName(t *testing.T, opts *server.Options, user, appname string) *nats.Conn { + t.Helper() + url := fmt.Sprintf("nats://%s:pass@%s:%d", user, opts.Host, opts.Port) + nc, err := nats.Connect(url, nats.Name(appname)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + return nc +} + +func clientConnect(t *testing.T, opts *server.Options, user string) *nats.Conn { + t.Helper() + return clientConnectWithName(t, opts, user, "") +} + +func clientConnectOldRequest(t *testing.T, opts *server.Options, user string) *nats.Conn { + t.Helper() + url := fmt.Sprintf("nats://%s:pass@%s:%d", user, opts.Host, opts.Port) + nc, err := nats.Connect(url, nats.UseOldRequestStyle()) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + return nc +} + +func checkServiceLatency(t *testing.T, sl server.ServiceLatency, start time.Time, serviceTime time.Duration) { + t.Helper() + + if sl.Status != 200 { + t.Fatalf("Bad status received, wanted 200 got %d", sl.Status) + } + + serviceTime = serviceTime.Round(time.Millisecond) + + startDelta := sl.RequestStart.Sub(start) + // Original test was 5ms, but got GitHub Action failure with "Bad start delta 5.033929ms", + // and Travis will get something like: "Bad start delta 15.046059ms", so be more generous. + if startDelta > 20*time.Millisecond { + t.Fatalf("Bad start delta %v", startDelta) + } + // Since RTT during tests is estimate we remove from calculation. + if (sl.ServiceLatency + sl.Responder.RTT) < time.Duration(float64(serviceTime)*0.8) { + t.Fatalf("Bad service latency: %v (%v)", sl.ServiceLatency, serviceTime) + } + if sl.TotalLatency < sl.ServiceLatency { + t.Fatalf("Bad total latency: %v (%v)", sl.TotalLatency, sl.ServiceLatency) + } + // We should have NATS latency here that is non-zero with real clients. + if sl.Requestor.RTT == 0 { + t.Fatalf("Expected non-zero NATS Requestor latency") + } + if sl.Responder.RTT == 0 { + t.Fatalf("Expected non-zero NATS Requestor latency") + } + + // Make sure they add up + got := sl.TotalLatency + expected := sl.ServiceLatency + sl.NATSTotalTime() + if got != expected { + t.Fatalf("Numbers do not add up: %+v,\ngot: %v\nexpected: %v", sl, got, expected) + } +} + +func extendedCheck(t *testing.T, lc *server.ClientInfo, eUser, appName, eServer string) { + t.Helper() + if lc.User != eUser { + t.Fatalf("Expected user of %q, got %q", eUser, lc.User) + } + if appName != "" && appName != lc.Name { + t.Fatalf("Expected appname of %q, got %q\n", appName, lc.Name) + } + if lc.Host == "" { + t.Fatalf("Expected non-empty IP") + } + if lc.ID < 1 || lc.ID > 20 { + t.Fatalf("Expected a ID in range, got %d", lc.ID) + } + if eServer != "" && eServer != lc.Server { + t.Fatalf("Expected server of %q, got %q", eServer, lc.Server) + } +} + +func noShareCheck(t *testing.T, lc *server.ClientInfo) { + t.Helper() + if lc.Name != "" { + t.Fatalf("appname should not have been shared, got %q", lc.Name) + } + if lc.User != "" { + t.Fatalf("user should not have been shared, got %q", lc.User) + } + if lc.Host != "" { + t.Fatalf("client ip should not have been shared, got %q", lc.Host) + } + if lc.ID != 0 { + t.Fatalf("client id should not have been shared, got %d", lc.ID) + } + if lc.Server != "" { + t.Fatalf("client' server should not have been shared, got %q", lc.Server) + } +} + +func TestServiceLatencySingleServerConnect(t *testing.T) { + sc := createSuperCluster(t, 3, 2) + defer sc.shutdown() + + // Now add in new service export to FOO and have bar import that with tracking enabled. + sc.setupLatencyTracking(t, 100) + + // Now we can setup and test, do single node only first. + // This is the service provider. + nc := clientConnectWithName(t, sc.clusters[0].opts[0], "foo", "service22") + defer nc.Close() + + // The service listener. + serviceTime := 25 * time.Millisecond + nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { + time.Sleep(serviceTime) + msg.Respond([]byte("22 msgs")) + }) + + // Listen for metrics + rsub, _ := nc.SubscribeSync("results") + + // Requestor + nc2 := clientConnect(t, sc.clusters[0].opts[0], "bar") + defer nc2.Close() + + // Send the request. + start := time.Now() + if _, err := nc2.Request("ngs.usage", []byte("1h"), time.Second); err != nil { + t.Fatalf("Expected a response") + } + + var sl server.ServiceLatency + rmsg, _ := rsub.NextMsg(time.Second) + json.Unmarshal(rmsg.Data, &sl) + + checkServiceLatency(t, sl, start, serviceTime) + + rs := sc.clusters[0].servers[0] + extendedCheck(t, sl.Responder, "foo", "service22", rs.Name()) + // Normally requestor's don't share + noShareCheck(t, sl.Requestor) + + // Now make sure normal use case works with old request style. + nc3 := clientConnectOldRequest(t, sc.clusters[0].opts[0], "bar") + defer nc3.Close() + + start = time.Now() + if _, err := nc3.Request("ngs.usage", []byte("1h"), time.Second); err != nil { + t.Fatalf("Expected a response") + } + nc3.Close() + + checkServiceLatency(t, sl, start, serviceTime) + extendedCheck(t, sl.Responder, "foo", "service22", rs.Name()) + // Normally requestor's don't share + noShareCheck(t, sl.Requestor) +} + +// If a client has a longer RTT that the effective RTT for NATS + responder +// the requestor RTT will be marked as 0. This can happen quite often with +// utility programs that are far away from a cluster like NGS but the service +// response time has a shorter RTT. +func TestServiceLatencyClientRTTSlowerVsServiceRTT(t *testing.T) { + sc := createSuperCluster(t, 2, 2) + defer sc.shutdown() + + // Now add in new service export to FOO and have BAR import that with tracking enabled. + sc.setupLatencyTracking(t, 100) + + nc := clientConnect(t, sc.clusters[0].opts[0], "foo") + defer nc.Close() + + // The service listener. Mostly instant response. + nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { + time.Sleep(time.Millisecond) + msg.Respond([]byte("22 msgs")) + }) + + // Listen for metrics + rsub, _ := nc.SubscribeSync("results") + nc.Flush() + + // Requestor and processing + requestAndCheck := func(cindex, sindex int) { + t.Helper() + sopts := sc.clusters[cindex].opts[sindex] + + if nmsgs, _, err := rsub.Pending(); err != nil || nmsgs != 0 { + t.Fatalf("Did not expect any latency results, got %d", nmsgs) + } + + rtt := 10 * time.Millisecond + bw := 1024 * 1024 + sp := newSlowProxy(rtt+5*time.Millisecond, bw, bw, sopts) + defer sp.stop() + + nc2 := clientConnect(t, sp.opts(), "bar") + defer nc2.Close() + + start := time.Now() + nc2.Flush() + // Check rtt for slow proxy + if d := time.Since(start); d < rtt { + t.Fatalf("Expected an rtt of at least %v, got %v", rtt, d) + } + + // Send the request. + _, err := nc2.Request("ngs.usage", []byte("1h"), time.Second) + if err != nil { + t.Fatalf("Expected a response") + } + + var sl server.ServiceLatency + rmsg, err := rsub.NextMsg(time.Second) + if err != nil || rmsg == nil { + t.Fatalf("Did not receive latency results") + } + json.Unmarshal(rmsg.Data, &sl) + + if sl.Status != 200 { + t.Fatalf("Expected a status 200 Ok, got [%d] %q", sl.Status, sl.Error) + } + // We want to test here that when the client requestor RTT is larger then the response time + // we still deliver a requestor value > 0. + // Now check that it is close to rtt. + if sl.Requestor.RTT < rtt { + t.Fatalf("Expected requestor latency to be > %v, got %v", rtt, sl.Requestor.RTT) + } + if sl.TotalLatency < rtt { + t.Fatalf("Expected total latency to be > %v, got %v", rtt, sl.TotalLatency) + } + + rs := sc.clusters[0].servers[0] + extendedCheck(t, sl.Responder, "foo", "", rs.Name()) + // Normally requestor's don't share + noShareCheck(t, sl.Requestor) + + // Check for trailing duplicates.. + rmsg, err = rsub.NextMsg(100 * time.Millisecond) + if err == nil { + t.Fatalf("Duplicate metric result, %q", rmsg.Data) + } + } + + // Check same server. + requestAndCheck(0, 0) + // Check from remote server across GW. + requestAndCheck(1, 1) + // Same cluster but different server + requestAndCheck(0, 1) +} + +func connRTT(nc *nats.Conn) time.Duration { + // Do 5x to flatten + total := time.Duration(0) + for i := 0; i < 5; i++ { + start := time.Now() + nc.Flush() + total += time.Since(start) + } + return total / 5 +} + +func TestServiceLatencyRemoteConnect(t *testing.T) { + sc := createSuperCluster(t, 3, 2) + defer sc.shutdown() + + // Now add in new service export to FOO and have bar import that with tracking enabled. + sc.setupLatencyTracking(t, 100) + + // Now we can setup and test, do single node only first. + // This is the service provider. + nc := clientConnect(t, sc.clusters[0].opts[0], "foo") + defer nc.Close() + + subsBefore := int(sc.clusters[0].servers[0].NumSubscriptions()) + + // The service listener. + serviceTime := 25 * time.Millisecond + nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { + time.Sleep(serviceTime) + msg.Respond([]byte("22 msgs")) + }) + + // Listen for metrics + rsub, _ := nc.SubscribeSync("results") + nc.Flush() + + if err := checkExpectedSubs(subsBefore+2, sc.clusters[0].servers...); err != nil { + t.Fatal(err.Error()) + } + + // Same Cluster Requestor + nc2 := clientConnect(t, sc.clusters[0].opts[2], "bar") + defer nc2.Close() + + // Send the request. + start := time.Now() + _, err := nc2.Request("ngs.usage", []byte("1h"), time.Second) + if err != nil { + t.Fatalf("Expected a response") + } + + var sl server.ServiceLatency + rmsg, err := rsub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Error getting latency measurement: %v", err) + } + json.Unmarshal(rmsg.Data, &sl) + checkServiceLatency(t, sl, start, serviceTime) + rs := sc.clusters[0].servers[0] + extendedCheck(t, sl.Responder, "foo", "", rs.Name()) + // Normally requestor's don't share + noShareCheck(t, sl.Requestor) + + // Lastly here, we need to make sure we are properly tracking the extra hops. + // We will make sure that NATS latency is close to what we see from the outside in terms of RTT. + if crtt := connRTT(nc) + connRTT(nc2); sl.NATSTotalTime() < crtt { + t.Fatalf("Not tracking second measurement for NATS latency across servers: %v vs %v", sl.NATSTotalTime(), crtt) + } + + // Gateway Requestor + nc2 = clientConnect(t, sc.clusters[1].opts[1], "bar") + defer nc2.Close() + + // Send the request. + start = time.Now() + _, err = nc2.Request("ngs.usage", []byte("1h"), time.Second) + if err != nil { + t.Fatalf("Expected a response") + } + + rmsg, err = rsub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Error getting latency measurement: %v", err) + } + json.Unmarshal(rmsg.Data, &sl) + checkServiceLatency(t, sl, start, serviceTime) + extendedCheck(t, sl.Responder, "foo", "", rs.Name()) + // Normally requestor's don't share + noShareCheck(t, sl.Requestor) + + // Lastly here, we need to make sure we are properly tracking the extra hops. + // We will make sure that NATS latency is close to what we see from the outside in terms of RTT. + if crtt := connRTT(nc) + connRTT(nc2); sl.NATSTotalTime() < crtt { + t.Fatalf("Not tracking second measurement for NATS latency across servers: %v vs %v", sl.NATSTotalTime(), crtt) + } + + // Now turn off and make sure we no longer receive updates. + sc.removeLatencyTracking(t) + _, err = nc2.Request("ngs.usage", []byte("1h"), time.Second) + if err != nil { + t.Fatalf("Expected a response") + } + + _, err = rsub.NextMsg(100 * time.Millisecond) + if err == nil { + t.Fatalf("Did not expect to receive a latency metric") + } +} + +func TestServiceLatencySampling(t *testing.T) { + sc := createSuperCluster(t, 3, 2) + defer sc.shutdown() + + // Now add in new service export to FOO and have bar import that with tracking enabled. + sc.setupLatencyTracking(t, 50) + + // Now we can setup and test, do single node only first. + // This is the service provider. + nc := clientConnect(t, sc.clusters[0].opts[0], "foo") + defer nc.Close() + + // The service listener. + nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { + msg.Respond([]byte("22 msgs")) + }) + + // Listen for metrics + received := int32(0) + + nc.Subscribe("results", func(msg *nats.Msg) { + atomic.AddInt32(&received, 1) + }) + + // Same Cluster Requestor + nc2 := clientConnect(t, sc.clusters[0].opts[2], "bar") + defer nc2.Close() + + toSend := 1000 + for i := 0; i < toSend; i++ { + nc2.Request("ngs.usage", []byte("1h"), time.Second) + } + // Wait for results to flow in. + time.Sleep(100 * time.Millisecond) + + mid := toSend / 2 + delta := toSend / 10 // 10% + got := int(atomic.LoadInt32(&received)) + + if got > mid+delta || got < mid-delta { + t.Fatalf("Sampling number incorrect: %d vs %d", mid, got) + } +} + +func TestServiceLatencyNoSubsLeak(t *testing.T) { + sc := createSuperCluster(t, 3, 3) + defer sc.shutdown() + + // Now add in new service export to FOO and have bar import that with tracking enabled. + sc.setupLatencyTracking(t, 100) + + nc := clientConnectWithName(t, sc.clusters[0].opts[1], "foo", "dlc22") + defer nc.Close() + + // The service listener. + nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { + msg.Respond([]byte("22 msgs")) + }) + nc.Flush() + // Propagation of sub through super cluster. + time.Sleep(100 * time.Millisecond) + + startSubs := sc.totalSubs() + + fooAcc, _ := sc.clusters[1].servers[1].LookupAccount("FOO") + startNumSis := fooAcc.NumServiceImports() + + for i := 0; i < 100; i++ { + nc := clientConnect(t, sc.clusters[1].opts[1], "bar") + defer nc.Close() + if _, err := nc.Request("ngs.usage", []byte("1h"), time.Second); err != nil { + t.Fatalf("Error on request: %v", err) + } + nc.Close() + } + + // We are adding 3 here for the wildcard response subject for service replies. + // we only have one but it will show in three places. + startSubs += 3 + + checkFor(t, time.Second, 50*time.Millisecond, func() error { + if numSubs := sc.totalSubs(); numSubs != startSubs { + return fmt.Errorf("Leaked %d subs", numSubs-startSubs) + } + return nil + }) + + // Now also check to make sure the service imports created for the request go away as well. + checkFor(t, time.Second, 50*time.Millisecond, func() error { + if numSis := fooAcc.NumServiceImports(); numSis != startNumSis { + return fmt.Errorf("Leaked %d service imports", numSis-startNumSis) + } + return nil + }) +} + +func TestServiceLatencyWithName(t *testing.T) { + sc := createSuperCluster(t, 1, 1) + defer sc.shutdown() + + // Now add in new service export to FOO and have bar import that with tracking enabled. + sc.setupLatencyTracking(t, 100) + + opts := sc.clusters[0].opts[0] + + nc := clientConnectWithName(t, opts, "foo", "dlc22") + defer nc.Close() + + // The service listener. + nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { + msg.Respond([]byte("22 msgs")) + }) + + // Listen for metrics + rsub, _ := nc.SubscribeSync("results") + nc.Flush() + + nc2 := clientConnect(t, opts, "bar") + defer nc2.Close() + nc2.Request("ngs.usage", []byte("1h"), time.Second) + + var sl server.ServiceLatency + rmsg, err := rsub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Error getting message: %v", err) + } + json.Unmarshal(rmsg.Data, &sl) + + // Make sure we have AppName set. + rs := sc.clusters[0].servers[0] + extendedCheck(t, sl.Responder, "foo", "dlc22", rs.Name()) + // Normally requestor's don't share + noShareCheck(t, sl.Requestor) +} + +func TestServiceLatencyWithNameMultiServer(t *testing.T) { + sc := createSuperCluster(t, 3, 2) + defer sc.shutdown() + + // Now add in new service export to FOO and have bar import that with tracking enabled. + sc.setupLatencyTracking(t, 100) + + nc := clientConnectWithName(t, sc.clusters[0].opts[1], "foo", "dlc22") + defer nc.Close() + + // The service listener. + nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { + msg.Respond([]byte("22 msgs")) + }) + + // Listen for metrics + rsub, _ := nc.SubscribeSync("results") + nc.Flush() + + nc2 := clientConnect(t, sc.clusters[1].opts[1], "bar") + defer nc2.Close() + nc2.Request("ngs.usage", []byte("1h"), time.Second) + + var sl server.ServiceLatency + checkFor(t, 3*time.Second, time.Second, func() error { + rmsg, err := rsub.NextMsg(500 * time.Millisecond) + if err != nil { + return err + } + json.Unmarshal(rmsg.Data, &sl) + return nil + }) + + // Make sure we have AppName set. + rs := sc.clusters[0].servers[1] + extendedCheck(t, sl.Responder, "foo", "dlc22", rs.Name()) + // Normally requestor's don't share + noShareCheck(t, sl.Requestor) +} + +func createAccountWithJWT(t *testing.T) (string, nkeys.KeyPair, *jwt.AccountClaims) { + t.Helper() + okp, _ := nkeys.FromSeed(oSeed) + akp, _ := nkeys.CreateAccount() + pub, _ := akp.PublicKey() + nac := jwt.NewAccountClaims(pub) + jwt, _ := nac.Encode(okp) + return jwt, akp, nac +} + +func TestServiceLatencyWithJWT(t *testing.T) { + okp, _ := nkeys.FromSeed(oSeed) + + // Create three accounts, system, service and normal account. + sysJWT, sysKP, _ := createAccountWithJWT(t) + sysPub, _ := sysKP.PublicKey() + + _, svcKP, svcAcc := createAccountWithJWT(t) + svcPub, _ := svcKP.PublicKey() + + // Add in the service export with latency tracking here. + serviceExport := &jwt.Export{Subject: "req.*", Type: jwt.Service} + svcAcc.Exports.Add(serviceExport) + svcJWT, err := svcAcc.Encode(okp) + if err != nil { + t.Fatalf("Error generating account JWT: %v", err) + } + + _, accKP, accAcc := createAccountWithJWT(t) + accPub, _ := accKP.PublicKey() + + // Add in the import. + serviceImport := &jwt.Import{Account: svcPub, Subject: "request", To: "req.echo", Type: jwt.Service} + accAcc.Imports.Add(serviceImport) + accJWT, err := accAcc.Encode(okp) + if err != nil { + t.Fatalf("Error generating account JWT: %v", err) + } + + cf := ` + listen: 127.0.0.1:-1 + cluster { + name: "A" + listen: 127.0.0.1:-1 + authorization { + timeout: 2.2 + } %s + } + + operator = "./configs/nkeys/op.jwt" + system_account = "%s" + + resolver = MEMORY + resolver_preload = { + %s : "%s" + %s : "%s" + %s : "%s" + } + ` + contents := strings.Replace(fmt.Sprintf(cf, "", sysPub, sysPub, sysJWT, svcPub, svcJWT, accPub, accJWT), "\n\t", "\n", -1) + conf := createConfFile(t, []byte(contents)) + + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + // Create a new server and route to main one. + routeStr := fmt.Sprintf("\n\t\troutes = [nats-route://%s:%d]", opts.Cluster.Host, opts.Cluster.Port) + contents2 := strings.Replace(fmt.Sprintf(cf, routeStr, sysPub, sysPub, sysJWT, svcPub, svcJWT, accPub, accJWT), "\n\t", "\n", -1) + + conf2 := createConfFile(t, []byte(contents2)) + + s2, opts2 := RunServerWithConfig(conf2) + defer s2.Shutdown() + + checkClusterFormed(t, s, s2) + + // Create service provider. + url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + copt, pubUser := createUserCredsOption(t, s, svcKP) + nc, err := nats.Connect(url, copt, nats.Name("fooService")) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // The service listener. + serviceTime := 25 * time.Millisecond + nc.Subscribe("req.echo", func(msg *nats.Msg) { + time.Sleep(serviceTime) + msg.Respond(msg.Data) + }) + + // Listen for metrics + rsub, _ := nc.SubscribeSync("results") + nc.Flush() + + // Create second client and send request from this one. + url2 := fmt.Sprintf("nats://%s:%d/", opts2.Host, opts2.Port) + nc2, err := nats.Connect(url2, createUserCreds(t, s2, accKP)) + if err != nil { + t.Fatalf("Error creating client: %v\n", err) + } + defer nc2.Close() + + // Send the request. + _, err = nc2.Request("request", []byte("hello"), time.Second) + if err != nil { + t.Fatalf("Expected a response") + } + + // We should not receive latency at this time. + _, err = rsub.NextMsg(100 * time.Millisecond) + if err == nil { + t.Fatalf("Did not expect to receive a latency metric") + } + + // Now turn it on.. + updateAccount := func() { + t.Helper() + for _, s := range []*server.Server{s, s2} { + svcAccount, err := s.LookupAccount(svcPub) + if err != nil { + t.Fatalf("Could not lookup service account from server %+v", s) + } + s.UpdateAccountClaims(svcAccount, svcAcc) + } + } + serviceExport.Latency = &jwt.ServiceLatency{Sampling: 100, Results: "results"} + updateAccount() + + // Grab the service responder's user. + + // Send the request. + start := time.Now() + _, err = nc2.Request("request", []byte("hello"), time.Second) + if err != nil { + t.Fatalf("Expected a response") + } + + var sl server.ServiceLatency + rmsg, err := rsub.NextMsg(time.Second) + if err != nil || rmsg == nil { + t.Fatalf("Did not receive a latency metric") + } + json.Unmarshal(rmsg.Data, &sl) + checkServiceLatency(t, sl, start, serviceTime) + extendedCheck(t, sl.Responder, pubUser, "fooService", s.Name()) + // Normally requestor's don't share + noShareCheck(t, sl.Requestor) + + // Now we will remove tracking. Do this by simulating a JWT update. + serviceExport.Latency = nil + updateAccount() + + // Now we should not get any tracking data. + _, err = nc2.Request("request", []byte("hello"), time.Second) + if err != nil { + t.Fatalf("Expected a response") + } + _, err = rsub.NextMsg(100 * time.Millisecond) + if err == nil { + t.Fatalf("Did not expect to receive a latency metric") + } +} + +func TestServiceLatencyAdjustNegativeLatencyValues(t *testing.T) { + sc := createSuperCluster(t, 3, 2) + defer sc.shutdown() + + // Now add in new service export to FOO and have bar import + // that with tracking enabled. + sc.setupLatencyTracking(t, 100) + + // Now we can setup and test, do single node only first. + // This is the service provider. + nc := clientConnect(t, sc.clusters[0].opts[0], "foo") + defer nc.Close() + + // The service listener. + nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { + msg.Respond([]byte("22 msgs")) + }) + + // Listen for metrics + rsub, err := nc.SubscribeSync("results") + if err != nil { + t.Fatal(err) + } + nc.Flush() + + // Requestor + nc2 := clientConnect(t, sc.clusters[0].opts[0], "bar") + defer nc2.Close() + + // Send the request. + totalSamples := 50 + for i := 0; i < totalSamples; i++ { + if _, err := nc2.Request("ngs.usage", []byte("1h"), time.Second); err != nil { + t.Fatalf("Expected a response") + } + } + + var sl server.ServiceLatency + for i := 0; i < totalSamples; i++ { + rmsg, err := rsub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Expected to receive latency metric: %d, %s", i, err) + } + if err := json.Unmarshal(rmsg.Data, &sl); err != nil { + t.Errorf("Unexpected error processing latency metric: %s", err) + } + if sl.ServiceLatency < 0 { + t.Fatalf("Unexpected negative latency value: %v", sl) + } + } +} + +func TestServiceLatencyRemoteConnectAdjustNegativeValues(t *testing.T) { + sc := createSuperCluster(t, 3, 2) + defer sc.shutdown() + + // Now add in new service export to FOO and have bar import that with tracking enabled. + sc.setupLatencyTracking(t, 100) + + // Now we can setup and test, do single node only first. + // This is the service provider. + nc := clientConnect(t, sc.clusters[0].opts[0], "foo") + defer nc.Close() + + // The service listener. + nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { + msg.Respond([]byte("22 msgs")) + }) + + // Listen for metrics + rsub, err := nc.SubscribeSync("results") + if err != nil { + t.Fatal(err) + } + nc.Flush() + + // Same Cluster Requestor + nc2 := clientConnect(t, sc.clusters[0].opts[2], "bar") + defer nc2.Close() + + // Gateway Requestor + nc3 := clientConnect(t, sc.clusters[1].opts[1], "bar") + defer nc3.Close() + + // Send a few initial requests to ensure interest is propagated + // both for cluster and gateway requestors. + checkFor(t, 3*time.Second, time.Second, func() error { + _, err1 := nc2.Request("ngs.usage", []byte("1h"), time.Second) + _, err2 := nc3.Request("ngs.usage", []byte("1h"), time.Second) + + if err1 != nil || err2 != nil { + return fmt.Errorf("Timed out waiting for super cluster to be ready") + } + return nil + }) + + // Send the request. + totalSamples := 20 + for i := 0; i < totalSamples; i++ { + if _, err := nc2.Request("ngs.usage", []byte("1h"), time.Second); err != nil { + t.Fatalf("Expected a response") + } + } + + for i := 0; i < totalSamples; i++ { + if _, err := nc3.Request("ngs.usage", []byte("1h"), time.Second); err != nil { + t.Fatalf("Expected a response") + } + } + + var sl server.ServiceLatency + for i := 0; i < totalSamples*2; i++ { + rmsg, err := rsub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Expected to receive latency metric: %d, %s", i, err) + } + if err = json.Unmarshal(rmsg.Data, &sl); err != nil { + t.Errorf("Unexpected error processing latency metric: %s", err) + } + if sl.ServiceLatency < 0 { + t.Fatalf("Unexpected negative service latency value: %v", sl) + } + if sl.SystemLatency < 0 { + t.Fatalf("Unexpected negative system latency value: %v", sl) + } + } +} + +func TestServiceLatencyFailureReportingSingleServer(t *testing.T) { + sc := createSuperCluster(t, 1, 1) + defer sc.shutdown() + + // Now add in new service export to FOO and have bar import that with tracking enabled. + sc.setupLatencyTracking(t, 100) + sc.setResponseThreshold(t, 20*time.Millisecond) + + nc := clientConnect(t, sc.clusters[0].opts[0], "foo") + defer nc.Close() + + // Listen for metrics + rsub, err := nc.SubscribeSync("results") + if err != nil { + t.Fatal(err) + } + nc.Flush() + + getMetricResult := func() *server.ServiceLatency { + t.Helper() + rmsg, err := rsub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Expected to receive latency metric: %v", err) + } + var sl server.ServiceLatency + if err = json.Unmarshal(rmsg.Data, &sl); err != nil { + t.Errorf("Unexpected error processing latency metric: %s", err) + } + return &sl + } + + // Same server + nc2 := clientConnect(t, sc.clusters[0].opts[0], "bar") + defer nc2.Close() + + // Test a request with no reply subject + nc2.Publish("ngs.usage", []byte("1h")) + sl := getMetricResult() + + if sl.Status != 400 { + t.Fatalf("Expected to get a bad request status [400], got %d", sl.Status) + } + + // Proper request, but no responders. + nc2.Request("ngs.usage", []byte("1h"), 20*time.Millisecond) + sl = getMetricResult() + if sl.Status != 503 { + t.Fatalf("Expected to get a service unavailable status [503], got %d", sl.Status) + } + + // The service listener. Make it slow. 20ms is respThreshold, so take 2X + sub, _ := nc.Subscribe("ngs.usage.bar", func(msg *nats.Msg) { + time.Sleep(40 * time.Millisecond) + msg.Respond([]byte("22 msgs")) + }) + nc.Flush() + + nc2.Request("ngs.usage", []byte("1h"), 20*time.Millisecond) + sl = getMetricResult() + if sl.Status != 504 { + t.Fatalf("Expected to get a service timeout status [504], got %d", sl.Status) + } + + // Make sure we do not get duplicates. + if rmsg, err := rsub.NextMsg(50 * time.Millisecond); err == nil { + t.Fatalf("Unexpected second response metric: %q\n", rmsg.Data) + } + + // Now setup a responder that will respond under the threshold. + sub.Unsubscribe() + nc.Subscribe("ngs.usage.bar", func(msg *nats.Msg) { + time.Sleep(5 * time.Millisecond) + msg.Respond([]byte("22 msgs")) + }) + nc.Flush() + time.Sleep(100 * time.Millisecond) + + // Now create a responder using old request and we will do a short timeout. + nc3 := clientConnectOldRequest(t, sc.clusters[0].opts[0], "bar") + defer nc3.Close() + + nc3.Request("ngs.usage", []byte("1h"), time.Millisecond) + sl = getMetricResult() + if sl.Status != 408 { + t.Fatalf("Expected to get a request timeout status [408], got %d", sl.Status) + } +} + +func TestServiceLatencyFailureReportingMultipleServers(t *testing.T) { + sc := createSuperCluster(t, 3, 3) + defer sc.shutdown() + + // Now add in new service export to FOO and have bar import that with tracking enabled. + sc.setupLatencyTracking(t, 100) + sc.setResponseThreshold(t, 10*time.Millisecond) + + nc := clientConnect(t, sc.clusters[0].opts[0], "foo") + defer nc.Close() + + // Listen for metrics + rsub, err := nc.SubscribeSync("results") + if err != nil { + t.Fatal(err) + } + nc.Flush() + + getMetricResult := func() *server.ServiceLatency { + t.Helper() + rmsg, err := rsub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Expected to receive latency metric: %v", err) + } + var sl server.ServiceLatency + if err = json.Unmarshal(rmsg.Data, &sl); err != nil { + t.Errorf("Unexpected error processing latency metric: %s", err) + } + return &sl + } + + cases := []struct { + ci, si int + desc string + }{ + {0, 0, "same server"}, + {0, 1, "same cluster, different server"}, + {1, 1, "different cluster"}, + } + + for _, cs := range cases { + // Select the server to send request from. + nc2 := clientConnect(t, sc.clusters[cs.ci].opts[cs.si], "bar") + defer nc2.Close() + + // Test a request with no reply subject + nc2.Publish("ngs.usage", []byte("1h")) + sl := getMetricResult() + if sl.Status != 400 { + t.Fatalf("Test %q, Expected to get a bad request status [400], got %d", cs.desc, sl.Status) + } + + // We wait here for the gateways to report no interest b/c optimistic mode. + time.Sleep(50 * time.Millisecond) + + // Proper request, but no responders. + nc2.Request("ngs.usage", []byte("1h"), 10*time.Millisecond) + sl = getMetricResult() + if sl.Status != 503 { + t.Fatalf("Test %q, Expected to get a service unavailable status [503], got %d", cs.desc, sl.Status) + } + + // The service listener. Make it slow. 10ms is respThreshold, so make 3X + sub, _ := nc.Subscribe("ngs.usage.bar", func(msg *nats.Msg) { + time.Sleep(30 * time.Millisecond) + msg.Respond([]byte("22 msgs")) + }) + defer sub.Unsubscribe() + nc.Flush() + // Wait to propagate. + time.Sleep(200 * time.Millisecond) + + nc2.Request("ngs.usage", []byte("1h"), 10*time.Millisecond) + sl = getMetricResult() + if sl.Status != 504 { + t.Fatalf("Test %q, Expected to get a service timeout status [504], got %d", cs.desc, sl.Status) + } + + // Clean up subscriber and requestor + nc2.Close() + sub.Unsubscribe() + nc.Flush() + // Wait to propagate. + time.Sleep(200 * time.Millisecond) + } +} + +// To test a bug rip@nats.io is seeing. +func TestServiceLatencyOldRequestStyleSingleServer(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + accounts: { + SVC: { + users: [ {user: svc, password: pass} ] + exports: [ { + service: "svc.echo" + accounts: [CLIENT] + latency: { + sampling: 100% + subject: latency.svc + } + } ] + }, + CLIENT: { + users: [{user: client, password: pass} ] + imports: [ {service: {account: SVC, subject: svc.echo}, to: SVC} ] + }, + SYS: { users: [{user: admin, password: pass}] } + } + + system_account: SYS + `)) + + srv, opts := RunServerWithConfig(conf) + defer srv.Shutdown() + + // Responder + nc, err := nats.Connect(fmt.Sprintf("nats://svc:pass@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // Listen for metrics + rsub, _ := nc.SubscribeSync("latency.svc") + + // Requestor + nc2, err := nats.Connect(fmt.Sprintf("nats://client:pass@%s:%d", opts.Host, opts.Port), nats.UseOldRequestStyle()) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + // Setup responder + serviceTime := 25 * time.Millisecond + sub, _ := nc.Subscribe("svc.echo", func(msg *nats.Msg) { + time.Sleep(serviceTime) + msg.Respond([]byte("world")) + }) + nc.Flush() + defer sub.Unsubscribe() + + // Send a request + start := time.Now() + if _, err := nc2.Request("SVC", []byte("1h"), time.Second); err != nil { + t.Fatalf("Expected a response") + } + + var sl server.ServiceLatency + rmsg, _ := rsub.NextMsg(time.Second) + json.Unmarshal(rmsg.Data, &sl) + + checkServiceLatency(t, sl, start, serviceTime) + extendedCheck(t, sl.Responder, "svc", "", srv.Name()) + noShareCheck(t, sl.Requestor) +} + +// To test a bug wally@nats.io is seeing. +func TestServiceAndStreamStackOverflow(t *testing.T) { + conf := createConfFile(t, []byte(` + accounts { + STATIC { + users = [ { user: "static", pass: "foo" } ] + exports [ + { stream: > } + { service: my.service } + ] + } + DYN { + users = [ { user: "foo", pass: "bar" } ] + imports [ + { stream { subject: >, account: STATIC } } + { service { subject: my.service, account: STATIC } } + ] + } + } + `)) + + srv, opts := RunServerWithConfig(conf) + defer srv.Shutdown() + + // Responder (just request sub) + nc, err := nats.Connect(fmt.Sprintf("nats://static:foo@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + sub, _ := nc.SubscribeSync("my.service") + nc.Flush() + + // Requestor + nc2, err := nats.Connect(fmt.Sprintf("nats://foo:bar@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + // Send a single request. + nc2.PublishRequest("my.service", "foo", []byte("hi")) + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if nm, _, err := sub.Pending(); err != nil || nm != 1 { + return fmt.Errorf("Expected one request, got %d", nm) + } + return nil + }) + + // Make sure works for queue subscribers as well. + sub.Unsubscribe() + sub, _ = nc.QueueSubscribeSync("my.service", "prod") + nc.Flush() + + // Send a single request. + nc2.PublishRequest("my.service", "foo", []byte("hi")) + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if nm, _, err := sub.Pending(); err != nil || nm != 1 { + return fmt.Errorf("Expected one request, got %d", nm) + } + return nil + }) + + // Now create an interest in the stream from nc2. that is a queue subscriber. + sub2, _ := nc2.QueueSubscribeSync("my.service", "prod") + defer sub2.Unsubscribe() + nc2.Flush() + + // Send a single request. + nc2.PublishRequest("my.service", "foo", []byte("hi")) + time.Sleep(10 * time.Millisecond) + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if nm, _, err := sub.Pending(); err != nil || nm != 2 { + return fmt.Errorf("Expected two requests, got %d", nm) + } + return nil + }) +} + +// Check we get the proper detailed information for the requestor when allowed. +func TestServiceLatencyRequestorSharesDetailedInfo(t *testing.T) { + sc := createSuperCluster(t, 3, 3) + defer sc.shutdown() + + // Now add in new service export to FOO and have bar import that with tracking enabled. + sc.setupLatencyTracking(t, 100) + sc.setResponseThreshold(t, 10*time.Millisecond) + sc.setImportShare(t) + + nc := clientConnect(t, sc.clusters[0].opts[0], "foo") + defer nc.Close() + + // Listen for metrics + rsub, err := nc.SubscribeSync("results") + if err != nil { + t.Fatal(err) + } + nc.Flush() + + getMetricResult := func() *server.ServiceLatency { + t.Helper() + rmsg, err := rsub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Expected to receive latency metric: %v", err) + } + var sl server.ServiceLatency + if err = json.Unmarshal(rmsg.Data, &sl); err != nil { + t.Errorf("Unexpected error processing latency metric: %s", err) + } + return &sl + } + + cases := []struct { + ci, si int + desc string + }{ + {0, 0, "same server"}, + {0, 1, "same cluster, different server"}, + {1, 1, "different cluster"}, + } + + for _, cs := range cases { + // Select the server to send request from. + nc2 := clientConnect(t, sc.clusters[cs.ci].opts[cs.si], "bar") + defer nc2.Close() + + rs := sc.clusters[cs.ci].servers[cs.si] + + // Test a request with no reply subject + nc2.Publish("ngs.usage", []byte("1h")) + sl := getMetricResult() + if sl.Status != 400 { + t.Fatalf("Test %q, Expected to get a bad request status [400], got %d", cs.desc, sl.Status) + } + extendedCheck(t, sl.Requestor, "bar", "", rs.Name()) + + // We wait here for the gateways to report no interest b/c optimistic mode. + time.Sleep(50 * time.Millisecond) + + // Proper request, but no responders. + nc2.Request("ngs.usage", []byte("1h"), 10*time.Millisecond) + sl = getMetricResult() + if sl.Status != 503 { + t.Fatalf("Test %q, Expected to get a service unavailable status [503], got %d", cs.desc, sl.Status) + } + extendedCheck(t, sl.Requestor, "bar", "", rs.Name()) + + // The service listener. Make it slow. 10ms is respThreshold, so take 2.5X + sub, _ := nc.Subscribe("ngs.usage.bar", func(msg *nats.Msg) { + time.Sleep(25 * time.Millisecond) + msg.Respond([]byte("22 msgs")) + }) + defer sub.Unsubscribe() + nc.Flush() + // Wait to propagate. + time.Sleep(200 * time.Millisecond) + + nc2.Request("ngs.usage", []byte("1h"), 10*time.Millisecond) + sl = getMetricResult() + if sl.Status != 504 { + t.Fatalf("Test %q, Expected to get a service timeout status [504], got %d", cs.desc, sl.Status) + } + extendedCheck(t, sl.Requestor, "bar", "", rs.Name()) + + // Clean up subscriber and requestor + nc2.Close() + sub.Unsubscribe() + nc.Flush() + // Wait to propagate. + time.Sleep(200 * time.Millisecond) + } +} + +func TestServiceLatencyRequestorSharesConfig(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + accounts: { + SVC: { + users: [ {user: svc, password: pass} ] + exports: [ { + service: "svc.echo" + accounts: [CLIENT] + latency: { + sampling: 100% + subject: latency.svc + } + } ] + }, + CLIENT: { + users: [{user: client, password: pass} ] + imports: [ {service: {account: SVC, subject: svc.echo}, to: SVC, share:true} ] + }, + SYS: { users: [{user: admin, password: pass}] } + } + + system_account: SYS + `)) + + srv, opts := RunServerWithConfig(conf) + defer srv.Shutdown() + + // Responder + nc, err := nats.Connect(fmt.Sprintf("nats://svc:pass@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // Listen for metrics + rsub, _ := nc.SubscribeSync("latency.svc") + + // Requestor + nc2, err := nats.Connect(fmt.Sprintf("nats://client:pass@%s:%d", opts.Host, opts.Port), nats.UseOldRequestStyle()) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + // Setup responder + serviceTime := 25 * time.Millisecond + sub, _ := nc.Subscribe("svc.echo", func(msg *nats.Msg) { + time.Sleep(serviceTime) + msg.Respond([]byte("world")) + }) + nc.Flush() + defer sub.Unsubscribe() + + // Send a request + start := time.Now() + if _, err := nc2.Request("SVC", []byte("1h"), time.Second); err != nil { + t.Fatalf("Expected a response") + } + + var sl server.ServiceLatency + rmsg, _ := rsub.NextMsg(time.Second) + json.Unmarshal(rmsg.Data, &sl) + + checkServiceLatency(t, sl, start, serviceTime) + extendedCheck(t, sl.Responder, "svc", "", srv.Name()) + extendedCheck(t, sl.Requestor, "client", "", srv.Name()) + + // Check reload. + newConf := []byte(` + listen: 127.0.0.1:-1 + accounts: { + SVC: { + users: [ {user: svc, password: pass} ] + exports: [ { + service: "svc.echo" + accounts: [CLIENT] + latency: { + sampling: 100% + subject: latency.svc + } + } ] + }, + CLIENT: { + users: [{user: client, password: pass} ] + imports: [ {service: {account: SVC, subject: svc.echo}, to: SVC} ] + }, + SYS: { users: [{user: admin, password: pass}] } + } + + system_account: SYS + `) + if err := os.WriteFile(conf, newConf, 0600); err != nil { + t.Fatalf("Error rewriting server's config file: %v", err) + } + if err := srv.Reload(); err != nil { + t.Fatalf("Error on server reload: %v", err) + } + + if _, err = nc2.Request("SVC", []byte("1h"), time.Second); err != nil { + t.Fatalf("Expected a response") + } + + var sl2 server.ServiceLatency + rmsg, _ = rsub.NextMsg(time.Second) + json.Unmarshal(rmsg.Data, &sl2) + noShareCheck(t, sl2.Requestor) +} + +func TestServiceLatencyLossTest(t *testing.T) { + // assure that behavior with respect to requests timing out (and samples being reordered) is as expected. + conf := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + accounts: { + SVC: { + users: [ {user: svc, password: pass} ] + exports: [ { + service: "svc.echo" + threshold: "500ms" + accounts: [CLIENT] + latency: { + sampling: headers + subject: latency.svc + } + } ] + }, + CLIENT: { + users: [{user: client, password: pass} ] + imports: [ {service: {account: SVC, subject: svc.echo}, to: SVC, share:true} ] + }, + SYS: { users: [{user: admin, password: pass}] } + } + system_account: SYS + `)) + srv, opts := RunServerWithConfig(conf) + defer srv.Shutdown() + + // Responder connection + ncr, err := nats.Connect(fmt.Sprintf("nats://svc:pass@%s:%d", opts.Host, opts.Port), nats.Name("responder")) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncr.Close() + + ncl, err := nats.Connect(fmt.Sprintf("nats://svc:pass@%s:%d", opts.Host, opts.Port), nats.Name("latency")) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer ncl.Close() + // Table of expected state for which message. + // This also codifies that the first message, in respsonse to second request is ok. + // Second message, in response to first request times out. + expectedState := map[int]int{1: http.StatusOK, 2: http.StatusGatewayTimeout} + msgCnt := 0 + start := time.Now().Add(250 * time.Millisecond) + + var latErr []error + // Listen for metrics + wg := sync.WaitGroup{} + wg.Add(2) + rsub, _ := ncl.Subscribe("latency.svc", func(rmsg *nats.Msg) { + defer wg.Done() + var sl server.ServiceLatency + json.Unmarshal(rmsg.Data, &sl) + msgCnt++ + if want := expectedState[msgCnt]; want != sl.Status { + latErr = append(latErr, fmt.Errorf("Expected different status for msg #%d: %d != %d", msgCnt, want, sl.Status)) + } + if msgCnt > 1 { + if start.Before(sl.RequestStart) { + latErr = append(latErr, fmt.Errorf("start times should indicate reordering %v : %v", start, sl.RequestStart)) + } + } + start = sl.RequestStart + if strings.EqualFold(sl.RequestHeader.Get("Uber-Trace-Id"), fmt.Sprintf("msg-%d", msgCnt)) { + latErr = append(latErr, fmt.Errorf("no header present")) + } + }) + defer rsub.Unsubscribe() + // Setup requestor + nc2, err := nats.Connect(fmt.Sprintf("nats://client:pass@%s:%d", opts.Host, opts.Port), + nats.UseOldRequestStyle(), nats.Name("requestor")) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + respCnt := int64(0) + reply := nc2.NewRespInbox() + repSub, _ := nc2.Subscribe(reply, func(msg *nats.Msg) { + atomic.AddInt64(&respCnt, 1) + }) + defer repSub.Unsubscribe() + nc2.Flush() + // use dedicated send that publishes requests using same reply subject + send := func(msg string) { + if err := nc2.PublishMsg(&nats.Msg{Subject: "SVC", Data: []byte(msg), Reply: reply, + Header: nats.Header{"X-B3-Sampled": []string{"1"}}}); err != nil { + t.Fatalf("Expected a response got: %v", err) + } + } + // Setup responder that skips responding and triggers next request OR responds + sub, _ := ncr.Subscribe("svc.echo", func(msg *nats.Msg) { + if string(msg.Data) != "msg2" { + msg.Respond([]byte("response")) + } else { + wg.Add(1) + go func() { // second request (use go routine to not block in responders callback) + defer wg.Done() + time.Sleep(250 * time.Millisecond) + send("msg1") // will cause the first latency measurement + }() + } + }) + ncr.Flush() + ncl.Flush() + nc2.Flush() + defer sub.Unsubscribe() + // Send first request, which is expected to timeout + send("msg2") + // wait till we got enough responses + wg.Wait() + if len(latErr) > 0 { + t.Fatalf("Got errors %v", latErr) + } + if atomic.LoadInt64(&respCnt) != 1 { + t.Fatalf("Expected only one message") + } +} + +func TestServiceLatencyHeaderTriggered(t *testing.T) { + receiveAndTest := func(t *testing.T, rsub *nats.Subscription, shared bool, header nats.Header, status int, srvName string) server.ServiceLatency { + t.Helper() + var sl server.ServiceLatency + rmsg, _ := rsub.NextMsg(time.Second) + if rmsg == nil { + t.Fatal("Expected message") + return sl + } + json.Unmarshal(rmsg.Data, &sl) + if sl.Status != status { + t.Fatalf("Expected different status %d != %d", status, sl.Status) + } + if status == http.StatusOK { + extendedCheck(t, sl.Responder, "svc", "", srvName) + } + if shared { + extendedCheck(t, sl.Requestor, "client", "", srvName) + } else { + noShareCheck(t, sl.Requestor) + } + // header are always included + if v := sl.RequestHeader.Get("Some-Other"); v != "" { + t.Fatalf("Expected header to be gone") + } + for k, value := range header { + if v := sl.RequestHeader.Get(k); v != value[0] { + t.Fatalf("Expected header %q to be set", k) + } + } + return sl + } + for _, v := range []struct { + shared bool + header nats.Header + }{ + {true, nats.Header{"Uber-Trace-Id": []string{"479fefe9525eddb:479fefe9525eddb:0:1"}}}, + {true, nats.Header{"X-B3-Sampled": []string{"1"}}}, + {true, nats.Header{"X-B3-TraceId": []string{"80f198ee56343ba864fe8b2a57d3eff7"}}}, + {true, nats.Header{"B3": []string{"80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-1-05e3ac9a4f6e3b90"}}}, + {true, nats.Header{"Traceparent": []string{"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"}}}, + {false, nats.Header{"Uber-Trace-Id": []string{"479fefe9525eddb:479fefe9525eddb:0:1"}}}, + {false, nats.Header{"X-B3-Sampled": []string{"1"}}}, + {false, nats.Header{"X-B3-TraceId": []string{"80f198ee56343ba864fe8b2a57d3eff7"}}}, + {false, nats.Header{"B3": []string{"80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-1-05e3ac9a4f6e3b90"}}}, + {false, nats.Header{"Traceparent": []string{"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"}}}, + {false, nats.Header{ + "X-B3-TraceId": []string{"80f198ee56343ba864fe8b2a57d3eff7"}, + "X-B3-ParentSpanId": []string{"05e3ac9a4f6e3b90"}, + "X-B3-SpanId": []string{"e457b5a2e4d86bd1"}, + "X-B3-Sampled": []string{"1"}, + }}, + {false, nats.Header{ + "X-B3-TraceId": []string{"80f198ee56343ba864fe8b2a57d3eff7"}, + "X-B3-ParentSpanId": []string{"05e3ac9a4f6e3b90"}, + "X-B3-SpanId": []string{"e457b5a2e4d86bd1"}, + }}, + {false, nats.Header{ + "Uber-Trace-Id": []string{"479fefe9525eddb:479fefe9525eddb:0:1"}, + "Uberctx-X": []string{"foo"}, + }}, + {false, nats.Header{ + "Traceparent": []string{"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"}, + "Tracestate": []string{"rojo=00f067aa0ba902b7,congo=t61rcWkgMzE"}, + }}, + } { + t.Run(fmt.Sprintf("%s_%t_%s", t.Name(), v.shared, v.header), func(t *testing.T) { + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:-1 + accounts: { + SVC: { + users: [ {user: svc, password: pass} ] + exports: [ { + service: "svc.echo" + accounts: [CLIENT] + latency: { + sampling: headers + subject: latency.svc + } + }] + }, + CLIENT: { + users: [{user: client, password: pass} ] + imports: [ {service: {account: SVC, subject: svc.echo}, to: SVC, share:%t} ] + }, + SYS: { users: [{user: admin, password: pass}] } + } + + system_account: SYS + `, v.shared))) + srv, opts := RunServerWithConfig(conf) + defer srv.Shutdown() + + // Responder + nc, err := nats.Connect(fmt.Sprintf("nats://svc:pass@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // Listen for metrics + rsub, _ := nc.SubscribeSync("latency.svc") + defer rsub.Unsubscribe() + + // Setup responder + serviceTime := 25 * time.Millisecond + sub, _ := nc.Subscribe("svc.echo", func(msg *nats.Msg) { + time.Sleep(serviceTime) + msg.Respond([]byte("world")) + }) + nc.Flush() + defer sub.Unsubscribe() + + // Setup requestor + nc2, err := nats.Connect(fmt.Sprintf("nats://client:pass@%s:%d", opts.Host, opts.Port), nats.UseOldRequestStyle()) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + // Send a request + start := time.Now() + msg := &nats.Msg{ + Subject: "SVC", + Data: []byte("1h"), + Header: make(nats.Header), + } + for k, v := range v.header { + for _, val := range v { + msg.Header.Add(k, val) + } + } + msg.Header.Add("Some-Other", "value") + if _, err := nc2.RequestMsg(msg, 50*time.Millisecond); err != nil { + t.Fatalf("Expected a response") + } + sl := receiveAndTest(t, rsub, v.shared, v.header, http.StatusOK, srv.Name()) + checkServiceLatency(t, sl, start, serviceTime) + // shut down responder to test various error scenarios + sub.Unsubscribe() + nc.Flush() + // Send a request without responder + if _, err := nc2.RequestMsg(msg, 50*time.Millisecond); err == nil { + t.Fatalf("Expected no response") + } + receiveAndTest(t, rsub, v.shared, v.header, http.StatusServiceUnavailable, srv.Name()) + + // send a message without a response + msg.Reply = "" + if err := nc2.PublishMsg(msg); err != nil { + t.Fatalf("Expected no error got %v", err) + } + receiveAndTest(t, rsub, v.shared, v.header, http.StatusBadRequest, srv.Name()) + }) + } +} + +// From a report by rip@nats.io on simple latency reporting missing in 2 server cluster setup. +func TestServiceLatencyMissingResults(t *testing.T) { + accConf := createConfFile(t, []byte(` + accounts { + one: { + users = [ {user: one, password: password} ] + imports = [ {service: {account: weather, subject: service.weather.requests.>}, to: service.weather.>, share: true} ] + } + weather: { + users = [ {user: weather, password: password} ] + exports = [ { + service: service.weather.requests.> + accounts: [one] + latency: { sampling: 100%, subject: service.weather.latency } + } ] + } + } + `)) + + s1Conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:-1 + server_name: s1 + cluster { port: -1 } + include %q + `, filepath.Base(accConf)))) + + // Link accConf for relative import from s1Conf + os.Link(accConf, filepath.Join(filepath.Dir(s1Conf), filepath.Base(accConf))) + + s1, opts1 := RunServerWithConfig(s1Conf) + defer s1.Shutdown() + + s2Conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:-1 + server_name: s2 + cluster { + port: -1 + routes = [ nats-route://127.0.0.1:%d ] + } + include %q + `, opts1.Cluster.Port, filepath.Base(accConf)))) + + // Link accConf for relative import from s2Conf + os.Link(accConf, filepath.Join(filepath.Dir(s2Conf), filepath.Base(accConf))) + + s2, opts2 := RunServerWithConfig(s2Conf) + defer s2.Shutdown() + + checkClusterFormed(t, s1, s2) + + nc1, err := nats.Connect(fmt.Sprintf("nats://%s:%s@%s:%d", "weather", "password", opts1.Host, opts1.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc1.Close() + + // Create responder + sub, _ := nc1.Subscribe("service.weather.requests.>", func(msg *nats.Msg) { + time.Sleep(25 * time.Millisecond) + msg.Respond([]byte("sunny!")) + }) + defer sub.Unsubscribe() + + // Create sync listener for latency. + latSubj := "service.weather.latency" + lsub, _ := nc1.SubscribeSync(latSubj) + defer lsub.Unsubscribe() + nc1.Flush() + + // Make sure the subscription propagates to s2 server. + checkSubInterest(t, s2, "weather", latSubj, time.Second) + + // Create requestor on s2. + nc2, err := nats.Connect(fmt.Sprintf("nats://%s:%s@%s:%d", "one", "password", opts2.Host, opts2.Port), nats.UseOldRequestStyle()) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + nc2.Request("service.weather.los_angeles", nil, time.Second) + + lr, err := lsub.NextMsg(time.Second) + if err != nil { + t.Fatalf("Expected a latency result, got %v", err) + } + // Make sure we reported ok and have valid results for service, system and total. + var sl server.ServiceLatency + json.Unmarshal(lr.Data, &sl) + + if sl.Status != 200 { + t.Fatalf("Expected a 200 status, got %d\n", sl.Status) + } + if sl.ServiceLatency == 0 || sl.SystemLatency == 0 || sl.TotalLatency == 0 { + t.Fatalf("Received invalid tracking measurements, %d %d %d", sl.ServiceLatency, sl.SystemLatency, sl.TotalLatency) + } +} + +// To test a bug I was seeing. +func TestServiceLatencyDoubleResponse(t *testing.T) { + sc := createSuperCluster(t, 3, 1) + defer sc.shutdown() + + // Now add in new service export to FOO and have bar import that with tracking enabled. + sc.setupLatencyTracking(t, 100) + + // Responder + nc := clientConnectWithName(t, sc.clusters[0].opts[0], "foo", "service22") + defer nc.Close() + + // Setup responder + sub, _ := nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { + msg.Respond([]byte("22 msgs")) + msg.Respond([]byte("boom")) + }) + nc.Flush() + defer sub.Unsubscribe() + + // Listen for metrics + rsub, _ := nc.SubscribeSync("latency.svc") + + // Requestor + nc2 := clientConnect(t, sc.clusters[0].opts[2], "bar") + defer nc2.Close() + + // Send a request + if _, err := nc2.Request("ngs.usage", []byte("1h"), time.Second); err != nil { + t.Fatalf("Expected a response") + } + + rsub.NextMsg(time.Second) + time.Sleep(time.Second) +} diff --git a/test/services_test.go b/test/services_test.go new file mode 100644 index 0000000..2aa8bce --- /dev/null +++ b/test/services_test.go @@ -0,0 +1,616 @@ +// Copyright 2020-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "bytes" + "fmt" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" +) + +var basicMASetupContents = []byte(` + server_name: A + listen: 127.0.0.1:-1 + + accounts: { + A: { + users: [ {user: a, password: pwd} ] + exports: [{service: "foo", response: stream}] + }, + B: { + users: [{user: b, password: pwd} ] + imports: [{ service: { account: A, subject: "foo"}, to: "foo_request" }] + } + } +`) + +func TestServiceImportWithStreamed(t *testing.T) { + conf := createConfFile(t, basicMASetupContents) + + srv, opts := RunServerWithConfig(conf) + defer srv.Shutdown() + + // Limit max response maps here for the test. + accB, err := srv.LookupAccount("B") + if err != nil { + t.Fatalf("Error looking up account: %v", err) + } + + // connect and offer a service + nc, err := nats.Connect(fmt.Sprintf("nats://a:pwd@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + nc.Subscribe("foo", func(msg *nats.Msg) { + if err := msg.Respond([]byte("world")); err != nil { + t.Fatalf("Error on respond: %v", err) + } + }) + nc.Flush() + + nc2, err := nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + numRequests := 10 + for i := 0; i < numRequests; i++ { + resp, err := nc2.Request("foo_request", []byte("hello"), 2*time.Second) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if resp == nil || strings.Compare("world", string(resp.Data)) != 0 { + t.Fatal("Did not receive the correct message") + } + } + + // Since we are using a new client that multiplexes, until the client itself goes away + // we will have the full number of entries, even with the tighter ResponseEntriesPruneThreshold. + accA, err := srv.LookupAccount("A") + if err != nil { + t.Fatalf("Error looking up account: %v", err) + } + + // These should always be the same now. + if nre := accB.NumPendingReverseResponses(); nre != numRequests { + t.Fatalf("Expected %d entries, got %d", numRequests, nre) + } + if nre := accA.NumPendingAllResponses(); nre != numRequests { + t.Fatalf("Expected %d entries, got %d", numRequests, nre) + } + + // Now kill of the client that was doing the requests. + nc2.Close() + + checkFor(t, time.Second, 10*time.Millisecond, func() error { + aNrssi := accA.NumPendingAllResponses() + bNre := accB.NumPendingReverseResponses() + if aNrssi != 0 || bNre != 0 { + return fmt.Errorf("Response imports and response entries should all be 0, got %d %d", aNrssi, bNre) + } + return nil + }) + + // Now let's test old style request and reply that uses a new inbox each time. This should work ok.. + nc2, err = nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port), nats.UseOldRequestStyle()) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + for i := 0; i < numRequests; i++ { + resp, err := nc2.Request("foo_request", []byte("hello"), 2*time.Second) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if resp == nil || strings.Compare("world", string(resp.Data)) != 0 { + t.Fatal("Did not receive the correct message") + } + } + + checkFor(t, time.Second, 10*time.Millisecond, func() error { + aNrssi := accA.NumPendingAllResponses() + bNre := accB.NumPendingReverseResponses() + if aNrssi != 0 || bNre != 0 { + return fmt.Errorf("Response imports and response entries should all be 0, got %d %d", aNrssi, bNre) + } + return nil + }) +} + +func TestServiceImportWithStreamedResponseAndEOF(t *testing.T) { + conf := createConfFile(t, basicMASetupContents) + + srv, opts := RunServerWithConfig(conf) + defer srv.Shutdown() + + accA, err := srv.LookupAccount("A") + if err != nil { + t.Fatalf("Error looking up account: %v", err) + } + accB, err := srv.LookupAccount("B") + if err != nil { + t.Fatalf("Error looking up account: %v", err) + } + + nc, err := nats.Connect(fmt.Sprintf("nats://a:pwd@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // We will send four responses and then and nil message signaling EOF + nc.Subscribe("foo", func(msg *nats.Msg) { + // Streamed response. + msg.Respond([]byte("world-1")) + msg.Respond([]byte("world-2")) + msg.Respond([]byte("world-3")) + msg.Respond([]byte("world-4")) + msg.Respond(nil) + }) + nc.Flush() + + // Now setup requester. + nc2, err := nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + numRequests := 10 + expectedResponses := 5 + + for i := 0; i < numRequests; i++ { + // Create an inbox + reply := nats.NewInbox() + sub, _ := nc2.SubscribeSync(reply) + defer sub.Unsubscribe() + + if err := nc2.PublishRequest("foo_request", reply, []byte("XOXO")); err != nil { + t.Fatalf("Error sending request: %v", err) + } + + // Wait and make sure we get all the responses. Should be five. + checkFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error { + if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != expectedResponses { + return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, expectedResponses) + } + return nil + }) + } + + if nre := accA.NumPendingAllResponses(); nre != 0 { + t.Fatalf("Expected no entries, got %d", nre) + } + if nre := accB.NumPendingReverseResponses(); nre != 0 { + t.Fatalf("Expected no entries, got %d", nre) + } +} + +func TestServiceExportsResponseFiltering(t *testing.T) { + conf := createConfFile(t, []byte(` + server_name: A + listen: 127.0.0.1:-1 + + accounts: { + A: { + users: [ {user: a, password: pwd} ] + exports: [ {service: "foo"}, {service: "bar"} ] + }, + B: { + users: [{user: b, password: pwd} ] + imports: [ {service: { account: A, subject: "foo"}}, {service: { account: A, subject: "bar"}, to: "baz"} ] + } + } + `)) + + srv, opts := RunServerWithConfig(conf) + defer srv.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("nats://a:pwd@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // If we do not subscribe the system is now smart enough to not setup the response service imports. + nc.SubscribeSync("foo") + nc.SubscribeSync("bar") + nc.Flush() + + nc2, err := nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + // We don't expect responses, so just do publishes. + // 5 for foo + sendFoo := 5 + for i := 0; i < sendFoo; i++ { + nc2.PublishRequest("foo", "reply", nil) + } + // 17 for bar + sendBar := 17 + for i := 0; i < sendBar; i++ { + nc2.PublishRequest("baz", "reply", nil) + } + nc2.Flush() + + accA, err := srv.LookupAccount("A") + if err != nil { + t.Fatalf("Error looking up account: %v", err) + } + + sendTotal := sendFoo + sendBar + if nre := accA.NumPendingAllResponses(); nre != sendTotal { + t.Fatalf("Expected %d entries, got %d", sendTotal, nre) + } + + if nre := accA.NumPendingResponses("foo"); nre != sendFoo { + t.Fatalf("Expected %d entries, got %d", sendFoo, nre) + } + + if nre := accA.NumPendingResponses("bar"); nre != sendBar { + t.Fatalf("Expected %d entries, got %d", sendBar, nre) + } +} + +func TestServiceExportsAutoDirectCleanup(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + accounts: { + A: { + users: [ {user: a, password: pwd} ] + exports: [ {service: "foo"} ] + }, + B: { + users: [{user: b, password: pwd} ] + imports: [ {service: { account: A, subject: "foo"}} ] + } + } + `)) + + srv, opts := RunServerWithConfig(conf) + defer srv.Shutdown() + + acc, err := srv.LookupAccount("A") + if err != nil { + t.Fatalf("Error looking up account: %v", err) + } + + // Potential resonder. + nc, err := nats.Connect(fmt.Sprintf("nats://a:pwd@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // Requestor + nc2, err := nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + expectNone := func() { + t.Helper() + if nre := acc.NumPendingAllResponses(); nre != 0 { + t.Fatalf("Expected no entries, got %d", nre) + } + } + + toSend := 10 + + // With no responders we should never register service import responses etc. + for i := 0; i < toSend; i++ { + nc2.PublishRequest("foo", "reply", nil) + } + nc2.Flush() + expectNone() + + // Now register a responder. + sub, _ := nc.Subscribe("foo", func(msg *nats.Msg) { + msg.Respond([]byte("world")) + }) + nc.Flush() + defer sub.Unsubscribe() + + // With no reply subject on a request we should never register service import responses etc. + for i := 0; i < toSend; i++ { + nc2.Publish("foo", nil) + } + nc2.Flush() + expectNone() + + // Create an old request style client. + nc3, err := nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port), nats.UseOldRequestStyle()) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc3.Close() + + // If the request loses interest before the response we should not queue up service import responses either. + // This only works for old style requests at the moment where we can detect interest going away. + delay := 25 * time.Millisecond + sub.Unsubscribe() + sub, _ = nc.Subscribe("foo", func(msg *nats.Msg) { + time.Sleep(delay) + msg.Respond([]byte("world")) + }) + nc.Flush() + defer sub.Unsubscribe() + + for i := 0; i < toSend; i++ { + nc3.Request("foo", nil, time.Millisecond) + } + nc3.Flush() + time.Sleep(time.Duration(toSend) * delay * 2) + expectNone() +} + +// In some instances we do not have a forceful trigger that signals us to clean up. +// Like a stream that does not send EOF or a responder who receives requests but does +// not answer. For these we will have an expectation of a response threshold which +// tells the system we should have seen a response by T, say 2 minutes, 30 seconds etc. +func TestServiceExportsPruningCleanup(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + accounts: { + A: { + users: [ {user: a, password: pwd} ] + exports: [ {service: "foo", response: stream} ] + }, + B: { + users: [{user: b, password: pwd} ] + imports: [ {service: { account: A, subject: "foo"}} ] + } + } + `)) + + srv, opts := RunServerWithConfig(conf) + defer srv.Shutdown() + + // Potential resonder. + nc, err := nats.Connect(fmt.Sprintf("nats://a:pwd@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // We will subscribe but not answer. + sub, _ := nc.Subscribe("foo", func(msg *nats.Msg) {}) + nc.Flush() + defer sub.Unsubscribe() + + acc, err := srv.LookupAccount("A") + if err != nil { + t.Fatalf("Error looking up account: %v", err) + } + + // Check on response thresholds. + rt, err := acc.ServiceExportResponseThreshold("foo") + if err != nil { + t.Fatalf("Error retrieving response threshold, %v", err) + } + + if rt != server.DEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD { + t.Fatalf("Expected the response threshold to be %v, got %v", + server.DEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD, rt) + } + // now set it + newRt := 10 * time.Millisecond + if err := acc.SetServiceExportResponseThreshold("foo", newRt); err != nil { + t.Fatalf("Expected no error setting response threshold, got %v", err) + } + + expectedPending := func(expected int) { + t.Helper() + // Caller is sleeping a bit before, but avoid flappers. + checkFor(t, time.Second, 15*time.Millisecond, func() error { + if nre := acc.NumPendingResponses("foo"); nre != expected { + return fmt.Errorf("Expected %d entries, got %d", expected, nre) + } + return nil + }) + } + + // Requestor + nc2, err := nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + toSend := 10 + + // This should register and they will dangle. Make sure we clean them up. + for i := 0; i < toSend; i++ { + nc2.PublishRequest("foo", "reply", nil) + } + nc2.Flush() + + expectedPending(10) + time.Sleep(4 * newRt) + expectedPending(0) + + // Do it again. + for i := 0; i < toSend; i++ { + nc2.PublishRequest("foo", "reply", nil) + } + nc2.Flush() + + expectedPending(10) + time.Sleep(4 * newRt) + expectedPending(0) +} + +func TestServiceExportsResponseThreshold(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + accounts: { + A: { + users: [ {user: a, password: pwd} ] + exports: [ {service: "foo", response: stream, threshold: "1s"} ] + }, + } + `)) + + srv, opts := RunServerWithConfig(conf) + defer srv.Shutdown() + + // Potential responder. + nc, err := nats.Connect(fmt.Sprintf("nats://a:pwd@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + // We will subscribe but not answer. + sub, _ := nc.Subscribe("foo", func(msg *nats.Msg) {}) + nc.Flush() + defer sub.Unsubscribe() + + acc, err := srv.LookupAccount("A") + if err != nil { + t.Fatalf("Error looking up account: %v", err) + } + + // Check on response thresholds. + rt, err := acc.ServiceExportResponseThreshold("foo") + if err != nil { + t.Fatalf("Error retrieving response threshold, %v", err) + } + if rt != 1*time.Second { + t.Fatalf("Expected response threshold to be %v, got %v", 1*time.Second, rt) + } +} + +func TestServiceExportsResponseThresholdChunked(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + accounts: { + A: { + users: [ {user: a, password: pwd} ] + exports: [ {service: "foo", response: chunked, threshold: "10ms"} ] + }, + B: { + users: [{user: b, password: pwd} ] + imports: [ {service: { account: A, subject: "foo"}} ] + } + } + `)) + + srv, opts := RunServerWithConfig(conf) + defer srv.Shutdown() + + // Responder. + nc, err := nats.Connect(fmt.Sprintf("nats://a:pwd@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + numChunks := 10 + + // Respond with 5ms gaps for total response time for all chunks and EOF > 50ms. + nc.Subscribe("foo", func(msg *nats.Msg) { + // Streamed response. + for i := 1; i <= numChunks; i++ { + time.Sleep(5 * time.Millisecond) + msg.Respond([]byte(fmt.Sprintf("chunk-%d", i))) + } + msg.Respond(nil) + }) + nc.Flush() + + // Now setup requester. + nc2, err := nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + // Create an inbox + reply := nats.NewInbox() + sub, _ := nc2.SubscribeSync(reply) + defer sub.Unsubscribe() + + if err := nc2.PublishRequest("foo", reply, nil); err != nil { + t.Fatalf("Error sending request: %v", err) + } + + checkFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error { + if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != numChunks+1 { + return fmt.Errorf("Did not receive correct number of chunks: %d vs %d", nmsgs, numChunks+1) + } + return nil + }) +} + +func TestServiceAllowResponsesPerms(t *testing.T) { + conf := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + accounts: { + A: { + users: [ {user: a, password: pwd, permissions = {subscribe=foo, allow_responses=true}} ] + exports: [ {service: "foo"} ] + }, + B: { + users: [{user: b, password: pwd} ] + imports: [ {service: { account: A, subject: "foo"}} ] + } + } + `)) + + srv, opts := RunServerWithConfig(conf) + defer srv.Shutdown() + + // Responder. + nc, err := nats.Connect(fmt.Sprintf("nats://a:pwd@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc.Close() + + reply := []byte("Hello") + // Respond with 5ms gaps for total response time for all chunks and EOF > 50ms. + nc.Subscribe("foo", func(msg *nats.Msg) { + msg.Respond(reply) + }) + nc.Flush() + + // Now setup requester. + nc2, err := nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port)) + if err != nil { + t.Fatalf("Error on connect: %v", err) + } + defer nc2.Close() + + resp, err := nc2.Request("foo", []byte("help"), time.Second) + if err != nil { + t.Fatalf("Error expecting response %v", err) + } + if !bytes.Equal(resp.Data, reply) { + t.Fatalf("Did not get correct response, %q vs %q", resp.Data, reply) + } +} diff --git a/test/system_services_test.go b/test/system_services_test.go new file mode 100644 index 0000000..7792116 --- /dev/null +++ b/test/system_services_test.go @@ -0,0 +1,355 @@ +// Copyright 2019-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "math/rand/v2" + "net/url" + "strconv" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" +) + +const dbgSubs = "$SYS.DEBUG.SUBSCRIBERS" + +func (sc *supercluster) selectRandomServer() *server.Options { + ci := rand.Int32N(int32(len(sc.clusters))) + si := rand.Int32N(int32(len(sc.clusters[ci].servers))) + return sc.clusters[ci].opts[si] +} + +func (sc *supercluster) setupSystemServicesImports(t *testing.T, account string) { + t.Helper() + for _, c := range sc.clusters { + for _, s := range c.servers { + sysAcc := s.SystemAccount() + if sysAcc == nil { + t.Fatalf("System account not set") + } + acc, err := s.LookupAccount(account) + if err != nil { + t.Fatalf("Error looking up account '%s': %v", account, err) + } + if err := acc.AddServiceImport(sysAcc, dbgSubs, dbgSubs); err != nil { + t.Fatalf("Error adding subscribers debug service to '%s': %v", account, err) + } + } + } +} + +func numSubs(t *testing.T, msg *nats.Msg) int { + t.Helper() + if msg == nil || msg.Data == nil { + t.Fatalf("No response") + } + n, err := strconv.Atoi(string(msg.Data)) + if err != nil { + t.Fatalf("Got non-number response: %v", err) + } + return n +} + +func checkDbgNumSubs(t *testing.T, nc *nats.Conn, subj string, expected int) { + t.Helper() + var n int + for i := 0; i < 3; i++ { + response, err := nc.Request(dbgSubs, []byte(subj), 250*time.Millisecond) + if err != nil { + continue + } + if n = numSubs(t, response); n == expected { + return + } + } + t.Fatalf("Expected %d subscribers, got %d", expected, n) +} + +func TestSystemServiceSubscribers(t *testing.T) { + numServers, numClusters := 3, 3 + sc := createSuperCluster(t, numServers, numClusters) + defer sc.shutdown() + + sc.setupSystemServicesImports(t, "FOO") + + nc := clientConnect(t, sc.clusters[0].opts[0], "foo") + defer nc.Close() + + checkInterest := func(expected int) { + t.Helper() + checkDbgNumSubs(t, nc, "foo.bar", expected) + } + + checkInterest(0) + + // Now add in local subscribers. + for i := 0; i < 5; i++ { + nc := clientConnect(t, sc.clusters[0].opts[0], "foo") + defer nc.Close() + nc.SubscribeSync("foo.bar") + nc.SubscribeSync("foo.*") + nc.Flush() + } + + checkInterest(10) + + // Now create remote subscribers at random. + for i := 0; i < 90; i++ { + nc := clientConnect(t, sc.selectRandomServer(), "foo") + defer nc.Close() + nc.SubscribeSync("foo.bar") + nc.Flush() + } + + checkInterest(100) +} + +// Test that we can match wildcards. So sub may be foo.bar and we ask about foo.*, that should work. +func TestSystemServiceSubscribersWildcards(t *testing.T) { + numServers, numClusters := 3, 3 + sc := createSuperCluster(t, numServers, numClusters) + defer sc.shutdown() + + sc.setupSystemServicesImports(t, "FOO") + + nc := clientConnect(t, sc.clusters[0].opts[0], "foo") + defer nc.Close() + + for i := 0; i < 50; i++ { + nc := clientConnect(t, sc.selectRandomServer(), "foo") + defer nc.Close() + nc.SubscribeSync(fmt.Sprintf("foo.bar.%d", i+1)) + nc.SubscribeSync(fmt.Sprintf("%d", i+1)) + nc.Flush() + } + + checkDbgNumSubs(t, nc, "foo.bar.*", 50) + + checkDbgNumSubs(t, nc, "foo.>", 50) + + checkDbgNumSubs(t, nc, "foo.bar.22", 1) + + response, _ := nc.Request(dbgSubs, []byte("_INBOX.*.*"), time.Second) + hasInbox := numSubs(t, response) + + checkDbgNumSubs(t, nc, ">", 100+hasInbox) +} + +// Test that we can match on queue groups as well. Separate request payload with any whitespace. +func TestSystemServiceSubscribersQueueGroups(t *testing.T) { + numServers, numClusters := 3, 3 + sc := createSuperCluster(t, numServers, numClusters) + defer sc.shutdown() + + sc.setupSystemServicesImports(t, "FOO") + + nc := clientConnect(t, sc.clusters[0].opts[0], "foo") + defer nc.Close() + + for i := 0; i < 10; i++ { + nc := clientConnect(t, sc.selectRandomServer(), "foo") + defer nc.Close() + subj := fmt.Sprintf("foo.bar.%d", i+1) + nc.QueueSubscribeSync(subj, "QG.11") + nc.QueueSubscribeSync("foo.baz", "QG.33") + nc.Flush() + } + + for i := 0; i < 23; i++ { + nc := clientConnect(t, sc.selectRandomServer(), "foo") + defer nc.Close() + subj := fmt.Sprintf("foo.bar.%d", i+1) + nc.QueueSubscribeSync(subj, "QG.22") + nc.QueueSubscribeSync("foo.baz", "QG.22") + nc.Flush() + } + + checkDbgNumSubs(t, nc, "foo.bar.*", 33) + + checkDbgNumSubs(t, nc, "foo.bar.22 QG.22", 1) + + checkDbgNumSubs(t, nc, "foo.bar.2", 2) + + checkDbgNumSubs(t, nc, "foo.baz", 33) + + checkDbgNumSubs(t, nc, "foo.baz QG.22", 23) + + // Now check qfilters work on wildcards too. + checkDbgNumSubs(t, nc, "foo.bar.> QG.11", 10) + + checkDbgNumSubs(t, nc, "*.baz QG.22", 23) + + checkDbgNumSubs(t, nc, "foo.*.22 QG.22", 1) +} + +func TestSystemServiceSubscribersLeafNodesWithoutSystem(t *testing.T) { + numServers, numClusters := 3, 3 + sc := createSuperCluster(t, numServers, numClusters) + defer sc.shutdown() + + sc.setupSystemServicesImports(t, "FOO") + + ci := rand.Int32N(int32(len(sc.clusters))) + si := rand.Int32N(int32(len(sc.clusters[ci].servers))) + s, opts := sc.clusters[ci].servers[si], sc.clusters[ci].opts[si] + url := fmt.Sprintf("nats://%s:pass@%s:%d", "foo", opts.Host, opts.LeafNode.Port) + ls, lopts := runSolicitLeafServerToURL(url) + defer ls.Shutdown() + + checkLeafNodeConnected(t, s) + + // This is so we can test when the subs on a leafnode are flushed to the connected supercluster. + fsubj := "__leaf.flush__" + fc := clientConnect(t, opts, "foo") + defer fc.Close() + fc.Subscribe(fsubj, func(m *nats.Msg) { + m.Respond(nil) + }) + + lnc := clientConnect(t, lopts, "$G") + defer lnc.Close() + + flushLeaf := func() { + if _, err := lnc.Request(fsubj, nil, time.Second); err != nil { + t.Fatalf("Did not flush through to the supercluster: %v", err) + } + } + + for i := 0; i < 10; i++ { + nc := clientConnect(t, sc.selectRandomServer(), "foo") + defer nc.Close() + nc.SubscribeSync(fmt.Sprintf("foo.bar.%d", i+1)) + nc.QueueSubscribeSync("foo.bar.baz", "QG.22") + nc.Flush() + } + + nc := clientConnect(t, sc.clusters[0].opts[0], "foo") + defer nc.Close() + + checkDbgNumSubs(t, nc, "foo.bar.*", 20) + + checkDbgNumSubs(t, nc, "foo.bar.3", 1) + + lnc.SubscribeSync("foo.bar.3") + lnc.QueueSubscribeSync("foo.bar.baz", "QG.22") + + // We could flush here but that does not guarantee we have flushed through to the supercluster. + flushLeaf() + + checkDbgNumSubs(t, nc, "foo.bar.3", 2) + + checkDbgNumSubs(t, nc, "foo.bar.baz QG.22", 11) + + lnc.SubscribeSync("foo.bar.3") + lnc.QueueSubscribeSync("foo.bar.baz", "QG.22") + flushLeaf() + + // For now we do not see all the details behind a leafnode if the leafnode is not enabled. + checkDbgNumSubs(t, nc, "foo.bar.3", 2) + + checkDbgNumSubs(t, nc, "foo.bar.baz QG.22", 12) +} + +func runSolicitLeafServerWithSystemToURL(surl string) (*server.Server, *server.Options) { + o := DefaultTestOptions + o.Port = -1 + fooAcc := server.NewAccount("FOO") + o.Accounts = []*server.Account{server.NewAccount("$SYS"), fooAcc} + o.SystemAccount = "$SYS" + o.Users = []*server.User{ + {Username: "foo", Password: "pass", Permissions: nil, Account: fooAcc}, + } + rurl, _ := url.Parse(surl) + sysUrl, _ := url.Parse(strings.Replace(surl, rurl.User.Username(), "sys", -1)) + o.LeafNode.Remotes = []*server.RemoteLeafOpts{ + { + URLs: []*url.URL{rurl}, + LocalAccount: "FOO", + }, + { + URLs: []*url.URL{sysUrl}, + LocalAccount: "$SYS", + }, + } + o.LeafNode.ReconnectInterval = 100 * time.Millisecond + return RunServer(&o), &o +} + +func TestSystemServiceSubscribersLeafNodesWithSystem(t *testing.T) { + numServers, numClusters := 3, 3 + sc := createSuperCluster(t, numServers, numClusters) + defer sc.shutdown() + + sc.setupSystemServicesImports(t, "FOO") + + ci := rand.Int32N(int32(len(sc.clusters))) + si := rand.Int32N(int32(len(sc.clusters[ci].servers))) + s, opts := sc.clusters[ci].servers[si], sc.clusters[ci].opts[si] + url := fmt.Sprintf("nats://%s:pass@%s:%d", "foo", opts.Host, opts.LeafNode.Port) + ls, lopts := runSolicitLeafServerWithSystemToURL(url) + defer ls.Shutdown() + + checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { + if nln := s.NumLeafNodes(); nln != 2 { + return fmt.Errorf("Expected a connected leafnode for server %q, got none", s.ID()) + } + return nil + }) + + // This is so we can test when the subs on a leafnode are flushed to the connected supercluster. + fsubj := "__leaf.flush__" + fc := clientConnect(t, opts, "foo") + defer fc.Close() + fc.Subscribe(fsubj, func(m *nats.Msg) { + m.Respond(nil) + }) + + lnc := clientConnect(t, lopts, "foo") + defer lnc.Close() + + flushLeaf := func() { + if _, err := lnc.Request(fsubj, nil, time.Second); err != nil { + t.Fatalf("Did not flush through to the supercluster: %v", err) + } + } + + for i := 0; i < 10; i++ { + nc := clientConnect(t, sc.selectRandomServer(), "foo") + defer nc.Close() + nc.SubscribeSync(fmt.Sprintf("foo.bar.%d", i+1)) + nc.QueueSubscribeSync("foo.bar.baz", "QG.22") + nc.Flush() + } + + nc := clientConnect(t, sc.clusters[0].opts[0], "foo") + defer nc.Close() + + checkDbgNumSubs(t, nc, "foo.bar.3", 1) + + checkDbgNumSubs(t, nc, "foo.bar.*", 20) + + lnc.SubscribeSync("foo.bar.3") + lnc.QueueSubscribeSync("foo.bar.baz", "QG.22") + flushLeaf() + + // Since we are doing real tracking now on the other side, this will be off by 1 since we are counting + // the leaf and the real sub. + checkDbgNumSubs(t, nc, "foo.bar.3", 3) + + checkDbgNumSubs(t, nc, "foo.bar.baz QG.22", 12) +} diff --git a/test/test.go b/test/test.go new file mode 100644 index 0000000..ebc464d --- /dev/null +++ b/test/test.go @@ -0,0 +1,662 @@ +// Copyright 2012-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net" + "os" + "regexp" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + + srvlog "github.com/nats-io/nats-server/v2/logger" +) + +// So we can pass tests and benchmarks.. +type tLogger interface { + Fatalf(format string, args ...any) + Errorf(format string, args ...any) +} + +// DefaultTestOptions are default options for the unit tests. +var DefaultTestOptions = server.Options{ + Host: "127.0.0.1", + Port: 4222, + NoLog: true, + NoSigs: true, + MaxControlLine: 4096, + DisableShortFirstPing: true, +} + +// RunDefaultServer starts a new Go routine based server using the default options +func RunDefaultServer() *server.Server { + dto := DefaultTestOptions + return RunServer(&dto) +} + +func RunRandClientPortServer() *server.Server { + opts := DefaultTestOptions + opts.Port = -1 + return RunServer(&opts) +} + +// To turn on server tracing and debugging and logging which are +// normally suppressed. +var ( + doLog = false + doTrace = false + doDebug = false +) + +// RunServer starts a new Go routine based server +func RunServer(opts *server.Options) *server.Server { + return RunServerCallback(opts, nil) +} + +func RunServerCallback(opts *server.Options, callback func(*server.Server)) *server.Server { + if opts == nil { + dto := DefaultTestOptions + opts = &dto + } + // Optionally override for individual debugging of tests + opts.NoLog = !doLog + opts.Trace = doTrace + opts.Debug = doDebug + // For all tests in the "test" package, we will disable route pooling. + opts.Cluster.PoolSize = -1 + // Also disable compression for "test" package. + opts.Cluster.Compression.Mode = server.CompressionOff + opts.LeafNode.Compression.Mode = server.CompressionOff + + s, err := server.NewServer(opts) + if err != nil || s == nil { + panic(fmt.Sprintf("No NATS Server object returned: %v", err)) + } + + if doLog { + s.ConfigureLogger() + } + + if ll := os.Getenv("NATS_LOGGING"); ll != "" { + log := srvlog.NewTestLogger(fmt.Sprintf("[%s] | ", s), true) + debug := ll == "debug" || ll == "trace" + trace := ll == "trace" + s.SetLoggerV2(log, debug, trace, false) + } + + if callback != nil { + callback(s) + } + + // Run server in Go routine. + go s.Start() + + // Wait for accept loop(s) to be started + if !s.ReadyForConnections(10 * time.Second) { + panic("Unable to start NATS Server in Go Routine") + } + return s +} + +// LoadConfig loads a configuration from a filename +func LoadConfig(configFile string) *server.Options { + opts, err := server.ProcessConfigFile(configFile) + if err != nil { + panic(fmt.Sprintf("Error processing configuration file: %v", err)) + } + return opts +} + +// RunServerWithConfig starts a new Go routine based server with a configuration file. +func RunServerWithConfig(configFile string) (srv *server.Server, opts *server.Options) { + opts = LoadConfig(configFile) + srv = RunServer(opts) + return +} + +// RunServerWithConfigOverrides starts a new Go routine based server with a configuration file, +// providing a callback to update the options configured. +func RunServerWithConfigOverrides(configFile string, optsCallback func(*server.Options), svrCallback func(*server.Server)) (srv *server.Server, opts *server.Options) { + opts = LoadConfig(configFile) + if optsCallback != nil { + optsCallback(opts) + } + srv = RunServerCallback(opts, svrCallback) + return +} + +func stackFatalf(t tLogger, f string, args ...any) { + lines := make([]string, 0, 32) + msg := fmt.Sprintf(f, args...) + lines = append(lines, msg) + + // Ignore ourselves + _, testFile, _, _ := runtime.Caller(0) + + // Generate the Stack of callers: + for i := 0; true; i++ { + _, file, line, ok := runtime.Caller(i) + if !ok { + break + } + if file == testFile { + continue + } + msg := fmt.Sprintf("%d - %s:%d", i, file, line) + lines = append(lines, msg) + } + + t.Fatalf("%s", strings.Join(lines, "\n")) +} + +func acceptRouteConn(t tLogger, host string, timeout time.Duration) net.Conn { + l, e := net.Listen("tcp", host) + if e != nil { + stackFatalf(t, "Error listening for route connection on %v: %v", host, e) + } + defer l.Close() + + tl := l.(*net.TCPListener) + tl.SetDeadline(time.Now().Add(timeout)) + conn, err := l.Accept() + tl.SetDeadline(time.Time{}) + + if err != nil { + stackFatalf(t, "Did not receive a route connection request: %v", err) + } + return conn +} + +func createRouteConn(t tLogger, host string, port int) net.Conn { + return createClientConn(t, host, port) +} + +func createClientConn(t tLogger, host string, port int) net.Conn { + addr := net.JoinHostPort(host, fmt.Sprintf("%d", port)) + c, err := net.DialTimeout("tcp", addr, 3*time.Second) + if err != nil { + stackFatalf(t, "Could not connect to server: %v\n", err) + } + return c +} + +func checkSocket(t tLogger, addr string, wait time.Duration) { + end := time.Now().Add(wait) + for time.Now().Before(end) { + conn, err := net.Dial("tcp", addr) + if err != nil { + // Retry after 50ms + time.Sleep(50 * time.Millisecond) + continue + } + conn.Close() + // Wait a bit to give a chance to the server to remove this + // "client" from its state, which may otherwise interfere with + // some tests. + time.Sleep(25 * time.Millisecond) + return + } + // We have failed to bind the socket in the time allowed. + t.Fatalf("Failed to connect to the socket: %q", addr) +} + +func checkInfoMsg(t tLogger, c net.Conn) server.Info { + buf := expectResult(t, c, infoRe) + js := infoRe.FindAllSubmatch(buf, 1)[0][1] + var sinfo server.Info + err := json.Unmarshal(js, &sinfo) + if err != nil { + stackFatalf(t, "Could not unmarshal INFO json: %v\n", err) + } + return sinfo +} + +func doHeadersConnect(t tLogger, c net.Conn, verbose, pedantic, ssl, headers bool) { + checkInfoMsg(t, c) + cs := fmt.Sprintf("CONNECT {\"verbose\":%v,\"pedantic\":%v,\"tls_required\":%v,\"headers\":%v}\r\n", + verbose, pedantic, ssl, headers) + sendProto(t, c, cs) +} + +func doConnect(t tLogger, c net.Conn, verbose, pedantic, ssl bool) { + doHeadersConnect(t, c, verbose, pedantic, ssl, false) +} + +func doDefaultHeadersConnect(t tLogger, c net.Conn) { + doHeadersConnect(t, c, false, false, false, true) +} + +func doDefaultConnect(t tLogger, c net.Conn) { + // Basic Connect + doConnect(t, c, false, false, false) +} + +const routeConnectProto = "CONNECT {\"verbose\":false,\"user\":\"%s\",\"pass\":\"%s\",\"name\":\"%s\",\"cluster\":\"xyz\"}\r\n" + +func doRouteAuthConnect(t tLogger, c net.Conn, user, pass, id string) { + cs := fmt.Sprintf(routeConnectProto, user, pass, id) + sendProto(t, c, cs) +} + +func setupRouteEx(t tLogger, c net.Conn, opts *server.Options, id string) (sendFun, expectFun) { + user := opts.Cluster.Username + pass := opts.Cluster.Password + doRouteAuthConnect(t, c, user, pass, id) + return sendCommand(t, c), expectCommand(t, c) +} + +func setupRoute(t tLogger, c net.Conn, opts *server.Options) (sendFun, expectFun) { + u := make([]byte, 16) + io.ReadFull(rand.Reader, u) + id := fmt.Sprintf("ROUTER:%s", hex.EncodeToString(u)) + return setupRouteEx(t, c, opts, id) +} + +func setupHeaderConn(t tLogger, c net.Conn) (sendFun, expectFun) { + doDefaultHeadersConnect(t, c) + return sendCommand(t, c), expectCommand(t, c) +} + +func setupConn(t tLogger, c net.Conn) (sendFun, expectFun) { + doDefaultConnect(t, c) + return sendCommand(t, c), expectCommand(t, c) +} + +func setupConnWithProto(t tLogger, c net.Conn, proto int) (sendFun, expectFun) { + checkInfoMsg(t, c) + cs := fmt.Sprintf("CONNECT {\"verbose\":%v,\"pedantic\":%v,\"tls_required\":%v,\"protocol\":%d}\r\n", false, false, false, proto) + sendProto(t, c, cs) + return sendCommand(t, c), expectCommand(t, c) +} + +func setupConnWithAccount(t tLogger, s *server.Server, c net.Conn, account string) (sendFun, expectFun) { + info := checkInfoMsg(t, c) + s.RegisterAccount(account) + acc, err := s.LookupAccount(account) + if err != nil { + t.Fatalf("Unexpected Error: %v", err) + } + cs := fmt.Sprintf("CONNECT {\"verbose\":%v,\"pedantic\":%v,\"tls_required\":%v}\r\n", false, false, false) + sendProto(t, c, cs) + + send, expect := sendCommand(t, c), expectCommand(t, c) + send("PING\r\n") + expect(pongRe) + + nc := s.GetClient(info.CID) + if nc == nil { + t.Fatalf("Could not get client for CID:%d", info.CID) + } + nc.RegisterUser(&server.User{Account: acc}) + + return send, expect +} + +func setupConnWithUserPass(t tLogger, c net.Conn, username, password string) (sendFun, expectFun) { + checkInfoMsg(t, c) + cs := fmt.Sprintf("CONNECT {\"verbose\":%v,\"pedantic\":%v,\"tls_required\":%v,\"user\":%q,\"pass\":%q}\r\n", + false, false, false, username, password) + sendProto(t, c, cs) + return sendCommand(t, c), expectLefMostCommand(t, c) +} + +type sendFun func(string) +type expectFun func(*regexp.Regexp) []byte + +// Closure version for easier reading +func sendCommand(t tLogger, c net.Conn) sendFun { + return func(op string) { + sendProto(t, c, op) + } +} + +// Closure version for easier reading +func expectCommand(t tLogger, c net.Conn) expectFun { + return func(re *regexp.Regexp) []byte { + return expectResult(t, c, re) + } +} + +// Closure version for easier reading +func expectLefMostCommand(t tLogger, c net.Conn) expectFun { + var buf []byte + return func(re *regexp.Regexp) []byte { + return expectLeftMostResult(t, c, re, &buf) + } +} + +// Send the protocol command to the server. +func sendProto(t tLogger, c net.Conn, op string) { + n, err := c.Write([]byte(op)) + if err != nil { + stackFatalf(t, "Error writing command to conn: %v\n", err) + } + if n != len(op) { + stackFatalf(t, "Partial write: %d vs %d\n", n, len(op)) + } +} + +var ( + anyRe = regexp.MustCompile(`.*`) + infoRe = regexp.MustCompile(`INFO\s+([^\r\n]+)\r\n`) + infoStartRe = regexp.MustCompile(`^INFO\s+([^\r\n]+)\r\n`) + pingRe = regexp.MustCompile(`^PING\r\n`) + pongRe = regexp.MustCompile(`^PONG\r\n`) + hmsgRe = regexp.MustCompile(`(?:(?:HMSG\s+([^\s]+)\s+([^\s]+)\s+(([^\s]+)[^\S\r\n]+)?(\d+)\s+(\d+)\s*\r\n([^\\r\\n]*?)\r\n)+?)`) + msgRe = regexp.MustCompile(`(?:(?:MSG\s+([^\s]+)\s+([^\s]+)\s+(([^\s]+)[^\S\r\n]+)?(\d+)\s*\r\n([^\\r\\n]*?)\r\n)+?)`) + rawMsgRe = regexp.MustCompile(`(?:(?:MSG\s+([^\s]+)\s+([^\s]+)\s+(([^\s]+)[^\S\r\n]+)?(\d+)\s*\r\n(.*?)))`) + okRe = regexp.MustCompile(`\A\+OK\r\n`) + errRe = regexp.MustCompile(`\A\-ERR\s+([^\r\n]+)\r\n`) + connectRe = regexp.MustCompile(`CONNECT\s+([^\r\n]+)\r\n`) + rsubRe = regexp.MustCompile(`RS\+\s+([^\s]+)\s+([^\s]+)\s*([^\s]+)?\s*(\d+)?\r\n`) + runsubRe = regexp.MustCompile(`RS\-\s+([^\s]+)\s+([^\s]+)\s*([^\s]+)?\r\n`) + rmsgRe = regexp.MustCompile(`(?:(?:RMSG\s+([^\s]+)\s+([^\s]+)\s+(?:([|+]\s+([\w\s]+)|[^\s]+)[^\S\r\n]+)?(\d+)\s*\r\n([^\\r\\n]*?)\r\n)+?)`) + asubRe = regexp.MustCompile(`A\+\s+([^\r\n]+)\r\n`) + aunsubRe = regexp.MustCompile(`A\-\s+([^\r\n]+)\r\n`) + lsubRe = regexp.MustCompile(`LS\+\s+([^\s]+)\s*([^\s]+)?\s*(\d+)?\r\n`) + lunsubRe = regexp.MustCompile(`LS\-\s+([^\s]+)\s*([^\s]+)\s*([^\s]+)?\r\n`) + lmsgRe = regexp.MustCompile(`(?:(?:LMSG\s+([^\s]+)\s+(?:([|+]\s+([\w\s]+)|[^\s]+)[^\S\r\n]+)?(\d+)\s*\r\n([^\\r\\n]*?)\r\n)+?)`) + rlsubRe = regexp.MustCompile(`LS\+\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s*([^\s]+)?\s*(\d+)?\r\n`) + rlunsubRe = regexp.MustCompile(`LS\-\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s*([^\s]+)?\r\n`) +) + +const ( + // Regular Messages + subIndex = 1 + sidIndex = 2 + replyIndex = 4 + lenIndex = 5 + msgIndex = 6 + // Headers + hlenIndex = 5 + tlenIndex = 6 + hmsgIndex = 7 + + // Routed Messages + accIndex = 1 + rsubIndex = 2 + replyAndQueueIndex = 3 +) + +// Test result from server against regexp and return left most match +func expectLeftMostResult(t tLogger, c net.Conn, re *regexp.Regexp, buf *[]byte) []byte { + recv := func() []byte { + expBuf := make([]byte, 32768) + // Wait for commands to be processed and results queued for read + c.SetReadDeadline(time.Now().Add(2 * time.Second)) + n, err := c.Read(expBuf) + c.SetReadDeadline(time.Time{}) + + if n <= 0 && err != nil { + stackFatalf(t, "Error reading from conn: %v\n", err) + } + return expBuf[:n] + } + if len(*buf) == 0 { + *buf = recv() + } + emptyCnt := 0 + for { + result := re.Find(*buf) + if result == nil { + emptyCnt++ + if emptyCnt > 5 { + stackFatalf(t, "Reading empty data too often\n") + } + *buf = append(*buf, recv()...) + } else { + cutIdx := strings.Index(string(*buf), string(result)) + len(result) + *buf = (*buf)[cutIdx:] + return result + } + } +} + +// Test result from server against regexp +func expectResult(t tLogger, c net.Conn, re *regexp.Regexp) []byte { + expBuf := make([]byte, 32768) + // Wait for commands to be processed and results queued for read + c.SetReadDeadline(time.Now().Add(2 * time.Second)) + n, err := c.Read(expBuf) + c.SetReadDeadline(time.Time{}) + + if n <= 0 && err != nil { + stackFatalf(t, "Error reading from conn: %v\n", err) + } + buf := expBuf[:n] + if !re.Match(buf) { + stackFatalf(t, "Response did not match expected: \n\tReceived:'%q'\n\tExpected:'%s'", buf, re) + } + return buf +} + +func peek(c net.Conn) []byte { + expBuf := make([]byte, 32768) + c.SetReadDeadline(time.Now().Add(50 * time.Millisecond)) + n, err := c.Read(expBuf) + c.SetReadDeadline(time.Time{}) + if err != nil || n <= 0 { + return nil + } + return expBuf +} + +func expectDisconnect(t *testing.T, c net.Conn) { + t.Helper() + var b [8]byte + c.SetReadDeadline(time.Now().Add(200 * time.Millisecond)) + _, err := c.Read(b[:]) + c.SetReadDeadline(time.Time{}) + if err != io.EOF { + t.Fatalf("Expected a disconnect") + } +} + +func expectNothing(t tLogger, c net.Conn) { + expectNothingTimeout(t, c, time.Now().Add(100*time.Millisecond)) +} + +func expectNothingTimeout(t tLogger, c net.Conn, dl time.Time) { + expBuf := make([]byte, 32) + c.SetReadDeadline(dl) + n, err := c.Read(expBuf) + c.SetReadDeadline(time.Time{}) + if err == nil && n > 0 { + stackFatalf(t, "Expected nothing, received: '%q'\n", expBuf[:n]) + } +} + +// This will check that we got what we expected from a normal message. +func checkMsg(t tLogger, m [][]byte, subject, sid, reply, len, msg string) { + if string(m[subIndex]) != subject { + stackFatalf(t, "Did not get correct subject: expected '%s' got '%s'\n", subject, m[subIndex]) + } + if sid != "" && string(m[sidIndex]) != sid { + stackFatalf(t, "Did not get correct sid: expected '%s' got '%s'\n", sid, m[sidIndex]) + } + if string(m[replyIndex]) != reply { + stackFatalf(t, "Did not get correct reply: expected '%s' got '%s'\n", reply, m[replyIndex]) + } + if string(m[lenIndex]) != len { + stackFatalf(t, "Did not get correct msg length: expected '%s' got '%s'\n", len, m[lenIndex]) + } + if string(m[msgIndex]) != msg { + stackFatalf(t, "Did not get correct msg: expected '%s' got '%s'\n", msg, m[msgIndex]) + } +} + +func checkRmsg(t tLogger, m [][]byte, account, subject, replyAndQueues, len, msg string) { + if string(m[accIndex]) != account { + stackFatalf(t, "Did not get correct account: expected '%s' got '%s'\n", account, m[accIndex]) + } + if string(m[rsubIndex]) != subject { + stackFatalf(t, "Did not get correct subject: expected '%s' got '%s'\n", subject, m[rsubIndex]) + } + if string(m[lenIndex]) != len { + stackFatalf(t, "Did not get correct msg length: expected '%s' got '%s'\n", len, m[lenIndex]) + } + if string(m[replyAndQueueIndex]) != replyAndQueues { + stackFatalf(t, "Did not get correct reply/queues: expected '%s' got '%s'\n", replyAndQueues, m[replyAndQueueIndex]) + } +} + +func checkLmsg(t tLogger, m [][]byte, subject, replyAndQueues, len, msg string) { + if string(m[rsubIndex-1]) != subject { + stackFatalf(t, "Did not get correct subject: expected '%s' got '%s'\n", subject, m[rsubIndex-1]) + } + if string(m[lenIndex-1]) != len { + stackFatalf(t, "Did not get correct msg length: expected '%s' got '%s'\n", len, m[lenIndex-1]) + } + if string(m[replyAndQueueIndex-1]) != replyAndQueues { + stackFatalf(t, "Did not get correct reply/queues: expected '%s' got '%s'\n", replyAndQueues, m[replyAndQueueIndex-1]) + } +} + +// This will check that we got what we expected from a header message. +func checkHmsg(t tLogger, m [][]byte, subject, sid, reply, hlen, len, hdr, msg string) { + if string(m[subIndex]) != subject { + stackFatalf(t, "Did not get correct subject: expected '%s' got '%s'\n", subject, m[subIndex]) + } + if sid != "" && string(m[sidIndex]) != sid { + stackFatalf(t, "Did not get correct sid: expected '%s' got '%s'\n", sid, m[sidIndex]) + } + if string(m[replyIndex]) != reply { + stackFatalf(t, "Did not get correct reply: expected '%s' got '%s'\n", reply, m[replyIndex]) + } + if string(m[hlenIndex]) != hlen { + stackFatalf(t, "Did not get correct header length: expected '%s' got '%s'\n", hlen, m[hlenIndex]) + } + if string(m[tlenIndex]) != len { + stackFatalf(t, "Did not get correct msg length: expected '%s' got '%s'\n", len, m[tlenIndex]) + } + // Extract the payload and break up the headers and msg. + payload := string(m[hmsgIndex]) + hi, _ := strconv.Atoi(hlen) + rhdr, rmsg := payload[:hi], payload[hi:] + if rhdr != hdr { + stackFatalf(t, "Did not get correct headers: expected '%s' got '%s'\n", hdr, rhdr) + } + if rmsg != msg { + stackFatalf(t, "Did not get correct msg: expected '%s' got '%s'\n", msg, rmsg) + } +} + +// Closure for expectMsgs +func expectRmsgsCommand(t tLogger, ef expectFun) func(int) [][][]byte { + return func(expected int) [][][]byte { + buf := ef(rmsgRe) + matches := rmsgRe.FindAllSubmatch(buf, -1) + if len(matches) != expected { + stackFatalf(t, "Did not get correct # routed msgs: %d vs %d\n", len(matches), expected) + } + return matches + } +} + +// Closure for expectHMsgs +func expectHeaderMsgsCommand(t tLogger, ef expectFun) func(int) [][][]byte { + return func(expected int) [][][]byte { + buf := ef(hmsgRe) + matches := hmsgRe.FindAllSubmatch(buf, -1) + if len(matches) != expected { + stackFatalf(t, "Did not get correct # msgs: %d vs %d\n", len(matches), expected) + } + return matches + } +} + +// Closure for expectMsgs +func expectMsgsCommand(t tLogger, ef expectFun) func(int) [][][]byte { + return func(expected int) [][][]byte { + buf := ef(msgRe) + matches := msgRe.FindAllSubmatch(buf, -1) + if len(matches) != expected { + stackFatalf(t, "Did not get correct # msgs: %d vs %d\n", len(matches), expected) + } + return matches + } +} + +// This will check that the matches include at least one of the sids. Useful for checking +// that we received messages on a certain queue group. +func checkForQueueSid(t tLogger, matches [][][]byte, sids []string) { + seen := make(map[string]int, len(sids)) + for _, sid := range sids { + seen[sid] = 0 + } + for _, m := range matches { + sid := string(m[sidIndex]) + if _, ok := seen[sid]; ok { + seen[sid]++ + } + } + // Make sure we only see one and exactly one. + total := 0 + for _, n := range seen { + total += n + } + if total != 1 { + stackFatalf(t, "Did not get a msg for queue sids group: expected 1 got %d\n", total) + } +} + +// This will check that the matches include all of the sids. Useful for checking +// that we received messages on all subscribers. +func checkForPubSids(t tLogger, matches [][][]byte, sids []string) { + seen := make(map[string]int, len(sids)) + for _, sid := range sids { + seen[sid] = 0 + } + for _, m := range matches { + sid := string(m[sidIndex]) + if _, ok := seen[sid]; ok { + seen[sid]++ + } + } + // Make sure we only see one and exactly one for each sid. + for sid, n := range seen { + if n != 1 { + stackFatalf(t, "Did not get a msg for sid[%s]: expected 1 got %d\n", sid, n) + + } + } +} + +// Helper function to generate next opts to make sure no port conflicts etc. +func nextServerOpts(opts *server.Options) *server.Options { + nopts := opts.Clone() + nopts.Port++ + nopts.Cluster.Port++ + nopts.HTTPPort++ + return nopts +} + +func createTempFile(t testing.TB, prefix string) *os.File { + t.Helper() + file, err := os.CreateTemp(t.TempDir(), prefix) + if err != nil { + t.Fatal(err) + } + return file +} diff --git a/test/test_test.go b/test/test_test.go new file mode 100644 index 0000000..4e72b42 --- /dev/null +++ b/test/test_test.go @@ -0,0 +1,164 @@ +// Copyright 2016-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "context" + "fmt" + "net" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "golang.org/x/time/rate" +) + +func checkFor(t testing.TB, totalWait, sleepDur time.Duration, f func() error) { + t.Helper() + timeout := time.Now().Add(totalWait) + var err error + for time.Now().Before(timeout) { + err = f() + if err == nil { + return + } + time.Sleep(sleepDur) + } + if err != nil { + t.Fatal(err.Error()) + } +} + +// Slow Proxy - For introducing RTT and BW constraints. +type slowProxy struct { + listener net.Listener + conns []net.Conn + o *server.Options + u string +} + +func newSlowProxy(rtt time.Duration, up, down int, opts *server.Options) *slowProxy { + saddr := net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port)) + hp := net.JoinHostPort("127.0.0.1", "0") + l, e := net.Listen("tcp", hp) + if e != nil { + panic(fmt.Sprintf("Error listening on port: %s, %q", hp, e)) + } + port := l.Addr().(*net.TCPAddr).Port + sp := &slowProxy{listener: l} + go func() { + client, err := l.Accept() + if err != nil { + return + } + server, err := net.DialTimeout("tcp", saddr, time.Second) + if err != nil { + panic("Can't connect to server") + } + sp.conns = append(sp.conns, client, server) + go sp.loop(rtt, up, client, server) + go sp.loop(rtt, down, server, client) + }() + sp.o = &server.Options{Host: "127.0.0.1", Port: port} + sp.u = fmt.Sprintf("nats://%s:%d", sp.o.Host, sp.o.Port) + return sp +} + +func (sp *slowProxy) opts() *server.Options { + return sp.o +} + +//lint:ignore U1000 Referenced in norace_test.go +func (sp *slowProxy) clientURL() string { + return sp.u +} + +func (sp *slowProxy) loop(rtt time.Duration, tbw int, r, w net.Conn) { + delay := rtt / 2 + const rbl = 1024 + var buf [rbl]byte + ctx := context.Background() + + rl := rate.NewLimiter(rate.Limit(tbw), rbl) + + for fr := true; ; { + sr := time.Now() + n, err := r.Read(buf[:]) + if err != nil { + return + } + // RTT delays + if fr || time.Since(sr) > 2*time.Millisecond { + fr = false + time.Sleep(delay) + } + if err := rl.WaitN(ctx, n); err != nil { + return + } + if _, err = w.Write(buf[:n]); err != nil { + return + } + } +} + +func (sp *slowProxy) stop() { + if sp.listener != nil { + sp.listener.Close() + sp.listener = nil + for _, c := range sp.conns { + c.Close() + } + } +} + +// Dummy Logger +type dummyLogger struct { + sync.Mutex + msg string +} + +func (d *dummyLogger) Fatalf(format string, args ...any) { + d.Lock() + d.msg = fmt.Sprintf(format, args...) + d.Unlock() +} + +func (d *dummyLogger) Errorf(format string, args ...any) { +} + +func (d *dummyLogger) Debugf(format string, args ...any) { +} + +func (d *dummyLogger) Tracef(format string, args ...any) { +} + +func (d *dummyLogger) Noticef(format string, args ...any) { +} + +func (d *dummyLogger) Warnf(format string, args ...any) { +} + +func TestStackFatal(t *testing.T) { + d := &dummyLogger{} + stackFatalf(d, "test stack %d", 1) + if !strings.HasPrefix(d.msg, "test stack 1") { + t.Fatalf("Unexpected start of stack: %v", d.msg) + } + if !strings.Contains(d.msg, "test_test.go") { + t.Fatalf("Unexpected stack: %v", d.msg) + } +} diff --git a/test/tls_test.go b/test/tls_test.go new file mode 100644 index 0000000..0952ec3 --- /dev/null +++ b/test/tls_test.go @@ -0,0 +1,2138 @@ +// Copyright 2015-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "bufio" + "crypto/ed25519" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + + "github.com/nats-io/nats-server/v2/server" +) + +var noOpErrHandler = func(_ *nats.Conn, _ *nats.Subscription, _ error) {} + +func TestTLSConnection(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/tls.conf") + defer srv.Shutdown() + + endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) + nurl := fmt.Sprintf("tls://%s:%s@%s/", opts.Username, opts.Password, endpoint) + nc, err := nats.Connect(nurl) + if err == nil { + nc.Close() + t.Fatalf("Expected error trying to connect to secure server") + } + + // Do simple SecureConnect + nc, err = nats.Connect(fmt.Sprintf("tls://%s/", endpoint)) + if err == nil { + nc.Close() + t.Fatalf("Expected error trying to connect to secure server with no auth") + } + + // Now do more advanced checking, verifying servername and using rootCA. + + nc, err = nats.Connect(nurl, nats.RootCAs("./configs/certs/ca.pem")) + if err != nil { + t.Fatalf("Got an error on Connect with Secure Options: %+v\n", err) + } + defer nc.Close() + + subj := "foo-tls" + sub, _ := nc.SubscribeSync(subj) + + nc.Publish(subj, []byte("We are Secure!")) + nc.Flush() + nmsgs, _, _ := sub.Pending() + if nmsgs != 1 { + t.Fatalf("Expected to receive a message over the TLS connection") + } +} + +// TestTLSInProcessConnection checks that even if TLS is enabled on the server, +// that an in-process connection that does *not* use TLS still connects successfully. +func TestTLSInProcessConnection(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/tls.conf") + defer srv.Shutdown() + + nc, err := nats.Connect("", nats.InProcessServer(srv), nats.UserInfo(opts.Username, opts.Password)) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + + if nc.TLSRequired() { + t.Fatalf("Shouldn't have required TLS for in-process connection") + } + + if _, err = nc.TLSConnectionState(); err == nil { + t.Fatal("Should have got an error retrieving TLS connection state") + } +} + +func TestTLSClientCertificate(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/tlsverify.conf") + defer srv.Shutdown() + + nurl := fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port) + + _, err := nats.Connect(nurl) + if err == nil { + t.Fatalf("Expected error trying to connect to secure server without a certificate") + } + + // Load client certificate to successfully connect. + certFile := "./configs/certs/client-cert.pem" + keyFile := "./configs/certs/client-key.pem" + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + t.Fatalf("error parsing X509 certificate/key pair: %v", err) + } + + // Load in root CA for server verification + rootPEM, err := os.ReadFile("./configs/certs/ca.pem") + if err != nil || rootPEM == nil { + t.Fatalf("failed to read root certificate") + } + pool := x509.NewCertPool() + ok := pool.AppendCertsFromPEM([]byte(rootPEM)) + if !ok { + t.Fatalf("failed to parse root certificate") + } + + config := &tls.Config{ + Certificates: []tls.Certificate{cert}, + ServerName: opts.Host, + RootCAs: pool, + MinVersion: tls.VersionTLS12, + } + + copts := nats.GetDefaultOptions() + copts.Url = nurl + copts.Secure = true + copts.TLSConfig = config + + nc, err := copts.Connect() + if err != nil { + t.Fatalf("Got an error on Connect with Secure Options: %+v\n", err) + } + nc.Flush() + defer nc.Close() +} + +func TestTLSClientCertificateHasUserID(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/tls_cert_id.conf") + defer srv.Shutdown() + nurl := fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(nurl, + nats.ClientCert("./configs/certs/client-id-auth-cert.pem", "./configs/certs/client-id-auth-key.pem"), + nats.RootCAs("./configs/certs/ca.pem")) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer nc.Close() +} + +func TestTLSClientCertificateCheckWithAllowedConnectionTypes(t *testing.T) { + conf := createConfFile(t, []byte( + ` + listen: "127.0.0.1:-1" + tls { + cert_file: "./configs/certs/server-cert.pem" + key_file: "./configs/certs/server-key.pem" + timeout: 2 + ca_file: "./configs/certs/ca.pem" + verify_and_map: true + } + authorization { + users = [ + {user: derek@nats.io, permissions: { publish:"foo" }, allowed_connection_types: ["WEBSOCKET"]} + ] + } + `)) + s, o := RunServerWithConfig(conf) + defer s.Shutdown() + + nurl := fmt.Sprintf("tls://%s:%d", o.Host, o.Port) + nc, err := nats.Connect(nurl, + nats.ClientCert("./configs/certs/client-id-auth-cert.pem", "./configs/certs/client-id-auth-key.pem"), + nats.RootCAs("./configs/certs/ca.pem")) + if err == nil { + if nc != nil { + nc.Close() + } + t.Fatal("Expected connection to fail, it did not") + } +} + +func TestTLSClientCertificateCNBasedAuth(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/tls_cert_cn.conf") + defer srv.Shutdown() + nurl := fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port) + errCh1 := make(chan error) + errCh2 := make(chan error) + + // Using the default permissions + nc1, err := nats.Connect(nurl, + nats.ClientCert("./configs/certs/tlsauth/client.pem", "./configs/certs/tlsauth/client-key.pem"), + nats.RootCAs("./configs/certs/tlsauth/ca.pem"), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + errCh1 <- err + }), + ) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer nc1.Close() + + // Admin permissions can publish to '>' + nc2, err := nats.Connect(nurl, + nats.ClientCert("./configs/certs/tlsauth/client2.pem", "./configs/certs/tlsauth/client2-key.pem"), + nats.RootCAs("./configs/certs/tlsauth/ca.pem"), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + errCh2 <- err + }), + ) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer nc2.Close() + + err = nc1.Publish("foo.bar", []byte("hi")) + if err != nil { + t.Fatal(err) + } + _, err = nc1.SubscribeSync("foo.>") + if err != nil { + t.Fatal(err) + } + nc1.Flush() + + sub, err := nc2.SubscribeSync(">") + if err != nil { + t.Fatal(err) + } + nc2.Flush() + err = nc2.Publish("hello", []byte("hi")) + if err != nil { + t.Fatal(err) + } + nc2.Flush() + + _, err = sub.NextMsg(1 * time.Second) + if err != nil { + t.Fatalf("Error during wait for next message: %s", err) + } + + // Wait for a couple of errors + var count int +Loop: + for { + select { + case err := <-errCh1: + if err != nil { + count++ + } + if count == 2 { + break Loop + } + case err := <-errCh2: + if err != nil { + t.Fatalf("Received unexpected auth error from client: %s", err) + } + case <-time.After(2 * time.Second): + t.Fatalf("Timed out expecting auth errors") + } + } +} + +func TestTLSClientCertificateSANsBasedAuth(t *testing.T) { + // In this test we have 3 clients, one with permissions defined + // for SAN 'app.nats.dev', other for SAN 'app.nats.prod' and another + // one without the default permissions for the CN. + srv, opts := RunServerWithConfig("./configs/tls_cert_san_auth.conf") + defer srv.Shutdown() + nurl := fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port) + defaultErrCh := make(chan error) + devErrCh := make(chan error) + prodErrCh := make(chan error) + + // default: Using the default permissions (no SANs) + // + // Subject: CN = www.nats.io + // + defaultc, err := nats.Connect(nurl, + nats.ClientCert("./configs/certs/sans/client.pem", "./configs/certs/sans/client-key.pem"), + nats.RootCAs("./configs/certs/sans/ca.pem"), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + defaultErrCh <- err + }), + ) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer defaultc.Close() + + // dev: Using SAN 'app.nats.dev' with permissions to its own sandbox. + // + // Subject: CN = www.nats.io + // + // X509v3 Subject Alternative Name: + // DNS:app.nats.dev, DNS:*.app.nats.dev + // + devc, err := nats.Connect(nurl, + nats.ClientCert("./configs/certs/sans/dev.pem", "./configs/certs/sans/dev-key.pem"), + nats.RootCAs("./configs/certs/sans/ca.pem"), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + devErrCh <- err + }), + ) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer devc.Close() + + // prod: Using SAN 'app.nats.prod' with all permissions. + // + // Subject: CN = www.nats.io + // + // X509v3 Subject Alternative Name: + // DNS:app.nats.prod, DNS:*.app.nats.prod + // + prodc, err := nats.Connect(nurl, + nats.ClientCert("./configs/certs/sans/prod.pem", "./configs/certs/sans/prod-key.pem"), + nats.RootCAs("./configs/certs/sans/ca.pem"), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + prodErrCh <- err + }), + ) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer prodc.Close() + + // No permissions to publish or subscribe on foo.> + err = devc.Publish("foo.bar", []byte("hi")) + if err != nil { + t.Fatal(err) + } + _, err = devc.SubscribeSync("foo.>") + if err != nil { + t.Fatal(err) + } + devc.Flush() + + prodSub, err := prodc.SubscribeSync(">") + if err != nil { + t.Fatal(err) + } + prodc.Flush() + err = prodc.Publish("hello", []byte("hi")) + if err != nil { + t.Fatal(err) + } + prodc.Flush() + + // prod: can receive message on wildcard subscription. + _, err = prodSub.NextMsg(1 * time.Second) + if err != nil { + t.Fatalf("Error during wait for next message: %s", err) + } + + // dev: enough permissions to publish to sandbox subject. + devSub, err := devc.SubscribeSync("sandbox.>") + if err != nil { + t.Fatal(err) + } + devc.Flush() + err = devc.Publish("sandbox.foo.bar", []byte("hi!")) + if err != nil { + t.Fatal(err) + } + // both dev and prod clients can receive message + _, err = devSub.NextMsg(1 * time.Second) + if err != nil { + t.Fatalf("Error during wait for next message: %s", err) + } + _, err = prodSub.NextMsg(1 * time.Second) + if err != nil { + t.Fatalf("Error during wait for next message: %s", err) + } + + // default: no enough permissions + _, err = defaultc.SubscribeSync("sandbox.>") + if err != nil { + t.Fatal(err) + } + defaultSub, err := defaultc.SubscribeSync("public.>") + if err != nil { + t.Fatal(err) + } + defaultc.Flush() + err = devc.Publish("public.foo.bar", []byte("hi!")) + if err != nil { + t.Fatal(err) + } + _, err = defaultSub.NextMsg(1 * time.Second) + if err != nil { + t.Fatalf("Error during wait for next message: %s", err) + } + + // Wait for a couple of errors + var count int +Loop: + for { + select { + case err := <-defaultErrCh: + if err != nil { + count++ + } + if count == 3 { + break Loop + } + case err := <-devErrCh: + if err != nil { + count++ + } + if count == 3 { + break Loop + } + case err := <-prodErrCh: + if err != nil { + t.Fatalf("Received unexpected auth error from client: %s", err) + } + case <-time.After(2 * time.Second): + t.Fatalf("Timed out expecting auth errors") + } + } +} + +func TestTLSClientCertificateNoSubjectWithSANAuth(t *testing.T) { + // A certificate with an empty Subject DN but a DNS SAN should be accepted + // and mapped to the SAN value, exercising the hasSANs branch in + // checkClientTLSCertSubject. + dir := t.TempDir() + ca := issueCert(t, dir, "ca", nil, pkix.Name{CommonName: "test-ca"}, nil, nil) + server := issueCert(t, dir, "server", ca, pkix.Name{CommonName: "localhost"}, []string{"localhost"}, []net.IP{net.ParseIP("127.0.0.1")}) + client := issueCert(t, dir, "client-nosubj", ca, pkix.Name{}, []string{"localhost"}, nil) + + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: "127.0.0.1:-1" + tls { + cert_file: %q + key_file: %q + ca_file: %q + verify_and_map: true + } + authorization { + users = [ + { user: "localhost" } + ] + } + `, server.certPath, server.keyPath, ca.certPath))) + srv, opts := RunServerWithConfig(conf) + defer srv.Shutdown() + nurl := fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port) + + t.Run("no-subject cert with SAN connects", func(t *testing.T) { + nc, err := nats.Connect(nurl, + nats.ClientCert(client.certPath, client.keyPath), + nats.RootCAs(ca.certPath), + ) + if err != nil { + t.Fatalf("Expected to connect with no-subject cert, got %v", err) + } + defer nc.Close() + }) + + t.Run("no client cert is rejected", func(t *testing.T) { + nc, err := nats.Connect(nurl, nats.RootCAs(ca.certPath)) + if err == nil { + nc.Close() + t.Fatal("Expected connection without client cert to fail, but it succeeded") + } + }) +} + +type issuedCert struct { + cert *x509.Certificate + key ed25519.PrivateKey + certPath string + keyPath string +} + +// issueCert generates an Ed25519 keypair and an X.509 certificate written under +// dir as -cert.pem and -key.pem. parent==nil produces a self-signed +// CA; otherwise the cert is signed by parent and gets server+client EKU. +func issueCert(t *testing.T, dir, name string, parent *issuedCert, subject pkix.Name, dnsNames []string, ips []net.IP) *issuedCert { + t.Helper() + _, key, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), + Subject: subject, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + DNSNames: dnsNames, + IPAddresses: ips, + } + parentCert, parentKey := tmpl, key + if parent == nil { + tmpl.IsCA = true + tmpl.BasicConstraintsValid = true + tmpl.KeyUsage |= x509.KeyUsageCertSign + } else { + tmpl.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth} + parentCert, parentKey = parent.cert, parent.key + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, parentCert, key.Public(), parentKey) + if err != nil { + t.Fatal(err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal(err) + } + out := &issuedCert{ + cert: cert, + key: key, + certPath: filepath.Join(dir, name+"-cert.pem"), + keyPath: filepath.Join(dir, name+"-key.pem"), + } + if err := os.WriteFile(out.certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(out.keyPath, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0600); err != nil { + t.Fatal(err) + } + return out +} + +func TestTLSClientCertificateTLSAuthMultipleOptions(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/tls_cert_san_emails.conf") + defer srv.Shutdown() + nurl := fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port) + defaultErrCh := make(chan error) + devErrCh := make(chan error) + prodErrCh := make(chan error) + + // default: Using the default permissions, there are SANs + // present in the cert but they are not users in the NATS config + // so the subject is used instead. + // + // Subject: CN = www.nats.io + // + // X509v3 Subject Alternative Name: + // DNS:app.nats.dev, DNS:*.app.nats.dev + // + defaultc, err := nats.Connect(nurl, + nats.ClientCert("./configs/certs/sans/dev.pem", "./configs/certs/sans/dev-key.pem"), + nats.RootCAs("./configs/certs/sans/ca.pem"), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + defaultErrCh <- err + }), + ) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer defaultc.Close() + + // dev: Using SAN to validate user, even if emails are present in the config. + // + // Subject: CN = www.nats.io + // + // X509v3 Subject Alternative Name: + // DNS:app.nats.dev, email:admin@app.nats.dev, email:root@app.nats.dev + // + devc, err := nats.Connect(nurl, + nats.ClientCert("./configs/certs/sans/dev-email.pem", "./configs/certs/sans/dev-email-key.pem"), + nats.RootCAs("./configs/certs/sans/ca.pem"), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + devErrCh <- err + }), + ) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer devc.Close() + + // prod: Using SAN '*.app.nats.prod' with all permissions, which is not the first SAN. + // + // Subject: CN = www.nats.io + // + // X509v3 Subject Alternative Name: + // DNS:app.nats.prod, DNS:*.app.nats.prod + // + prodc, err := nats.Connect(nurl, + nats.ClientCert("./configs/certs/sans/prod.pem", "./configs/certs/sans/prod-key.pem"), + nats.RootCAs("./configs/certs/sans/ca.pem"), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + prodErrCh <- err + }), + ) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer prodc.Close() + + // No permissions to publish or subscribe on foo.> + err = devc.Publish("foo.bar", []byte("hi")) + if err != nil { + t.Fatal(err) + } + _, err = devc.SubscribeSync("foo.>") + if err != nil { + t.Fatal(err) + } + devc.Flush() + + prodSub, err := prodc.SubscribeSync(">") + if err != nil { + t.Fatal(err) + } + prodc.Flush() + err = prodc.Publish("hello", []byte("hi")) + if err != nil { + t.Fatal(err) + } + prodc.Flush() + + // prod: can receive message on wildcard subscription. + _, err = prodSub.NextMsg(1 * time.Second) + if err != nil { + t.Fatalf("Error during wait for next message: %s", err) + } + + // dev: enough permissions to publish to sandbox subject. + devSub, err := devc.SubscribeSync("sandbox.>") + if err != nil { + t.Fatal(err) + } + devc.Flush() + err = devc.Publish("sandbox.foo.bar", []byte("hi!")) + if err != nil { + t.Fatal(err) + } + // both dev and prod clients can receive message + _, err = devSub.NextMsg(1 * time.Second) + if err != nil { + t.Fatalf("Error during wait for next message: %s", err) + } + _, err = prodSub.NextMsg(1 * time.Second) + if err != nil { + t.Fatalf("Error during wait for next message: %s", err) + } + + // default: no enough permissions + _, err = defaultc.SubscribeSync("sandbox.>") + if err != nil { + t.Fatal(err) + } + defaultSub, err := defaultc.SubscribeSync("public.>") + if err != nil { + t.Fatal(err) + } + defaultc.Flush() + err = devc.Publish("public.foo.bar", []byte("hi!")) + if err != nil { + t.Fatal(err) + } + _, err = defaultSub.NextMsg(1 * time.Second) + if err != nil { + t.Fatalf("Error during wait for next message: %s", err) + } + + // Wait for a couple of errors + var count int +Loop: + for { + select { + case err := <-defaultErrCh: + if err != nil { + count++ + } + if count == 3 { + break Loop + } + case err := <-devErrCh: + if err != nil { + count++ + } + if count == 3 { + break Loop + } + case err := <-prodErrCh: + if err != nil { + t.Fatalf("Received unexpected auth error from client: %s", err) + } + case <-time.After(2 * time.Second): + t.Fatalf("Timed out expecting auth errors") + } + } +} + +func TestTLSRoutesCertificateCNBasedAuth(t *testing.T) { + // Base config for the servers + optsA := LoadConfig("./configs/tls_cert_cn_routes.conf") + optsB := LoadConfig("./configs/tls_cert_cn_routes.conf") + optsC := LoadConfig("./configs/tls_cert_cn_routes_invalid_auth.conf") + + // TLS map should have been enabled + if !optsA.Cluster.TLSMap || !optsB.Cluster.TLSMap || !optsC.Cluster.TLSMap { + t.Error("Expected Cluster TLS verify and map feature to be activated") + } + + routeURLs := "nats://localhost:9935, nats://localhost:9936, nats://localhost:9937" + + optsA.Host = "127.0.0.1" + optsA.Port = 9335 + optsA.Cluster.Name = "xyz" + optsA.Cluster.Host = optsA.Host + optsA.Cluster.Port = 9935 + optsA.Routes = server.RoutesFromStr(routeURLs) + optsA.NoSystemAccount = true + srvA := RunServer(optsA) + defer srvA.Shutdown() + + optsB.Host = "127.0.0.1" + optsB.Port = 9336 + optsB.Cluster.Name = "xyz" + optsB.Cluster.Host = optsB.Host + optsB.Cluster.Port = 9936 + optsB.Routes = server.RoutesFromStr(routeURLs) + optsB.NoSystemAccount = true + srvB := RunServer(optsB) + defer srvB.Shutdown() + + optsC.Host = "127.0.0.1" + optsC.Port = 9337 + optsC.Cluster.Name = "xyz" + optsC.Cluster.Host = optsC.Host + optsC.Cluster.Port = 9937 + optsC.Routes = server.RoutesFromStr(routeURLs) + optsC.NoSystemAccount = true + srvC := RunServer(optsC) + defer srvC.Shutdown() + + // srvC is not connected to srvA and srvB due to wrong cert + checkClusterFormed(t, srvA, srvB) + + nc1, err := nats.Connect(fmt.Sprintf("%s:%d", optsA.Host, optsA.Port), nats.Name("A")) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer nc1.Close() + + nc2, err := nats.Connect(fmt.Sprintf("%s:%d", optsB.Host, optsB.Port), nats.Name("B")) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer nc2.Close() + + // This client is partitioned from rest of cluster due to using wrong cert. + nc3, err := nats.Connect(fmt.Sprintf("%s:%d", optsC.Host, optsC.Port), nats.Name("C")) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer nc3.Close() + + // Allowed via permissions to broadcast to other members of cluster. + publicSub, err := nc1.SubscribeSync("public.>") + if err != nil { + t.Fatal(err) + } + + // Not part of cluster permissions so message is not forwarded. + privateSub, err := nc1.SubscribeSync("private.>") + if err != nil { + t.Fatal(err) + } + // This ensures that srvA has received the sub, not srvB + nc1.Flush() + + checkFor(t, time.Second, 15*time.Millisecond, func() error { + if n := srvB.NumSubscriptions(); n != 1 { + return fmt.Errorf("Expected 1 sub, got %v", n) + } + return nil + }) + + // Not forwarded by cluster so won't be received. + err = nc2.Publish("private.foo", []byte("private message on server B")) + if err != nil { + t.Fatal(err) + } + err = nc2.Publish("public.foo", []byte("public message from server B to server A")) + if err != nil { + t.Fatal(err) + } + nc2.Flush() + err = nc3.Publish("public.foo", []byte("dropped message since unauthorized server")) + if err != nil { + t.Fatal(err) + } + nc3.Flush() + + // Message will be received since allowed via permissions + _, err = publicSub.NextMsg(1 * time.Second) + if err != nil { + t.Fatalf("Error during wait for next message: %s", err) + } + msg, err := privateSub.NextMsg(500 * time.Millisecond) + if err == nil { + t.Errorf("Received unexpected message from private sub: %+v", msg) + } + msg, err = publicSub.NextMsg(500 * time.Millisecond) + if err == nil { + t.Errorf("Received unexpected message from private sub: %+v", msg) + } +} + +func TestTLSGatewaysCertificateCNBasedAuth(t *testing.T) { + // Base config for the servers + optsA := LoadConfig("./configs/tls_cert_cn_gateways.conf") + optsB := LoadConfig("./configs/tls_cert_cn_gateways.conf") + optsC := LoadConfig("./configs/tls_cert_cn_gateways_invalid_auth.conf") + + // TLS map should have been enabled + if !optsA.Gateway.TLSMap || !optsB.Gateway.TLSMap || !optsC.Gateway.TLSMap { + t.Error("Expected Cluster TLS verify and map feature to be activated") + } + + gwA, err := url.Parse("nats://localhost:9995") + if err != nil { + t.Fatal(err) + } + gwB, err := url.Parse("nats://localhost:9996") + if err != nil { + t.Fatal(err) + } + gwC, err := url.Parse("nats://localhost:9997") + if err != nil { + t.Fatal(err) + } + + optsA.Host = "127.0.0.1" + optsA.Port = -1 + optsA.Gateway.Name = "A" + optsA.Gateway.Port = 9995 + + optsB.Host = "127.0.0.1" + optsB.Port = -1 + optsB.Gateway.Name = "B" + optsB.Gateway.Port = 9996 + + optsC.Host = "127.0.0.1" + optsC.Port = -1 + optsC.Gateway.Name = "C" + optsC.Gateway.Port = 9997 + + gateways := make([]*server.RemoteGatewayOpts, 3) + gateways[0] = &server.RemoteGatewayOpts{ + Name: optsA.Gateway.Name, + URLs: []*url.URL{gwA}, + } + gateways[1] = &server.RemoteGatewayOpts{ + Name: optsB.Gateway.Name, + URLs: []*url.URL{gwB}, + } + gateways[2] = &server.RemoteGatewayOpts{ + Name: optsC.Gateway.Name, + URLs: []*url.URL{gwC}, + } + optsA.Gateway.Gateways = gateways + optsB.Gateway.Gateways = gateways + optsC.Gateway.Gateways = gateways + + server.SetGatewaysSolicitDelay(100 * time.Millisecond) + defer server.ResetGatewaysSolicitDelay() + + srvA := RunServer(optsA) + defer srvA.Shutdown() + + srvB := RunServer(optsB) + defer srvB.Shutdown() + + srvC := RunServer(optsC) + defer srvC.Shutdown() + + // Because we need to use "localhost" in the gw URLs (to match + // hostname in the user/CN), the server may try to connect to + // a [::1], etc.. that may or may not work, so give a lot of + // time for that to complete ok. + waitForOutboundGateways(t, srvA, 1, 5*time.Second) + waitForOutboundGateways(t, srvB, 1, 5*time.Second) + + nc1, err := nats.Connect(srvA.Addr().String(), nats.Name("A")) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer nc1.Close() + + nc2, err := nats.Connect(srvB.Addr().String(), nats.Name("B")) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer nc2.Close() + + nc3, err := nats.Connect(srvC.Addr().String(), nats.Name("C")) + if err != nil { + t.Fatalf("Expected to connect, got %v", err) + } + defer nc3.Close() + + received := make(chan *nats.Msg) + _, err = nc1.Subscribe("foo", func(msg *nats.Msg) { + select { + case received <- msg: + default: + } + }) + if err != nil { + t.Error(err) + } + _, err = nc3.Subscribe("help", func(msg *nats.Msg) { + nc3.Publish(msg.Reply, []byte("response")) + }) + if err != nil { + t.Error(err) + } + + go func() { + for range time.NewTicker(10 * time.Millisecond).C { + if nc2.IsClosed() { + return + } + nc2.Publish("foo", []byte("bar")) + } + }() + + select { + case <-received: + case <-time.After(2 * time.Second): + t.Error("Timed out waiting for gateway messages") + } + + msg, err := nc2.Request("help", []byte("should fail"), 100*time.Millisecond) + if err == nil { + t.Errorf("Expected to not receive any messages, got: %+v", msg) + } +} + +func TestTLSVerifyClientCertificate(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/tlsverify_noca.conf") + defer srv.Shutdown() + + nurl := fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port) + + // The client is configured properly, but the server has no CA + // to verify the client certificate. Connection should fail. + nc, err := nats.Connect(nurl, + nats.ClientCert("./configs/certs/client-cert.pem", "./configs/certs/client-key.pem"), + nats.RootCAs("./configs/certs/ca.pem")) + if err == nil { + nc.Close() + t.Fatal("Expected failure to connect, did not") + } +} + +func TestTLSConnectionTimeout(t *testing.T) { + opts := LoadConfig("./configs/tls.conf") + opts.TLSTimeout = 0.25 + + srv := RunServer(opts) + defer srv.Shutdown() + + // Dial with normal TCP + endpoint := net.JoinHostPort(opts.Host, fmt.Sprintf("%d", opts.Port)) + conn, err := net.Dial("tcp", endpoint) + if err != nil { + t.Fatalf("Could not connect to %q", endpoint) + } + defer conn.Close() + + // Read deadlines + conn.SetReadDeadline(time.Now().Add(2 * time.Second)) + + // Read the INFO string. + br := bufio.NewReader(conn) + info, err := br.ReadString('\n') + if err != nil { + t.Fatalf("Failed to read INFO - %v", err) + } + if !strings.HasPrefix(info, "INFO ") { + t.Fatalf("INFO response incorrect: %s\n", info) + } + wait := time.Duration(opts.TLSTimeout * float64(time.Second)) + time.Sleep(wait) + // Read deadlines + conn.SetReadDeadline(time.Now().Add(2 * time.Second)) + tlsErr, err := br.ReadString('\n') + if err == nil && !strings.Contains(tlsErr, "-ERR 'Secure Connection - TLS Required") { + t.Fatalf("TLS Timeout response incorrect: %q\n", tlsErr) + } +} + +// Ensure there is no race between authorization timeout and TLS handshake. +func TestTLSAuthorizationShortTimeout(t *testing.T) { + opts := LoadConfig("./configs/tls.conf") + opts.AuthTimeout = 0.001 + + srv := RunServer(opts) + defer srv.Shutdown() + + endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) + nurl := fmt.Sprintf("tls://%s:%s@%s/", opts.Username, opts.Password, endpoint) + + // Expect an error here (no CA) but not a TLS oversized record error which + // indicates the authorization timeout fired too soon. + _, err := nats.Connect(nurl) + if err == nil { + t.Fatal("Expected error trying to connect to secure server") + } + if strings.Contains(err.Error(), "oversized record") { + t.Fatal("Corrupted TLS handshake:", err) + } +} + +func TestClientTLSAndNonTLSConnections(t *testing.T) { + s, opts := RunServerWithConfig("./configs/tls_mixed.conf") + defer s.Shutdown() + + surl := fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port) + nc, err := nats.Connect(surl, nats.RootCAs("./configs/certs/ca.pem")) + if err != nil { + t.Fatalf("Failed to connect with TLS: %v", err) + } + defer nc.Close() + + // Now also make sure we can connect through plain text. + nurl := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) + nc2, err := nats.Connect(nurl) + if err != nil { + t.Fatalf("Failed to connect without TLS: %v", err) + } + defer nc2.Close() + + // Make sure they can go back and forth. + sub, err := nc.SubscribeSync("foo") + if err != nil { + t.Fatalf("Error subscribing: %v", err) + } + sub2, err := nc2.SubscribeSync("bar") + if err != nil { + t.Fatalf("Error subscribing: %v", err) + } + nc.Flush() + nc2.Flush() + + nmsgs := 100 + for i := 0; i < nmsgs; i++ { + nc2.Publish("foo", []byte("HELLO FROM PLAINTEXT")) + nc.Publish("bar", []byte("HELLO FROM TLS")) + } + // Now wait for the messages. + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if msgs, _, err := sub.Pending(); err != nil || msgs != nmsgs { + return fmt.Errorf("Did not receive the correct number of messages: %d", msgs) + } + if msgs, _, err := sub2.Pending(); err != nil || msgs != nmsgs { + return fmt.Errorf("Did not receive the correct number of messages: %d", msgs) + } + return nil + }) + +} + +func stressConnect(t *testing.T, wg *sync.WaitGroup, errCh chan error, url string, index int) { + defer wg.Done() + + subName := fmt.Sprintf("foo.%d", index) + + for i := 0; i < 33; i++ { + nc, err := nats.Connect(url, nats.RootCAs("./configs/certs/ca.pem")) + if err != nil { + errCh <- fmt.Errorf("Unable to create TLS connection: %v\n", err) + return + } + defer nc.Close() + + sub, err := nc.SubscribeSync(subName) + if err != nil { + errCh <- fmt.Errorf("Unable to subscribe on '%s': %v\n", subName, err) + return + } + + if err := nc.Publish(subName, []byte("secure data")); err != nil { + errCh <- fmt.Errorf("Unable to send on '%s': %v\n", subName, err) + } + + if _, err := sub.NextMsg(2 * time.Second); err != nil { + errCh <- fmt.Errorf("Unable to get next message: %v\n", err) + } + + nc.Close() + } + + errCh <- nil +} + +func TestTLSStressConnect(t *testing.T) { + opts, err := server.ProcessConfigFile("./configs/tls.conf") + if err != nil { + panic(fmt.Sprintf("Error processing configuration file: %v", err)) + } + opts.NoSigs, opts.NoLog = true, true + + // For this test, remove the authorization + opts.Username = "" + opts.Password = "" + + // Increase ssl timeout + opts.TLSTimeout = 2.0 + + srv := RunServer(opts) + defer srv.Shutdown() + + nurl := fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port) + + threadCount := 3 + + errCh := make(chan error, threadCount) + + var wg sync.WaitGroup + wg.Add(threadCount) + + for i := 0; i < threadCount; i++ { + go stressConnect(t, &wg, errCh, nurl, i) + } + + wg.Wait() + + var lastError error + for i := 0; i < threadCount; i++ { + err := <-errCh + if err != nil { + lastError = err + } + } + + if lastError != nil { + t.Fatalf("%v\n", lastError) + } +} + +func TestTLSBadAuthError(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/tls.conf") + defer srv.Shutdown() + + endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) + nurl := fmt.Sprintf("tls://%s:%s@%s/", opts.Username, "NOT_THE_PASSWORD", endpoint) + + _, err := nats.Connect(nurl, nats.RootCAs("./configs/certs/ca.pem")) + if err == nil { + t.Fatalf("Expected error trying to connect to secure server") + } + if strings.ToLower(err.Error()) != nats.ErrAuthorization.Error() { + t.Fatalf("Expected and auth violation, got %v\n", err) + } +} + +func TestTLSConnectionCurvePref(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/tls_curve_pref.conf") + defer srv.Shutdown() + + if len(opts.TLSConfig.CurvePreferences) != 1 { + t.Fatal("Invalid curve preference loaded.") + } + + if opts.TLSConfig.CurvePreferences[0] != tls.CurveP256 { + t.Fatalf("Invalid curve preference loaded [%v].", opts.TLSConfig.CurvePreferences[0]) + } + + endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) + nurl := fmt.Sprintf("tls://%s:%s@%s/", opts.Username, opts.Password, endpoint) + nc, err := nats.Connect(nurl) + if err == nil { + nc.Close() + t.Fatalf("Expected error trying to connect to secure server") + } + + // Do simple SecureConnect + nc, err = nats.Connect(fmt.Sprintf("tls://%s/", endpoint)) + if err == nil { + nc.Close() + t.Fatalf("Expected error trying to connect to secure server with no auth") + } + + // Now do more advanced checking, verifying servername and using rootCA. + + nc, err = nats.Connect(nurl, nats.RootCAs("./configs/certs/ca.pem")) + if err != nil { + t.Fatalf("Got an error on Connect with Secure Options: %+v\n", err) + } + defer nc.Close() + + subj := "foo-tls" + sub, _ := nc.SubscribeSync(subj) + + nc.Publish(subj, []byte("We are Secure!")) + nc.Flush() + nmsgs, _, _ := sub.Pending() + if nmsgs != 1 { + t.Fatalf("Expected to receive a message over the TLS connection") + } +} + +type captureSlowConsumerLogger struct { + dummyLogger + ch chan string + gotIt bool +} + +func (l *captureSlowConsumerLogger) Noticef(format string, v ...any) { + msg := fmt.Sprintf(format, v...) + if strings.Contains(msg, "Slow Consumer") { + l.Lock() + if !l.gotIt { + l.gotIt = true + l.ch <- msg + } + l.Unlock() + } +} + +func TestTLSTimeoutNotReportSlowConsumer(t *testing.T) { + oa, err := server.ProcessConfigFile("./configs/srv_a_tls.conf") + if err != nil { + t.Fatalf("Unable to load config file: %v", err) + } + + // Override TLSTimeout to very small value so that handshake fails + oa.Cluster.TLSTimeout = 0.0000001 + sa := RunServer(oa) + defer sa.Shutdown() + + ch := make(chan string, 1) + cscl := &captureSlowConsumerLogger{ch: ch} + sa.SetLogger(cscl, false, false) + + ob, err := server.ProcessConfigFile("./configs/srv_b_tls.conf") + if err != nil { + t.Fatalf("Unable to load config file: %v", err) + } + + sb := RunServer(ob) + defer sb.Shutdown() + + // Watch the logger for a bit and make sure we don't get any + // Slow Consumer error. + select { + case e := <-ch: + t.Fatalf("Unexpected slow consumer error: %s", e) + case <-time.After(500 * time.Millisecond): + // ok + } +} + +func TestTLSHandshakeFailureMemUsage(t *testing.T) { + for i, test := range []struct { + name string + config string + }{ + { + "connect to TLS client port", + ` + port: -1 + %s + `, + }, + { + "connect to TLS route port", + ` + port: -1 + cluster { + port: -1 + %s + } + `, + }, + { + "connect to TLS gateway port", + ` + port: -1 + gateway { + name: "A" + port: -1 + %s + } + `, + }, + } { + t.Run(test.name, func(t *testing.T) { + content := fmt.Sprintf(test.config, ` + tls { + cert_file: "configs/certs/server-cert.pem" + key_file: "configs/certs/server-key.pem" + ca_file: "configs/certs/ca.pem" + timeout: 5 + } + `) + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + var port int + switch i { + case 0: + port = opts.Port + case 1: + port = s.ClusterAddr().Port + case 2: + port = s.GatewayAddr().Port + } + + varz, _ := s.Varz(nil) + base := varz.Mem + buf := make([]byte, 1024*1024) + for i := 0; i < 100; i++ { + conn, err := net.Dial("tcp", fmt.Sprintf("localhost:%d", port)) + if err != nil { + t.Fatalf("Error on dial: %v", err) + } + conn.SetWriteDeadline(time.Now().Add(10 * time.Millisecond)) + conn.Write(buf) + conn.Close() + } + + varz, _ = s.Varz(nil) + newval := (varz.Mem - base) / (1024 * 1024) + // May need to adjust that, but right now, with 100 clients + // we are at about 20MB for client port, 11MB for route + // and 6MB for gateways, so pick 50MB as the threshold + if newval >= 50 { + t.Fatalf("Consumed too much memory: %v MB", newval) + } + }) + } +} + +func TestTLSClientAuthWithRDNSequence(t *testing.T) { + for _, test := range []struct { + name string + config string + certs nats.Option + err error + rerr error + }{ + // To generate certs for these tests: + // USE `regenerate_rdns_svid.sh` TO REGENERATE + // + // ``` + // openssl req -newkey rsa:2048 -nodes -keyout client-$CLIENT_ID.key -subj "/C=US/ST=CA/L=Los Angeles/OU=NATS/O=NATS/CN=*.example.com/DC=example/DC=com" -addext extendedKeyUsage=clientAuth -out client-$CLIENT_ID.csr + // openssl x509 -req -extfile <(printf "subjectAltName=DNS:localhost,DNS:example.com,DNS:www.example.com") -days 1825 -in client-$CLIENT_ID.csr -CA ca.pem -CAkey ca.key -CAcreateserial -out client-$CLIENT_ID.pem + // ``` + // + // To confirm subject from cert: + // + // ``` + // openssl x509 -in client-$CLIENT_ID.pem -text | grep Subject: + // ``` + // + { + "connect with tls using full RDN sequence", + ` + port: -1 + %s + + authorization { + users = [ + { user = "CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=foo1,DC=foo2" } + ] + } + `, + // C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo1, DC = foo2 + nats.ClientCert("./configs/certs/rdns/client-a.pem", "./configs/certs/rdns/client-a.key"), + nil, + nil, + }, + { + "connect with tls using full RDN sequence in original order", + ` + port: -1 + %s + + authorization { + users = [ + { user = "DC=foo2,DC=foo1,CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US" } + ] + } + `, + // C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo1, DC = foo2 + nats.ClientCert("./configs/certs/rdns/client-a.pem", "./configs/certs/rdns/client-a.key"), + nil, + nil, + }, + { + "connect with tls using partial RDN sequence has different permissions", + ` + port: -1 + %s + + authorization { + users = [ + { user = "DC=foo2,DC=foo1,CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US" }, + { user = "CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=foo1,DC=foo2" }, + { user = "CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US", + permissions = { subscribe = { deny = ">" }} } + ] + } + `, + // C = US, ST = California, L = Los Angeles, O = NATS, OU = NATS, CN = localhost + nats.ClientCert("./configs/certs/rdns/client-b.pem", "./configs/certs/rdns/client-b.key"), + nil, + errors.New("nats: timeout"), + }, + { + "connect with tls and RDN sequence partially matches", + ` + port: -1 + %s + + authorization { + users = [ + { user = "DC=foo2,DC=foo1,CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US" } + { user = "CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=foo1,DC=foo2" } + { user = "CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US"}, + ] + } + `, + // Cert is: + // + // C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo3, DC = foo4 + // + // but it will actually match the user without DCs so will not get an error: + // + // C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost + // + nats.ClientCert("./configs/certs/rdns/client-c.pem", "./configs/certs/rdns/client-c.key"), + nil, + nil, + }, + { + "connect with tls and RDN sequence does not match", + ` + port: -1 + %s + + authorization { + users = [ + { user = "CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=foo1,DC=foo2" }, + { user = "DC=foo2,DC=foo1,CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US" } + ] + } + `, + // C = US, ST = California, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo3, DC = foo4 + // + nats.ClientCert("./configs/certs/rdns/client-c.pem", "./configs/certs/rdns/client-c.key"), + errors.New("nats: Authorization Violation"), + nil, + }, + { + "connect with tls and RDN sequence with space after comma not should matter", + ` + port: -1 + %s + + authorization { + users = [ + { user = "DC=foo2, DC=foo1, CN=localhost, OU=NATS, O=NATS, L=Los Angeles, ST=CA, C=US" } + ] + } + `, + // C=US/ST=California/L=Los Angeles/O=NATS/OU=NATS/CN=localhost/DC=foo1/DC=foo2 + // + nats.ClientCert("./configs/certs/rdns/client-a.pem", "./configs/certs/rdns/client-a.key"), + nil, + nil, + }, + { + "connect with tls and full RDN sequence respects order", + ` + port: -1 + %s + + authorization { + users = [ + { user = "DC=com, DC=example, CN=*.example.com, O=NATS, OU=NATS, L=Los Angeles, ST=CA, C=US" } + ] + } + `, + // + // C = US, ST = CA, L = Los Angeles, OU = NATS, O = NATS, CN = *.example.com, DC = example, DC = com + // + nats.ClientCert("./configs/certs/rdns/client-d.pem", "./configs/certs/rdns/client-d.key"), + nil, + nil, + }, + { + "connect with tls and full RDN sequence with added domainComponents and spaces also matches", + ` + port: -1 + %s + + authorization { + users = [ + { user = "CN=*.example.com,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=example,DC=com" } + ] + } + `, + // + // C = US, ST = CA, L = Los Angeles, OU = NATS, O = NATS, CN = *.example.com, DC = example, DC = com + // + nats.ClientCert("./configs/certs/rdns/client-d.pem", "./configs/certs/rdns/client-d.key"), + nil, + nil, + }, + { + "connect with tls and full RDN sequence with correct order takes precedence over others matches", + ` + port: -1 + %s + + authorization { + users = [ + { user = "CN=*.example.com,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=example,DC=com", + permissions = { subscribe = { deny = ">" }} } + + # This should take precedence since it is in the RFC2253 order. + { user = "DC=com,DC=example,CN=*.example.com,O=NATS,OU=NATS,L=Los Angeles,ST=CA,C=US" } + + { user = "CN=*.example.com,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US", + permissions = { subscribe = { deny = ">" }} } + ] + } + `, + // + // C = US, ST = CA, L = Los Angeles, OU = NATS, O = NATS, CN = *.example.com, DC = example, DC = com + // + nats.ClientCert("./configs/certs/rdns/client-d.pem", "./configs/certs/rdns/client-d.key"), + nil, + nil, + }, + { + "connect with tls and RDN includes multiple CN elements", + ` + port: -1 + %s + + authorization { + users = [ + { user = "DC=com,DC=acme,OU=Organic Units,OU=Users,CN=jdoe,CN=123456,CN=John Doe" } + ] + } + `, + // + // OpenSSL: -subj "/CN=John Doe/CN=123456/CN=jdoe/OU=Users/OU=Organic Units/DC=acme/DC=com" + // Go: CN=jdoe,OU=Users+OU=Organic Units + // RFC2253: DC=com,DC=acme,OU=Organic Units,OU=Users,CN=jdoe,CN=123456,CN=John Doe + // + nats.ClientCert("./configs/certs/rdns/client-e.pem", "./configs/certs/rdns/client-e.key"), + nil, + nil, + }, + { + "connect with tls and DN includes a multi value RDN", + ` + port: -1 + %s + + authorization { + users = [ + { user = "CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org" } + ] + } + `, + // + // OpenSSL: -subj "/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe" -multivalue-rdn + // Go: CN=John Doe,O=users + // RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org + // + nats.ClientCert("./configs/certs/rdns/client-f.pem", "./configs/certs/rdns/client-f.key"), + nil, + nil, + }, + { + "connect with tls and DN includes a multi value RDN but there is no match", + ` + port: -1 + %s + + authorization { + users = [ + { user = "CN=John Doe,DC=DEV,DC=OpenSSL,DC=org" } + ] + } + `, + // + // OpenSSL: -subj "/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe" -multivalue-rdn + // Go: CN=John Doe,O=users + // RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org + // + nats.ClientCert("./configs/certs/rdns/client-f.pem", "./configs/certs/rdns/client-f.key"), + errors.New("nats: Authorization Violation"), + nil, + }, + { + "connect with tls and DN includes a multi value RDN that are reordered", + ` + port: -1 + %s + + authorization { + users = [ + { user = "CN=John Doe,O=users+DC=DEV,DC=OpenSSL,DC=org" } + ] + } + `, + // + // OpenSSL: -subj "/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe" -multivalue-rdn + // Go: CN=John Doe,O=users + // RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org + // + nats.ClientCert("./configs/certs/rdns/client-f.pem", "./configs/certs/rdns/client-f.key"), + nil, + nil, + }, + } { + t.Run(test.name, func(t *testing.T) { + content := fmt.Sprintf(test.config, ` + tls { + cert_file: "configs/certs/rdns/server.pem" + key_file: "configs/certs/rdns/server.key" + ca_file: "configs/certs/rdns/ca.pem" + timeout: 5 + verify_and_map: true + } + `) + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + test.certs, + nats.RootCAs("./configs/certs/rdns/ca.pem"), + nats.ErrorHandler(noOpErrHandler), + ) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + + nc.Subscribe("ping", func(m *nats.Msg) { + m.Respond([]byte("pong")) + }) + nc.Flush() + + _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) + if test.rerr != nil && err == nil { + t.Errorf("Expected error getting response") + } else if test.rerr == nil && err != nil { + t.Errorf("Expected response") + } + }) + } +} + +func TestTLSClientAuthWithRDNSequenceReordered(t *testing.T) { + for _, test := range []struct { + name string + config string + certs nats.Option + err error + rerr error + }{ + { + "connect with tls and DN includes a multi value RDN that are reordered", + ` + port: -1 + %s + + authorization { + users = [ + { user = "DC=org,DC=OpenSSL,O=users+DC=DEV,CN=John Doe" } + ] + } + `, + // + // OpenSSL: -subj "/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe" -multivalue-rdn + // Go: CN=John Doe,O=users + // RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org + // + nats.ClientCert("./configs/certs/rdns/client-f.pem", "./configs/certs/rdns/client-f.key"), + nil, + nil, + }, + { + "connect with tls and DN includes a multi value RDN that are reordered but not equal RDNs", + ` + port: -1 + %s + + authorization { + users = [ + { user = "DC=org,DC=OpenSSL,O=users,CN=John Doe" } + ] + } + `, + // + // OpenSSL: -subj "/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe" -multivalue-rdn + // Go: CN=John Doe,O=users + // RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org + // + nats.ClientCert("./configs/certs/rdns/client-f.pem", "./configs/certs/rdns/client-f.key"), + errors.New("nats: Authorization Violation"), + nil, + }, + { + "connect with tls and DN includes a multi value RDN that are reordered but not equal RDNs", + ` + port: -1 + %s + + authorization { + users = [ + { user = "DC=OpenSSL, DC=org, O=users, CN=John Doe" } + ] + } + `, + // + // OpenSSL: -subj "/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe" -multivalue-rdn + // Go: CN=John Doe,O=users + // RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org + // + nats.ClientCert("./configs/certs/rdns/client-f.pem", "./configs/certs/rdns/client-f.key"), + errors.New("nats: Authorization Violation"), + nil, + }, + } { + t.Run(test.name, func(t *testing.T) { + content := fmt.Sprintf(test.config, ` + tls { + cert_file: "configs/certs/rdns/server.pem" + key_file: "configs/certs/rdns/server.key" + ca_file: "configs/certs/rdns/ca.pem" + timeout: 5 + verify_and_map: true + } + `) + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + test.certs, + nats.RootCAs("./configs/certs/rdns/ca.pem"), + ) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + + nc.Subscribe("ping", func(m *nats.Msg) { + m.Respond([]byte("pong")) + }) + nc.Flush() + + _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) + if test.rerr != nil && err == nil { + t.Errorf("Expected error getting response") + } else if test.rerr == nil && err != nil { + t.Errorf("Expected response") + } + }) + } +} + +func TestTLSClientSVIDAuth(t *testing.T) { + for _, test := range []struct { + name string + config string + certs nats.Option + err error + rerr error + }{ + { + "connect with tls using certificate with URIs", + ` + port: -1 + %s + + authorization { + users = [ + { + user = "spiffe://localhost/my-nats-service/user-a" + } + ] + } + `, + nats.ClientCert("./configs/certs/svid/client-a.pem", "./configs/certs/svid/client-a.key"), + nil, + nil, + }, + { + "connect with tls using certificate with limited different permissions", + ` + port: -1 + %s + + authorization { + users = [ + { + user = "spiffe://localhost/my-nats-service/user-a" + }, + { + user = "spiffe://localhost/my-nats-service/user-b" + permissions = { subscribe = { deny = ">" }} + } + ] + } + `, + nats.ClientCert("./configs/certs/svid/client-b.pem", "./configs/certs/svid/client-b.key"), + nil, + errors.New("nats: timeout"), + }, + { + "connect with tls without URIs in permissions will still match SAN", + ` + port: -1 + %s + + authorization { + users = [ + { + user = "O=SPIRE,C=US" + } + ] + } + `, + nats.ClientCert("./configs/certs/svid/client-b.pem", "./configs/certs/svid/client-b.key"), + nil, + nil, + }, + { + "connect with tls but no permissions", + ` + port: -1 + %s + + authorization { + users = [ + { + user = "spiffe://localhost/my-nats-service/user-c" + } + ] + } + `, + nats.ClientCert("./configs/certs/svid/client-a.pem", "./configs/certs/svid/client-a.key"), + errors.New("nats: Authorization Violation"), + nil, + }, + } { + t.Run(test.name, func(t *testing.T) { + content := fmt.Sprintf(test.config, ` + tls { + cert_file: "configs/certs/svid/server.pem" + key_file: "configs/certs/svid/server.key" + ca_file: "configs/certs/svid/ca.pem" + timeout: 5 + insecure: true + verify_and_map: true + } + `) + conf := createConfFile(t, []byte(content)) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + + nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), + test.certs, + nats.RootCAs("./configs/certs/svid/ca.pem"), + nats.ErrorHandler(noOpErrHandler), + ) + if test.err == nil && err != nil { + t.Errorf("Expected to connect, got %v", err) + } else if test.err != nil && err == nil { + t.Errorf("Expected error on connect") + } else if test.err != nil && err != nil { + // Error on connect was expected + if test.err.Error() != err.Error() { + t.Errorf("Expected error %s, got: %s", test.err, err) + } + return + } + defer nc.Close() + + nc.Subscribe("ping", func(m *nats.Msg) { + m.Respond([]byte("pong")) + }) + nc.Flush() + + _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) + if test.rerr != nil && err == nil { + t.Errorf("Expected error getting response") + } else if test.rerr == nil && err != nil { + t.Errorf("Expected response") + } + }) + } +} + +func TestTLSPinnedCertsClient(t *testing.T) { + tmpl := ` + host: localhost + port: -1 + tls { + ca_file: "configs/certs/ca.pem" + cert_file: "configs/certs/server-cert.pem" + key_file: "configs/certs/server-key.pem" + # Require a client certificate and map user id from certificate + verify: true + pinned_certs: ["%s"] + }` + + confFileName := createConfFile(t, []byte(fmt.Sprintf(tmpl, "aaaaaaaa09fde09451411ba3b42c0f74727d61a974c69fd3cf5257f39c75f0e9"))) + srv, o := RunServerWithConfig(confFileName) + defer srv.Shutdown() + + if len(o.TLSPinnedCerts) != 1 { + t.Fatal("expected one pinned cert") + } + + opts := []nats.Option{ + nats.RootCAs("configs/certs/ca.pem"), + nats.ClientCert("./configs/certs/client-cert.pem", "./configs/certs/client-key.pem"), + } + + nc, err := nats.Connect(srv.ClientURL(), opts...) + if err == nil { + nc.Close() + t.Fatalf("Expected error trying to connect without a certificate in pinned_certs") + } + + os.WriteFile(confFileName, []byte(fmt.Sprintf(tmpl, "bf6f821f09fde09451411ba3b42c0f74727d61a974c69fd3cf5257f39c75f0e9")), 0660) + if err := srv.Reload(); err != nil { + t.Fatalf("on Reload got %v", err) + } + // reload pinned to the certs used + nc, err = nats.Connect(srv.ClientURL(), opts...) + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + nc.Close() +} + +type captureWarnLogger struct { + dummyLogger + receive chan string +} + +func newCaptureWarnLogger() *captureWarnLogger { + return &captureWarnLogger{ + receive: make(chan string, 100), + } +} + +func (l *captureWarnLogger) Warnf(format string, v ...any) { + l.receive <- fmt.Sprintf(format, v...) +} + +func (l *captureWarnLogger) waitFor(expect string, timeout time.Duration) bool { + for { + select { + case msg := <-l.receive: + if strings.Contains(msg, expect) { + return true + } + case <-time.After(timeout): + return false + } + } +} + +func TestTLSConnectionRate(t *testing.T) { + config := ` + listen: "127.0.0.1:-1" + tls { + cert_file: "./configs/certs/server-cert.pem" + key_file: "./configs/certs/server-key.pem" + connection_rate_limit: 3 + } + ` + + confFileName := createConfFile(t, []byte(config)) + + srv, _ := RunServerWithConfig(confFileName) + logger := newCaptureWarnLogger() + srv.SetLogger(logger, false, false) + defer srv.Shutdown() + + var err error + count := 0 + for count < 10 { + var nc *nats.Conn + nc, err = nats.Connect(srv.ClientURL(), nats.RootCAs("./configs/certs/ca.pem")) + + if err != nil { + break + } + nc.Close() + count++ + } + + if count != 3 { + t.Fatalf("Expected 3 connections per second, got %d (%v)", count, err) + } + + if !logger.waitFor("connections due to TLS rate limiting", time.Second) { + t.Fatalf("did not log 'TLS rate limiting' warning") + } +} + +func TestTLSPinnedCertsRoute(t *testing.T) { + tmplSeed := ` + host: localhost + port: -1 + cluster { + port: -1 + pool_size: -1 + tls { + ca_file: "configs/certs/ca.pem" + cert_file: "configs/certs/server-cert.pem" + key_file: "configs/certs/server-key.pem" + } + }` + // this server connects to seed, but is set up to not trust seeds cert + tmplSrv := ` + host: localhost + port: -1 + cluster { + port: -1 + routes = [nats-route://localhost:%d] + tls { + ca_file: "configs/certs/ca.pem" + cert_file: "configs/certs/server-cert.pem" + key_file: "configs/certs/server-key.pem" + # Require a client certificate and map user id from certificate + verify: true + # expected to fail the seed server + pinned_certs: ["%s"] + } + }` + + confSeed := createConfFile(t, []byte(tmplSeed)) + srvSeed, o := RunServerWithConfig(confSeed) + defer srvSeed.Shutdown() + + confSrv := createConfFile(t, []byte(fmt.Sprintf(tmplSrv, o.Cluster.Port, "89386860ea1222698ea676fc97310bdf2bff6f7e2b0420fac3b3f8f5a08fede5"))) + srv, _ := RunServerWithConfig(confSrv) + defer srv.Shutdown() + + checkClusterFormed(t, srvSeed, srv) + + // this change will result in the server being and remaining disconnected + os.WriteFile(confSrv, []byte(fmt.Sprintf(tmplSrv, o.Cluster.Port, "aaaaaaaa09fde09451411ba3b42c0f74727d61a974c69fd3cf5257f39c75f0e9")), 0660) + if err := srv.Reload(); err != nil { + t.Fatalf("on Reload got %v", err) + } + + checkNumRoutes(t, srvSeed, 0) + checkNumRoutes(t, srv, 0) +} + +func TestAllowNonTLSReload(t *testing.T) { + tmpl := ` + listen: "127.0.0.1:-1" + ping_interval: "%s" + tls { + ca_file: "configs/certs/ca.pem" + cert_file: "configs/certs/server-cert.pem" + key_file: "configs/certs/server-key.pem" + } + allow_non_tls: true + ` + conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, "10s"))) + s, o := RunServerWithConfig(conf) + defer s.Shutdown() + + check := func() { + t.Helper() + nc := createClientConn(t, "127.0.0.1", o.Port) + defer nc.Close() + info := checkInfoMsg(t, nc) + if !info.TLSAvailable { + t.Fatal("TLSAvailable should be true, was false") + } + if info.TLSRequired { + t.Fatal("TLSRequired should be false, was true") + } + } + check() + + os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, "20s")), 0660) + if err := s.Reload(); err != nil { + t.Fatalf("Error on reload: %v", err) + } + check() +} diff --git a/test/user_authorization_test.go b/test/user_authorization_test.go new file mode 100644 index 0000000..feca53e --- /dev/null +++ b/test/user_authorization_test.go @@ -0,0 +1,316 @@ +// Copyright 2016-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "regexp" + "testing" + "time" +) + +const DefaultPass = "foo" + +var permErrRe = regexp.MustCompile(`\A\-ERR\s+'Permissions Violation([^\r\n]+)\r\n`) + +func TestUserAuthorizationProto(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/authorization.conf") + defer srv.Shutdown() + + // Alice can do anything, check a few for OK result. + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, "", "alice", DefaultPass) + expectResult(t, c, okRe) + sendProto(t, c, "PUB foo 2\r\nok\r\n") + expectResult(t, c, okRe) + sendProto(t, c, "SUB foo 1\r\n") + expectResult(t, c, okRe) + + // Check that _ is ok + sendProto(t, c, "PUB _ 2\r\nok\r\n") + expectResult(t, c, okRe) + + c.Close() + + // Bob is a requestor only, e.g. req.foo, req.bar for publish, subscribe only to INBOXes. + c = createClientConn(t, opts.Host, opts.Port) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, "", "bob", DefaultPass) + expectResult(t, c, okRe) + + // These should error. + sendProto(t, c, "SUB foo 1\r\n") + expectResult(t, c, permErrRe) + sendProto(t, c, "PUB foo 2\r\nok\r\n") + expectResult(t, c, permErrRe) + + // These should work ok. + sendProto(t, c, "SUB _INBOX.abcd 1\r\n") + expectResult(t, c, okRe) + sendProto(t, c, "PUB req.foo 2\r\nok\r\n") + expectResult(t, c, okRe) + sendProto(t, c, "PUB req.bar 2\r\nok\r\n") + expectResult(t, c, okRe) + c.Close() + + // Joe is a default user + c = createClientConn(t, opts.Host, opts.Port) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, "", "joe", DefaultPass) + expectResult(t, c, okRe) + + // These should error. + sendProto(t, c, "SUB foo.bar.* 1\r\n") + expectResult(t, c, permErrRe) + sendProto(t, c, "PUB foo.bar.baz 2\r\nok\r\n") + expectResult(t, c, permErrRe) + + // These should work ok. + sendProto(t, c, "SUB _INBOX.abcd 1\r\n") + expectResult(t, c, okRe) + sendProto(t, c, "SUB PUBLIC.abcd 1\r\n") + expectResult(t, c, okRe) + + sendProto(t, c, "PUB SANDBOX.foo 2\r\nok\r\n") + expectResult(t, c, okRe) + sendProto(t, c, "PUB SANDBOX.bar 2\r\nok\r\n") + expectResult(t, c, okRe) + + // Since only PWC, this should fail (too many tokens). + sendProto(t, c, "PUB SANDBOX.foo.bar 2\r\nok\r\n") + expectResult(t, c, permErrRe) + + c.Close() + + // This is the new style permissions with allow and deny clauses. + c = createClientConn(t, opts.Host, opts.Port) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, "", "ns", DefaultPass) + expectResult(t, c, okRe) + + // These should work + sendProto(t, c, "PUB SANDBOX.foo 2\r\nok\r\n") + expectResult(t, c, okRe) + sendProto(t, c, "PUB baz.bar 2\r\nok\r\n") + expectResult(t, c, okRe) + sendProto(t, c, "PUB baz.foo 2\r\nok\r\n") + expectResult(t, c, okRe) + + // These should error. + sendProto(t, c, "PUB foo 2\r\nok\r\n") + expectResult(t, c, permErrRe) + sendProto(t, c, "PUB bar 2\r\nok\r\n") + expectResult(t, c, permErrRe) + sendProto(t, c, "PUB foo.bar 2\r\nok\r\n") + expectResult(t, c, permErrRe) + sendProto(t, c, "PUB foo.bar.baz 2\r\nok\r\n") + expectResult(t, c, permErrRe) + sendProto(t, c, "PUB SYS.1 2\r\nok\r\n") + expectResult(t, c, permErrRe) + + // Subscriptions + + // These should work ok. + sendProto(t, c, "SUB foo.bar 1\r\n") + expectResult(t, c, okRe) + sendProto(t, c, "SUB foo.foo 1\r\n") + expectResult(t, c, okRe) + + // These should error. + sendProto(t, c, "SUB foo 1\r\n") + expectResult(t, c, permErrRe) + sendProto(t, c, "SUB foo.baz 1\r\n") + expectResult(t, c, permErrRe) + sendProto(t, c, "SUB foo.baz 1\r\n") + expectResult(t, c, permErrRe) + sendProto(t, c, "SUB foo.baz 1\r\n") + expectResult(t, c, permErrRe) + + // Deny clauses for subscriptions need to be able to allow subscriptions + // on larger scoped wildcards, but prevent delivery of a message whose + // subject matches a deny clause. + + // Clear old stuff + c.Close() + + c = createClientConn(t, opts.Host, opts.Port) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, "", "ns", DefaultPass) + expectResult(t, c, okRe) + + sendProto(t, c, "SUB foo.* 1\r\n") + expectResult(t, c, okRe) + + sendProto(t, c, "SUB foo.* bar 2\r\n") + expectResult(t, c, okRe) + + // Now send on foo.baz which should not be received on first client. + // Joe is a default user + nc := createClientConn(t, opts.Host, opts.Port) + defer nc.Close() + expectAuthRequired(t, nc) + doAuthConnect(t, nc, "", "ns-pub", DefaultPass) + expectResult(t, nc, okRe) + + sendProto(t, nc, "PUB foo.baz 2\r\nok\r\n") + expectResult(t, nc, okRe) + + // Expect nothing from the wildcard subscription. + expectNothing(t, c) + + sendProto(t, c, "PING\r\n") + expectResult(t, c, pongRe) + + // Now create a queue sub on our ns-pub user. We want to test that + // queue subscribers can be denied and delivery will route around. + sendProto(t, nc, "SUB foo.baz bar 2\r\n") + expectResult(t, nc, okRe) + + // Make sure we always get the message on our queue subscriber. + // Do this several times since we should select the other subscriber + // but get permission denied.. + for i := 0; i < 20; i++ { + sendProto(t, nc, "PUB foo.baz 2\r\nok\r\n") + buf := expectResult(t, nc, okRe) + if msgRe.Match(buf) { + continue + } else { + expectResult(t, nc, msgRe) + } + } + + // Clear old stuff + c.Close() + + c = createClientConn(t, opts.Host, opts.Port) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, "", "ns", DefaultPass) + expectResult(t, c, okRe) + + sendProto(t, c, "SUB foo.bar 1\r\n") + expectResult(t, c, okRe) + + sendProto(t, c, "SUB foo.bar.baz 2\r\n") + expectResult(t, c, errRe) + + sendProto(t, c, "SUB > 3\r\n") + expectResult(t, c, errRe) + + sendProto(t, c, "SUB SYS.> 4\r\n") + expectResult(t, c, errRe) + + sendProto(t, c, "SUB SYS.TEST.foo 5\r\n") + expectResult(t, c, okRe) + + sendProto(t, c, "SUB SYS.bar 5\r\n") + expectResult(t, c, errRe) +} + +func TestUserAuthorizationAllowResponses(t *testing.T) { + srv, opts := RunServerWithConfig("./configs/authorization.conf") + defer srv.Shutdown() + + // Alice can do anything, so she will be our requestor + rc := createClientConn(t, opts.Host, opts.Port) + defer rc.Close() + expectAuthRequired(t, rc) + doAuthConnect(t, rc, "", "alice", DefaultPass) + expectResult(t, rc, okRe) + + // MY_SERVICE can subscribe to a single request subject but can + // respond to any reply subject that it receives, but only + // for one response. + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, "", "svca", DefaultPass) + expectResult(t, c, okRe) + + sendProto(t, c, "SUB my.service.req 1\r\n") + expectResult(t, c, okRe) + + sendProto(t, rc, "PUB my.service.req resp.bar.22 2\r\nok\r\n") + expectResult(t, rc, okRe) + + matches := msgRe.FindAllSubmatch(expectResult(t, c, msgRe), -1) + checkMsg(t, matches[0], "my.service.req", "1", "resp.bar.22", "2", "ok") + + // This should be allowed + sendProto(t, c, "PUB resp.bar.22 2\r\nok\r\n") + expectResult(t, c, okRe) + + // This should not be allowed + sendProto(t, c, "PUB resp.bar.33 2\r\nok\r\n") + expectResult(t, c, errRe) + + // This should also not be allowed now since we already sent a response and max is 1. + sendProto(t, c, "PUB resp.bar.22 2\r\nok\r\n") + expectResult(t, c, errRe) + + c.Close() // from MY_SERVICE + + // MY_STREAM_SERVICE can subscribe to a single request subject but can + // respond to any reply subject that it receives, and send up to 10 responses. + // Each permission for a response can last up to 10ms. + c = createClientConn(t, opts.Host, opts.Port) + defer c.Close() + expectAuthRequired(t, c) + doAuthConnect(t, c, "", "svcb", DefaultPass) + expectResult(t, c, okRe) + + sendProto(t, c, "SUB my.service.req 1\r\n") + expectResult(t, c, okRe) + + // Same rules as above. + sendProto(t, rc, "PUB my.service.req resp.bar.22 2\r\nok\r\n") + expectResult(t, rc, okRe) + + matches = msgRe.FindAllSubmatch(expectResult(t, c, msgRe), -1) + checkMsg(t, matches[0], "my.service.req", "1", "resp.bar.22", "2", "ok") + + // This should be allowed + sendProto(t, c, "PUB resp.bar.22 2\r\nok\r\n") + expectResult(t, c, okRe) + + // This should not be allowed + sendProto(t, c, "PUB resp.bar.33 2\r\nok\r\n") + expectResult(t, c, errRe) + + // We should be able to send 9 more here since we are allowed 10 total + for i := 0; i < 9; i++ { + sendProto(t, c, "PUB resp.bar.22 2\r\nok\r\n") + expectResult(t, c, okRe) + } + // Now this should fail since we already sent 10 responses. + sendProto(t, c, "PUB resp.bar.22 2\r\nok\r\n") + expectResult(t, c, errRe) + + // Now test timeout. + sendProto(t, rc, "PUB my.service.req resp.bar.11 2\r\nok\r\n") + expectResult(t, rc, okRe) + + matches = msgRe.FindAllSubmatch(expectResult(t, c, msgRe), -1) + checkMsg(t, matches[0], "my.service.req", "1", "resp.bar.11", "2", "ok") + + time.Sleep(100 * time.Millisecond) + + sendProto(t, c, "PUB resp.bar.11 2\r\nok\r\n") + expectResult(t, c, errRe) +} diff --git a/test/verbose_test.go b/test/verbose_test.go new file mode 100644 index 0000000..d370a2d --- /dev/null +++ b/test/verbose_test.go @@ -0,0 +1,82 @@ +// Copyright 2012-2018 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "testing" +) + +func TestVerbosePing(t *testing.T) { + s := runProtoServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) + defer c.Close() + + doConnect(t, c, true, false, false) + + send := sendCommand(t, c) + expect := expectCommand(t, c) + + expect(okRe) + + // Ping should still be same + send("PING\r\n") + expect(pongRe) +} + +func TestVerboseConnect(t *testing.T) { + s := runProtoServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) + defer c.Close() + + doConnect(t, c, true, false, false) + + send := sendCommand(t, c) + expect := expectCommand(t, c) + + expect(okRe) + + // Connect + send("CONNECT {\"verbose\":true,\"pedantic\":true,\"tls_required\":false}\r\n") + expect(okRe) +} + +func TestVerbosePubSub(t *testing.T) { + s := runProtoServer() + defer s.Shutdown() + + c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) + defer c.Close() + + doConnect(t, c, true, false, false) + send := sendCommand(t, c) + expect := expectCommand(t, c) + + expect(okRe) + + // Pub + send("PUB foo 2\r\nok\r\n") + expect(okRe) + + // Sub + send("SUB foo 1\r\n") + expect(okRe) + + // UnSub + send("UNSUB 1\r\n") + expect(okRe) +} diff --git a/util/nats-server-hardened.service b/util/nats-server-hardened.service new file mode 100644 index 0000000..9c458fc --- /dev/null +++ b/util/nats-server-hardened.service @@ -0,0 +1,90 @@ +[Unit] +Description=NATS Server +After=network-online.target ntp.service + +# If you use a dedicated filesystem for JetStream data, then you might use something like: +# ConditionPathIsMountPoint=/srv/jetstream +# See also Service.ReadWritePaths + +[Service] +Type=simple +EnvironmentFile=-/etc/default/nats-server +ExecStart=/usr/sbin/nats-server -c /etc/nats-server.conf +ExecReload=/bin/kill -s HUP $MAINPID + +# The nats-server uses SIGUSR2 to trigger Lame Duck Mode (LDM) shutdown +# https://docs.nats.io/running-a-nats-service/nats_admin/lame_duck_mode +ExecStop=/bin/kill -s SIGUSR2 $MAINPID + +User=nats +Group=nats + +Restart=on-failure +RestartSec=5 + +# This should be `lame_duck_duration` + some buffer to finish the shutdown. +# By default, `lame_duck_duration` is 2 mins. +TimeoutStopSec=150 + +# Capacity Limits +# JetStream requires 2 FDs open per stream. +LimitNOFILE=800000 +# Environment=GOMEMLIMIT=12GiB +# You might find it better to set GOMEMLIMIT via /etc/default/nats-server, +# so that you can change limits without needing a systemd daemon-reload. + +# Hardening +CapabilityBoundingSet= +LockPersonality=true +MemoryDenyWriteExecute=true +NoNewPrivileges=true +PrivateDevices=true +PrivateTmp=true +PrivateUsers=true +ProcSubset=pid +ProtectClock=true +ProtectControlGroups=true +ProtectHome=true +ProtectHostname=true +ProtectKernelLogs=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectSystem=strict +ReadOnlyPaths= +RestrictAddressFamilies=AF_INET AF_INET6 +RestrictNamespaces=true +RestrictRealtime=true +RestrictSUIDSGID=true +SystemCallFilter=@system-service ~@privileged ~@resources +UMask=0077 + +# Consider locking down all areas of /etc which hold machine identity keys, etc +InaccessiblePaths=/etc/ssh + +# If you have systemd >= 247 +ProtectProc=invisible + +# If you have systemd >= 248 +PrivateIPC=true + +# Optional: writable directory for JetStream. +# See also: Unit.ConditionPathIsMountPoint +ReadWritePaths=/var/lib/nats + +# Optional: resource control. +# Replace weights by values that make sense for your situation. +# For a list of all options see: +# https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html +#CPUAccounting=true +#CPUWeight=100 # of 10000 +#IOAccounting=true +#IOWeight=100 # of 10000 +#MemoryAccounting=true +#MemoryMax=1GB +#IPAccounting=true + +[Install] +WantedBy=multi-user.target +# If you install this service as nats-server.service and want 'nats' +# to work as an alias, then uncomment this next line: +#Alias=nats.service diff --git a/util/nats-server.service b/util/nats-server.service new file mode 100644 index 0000000..665f246 --- /dev/null +++ b/util/nats-server.service @@ -0,0 +1,25 @@ +[Unit] +Description=NATS Server +After=network-online.target ntp.service + +[Service] +PrivateTmp=true +Type=simple +ExecStart=/usr/sbin/nats-server -c /etc/nats-server.conf +ExecReload=/bin/kill -s HUP $MAINPID + +# The nats-server uses SIGUSR2 to trigger Lame Duck Mode (LDM) shutdown +# https://docs.nats.io/running-a-nats-service/nats_admin/lame_duck_mode +ExecStop=/bin/kill -s SIGUSR2 $MAINPID + +# This should be `lame_duck_duration` + some buffer to finish the shutdown. +# By default, `lame_duck_duration` is 2 mins. +TimeoutStopSec=150 + +Restart=on-failure + +User=nats +Group=nats + +[Install] +WantedBy=multi-user.target