commit 89f5be005ddc1798fbb962c021305de8e128b69e Author: upstream-mirror Date: Wed Sep 16 03:23:42 2026 +0500 snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted) diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 0000000..038fb65 --- /dev/null +++ b/.codespellrc @@ -0,0 +1,23 @@ +[codespell] +# Ref: https://github.com/codespell-project/codespell#using-a-config-file +skip = .git,.git-meta,.gitignore,.gitattributes,*.svg,go.sum,vendor,*.lock,*.css,*.min.*,.codespellrc,*_templ.go +check-hidden = true +# Ignore camelCase and PascalCase identifiers (very common in Go/Rust/JS +# source, e.g. allLocations, publishErr, ReadInside, FlushInterval). +ignore-regex = \b[a-z]+[A-Z]\w*\b|\b[A-Z][a-z]+[A-Z]\w*\b +# visibles: variable name for VisibleInterval collections in filer/mount code +# fo: `*FilerOptions` receiver name (e.g. `func (fo *FilerOptions) ...`) +# te: "truncate error" local variable (e.g. `if te := w.Truncate(end); te != nil`) +# ser: Rust serde serializer variable (`serde_json::ser`, `let mut ser = ...`) +# bject: intentional wildcard test data (e.g. `s3:Get?bject` matching `s3:GetObject`) +# unparseable: accepted alternate spelling used throughout the codebase +# keep-alives: correct plural of the technical term (SSH/HTTP keep-alive) +# tread: valid English word in the idiom "tread carefully" (help text) +# anc: variable abbreviation for "ancestor" in tree/path tests +# ue: appears inside JSON test fixtures with embedded escaped quotes (Bl\"ue) +# auther: local variable meaning "authenticator" (tls.go: `auther := Authenticator{}`) +# thirdparty: literal Maven groupId `org.apache.hadoop.thirdparty` (external, cannot rename) +# unknwon: GitHub username / Go module path (`github.com/unknwon/goconfig`) +# atleast: CLI mode literal string in test/benchmark/fuse_db/bin/sqlite_verify.py +# sme: local variable for a *streamMutateError in mount tests +ignore-words-list = visibles,fo,te,ser,bject,unparseable,keep-alives,tread,anc,ue,auther,thirdparty,unknwon,atleast,sme diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..b0d3b33 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,4 @@ +# These are supported funding model platforms + +github: chrislusf +patreon: seaweedfs diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..77d2eb0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,35 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +Sponsors SeaweedFS via Patreon https://www.patreon.com/seaweedfs +Report issues here. Ask questions here https://stackoverflow.com/questions/tagged/seaweedfs +Please ask questions in https://github.com/seaweedfs/seaweedfs/discussions + +example of a good issue report: +https://github.com/seaweedfs/seaweedfs/issues/1005 +example of a bad issue report: +https://github.com/seaweedfs/seaweedfs/issues/1008 + +**Describe the bug** +A clear and concise description of what the bug is. + +**System Setup** +- List the command line to start "weed master", "weed volume", "weed filer", "weed s3", "weed mount". +- OS version +- output of `weed version` +- if using filer, show the content of `filer.toml` + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Additional context** +Add any other context about the problem here. diff --git a/.github/actions/fix-fusermount-setuid/action.yml b/.github/actions/fix-fusermount-setuid/action.yml new file mode 100644 index 0000000..7aad5bf --- /dev/null +++ b/.github/actions/fix-fusermount-setuid/action.yml @@ -0,0 +1,66 @@ +name: Fix fusermount3 setuid +description: > + Make sure the fusermount3 an unprivileged mount will find can actually mount. + Some runner images carry a second, source-built fusermount3 in /usr/local/bin + that shadows the distro one in PATH; it is neither setuid nor root-owned, so + every mount fails with "mount failed: Operation not permitted". Both the Go + mount and sw-fuse resolve the helper through PATH. + Run it after the step that apt-installs fuse3. + +runs: + using: composite + steps: + - name: Point PATH at a fusermount3 that can mount + shell: bash + run: | + set -euo pipefail + + # setuid only grants root when root owns the file, and exec follows + # symlinks, so judge the target. + can_mount() { + [ -u "$1" ] && [ "$(stat -Lc %u "$1")" = 0 ] + } + + bin=$(command -v fusermount3 || true) + if [ -z "$bin" ]; then + echo "no fusermount3 in PATH" >&2 + exit 1 + fi + if can_mount "$bin"; then + ls -l "$bin" + exit 0 + fi + echo "$bin cannot mount unprivileged:" + ls -l "$bin" + + # The distro fusermount3 is setuid root and is the one meant to be used. + # Reach it through a symlink earlier in PATH: exec resolves the link, so + # the target keeps its setuid bit, and nothing on the image is modified. + for distro in /usr/bin/fusermount3 /bin/fusermount3; do + if [ "$distro" != "$bin" ] && can_mount "$distro"; then + mkdir -p "$RUNNER_TEMP/fuse-bin" + ln -sf "$distro" "$RUNNER_TEMP/fuse-bin/fusermount3" + echo "$RUNNER_TEMP/fuse-bin" >> "$GITHUB_PATH" + echo "using $distro instead" + ls -l "$distro" + exit 0 + fi + done + + # No usable distro binary, so setting the bit is the only way out - but + # the target came from PATH: do that only for a root-owned system binary, + # never for anything else that happens to sit there. + case "$bin" in + /bin/*|/sbin/*|/usr/bin/*|/usr/sbin/*|/usr/local/bin/*|/usr/local/sbin/*) ;; + *) echo "refusing to setuid $bin: outside the system bin paths" >&2; exit 1 ;; + esac + if [ -L "$bin" ] || [ ! -f "$bin" ] || [ ! -x "$bin" ]; then + echo "refusing to setuid $bin: not a regular executable file" >&2 + exit 1 + fi + if [ "$(stat -c %u "$bin")" != 0 ] || [ "$(stat -c %g "$bin")" != 0 ]; then + echo "refusing to setuid $bin: not owned by root:root" >&2 + exit 1 + fi + sudo chmod u+s "$bin" + ls -l "$bin" diff --git a/.github/actions/sign-image/action.yml b/.github/actions/sign-image/action.yml new file mode 100644 index 0000000..82cc046 --- /dev/null +++ b/.github/actions/sign-image/action.yml @@ -0,0 +1,44 @@ +name: Sign container images +description: > + Keyless cosign signature on each image, then a verification pass against the + identity the signature should carry, so a misconfigured job fails here and not + on someone's cluster. That identity is the calling workflow's own, + https://github.com///.github/workflows/@. + The calling job needs `id-token: write` and a registry login for every image. + +inputs: + images: + description: Image references by digest (name@sha256:...), whitespace separated. + required: true + +runs: + using: composite + steps: + - name: Install cosign + uses: sigstore/cosign-installer@v4.1.2 + + - name: Sign + shell: bash + env: + IMAGES: ${{ inputs.images }} + run: | + set -euo pipefail + # The .sig tag layout: the OCI-referrer bundle cosign 3 writes by default + # is not read by the Kyverno and policy-controller releases in use today. + # Cosign 3 also defaults to --use-signing-config, which insists on a + # bundle for its output; turning it off falls back to the default + # Fulcio and Rekor URLs, which is all the .sig layout ever used. + cosign sign --yes --recursive \ + --new-bundle-format=false --use-signing-config=false \ + $IMAGES + + - name: Verify + shell: bash + env: + IMAGES: ${{ inputs.images }} + run: | + set -euo pipefail + cosign verify \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity "https://github.com/$GITHUB_WORKFLOW_REF" \ + $IMAGES diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..57d44a6 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: +- package-ecosystem: "github-actions" + directories: + - "/" + - "/.github/actions/sign-image" + schedule: + interval: "weekly" +- package-ecosystem: gomod + directory: "/" + schedule: + interval: weekly diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..5098e5e --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,19 @@ +# What problem are we solving? + + + +# How are we solving the problem? + + +# How is the PR tested? + + + +# Checks +- [ ] I have added unit tests if possible. +- [ ] I will add related wiki document changes and link to this PR after merging. +- [ ] All AI code review comments have been addressed. No more comments to fix if reviewed again. Reviewer may request additional gemini and copilot reviews. + +# Checks for AI generated PRs +- [ ] I have reviewed every line of code. +- [ ] The PR is kept as minimum as possible. Large PRs would not be accepted. diff --git a/.github/scripts/star_history.py b/.github/scripts/star_history.py new file mode 100644 index 0000000..fa61a8b --- /dev/null +++ b/.github/scripts/star_history.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Render the repository's star history to note/star_history.svg. + +Uses the GitHub REST stargazers endpoint with the starred-at accept header, +which caps at 40,000 entries (400 pages of 100). The chart is regenerated on +a schedule; if the repo grows past that cap the script stops at 40,000 and +logs a warning rather than under-reporting. +""" +import json +import os +import sys +import urllib.request +from datetime import datetime + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib.dates import AutoDateLocator, DateFormatter # noqa: E402 + +REPO = "seaweedfs/seaweedfs" +TOKEN = os.environ["GITHUB_TOKEN"] +OUT = os.environ.get("OUT", "note/star_history.svg") +PAGE_CAP = 400 # GitHub's hard limit on stargazer pagination + + +def fetch_stargazers(): + stars = [] + page = 1 + while page <= PAGE_CAP: + url = f"https://api.github.com/repos/{REPO}/stargazers?per_page=100&page={page}" + req = urllib.request.Request( + url, + headers={ + "Accept": "application/vnd.github.star+json", + "Authorization": f"Bearer {TOKEN}", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "seaweedfs-star-history", + }, + ) + with urllib.request.urlopen(req) as resp: + batch = json.load(resp) + if not batch: + break + for u in batch: + sa = u.get("starred_at") + if sa: + stars.append(datetime.fromisoformat(sa.replace("Z", "+00:00"))) + if len(batch) < 100: + break + page += 1 + if page > PAGE_CAP: + print( + f"::warning::Hit the {PAGE_CAP}-page stargazer pagination cap; " + "chart reflects the first 40,000 stars only." + ) + return stars + + +def render(stars, out): + stars.sort() + counts = list(range(1, len(stars) + 1)) + fig, ax = plt.subplots(figsize=(10, 6), dpi=130) + ax.plot(stars, counts, color="#0969da", linewidth=1.6) + ax.set_xlabel("Date") + ax.set_ylabel("Stars") + ax.set_title(f"{REPO} star history") + ax.grid(True, linestyle="--", alpha=0.3) + ax.xaxis.set_major_locator(AutoDateLocator()) + ax.xaxis.set_major_formatter(DateFormatter("%Y-%m")) + fig.autofmt_xdate() + fig.tight_layout() + fig.savefig(out, format="svg", transparent=False) + plt.close(fig) + + +def main(): + stars = fetch_stargazers() + if not stars: + print("::error::No stargazers fetched; not updating the chart.") + sys.exit(1) + render(stars, OUT) + print(f"Rendered {len(stars)} stars to {OUT}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/binaries_dev.yml b/.github/workflows/binaries_dev.yml new file mode 100644 index 0000000..ed9d8cc --- /dev/null +++ b/.github/workflows/binaries_dev.yml @@ -0,0 +1,134 @@ +name: "go: build dev binaries" + +on: + push: + branches: [ master ] + paths: + - 'weed/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/binaries_dev.yml' + +concurrency: + group: binaries-dev-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + + cleanup: + permissions: + contents: write # for mknejp/delete-release-assets to delete release assets + runs-on: ubuntu-latest + + steps: + + - name: Delete old release assets + uses: mknejp/delete-release-assets@v1 + continue-on-error: true + with: + token: ${{ github.token }} + tag: dev + fail-if-no-assets: false + assets: | + weed-* + + build_dev_linux_windows: + permissions: + contents: write # for wangyoucao577/go-release-action to upload release assets + needs: cleanup + runs-on: ubuntu-latest + strategy: + matrix: + goos: [linux, windows] + goarch: [amd64] + + steps: + + - name: Check out code into the Go module directory + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2 + + - name: Set BUILD_TIME env + run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV} + + - name: Go Release Binaries Large Disk + uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + release_tag: dev + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + build_flags: -tags 5BytesOffset # optional, default is + ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed-large-disk + asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" + + - name: Go Release Binaries Normal Volume Size + uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + release_tag: dev + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed-normal-disk + asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" + + build_dev_darwin: + permissions: + contents: write # for wangyoucao577/go-release-action to upload release assets + needs: build_dev_linux_windows + runs-on: ubuntu-latest + strategy: + matrix: + goos: [darwin] + goarch: [amd64, arm64] + + steps: + + - name: Check out code into the Go module directory + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2 + + - name: Set BUILD_TIME env + run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV} + + - name: Go Release Binaries Large Disk + uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + release_tag: dev + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + build_flags: -tags 5BytesOffset # optional, default is + ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed-large-disk + asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" + + - name: Go Release Binaries Normal Volume Size + uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + release_tag: dev + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed-normal-disk + asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" diff --git a/.github/workflows/binaries_release0.yml b/.github/workflows/binaries_release0.yml new file mode 100644 index 0000000..045f094 --- /dev/null +++ b/.github/workflows/binaries_release0.yml @@ -0,0 +1,59 @@ +# This is a basic workflow to help you get started with Actions + +name: "go: build versioned binaries for windows" + +on: + push: + tags: + - '*' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +permissions: + contents: read + +jobs: + + build-release-binaries_windows: + permissions: + contents: write # for wangyoucao577/go-release-action to upload release assets + runs-on: ubuntu-latest + strategy: + matrix: + goos: [windows] + goarch: [amd64] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2 + - name: Go Release Binaries Normal Volume Size + uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + # build_flags: -tags 5BytesOffset # optional, default is + ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" + - name: Go Release Large Disk Binaries + uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + build_flags: -tags 5BytesOffset # optional, default is + ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk" diff --git a/.github/workflows/binaries_release1.yml b/.github/workflows/binaries_release1.yml new file mode 100644 index 0000000..9130ac3 --- /dev/null +++ b/.github/workflows/binaries_release1.yml @@ -0,0 +1,59 @@ +# This is a basic workflow to help you get started with Actions + +name: "go: build versioned binaries for linux" + +on: + push: + tags: + - '*' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +permissions: + contents: read + +jobs: + + build-release-binaries_linux: + permissions: + contents: write # for wangyoucao577/go-release-action to upload release assets + runs-on: ubuntu-latest + strategy: + matrix: + goos: [linux] + goarch: [amd64, arm, arm64] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2 + - name: Go Release Binaries Normal Volume Size + uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + # build_flags: -tags 5BytesOffset # optional, default is + ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" + - name: Go Release Large Disk Binaries + uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + build_flags: -tags 5BytesOffset # optional, default is + ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk" diff --git a/.github/workflows/binaries_release2.yml b/.github/workflows/binaries_release2.yml new file mode 100644 index 0000000..99f0755 --- /dev/null +++ b/.github/workflows/binaries_release2.yml @@ -0,0 +1,59 @@ +# This is a basic workflow to help you get started with Actions + +name: "go: build versioned binaries for darwin" + +on: + push: + tags: + - '*' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +permissions: + contents: read + +jobs: + + build-release-binaries_darwin: + permissions: + contents: write # for wangyoucao577/go-release-action to upload release assets + runs-on: ubuntu-latest + strategy: + matrix: + goos: [darwin] + goarch: [amd64, arm64] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2 + - name: Go Release Binaries Normal Volume Size + uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + # build_flags: -tags 5BytesOffset # optional, default is + ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" + - name: Go Release Large Disk Binaries + uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + build_flags: -tags 5BytesOffset # optional, default is + ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk" diff --git a/.github/workflows/binaries_release3.yml b/.github/workflows/binaries_release3.yml new file mode 100644 index 0000000..0b429f3 --- /dev/null +++ b/.github/workflows/binaries_release3.yml @@ -0,0 +1,59 @@ +# This is a basic workflow to help you get started with Actions + +name: "go: build versioned binaries for freebsd" + +on: + push: + tags: + - '*' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +permissions: + contents: read + +jobs: + + build-release-binaries_freebsd: + permissions: + contents: write # for wangyoucao577/go-release-action to upload release assets + runs-on: ubuntu-latest + strategy: + matrix: + goos: [freebsd] + goarch: [amd64, arm, arm64] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2 + - name: Go Release Binaries Normal Volume Size + uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + # build_flags: -tags 5BytesOffset # optional, default is + ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" + - name: Go Release Large Disk Binaries + uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + build_flags: -tags 5BytesOffset # optional, default is + ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk" diff --git a/.github/workflows/binaries_release4.yml b/.github/workflows/binaries_release4.yml new file mode 100644 index 0000000..11871e6 --- /dev/null +++ b/.github/workflows/binaries_release4.yml @@ -0,0 +1,60 @@ +# This is a basic workflow to help you get started with Actions + +name: "go: build versioned binaries for linux with all tags" + +on: + push: + tags: + - '*' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +permissions: + contents: read + +jobs: + + build-release-binaries_linux: + permissions: + contents: write # for wangyoucao577/go-release-action to upload release assets + runs-on: ubuntu-latest + strategy: + matrix: + goos: [linux] + goarch: [amd64] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2 + - name: Go Release Binaries Normal Volume Size + uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + build_flags: -tags elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + # build_flags: -tags 5BytesOffset # optional, default is + ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full" + - name: Go Release Large Disk Binaries + uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + build_flags: -tags 5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb + ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full_large_disk" diff --git a/.github/workflows/binaries_release5.yml b/.github/workflows/binaries_release5.yml new file mode 100644 index 0000000..1d254dc --- /dev/null +++ b/.github/workflows/binaries_release5.yml @@ -0,0 +1,59 @@ +# This is a basic workflow to help you get started with Actions + +name: "go: build versioned binaries for openbsd" + +on: + push: + tags: + - '*' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +permissions: + contents: read + +jobs: + + build-release-binaries_openbsd: + permissions: + contents: write # for wangyoucao577/go-release-action to upload release assets + runs-on: ubuntu-latest + strategy: + matrix: + goos: [openbsd] + goarch: [amd64, arm, arm64] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2 + - name: Go Release Binaries Normal Volume Size + uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + # build_flags: -tags 5BytesOffset # optional, default is + ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" + - name: Go Release Large Disk Binaries + uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + build_flags: -tags 5BytesOffset # optional, default is + ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..10d86e1 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,52 @@ +name: "Code Scanning - Action" + +on: + pull_request: + paths: + - '**/*.go' + - 'go.mod' + - 'go.sum' + - '.github/workflows/codeql.yml' + +concurrency: + group: ${{ github.head_ref }}/codeql + cancel-in-progress: true + +jobs: + CodeQL-Build: + # CodeQL runs on ubuntu-latest, windows-latest, and macos-latest + runs-on: ubuntu-latest + + permissions: + # required for all workflows + security-events: write + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v4.38.0 + # Override language selection by uncommenting this and choosing your languages + with: + languages: go + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below). + - name: Autobuild + uses: github/codeql-action/autobuild@v4.38.0 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following + # three lines and modify them (or add more) to build your code if your + # project uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4.38.0 diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml new file mode 100644 index 0000000..9c4d99a --- /dev/null +++ b/.github/workflows/codespell.yml @@ -0,0 +1,23 @@ +# Codespell configuration is within .codespellrc +--- +name: Codespell + +on: + push: + branches: [master] + pull_request: + branches: [master] + +permissions: + contents: read + +jobs: + codespell: + name: Check for spelling errors + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: Codespell + uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 # v2.2 diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml new file mode 100644 index 0000000..7a922a6 --- /dev/null +++ b/.github/workflows/container_dev.yml @@ -0,0 +1,202 @@ +name: "docker: build dev containers" + +on: + push: + branches: [ master ] + paths: + - 'weed/**' + - 'seaweed-volume/**' + - 'seaweed-worker/**' + - 'docker/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/container_dev.yml' + +permissions: + contents: read + +jobs: + + # ── Pre-build the Rust binaries natively ──────────────────────────── + build-rust-binaries: + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - target: x86_64-unknown-linux-musl + arch: amd64 + - target: aarch64-unknown-linux-musl + arch: arm64 + cross: true + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install musl tools (amd64) + if: ${{ !matrix.cross }} + run: sudo apt-get install -y musl-tools + + - name: Install cross-compilation tools (arm64) + if: matrix.cross + run: | + sudo apt-get install -y gcc-aarch64-linux-gnu + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV" + # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl + echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV" + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + seaweed-worker/target/${{ matrix.target }}/release + key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} + restore-keys: | + rust-docker-dev-${{ matrix.target }}- + + # lance's build scripts compile their own protos and look for a protoc. + # Point them at the one protoc-bin-vendored ships, which seaweed-worker's + # own build already uses, so no job depends on a system package and every + # build sees the same version. + - name: Use the vendored protoc + run: | + cd seaweed-worker + cargo fetch + # The version from the lock, not whatever else a restored cache holds. + version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock) + test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; } + protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1) + test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; } + echo "PROTOC=$protoc" >> "$GITHUB_ENV" + + - name: Build normal variant + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} --no-default-features + cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }} + + - name: Build the Rust maintenance worker + run: | + cd seaweed-worker + cargo build --release -p weed-lance-worker --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + + - name: Upload artifacts + uses: actions/upload-artifact@v7 + with: + name: rust-bins-${{ matrix.arch }} + path: | + weed-volume-normal-${{ matrix.arch }} + weed-worker-${{ matrix.arch }} + + build-dev-containers: + needs: [build-rust-binaries] + runs-on: [ubuntu-latest] + permissions: + contents: read + id-token: write + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Download pre-built Rust binaries + uses: actions/download-artifact@v8 + with: + pattern: rust-bins-* + merge-multiple: true + path: ./rust-bins + + - name: Place Rust binaries in Docker context + run: | + mkdir -p docker/weed-volume-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-volume-normal-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}" + echo "Placed pre-built Rust binary for ${arch}" + fi + done + mkdir -p docker/weed-worker-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-worker-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}" + echo "Placed pre-built Rust worker for ${arch}" + fi + done + ls -la docker/weed-volume-prebuilt/ + ls -la docker/weed-worker-prebuilt/ + + - name: Docker meta + id: docker_meta + uses: docker/metadata-action@v6 + with: + images: | + chrislusf/seaweedfs + ghcr.io/chrislusf/seaweedfs + tags: | + type=raw,value=dev + labels: | + org.opencontainers.image.title=seaweedfs + org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! + org.opencontainers.image.vendor=Chris Lu + + - name: Set up QEMU + uses: docker/setup-qemu-action@v4.3.0 + + - name: Create BuildKit config + run: | + cat > /tmp/buildkitd.toml <- + chrislusf/seaweedfs@${{ steps.build.outputs.digest }} + ghcr.io/chrislusf/seaweedfs@${{ steps.build.outputs.digest }} diff --git a/.github/workflows/container_foundationdb_version.yml b/.github/workflows/container_foundationdb_version.yml new file mode 100644 index 0000000..e3f9029 --- /dev/null +++ b/.github/workflows/container_foundationdb_version.yml @@ -0,0 +1,182 @@ +name: "docker: build foundationdb image by version" + +on: + pull_request: + branches: [ master, main ] + paths: + - 'weed/filer/foundationdb/**' + - 'test/foundationdb/**' + - 'docker/Dockerfile.foundationdb_large' + - 'docker/filer_foundationdb.toml' + - '.github/workflows/container_foundationdb_version.yml' + workflow_dispatch: + inputs: + fdb_version: + description: 'FoundationDB version to build (e.g. 7.4.5)' + required: true + default: '7.4.5' + seaweedfs_ref: + description: 'SeaweedFS git tag, branch, or commit to build' + required: true + default: 'master' + image_tag: + description: 'Optional Docker tag suffix (defaults to foundationdb__seaweedfs_)' + required: false + default: '' + +permissions: + contents: read + +jobs: + build-foundationdb-image: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Install FoundationDB client libraries + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y ca-certificates wget + FDB_VERSION="${{ inputs.fdb_version || '7.4.5' }}" + case "${FDB_VERSION}_amd64" in + "7.4.5_amd64") EXPECTED_SHA256="eea6b98cf386a0848655b2e196d18633662a7440a7ee061c10e32153c7e7e112" ;; + "7.3.43_amd64") EXPECTED_SHA256="c3fa0a59c7355b914a1455dac909238d5ea3b6c6bc7b530af8597e6487c1651a" ;; + *) + echo "Unsupported FoundationDB version ${FDB_VERSION} for CI client install" >&2 + exit 1 ;; + esac + PACKAGE="foundationdb-clients_${FDB_VERSION}-1_amd64.deb" + wget --timeout=30 --tries=3 -O "${PACKAGE}" "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/${PACKAGE}" + echo "${EXPECTED_SHA256} ${PACKAGE}" | sha256sum -c - + sudo dpkg -i "${PACKAGE}" + rm "${PACKAGE}" + sudo ldconfig + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + + - name: Run FoundationDB tagged tests + env: + CGO_ENABLED: 1 + run: | + go test ./weed/filer/foundationdb -tags foundationdb -count=1 + + - name: Prepare Docker tag + id: tag + env: + FDB_VERSION_INPUT: ${{ inputs.fdb_version }} + SEAWEEDFS_REF_INPUT: ${{ inputs.seaweedfs_ref }} + CUSTOM_TAG_INPUT: ${{ inputs.image_tag }} + EVENT_NAME: ${{ github.event_name }} + HEAD_REF: ${{ github.head_ref }} + REF_NAME: ${{ github.ref_name }} + run: | + set -euo pipefail + sanitize() { + local value="$1" + value="${value,,}" + value="${value// /-}" + value="${value//[^a-z0-9_.-]/-}" + value="${value#-}" + value="${value%-}" + printf '%s' "$value" + } + version="${FDB_VERSION_INPUT}" + seaweed="${SEAWEEDFS_REF_INPUT}" + tag="${CUSTOM_TAG_INPUT}" + # Use defaults for PR builds + if [ -z "$version" ]; then + version="7.4.5" + fi + if [ -z "$seaweed" ]; then + if [ "$EVENT_NAME" = "pull_request" ]; then + seaweed="${HEAD_REF}" + else + seaweed="${REF_NAME}" + fi + fi + sanitized_version="$(sanitize "$version")" + if [ -z "$sanitized_version" ]; then + echo "Unable to sanitize FoundationDB version '$version'." >&2 + exit 1 + fi + sanitized_seaweed="$(sanitize "$seaweed")" + if [ -z "$sanitized_seaweed" ]; then + echo "Unable to sanitize SeaweedFS ref '$seaweed'." >&2 + exit 1 + fi + if [ -z "$tag" ]; then + tag="foundationdb_${sanitized_version}_seaweedfs_${sanitized_seaweed}" + else + tag="$(sanitize "$tag")" + fi + if [ -z "$tag" ]; then + echo "Resulting Docker tag is empty." >&2 + exit 1 + fi + echo "docker_tag=$tag" >> "$GITHUB_OUTPUT" + echo "full_image=chrislusf/seaweedfs:$tag" >> "$GITHUB_OUTPUT" + echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v4.3.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v4.6.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Determine branch to build + id: branch + env: + INPUT_REF: ${{ inputs.seaweedfs_ref }} + HEAD_REF: ${{ github.head_ref }} + REF_NAME: ${{ github.ref_name }} + run: | + if [ -n "$INPUT_REF" ]; then + echo "branch=$INPUT_REF" >> "$GITHUB_OUTPUT" + elif [ "${{ github.event_name }}" = "pull_request" ]; then + echo "branch=$HEAD_REF" >> "$GITHUB_OUTPUT" + else + echo "branch=$REF_NAME" >> "$GITHUB_OUTPUT" + fi + + - name: Build and push image + id: build + uses: docker/build-push-action@v7 + with: + context: ./docker + push: ${{ github.event_name != 'pull_request' }} + file: ./docker/Dockerfile.foundationdb_large + build-args: | + FDB_VERSION=${{ inputs.fdb_version || '7.4.5' }} + BRANCH=${{ steps.branch.outputs.branch }} + # Note: ARM64 support requires FoundationDB ARM64 packages which are not available for all versions + # Currently only building for amd64. To enable ARM64, verify package availability and add checksums. + platforms: linux/amd64 + tags: ${{ steps.tag.outputs.full_image || 'seaweedfs:foundationdb-test' }} + labels: | + org.opencontainers.image.title=seaweedfs + org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! + org.opencontainers.image.vendor=Chris Lu + + - name: Sign + if: github.event_name != 'pull_request' + uses: ./.github/actions/sign-image + with: + images: chrislusf/seaweedfs@${{ steps.build.outputs.digest }} diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml new file mode 100644 index 0000000..b7e79db --- /dev/null +++ b/.github/workflows/container_latest.yml @@ -0,0 +1,566 @@ +name: "docker: build latest container" + +# Manual fallback only. On tag push, container_release_unified.yml already +# re-tags the released versioned image as `latest` / `latest_large_disk`, +# so a full rebuild here is unnecessary. Run this manually if you need to +# rebuild `latest` from an arbitrary ref. +on: + workflow_dispatch: + inputs: + source_ref: + description: 'Git ref to build (branch, tag, or commit SHA)' + required: true + default: 'master' + image_tag: + description: 'Docker tag to publish (without variant suffix)' + required: true + default: 'latest' + variant: + description: 'Variant to build manually' + required: true + type: choice + default: all + options: + - all + - standard + - large_disk + publish: + description: 'Publish images and manifests' + required: true + type: boolean + default: false + +permissions: + contents: read + security-events: write + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + variants: ${{ steps.set-variants.outputs.variants }} + publish: ${{ steps.set-publish.outputs.publish }} + steps: + - name: Select variants for this run + id: set-variants + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ github.event.inputs.variant }}" != "all" ]; then + variants="[\"${{ github.event.inputs.variant }}\"]" + else + variants='["standard","large_disk"]' + fi + echo "variants=$variants" >> "$GITHUB_OUTPUT" + - name: Select publish mode + id: set-publish + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "publish=${{ github.event.inputs.publish }}" >> "$GITHUB_OUTPUT" + else + echo "publish=true" >> "$GITHUB_OUTPUT" + fi + + # ── Pre-build the Rust binaries natively ──────────────────────────── + build-rust-binaries: + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - target: x86_64-unknown-linux-musl + arch: amd64 + - target: aarch64-unknown-linux-musl + arch: arm64 + cross: true + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }} + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install musl tools (amd64) + if: ${{ !matrix.cross }} + run: sudo apt-get install -y musl-tools + + - name: Install cross-compilation tools (arm64) + if: matrix.cross + run: | + sudo apt-get install -y gcc-aarch64-linux-gnu + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV" + # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl + echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV" + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + seaweed-worker/target/${{ matrix.target }}/release + key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} + restore-keys: | + rust-docker-${{ matrix.target }}- + + # lance's build scripts compile their own protos and look for a protoc. + # Point them at the one protoc-bin-vendored ships, which seaweed-worker's + # own build already uses, so no job depends on a system package and every + # build sees the same version. + - name: Use the vendored protoc + run: | + cd seaweed-worker + cargo fetch + # The version from the lock, not whatever else a restored cache holds. + version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock) + test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; } + protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1) + test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; } + echo "PROTOC=$protoc" >> "$GITHUB_ENV" + + - name: Build large-disk variant + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }} + + - name: Build normal variant + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} --no-default-features + cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }} + + - name: Build the Rust maintenance worker + run: | + cd seaweed-worker + cargo build --release -p weed-lance-worker --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + + - name: Upload artifacts + uses: actions/upload-artifact@v7 + with: + name: rust-bins-${{ matrix.arch }} + path: | + weed-volume-large-disk-${{ matrix.arch }} + weed-volume-normal-${{ matrix.arch }} + weed-worker-${{ matrix.arch }} + + build: + needs: [setup, build-rust-binaries] + runs-on: ubuntu-latest + strategy: + matrix: + platform: [amd64, arm64, arm, 386] + variant: ${{ fromJSON(needs.setup.outputs.variants) }} + + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }} + - name: Free Disk Space + run: | + echo "Available disk space before cleanup:" + df -h + # Remove pre-installed tools + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + # Clean package managers + sudo apt-get clean + sudo rm -rf /var/lib/apt/lists/* + # Clean Docker aggressively + sudo docker system prune -af --volumes + # Clean Go cache if it exists + [ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true + [ -d /go/pkg ] && rm -rf /go/pkg || true + echo "Available disk space after cleanup:" + df -h + + - name: Configure variant + id: config + run: | + if [ "${{ matrix.variant }}" == "large_disk" ]; then + echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT + echo "build_args=TAGS=5BytesOffset" >> $GITHUB_OUTPUT + echo "rust_variant=large-disk" >> $GITHUB_OUTPUT + else + echo "tag_suffix=" >> $GITHUB_OUTPUT + echo "build_args=" >> $GITHUB_OUTPUT + echo "rust_variant=normal" >> $GITHUB_OUTPUT + fi + + - name: Download pre-built Rust binaries + uses: actions/download-artifact@v8 + with: + pattern: rust-bins-* + merge-multiple: true + path: ./rust-bins + + - name: Place Rust binaries in Docker context + run: | + mkdir -p docker/weed-volume-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-volume-${{ steps.config.outputs.rust_variant }}-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}" + echo "Placed pre-built Rust binary for ${arch}" + fi + done + mkdir -p docker/weed-worker-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-worker-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}" + echo "Placed pre-built Rust worker for ${arch}" + fi + done + ls -la docker/weed-volume-prebuilt/ + ls -la docker/weed-worker-prebuilt/ + + - name: Docker meta + id: docker_meta + uses: docker/metadata-action@v6 + with: + images: | + chrislusf/seaweedfs + ghcr.io/chrislusf/seaweedfs + tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }} + labels: | + org.opencontainers.image.title=seaweedfs + org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! + org.opencontainers.image.vendor=Chris Lu + - name: Set up QEMU + if: matrix.platform != 'amd64' + uses: docker/setup-qemu-action@v4.3.0 + - name: Create BuildKit config + run: | + cat > /tmp/buildkitd.toml <> $GITHUB_OUTPUT + else + echo "tag_suffix=" >> $GITHUB_OUTPUT + fi + - name: Login to GHCR + if: needs.setup.outputs.publish == 'true' + uses: docker/login-action@v4.6.0 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + - name: Checkout for local scan build + if: needs.setup.outputs.publish != 'true' + uses: actions/checkout@v7 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }} + - name: Download pre-built Rust binaries for local scan + if: needs.setup.outputs.publish != 'true' + uses: actions/download-artifact@v8 + with: + pattern: rust-bins-* + merge-multiple: true + path: ./rust-bins + - name: Place Rust binaries in Docker context for local scan + if: needs.setup.outputs.publish != 'true' + run: | + rust_variant="normal" + if [ "${{ matrix.variant }}" == "large_disk" ]; then + rust_variant="large-disk" + fi + mkdir -p docker/weed-volume-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-volume-${rust_variant}-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}" + echo "Placed pre-built Rust binary for ${arch}" + fi + done + mkdir -p docker/weed-worker-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-worker-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}" + echo "Placed pre-built Rust worker for ${arch}" + fi + done + ls -la docker/weed-volume-prebuilt/ + ls -la docker/weed-worker-prebuilt/ + - name: Create BuildKit config for local scan build + if: needs.setup.outputs.publish != 'true' + run: | + cat > /tmp/buildkitd.toml <> $GITHUB_OUTPUT + else + echo "tag_suffix=" >> $GITHUB_OUTPUT + fi + + - name: Docker meta + id: docker_meta + uses: docker/metadata-action@v6 + with: + images: | + chrislusf/seaweedfs + ghcr.io/chrislusf/seaweedfs + tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }} + - name: Login to Docker Hub + uses: docker/login-action@v4.6.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Login to GHCR + uses: docker/login-action@v4.6.0 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + - name: Install crane + run: | + # Install crane for efficient multi-arch image copying + cd $(mktemp -d) + curl -sLO https://github.com/google/go-containerregistry/releases/download/v0.22.0/go-containerregistry_Linux_x86_64.tar.gz + echo "edb74d53fad9a596860f59d1c5d04a43dfb5f441dc71f57060dd0bf39483c833 go-containerregistry_Linux_x86_64.tar.gz" | sha256sum -c - + tar xzf go-containerregistry_Linux_x86_64.tar.gz crane + sudo mv crane /usr/local/bin/ + crane version + - name: Create and push manifest + id: manifest + env: + BASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }} + run: | + SUFFIX="${{ steps.config.outputs.tag_suffix }}" + + # Create manifest on GHCR first (no rate limits) + echo "Creating GHCR manifest (no rate limits)..." + docker buildx imagetools create -t ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \ + --metadata-file /tmp/manifest.json \ + ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \ + ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \ + ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \ + ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386 + # The copy and the signature below use the digest this run pushed, not whatever the tag points at by then. + DIGEST=$(jq -er '."containerimage.descriptor".digest' /tmp/manifest.json) + echo "digest=${DIGEST}" >> "$GITHUB_OUTPUT" + + # Copy the complete multi-arch image from GHCR to Docker Hub + # This only requires one pull from GHCR (no rate limit) and one push to Docker Hub + echo "Copying manifest from GHCR to Docker Hub..." + + # Function to retry with exponential backoff for Docker Hub operations + retry_with_backoff() { + local max_attempts=5 + local timeout=1 + local attempt=1 + local exit_code=0 + + while [ $attempt -le $max_attempts ]; do + if "$@"; then + return 0 + else + exit_code=$? + fi + + if [ $attempt -lt $max_attempts ]; then + echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2 + sleep $timeout + timeout=$((timeout * 2)) + fi + + attempt=$((attempt + 1)) + done + + echo "Command failed after $max_attempts attempts" >&2 + return $exit_code + } + + # Use crane or skopeo to copy, fallback to docker if not available + if command -v crane &> /dev/null; then + echo "Using crane to copy..." + retry_with_backoff crane copy ghcr.io/chrislusf/seaweedfs@${DIGEST} chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} + elif command -v skopeo &> /dev/null; then + echo "Using skopeo to copy..." + retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs@${DIGEST} docker://chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} + else + echo "Using docker buildx imagetools (pulling 4 images from Docker Hub)..." + # Fallback: create manifest directly on Docker Hub (pulls from Docker Hub - rate limited) + retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \ + ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \ + ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \ + ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \ + ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386 + fi + + - name: Sign + uses: ./.github/actions/sign-image + with: + images: >- + ghcr.io/chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }} + chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }} diff --git a/.github/workflows/container_release_foundationdb.yml b/.github/workflows/container_release_foundationdb.yml new file mode 100644 index 0000000..2dc67f9 --- /dev/null +++ b/.github/workflows/container_release_foundationdb.yml @@ -0,0 +1,89 @@ +name: "docker: build release containers for foundationdb" + +on: + push: + tags: + - '*' + workflow_dispatch: + inputs: + release_tag: + description: 'Release tag to publish (e.g. 3.93)' + required: true + default: '' + +permissions: + contents: read + +env: + RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }} + +jobs: + + build-large-release-container_foundationdb: + runs-on: [ubuntu-latest] + permissions: + contents: read + id-token: write + + steps: + - + name: Checkout + uses: actions/checkout@v7 + - + name: Docker meta + id: docker_meta + uses: docker/metadata-action@v6 + with: + images: | + chrislusf/seaweedfs + tags: | + type=raw,value=${{ env.RELEASE_TAG }}_large_disk_foundationdb + flavor: | + latest=false + labels: | + org.opencontainers.image.title=seaweedfs + org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! + org.opencontainers.image.vendor=Chris Lu + - + name: Set up QEMU + uses: docker/setup-qemu-action@v4.3.0 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + - + name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v4.6.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Determine branch to build + id: branch + run: | + if [ "${{ github.event_name }}" = "push" ] && [ -n "${{ github.ref_name }}" ]; then + echo "branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT" + else + echo "branch=master" >> "$GITHUB_OUTPUT" + fi + - + name: Build + id: build + uses: docker/build-push-action@v7 + with: + context: ./docker + push: ${{ github.event_name != 'pull_request' }} + file: ./docker/Dockerfile.foundationdb_large + build-args: | + BRANCH=${{ steps.branch.outputs.branch }} + # Note: ARM64 support requires FoundationDB ARM64 packages which are not available for all versions + platforms: linux/amd64 + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + - + name: Sign + if: github.event_name != 'pull_request' + uses: ./.github/actions/sign-image + with: + images: chrislusf/seaweedfs@${{ steps.build.outputs.digest }} + diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml new file mode 100644 index 0000000..b3b8911 --- /dev/null +++ b/.github/workflows/container_release_unified.yml @@ -0,0 +1,553 @@ +name: "docker: build all release containers (unified)" + +on: + push: + tags: + - '*' + workflow_dispatch: + inputs: + variant: + description: 'Variant to build manually' + required: true + type: choice + default: all + options: + - all + - normal + - large_disk + - full + - large_disk_full + - rocksdb + release_tag: + description: 'Release tag to publish (e.g. 3.93)' + required: true + default: '' + rocksdb_version: + description: 'RocksDB git tag to use when variant=rocksdb' + required: false + default: 'v10.10.1' + +permissions: + contents: read + security-events: write + +env: + RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }} + IMAGE: ghcr.io/chrislusf/seaweedfs + +# Limit concurrent builds to avoid rate limits +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +jobs: + + # ── Pre-build the Rust binaries natively ──────────────────────────── + # The volume server and the Rust maintenance worker, cross-compiled for + # amd64 and arm64 without QEMU, turning a 5-hour emulated cargo build into + # ~15 minutes of native compilation. + build-rust-binaries: + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - target: x86_64-unknown-linux-musl + arch: amd64 + - target: aarch64-unknown-linux-musl + arch: arm64 + cross: true + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install musl tools (amd64) + if: ${{ !matrix.cross }} + run: sudo apt-get install -y musl-tools + + - name: Install cross-compilation tools (arm64) + if: matrix.cross + run: | + sudo apt-get install -y gcc-aarch64-linux-gnu + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV" + # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl + echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV" + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + seaweed-worker/target/${{ matrix.target }}/release + key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} + restore-keys: | + rust-docker-${{ matrix.target }}- + + # lance's build scripts compile their own protos and look for a protoc. + # Point them at the one protoc-bin-vendored ships, which seaweed-worker's + # own build already uses, so no job depends on a system package and every + # build sees the same version. + - name: Use the vendored protoc + run: | + cd seaweed-worker + cargo fetch + # The version from the lock, not whatever else a restored cache holds. + version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock) + test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; } + protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1) + test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; } + echo "PROTOC=$protoc" >> "$GITHUB_ENV" + + - name: Build large-disk variant + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }} + + - name: Build normal variant + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} --no-default-features + cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }} + + - name: Build the Rust maintenance worker + run: | + cd seaweed-worker + cargo build --release -p weed-lance-worker --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + + - name: Upload artifacts + uses: actions/upload-artifact@v7 + with: + name: rust-bins-${{ matrix.arch }} + path: | + weed-volume-large-disk-${{ matrix.arch }} + weed-volume-normal-${{ matrix.arch }} + weed-worker-${{ matrix.arch }} + + # One job per (variant, platform) on a native runner, pushed by digest; + # the merge job stitches the digests into one multi-arch tag. + build: + needs: [build-rust-binaries] + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + # Normal volume - multi-arch + - { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false } + - { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false } + - { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest, qemu: true } + - { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/386, arch: i386, runner: ubuntu-latest, qemu: false } + + # Large disk - multi-arch + - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false } + - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false } + - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest, qemu: true } + - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/386, arch: i386, runner: ubuntu-latest, qemu: false } + + # Full tags - multi-arch + - { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false } + - { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false } + + # Large disk + full tags - multi-arch + - { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false } + - { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false } + + # RocksDB large disk - amd64 only + - { variant: rocksdb, tag_suffix: _large_disk_rocksdb, dockerfile: ./docker/Dockerfile.rocksdb_large, build_args: "", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false } + steps: + - name: Skip unselected variant + if: github.event_name == 'workflow_dispatch' && github.event.inputs.variant != 'all' && github.event.inputs.variant != matrix.variant + run: echo "Skipping ${{ matrix.variant }} (${{ matrix.platform }})" && exit 0 + + - name: Checkout + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + uses: actions/checkout@v7 + + - name: Download pre-built Rust binaries + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + uses: actions/download-artifact@v8 + with: + pattern: rust-bins-* + merge-multiple: true + path: ./rust-bins + + - name: Place Rust binaries in Docker context + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + run: | + mkdir -p docker/weed-volume-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-volume-${{ matrix.rust_variant }}-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}" + echo "Placed pre-built Rust binary for ${arch}" + fi + done + mkdir -p docker/weed-worker-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-worker-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}" + echo "Placed pre-built Rust worker for ${arch}" + fi + done + ls -la docker/weed-volume-prebuilt/ + ls -la docker/weed-worker-prebuilt/ + + - name: Free Disk Space + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + sudo apt-get clean + sudo rm -rf /var/lib/apt/lists/* + sudo docker system prune -af --volumes + [ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true + [ -d /go/pkg ] && rm -rf /go/pkg || true + df -h + + - name: Docker meta + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + id: docker_meta + uses: docker/metadata-action@v6 + with: + images: ${{ env.IMAGE }} + tags: type=raw,value=${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} + flavor: latest=false + labels: | + org.opencontainers.image.title=seaweedfs + org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! + org.opencontainers.image.vendor=Chris Lu + + - name: Set up QEMU + if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && matrix.qemu + uses: docker/setup-qemu-action@v4.3.0 + + - name: Create BuildKit config + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + run: | + cat > /tmp/buildkitd.toml <> "$GITHUB_OUTPUT" + + - name: Login to Docker Hub + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + uses: docker/login-action@v4.6.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Install crane + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + run: | + cd $(mktemp -d) + curl -sLO https://github.com/google/go-containerregistry/releases/download/v0.22.0/go-containerregistry_Linux_x86_64.tar.gz + echo "edb74d53fad9a596860f59d1c5d04a43dfb5f441dc71f57060dd0bf39483c833 go-containerregistry_Linux_x86_64.tar.gz" | sha256sum -c - + tar xzf go-containerregistry_Linux_x86_64.tar.gz crane + sudo mv crane /usr/local/bin/ + crane version + + - name: Copy ${{ matrix.variant }} to Docker Hub + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + run: | + retry_with_backoff() { + local max_attempts=5 + local timeout=1 + local attempt=1 + local exit_code=0 + while [ $attempt -le $max_attempts ]; do + if "$@"; then + return 0 + else + exit_code=$? + fi + if [ $attempt -lt $max_attempts ]; then + echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2 + sleep $timeout + timeout=$((timeout * 2)) + fi + attempt=$((attempt + 1)) + done + echo "Command failed after $max_attempts attempts" >&2 + return $exit_code + } + + echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..." + retry_with_backoff crane copy \ + ${{ env.IMAGE }}@${{ steps.manifest.outputs.digest }} \ + chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} + echo "Copied ${{ matrix.variant }} to Docker Hub" + + - name: Sign ${{ matrix.variant }} + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + uses: ./.github/actions/sign-image + with: + images: >- + ${{ env.IMAGE }}@${{ steps.manifest.outputs.digest }} + chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }} + + # Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub + # Security for visibility, but never blocks the release. Releases (including + # `latest`) ship regardless — vulnerabilities are tracked, not gated, since + # we sometimes need to publish through known findings (e.g. unfixed upstream + # CVE, base-image lag). + trivy-scan: + runs-on: ubuntu-latest + needs: [merge] + if: github.event_name == 'push' + continue-on-error: true + strategy: + fail-fast: false + matrix: + include: + - source_suffix: "" + variant: normal + - source_suffix: _large_disk + variant: large_disk + steps: + - name: Login to GHCR + uses: docker/login-action@v4.6.0 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + + - name: Trivy report (${{ matrix.variant }}) + # Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23) + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + scan-type: image + # Scan the multi-arch tag on GHCR (already pushed by the build job). + # Trivy scans the runner's native platform; OS packages are identical + # across architectures since they all share the same alpine base. + image-ref: ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.source_suffix }} + scanners: vuln + vuln-type: os,library + severity: HIGH,CRITICAL + ignore-unfixed: true + limit-severities-for-sarif: true + format: sarif + output: trivy-results.sarif + exit-code: '0' + + - name: Upload Trivy scan results to GitHub Security + if: always() + uses: github/codeql-action/upload-sarif@v4.38.0 + with: + sarif_file: trivy-results.sarif + category: trivy-${{ matrix.variant }} + + # Point `latest` (and `latest_large_disk`) at the just-released versioned + # image. crane tag adds an extra tag to an existing manifest — no rebuild, + # no QEMU, no separate workflow. Replaces the old container_latest.yml + # rebuild that often failed or lagged behind the release. Independent of + # trivy-scan: vuln findings are reported but do not block `latest`. The + # cosign signature is attached to the digest, so `latest` carries it too. + tag-latest: + runs-on: ubuntu-latest + needs: [merge] + if: github.event_name == 'push' + strategy: + matrix: + include: + - source_suffix: "" + latest_tag: latest + - source_suffix: _large_disk + latest_tag: latest_large_disk + steps: + - name: Login to Docker Hub + uses: docker/login-action@v4.6.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Login to GHCR + uses: docker/login-action@v4.6.0 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + + - name: Install crane + run: | + cd $(mktemp -d) + curl -sLO https://github.com/google/go-containerregistry/releases/download/v0.22.0/go-containerregistry_Linux_x86_64.tar.gz + echo "edb74d53fad9a596860f59d1c5d04a43dfb5f441dc71f57060dd0bf39483c833 go-containerregistry_Linux_x86_64.tar.gz" | sha256sum -c - + tar xzf go-containerregistry_Linux_x86_64.tar.gz crane + sudo mv crane /usr/local/bin/ + crane version + + - name: Re-tag ${{ env.RELEASE_TAG }}${{ matrix.source_suffix }} as ${{ matrix.latest_tag }} + run: | + retry_with_backoff() { + local max_attempts=5 + local timeout=1 + local attempt=1 + local exit_code=0 + while [ $attempt -le $max_attempts ]; do + if "$@"; then + return 0 + else + exit_code=$? + fi + if [ $attempt -lt $max_attempts ]; then + echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2 + sleep $timeout + timeout=$((timeout * 2)) + fi + attempt=$((attempt + 1)) + done + echo "Command failed after $max_attempts attempts" >&2 + return $exit_code + } + + SRC_TAG="${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}" + DST_TAG="${{ matrix.latest_tag }}" + + echo "Tagging ${{ env.IMAGE }}:${SRC_TAG} as ${DST_TAG}" + retry_with_backoff crane tag "${{ env.IMAGE }}:${SRC_TAG}" "${DST_TAG}" + + echo "Tagging chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}" + retry_with_backoff crane tag "chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}" + + helm-release: + runs-on: ubuntu-latest + needs: [build] + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' + permissions: + contents: write + pages: write + steps: + - uses: actions/checkout@v7 + - name: Publish Helm charts + uses: stefanprodan/helm-gh-pages@v1.7.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + charts_dir: k8s/charts + target_dir: helm + branch: gh-pages + helm_version: "3.18.4" diff --git a/.github/workflows/container_rocksdb_version.yml b/.github/workflows/container_rocksdb_version.yml new file mode 100644 index 0000000..4951e37 --- /dev/null +++ b/.github/workflows/container_rocksdb_version.yml @@ -0,0 +1,119 @@ +name: "docker: build rocksdb image by version" + +on: + workflow_dispatch: + inputs: + rocksdb_version: + description: 'RocksDB git tag or branch to build (e.g. v10.10.1)' + required: true + default: 'v10.10.1' + seaweedfs_ref: + description: 'SeaweedFS git tag, branch, or commit to build' + required: true + default: 'master' + image_tag: + description: 'Optional Docker tag suffix (defaults to rocksdb__seaweedfs_)' + required: false + default: '' + +permissions: + contents: read + +jobs: + build-rocksdb-image: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2 + + - name: Prepare Docker tag + id: tag + env: + ROCKSDB_VERSION_INPUT: ${{ inputs.rocksdb_version }} + SEAWEEDFS_REF_INPUT: ${{ inputs.seaweedfs_ref }} + CUSTOM_TAG_INPUT: ${{ inputs.image_tag }} + run: | + set -euo pipefail + sanitize() { + local value="$1" + value="${value,,}" + value="${value// /-}" + value="${value//[^a-z0-9_.-]/-}" + value="${value#-}" + value="${value%-}" + printf '%s' "$value" + } + version="${ROCKSDB_VERSION_INPUT}" + seaweed="${SEAWEEDFS_REF_INPUT}" + tag="${CUSTOM_TAG_INPUT}" + if [ -z "$version" ]; then + echo "RocksDB version input is required." >&2 + exit 1 + fi + if [ -z "$seaweed" ]; then + echo "SeaweedFS ref input is required." >&2 + exit 1 + fi + sanitized_version="$(sanitize "$version")" + if [ -z "$sanitized_version" ]; then + echo "Unable to sanitize RocksDB version '$version'." >&2 + exit 1 + fi + sanitized_seaweed="$(sanitize "$seaweed")" + if [ -z "$sanitized_seaweed" ]; then + echo "Unable to sanitize SeaweedFS ref '$seaweed'." >&2 + exit 1 + fi + if [ -z "$tag" ]; then + tag="rocksdb_${sanitized_version}_seaweedfs_${sanitized_seaweed}" + fi + tag="${tag,,}" + tag="${tag// /-}" + tag="${tag//[^a-z0-9_.-]/-}" + tag="${tag#-}" + tag="${tag%-}" + if [ -z "$tag" ]; then + echo "Resulting Docker tag is empty." >&2 + exit 1 + fi + echo "docker_tag=$tag" >> "$GITHUB_OUTPUT" + echo "full_image=chrislusf/seaweedfs:$tag" >> "$GITHUB_OUTPUT" + echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT" + + - name: Set up QEMU + uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1 + + - name: Login to Docker Hub + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push image + id: build + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v2 + with: + context: ./docker + push: true + file: ./docker/Dockerfile.rocksdb_large + build-args: | + ROCKSDB_VERSION=${{ inputs.rocksdb_version }} + BRANCH=${{ inputs.seaweedfs_ref }} + platforms: linux/amd64 + tags: ${{ steps.tag.outputs.full_image }} + labels: | + org.opencontainers.image.title=seaweedfs + org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! + org.opencontainers.image.vendor=Chris Lu + + - name: Sign + uses: ./.github/actions/sign-image + with: + images: chrislusf/seaweedfs@${{ steps.build.outputs.digest }} diff --git a/.github/workflows/deploy_telemetry.yml b/.github/workflows/deploy_telemetry.yml new file mode 100644 index 0000000..2c67e51 --- /dev/null +++ b/.github/workflows/deploy_telemetry.yml @@ -0,0 +1,174 @@ +# This workflow will build and deploy the SeaweedFS telemetry server +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go + +name: Deploy Telemetry Server + +on: + workflow_dispatch: + inputs: + setup: + description: 'Run first-time server setup' + required: true + type: boolean + default: false + deploy: + description: 'Deploy telemetry server to remote server' + required: true + type: boolean + default: false + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'telemetry/server/go.mod' + + - name: Build Telemetry Server + if: github.event_name == 'workflow_dispatch' && inputs.deploy + run: | + # telemetry/server is its own Go module; build from within it + cd telemetry/server + go mod tidy + echo "Building telemetry server..." + GOOS=linux GOARCH=amd64 go build -o ../../telemetry-server . + cd ../.. + ls -la telemetry-server + echo "Build completed successfully" + + - name: First-time Server Setup + if: github.event_name == 'workflow_dispatch' && inputs.setup + env: + SSH_PRIVATE_KEY: ${{ secrets.TELEMETRY_SSH_PRIVATE_KEY }} + REMOTE_HOST: ${{ secrets.TELEMETRY_HOST }} + REMOTE_USER: ${{ secrets.TELEMETRY_USER }} + run: | + mkdir -p ~/.ssh + echo "$SSH_PRIVATE_KEY" > ~/.ssh/deploy_key + chmod 600 ~/.ssh/deploy_key + echo "Host *" > ~/.ssh/config + echo " StrictHostKeyChecking no" >> ~/.ssh/config + + # Create all required directories with proper permissions + ssh -i ~/.ssh/deploy_key $REMOTE_USER@$REMOTE_HOST " + mkdir -p ~/seaweedfs-telemetry/bin ~/seaweedfs-telemetry/logs ~/seaweedfs-telemetry/data ~/seaweedfs-telemetry/tmp && \ + chmod 755 ~/seaweedfs-telemetry/logs && \ + chmod 755 ~/seaweedfs-telemetry/data && \ + touch ~/seaweedfs-telemetry/logs/telemetry.log ~/seaweedfs-telemetry/logs/telemetry.error.log && \ + chmod 644 ~/seaweedfs-telemetry/logs/*.log" + + # Create systemd service file + echo " + [Unit] + Description=SeaweedFS Telemetry Server + After=network.target + + [Service] + Type=simple + User=$REMOTE_USER + WorkingDirectory=/home/$REMOTE_USER/seaweedfs-telemetry + ExecStart=/home/$REMOTE_USER/seaweedfs-telemetry/bin/telemetry-server -port=8353 + Restart=always + RestartSec=5 + StandardOutput=append:/home/$REMOTE_USER/seaweedfs-telemetry/logs/telemetry.log + StandardError=append:/home/$REMOTE_USER/seaweedfs-telemetry/logs/telemetry.error.log + + [Install] + WantedBy=multi-user.target" > telemetry.service + + # Setup logrotate configuration + echo "# SeaweedFS Telemetry service log rotation + /home/$REMOTE_USER/seaweedfs-telemetry/logs/*.log { + daily + rotate 30 + compress + delaycompress + missingok + notifempty + create 644 $REMOTE_USER $REMOTE_USER + postrotate + systemctl restart telemetry.service + endscript + }" > telemetry_logrotate + + # Copy configuration files + scp -i ~/.ssh/deploy_key telemetry/grafana-dashboard.json $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/ + scp -i ~/.ssh/deploy_key telemetry/prometheus.yml $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/ + + # Copy and install service and logrotate files + scp -i ~/.ssh/deploy_key telemetry.service telemetry_logrotate $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/ + ssh -i ~/.ssh/deploy_key $REMOTE_USER@$REMOTE_HOST " + sudo mv ~/seaweedfs-telemetry/telemetry.service /etc/systemd/system/ && \ + sudo mv ~/seaweedfs-telemetry/telemetry_logrotate /etc/logrotate.d/seaweedfs-telemetry && \ + sudo systemctl daemon-reload && \ + sudo systemctl enable telemetry.service" + + echo "✅ First-time setup completed successfully!" + echo "📋 Next step: Run the deployment to install the telemetry server binary" + echo " 1. Go to GitHub Actions → Deploy Telemetry Server" + echo " 2. Click 'Run workflow'" + echo " 3. Check 'Deploy telemetry server to remote server'" + echo " 4. Click 'Run workflow'" + + rm -f ~/.ssh/deploy_key + + - name: Deploy Telemetry Server to Remote Server + if: github.event_name == 'workflow_dispatch' && inputs.deploy + env: + SSH_PRIVATE_KEY: ${{ secrets.TELEMETRY_SSH_PRIVATE_KEY }} + REMOTE_HOST: ${{ secrets.TELEMETRY_HOST }} + REMOTE_USER: ${{ secrets.TELEMETRY_USER }} + run: | + mkdir -p ~/.ssh + echo "$SSH_PRIVATE_KEY" > ~/.ssh/deploy_key + chmod 600 ~/.ssh/deploy_key + echo "Host *" > ~/.ssh/config + echo " StrictHostKeyChecking no" >> ~/.ssh/config + + # Create temp directory and copy binary + ssh -i ~/.ssh/deploy_key $REMOTE_USER@$REMOTE_HOST "mkdir -p ~/seaweedfs-telemetry/tmp" + scp -i ~/.ssh/deploy_key telemetry-server $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/tmp/ + + # Copy updated configuration files + scp -i ~/.ssh/deploy_key telemetry/grafana-dashboard.json $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/ + scp -i ~/.ssh/deploy_key telemetry/prometheus.yml $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/ + + # Check if service exists and deploy accordingly + ssh -i ~/.ssh/deploy_key $REMOTE_USER@$REMOTE_HOST " + if systemctl list-unit-files telemetry.service >/dev/null 2>&1; then + echo 'Service exists, performing update...' + sudo systemctl stop telemetry.service + mkdir -p ~/seaweedfs-telemetry/bin + mv ~/seaweedfs-telemetry/tmp/telemetry-server ~/seaweedfs-telemetry/bin/ + chmod +x ~/seaweedfs-telemetry/bin/telemetry-server + sudo systemctl start telemetry.service + sudo systemctl status telemetry.service + else + echo 'ERROR: telemetry.service not found!' + echo 'Please run the first-time setup before deploying.' + echo 'Go to GitHub Actions → Deploy Telemetry Server → Run workflow → Check \"Run first-time server setup\"' + exit 1 + fi" + + # Verify deployment + ssh -i ~/.ssh/deploy_key $REMOTE_USER@$REMOTE_HOST " + echo 'Waiting for service to start...' + sleep 5 + curl -f http://localhost:8353/health || echo 'Health check failed'" + + rm -f ~/.ssh/deploy_key + + - name: Notify Deployment Status + if: always() + run: | + if [ "${{ job.status }}" == "success" ]; then + echo "✅ Telemetry server deployment successful" + echo "Dashboard: http://${{ secrets.TELEMETRY_HOST }}:8353" + echo "Metrics: http://${{ secrets.TELEMETRY_HOST }}:8353/metrics" + else + echo "❌ Telemetry server deployment failed" + fi \ No newline at end of file diff --git a/.github/workflows/depsreview.yml b/.github/workflows/depsreview.yml new file mode 100644 index 0000000..e4338c2 --- /dev/null +++ b/.github/workflows/depsreview.yml @@ -0,0 +1,14 @@ +name: 'Dependency Review' +on: [pull_request] + +permissions: + contents: read + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: 'Checkout Repository' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - name: 'Dependency Review' + uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..0e58880 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,166 @@ +name: "End to End" + +on: + push: + branches: [ master ] + paths: + - 'weed/**' + - 'docker/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/e2e.yml' + pull_request: + branches: [ master ] + paths: + - 'weed/**' + - 'docker/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/e2e.yml' + +concurrency: + group: ${{ github.head_ref }}/e2e + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + working-directory: docker + +jobs: + e2e: + name: FUSE Mount + runs-on: ubuntu-22.04 + timeout-minutes: 30 + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Cache Docker layers + uses: actions/cache@v6 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-e2e-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-e2e- + + - name: Install dependencies + run: | + sudo rm -f /etc/apt/sources.list.d/azure-cli.list /etc/apt/sources.list.d/microsoft-prod.list + # Same helper the e2e image installs through: the runner's own list is + # azure-only too, and an outage there fails this step outright. + sudo ./apt-install fuse + + # Verify FUSE installation + echo "FUSE version: $(fusermount --version 2>&1 || echo 'fusermount not found')" + echo "FUSE device: $(ls -la /dev/fuse 2>&1 || echo '/dev/fuse not found')" + + - name: Start SeaweedFS + timeout-minutes: 15 + run: | + # Enable Docker buildkit for better caching + export DOCKER_BUILDKIT=1 + export COMPOSE_DOCKER_CLI_BUILD=1 + + # Build with retry logic + for i in {1..3}; do + echo "Build attempt $i/3" + if make build_e2e; then + echo "Build successful on attempt $i" + break + elif [ $i -eq 3 ]; then + echo "Build failed after 3 attempts" + exit 1 + else + echo "Build attempt $i failed, retrying in 30 seconds..." + sleep 30 + fi + done + + # Start services with wait + docker compose -f ./compose/e2e-mount.yml up --wait + + - name: Rotate buildx cache + if: always() + run: | + # Without this, --cache-to writes to .buildx-cache-new but actions/cache only + # uploads .buildx-cache, so layers (notably the slow apt RUN) never persist. + rm -rf /tmp/.buildx-cache + if [ -d /tmp/.buildx-cache-new ]; then mv /tmp/.buildx-cache-new /tmp/.buildx-cache; fi + + - name: Run FIO 4k + timeout-minutes: 15 + run: | + echo "Starting FIO at: $(date)" + # Concurrent r/w + echo 'Run randrw with size=16M bs=4k' + docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --group_reporting --runtime=30 --time_based=1 + + echo "Verify FIO at: $(date)" + # Verified write + echo 'Run randwrite with size=16M bs=4k' + docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1 + + - name: Run FIO 128k + timeout-minutes: 15 + run: | + echo "Starting FIO at: $(date)" + # Concurrent r/w + echo 'Run randrw with size=16M bs=128k' + docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 + + echo "Verify FIO at: $(date)" + # Verified write + echo 'Run randwrite with size=16M bs=128k' + docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1 + + - name: Run FIO 1MB + timeout-minutes: 15 + run: | + echo "Starting FIO at: $(date)" + # Concurrent r/w + echo 'Run randrw with size=16M bs=1m' + docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 + + echo "Verify FIO at: $(date)" + # Verified write + echo 'Run randwrite with size=16M bs=1m' + docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1 + + - name: Save logs + if: always() + run: | + docker compose -f ./compose/e2e-mount.yml logs > output.log + echo 'Showing last 500 log lines of mount service:' + docker compose -f ./compose/e2e-mount.yml logs --tail 500 mount + + - name: Check for data races + if: always() + continue-on-error: true # TODO: remove this comment to enable build failure on data races (after all are fixed) + run: grep -A50 'DATA RACE' output.log && exit 1 || exit 0 + + - name: Archive logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: output-logs + path: docker/output.log + + - name: Cleanup + if: always() + run: docker compose -f ./compose/e2e-mount.yml down --volumes --remove-orphans --rmi all diff --git a/.github/workflows/ec-integration-tests.yml b/.github/workflows/ec-integration-tests.yml new file mode 100644 index 0000000..e078e01 --- /dev/null +++ b/.github/workflows/ec-integration-tests.yml @@ -0,0 +1,76 @@ +name: "EC Integration Tests" + +on: + push: + branches: [ master ] + paths: + - 'weed/**' + - 'test/erasure_coding/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/ec-integration-tests.yml' + pull_request: + branches: [ master ] + paths: + - 'weed/**' + - 'test/erasure_coding/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/ec-integration-tests.yml' + +permissions: + contents: read + +jobs: + ec-integration-tests: + name: EC Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version: ^1.26 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + + - name: Build weed binary + run: | + cd weed && go build -o weed . + + - name: Run EC Integration Tests + working-directory: test/erasure_coding + run: | + # The suite now includes the interruption matrix and runs close to Go's + # default 10m binary timeout on slower runners; bound it by the job's + # 30m budget instead. + go test -v -timeout 25m + + - name: Collect server logs on failure + if: failure() + run: | + echo "Collecting server logs from temp directories..." + mkdir -p /tmp/ec-test-logs + # Find all temp directories created by the tests (they persist on failure with t.TempDir()) + find /tmp -maxdepth 1 -type d \( -name "TestEC*" -o -name "TestDisk*" -o -name "TestCross*" -o -name "TestEvacuation*" \) 2>/dev/null | while read dir; do + if [ -d "$dir" ]; then + echo "Found test directory: $dir" + # Copy the entire directory structure to preserve organization + cp -r "$dir" /tmp/ec-test-logs/ 2>/dev/null || true + fi + done + # List what we collected + echo "Collected logs:" + find /tmp/ec-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found" + + - name: Archive logs + if: failure() + uses: actions/upload-artifact@v7 + with: + name: ec-integration-test-logs + path: | + /tmp/ec-test-logs/ + test/erasure_coding/ + if-no-files-found: warn \ No newline at end of file diff --git a/.github/workflows/fuse-dlm-integration.yml b/.github/workflows/fuse-dlm-integration.yml new file mode 100644 index 0000000..47a27cc --- /dev/null +++ b/.github/workflows/fuse-dlm-integration.yml @@ -0,0 +1,69 @@ +name: "FUSE DLM Integration Tests" + +on: + pull_request: + paths: + - 'weed/command/mount*.go' + - 'weed/mount/**' + - 'weed/cluster/**' + - 'test/fuse_dlm/**' + - '.github/workflows/fuse-dlm-integration.yml' + - '.github/actions/fix-fusermount-setuid/**' + push: + branches: [master] + paths: + - 'weed/command/mount*.go' + - 'weed/mount/**' + - 'weed/cluster/**' + - 'test/fuse_dlm/**' + - '.github/actions/fix-fusermount-setuid/**' + +concurrency: + group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration + cancel-in-progress: true + +permissions: + contents: read + +jobs: + fuse-dlm-integration: + name: FUSE DLM Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Configure FUSE + run: | + # Nothing to install: fuse3 ships fusermount3 and is pre-installed, + # and go-fuse is pure Go, so the libfuse headers were never linked + # against. + echo 'user_allow_other' | sudo tee -a /etc/fuse.conf + sudo chmod 644 /etc/fuse.conf + + - name: Repair the fusermount3 setuid bit + uses: ./.github/actions/fix-fusermount-setuid + + - name: Build SeaweedFS + run: go build -o weed/weed -buildvcs=false ./weed + + - name: Run DLM integration tests + timeout-minutes: 25 + env: + WEED_BINARY: ${{ github.workspace }}/weed/weed + run: go test -v -count=1 -timeout=20m ./test/fuse_dlm/... + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: fuse-dlm-test-logs + path: /tmp/seaweedfs-fuse-dlm-logs/ + retention-days: 3 diff --git a/.github/workflows/fuse-failover.yml b/.github/workflows/fuse-failover.yml new file mode 100644 index 0000000..c11ca3e --- /dev/null +++ b/.github/workflows/fuse-failover.yml @@ -0,0 +1,76 @@ +name: "FUSE Volume Server Failover Tests" + +on: + pull_request: + paths: + - 'weed/command/mount*.go' + - 'weed/mount/**' + - 'weed/filer/**' + - 'weed/wdclient/**' + - 'weed/operation/upload_content.go' + - 'test/fuse_failover/**' + - '.github/workflows/fuse-failover.yml' + - '.github/actions/fix-fusermount-setuid/**' + push: + branches: [master] + paths: + - 'weed/command/mount*.go' + - 'weed/mount/**' + - 'weed/filer/**' + - 'weed/wdclient/**' + - 'weed/operation/upload_content.go' + - 'test/fuse_failover/**' + - '.github/workflows/fuse-failover.yml' + - '.github/actions/fix-fusermount-setuid/**' + +concurrency: + group: ${{ github.head_ref || github.ref }}/fuse-failover + cancel-in-progress: true + +permissions: + contents: read + +jobs: + fuse-failover: + name: FUSE Volume Server Failover + runs-on: ubuntu-22.04 + timeout-minutes: 40 + + steps: + - name: Check out code + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Configure FUSE + run: | + # Nothing to install: fuse3 ships fusermount3 and is pre-installed, + # and go-fuse is pure Go, so the libfuse headers were never linked + # against. + echo 'user_allow_other' | sudo tee -a /etc/fuse.conf + sudo chmod 644 /etc/fuse.conf + + - name: Repair the fusermount3 setuid bit + uses: ./.github/actions/fix-fusermount-setuid + + - name: Build SeaweedFS + run: go build -o weed/weed -buildvcs=false ./weed + + - name: Run failover integration tests + timeout-minutes: 35 + env: + WEED_BINARY: ${{ github.workspace }}/weed/weed + run: go test -v -count=1 -timeout=30m ./test/fuse_failover/... + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: fuse-failover-test-logs + path: /tmp/seaweedfs-fuse-failover-logs/ + retention-days: 3 diff --git a/.github/workflows/fuse-integration.yml b/.github/workflows/fuse-integration.yml new file mode 100644 index 0000000..3b23cb2 --- /dev/null +++ b/.github/workflows/fuse-integration.yml @@ -0,0 +1,84 @@ +name: "FUSE Integration Tests" + +on: + push: + branches: [ master, main ] + paths: + - 'weed/**' + - 'test/fuse_integration/**' + - '.github/workflows/fuse-integration.yml' + - '.github/actions/fix-fusermount-setuid/**' + pull_request: + branches: [ master, main ] + paths: + - 'weed/**' + - 'test/fuse_integration/**' + - '.github/workflows/fuse-integration.yml' + - '.github/actions/fix-fusermount-setuid/**' + +concurrency: + group: ${{ github.head_ref }}/fuse-integration + cancel-in-progress: true + +permissions: + contents: read + +jobs: + fuse-integration: + name: FUSE Integration Testing + runs-on: ubuntu-22.04 + timeout-minutes: 50 + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Configure FUSE + run: | + # Nothing to install: fuse3 ships fusermount3 and is pre-installed, and + # go-fuse is pure Go, so the libfuse headers were never linked against. + # Allow non-root FUSE mounts with allow_other + echo 'user_allow_other' | sudo tee -a /etc/fuse.conf + sudo chmod 644 /etc/fuse.conf + # Verify FUSE installation + fusermount3 --version || fusermount --version || true + ls -la /dev/fuse + + - name: Repair the fusermount3 setuid bit + uses: ./.github/actions/fix-fusermount-setuid + + - name: Build SeaweedFS + run: | + cd weed + go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -o weed . + chmod +x weed + ./weed version + # Make weed binary available in PATH for the test framework + sudo cp weed /usr/local/bin/weed + + - name: Install test dependencies + run: | + cd test/fuse_integration + go mod download + + - name: Run FUSE Integration Tests + run: | + set -o pipefail + cd test/fuse_integration + echo "Running full FUSE integration test suite..." + go test -v -count=1 -timeout=45m ./... 2>&1 | tee /tmp/fuse-test-output.log + + - name: Upload Test Logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: fuse-integration-test-results + path: | + /tmp/fuse-test-output.log + /tmp/seaweedfs-fuse-logs/ + retention-days: 7 diff --git a/.github/workflows/fuse-p2p-integration.yml b/.github/workflows/fuse-p2p-integration.yml new file mode 100644 index 0000000..28771e1 --- /dev/null +++ b/.github/workflows/fuse-p2p-integration.yml @@ -0,0 +1,75 @@ +name: "FUSE P2P Peer Chunk Sharing Integration Tests" + +on: + pull_request: + paths: + - 'weed/command/mount*.go' + - 'weed/mount/**' + - 'weed/filer/mount_peer_registry*.go' + - 'weed/server/filer_grpc_server_mount_peer.go' + - 'weed/pb/mount_peer.proto' + - 'weed/pb/filer.proto' + - 'test/fuse_p2p/**' + - '.github/workflows/fuse-p2p-integration.yml' + - '.github/actions/fix-fusermount-setuid/**' + push: + branches: [master] + paths: + - 'weed/command/mount*.go' + - 'weed/mount/**' + - 'weed/filer/mount_peer_registry*.go' + - 'weed/server/filer_grpc_server_mount_peer.go' + - 'weed/pb/mount_peer.proto' + - 'weed/pb/filer.proto' + - 'test/fuse_p2p/**' + - '.github/actions/fix-fusermount-setuid/**' + +concurrency: + group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration + cancel-in-progress: true + +permissions: + contents: read + +jobs: + fuse-p2p-integration: + name: FUSE P2P Peer Chunk Sharing + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Configure FUSE + run: | + # Nothing to install: fuse3 ships fusermount3 and is pre-installed, + # and go-fuse is pure Go, so the libfuse headers were never linked + # against. + echo 'user_allow_other' | sudo tee -a /etc/fuse.conf + sudo chmod 644 /etc/fuse.conf + + - name: Repair the fusermount3 setuid bit + uses: ./.github/actions/fix-fusermount-setuid + + - name: Build SeaweedFS + run: go build -o weed/weed -buildvcs=false ./weed + + - name: Run P2P integration tests + timeout-minutes: 15 + env: + WEED_BINARY: ${{ github.workspace }}/weed/weed + run: go test -v -count=1 -timeout=12m ./test/fuse_p2p/... + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: fuse-p2p-test-logs + path: /tmp/seaweedfs-fuse-p2p-logs/ + retention-days: 3 diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..0a6116e --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,150 @@ +name: "go: build binary" + +on: + push: + branches: [ master ] + paths: + - '**/*.go' + - 'go.mod' + - 'go.sum' + - '.github/workflows/go.yml' + pull_request: + branches: [ master ] + paths: + - '**/*.go' + - 'go.mod' + - 'go.sum' + - '.github/workflows/go.yml' + +concurrency: + group: ${{ github.head_ref }}/go + cancel-in-progress: true + +permissions: + contents: read + +jobs: + + vet: + name: Go Vet + runs-on: ubuntu-latest + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + - name: Get dependencies + run: | + cd weed; go get -v -t -d ./... + - name: Go Vet (excluding protobuf lock copying) + run: | + cd weed + # Run go vet and filter out known protobuf MessageState lock copying warnings + # These are expected in generated protobuf code with embedded sync.Mutex and are safe in practice + go vet -v ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-output.txt + # Fail only if there are actual vet errors (not counting the filtered lock warnings) + if grep -q "vet:" vet-output.txt; then exit 1; fi + + vet-32bit: + name: Go Vet 32-bit + runs-on: ubuntu-latest + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + - name: Go Vet linux/386 (type-checks code and tests for 32-bit int overflows) + run: | + GOOS=linux GOARCH=386 go vet ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-32bit-output.txt + if grep -q "vet:" vet-32bit-output.txt; then exit 1; fi + + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + - name: Build + run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v . + + build-cross: + name: Build cross-platform (${{ matrix.goos }}/${{ matrix.goarch }}) + runs-on: ubuntu-latest + strategy: + # One target's breakage should not hide the other three. + fail-fast: false + matrix: + include: + - { goos: windows, goarch: amd64 } + - { goos: windows, goarch: arm64 } + - { goos: freebsd, goarch: amd64 } + - { goos: darwin, goarch: arm64 } + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + # Nothing else on a PR compiles these, so per-OS syscall constants creep + # back into shared files unnoticed and only a release build catches it. + - name: Cross-compile + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + run: | + go build ./weed/... + # Tests too: they reach for per-OS syscall constants the build does + # not, and only compiling them catches an untagged one. + go vet ./weed/mount/... ./weed/command/... 2>&1 | + grep -v "MessageState contains sync.Mutex" | tee /tmp/vet.txt + if grep -q "vet:" /tmp/vet.txt; then exit 1; fi + + test: + name: Test + runs-on: ubuntu-latest + services: + redis: + image: redis:8 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + - name: Test + env: + RUN_REDIS_TESTS: "1" + run: cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./... + + test-32bit: + name: Test 32-bit + runs-on: ubuntu-latest + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + # 386 test binaries run natively on the amd64 runner. This catches what vet + # can't: unaligned 64-bit atomics and arithmetic that wraps at runtime. + # -short skips the e2e suites already covered on amd64. + - name: Test linux/386 + run: cd weed; GOOS=linux GOARCH=386 go test -short ./... diff --git a/.github/workflows/helm_ci.yml b/.github/workflows/helm_ci.yml new file mode 100644 index 0000000..9873e73 --- /dev/null +++ b/.github/workflows/helm_ci.yml @@ -0,0 +1,1753 @@ +name: "helm: lint and test charts" + +on: + push: + branches: [ master ] + paths: ['k8s/**', '.github/workflows/helm_ci.yml'] + pull_request: + branches: [ master ] + paths: ['k8s/**', '.github/workflows/helm_ci.yml'] + +permissions: + contents: read + +jobs: + lint-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v5 + with: + version: v3.18.4 + + - uses: actions/setup-python@v7 + with: + python-version: '3.10' + check-latest: true + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.8.0 + + - name: Run chart-testing (list-changed) + id: list-changed + run: | + changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }} --chart-dirs k8s/charts) + if [[ -n "$changed" ]]; then + echo "::set-output name=changed::true" + fi + + - name: Run chart-testing (lint) + run: ct lint --target-branch ${{ github.event.repository.default_branch }} --all --validate-maintainers=false --chart-dirs k8s/charts + + - name: Verify template rendering + run: | + set -e + CHART_DIR="k8s/charts/seaweedfs" + + echo "=== Testing default configuration ===" + helm template test $CHART_DIR > /tmp/default.yaml + echo "Default configuration renders successfully" + + echo "=== Testing with S3 enabled ===" + helm template test $CHART_DIR --set s3.enabled=true > /tmp/s3.yaml + grep -q "kind: Deployment" /tmp/s3.yaml && grep -q "seaweedfs-s3" /tmp/s3.yaml + echo "S3 deployment renders correctly" + + echo "=== Testing S3 credentials from an existing secret ===" + credential_args=( + --set s3.credentials.admin.existingSecret=minio-root + --set s3.credentials.admin.accessKeyKey=root-user + --set s3.credentials.admin.secretKeyKey=root-password + --set s3.credentials.read.existingSecret=minio-root + ) + for workload in \ + "s3.enabled=true,s3.enableAuth=true" \ + "filer.s3.enabled=true,filer.s3.enableAuth=true" \ + "allInOne.enabled=true,allInOne.s3.enabled=true,allInOne.s3.enableAuth=true" + do + helm template test $CHART_DIR --set "$workload" "${credential_args[@]}" > /tmp/s3-existing-credentials.yaml + # The identities file names the variables, and each is bound to the key it was pointed at. + grep -q 'accessKey":"${SEAWEEDFS_S3_ADMIN_ACCESS_KEY_ID}' /tmp/s3-existing-credentials.yaml + grep -q 'secretKey":"${SEAWEEDFS_S3_ADMIN_SECRET_ACCESS_KEY}' /tmp/s3-existing-credentials.yaml + grep -q 'accessKey":"${SEAWEEDFS_S3_READ_ACCESS_KEY_ID}' /tmp/s3-existing-credentials.yaml + for pair in \ + "SEAWEEDFS_S3_ADMIN_ACCESS_KEY_ID root-user" \ + "SEAWEEDFS_S3_ADMIN_SECRET_ACCESS_KEY root-password" \ + "SEAWEEDFS_S3_READ_ACCESS_KEY_ID read_access_key_id" \ + "SEAWEEDFS_S3_READ_SECRET_ACCESS_KEY read_secret_access_key" + do + set -- $pair + grep -A 4 -- "- name: $1\$" /tmp/s3-existing-credentials.yaml | grep -q "name: \"minio-root\"" + grep -A 4 -- "- name: $1\$" /tmp/s3-existing-credentials.yaml | grep -q "key: \"$2\"" + done + # The keys stay in the user's secret rather than being copied into the chart's. + ! grep -qE "^ (admin|read)_(access_key_id|secret_access_key):" /tmp/s3-existing-credentials.yaml + echo "S3 credentials reference the existing secret for $workload" + done + + echo "=== Testing ingress labels ===" + helm template test $CHART_DIR --set s3.enabled=true,s3.ingress.enabled=true \ + --set s3.ingress.labels.external-dns=s3 > /tmp/s3-ingress-labels.yaml + grep -A 12 "^kind: Ingress" /tmp/s3-ingress-labels.yaml | grep -q "^ external-dns: s3" + echo "S3 ingress renders custom labels" + + echo "=== Testing with all-in-one mode ===" + helm template test $CHART_DIR --set allInOne.enabled=true > /tmp/allinone.yaml + grep -q "seaweedfs-all-in-one" /tmp/allinone.yaml + echo "All-in-one deployment renders correctly" + + echo "=== Testing the all-in-one s3 secret is created by every flag that mounts it ===" + for auth in allInOne.s3.enableAuth s3.enableAuth filer.s3.enableAuth; do + helm template test $CHART_DIR \ + --set allInOne.enabled=true --set allInOne.s3.enabled=true --set "$auth=true" \ + > /tmp/allinone-s3-auth.yaml + grep -q "secretName: test-seaweedfs-s3-secret" /tmp/allinone-s3-auth.yaml + grep -q "name: test-seaweedfs-s3-secret" /tmp/allinone-s3-auth.yaml + echo "All-in-one s3 secret renders for $auth" + done + + echo "=== Testing with security enabled ===" + helm template test $CHART_DIR --set global.seaweedfs.enableSecurity=true > /tmp/security.yaml + grep -q "security-config" /tmp/security.yaml + echo "Security configuration renders correctly" + + echo "" + echo "=== Testing JWT expiration overrides ===" + helm template test $CHART_DIR \ + --set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.volumeWrite=11 \ + > /tmp/jwt-volume-write-expiration.yaml + grep -q "security-config" /tmp/jwt-volume-write-expiration.yaml + grep -q "expires_after_seconds = 11" /tmp/jwt-volume-write-expiration.yaml + + helm template test $CHART_DIR \ + --set global.seaweedfs.securityConfig.jwtSigning.volumeRead=true \ + --set global.seaweedfs.securityConfig.jwtSigning.filerWrite=true \ + --set global.seaweedfs.securityConfig.jwtSigning.filerRead=true \ + --set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.volumeWrite=11 \ + --set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.volumeRead=22 \ + --set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.filerWrite=33 \ + --set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.filerRead=44 \ + > /tmp/jwt-expiration.yaml + + assert_jwt_expiration() { + local section="$1" + local seconds="$2" + awk -v section="[$section]" -v seconds="$seconds" ' + /^[[:space:]]*\[.*\][[:space:]]*$/ { + in_section = index($0, section) > 0 + } + in_section && $0 ~ "^[[:space:]]*expires_after_seconds = " seconds "$" { + found = 1 + } + END { exit !found } + ' /tmp/jwt-expiration.yaml + } + + assert_jwt_expiration jwt.signing 11 + assert_jwt_expiration jwt.signing.read 22 + assert_jwt_expiration jwt.filer_signing 33 + assert_jwt_expiration jwt.filer_signing.read 44 + + helm template test $CHART_DIR \ + --set global.seaweedfs.enableSecurity=true \ + --set global.seaweedfs.securityConfig.jwtSigning.volumeRead=true \ + --set global.seaweedfs.securityConfig.jwtSigning.filerWrite=true \ + --set global.seaweedfs.securityConfig.jwtSigning.filerRead=true \ + > /tmp/jwt-default-expiration.yaml + if grep -q "expires_after_seconds =" /tmp/jwt-default-expiration.yaml; then + echo "FAIL: zero JWT expiration values should preserve runtime defaults" + exit 1 + fi + echo "JWT expiration overrides render correctly" + + echo "" + echo "=== Testing IAM gRPC opt-in path ===" + # Regression test: the filer registers the IAM gRPC service the + # Admin UI Users tab calls only when jwt.filer_signing.key is in + # security.toml. Operators must be able to enable that without + # the cert-manager mTLS bundle. + # Install PyYAML explicitly: this block runs before the later + # security+S3 block that does the same install, and we don't + # want to rely on the runner image shipping it. + pip install pyyaml -q + python3 - "$CHART_DIR" <<'PYEOF' + import subprocess, sys, yaml + chart = sys.argv[1] + + def render(values): + args = ["helm", "template", "test", chart] + for k, v in values.items(): + args += ["--set", f"{k}={v}"] + return subprocess.check_output(args, text=True) + + def docs(manifest): + return [d for d in yaml.safe_load_all(manifest) if d] + + def configmap(manifest, name): + for d in docs(manifest): + if d.get("kind") == "ConfigMap" and d["metadata"]["name"] == name: + return d + return None + + def workload_mounts(manifest, name): + for d in docs(manifest): + if d.get("kind") not in ("Deployment", "StatefulSet"): + continue + if d["metadata"]["name"] != name: + continue + pod = d["spec"]["template"]["spec"] + vols = {v["name"] for v in pod.get("volumes", [])} + mounts = set() + for c in pod.get("containers", []): + for vm in c.get("volumeMounts", []): + mounts.add(vm["name"]) + return vols, mounts + return None, None + + failed = [] + + # Case 1: defaults. The chart historically rendered nothing + # security-related; preserve that so this PR is non-breaking on + # existing installs. + out = render({}) + if configmap(out, "test-seaweedfs-security-config") is not None: + failed.append("defaults: security ConfigMap should not render") + else: + print("defaults: no security-config ConfigMap (unchanged)") + + # Case 2: filerWrite=true alone is the documented opt-in for + # the Admin UI Users tab. Configmap must render with + # [jwt.filer_signing] and NO [grpc.*] sections (cert paths + # only exist with mTLS). + out = render({ + "global.seaweedfs.securityConfig.jwtSigning.filerWrite": "true", + "admin.enabled": "true", + # admin.ip defaults to 0.0.0.0 (non-loopback), which weed admin 4.46 + # refuses to bind without authentication. + "admin.secret.adminPassword": "ci-admin-password", + }) + cm = configmap(out, "test-seaweedfs-security-config") + if cm is None: + failed.append("filerWrite=true: security ConfigMap missing") + else: + toml = cm["data"]["security.toml"] + if "[jwt.filer_signing]" not in toml: + failed.append("filerWrite=true: security.toml missing [jwt.filer_signing]") + if "[grpc" in toml: + failed.append("filerWrite=true: security.toml unexpectedly has [grpc.*] (would need cert mounts)") + if "[jwt.filer_signing]" in toml and "[grpc" not in toml: + print("filerWrite=true: security.toml has [jwt.filer_signing], no [grpc.*]") + + # Case 3: filer + admin pods must MOUNT the security ConfigMap + # under filerWrite=true so the JWT key reaches both processes. + # Cert volumes must NOT be present (no mTLS). + for wl in ("test-seaweedfs-filer", "test-seaweedfs-admin"): + vols, mounts = workload_mounts(out, wl) + if vols is None: + failed.append(f"filerWrite=true: workload {wl} not found") + continue + if "security-config" not in vols or "security-config" not in mounts: + failed.append(f"filerWrite=true: {wl} does not mount security-config (IAM gRPC would still fail)") + else: + print(f"filerWrite=true: {wl} mounts security-config") + cert_vols = {v for v in vols if v.endswith("-cert")} + if cert_vols: + failed.append(f"filerWrite=true: {wl} unexpectedly has cert volumes {sorted(cert_vols)}") + + # Case 4: enableSecurity=true must still render the full toml + # with both [jwt.signing] and [grpc.*]. Guards against the + # decoupling change accidentally regressing the mTLS path. + out = render({"global.seaweedfs.enableSecurity": "true"}) + cm = configmap(out, "test-seaweedfs-security-config") + if cm is None: + failed.append("enableSecurity=true: security ConfigMap missing") + else: + toml = cm["data"]["security.toml"] + missing = [s for s in ("[jwt.signing]", "[grpc.master]") if s not in toml] + if missing: + failed.append(f"enableSecurity=true: security.toml missing {missing}") + else: + print("enableSecurity=true: security.toml has [jwt.signing] + [grpc.*] preserved") + + # Case 5: helper must tolerate explicit nulls (gemini-code-assist + # PR review). securityConfig=null was the parens-pattern crash + # the helper review caught. + for null_path in ("global.seaweedfs.securityConfig", + "global.seaweedfs.securityConfig.jwtSigning"): + try: + out = render({null_path: "null"}) + except subprocess.CalledProcessError as e: + failed.append(f"{null_path}=null: render failed: {e.output[:200] if e.output else e}") + continue + if configmap(out, "test-seaweedfs-security-config") is not None: + failed.append(f"{null_path}=null: should not render configmap") + else: + print(f"{null_path}=null: render tolerates explicit null") + + if failed: + print("\nFAIL:", file=sys.stderr) + for f in failed: + print(f" - {f}", file=sys.stderr) + sys.exit(1) + PYEOF + echo "IAM gRPC decoupling tests passed" + + echo "" + echo "=== Testing bucket hook credentials ===" + # The bucket hook pipes s3.configure into `weed shell`, which needs + # the filer JWT signing key once jwtSigning.filerWrite is on: the + # filer rejects unsigned IAM gRPC calls, and `weed shell` still + # exits 0, so the Job goes green while anonymousRead is dropped. + # The key reaches the hook either through security.toml or through + # an environment override in extraEnvironmentVars. + python3 - "$CHART_DIR" <<'PYEOF' + import re, subprocess, sys, yaml + + chart = sys.argv[1] + + def render(values): + args = ["helm", "template", "test", chart] + for k, v in values.items(): + args += ["--set", f"{k}={v}"] + return subprocess.check_output(args, text=True) + + def hook_pod(manifest): + for d in yaml.safe_load_all(manifest): + if d and d.get("kind") == "Job" and d["metadata"]["name"].endswith("-bucket-hook"): + return d["spec"]["template"]["spec"] + return None + + def cluster_endpoints(manifest): + """Where the chart's own Services put master and filer. + + The hook's WEED_CLUSTER_* values have to match these: a defined + name pointing anywhere else fails the readiness wait exactly as + an undefined one does. + """ + wanted = {"-master": "swfs-master", "-filer-client": "swfs-filer"} + found = {} + for d in yaml.safe_load_all(manifest): + if not d or d.get("kind") != "Service": + continue + for suffix, portName in wanted.items(): + if not d["metadata"]["name"].endswith(suffix): + continue + port = next((p["port"] for p in d["spec"]["ports"] if p.get("name") == portName), None) + if port: + found[suffix] = f"{d['metadata']['name']}.{d['metadata']['namespace']}:{port}" + return found.get("-master"), found.get("-filer-client") + + buckets = { + "s3.enabled": "true", + "s3.createBuckets[0].name": "data", + "s3.createBuckets[0].anonymousRead": "true", + } + jwt = dict(buckets, **{"global.seaweedfs.securityConfig.jwtSigning.filerWrite": "true"}) + + failed = [] + + # The script in the hook addresses master and filer through + # WEED_CLUSTER_*, now rendered next to extraEnvironmentVars. + # Duplicate names would be a rendering bug. + out = render(buckets) + pod = hook_pod(out) + if pod is None: + failed.append("createBuckets: bucket hook Job missing") + else: + env = pod["containers"][0]["env"] + names = [e["name"] for e in env] + dupes = sorted({n for n in names if names.count(n) > 1}) + if dupes: + failed.append(f"createBuckets: duplicate env entries {dupes}") + values = {e["name"]: e.get("value") for e in env} + if not values.get("WEED_CLUSTER_DEFAULT"): + failed.append("createBuckets: hook env WEED_CLUSTER_DEFAULT is empty") + rendered = (values.get("WEED_CLUSTER_SW_MASTER"), values.get("WEED_CLUSTER_SW_FILER")) + expected = cluster_endpoints(out) + if not all(expected): + failed.append(f"createBuckets: no master/filer-client Service to compare against, found {expected}") + elif rendered != expected: + failed.append(f"createBuckets: hook env holds {rendered}, the services are at {expected}") + elif not dupes: + print("createBuckets: hook env resolves the cluster addresses once") + if any(v["name"] == "security-config" for v in pod.get("volumes", [])): + failed.append("createBuckets: hook mounts security-config without JWT signing") + + # filerWrite=true: the hook needs the same security.toml the filer + # gets, otherwise s3.configure fails with "missing authorization + # metadata". + pod = hook_pod(render(jwt)) + if pod is None: + failed.append("filerWrite=true: bucket hook Job missing") + else: + vol = next((v for v in pod.get("volumes", []) if v["name"] == "security-config"), None) + mount = next((m for m in pod["containers"][0].get("volumeMounts", []) if m["name"] == "security-config"), None) + # The file has to land on one of weed's config search paths, so + # the target matters as much as the volume itself - and a volume + # of that name backed by anything but the chart's ConfigMap + # leaves `weed shell` without the signing key just the same. + target = ("/etc/seaweedfs/security.toml", "security.toml") + configmap = "test-seaweedfs-security-config" + if vol is None or mount is None: + failed.append("filerWrite=true: bucket hook does not mount security-config (s3.configure would fail)") + elif (mount.get("mountPath"), mount.get("subPath")) != target: + failed.append(f"filerWrite=true: bucket hook mounts security-config as {mount}") + elif vol.get("configMap", {}).get("name") != configmap: + failed.append(f"filerWrite=true: security-config is not backed by the {configmap} ConfigMap: {vol}") + else: + print("filerWrite=true: bucket hook mounts security-config at /etc/seaweedfs/security.toml") + + # The readiness waits dereference the cluster env names, so renaming + # the cluster alias has to rename them too: an undefined name leaves + # the hook polling "http://" for five minutes and then failing the + # release. The endpoints stay chart-computed, so an address kept in + # extraEnvironmentVars is replaced, not rendered twice. + out = render(dict(buckets, **{ + "global.seaweedfs.extraEnvironmentVars.WEED_CLUSTER_DEFAULT": "prod", + "global.seaweedfs.extraEnvironmentVars.WEED_CLUSTER_PROD_MASTER": "stale:9333", + "global.seaweedfs.extraEnvironmentVars.WEED_CLUSTER_SW_MASTER": "null", + "global.seaweedfs.extraEnvironmentVars.WEED_CLUSTER_SW_FILER": "null", + })) + pod = hook_pod(out) + if pod is None: + failed.append("cluster alias: bucket hook Job missing") + else: + container = pod["containers"][0] + env = {e["name"]: e.get("value") for e in container["env"]} + waited = re.findall(r'wait_for_service "http://\$(\w+)', container["command"][2]) + undefined = sorted(n for n in set(waited) if not env.get(n)) + renamed = (env.get("WEED_CLUSTER_PROD_MASTER"), env.get("WEED_CLUSTER_PROD_FILER")) + expected = cluster_endpoints(out) + if not waited: + failed.append("cluster alias: hook script has no readiness waits") + elif undefined: + failed.append(f"cluster alias: hook waits on undefined env {undefined}") + elif not all(expected): + failed.append(f"cluster alias: no master/filer-client Service to compare against, found {expected}") + elif renamed != expected: + # Both addresses under the renamed alias, not just the one + # the values tried to keep: the stale one would fail the + # release, and a missing filer address just as much. + failed.append(f"cluster alias: hook waits on {renamed}, the services are at {expected}") + else: + print("cluster alias: hook waits on the renamed names and the chart's addresses") + + # Keys kept in a Secret are referenced, not inlined, so the hook + # must render valueFrom for non-string extraEnvironmentVars. + out = render(dict(jwt, **{ + "global.seaweedfs.extraEnvironmentVars.WEED_JWT_FILER_SIGNING_KEY.secretKeyRef.name": "signing-keys", + "global.seaweedfs.extraEnvironmentVars.WEED_JWT_FILER_SIGNING_KEY.secretKeyRef.key": "filerWrite", + })) + pod = hook_pod(out) + if pod is None: + failed.append("secretKeyRef: bucket hook Job missing") + else: + entry = next((e for e in pod["containers"][0]["env"] if e["name"] == "WEED_JWT_FILER_SIGNING_KEY"), None) + if entry is None: + failed.append("secretKeyRef: hook env WEED_JWT_FILER_SIGNING_KEY missing") + elif entry.get("valueFrom", {}).get("secretKeyRef") != {"name": "signing-keys", "key": "filerWrite"}: + failed.append(f"secretKeyRef: hook env rendered as {entry}") + else: + print("secretKeyRef: hook env keeps the secret reference") + + if failed: + print("\nFAIL:", file=sys.stderr) + for f in failed: + print(f" - {f}", file=sys.stderr) + sys.exit(1) + PYEOF + echo "Bucket hook credential tests passed" + + echo "=== Testing with monitoring enabled ===" + helm template test $CHART_DIR \ + --set global.seaweedfs.monitoring.enabled=true \ + --set global.seaweedfs.monitoring.gatewayHost=prometheus \ + --set global.seaweedfs.monitoring.gatewayPort=9091 > /tmp/monitoring.yaml + echo "Monitoring configuration renders correctly" + + echo "=== Testing with PVC storage ===" + helm template test $CHART_DIR \ + --set master.data.type=persistentVolumeClaim \ + --set master.data.size=10Gi \ + --set master.data.storageClass=standard > /tmp/pvc.yaml + grep -q "PersistentVolumeClaim" /tmp/pvc.yaml + echo "PVC configuration renders correctly" + + echo "=== Testing with custom replicas ===" + helm template test $CHART_DIR \ + --set master.replicas=3 \ + --set filer.replicas=2 \ + --set volume.replicas=3 > /tmp/replicas.yaml + echo "Custom replicas configuration renders correctly" + + echo "=== Testing filer with S3 gateway ===" + helm template test $CHART_DIR \ + --set filer.s3.enabled=true \ + --set filer.s3.enableAuth=true > /tmp/filer-s3.yaml + echo "Filer S3 gateway renders correctly" + + echo "=== Testing the mysql filer store gates its secret and env ===" + helm template test $CHART_DIR \ + --set-string filer.extraEnvironmentVars.WEED_MONGODB_ENABLED=true \ + --set-string filer.extraEnvironmentVars.WEED_LEVELDB2_ENABLED=false > /tmp/filer-mongodb.yaml + ! grep -q "db-secret" /tmp/filer-mongodb.yaml + ! grep -q "WEED_MYSQL" /tmp/filer-mongodb.yaml + grep -q "name: WEED_MONGODB_ENABLED" /tmp/filer-mongodb.yaml + helm template test $CHART_DIR \ + --set-string filer.extraEnvironmentVars.WEED_MYSQL_ENABLED=true > /tmp/filer-mysql.yaml + grep -q "name: test-seaweedfs-db-secret" /tmp/filer-mysql.yaml + grep -q "name: WEED_MYSQL_USERNAME" /tmp/filer-mysql.yaml + grep -q "name: WEED_MYSQL_PASSWORD" /tmp/filer-mysql.yaml + grep -q "name: WEED_MYSQL_HOSTNAME" /tmp/filer-mysql.yaml + # Secret-backed keys follow the same rule as the plain ones. + helm template test $CHART_DIR \ + --set filer.secretExtraEnvironmentVars.WEED_MYSQL_PASSWORD.secretKeyRef.name=db \ + --set filer.secretExtraEnvironmentVars.WEED_MYSQL_PASSWORD.secretKeyRef.key=password > /tmp/filer-mysql-off-secret.yaml + ! grep -q "WEED_MYSQL" /tmp/filer-mysql-off-secret.yaml + helm template test $CHART_DIR \ + --set-string filer.extraEnvironmentVars.WEED_MYSQL_ENABLED=true \ + --set filer.secretExtraEnvironmentVars.WEED_MYSQL_PASSWORD.secretKeyRef.name=db \ + --set filer.secretExtraEnvironmentVars.WEED_MYSQL_PASSWORD.secretKeyRef.key=password > /tmp/filer-mysql-on-secret.yaml + grep -A 4 -- "- name: WEED_MYSQL_PASSWORD$" /tmp/filer-mysql-on-secret.yaml | grep -q "name: db" + # A flag the chart cannot read counts as selected, not as off. + helm template test $CHART_DIR \ + --set filer.secretExtraEnvironmentVars.WEED_MYSQL_ENABLED.secretKeyRef.name=store \ + --set filer.secretExtraEnvironmentVars.WEED_MYSQL_ENABLED.secretKeyRef.key=enabled > /tmp/filer-mysql-secret.yaml + grep -q "name: test-seaweedfs-db-secret" /tmp/filer-mysql-secret.yaml + grep -q "name: WEED_MYSQL_HOSTNAME" /tmp/filer-mysql-secret.yaml + helm template test $CHART_DIR \ + --set-string filer.extraEnvironmentVars.WEED_MYSQL2_HOSTNAME=other > /tmp/filer-mysql2.yaml + grep -q "name: WEED_MYSQL2_HOSTNAME" /tmp/filer-mysql2.yaml + ! grep -q "WEED_MYSQL_" /tmp/filer-mysql2.yaml + echo "The mysql secret and env follow the selected filer store" + + echo "=== Testing SFTP enabled ===" + helm template test $CHART_DIR --set sftp.enabled=true > /tmp/sftp.yaml + grep -q "seaweedfs-sftp" /tmp/sftp.yaml + echo "SFTP deployment renders correctly" + + echo "" + echo "=== Testing SFTP host key generation ===" + # The chart must not ship host key material: keys are generated + # per install, land in the secret the deployments mount at + # sftp.hostKeysFolder, and must be PKCS#8 ed25519 private keys, + # the shape the server's host key loader parses; it fails to + # start otherwise. + pip install pyyaml -q + python3 - "$CHART_DIR" <<'PYEOF' + import base64, re, subprocess, sys, yaml + + chart = sys.argv[1] + + def render(values): + args = ["helm", "template", "test", chart] + for k, v in values.items(): + args += ["--set", f"{k}={v}"] + return subprocess.check_output(args, text=True) + + def docs(manifest): + return [d for d in yaml.safe_load_all(manifest) if d] + + def secret(manifest, name): + for d in docs(manifest): + if d.get("kind") == "Secret" and d["metadata"]["name"] == name: + return d + return None + + def pod_spec(manifest, name): + for d in docs(manifest): + if d.get("kind") in ("Deployment", "StatefulSet") and d["metadata"]["name"] == name: + return d["spec"]["template"]["spec"] + return None + + def script_of(spec): + for c in spec["containers"]: + cmd = c.get("command", []) + if len(cmd) >= 3 and cmd[0] == "/bin/sh": + return cmd[2] + raise AssertionError("no shell command block found") + + def volume_secret(spec, volname): + for v in spec.get("volumes", []): + if v["name"] == volname: + return v["secret"]["secretName"] + return None + + def mount_path(spec, volname): + for c in spec["containers"]: + for vm in c.get("volumeMounts", []): + if vm["name"] == volname: + return vm["mountPath"] + return None + + def parse_ed25519(pem): + m = re.match(r"-----BEGIN PRIVATE KEY-----\n(.+?)-----END PRIVATE KEY-----", pem.strip(), re.S) + if not m: + raise AssertionError("not a PKCS#8 PEM private key") + der = base64.b64decode(m.group(1)) + # RFC 8410: fixed PKCS#8 prefix, then the 32-byte seed + prefix = bytes.fromhex("302e020100300506032b657004220420") + if len(der) != 48 or not der.startswith(prefix): + raise AssertionError("not an ed25519 PKCS#8 key") + + failed = [] + # public-key material of the key the chart used to bundle + BUNDLED = "H4McwcDphteXVullu6q7ephEN1N60z" + + out1 = render({"sftp.enabled": "true"}) + out2 = render({"sftp.enabled": "true"}) + + for label, out in (("first", out1), ("second", out2)): + if BUNDLED in out: + failed.append(f"{label} render still contains the formerly bundled host key") + + def folder_key(out): + s = secret(out, "test-seaweedfs-sftp-ssh-secret") + if s is None: + return None + return base64.b64decode(s["data"]["ssh_host_ed25519_key"]).decode() + + k1, k2 = folder_key(out1), folder_key(out2) + if k1 is None or k2 is None: + failed.append("sftp-ssh-secret not rendered with sftp.enabled=true") + else: + try: + parse_ed25519(k1) + print("generated host key parses as ed25519") + except Exception as e: + failed.append(f"generated host key does not parse: {e}") + if k1 == k2: + failed.append("two renders produced the same host key (key is not generated per install)") + else: + print("host key differs between installs") + + legacy1 = secret(out1, "test-seaweedfs-sftp-secret")["stringData"]["seaweedfs_sftp_ssh_private_key"] + legacy2 = secret(out2, "test-seaweedfs-sftp-secret")["stringData"]["seaweedfs_sftp_ssh_private_key"] + try: + parse_ed25519(legacy1) + except Exception as e: + failed.append(f"sftp-secret ssh key does not parse: {e}") + if legacy1 == legacy2: + failed.append("sftp-secret ssh key identical across renders") + else: + print("sftp-secret ssh key is generated per install") + + spec = pod_spec(out1, "test-seaweedfs-sftp") + script = script_of(spec) + if "-sshPrivateKey" in script: + failed.append("sftp deployment passes -sshPrivateKey by default; the file only exists " + "when enableAuth mounts /etc/sw and a missing key file is fatal") + if "-hostKeysFolder=/etc/sw/ssh" not in script: + failed.append("sftp deployment missing -hostKeysFolder=/etc/sw/ssh") + if volume_secret(spec, "config-ssh") != "test-seaweedfs-sftp-ssh-secret": + failed.append("sftp config-ssh volume does not reference the generated secret") + else: + print("sftp deployment mounts the generated secret at the host keys folder") + + out = render({"sftp.enabled": "true", "sftp.existingSshConfigSecret": "my-keys"}) + if secret(out, "test-seaweedfs-sftp-ssh-secret") is not None: + failed.append("existingSshConfigSecret set but the default ssh secret still renders") + if volume_secret(pod_spec(out, "test-seaweedfs-sftp"), "config-ssh") != "my-keys": + failed.append("existingSshConfigSecret is not the config-ssh volume source") + else: + print("existingSshConfigSecret replaces the generated secret") + + out = render({"allInOne.enabled": "true", "allInOne.sftp.enabled": "true"}) + spec = pod_spec(out, "test-seaweedfs-all-in-one") + if secret(out, "test-seaweedfs-sftp-ssh-secret") is None: + failed.append("all-in-one: ssh secret not rendered") + if "-sftp.hostKeysFolder=/etc/sw/ssh" not in script_of(spec): + failed.append("all-in-one: missing -sftp.hostKeysFolder=/etc/sw/ssh") + if volume_secret(spec, "config-ssh") != "test-seaweedfs-sftp-ssh-secret": + failed.append("all-in-one: config-ssh volume does not reference the generated secret") + else: + print("all-in-one mounts the generated secret") + + out = render({"sftp.enabled": "true", "sftp.hostKeysFolder": "/keys"}) + spec = pod_spec(out, "test-seaweedfs-sftp") + if "-hostKeysFolder=/keys" not in script_of(spec) or mount_path(spec, "config-ssh") != "/keys": + failed.append("custom hostKeysFolder: flag and secret mount do not agree") + else: + print("custom hostKeysFolder keeps flag and mount aligned") + + out = render({"allInOne.enabled": "true", "allInOne.sftp.enabled": "true", + "allInOne.sftp.hostKeysFolder": "/keys"}) + spec = pod_spec(out, "test-seaweedfs-all-in-one") + if "-sftp.hostKeysFolder=/keys" not in script_of(spec) or mount_path(spec, "config-ssh") != "/keys": + failed.append("all-in-one custom hostKeysFolder: flag and secret mount do not agree") + else: + print("all-in-one custom hostKeysFolder keeps flag and mount aligned") + + if failed: + print("\nFAIL:", file=sys.stderr) + for f in failed: + print(f" - {f}", file=sys.stderr) + sys.exit(1) + PYEOF + echo "SFTP host key generation tests passed" + + echo "=== Testing ingress configurations ===" + helm template test $CHART_DIR \ + --set master.ingress.enabled=true \ + --set filer.ingress.enabled=true \ + --set s3.enabled=true \ + --set s3.ingress.enabled=true > /tmp/ingress.yaml + grep -q "kind: Ingress" /tmp/ingress.yaml + echo "Ingress configurations render correctly" + + echo "=== Testing COSI driver ===" + helm template test $CHART_DIR --set cosi.enabled=true > /tmp/cosi.yaml + grep -q "seaweedfs-cosi" /tmp/cosi.yaml + echo "COSI driver renders correctly" + + echo "" + echo "=== Testing long release name: service names match DNS references ===" + # Use a release name that, combined with chart name "seaweedfs", exceeds 63 chars. + # fullname = "my-very-long-release-name-that-will-cause-truncation-seaweedfs" (65 chars before trunc) + LONG_RELEASE="my-very-long-release-name-that-will-cause-truncation" + + # --- Normal mode: master + filer-client services vs helper-produced addresses --- + helm template "$LONG_RELEASE" $CHART_DIR \ + --set s3.enabled=true \ + --set global.seaweedfs.createBuckets[0].name=test > /tmp/longname.yaml + + # Extract Service names from metadata + MASTER_SVC=$(awk '/kind: Service/{found=1} found && /^ *name:/{print $2; found=0}' /tmp/longname.yaml \ + | grep -- '-master$') + FILER_CLIENT_SVC=$(awk '/kind: Service/{found=1} found && /^ *name:/{print $2; found=0}' /tmp/longname.yaml \ + | grep -- '-filer-client$') + + # Extract the hostname from WEED_CLUSTER_SW_MASTER in post-install-bucket-hook + MASTER_ADDR=$(grep 'WEED_CLUSTER_SW_MASTER' -A1 /tmp/longname.yaml \ + | grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/') + FILER_ADDR=$(grep 'WEED_CLUSTER_SW_FILER' -A1 /tmp/longname.yaml \ + | grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/') + + # Extract the hostname from S3 deployment -filer= argument + S3_FILER_HOST=$(grep '\-filer=' /tmp/longname.yaml \ + | head -1 | sed 's/.*-filer=\([^:]*\).*/\1/') + + # The address helpers produce ".:"; extract just the svc name + MASTER_ADDR_SVC=$(echo "$MASTER_ADDR" | cut -d. -f1) + FILER_ADDR_SVC=$(echo "$FILER_ADDR" | cut -d. -f1) + S3_FILER_SVC=$(echo "$S3_FILER_HOST" | cut -d. -f1) + + echo " master Service.name: $MASTER_SVC" + echo " cluster.masterAddress svc: $MASTER_ADDR_SVC" + echo " filer-client Service.name: $FILER_CLIENT_SVC" + echo " cluster.filerAddress svc: $FILER_ADDR_SVC" + echo " S3 -filer= svc: $S3_FILER_SVC" + + [ "$MASTER_SVC" = "$MASTER_ADDR_SVC" ] || { echo "FAIL: master service name mismatch"; exit 1; } + [ "$FILER_CLIENT_SVC" = "$FILER_ADDR_SVC" ] || { echo "FAIL: filer-client service name mismatch"; exit 1; } + [ "$FILER_CLIENT_SVC" = "$S3_FILER_SVC" ] || { echo "FAIL: S3 -filer= does not match filer-client service"; exit 1; } + echo "Normal mode: service names match DNS references with long release name" + + # --- All-in-one mode: all-in-one service vs both helper addresses --- + helm template "$LONG_RELEASE" $CHART_DIR \ + --set allInOne.enabled=true \ + --set global.seaweedfs.createBuckets[0].name=test > /tmp/longname-aio.yaml + + AIO_SVC=$(awk '/kind: Service/{found=1} found && /^ *name:/{print $2; found=0}' /tmp/longname-aio.yaml \ + | grep -- '-all-in-one$') + AIO_MASTER_ADDR_SVC=$(grep 'WEED_CLUSTER_SW_MASTER' -A1 /tmp/longname-aio.yaml \ + | grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/' | cut -d. -f1) + AIO_FILER_ADDR_SVC=$(grep 'WEED_CLUSTER_SW_FILER' -A1 /tmp/longname-aio.yaml \ + | grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/' | cut -d. -f1) + + echo " all-in-one Service.name: $AIO_SVC" + echo " cluster.masterAddress svc: $AIO_MASTER_ADDR_SVC" + echo " cluster.filerAddress svc: $AIO_FILER_ADDR_SVC" + + [ "$AIO_SVC" = "$AIO_MASTER_ADDR_SVC" ] || { echo "FAIL: all-in-one master address mismatch"; exit 1; } + [ "$AIO_SVC" = "$AIO_FILER_ADDR_SVC" ] || { echo "FAIL: all-in-one filer address mismatch"; exit 1; } + echo "All-in-one mode: service names match DNS references with long release name" + + echo "" + echo "=== Testing security+S3: no blank lines in shell command blocks ===" + # Render the three manifests that include seaweedfs.s3.tlsArgs: + # filer-statefulset, s3-deployment, all-in-one-deployment + helm template test $CHART_DIR \ + --set global.seaweedfs.enableSecurity=true \ + --set filer.s3.enabled=true \ + --set s3.enabled=true > /tmp/security-s3.yaml + helm template test $CHART_DIR \ + --set global.seaweedfs.enableSecurity=true \ + --set allInOne.enabled=true \ + --set allInOne.s3.enabled=true > /tmp/security-aio.yaml + + pip install pyyaml -q + python3 - /tmp/security-s3.yaml /tmp/security-aio.yaml <<'PYEOF' + import yaml, sys + errors = [] + for path in sys.argv[1:]: + with open(path) as f: + docs = list(yaml.safe_load_all(f)) + for doc in docs: + if not doc or doc.get("kind") not in ("Deployment", "StatefulSet"): + continue + name = doc["metadata"]["name"] + for c in doc["spec"]["template"]["spec"].get("containers", []): + cmd = c.get("command", []) + if len(cmd) >= 3 and cmd[0] == "/bin/sh" and cmd[1] == "-ec": + script = cmd[2] + for i, line in enumerate(script.splitlines(), 1): + if line.strip() == "": + errors.append(f"{path}: {name}/{c['name']} has blank line at script line {i}") + if errors: + for e in errors: + print(f"FAIL: {e}", file=sys.stderr) + print("Rendered with: global.seaweedfs.enableSecurity=true, filer.s3.enabled=true, s3.enabled=true, allInOne.enabled=true", file=sys.stderr) + sys.exit(1) + print("No blank lines in security+S3 command blocks") + PYEOF + + echo "" + echo "=== Testing security+S3: -cert.file/-key.file gated on httpsPort (issue #9202) ===" + # Regression test: when enableSecurity=true but *.httpsPort is 0 (the default), + # the chart must NOT emit -cert.file / -key.file to the S3 frontend. Passing + # them promotes weed s3's main -port to HTTPS (see weed/command/s3.go), which + # makes the HTTP readinessProbe spam "TLS handshake error ... client sent an + # HTTP request to an HTTPS server" into the pod log. + # + # When *.httpsPort > 0, both -port.https and cert/key args MUST be emitted + # together so the opt-in HTTPS listener actually has credentials. + python3 - "$CHART_DIR" <<'PYEOF' + import subprocess, sys, yaml + chart = sys.argv[1] + + def render(values): + args = ["helm", "template", "test", chart] + for k, v in values.items(): + args += ["--set", f"{k}={v}"] + return subprocess.check_output(args, text=True) + + def script_of(manifest, kind_name): + for doc in yaml.safe_load_all(manifest): + if not doc or doc.get("kind") not in ("Deployment", "StatefulSet"): + continue + if doc["metadata"]["name"] != kind_name: + continue + for c in doc["spec"]["template"]["spec"]["containers"]: + cmd = c.get("command", []) + if len(cmd) >= 3 and cmd[0] == "/bin/sh" and cmd[1] == "-ec": + return cmd[2] + raise AssertionError(f"no container script for {kind_name}") + + cases = [ + # (values, workload-name, httpsPort-set?, arg-prefix) + ({"global.seaweedfs.enableSecurity": "true", + "s3.enabled": "true"}, + "test-seaweedfs-s3", False, ""), + ({"global.seaweedfs.enableSecurity": "true", + "s3.enabled": "true", + "s3.httpsPort": "8443"}, + "test-seaweedfs-s3", True, ""), + ({"global.seaweedfs.enableSecurity": "true", + "filer.s3.enabled": "true"}, + "test-seaweedfs-filer", False, "s3."), + ({"global.seaweedfs.enableSecurity": "true", + "filer.s3.enabled": "true", + "filer.s3.httpsPort": "8444"}, + "test-seaweedfs-filer", True, "s3."), + ({"global.seaweedfs.enableSecurity": "true", + "allInOne.enabled": "true", + "allInOne.s3.enabled": "true"}, + "test-seaweedfs-all-in-one", False, "s3."), + ({"global.seaweedfs.enableSecurity": "true", + "allInOne.enabled": "true", + "allInOne.s3.enabled": "true", + "allInOne.s3.httpsPort": "8445"}, + "test-seaweedfs-all-in-one", True, "s3."), + ] + + failed = False + for values, name, https_on, prefix in cases: + script = script_of(render(values), name) + cert_flag = f"-{prefix}cert.file=" + key_flag = f"-{prefix}key.file=" + https_flag = f"-{prefix}port.https=" + has_cert = cert_flag in script + has_key = key_flag in script + has_https = https_flag in script + label = f"{name} (httpsPort {'set' if https_on else 'unset'})" + if https_on: + if not (has_cert and has_key and has_https): + print(f"FAIL: {label}: expected {cert_flag}, {key_flag}, {https_flag} all present " + f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr) + failed = True + else: + print(f"{label}: cert/key/https args emitted together") + else: + if has_cert or has_key or has_https: + print(f"FAIL: {label}: expected none of {cert_flag}/{key_flag}/{https_flag}; " + f"main S3 -port would silently become HTTPS and break HTTP probes " + f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr) + failed = True + else: + print(f"{label}: no TLS args emitted, main -port stays HTTP") + + # bash -n: pin down that the rendered script parses. Guards against + # a future helper change that leaves a dangling `\` with nothing + # after it (every current caller already exits cleanly because + # bash treats trailing `\` as line-continuation to + # an empty line — but keep the contract explicit). + parse = subprocess.run(["bash", "-n"], input=script, text=True, + capture_output=True) + if parse.returncode != 0: + print(f"FAIL: {label}: bash -n rejected rendered script: {parse.stderr.strip()}", + file=sys.stderr) + failed = True + + sys.exit(1 if failed else 0) + PYEOF + + echo "" + echo "=== Testing all-in-one env: a key in both global and component renders once ===" + # Regression: all-in-one looped global and component extraEnvironmentVars + # in two separate ranges, emitting duplicate env entries for any key set + # in both maps. Render a shared key and assert it appears exactly once in + # the all-in-one container, with the component value winning (consistent + # with the merge helper the other components already use). pyyaml is + # installed by the earlier IAM gRPC block in this same step. + helm template test $CHART_DIR \ + --set allInOne.enabled=true \ + --set global.seaweedfs.extraEnvironmentVars.WEED_SHARED=fromGlobal \ + --set allInOne.extraEnvironmentVars.WEED_SHARED=fromComponent > /tmp/aio-env.yaml + python3 - /tmp/aio-env.yaml <<'PYEOF' + import sys, yaml + from collections import Counter + docs = [d for d in yaml.safe_load_all(open(sys.argv[1])) if d] + dep = next(d for d in docs if d.get("kind") == "Deployment" + and d["metadata"]["name"].endswith("all-in-one")) + envs = [e["name"] for c in dep["spec"]["template"]["spec"]["containers"] + for e in c.get("env", [])] + dups = {k: v for k, v in Counter(envs).items() if v > 1} + if dups: + print(f"FAIL: duplicate env entries in all-in-one container: {dups}", file=sys.stderr) + sys.exit(1) + val = next(e.get("value") for c in dep["spec"]["template"]["spec"]["containers"] + for e in c.get("env", []) if e["name"] == "WEED_SHARED") + if val != "fromComponent": + print(f"FAIL: WEED_SHARED should take the component value 'fromComponent', got '{val}'", + file=sys.stderr) + sys.exit(1) + print("all-in-one env: shared key renders once, component value wins") + PYEOF + + echo "=== Testing bucket versioning: YAML bool false suspends like string \"false\" ===" + # bool false used to be a silent no-op while string "false" suspended. + BOOL_FALSE=$(helm template test $CHART_DIR \ + --set s3.enabled=true \ + --set s3.createBuckets[0].name=verbucket \ + --set s3.createBuckets[0].versioning=false | grep 's3.bucket.versioning -name verbucket' || true) + echo "$BOOL_FALSE" | grep -q -- '-status Suspended' || { echo "FAIL: bool false versioning did not Suspend the bucket"; exit 1; } + echo "Bucket versioning: YAML bool false suspends consistently with string \"false\"" + + echo "" + echo "=== Testing hook Job labels ===" + # The hook Jobs were the only pods in the chart without the standard + # app.kubernetes.io label set, so nothing label-based could target + # them - NetworkPolicy podSelectors, monitoring, kubectl -l. Assert + # both the Job and its pod template carry the same name/instance/ + # component triple the other components use, and that the triple is + # the hook's own so a per-component selector cannot match it too. + # + # Only the bucket hook is covered: the volume resize hook is gated on + # lookup finding a StatefulSet with a smaller PVC than requested, and + # lookup returns nothing under helm template, so it never renders here. + python3 - "$CHART_DIR" <<'PYEOF' + import subprocess, sys, yaml + + chart = sys.argv[1] + # The three a selector keys on, and the full set the other workloads + # carry - a missing chart or managed-by label is not a selector + # problem, but it does leave the hook Jobs looking unlike everything + # else the release owns. + TRIPLE = ("app.kubernetes.io/name", "app.kubernetes.io/instance", + "app.kubernetes.io/component") + # The labels that identify the release rather than the workload, and + # so have to hold the same values everywhere. They are compared + # against a workload that already renders them instead of being + # spelled out here, which keeps the chart version out of the test. + # managed-by is not among them on purpose: the chart puts it on + # workload metadata but not on pod templates, so it cannot be part of + # a cross-workload comparison. Its presence is still checked below. + RELEASE = ("app.kubernetes.io/name", "app.kubernetes.io/instance", + "helm.sh/chart") + STANDARD = TRIPLE + ("helm.sh/chart", "app.kubernetes.io/managed-by") + + def render(values): + args = ["helm", "template", "test", chart] + for k, v in values.items(): + args += ["--set", f"{k}={v}"] + return subprocess.check_output(args, text=True) + + def docs(manifest): + return [d for d in yaml.safe_load_all(manifest) if d] + + def hook_job(manifest): + for d in docs(manifest): + if d.get("kind") == "Job" and d["metadata"]["name"].endswith("-bucket-hook"): + return d + return None + + def triple(labels): + return {k: labels.get(k) for k in TRIPLE} + + def release(labels): + return {k: labels.get(k) for k in RELEASE} + + def reference(manifest): + """Any long-running workload; they all carry the release labels.""" + for d in docs(manifest): + if d.get("kind") in ("Deployment", "StatefulSet"): + return d + return None + + modes = { + "s3": {"s3.enabled": "true", + "s3.createBuckets[0].name": "b"}, + "filer.s3": {"filer.s3.enabled": "true", + "filer.s3.createBuckets[0].name": "b"}, + "allInOne": {"allInOne.enabled": "true", + "allInOne.s3.enabled": "true", + "allInOne.s3.createBuckets[0].name": "b"}, + } + + failed = [] + for mode, values in modes.items(): + before = len(failed) + out = render(values) + job = hook_job(out) + if job is None: + failed.append(f"{mode}: bucket hook Job not rendered") + continue + job_labels = job["metadata"].get("labels", {}) + pod_labels = job["spec"]["template"]["metadata"].get("labels", {}) + ref = reference(out) + if ref is None: + failed.append(f"{mode}: no workload to compare the release labels against") + continue + ref_labels = release(ref["spec"]["template"]["metadata"].get("labels", {})) + for where, labels in (("Job", job_labels), ("pod", pod_labels)): + missing = [k for k in STANDARD if not labels.get(k)] + if missing: + failed.append(f"{mode}: bucket hook {where} has no {missing}, " + "nothing can select it") + component = labels.get("app.kubernetes.io/component") + if component != "bucket-hook": + failed.append(f"{mode}: bucket hook {where} component is " + f"{component!r}, expected 'bucket-hook'") + # Present is not enough: the values have to be the release's + # own, or a selector written for this release misses the hook. + if release(labels) != ref_labels: + failed.append(f"{mode}: bucket hook {where} release labels " + f"{release(labels)} differ from " + f"{ref['metadata']['name']}'s {ref_labels}") + # A selector written against the Job has to find its pods. + if triple(job_labels) != triple(pod_labels): + failed.append(f"{mode}: bucket hook Job and pod disagree: " + f"{triple(job_labels)} vs {triple(pod_labels)}") + # The triple must not also match another component's pods, or a + # selector meant for that component would pull the hook pod in. + for d in docs(out): + if d.get("kind") not in ("Deployment", "StatefulSet"): + continue + other = d["spec"]["template"]["metadata"].get("labels", {}) + if triple(other) == triple(pod_labels): + failed.append(f"{mode}: bucket hook pod shares its label triple " + f"with {d['metadata']['name']}") + if len(failed) == before: + print(f"{mode}: bucket hook Job and pod carry a distinct label triple") + + if failed: + print("\nFAIL:", file=sys.stderr) + for f in failed: + print(f" - {f}", file=sys.stderr) + sys.exit(1) + PYEOF + echo "Hook Job label tests passed" + + echo "" + echo "=== Testing NetworkPolicy rendering ===" + # The policies are only exercised for real by the networkpolicy-install + # job below. These assertions cover what template rendering can see: + # that the flag stays off by default, that every deployed component has + # exactly one policy, that each policy allows every port its workload + # declares, and that egress stays a separate opt-in. + python3 - "$CHART_DIR" <<'PYEOF' + import subprocess, sys, yaml + + chart = sys.argv[1] + + def render(values, json_values=None): + args = ["helm", "template", "test", chart] + for k, v in values.items(): + args += ["--set", f"{k}={v}"] + for k, v in (json_values or {}).items(): + args += ["--set-json", f"{k}={v}"] + return subprocess.check_output(args, text=True, stderr=subprocess.STDOUT) + + def expect_failure(values, needle, label, json_values=None): + try: + render(values, json_values) + except subprocess.CalledProcessError as e: + if needle not in (e.output or ""): + failed.append(f"{label}: unexpected error: {(e.output or '')[:200]}") + else: + print(label) + return + failed.append(f"{label}: render should have failed") + + def docs(manifest): + return [d for d in yaml.safe_load_all(manifest) if d] + + def component(labels): + return labels.get("app.kubernetes.io/component") + + def policies(manifest): + out = {} + for d in docs(manifest): + if d.get("kind") != "NetworkPolicy": + continue + out[component(d["spec"]["podSelector"]["matchLabels"])] = d + return out + + def workloads(manifest, kinds=("Deployment", "StatefulSet")): + out = {} + for d in docs(manifest): + if d.get("kind") in kinds: + out[component(d["spec"]["template"]["metadata"]["labels"])] = d + return out + + def allowed_ports(policy): + ports = set() + for rule in policy["spec"].get("ingress") or []: + for p in rule.get("ports") or []: + ports.add(p["port"]) + return ports + + def container_ports(workload): + ports = set() + for c in workload["spec"]["template"]["spec"].get("containers", []): + for p in c.get("ports") or []: + ports.add(p["containerPort"]) + return ports + + EVERYTHING = { + "s3.enabled": "true", + "sftp.enabled": "true", + "admin.enabled": "true", + # admin.ip defaults to 0.0.0.0 (non-loopback), which weed admin 4.46 + # refuses to bind without authentication. + "admin.secret.adminPassword": "ci-admin-password", + "worker.enabled": "true", + "cosi.enabled": "true", + "s3.createBuckets[0].name": "b", + "volumes.ssd.port": "8081", + "global.seaweedfs.monitoring.enabled": "true", + } + + failed = [] + + # Off by default: the chart has never shipped a NetworkPolicy and must not + # start now, or every existing release in a default-deny namespace changes + # behaviour on the next upgrade. + if policies(render(EVERYTHING)): + failed.append("networkPolicy.enabled unset: policies rendered anyway") + else: + print("networkPolicy off by default: no policies rendered") + + on = dict(EVERYTHING, **{"networkPolicy.enabled": "true"}) + out = render(on) + pols = policies(out) + wls = workloads(out) + + # Every workload gets exactly one policy, and every policy has a workload. + # A component with no policy is wide open under default-deny; a policy with + # no component is dead weight that hides a renamed label. + for comp in wls: + if comp not in pols: + failed.append(f"{comp}: workload has no NetworkPolicy") + for comp in pols: + # The bucket hook Job is covered too, and a Job is not a workload here. + if comp not in wls and comp != "bucket-hook": + failed.append(f"{comp}: NetworkPolicy selects a component that is not deployed") + + # The ports a component listens on come from the same values as its + # containerPorts, so the two must agree. This is what catches a port added to + # a workload and forgotten in the policy - the failure mode that only shows up + # once someone turns the flag on. + for comp, wl in wls.items(): + if comp not in pols: + continue + declared = container_ports(wl) + allowed = allowed_ports(pols[comp]) + missing = sorted(declared - allowed) + if missing: + failed.append(f"{comp}: listens on {missing} but its policy does not allow it") + if not failed: + print("every workload has a policy covering all of its containerPorts") + + # The bucket hook Job and its policy are gated on one shared helper, and + # this pins that down: a Job without its policy hangs under default-deny, + # a policy without its Job is noise. The combinations cover the modes that + # decide it, including the ones where a bucket list is ignored. + hook_cases = { + "nothing": {}, + "s3 without buckets": {"s3.enabled": "true"}, + "s3 with buckets": {"s3.enabled": "true", "s3.createBuckets[0].name": "b"}, + "filer.s3 without buckets": {"filer.s3.enabled": "true"}, + "filer.s3 with buckets": {"filer.s3.enabled": "true", + "filer.s3.createBuckets[0].name": "b"}, + "s3 gateway, buckets on filer.s3": {"s3.enabled": "true", + "filer.s3.createBuckets[0].name": "b"}, + "allInOne without buckets": {"allInOne.enabled": "true", + "allInOne.s3.enabled": "true"}, + "allInOne with buckets": {"allInOne.enabled": "true", + "allInOne.s3.enabled": "true", + "allInOne.s3.createBuckets[0].name": "b"}, + # allInOne reads only its own bucket list, so s3.createBuckets is + # not enough to produce the Job. + "allInOne, buckets on s3": {"allInOne.enabled": "true", + "allInOne.s3.enabled": "true", + "s3.createBuckets[0].name": "b"}, + "master off": {"master.enabled": "false", "s3.enabled": "true", + "s3.createBuckets[0].name": "b"}, + "buckets but no S3 endpoint": {"s3.createBuckets[0].name": "b"}, + } + for label, values in hook_cases.items(): + out = render(dict(values, **{"networkPolicy.enabled": "true"})) + job = any(d.get("kind") == "Job" + and d["metadata"]["name"].endswith("-bucket-hook") + for d in docs(out)) + policy = "bucket-hook" in policies(out) + if job != policy: + failed.append(f"{label}: bucket hook Job={job} but its policy={policy}; " + "the two are gated on seaweedfs.bucketHookEnabled and " + "have to appear together") + if not failed: + print(f"bucket hook Job and policy agree across {len(hook_cases)} modes") + + # Egress is its own opt-in: with it off the policies must not constrain + # outbound traffic at all, or enabling networkPolicy alone would cut the filer + # off from its store. + for comp, p in pols.items(): + if "Egress" in p["spec"]["policyTypes"]: + failed.append(f"{comp}: Egress in policyTypes while networkPolicy.egress.enabled is false") + if not any("Egress" in p["spec"]["policyTypes"] for p in pols.values()): + print("egress off by default: policies are ingress-only") + + # Components that need the API server must not silently lose it: rendering + # fails with a pointer to the value instead. + expect_failure(dict(on, **{"networkPolicy.egress.enabled": "true"}), + "kubeApiServer.cidrs is empty", + "empty kubeApiServer.cidrs fails the render with a pointer to the value") + + egress_on = dict(on, **{ + "networkPolicy.egress.enabled": "true", + "networkPolicy.egress.kubeApiServer.cidrs[0]": "10.96.0.1/32", + }) + + # An empty port or peer list is "everything" in a NetworkPolicy, not + # "nothing", so the two places that could be emptied have to be refused + # rather than quietly widened. + expect_failure(egress_on, "kubeApiServer.ports is empty", + "empty kubeApiServer.ports is refused instead of allowing every port", + json_values={"networkPolicy.egress.kubeApiServer.ports": "[]"}) + expect_failure(egress_on, "allowDNS is on but both", + "DNS with no selector is refused instead of allowing every pod", + json_values={"networkPolicy.egress.dnsNamespaceSelector": "null", + "networkPolicy.egress.dnsPodSelector": "null"}) + + # Dropping one of the two DNS selectors must leave the key out, not + # render it as null - null namespaceSelector means this namespace and + # null podSelector means every pod in the peer namespace. + for dropped in ("dnsNamespaceSelector", "dnsPodSelector"): + out = render(egress_on, {f"networkPolicy.egress.{dropped}": "null"}) + nulls = [(c, k) for c, p in policies(out).items() + for r in p["spec"]["egress"] + for t in r.get("to") or [] + for k, v in t.items() if v is None] + nulls += [(c, "ports") for c, p in policies(out).items() + for r in p["spec"]["egress"] if "ports" in r and r["ports"] is None] + if nulls: + failed.append(f"{dropped}=null: rendered null keys {sorted(set(nulls))}") + else: + print(f"{dropped}=null: the key is left out rather than rendered as null") + + # Helm allows a release named "123", and an unquoted label value then + # renders as a YAML integer. Label values are strings in the API, so the + # whole object gets rejected - a policy that never applies, silently. + # Every label the policies emit, in metadata and in every selector, has + # to survive that release name as a string. + numeric = subprocess.check_output( + ["helm", "template", "123", chart, + "--set", "networkPolicy.enabled=true", + "--set", "networkPolicy.egress.enabled=true", + "--set", "networkPolicy.egress.kubeApiServer.cidrs[0]=10.96.0.1/32", + "--set", "s3.enabled=true", "--set", "s3.createBuckets[0].name=b"], + text=True, stderr=subprocess.STDOUT) + + def label_maps(policy): + yield "metadata", policy["metadata"].get("labels", {}) + yield "podSelector", policy["spec"]["podSelector"]["matchLabels"] + for direction in ("ingress", "egress"): + for i, rule in enumerate(policy["spec"].get(direction) or []): + for peer in rule.get("to") or rule.get("from") or []: + for key in ("podSelector", "namespaceSelector"): + sel = peer.get(key) or {} + yield f"{direction}[{i}].{key}", sel.get("matchLabels", {}) + + coerced = [(comp, where, k, v) + for comp, p in policies(numeric).items() + for where, labels in label_maps(p) + for k, v in labels.items() if not isinstance(v, str)] + if coerced: + failed.append(f"release name 123: label values are not strings, so the API " + f"server rejects the policy: {coerced}") + else: + print("a numeric release name keeps every policy label a string") + + out = render(egress_on) + pols = policies(out) + + # DNS for everyone: every component addresses its peers by service name. + for comp, p in pols.items(): + dns = [r for r in p["spec"]["egress"] + if {x["port"] for x in r.get("ports") or []} == {53}] + if not dns: + failed.append(f"{comp}: egress on but no DNS rule") + if not failed: + print("every policy allows DNS when egress is on") + + # The API server rule goes only to what talks to it, which is the COSI + # sidecar and, on an upgrade that grows a PVC, the resize hook. No + # seaweedfs binary does - there is no client-go in go.mod - so admin + # must not be in here: it would force everyone running admin to name an + # API server address for a connection that is never made. + apiserver = {c for c, p in pols.items() + if any("ipBlock" in t for r in p["spec"]["egress"] for t in r.get("to") or [])} + expected = {"objectstorage-provisioner"} + if apiserver != expected: + failed.append(f"API server egress granted to {sorted(apiserver)}, expected {sorted(expected)}") + else: + print(f"API server egress limited to {sorted(expected)}") + + # Which means egress on its own must render for a release that runs + # neither COSI nor a resize: no component of it reaches the API server, + # so nothing may demand a CIDR for one. + for label, values in {"defaults": {}, "admin": {"admin.enabled": "true", "admin.secret.adminPassword": "ci-admin-password"}}.items(): + try: + render(dict(values, **{"networkPolicy.enabled": "true", + "networkPolicy.egress.enabled": "true"})) + print(f"egress on {label} renders without kubeApiServer.cidrs") + except subprocess.CalledProcessError as e: + failed.append(f"{label}: egress needs kubeApiServer.cidrs but nothing " + f"in the release talks to the API server: {(e.output or '')[:200]}") + + # The resize hook's policy is gated on the same lookup as its Job, so + # neither is ever in a rendered set - a policy on its own would be an + # orphaned hook resource on every install, since Helm does not collect + # those. The hook annotations it carries when the lookup does hit are + # only reachable against a live cluster. + if "volume-resize-hook" in pols: + failed.append("volume-resize-hook policy rendered without its Job; the two " + "are gated on seaweedfs.volumeResizeHookCommands and have to " + "appear together") + else: + print("volume-resize-hook policy tracks its Job rather than rendering always") + + # A components key that names nothing reads as "these rules are applied" + # and silently does not apply them. + expect_failure(dict(on, **{"networkPolicy.components.filerr.extraIngress[0].ports[0].port": "1"}), + '"filerr" is not a component of this chart', + "a misspelled networkPolicy.components key fails the render") + + # The bucket hook is post-install, so the release manifest is already applied; + # its policy must be a plain resource that uninstall cleans up. + if "helm.sh/hook" in (pols["bucket-hook"]["metadata"].get("annotations") or {}): + failed.append("bucket-hook policy is a hook resource; post-install runs after the manifest is applied") + else: + print("bucket-hook policy is a plain release resource") + + if failed: + print("\nFAIL:", file=sys.stderr) + for f in failed: + print(f" - {f}", file=sys.stderr) + sys.exit(1) + PYEOF + echo "NetworkPolicy rendering tests passed" + + echo "" + echo "=== Testing enterprise license mount + master persistence ===" + python3 - "$CHART_DIR" <<'PYEOF' + import subprocess, sys, yaml + chart = sys.argv[1] + + def render(values): + args = ["helm", "template", "test", chart] + for k, v in values.items(): + args += ["--set", f"{k}={v}"] + return subprocess.check_output(args, text=True) + + def workloads(manifest): + for d in yaml.safe_load_all(manifest): + if d and d.get("kind") in ("Deployment", "StatefulSet"): + yield d + + ALL_ON = { + "admin.enabled": "true", + # admin.ip defaults to 0.0.0.0 (non-loopback), which weed admin 4.46 + # refuses to bind without authentication. + "admin.secret.adminPassword": "ci-admin-password", + "s3.enabled": "true", + "sftp.enabled": "true", + "worker.enabled": "true", + } + failed = [] + + # Case 1: unconfigured renders nothing, so existing installs are unaffected. + out = render(ALL_ON) + if "SEAWEED_LICENSE" in out or "seaweedfs-license" in out: + failed.append("defaults: license plumbing should not render") + else: + print("defaults: no license volume/env (unchanged)") + + # Case 2: only the workloads that run a master carry the license. + LICENSE_PATH = "/etc/seaweedfs/license/seaweed-license.json" + out = render(dict(ALL_ON, **{ + "global.seaweedfs.license.existingSecret": "lic", + })) + carrying = set() + for d in workloads(out): + name = d["metadata"]["name"] + pod = d["spec"]["template"]["spec"] + vols = [v for v in pod.get("volumes", []) if v.get("name") == "seaweedfs-license"] + if not vols: + continue + carrying.add(name) + secret = vols[0].get("secret", {}) + if secret.get("secretName") != "lic": + failed.append(f"{name}: license volume points at {secret.get('secretName')!r}, not the configured Secret") + items = [i.get("key") for i in secret.get("items", [])] + if items != ["seaweed-license.json"]: + failed.append(f"{name}: license volume projects {items}, expected only the license key") + for c in pod.get("containers", []): + mounts = [m for m in c.get("volumeMounts", []) if m["name"] == "seaweedfs-license"] + if not mounts: + failed.append(f"{name}/{c['name']}: license volume not mounted") + continue + # subPath would freeze the file at container start. + if any("subPath" in m for m in mounts): + failed.append(f"{name}/{c['name']}: license mount uses subPath (renewals would not propagate)") + if not all(m.get("readOnly") for m in mounts): + failed.append(f"{name}/{c['name']}: license mount is not readOnly") + env = {e["name"]: e.get("value") for e in c.get("env", [])} + if env.get("SEAWEED_LICENSE") != LICENSE_PATH: + failed.append(f"{name}/{c['name']}: SEAWEED_LICENSE not set to the mounted file") + if carrying != {"test-seaweedfs-master"}: + failed.append(f"license: expected only the master to carry it, got {sorted(carrying)}") + else: + print("license: master only, key-scoped, readOnly, no subPath, SEAWEED_LICENSE set") + + # all-in-one runs `weed server -master`, so it needs the license too. + out = render({ + "allInOne.enabled": "true", + "master.enabled": "false", + "volume.enabled": "false", + "filer.enabled": "false", + "global.seaweedfs.license.existingSecret": "lic", + }) + aio = [d for d in workloads(out) if d["metadata"]["name"].endswith("-all-in-one")] + if len(aio) != 1: + failed.append(f"all-in-one: expected 1 workload, got {len(aio)}") + else: + pod = aio[0]["spec"]["template"]["spec"] + if not any(v.get("name") == "seaweedfs-license" for v in pod.get("volumes", [])): + failed.append("all-in-one: missing the license volume") + else: + print("all-in-one: carries the license") + + # SEAWEED_LICENSE is reserved on both env paths. + out = render({ + "global.seaweedfs.license.existingSecret": "lic", + "global.seaweedfs.extraEnvironmentVars.SEAWEED_LICENSE": "/tmp/bogus.json", + }) + for d in workloads(out): + if not d["metadata"]["name"].endswith("-master"): + continue + for c in d["spec"]["template"]["spec"].get("containers", []): + names = [e["name"] for e in c.get("env", [])] + if names.count("SEAWEED_LICENSE") != 1: + failed.append(f"SEAWEED_LICENSE rendered {names.count('SEAWEED_LICENSE')} times") + else: + value = next(e.get("value") for e in c["env"] if e["name"] == "SEAWEED_LICENSE") + if value != LICENSE_PATH: + failed.append(f"SEAWEED_LICENSE overridden to {value!r} by extraEnvironmentVars") + else: + print("SEAWEED_LICENSE: reserved, rendered once, points at the mount") + + # secretExtraEnvironmentVars is rendered outside the merge helper. + out = render({ + "allInOne.enabled": "true", + "master.enabled": "false", + "volume.enabled": "false", + "filer.enabled": "false", + "global.seaweedfs.license.existingSecret": "lic", + "allInOne.secretExtraEnvironmentVars.SEAWEED_LICENSE.secretKeyRef.name": "x", + "allInOne.secretExtraEnvironmentVars.SEAWEED_LICENSE.secretKeyRef.key": "y", + }) + for d in workloads(out): + if not d["metadata"]["name"].endswith("-all-in-one"): + continue + for c in d["spec"]["template"]["spec"].get("containers", []): + names = [e["name"] for e in c.get("env", [])] + if names.count("SEAWEED_LICENSE") != 1: + failed.append(f"all-in-one: SEAWEED_LICENSE rendered {names.count('SEAWEED_LICENSE')} times via secretExtraEnvironmentVars") + elif next(e.get("value") for e in c["env"] if e["name"] == "SEAWEED_LICENSE") != LICENSE_PATH: + failed.append("all-in-one: secretExtraEnvironmentVars overrode SEAWEED_LICENSE") + else: + print("all-in-one: SEAWEED_LICENSE reserved on the secret env path too") + + # Case 3: the default stays hostPath. volumeClaimTemplates is immutable, + # so growing one here would break helm upgrade on existing releases. + masters = [d for d in workloads(render({})) if d["metadata"]["name"].endswith("-master")] + if len(masters) != 1: + failed.append(f"master: expected 1 StatefulSet by default, got {len(masters)}") + else: + d = masters[0] + claims = [c["metadata"]["name"] for c in d["spec"].get("volumeClaimTemplates", [])] + if any(c.startswith("data-") for c in claims): + failed.append(f"master: default grew a data volumeClaimTemplate ({claims}); " + "that breaks helm upgrade on existing releases") + data_host = [v["name"] for v in d["spec"]["template"]["spec"].get("volumes", []) + if v.get("hostPath") and v["name"].startswith("data-")] + if not data_host: + failed.append("master: default no longer renders a hostPath data volume") + else: + print("master: default still hostPath (upgrade-compatible)") + + # Case 4: opting into a claim renders one, with the requested size. + masters = [d for d in workloads(render({"master.data.type": "persistentVolumeClaim"})) + if d["metadata"]["name"].endswith("-master")] + if len(masters) != 1: + failed.append(f"master.data.type=persistentVolumeClaim: expected 1 StatefulSet, got {len(masters)}") + else: + d = masters[0] + claims = {c["metadata"]["name"]: c["spec"]["resources"]["requests"]["storage"] + for c in d["spec"].get("volumeClaimTemplates", [])} + data = {k: v for k, v in claims.items() if k.startswith("data-")} + if not data: + failed.append(f"master.data.type=persistentVolumeClaim: no data claim rendered ({claims})") + elif set(data.values()) != {"1Gi"}: + failed.append(f"master.data.type=persistentVolumeClaim: unexpected size {data}") + else: + print(f"master.data.type=persistentVolumeClaim: renders {sorted(data)} at 1Gi") + if any(v.get("hostPath") and v["name"].startswith("data-") + for v in d["spec"]["template"]["spec"].get("volumes", [])): + failed.append("master.data.type=persistentVolumeClaim: data still on a hostPath") + + if failed: + print("\nFAIL:", file=sys.stderr) + for f in failed: + print(f" - {f}", file=sys.stderr) + sys.exit(1) + PYEOF + echo "License + master persistence tests passed" + + echo "All template rendering tests passed!" + + - name: Resolve an image tag that is published + run: | + set -e + # A release bumps appVersion on master ~40 minutes before the container + # build publishes that tag, and this workflow runs on the bump commit. + # Install the last released image for the length of that window instead + # of failing on ImagePullBackOff. + IMAGE=$(helm template test k8s/charts/seaweedfs \ + -s templates/master/master-statefulset.yaml | awk '$1 == "image:" {print $2; exit}') + REPO=${IMAGE%:*} + TAG=${IMAGE##*:} + # Anything but a published tag installs latest, which between releases + # is the same digest as the chart's own appVersion, so a registry blip + # costs nothing while failing the job on one would cost a red build. + STATUS=$(curl -sSL --connect-timeout 5 --max-time 15 -o /dev/null \ + -w '%{http_code}' "https://hub.docker.com/v2/repositories/$REPO/tags/$TAG" || true) + if [ "$STATUS" = 200 ]; then + echo "installing $IMAGE" + else + echo "$IMAGE is unavailable (HTTP ${STATUS:-none}), installing $REPO:latest" + TAG=latest + fi + echo "IMAGE_TAG=$TAG" >> $GITHUB_ENV + + - name: Create kind cluster + uses: helm/kind-action@v1.15.0 + + - name: Run chart-testing (install) + run: | + ct install --target-branch ${{ github.event.repository.default_branch }} --all --chart-dirs k8s/charts \ + --helm-extra-set-args "--set=image.tag=$IMAGE_TAG" + + - name: Verify SFTP host key secret lifecycle + run: | + set -e + CHART_DIR="k8s/charts/seaweedfs" + NS="sftp-hostkey" + SECRET="hk-seaweedfs-sftp-ssh-secret" + SFTP_ARGS="--set image.tag=$IMAGE_TAG --set sftp.enabled=true --set master.enabled=false --set volume.enabled=false --set filer.enabled=false" + kubectl create namespace "$NS" + + echo "=== install generates a host key, upgrade keeps it ===" + helm install hk $CHART_DIR -n "$NS" $SFTP_ARGS + KEY1=$(kubectl get secret "$SECRET" -n "$NS" -o jsonpath='{.data.ssh_host_ed25519_key}') + [ -n "$KEY1" ] || { echo "FAIL: install did not create a host key"; exit 1; } + echo "$KEY1" | base64 -d | grep -q "BEGIN PRIVATE KEY" || { echo "FAIL: host key is not a PEM private key"; exit 1; } + helm upgrade hk $CHART_DIR -n "$NS" $SFTP_ARGS + KEY2=$(kubectl get secret "$SECRET" -n "$NS" -o jsonpath='{.data.ssh_host_ed25519_key}') + [ "$KEY1" = "$KEY2" ] || { echo "FAIL: host key changed across upgrade"; exit 1; } + echo "host key survives upgrade" + + echo "=== the key the chart used to bundle is replaced ===" + kubectl delete secret "$SECRET" -n "$NS" + kubectl create secret generic "$SECRET" -n "$NS" \ + --from-literal=ssh_host_ed25519_key="stand-in H4McwcDphteXVullu6q7ephEN1N60z stand-in" + helm upgrade hk $CHART_DIR -n "$NS" $SFTP_ARGS + ROTATED=$(kubectl get secret "$SECRET" -n "$NS" -o jsonpath='{.data.ssh_host_ed25519_key}' | base64 -d) + case "$ROTATED" in + *H4McwcDphteXVullu6q7ephEN1N60z*) echo "FAIL: bundled key survived the upgrade"; exit 1;; + esac + echo "$ROTATED" | grep -q "BEGIN PRIVATE KEY" || { echo "FAIL: replacement is not a generated key"; exit 1; } + echo "bundled key rotated to a generated one" + + echo "=== operator-managed keys are kept as-is ===" + kubectl delete secret "$SECRET" -n "$NS" + ssh-keygen -q -t ed25519 -N "" -C "" -f /tmp/operator_key + kubectl create secret generic "$SECRET" -n "$NS" --from-file=my_key=/tmp/operator_key + helm upgrade hk $CHART_DIR -n "$NS" $SFTP_ARGS + kubectl get secret "$SECRET" -n "$NS" -o jsonpath='{.data.my_key}' | base64 -d | cmp -s - /tmp/operator_key \ + || { echo "FAIL: operator key was modified"; exit 1; } + NKEYS=$(kubectl get secret "$SECRET" -n "$NS" -o json | jq '.data | length') + [ "$NKEYS" = "1" ] || { echo "FAIL: expected only the operator key, found $NKEYS entries"; exit 1; } + echo "operator key kept, no extra key generated" + + kubectl delete namespace "$NS" + echo "SFTP host key lifecycle tests passed" + + - name: Verify install into a default-deny namespace + run: | + set -e + CHART_DIR="k8s/charts/seaweedfs" + NS="netpol" + + # kind enforces NetworkPolicy out of the box since v0.24 (kindnetd + # runs sigs.k8s.io/kube-network-policies), so the cluster created for + # chart-testing above is enough and no extra CNI is needed. The two + # probes at the end fail loudly if that ever stops being true, rather + # than letting this pass vacuously. + kubectl create namespace "$NS" + kubectl apply -n "$NS" -f - <<'EOF' + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: default-deny + spec: + podSelector: {} + policyTypes: [Ingress, Egress] + EOF + + # The backend address, not the kubernetes service ClusterIP: kube-proxy + # rewrites the destination, so an ipBlock has to name the real endpoint. + APISERVER=$(kubectl get endpointslice kubernetes \ + -o jsonpath='{.endpoints[0].addresses[0]}' 2>/dev/null || true) + if [ -z "$APISERVER" ]; then + APISERVER=$(kubectl get endpoints kubernetes -o jsonpath='{.subsets[0].addresses[0].ip}') + fi + echo "kube-apiserver at $APISERVER" + + echo "=== install with the policies on ===" + # Without them this hangs: the components cannot resolve or reach each + # other, and the post-install bucket hook waits on master and filer + # until it gives up. --wait covers the components, and helm fails the + # release if the hook Job does not finish, so a clean install is the + # assertion. + helm install np $CHART_DIR -n "$NS" --wait --timeout 8m \ + --set image.tag=$IMAGE_TAG \ + --set s3.enabled=true \ + --set s3.createBuckets[0].name=testbucket \ + --set networkPolicy.enabled=true \ + --set networkPolicy.egress.enabled=true \ + --set networkPolicy.egress.kubeApiServer.cidrs[0]="$APISERVER/32" + echo "release came up and the bucket hook finished under default-deny" + + FILER_IP=$(kubectl get pod -n "$NS" -l app.kubernetes.io/component=filer \ + -o jsonpath='{.items[0].status.podIP}') + echo "filer pod at $FILER_IP" + + # Both probes get their own all-egress policy, so the namespace-wide + # default-deny is not what decides the outcome: the only thing left in + # the way is the filer's own policy, which admits release pods only. + # Probing the pod IP keeps DNS out of it. + kubectl apply -n "$NS" -f - <<'EOF' + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: probe-egress + spec: + podSelector: + matchExpressions: + - key: probe + operator: Exists + policyTypes: [Egress] + egress: + - {} + EOF + + # Only the marker line, so a probe that never produced one cannot be + # mistaken for a verdict: "pod/x created" on stdout would read as + # "not exit=0", which is how the denied case passes. busybox wget + # reports a timeout on stderr even under -q and kubectl logs returns + # both streams, so the marker has to be picked out rather than taken + # as the whole log. Empty output stays empty - the callers fail on it. + probe() { + local name=$1 + local labels=$2 + kubectl run "$name" -n "$NS" --image=busybox:1.36 --restart=Never \ + --labels="probe=$name,$labels" --command -- \ + sh -c "wget -T 5 -q -O /dev/null http://$FILER_IP:8888/; echo exit=\$?" >/dev/null + kubectl wait -n "$NS" --for=jsonpath='{.status.phase}'=Succeeded \ + "pod/$name" --timeout=120s >/dev/null + kubectl logs -n "$NS" "$name" | grep '^exit=' || true + } + + # A pod carrying the release labels is what the filer's policy allows. + # This has to succeed, otherwise the policies are blocking traffic they + # are supposed to permit - or nothing is enforced and the next probe + # would be meaningless. + ALLOWED=$(probe probe-allowed "app.kubernetes.io/name=seaweedfs,app.kubernetes.io/instance=np") + echo "labelled probe: $ALLOWED" + case "$ALLOWED" in + exit=0) echo "a pod with the release labels reaches the filer";; + *) echo "FAIL: the filer policy rejects a pod carrying the release labels"; exit 1;; + esac + + # The same probe without those labels must not get through. Assert the + # wget failure rather than the absence of a success, so an empty log + # fails the job instead of reading as a denial. + DENIED=$(probe probe-denied "role=outsider") + echo "unlabelled probe: $DENIED" + case "$DENIED" in + exit=0) echo "FAIL: a pod outside the release reached the filer; the policy over-allows"; exit 1;; + exit=*) echo "the filer policy refuses a pod outside the release";; + *) echo "FAIL: the probe produced no result, so nothing was tested: '$DENIED'"; exit 1;; + esac + + kubectl delete namespace "$NS" --wait=false + echo "default-deny namespace tests passed" diff --git a/.github/workflows/helm_manual_release.yml b/.github/workflows/helm_manual_release.yml new file mode 100644 index 0000000..e4a2072 --- /dev/null +++ b/.github/workflows/helm_manual_release.yml @@ -0,0 +1,41 @@ +name: "helm: release" + +on: + push: + tags: + - '*' + workflow_dispatch: + +permissions: + contents: write + pages: write + packages: write + +jobs: + helm-release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Publish Helm charts to github pages + uses: stefanprodan/helm-gh-pages@v1.7.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + charts_dir: k8s/charts + target_dir: helm + branch: gh-pages + helm_version: "3.18.4" + + - name: Publish Helm charts to github container registry (ghcr.io) + uses: bitdeps/helm-oci-charts-releaser@v0.1.5 + with: + charts_dir: k8s/charts + github_token: ${{ secrets.GITHUB_TOKEN }} + oci_registry: ghcr.io/${{ github.repository_owner }} + oci_username: github-actions + oci_password: ${{ secrets.GITHUB_TOKEN }} + skip_dependencies: true + skip_helm_install: true + skip_gh_release: true diff --git a/.github/workflows/java_integration_tests.yml b/.github/workflows/java_integration_tests.yml new file mode 100644 index 0000000..075df5f --- /dev/null +++ b/.github/workflows/java_integration_tests.yml @@ -0,0 +1,193 @@ +name: Java Client Integration Tests + +on: + push: + branches: [ master ] + paths: + - 'other/java/**' + - 'weed/**' + - '.github/workflows/java_integration_tests.yml' + pull_request: + branches: [ master ] + paths: + - 'other/java/**' + - 'weed/**' + - '.github/workflows/java_integration_tests.yml' + +jobs: + test: + name: Java Integration Tests + runs-on: ubuntu-latest + + strategy: + matrix: + java: ['11', '17'] + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Set up Java + uses: actions/setup-java@v6 + with: + java-version: ${{ matrix.java }} + distribution: 'temurin' + cache: 'maven' + + - name: Build SeaweedFS + run: | + cd weed + go install -buildvcs=false + weed version + + - name: Start SeaweedFS Server + run: | + # Create clean data directory + export WEED_DATA_DIR="/tmp/seaweedfs-java-tests-$(date +%s)" + mkdir -p "$WEED_DATA_DIR" + + # Start SeaweedFS with optimized settings for CI + # Include S3 API for s3copier integration tests + weed server -dir="$WEED_DATA_DIR" \ + -master.raftHashicorp \ + -master.electionTimeout=1s \ + -master.volumeSizeLimitMB=100 \ + -volume.max=100 \ + -volume.preStopSeconds=1 \ + -master.peers=none \ + -filer -filer.maxMB=64 \ + -s3 -s3.port=8333 \ + -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" \ + -s3.allowDeleteBucketNotEmpty=true \ + -master.port=9333 \ + -volume.port=8080 \ + -filer.port=8888 \ + -metricsPort=9324 > seaweedfs.log 2>&1 & + + SERVER_PID=$! + echo "SERVER_PID=$SERVER_PID" >> $GITHUB_ENV + echo "WEED_DATA_DIR=$WEED_DATA_DIR" >> $GITHUB_ENV + echo "SeaweedFS server started with PID: $SERVER_PID" + + - name: Wait for SeaweedFS Components + run: | + echo "Waiting for SeaweedFS components to start..." + + # Wait for master + for i in {1..30}; do + if curl -s http://localhost:9333/cluster/status > /dev/null 2>&1; then + echo "✓ Master server is ready" + break + fi + echo "Waiting for master server... ($i/30)" + sleep 2 + done + + # Wait for volume + for i in {1..30}; do + if curl -s http://localhost:8080/status > /dev/null 2>&1; then + echo "✓ Volume server is ready" + break + fi + echo "Waiting for volume server... ($i/30)" + sleep 2 + done + + # Wait for filer + for i in {1..30}; do + if curl -s http://localhost:8888/ > /dev/null 2>&1; then + echo "✓ Filer is ready" + break + fi + echo "Waiting for filer... ($i/30)" + sleep 2 + done + + # Wait for S3 API + for i in {1..30}; do + if curl -s http://localhost:8333/healthz > /dev/null 2>&1; then + echo "✓ S3 API is ready" + break + fi + echo "Waiting for S3 API... ($i/30)" + sleep 2 + done + + echo "✓ All SeaweedFS components are ready!" + + # Display cluster status + echo "Cluster status:" + curl -s http://localhost:9333/cluster/status | head -20 + + - name: Build and Install SeaweedFS Client + working-directory: other/java/client + run: | + mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true + + - name: Run Client Unit Tests + working-directory: other/java/client + run: | + mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest + + - name: Run Client Integration Tests + working-directory: other/java/client + env: + SEAWEEDFS_TEST_ENABLED: true + run: | + mvn test -Dtest=*IntegrationTest + + - name: Run HDFS3 Configuration Tests + working-directory: other/java/hdfs3 + run: | + mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true + + - name: Run S3 ETag Validation Tests (Issue #7768) + working-directory: other/java/s3copier + env: + S3_ENDPOINT: http://127.0.0.1:8333 + S3_ACCESS_KEY: some_access_key1 + S3_SECRET_KEY: some_secret_key1 + run: | + echo "Running S3 ETag validation tests against $S3_ENDPOINT" + mvn test -Dtest=ETagValidationTest \ + -DS3_ENDPOINT=$S3_ENDPOINT \ + -DS3_ACCESS_KEY=$S3_ACCESS_KEY \ + -DS3_SECRET_KEY=$S3_SECRET_KEY \ + -Dmaven.javadoc.skip=true -Dgpg.skip=true + + - name: Display logs on failure + if: failure() + run: | + echo "=== SeaweedFS Server Log ===" + tail -100 seaweedfs.log || echo "No server log" + echo "" + echo "=== Cluster Status ===" + curl -s http://localhost:9333/cluster/status || echo "Cannot reach cluster" + echo "" + echo "=== Process Status ===" + ps aux | grep weed || echo "No weed processes" + + - name: Cleanup + if: always() + run: | + # Stop server using stored PID + if [ -n "$SERVER_PID" ]; then + echo "Stopping SeaweedFS server (PID: $SERVER_PID)" + kill -9 $SERVER_PID 2>/dev/null || true + fi + + # Fallback: kill any remaining weed processes + pkill -f "weed server" || true + + # Clean up data directory + if [ -n "$WEED_DATA_DIR" ]; then + echo "Cleaning up data directory: $WEED_DATA_DIR" + rm -rf "$WEED_DATA_DIR" || true + fi + diff --git a/.github/workflows/java_release.yml b/.github/workflows/java_release.yml new file mode 100644 index 0000000..282c562 --- /dev/null +++ b/.github/workflows/java_release.yml @@ -0,0 +1,183 @@ +name: "release: java clients" + +# Publishes the SeaweedFS Java clients (seaweedfs-client and +# seaweedfs-hadoop3-client) to Maven Central via the Sonatype Central Portal. +# +# Required repository secrets: +# MAVEN_CENTRAL_USERNAME - Central Portal user token username (central.sonatype.com -> Account -> Generate User Token) +# MAVEN_CENTRAL_PASSWORD - Central Portal user token password +# MAVEN_GPG_PRIVATE_KEY - ASCII-armored signing key: +# gpg --homedir --armor --export-secret-keys > key.asc +# MAVEN_GPG_PASSPHRASE - passphrase for that key +# +# A plain run publishes to Central. Check dry_run to exercise secrets, GPG +# signing, and the build without committing or publishing. + +on: + workflow_dispatch: + inputs: + version: + description: "Version to publish, e.g. 4.39. Leave blank to use the current SeaweedFS version." + type: string + required: false + dry_run: + description: "Build and GPG-sign only; skip the version commit and the Central upload" + type: boolean + default: false + +permissions: + contents: write + +concurrency: + group: java-release + cancel-in-progress: false + +env: + # Java 17 needs these opens for the Maven publishing plugins' reflective access. + MAVEN_OPTS: >- + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.text=ALL-UNNAMED + --add-opens=java.desktop/java.awt.font=ALL-UNNAMED + +jobs: + publish: + name: Publish to Maven Central + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + # Full history so the version-bump commit can rebase onto a moved master. + fetch-depth: 0 + + # An explicit input wins; otherwise fall back to the current SeaweedFS + # version (MAJOR.MINOR) from constants.go, matching its %d.%02d formatting. + - name: Resolve version + id: resolve + env: + INPUT_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + VERSION="$INPUT_VERSION" + if [ -z "$VERSION" ]; then + CONST=weed/util/version/constants.go + MAJOR=$(grep -oP 'MAJOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST") + MINOR=$(grep -oP 'MINOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST") + VERSION=$(printf '%d.%02d' "$MAJOR" "$MINOR") + echo "No version given; using current SeaweedFS version ${VERSION}." + fi + if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+$ ]]; then + echo "::error::version must be MAJOR.MINOR, e.g. 4.39 (got '$VERSION')" + exit 1 + fi + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + + - name: Set up JDK 17 + uses: actions/setup-java@v6 + with: + java-version: '17' + distribution: 'temurin' + cache: 'maven' + server-id: central + server-username: MAVEN_CENTRAL_USERNAME + server-password: MAVEN_CENTRAL_PASSWORD + gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + + # client carries the version literally; hdfs3 derives its own version and + # its seaweedfs-client dependency from the seaweedfs.client.version property. + - name: Set versions in poms + env: + VERSION: ${{ steps.resolve.outputs.version }} + run: | + set -euo pipefail + V=org.codehaus.mojo:versions-maven-plugin:2.16.2 + mvn -B -ntp -f other/java/client/pom.xml "${V}:set" \ + -DnewVersion="$VERSION" -DgenerateBackupPoms=false + mvn -B -ntp -f other/java/hdfs3/pom.xml "${V}:set-property" \ + -Dproperty=seaweedfs.client.version -DnewVersion="$VERSION" -DgenerateBackupPoms=false + + - name: Commit version bump + if: ${{ !inputs.dry_run }} + env: + VERSION: ${{ steps.resolve.outputs.version }} + run: | + set -euo pipefail + if git diff --quiet; then + echo "Poms already at ${VERSION}; nothing to commit." + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add other/java/client/pom.xml other/java/hdfs3/pom.xml + git commit -m "java ${VERSION}" + # Rebase and retry so a concurrent push to master doesn't lose the bump. + for attempt in 1 2 3 4 5; do + if git push origin HEAD:master; then + exit 0 + fi + echo "push rejected (attempt ${attempt}); rebasing onto latest master" + git pull --rebase origin master + done + echo "::error::could not push version bump after retries" + exit 1 + + # Tests are skipped here: the integration tests need a live filer, and the + # offline unit tests already run on every push in java_unit_tests.yml. + # dry_run stops at `install` (build + GPG sign, no upload). A real run deploys. + # client goes first either way so its install populates the local repo for hdfs3. + - name: Publish seaweedfs-client + working-directory: other/java/client + env: + MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }} + MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + set -euo pipefail + [ "$DRY_RUN" = "true" ] && GOAL="clean install" || GOAL="clean deploy" + mvn -B -ntp $GOAL -DskipTests + + - name: Publish seaweedfs-hadoop3-client + working-directory: other/java/hdfs3 + env: + MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }} + MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + set -euo pipefail + [ "$DRY_RUN" = "true" ] && GOAL="clean install" || GOAL="clean deploy" + mvn -B -ntp $GOAL -DskipTests + + # Bump every version the wiki quotes for the Java clients. Runs only after + # a successful real publish. Uses GITHUB_TOKEN; set a WIKI_TOKEN secret if + # the token can't push to the wiki. + - name: Update wiki client versions + if: ${{ !inputs.dry_run }} + env: + VERSION: ${{ steps.resolve.outputs.version }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WIKI_TOKEN: ${{ secrets.WIKI_TOKEN }} + run: | + set -euo pipefail + TOKEN="${WIKI_TOKEN:-$GH_TOKEN}" + git clone -q "https://x-access-token:${TOKEN}@github.com/${{ github.repository }}.wiki.git" wiki + cd wiki + for f in $(grep -rlE "seaweedfs-(client|hadoop3-client)" --include="*.md" .); do + NEWV="$VERSION" perl -0777 -i -pe ' + my $v = $ENV{NEWV}; + s{(seaweedfs-(?:client|hadoop3-client)\s*)[0-9]+\.[0-9]+()}{$1$v$2}g; + s{(seaweedfs-(?:client|hadoop3-client):)[0-9]+\.[0-9]+}{$1$v}g; + s{(seaweedfs-(?:client|hadoop3-client)-)[0-9]+\.[0-9]+(\.jar)}{$1$v$2}g; + s{(seaweedfs-(?:client|hadoop3-client)/)[0-9]+\.[0-9]+(/)}{$1$v$2}g; + ' "$f" + done + if git diff --quiet; then + echo "Wiki already references ${VERSION}." + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -am "java clients ${VERSION}" + git push diff --git a/.github/workflows/java_unit_tests.yml b/.github/workflows/java_unit_tests.yml new file mode 100644 index 0000000..753d170 --- /dev/null +++ b/.github/workflows/java_unit_tests.yml @@ -0,0 +1,58 @@ +name: Java Client Unit Tests + +on: + push: + branches: [ master ] + paths: + - 'other/java/**' + - '.github/workflows/java_unit_tests.yml' + pull_request: + branches: [ master ] + paths: + - 'other/java/**' + - '.github/workflows/java_unit_tests.yml' + +jobs: + test: + name: Java Unit Tests + runs-on: ubuntu-latest + + strategy: + matrix: + java: ['8', '11', '17', '21'] + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Java + uses: actions/setup-java@v6 + with: + java-version: ${{ matrix.java }} + distribution: 'temurin' + cache: 'maven' + + - name: Build and Install SeaweedFS Client + working-directory: other/java/client + run: | + mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true + + - name: Run Client Unit Tests + working-directory: other/java/client + run: | + mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest + + - name: Run HDFS3 Configuration Tests + working-directory: other/java/hdfs3 + run: | + mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true + + - name: Upload Test Reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: test-reports-java-${{ matrix.java }} + path: | + other/java/client/target/surefire-reports/ + other/java/hdfs3/target/surefire-reports/ + diff --git a/.github/workflows/kafka-quicktest.yml b/.github/workflows/kafka-quicktest.yml new file mode 100644 index 0000000..c18a870 --- /dev/null +++ b/.github/workflows/kafka-quicktest.yml @@ -0,0 +1,145 @@ +name: "Kafka Quick Test (Load Test with Schema Registry)" + +on: + push: + branches: [ master ] + paths: + - 'weed/mq/**' + - 'weed/pb/mq_pb/**' + - 'weed/pb/schema_pb/**' + - 'test/kafka/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/kafka-quicktest.yml' + pull_request: + branches: [ master ] + paths: + - 'weed/mq/**' + - 'weed/pb/mq_pb/**' + - 'weed/pb/schema_pb/**' + - 'test/kafka/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/kafka-quicktest.yml' + workflow_dispatch: # Allow manual trigger + +concurrency: + group: ${{ github.head_ref }}/kafka-quicktest + cancel-in-progress: true + +permissions: + contents: read + +jobs: + kafka-client-quicktest: + name: Kafka Client Load Test (Quick) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version: ^1.26 + cache: true + cache-dependency-path: | + **/go.sum + id: go + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Install dependencies + run: | + # Ensure make is available + sudo apt-get update -qq + sudo apt-get install -y make + + - name: Validate test setup + working-directory: test/kafka/kafka-client-loadtest + run: | + make validate-setup + + - name: Run quick-test + working-directory: test/kafka/kafka-client-loadtest + run: | + # Run the quick-test target which includes: + # 1. Building the gateway + # 2. Starting all services (SeaweedFS, MQ broker, Schema Registry) + # 3. Registering Avro schemas + # 4. Running a 1-minute load test with Avro messages + # Override GOARCH to build for AMD64 (GitHub Actions runners are x86_64) + GOARCH=amd64 make quick-test + env: + # Docker Compose settings + COMPOSE_HTTP_TIMEOUT: 300 + DOCKER_CLIENT_TIMEOUT: 300 + # Test parameters (set by quick-test, but can override) + TEST_DURATION: 60s + PRODUCER_COUNT: 1 + CONSUMER_COUNT: 1 + MESSAGE_RATE: 10 + VALUE_TYPE: avro + + - name: Show test results + if: always() + working-directory: test/kafka/kafka-client-loadtest + run: | + echo "=========================================" + echo "Test Results" + echo "=========================================" + make show-results || echo "Could not retrieve results" + + - name: Show service logs on failure + if: failure() + working-directory: test/kafka/kafka-client-loadtest + run: | + echo "=========================================" + echo "Service Logs" + echo "=========================================" + + echo "Checking running containers..." + docker compose ps || true + + echo "=========================================" + echo "Master Logs" + echo "=========================================" + docker compose logs --tail=100 seaweedfs-master 2>&1 || echo "No master logs available" + + echo "=========================================" + echo "MQ Broker Logs (Last 100 lines)" + echo "=========================================" + docker compose logs --tail=100 seaweedfs-mq-broker 2>&1 || echo "No broker logs available" + + echo "=========================================" + echo "Kafka Gateway Logs (FULL - Critical for debugging)" + echo "=========================================" + docker compose logs kafka-gateway 2>&1 || echo "ERROR: Could not retrieve kafka-gateway logs" + + echo "=========================================" + echo "Schema Registry Logs (FULL)" + echo "=========================================" + docker compose logs schema-registry 2>&1 || echo "ERROR: Could not retrieve schema-registry logs" + + echo "=========================================" + echo "Load Test Logs" + echo "=========================================" + docker compose logs --tail=100 kafka-client-loadtest 2>&1 || echo "No loadtest logs available" + + - name: Cleanup + if: always() + working-directory: test/kafka/kafka-client-loadtest + run: | + # Stop containers first + docker compose --profile loadtest --profile monitoring down -v --remove-orphans || true + # Clean up data with sudo to handle Docker root-owned files + sudo rm -rf data/* || true + # Clean up binary + rm -f weed-linux-* || true diff --git a/.github/workflows/kafka-tests.yml b/.github/workflows/kafka-tests.yml new file mode 100644 index 0000000..7a40ee3 --- /dev/null +++ b/.github/workflows/kafka-tests.yml @@ -0,0 +1,841 @@ +name: "Kafka Gateway Tests" + +on: + push: + branches: [ master ] + paths: + - 'weed/mq/**' + - 'weed/pb/mq_pb/**' + - 'weed/pb/schema_pb/**' + - 'test/kafka/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/kafka-tests.yml' + pull_request: + branches: [ master ] + paths: + - 'weed/mq/**' + - 'weed/pb/mq_pb/**' + - 'weed/pb/schema_pb/**' + - 'test/kafka/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/kafka-tests.yml' + +concurrency: + group: ${{ github.head_ref }}/kafka-tests + cancel-in-progress: true + +# Force different runners for better isolation +env: + FORCE_RUNNER_SEPARATION: true + +permissions: + contents: read + +jobs: + kafka-unit-tests: + name: Kafka Unit Tests + runs-on: ubuntu-latest + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + container-id: [unit-tests-1] + container: + image: golang:1.26-alpine + options: --cpus 1.0 --memory 1g --hostname kafka-unit-${{ matrix.container-id }} + env: + GOMAXPROCS: 1 + CGO_ENABLED: 0 + CONTAINER_ID: ${{ matrix.container-id }} + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version: ^1.26 + id: go + + - name: Check out code + uses: actions/checkout@v7 + + - name: Setup Container Environment + run: | + apk add --no-cache git + ulimit -n 1024 || echo "Warning: Could not set file descriptor limit" + + - name: Get dependencies + run: | + cd test/kafka + go mod download + go mod tidy + + - name: Run Kafka Gateway Unit Tests + run: | + cd test/kafka + # Set process limits for container isolation + ulimit -n 512 || echo "Warning: Could not set file descriptor limit" + ulimit -u 100 || echo "Warning: Could not set process limit" + go test -v -timeout 10s ./unit/... + + kafka-integration-tests: + name: Kafka Integration Tests (Critical) + runs-on: ubuntu-latest + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + container-id: [integration-1] + container: + image: golang:1.26-alpine + options: --cpus 2.0 --memory 2g --ulimit nofile=1024:1024 --hostname kafka-integration-${{ matrix.container-id }} + env: + GOMAXPROCS: 2 + CGO_ENABLED: 0 + KAFKA_TEST_ISOLATION: "true" + CONTAINER_ID: ${{ matrix.container-id }} + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version: ^1.26 + id: go + + - name: Check out code + uses: actions/checkout@v7 + + - name: Setup Integration Container Environment + run: | + apk add --no-cache git procps + ulimit -n 2048 || echo "Warning: Could not set file descriptor limit" + + - name: Get dependencies + run: | + cd test/kafka + go mod download + go mod tidy + + - name: Run Integration Tests + run: | + cd test/kafka + # Higher limits for integration tests + ulimit -n 1024 || echo "Warning: Could not set file descriptor limit" + ulimit -u 200 || echo "Warning: Could not set process limit" + go test -v -timeout 90s ./integration/... + env: + GOMAXPROCS: 2 + + kafka-e2e-tests: + name: Kafka End-to-End Tests (with SMQ) + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + container-id: [e2e-1] + container: + image: golang:1.26-alpine + options: --cpus 2.0 --memory 2g --hostname kafka-e2e-${{ matrix.container-id }} + env: + GOMAXPROCS: 2 + CGO_ENABLED: 0 + KAFKA_E2E_ISOLATION: "true" + CONTAINER_ID: ${{ matrix.container-id }} + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version: ^1.26 + cache: true + cache-dependency-path: | + **/go.sum + id: go + + - name: Setup E2E Container Environment + run: | + apk add --no-cache git procps curl netcat-openbsd + ulimit -n 2048 || echo "Warning: Could not set file descriptor limit" + + - name: Warm Go module cache + run: | + # Warm cache for root module + go mod download || true + # Warm cache for kafka test module + cd test/kafka + go mod download || true + + - name: Get dependencies + run: | + cd test/kafka + # Use go mod download with timeout to prevent hanging + timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules" + go mod tidy + + - name: Build and start SeaweedFS MQ + run: | + set -e + cd $GITHUB_WORKSPACE + # Build weed binary + go build -o /usr/local/bin/weed ./weed + # Start SeaweedFS components with MQ brokers + export WEED_DATA_DIR=/tmp/seaweedfs-e2e-$RANDOM + mkdir -p "$WEED_DATA_DIR" + + # Start SeaweedFS server (master, volume, filer) with consistent IP advertising + nohup weed -v 1 server \ + -ip="127.0.0.1" \ + -ip.bind="0.0.0.0" \ + -dir="$WEED_DATA_DIR" \ + -master.raftHashicorp \ + -master.port=9333 \ + -volume.port=8081 \ + -filer.port=8888 \ + -filer=true \ + -metricsPort=9325 \ + -master.peers=none \ + > /tmp/weed-server.log 2>&1 & + + # Wait for master to be ready + for i in $(seq 1 30); do + if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then + echo "SeaweedFS master HTTP is up"; break + fi + echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1 + done + + # Wait for master gRPC to be ready (this is what broker discovery uses) + echo "Waiting for master gRPC port..." + for i in $(seq 1 30); do + if nc -z 127.0.0.1 19333; then + echo "✓ SeaweedFS master gRPC is up (port 19333)" + break + fi + echo " Waiting for master gRPC... ($i/30)"; sleep 1 + done + + # Give server time to initialize all components including gRPC services + echo "Waiting for SeaweedFS components to initialize..." + sleep 15 + + # Additional wait specifically for gRPC services to be ready for streaming + echo "Allowing extra time for master gRPC streaming services to initialize..." + sleep 10 + + # Start MQ broker with maximum verbosity for debugging + echo "Starting MQ broker..." + nohup weed -v 3 mq.broker \ + -master="127.0.0.1:9333" \ + -ip="127.0.0.1" \ + -port=17777 \ + -logFlushInterval=0 \ + > /tmp/weed-mq-broker.log 2>&1 & + + # Wait for broker to be ready with better error reporting + sleep 15 + broker_ready=false + for i in $(seq 1 20); do + if nc -z 127.0.0.1 17777; then + echo "SeaweedFS MQ broker is up" + broker_ready=true + break + fi + echo "Waiting for MQ broker... ($i/20)"; sleep 1 + done + + # Give broker additional time to register with master + if [ "$broker_ready" = true ]; then + echo "Allowing broker to register with master..." + sleep 30 + + # Check if broker is properly registered by querying cluster nodes + echo "Cluster status after broker registration:" + curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status" + + echo "Checking cluster topology (includes registered components):" + curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status" + + echo "Verifying broker discovery via master client debug:" + echo "If broker registration is successful, it should appear in dir status" + + echo "Testing gRPC connectivity with weed binary:" + echo "This simulates what the gateway does during broker discovery..." + timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..." + echo "Shell test results:" + cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs" + fi + + # Check if broker failed to start and show logs + if [ "$broker_ready" = false ]; then + echo "ERROR: MQ broker failed to start. Broker logs:" + cat /tmp/weed-mq-broker.log || echo "No broker logs found" + echo "Server logs:" + tail -20 /tmp/weed-server.log || echo "No server logs found" + exit 1 + fi + + - name: Run End-to-End Tests + run: | + cd test/kafka + # Higher limits for E2E tests + ulimit -n 1024 || echo "Warning: Could not set file descriptor limit" + ulimit -u 200 || echo "Warning: Could not set process limit" + + # Allow additional time for all background processes to settle + echo "Allowing additional settlement time for SeaweedFS ecosystem..." + sleep 15 + + # Run tests and capture result + if ! go test -v -timeout 180s ./e2e/...; then + echo "=========================================" + echo "Tests failed! Showing debug information:" + echo "=========================================" + echo "Server logs (last 50 lines):" + tail -50 /tmp/weed-server.log || echo "No server logs" + echo "=========================================" + echo "Broker logs (last 50 lines):" + tail -50 /tmp/weed-mq-broker.log || echo "No broker logs" + echo "=========================================" + exit 1 + fi + env: + GOMAXPROCS: 2 + SEAWEEDFS_MASTERS: 127.0.0.1:9333 + + kafka-consumer-group-tests: + name: Kafka Consumer Group Tests (Highly Isolated) + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + container-id: [consumer-group-1] + container: + image: golang:1.26-alpine + options: --cpus 1.0 --memory 2g --ulimit nofile=512:512 --hostname kafka-consumer-${{ matrix.container-id }} + env: + GOMAXPROCS: 1 + CGO_ENABLED: 0 + KAFKA_CONSUMER_ISOLATION: "true" + CONTAINER_ID: ${{ matrix.container-id }} + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version: ^1.26 + cache: true + cache-dependency-path: | + **/go.sum + id: go + + - name: Setup Consumer Group Container Environment + run: | + apk add --no-cache git procps curl netcat-openbsd + ulimit -n 256 || echo "Warning: Could not set file descriptor limit" + + - name: Warm Go module cache + run: | + # Warm cache for root module + go mod download || true + # Warm cache for kafka test module + cd test/kafka + go mod download || true + + - name: Get dependencies + run: | + cd test/kafka + # Use go mod download with timeout to prevent hanging + timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules" + go mod tidy + + - name: Build and start SeaweedFS MQ + run: | + set -e + cd $GITHUB_WORKSPACE + # Build weed binary + go build -o /usr/local/bin/weed ./weed + # Start SeaweedFS components with MQ brokers + export WEED_DATA_DIR=/tmp/seaweedfs-mq-$RANDOM + mkdir -p "$WEED_DATA_DIR" + + # Start SeaweedFS server (master, volume, filer) with consistent IP advertising + nohup weed -v 1 server \ + -ip="127.0.0.1" \ + -ip.bind="0.0.0.0" \ + -dir="$WEED_DATA_DIR" \ + -master.raftHashicorp \ + -master.port=9333 \ + -volume.port=8081 \ + -filer.port=8888 \ + -filer=true \ + -metricsPort=9325 \ + -master.peers=none \ + > /tmp/weed-server.log 2>&1 & + + # Wait for master to be ready + for i in $(seq 1 30); do + if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then + echo "SeaweedFS master HTTP is up"; break + fi + echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1 + done + + # Wait for master gRPC to be ready (this is what broker discovery uses) + echo "Waiting for master gRPC port..." + for i in $(seq 1 30); do + if nc -z 127.0.0.1 19333; then + echo "✓ SeaweedFS master gRPC is up (port 19333)" + break + fi + echo " Waiting for master gRPC... ($i/30)"; sleep 1 + done + + # Give server time to initialize all components including gRPC services + echo "Waiting for SeaweedFS components to initialize..." + sleep 15 + + # Additional wait specifically for gRPC services to be ready for streaming + echo "Allowing extra time for master gRPC streaming services to initialize..." + sleep 10 + + # Start MQ broker with maximum verbosity for debugging + echo "Starting MQ broker..." + nohup weed -v 3 mq.broker \ + -master="127.0.0.1:9333" \ + -ip="127.0.0.1" \ + -port=17777 \ + -logFlushInterval=0 \ + > /tmp/weed-mq-broker.log 2>&1 & + + # Wait for broker to be ready with better error reporting + sleep 15 + broker_ready=false + for i in $(seq 1 20); do + if nc -z 127.0.0.1 17777; then + echo "SeaweedFS MQ broker is up" + broker_ready=true + break + fi + echo "Waiting for MQ broker... ($i/20)"; sleep 1 + done + + # Give broker additional time to register with master + if [ "$broker_ready" = true ]; then + echo "Allowing broker to register with master..." + sleep 30 + + # Check if broker is properly registered by querying cluster nodes + echo "Cluster status after broker registration:" + curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status" + + echo "Checking cluster topology (includes registered components):" + curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status" + + echo "Verifying broker discovery via master client debug:" + echo "If broker registration is successful, it should appear in dir status" + + echo "Testing gRPC connectivity with weed binary:" + echo "This simulates what the gateway does during broker discovery..." + timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..." + echo "Shell test results:" + cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs" + fi + + # Check if broker failed to start and show logs + if [ "$broker_ready" = false ]; then + echo "ERROR: MQ broker failed to start. Broker logs:" + cat /tmp/weed-mq-broker.log || echo "No broker logs found" + echo "Server logs:" + tail -20 /tmp/weed-server.log || echo "No server logs found" + exit 1 + fi + + - name: Run Consumer Group Tests + run: | + cd test/kafka + # Test consumer group functionality with explicit timeout + ulimit -n 512 || echo "Warning: Could not set file descriptor limit" + ulimit -u 100 || echo "Warning: Could not set process limit" + timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/... + env: + GOMAXPROCS: 1 + SEAWEEDFS_MASTERS: 127.0.0.1:9333 + + kafka-client-compatibility: + name: Kafka Client Compatibility (with SMQ) + runs-on: ubuntu-latest + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + container-id: [client-compat-1] + container: + image: golang:1.26-alpine + options: --cpus 1.0 --memory 1.5g --shm-size 256m --hostname kafka-client-${{ matrix.container-id }} + env: + GOMAXPROCS: 1 + CGO_ENABLED: 0 + KAFKA_CLIENT_ISOLATION: "true" + CONTAINER_ID: ${{ matrix.container-id }} + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version: ^1.26 + cache: true + cache-dependency-path: | + **/go.sum + id: go + + - name: Setup Client Container Environment + run: | + apk add --no-cache git procps curl netcat-openbsd + ulimit -n 1024 || echo "Warning: Could not set file descriptor limit" + + - name: Warm Go module cache + run: | + # Warm cache for root module + go mod download || true + # Warm cache for kafka test module + cd test/kafka + go mod download || true + + - name: Get dependencies + run: | + cd test/kafka + timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules" + go mod tidy + + - name: Build and start SeaweedFS MQ + run: | + set -e + cd $GITHUB_WORKSPACE + # Build weed binary + go build -o /usr/local/bin/weed ./weed + # Start SeaweedFS components with MQ brokers + export WEED_DATA_DIR=/tmp/seaweedfs-client-$RANDOM + mkdir -p "$WEED_DATA_DIR" + + # Start SeaweedFS server (master, volume, filer) with consistent IP advertising + nohup weed -v 1 server \ + -ip="127.0.0.1" \ + -ip.bind="0.0.0.0" \ + -dir="$WEED_DATA_DIR" \ + -master.raftHashicorp \ + -master.port=9333 \ + -volume.port=8081 \ + -filer.port=8888 \ + -filer=true \ + -metricsPort=9325 \ + -master.peers=none \ + > /tmp/weed-server.log 2>&1 & + + # Wait for master to be ready + for i in $(seq 1 30); do + if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then + echo "SeaweedFS master HTTP is up"; break + fi + echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1 + done + + # Wait for master gRPC to be ready (this is what broker discovery uses) + echo "Waiting for master gRPC port..." + for i in $(seq 1 30); do + if nc -z 127.0.0.1 19333; then + echo "✓ SeaweedFS master gRPC is up (port 19333)" + break + fi + echo " Waiting for master gRPC... ($i/30)"; sleep 1 + done + + # Give server time to initialize all components including gRPC services + echo "Waiting for SeaweedFS components to initialize..." + sleep 15 + + # Additional wait specifically for gRPC services to be ready for streaming + echo "Allowing extra time for master gRPC streaming services to initialize..." + sleep 10 + + # Start MQ broker with maximum verbosity for debugging + echo "Starting MQ broker..." + nohup weed -v 3 mq.broker \ + -master="127.0.0.1:9333" \ + -ip="127.0.0.1" \ + -port=17777 \ + -logFlushInterval=0 \ + > /tmp/weed-mq-broker.log 2>&1 & + + # Wait for broker to be ready with better error reporting + sleep 15 + broker_ready=false + for i in $(seq 1 20); do + if nc -z 127.0.0.1 17777; then + echo "SeaweedFS MQ broker is up" + broker_ready=true + break + fi + echo "Waiting for MQ broker... ($i/20)"; sleep 1 + done + + # Give broker additional time to register with master + if [ "$broker_ready" = true ]; then + echo "Allowing broker to register with master..." + sleep 30 + + # Check if broker is properly registered by querying cluster nodes + echo "Cluster status after broker registration:" + curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status" + + echo "Checking cluster topology (includes registered components):" + curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status" + + echo "Verifying broker discovery via master client debug:" + echo "If broker registration is successful, it should appear in dir status" + + echo "Testing gRPC connectivity with weed binary:" + echo "This simulates what the gateway does during broker discovery..." + timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..." + echo "Shell test results:" + cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs" + fi + + # Check if broker failed to start and show logs + if [ "$broker_ready" = false ]; then + echo "ERROR: MQ broker failed to start. Broker logs:" + cat /tmp/weed-mq-broker.log || echo "No broker logs found" + echo "Server logs:" + tail -20 /tmp/weed-server.log || echo "No server logs found" + exit 1 + fi + + - name: Run Client Compatibility Tests + run: | + cd test/kafka + go test -v -run "^TestClientCompatibility" -timeout 180s ./integration/... + env: + GOMAXPROCS: 1 + SEAWEEDFS_MASTERS: 127.0.0.1:9333 + + kafka-smq-integration-tests: + name: Kafka SMQ Integration Tests (Full Stack) + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + container-id: [smq-integration-1] + container: + image: golang:1.26-alpine + options: --cpus 1.0 --memory 2g --hostname kafka-smq-${{ matrix.container-id }} + env: + GOMAXPROCS: 1 + CGO_ENABLED: 0 + KAFKA_SMQ_INTEGRATION: "true" + CONTAINER_ID: ${{ matrix.container-id }} + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version: ^1.26 + cache: true + cache-dependency-path: | + **/go.sum + id: go + + - name: Setup SMQ Integration Container Environment + run: | + apk add --no-cache git procps curl netcat-openbsd + ulimit -n 1024 || echo "Warning: Could not set file descriptor limit" + + - name: Warm Go module cache + run: | + # Warm cache for root module + go mod download || true + # Warm cache for kafka test module + cd test/kafka + go mod download || true + + - name: Get dependencies + run: | + cd test/kafka + timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules" + go mod tidy + + - name: Build and start SeaweedFS MQ + run: | + set -e + cd $GITHUB_WORKSPACE + # Build weed binary + go build -o /usr/local/bin/weed ./weed + # Start SeaweedFS components with MQ brokers + export WEED_DATA_DIR=/tmp/seaweedfs-smq-$RANDOM + mkdir -p "$WEED_DATA_DIR" + + # Start SeaweedFS server (master, volume, filer) with consistent IP advertising + nohup weed -v 1 server \ + -ip="127.0.0.1" \ + -ip.bind="0.0.0.0" \ + -dir="$WEED_DATA_DIR" \ + -master.raftHashicorp \ + -master.port=9333 \ + -volume.port=8081 \ + -filer.port=8888 \ + -filer=true \ + -metricsPort=9325 \ + -master.peers=none \ + > /tmp/weed-server.log 2>&1 & + + # Wait for master to be ready + for i in $(seq 1 30); do + if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then + echo "SeaweedFS master HTTP is up"; break + fi + echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1 + done + + # Wait for master gRPC to be ready (this is what broker discovery uses) + echo "Waiting for master gRPC port..." + for i in $(seq 1 30); do + if nc -z 127.0.0.1 19333; then + echo "✓ SeaweedFS master gRPC is up (port 19333)" + break + fi + echo " Waiting for master gRPC... ($i/30)"; sleep 1 + done + + # Give server time to initialize all components including gRPC services + echo "Waiting for SeaweedFS components to initialize..." + sleep 15 + + # Additional wait specifically for gRPC services to be ready for streaming + echo "Allowing extra time for master gRPC streaming services to initialize..." + sleep 10 + + # Start MQ broker with maximum verbosity for debugging + echo "Starting MQ broker..." + nohup weed -v 3 mq.broker \ + -master="127.0.0.1:9333" \ + -ip="127.0.0.1" \ + -port=17777 \ + -logFlushInterval=0 \ + > /tmp/weed-mq-broker.log 2>&1 & + + # Wait for broker to be ready with better error reporting + sleep 15 + broker_ready=false + for i in $(seq 1 20); do + if nc -z 127.0.0.1 17777; then + echo "SeaweedFS MQ broker is up" + broker_ready=true + break + fi + echo "Waiting for MQ broker... ($i/20)"; sleep 1 + done + + # Give broker additional time to register with master + if [ "$broker_ready" = true ]; then + echo "Allowing broker to register with master..." + sleep 30 + + # Check if broker is properly registered by querying cluster nodes + echo "Cluster status after broker registration:" + curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status" + + echo "Checking cluster topology (includes registered components):" + curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status" + + echo "Verifying broker discovery via master client debug:" + echo "If broker registration is successful, it should appear in dir status" + + echo "Testing gRPC connectivity with weed binary:" + echo "This simulates what the gateway does during broker discovery..." + timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..." + echo "Shell test results:" + cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs" + fi + + # Check if broker failed to start and show logs + if [ "$broker_ready" = false ]; then + echo "ERROR: MQ broker failed to start. Broker logs:" + cat /tmp/weed-mq-broker.log || echo "No broker logs found" + echo "Server logs:" + tail -20 /tmp/weed-server.log || echo "No server logs found" + exit 1 + fi + + - name: Run SMQ Integration Tests + run: | + cd test/kafka + ulimit -n 512 || echo "Warning: Could not set file descriptor limit" + ulimit -u 100 || echo "Warning: Could not set process limit" + # Run the dedicated SMQ integration tests + go test -v -run "^TestSMQIntegration" -timeout 180s ./integration/... + env: + GOMAXPROCS: 1 + SEAWEEDFS_MASTERS: 127.0.0.1:9333 + + kafka-protocol-tests: + name: Kafka Protocol Tests (Isolated) + runs-on: ubuntu-latest + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + container-id: [protocol-1] + container: + image: golang:1.26-alpine + options: --cpus 1.0 --memory 1g --tmpfs /tmp:exec --hostname kafka-protocol-${{ matrix.container-id }} + env: + GOMAXPROCS: 1 + CGO_ENABLED: 0 + KAFKA_PROTOCOL_ISOLATION: "true" + CONTAINER_ID: ${{ matrix.container-id }} + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version: ^1.26 + id: go + + - name: Check out code + uses: actions/checkout@v7 + + - name: Setup Protocol Container Environment + run: | + apk add --no-cache git procps + # Ensure proper permissions for test execution + chmod -R 755 /tmp || true + export TMPDIR=/tmp + export GOCACHE=/tmp/go-cache + mkdir -p $GOCACHE + chmod 755 $GOCACHE + + - name: Get dependencies + run: | + cd test/kafka + go mod download + go mod tidy + + - name: Run Protocol Tests + run: | + cd test/kafka + export TMPDIR=/tmp + export GOCACHE=/tmp/go-cache + # Run protocol tests from the weed/mq/kafka directory since they test the protocol implementation + cd ../../weed/mq/kafka + go test -v -run "^Test.*" -timeout 10s ./... + env: + GOMAXPROCS: 1 + TMPDIR: /tmp + GOCACHE: /tmp/go-cache diff --git a/.github/workflows/kms-tests.yml b/.github/workflows/kms-tests.yml new file mode 100644 index 0000000..3c997fd --- /dev/null +++ b/.github/workflows/kms-tests.yml @@ -0,0 +1,140 @@ +name: "KMS Tests" + +on: + pull_request: + paths: + - 'weed/kms/**' + - 'weed/s3api/s3_sse_*.go' + - 'weed/s3api/s3api_object_handlers.go' + - 'weed/s3api/s3api_object_handlers_put.go' + - 'test/kms/**' + - '.github/workflows/kms-tests.yml' + push: + branches: [ master, main ] + paths: + - 'weed/kms/**' + - 'weed/s3api/s3_sse_*.go' + - 'weed/s3api/s3api_object_handlers.go' + - 'weed/s3api/s3api_object_handlers_put.go' + - 'test/kms/**' + +concurrency: + group: ${{ github.head_ref || github.ref }}-kms-tests + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + working-directory: weed + +jobs: + kms-provider-tests: + name: KMS Provider Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run KMS provider integration tests + timeout-minutes: 15 + working-directory: test/kms + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + docker --version + + make test-provider-ci + + - name: Show OpenBao logs on failure + if: failure() + run: | + echo "=== OpenBao Container Logs ===" + docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao container found" + echo "=== Setup Logs ===" + cat /tmp/openbao-ci-setup.log 2>/dev/null || echo "No setup log found" + + - name: Cleanup + if: always() + working-directory: test/kms + run: | + make stop-openbao-ci || true + + s3-kms-e2e-tests: + name: S3 KMS End-to-End Tests + runs-on: ubuntu-22.04 + timeout-minutes: 25 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run S3 KMS end-to-end tests + timeout-minutes: 20 + working-directory: test/kms + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + docker --version + aws --version + + make test-s3-kms-ci + + - name: Show logs on failure + if: failure() + working-directory: test/kms + run: | + echo "=== OpenBao Container Logs ===" + cat /tmp/openbao-ci-container.log 2>/dev/null || docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao logs found" + echo "=== SeaweedFS Server Logs ===" + tail -100 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found" + echo "=== Setup Logs ===" + cat /tmp/weed-kms-ci-setup.log 2>/dev/null || echo "No weed setup log" + echo "=== Process Information ===" + ps aux | grep -E "(weed|test)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-kms-e2e-logs + path: | + /tmp/seaweedfs-kms-mini.log + /tmp/openbao-ci-container.log + /tmp/weed-kms-ci-setup.log + retention-days: 3 + + - name: Cleanup + if: always() + working-directory: test/kms + run: | + make stop-seaweedfs-ci || true + make stop-openbao-ci || true + make clean-ci || true diff --git a/.github/workflows/master-cold-start-tests.yml b/.github/workflows/master-cold-start-tests.yml new file mode 100644 index 0000000..bd0ed72 --- /dev/null +++ b/.github/workflows/master-cold-start-tests.yml @@ -0,0 +1,79 @@ +name: "Master Cold Start Tests" + +on: + push: + branches: [ master ] + paths: + - 'weed/server/master_*.go' + - 'weed/topology/**' + - 'weed/operation/**' + - 'test/master_cold_start/**' + - 'test/testutil/**' + - '.github/workflows/master-cold-start-tests.yml' + pull_request: + branches: [ master ] + paths: + - 'weed/server/master_*.go' + - 'weed/topology/**' + - 'weed/operation/**' + - 'test/master_cold_start/**' + - 'test/testutil/**' + - '.github/workflows/master-cold-start-tests.yml' + +concurrency: + group: ${{ github.head_ref || github.ref }}/master-cold-start-tests + cancel-in-progress: true + +permissions: + contents: read + +jobs: + master-cold-start-tests: + name: Master Cold Start Tests + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - name: Check out code + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Build weed binary + run: | + cd weed && go install -buildvcs=false + + - name: Run master cold start tests + # test/master_cold_start boots a fresh master plus empty volume + # servers and requires the very first assign (HTTP and gRPC, no + # client retries) to complete a write: the assign that triggers + # volume growth must wait for it instead of failing with + # "volume growth in progress". + run: | + export WEED_BINARY=$(go env GOPATH)/bin/weed + go test -v -timeout=8m ./test/master_cold_start/... + + - name: Collect server logs on failure + if: failure() + run: | + # test/master_cold_start/cluster.go keeps failing-test dirs created + # via os.MkdirTemp("", "seaweedfs_master_cold_start_it_") with each + # process log under /logs/. + mkdir -p /tmp/master-cold-start-logs + find /tmp -maxdepth 1 -type d -name "seaweedfs_master_cold_start_it_*" 2>/dev/null | while read dir; do + echo "Found test directory: $dir" + cp -r "$dir" /tmp/master-cold-start-logs/ 2>/dev/null || true + done + find /tmp/master-cold-start-logs -type f -name "*.log" -print -exec tail -n 100 {} \; 2>/dev/null || echo "No logs found" + + - name: Archive logs + if: failure() + uses: actions/upload-artifact@v7 + with: + name: master-cold-start-test-logs + path: /tmp/master-cold-start-logs/ + retention-days: 7 diff --git a/.github/workflows/metadata-subscribe-tests.yml b/.github/workflows/metadata-subscribe-tests.yml new file mode 100644 index 0000000..b47c16b --- /dev/null +++ b/.github/workflows/metadata-subscribe-tests.yml @@ -0,0 +1,93 @@ +name: "Metadata Subscribe Integration Tests" + +on: + push: + branches: [ master ] + paths: + - 'weed/filer/**' + - 'weed/pb/filer_pb/**' + - 'weed/util/log_buffer/**' + - 'weed/server/filer_grpc_server_sub_meta.go' + - 'weed/server/master_grpc_server.go' + - 'weed/command/filer_backup.go' + - 'test/metadata_subscribe/**' + - '.github/workflows/metadata-subscribe-tests.yml' + pull_request: + branches: [ master ] + paths: + - 'weed/filer/**' + - 'weed/pb/filer_pb/**' + - 'weed/util/log_buffer/**' + - 'weed/server/filer_grpc_server_sub_meta.go' + - 'weed/server/master_grpc_server.go' + - 'weed/command/filer_backup.go' + - 'test/metadata_subscribe/**' + - '.github/workflows/metadata-subscribe-tests.yml' + +concurrency: + group: ${{ github.head_ref }}/metadata-subscribe-tests + cancel-in-progress: true + +permissions: + contents: read + +env: + TEST_TIMEOUT: '10m' + +jobs: + metadata-subscribe-integration: + name: Metadata Subscribe Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Build SeaweedFS + run: | + cd weed + go build -o weed . + chmod +x weed + ./weed version + + - name: Run Metadata Subscribe Integration Tests + run: | + cd test/metadata_subscribe + + echo "Running Metadata Subscribe integration tests..." + echo "============================================" + + # Run tests with verbose output + go test -v -timeout=${{ env.TEST_TIMEOUT }} ./... + + echo "============================================" + echo "Metadata Subscribe integration tests completed" + + - name: Archive logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: metadata-subscribe-test-logs + path: | + /tmp/seaweedfs_* + retention-days: 7 + + - name: Test Summary + if: always() + run: | + echo "## Metadata Subscribe Integration Test Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY + echo "- **Basic Subscription**: Subscribe to metadata changes and receive events" >> $GITHUB_STEP_SUMMARY + echo "- **Single-Filer No Stall**: Regression test for issue #4977" >> $GITHUB_STEP_SUMMARY + echo "- **Resume from Disk**: Verify subscription can resume from persisted logs" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Related Issues" >> $GITHUB_STEP_SUMMARY + echo "- [#4977](https://github.com/seaweedfs/seaweedfs/issues/4977): filer.backup synchronisation stall" >> $GITHUB_STEP_SUMMARY + diff --git a/.github/workflows/mount-benchmark.yml b/.github/workflows/mount-benchmark.yml new file mode 100644 index 0000000..90310ee --- /dev/null +++ b/.github/workflows/mount-benchmark.yml @@ -0,0 +1,203 @@ +name: "mount: benchmark" + +# Manual benchmark: native WinFsp mount vs rclone+WebDAV on the same Windows +# runner, with a Linux FUSE mount of the same build as a reference. Numbers +# from shared runners are noisy; this is for finding factor-of-N gaps, not +# regressions of a few percent. +on: + workflow_dispatch: + push: + branches: [ 'winfsp-bench**' ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + bench-windows: + name: Windows native vs rclone + runs-on: windows-latest + timeout-minutes: 60 + env: + CGO_ENABLED: 0 + + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Install WinFsp and rclone + run: choco install winfsp rclone -y --no-progress + + - name: Build weed.exe + run: go build -o weed.exe ./weed + + - name: Benchmark both mounts + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + + function Test-Port($port) { + $client = New-Object System.Net.Sockets.TcpClient + try { $client.Connect('127.0.0.1', $port); return $client.Connected } + catch { return $false } + finally { $client.Dispose() } + } + + function Wait-Drive($drive, $what) { + $deadline = (Get-Date).AddMinutes(2) + while ((Get-Date) -lt $deadline) { + if (Test-Path "${drive}\") { Write-Host "$what is mounted on $drive"; return } + Start-Sleep -Seconds 2 + } + throw "$what never appeared on $drive" + } + + function Invoke-Bench($dir, $label, $out) { + Write-Host "::group::bench $label" + & go run ./test/mount_bench -dir $dir -label $label -filer 127.0.0.1:8888 -out $out + $code = $LASTEXITCODE + Write-Host "::endgroup::" + if ($code -ne 0) { throw "bench $label failed with exit $code" } + } + + New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null + Start-Process -FilePath .\weed.exe ` + -ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' ` + -RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log + + $deadline = (Get-Date).AddMinutes(3) + while ((Get-Date) -lt $deadline) { + if ((Test-Port 8888) -and (Test-Port 18888) -and (Test-Port 7333)) { break } + Start-Sleep -Seconds 3 + } + if (-not ((Test-Port 8888) -and (Test-Port 18888) -and (Test-Port 7333))) { + Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue + throw "mini cluster never came up" + } + Write-Host "filer on 8888/18888, webdav on 7333" + + # --- native WinFsp mount --- + Start-Process -FilePath .\weed.exe ` + -ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' ` + -RedirectStandardOutput C:\seaweed-mount.log -RedirectStandardError C:\seaweed-mount.err.log + Wait-Drive 'S:' 'weed mount' + + Invoke-Bench 'S:\bench-native' 'winfsp-native' 'C:\results-native.json' + + Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" | + Where-Object { $_.CommandLine -like '*mount*' } | + ForEach-Object { Stop-Process -Id $_.ProcessId -Force } + $deadline = (Get-Date).AddMinutes(1) + while ((Get-Date) -lt $deadline -and (Test-Path S:\)) { Start-Sleep -Seconds 2 } + + # --- rclone + WebDAV on the same WinFsp --- + # rclone serves listings from a directory cache it fills lazily, so the + # big-listing files have to exist before it mounts or it never sees them. + & go run ./test/mount_bench -filer 127.0.0.1:8888 -seed bench-rclone/biglist + if ($LASTEXITCODE -ne 0) { throw "seeding failed" } + + $env:RCLONE_CONFIG_SEAWEED_TYPE = 'webdav' + $env:RCLONE_CONFIG_SEAWEED_URL = 'http://127.0.0.1:7333' + $env:RCLONE_CONFIG_SEAWEED_VENDOR = 'other' + Start-Process -FilePath rclone ` + -ArgumentList 'mount','seaweed:','T:','--vfs-cache-mode=writes','-v','--log-file=C:\rclone.log' + Wait-Drive 'T:' 'rclone mount' + + Invoke-Bench 'T:\bench-rclone' 'rclone-webdav' 'C:\results-rclone.json' + + Stop-Process -Name rclone -Force -ErrorAction SilentlyContinue + + # --- comparison --- + $table = & go run ./test/mount_bench -compare C:\results-native.json,C:\results-rclone.json + $table | Write-Host + "## Windows: native WinFsp vs rclone+WebDAV" | Out-File -Append $env:GITHUB_STEP_SUMMARY + $table | Out-File -Append $env:GITHUB_STEP_SUMMARY + + - name: Logs + if: always() + shell: pwsh + run: | + foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\rclone.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') { + if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 100 } + } + + - name: Results + if: always() + uses: actions/upload-artifact@v7 + with: + name: results-windows + path: C:\results-*.json + if-no-files-found: ignore + + bench-linux: + name: Linux FUSE reference + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Repair the fusermount3 setuid bit + uses: ./.github/actions/fix-fusermount-setuid + + - name: Allow non-root FUSE mounts with allow_other + run: | + echo 'user_allow_other' | sudo tee -a /etc/fuse.conf + sudo chmod 644 /etc/fuse.conf + + - name: Build weed + run: go build -o /tmp/weed ./weed + + - name: Benchmark FUSE mount + run: | + set -e + mkdir -p /tmp/seaweed-data + /tmp/weed -logtostderr mini -dir=/tmp/seaweed-data -ip=127.0.0.1 > /tmp/mini.log 2>&1 & + + for i in $(seq 1 60); do + if nc -z 127.0.0.1 8888 && nc -z 127.0.0.1 18888; then break; fi + sleep 3 + done + nc -z 127.0.0.1 8888 || { cat /tmp/mini.log; echo "filer never came up"; exit 1; } + + mkdir -p "$HOME/mnt" + /tmp/weed -logtostderr mount -filer=127.0.0.1:8888 -dir="$HOME/mnt" > /tmp/mount.log 2>&1 & + for i in $(seq 1 60); do + if mountpoint -q "$HOME/mnt"; then break; fi + sleep 2 + done + mountpoint -q "$HOME/mnt" || { cat /tmp/mount.log; echo "mount never appeared"; exit 1; } + + go run ./test/mount_bench -dir "$HOME/mnt/bench-linux" -label linux-fuse -filer 127.0.0.1:8888 -out /tmp/results-linux.json + + { + echo "## Linux FUSE reference" + go run ./test/mount_bench -compare /tmp/results-linux.json + } >> "$GITHUB_STEP_SUMMARY" + + - name: Logs + if: always() + run: | + tail -n 100 /tmp/mount.log /tmp/mini.log 2>/dev/null || true + + - name: Results + if: always() + uses: actions/upload-artifact@v7 + with: + name: results-linux + path: /tmp/results-linux.json + if-no-files-found: ignore diff --git a/.github/workflows/mount-windows-conformance.yml b/.github/workflows/mount-windows-conformance.yml new file mode 100644 index 0000000..6d490c5 --- /dev/null +++ b/.github/workflows/mount-windows-conformance.yml @@ -0,0 +1,124 @@ +name: "mount: windows conformance" + +on: + push: + branches: [ master ] + paths: + - 'weed/mount/**' + - 'weed/command/mount*.go' + - 'weed/storage/volume_vacuum*.go' + - 'weed/storage/volume_loading.go' + - 'weed/storage/disk_location.go' + - 'test/winfsp-conformance/**' + - '.github/workflows/mount-windows-conformance.yml' + # No base branch filter: this is the only thing that runs the Windows mount, + # so it should cover a pull request stacked on another one too. + pull_request: + paths: + - 'weed/mount/**' + - 'weed/command/mount*.go' + - 'weed/storage/volume_vacuum*.go' + - 'weed/storage/volume_loading.go' + - 'weed/storage/disk_location.go' + - 'test/winfsp-conformance/**' + - '.github/workflows/mount-windows-conformance.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + conformance: + name: WinFsp conformance + runs-on: windows-latest + timeout-minutes: 60 + env: + # The runner ships MinGW, so cgo is on by default and cgofuse picks its + # cgo variant, which wants WinFsp's headers. The nocgo variant loads + # winfsp-x64.dll at run time instead, which is how weed.exe is released. + CGO_ENABLED: 0 + + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + # cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but + # not to build. + - name: Install WinFsp + run: choco install winfsp -y --no-progress + + - name: Build weed.exe + run: go build -o weed.exe ./weed + + # The runner tears down a step's process tree when its shell exits, so a + # cluster started in one step is gone by the next. Everything that needs + # the cluster and the mount alive has to share a step. + - name: Mount and run winfsp-tests + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + + function Test-Port($port) { + # A plain connect, because Test-NetConnection has reported success + # here for a port nothing was listening on. + $client = New-Object System.Net.Sockets.TcpClient + try { $client.Connect('127.0.0.1', $port); return $client.Connected } + catch { return $false } + finally { $client.Dispose() } + } + + function Start-Mount($log) { + Start-Process -FilePath .\weed.exe ` + -ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' ` + -RedirectStandardOutput "C:\$log.log" -RedirectStandardError "C:\$log.err.log" + $deadline = (Get-Date).AddMinutes(2) + while ((Get-Date) -lt $deadline) { + if (Test-Path S:\) { Write-Host "S: is mounted"; return } + Start-Sleep -Seconds 2 + } + Get-Content "C:\$log.log", "C:\$log.err.log" -ErrorAction SilentlyContinue + throw "S: never appeared" + } + + New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null + # -ip pins the cluster to loopback; it otherwise advertises and binds + # the runner's LAN address, which 127.0.0.1 cannot reach. + Start-Process -FilePath .\weed.exe ` + -ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' ` + -RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log + + $deadline = (Get-Date).AddMinutes(3) + while ((Get-Date) -lt $deadline) { + # The mount dials grpc, not http, so both ports have to answer. + if ((Test-Port 8888) -and (Test-Port 18888)) { break } + Start-Sleep -Seconds 3 + } + if (-not ((Test-Port 8888) -and (Test-Port 18888))) { + Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue + throw "filer never came up" + } + Write-Host "filer is up on http 8888 and grpc 18888" + + Start-Mount 'seaweed-mount' + + Write-Host "::group::winfsp-tests" + & pwsh -File test/winfsp-conformance/run.ps1 -MountPoint S:\ + $code = $LASTEXITCODE + Write-Host "::endgroup::" + if ($code -ne 0) { throw "winfsp-tests failed with exit $code" } + + - name: Logs + if: always() + shell: pwsh + run: | + foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') { + if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 200 } + } diff --git a/.github/workflows/mount-windows.yml b/.github/workflows/mount-windows.yml new file mode 100644 index 0000000..91f61d8 --- /dev/null +++ b/.github/workflows/mount-windows.yml @@ -0,0 +1,205 @@ +name: "mount: windows" + +on: + push: + branches: [ master ] + paths: + - 'weed/mount/**' + - 'weed/command/mount*.go' + - 'weed/storage/volume_vacuum*.go' + - 'weed/storage/volume_loading.go' + - 'weed/storage/disk_location.go' + - 'test/winfsp/**' + - '.github/workflows/mount-windows.yml' + # No base branch filter: this is the only thing that runs the Windows mount, + # so it should cover a pull request stacked on another one too. + pull_request: + paths: + - 'weed/mount/**' + - 'weed/command/mount*.go' + - 'weed/storage/volume_vacuum*.go' + - 'weed/storage/volume_loading.go' + - 'weed/storage/disk_location.go' + - 'test/winfsp/**' + - '.github/workflows/mount-windows.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + mount-windows: + name: Mount on Windows + runs-on: windows-latest + timeout-minutes: 40 + env: + # The runner ships MinGW, so cgo is on by default and cgofuse picks its + # cgo variant, which wants WinFsp's headers. The nocgo variant loads + # winfsp-x64.dll at run time instead, which is how weed.exe is released. + CGO_ENABLED: 0 + + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + # cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but + # not to build. + - name: Install WinFsp + run: choco install winfsp -y --no-progress + + - name: Build weed.exe + run: go build -o weed.exe ./weed + + # The runner tears down a step's process tree when its shell exits, so a + # cluster started in one step is gone by the next. Everything that needs + # the cluster and the mount alive has to share a step. + - name: Mount and exercise + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + + function Test-Port($port) { + # A plain connect, because Test-NetConnection has reported success + # here for a port nothing was listening on. + $client = New-Object System.Net.Sockets.TcpClient + try { $client.Connect('127.0.0.1', $port); return $client.Connected } + catch { return $false } + finally { $client.Dispose() } + } + + function Start-Mount($log) { + Start-Process -FilePath .\weed.exe ` + -ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' ` + -RedirectStandardOutput "C:\$log.log" -RedirectStandardError "C:\$log.err.log" + $deadline = (Get-Date).AddMinutes(2) + while ((Get-Date) -lt $deadline) { + if (Test-Path S:\) { Write-Host "S: is mounted"; return } + Start-Sleep -Seconds 2 + } + Get-Content "C:\$log.log", "C:\$log.err.log" -ErrorAction SilentlyContinue + throw "S: never appeared" + } + + function Stop-Mount { + Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" | + Where-Object { $_.CommandLine -like '*mount*' } | + ForEach-Object { Stop-Process -Id $_.ProcessId -Force } + $deadline = (Get-Date).AddMinutes(1) + while ((Get-Date) -lt $deadline -and (Test-Path S:\)) { Start-Sleep -Seconds 2 } + if (Test-Path S:\) { throw "S: still present after stopping the mount" } + Write-Host "unmounted" + } + + function Invoke-Tests($label, [string[]]$goArgs) { + Write-Host "::group::$label" + & go @goArgs + $code = $LASTEXITCODE + Write-Host "::endgroup::" + if ($code -ne 0) { throw "$label failed with exit $code" } + } + + New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null + # -ip pins the cluster to loopback; it otherwise advertises and binds + # the runner's LAN address, which 127.0.0.1 cannot reach. + Start-Process -FilePath .\weed.exe ` + -ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' ` + -RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log + + $deadline = (Get-Date).AddMinutes(3) + while ((Get-Date) -lt $deadline) { + # The mount dials grpc, not http, so both ports have to answer. + if ((Test-Port 8888) -and (Test-Port 18888)) { break } + Start-Sleep -Seconds 3 + } + if (-not ((Test-Port 8888) -and (Test-Port 18888))) { + Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue + throw "filer never came up" + } + Write-Host "filer is up on http 8888 and grpc 18888" + + Start-Mount 'seaweed-mount' + + Invoke-Tests 'exercise' @('test','-v','-timeout','20m','./test/winfsp','-mountpoint=S:\') + Invoke-Tests 'persist-write' @('test','-v','-timeout','15m','./test/winfsp','-run','TestPersistence','-mountpoint=S:\','-phase=write','-filer=127.0.0.1:8888') + + Stop-Mount + Start-Mount 'seaweed-remount' + + Invoke-Tests 'persist-verify' @('test','-v','-timeout','15m','./test/winfsp','-run','TestPersistence','-mountpoint=S:\','-phase=verify') + + # WinFsp creates the mount directory itself, so the path must not + # exist; only its parent has to. + Write-Host "::group::mount over a directory" + Stop-Mount + Remove-Item C:\seaweed-mnt -Recurse -Force -ErrorAction SilentlyContinue + Start-Process -FilePath .\weed.exe ` + -ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=C:\seaweed-mnt' ` + -RedirectStandardOutput C:\seaweed-dirmount.log -RedirectStandardError C:\seaweed-dirmount.err.log + $deadline = (Get-Date).AddMinutes(2) + $ok = $false + while ((Get-Date) -lt $deadline) { + # Listing succeeds on the plain empty directory too, so wait for the + # reparse point WinFsp turns it into. Otherwise this step passes + # without a mount and writes to local disk. + $item = Get-Item C:\seaweed-mnt -Force -ErrorAction SilentlyContinue + if ($null -ne $item -and $item.Attributes.ToString() -like '*ReparsePoint*') { $ok = $true; break } + Start-Sleep -Seconds 2 + } + if (-not $ok) { + Get-Content C:\seaweed-dirmount.log, C:\seaweed-dirmount.err.log -ErrorAction SilentlyContinue + throw "mounting over a directory failed" + } + Set-Content -Path C:\seaweed-mnt\dirmount.txt -Value 'via directory mount' + if ((Get-Content C:\seaweed-mnt\dirmount.txt) -ne 'via directory mount') { throw "readback through the directory mount differs" } + Remove-Item C:\seaweed-mnt\dirmount.txt -Force + Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" | + Where-Object { $_.CommandLine -like '*mount*' } | + ForEach-Object { Stop-Process -Id $_.ProcessId -Force } + Start-Sleep -Seconds 5 + Write-Host "::endgroup::" + + Start-Mount 'seaweed-remount2' + + Write-Host "::group::explorer-style walk" + New-Item -ItemType Directory -Force -Path S:\walk | Out-Null + 1..200 | ForEach-Object { Set-Content -Path "S:\walk\f$_.txt" -Value "line $_" } + $names = @(Get-ChildItem S:\walk | ForEach-Object { $_.Name }) + if ($names.Count -ne 200) { + # Name the strays: a dot entry surfacing here is a different problem + # from a missing or duplicated file. + $unexpected = $names | Where-Object { $_ -notmatch '^f\d+\.txt$' } + throw "listed $($names.Count) entries, expected 200; unexpected: $($unexpected -join ', ')" + } + $body = Get-Content S:\walk\f42.txt + if ($body -ne 'line 42') { throw "unexpected content: $body" } + Copy-Item S:\walk\f42.txt S:\walk\copy.txt + Remove-Item S:\walk -Recurse -Force + if (Test-Path S:\walk) { throw "directory survived recursive delete" } + Write-Host "::endgroup::" + + # Tear down everything we started so the runner's later steps (Logs, + # post-checkout) launch into a clean environment. A WinFsp mount left + # active and a weed.exe holding winfsp-x64.dll have been seen to make + # the next step's pwsh.exe fail with STATUS_DLL_INIT_FAILED + # (0xC0000142), which fails a job whose actual test step passed. + try { Stop-Mount } catch { Write-Host "no mount to stop at teardown: $_" } + Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" | + ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } + Start-Sleep -Seconds 3 + + - name: Logs + if: always() + continue-on-error: true + shell: pwsh + run: | + foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\seaweed-remount.log','C:\seaweed-remount.err.log','C:\seaweed-remount2.log','C:\seaweed-remount2.err.log','C:\seaweed-dirmount.log','C:\seaweed-dirmount.err.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') { + if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 200 } + } diff --git a/.github/workflows/multi-master-tests.yml b/.github/workflows/multi-master-tests.yml new file mode 100644 index 0000000..2e0cfe8 --- /dev/null +++ b/.github/workflows/multi-master-tests.yml @@ -0,0 +1,77 @@ +name: "Multi-Master Tests" + +on: + push: + branches: [ master ] + paths: + - 'weed/server/master_*.go' + - 'weed/server/raft_*.go' + - 'weed/topology/**' + - 'test/multi_master/**' + - 'test/testutil/**' + - '.github/workflows/multi-master-tests.yml' + pull_request: + branches: [ master ] + paths: + - 'weed/server/master_*.go' + - 'weed/server/raft_*.go' + - 'weed/topology/**' + - 'test/multi_master/**' + - 'test/testutil/**' + - '.github/workflows/multi-master-tests.yml' + +concurrency: + group: ${{ github.head_ref || github.ref }}/multi-master-tests + cancel-in-progress: true + +permissions: + contents: read + +jobs: + multi-master-failover-tests: + name: Multi-Master Failover Tests + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Install SeaweedFS + run: | + cd weed && go install -buildvcs=false + + - name: Run multi-master failover tests + # The tests in test/multi_master spin up their own 3-node master raft + # cluster (using the freshly-installed `weed` binary) and exercise + # leader-election, failover and recovery scenarios. The shared + # test/testutil port-allocator regression test runs alongside since it + # is a prerequisite for the cluster fixtures. + run: | + go test -v -timeout=8m ./test/multi_master/... ./test/testutil/... + + - name: Collect server logs on failure + if: failure() + run: | + # test/multi_master/cluster.go creates per-test dirs via + # os.MkdirTemp("", "seaweedfs_multi_master_it_") and writes each + # node's log into /logs/master*.log. + echo "Collecting per-node master logs from temp directories..." + mkdir -p /tmp/multi-master-logs + find /tmp -maxdepth 1 -type d -name "seaweedfs_multi_master_it_*" 2>/dev/null | while read dir; do + echo "Found test directory: $dir" + cp -r "$dir" /tmp/multi-master-logs/ 2>/dev/null || true + done + find /tmp/multi-master-logs -type f -name "*.log" -print -exec tail -n 100 {} \; 2>/dev/null || echo "No logs found" + + - name: Archive logs + if: failure() + uses: actions/upload-artifact@v7 + with: + name: multi-master-test-logs + path: /tmp/multi-master-logs/ + retention-days: 7 diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 0000000..2be3782 --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,469 @@ +name: "Performance" + +on: + push: + branches: [ master ] + paths: + - '**/*.go' + - 'go.mod' + - 'go.sum' + - 'seaweed-volume/**' + - 'test/perf/**' + - '.github/workflows/performance.yml' + workflow_dispatch: + inputs: + profile_duration: + description: "CPU profiling duration in seconds" + required: false + default: "30" + type: string + benchmark_files: + description: "Number of files for the throughput benchmark" + required: false + default: "100000" + type: string + benchmark_concurrency: + description: "Concurrent read/write workers" + required: false + default: "16" + type: string + benchmark_size: + description: "Simulated file size in bytes" + required: false + default: "1024" + type: string + s3_objects: + description: "Number of objects for the S3 benchmark" + required: false + default: "20000" + type: string + s3_size: + description: "S3 object size in bytes" + required: false + default: "4096" + type: string + +concurrency: + group: ${{ github.head_ref || github.ref }}/performance + cancel-in-progress: true + +permissions: + contents: read + +env: + VOL_SIZE_LIMIT: "1024" + +jobs: + performance-profile: + name: CPU and Heap Profile + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Build weed + run: go build -o weed_bin ./weed + + - name: Start server with profiling enabled + run: | + mkdir -p ./perfdata + ./weed_bin -v=1 server -debug -debug.port=6060 -dir=./perfdata \ + -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=100 \ + -s3.port=8000 -s3.config=./docker/compose/s3.json \ + > weed.log 2>&1 & + echo "WEED_PID=$!" >> "$GITHUB_ENV" + for i in $(seq 1 60); do + if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then + echo "master is ready" + break + fi + sleep 1 + done + # give the volume server a moment to register with the master + sleep 3 + + - name: Start memory sampler + run: | + bash test/perf/mem_sample.sh mem-profile.csv "server=${WEED_PID}" & + echo "SAMPLER_PID=$!" >> "$GITHUB_ENV" + + - name: Capture profiles under load + run: | + DURATION="${{ github.event.inputs.profile_duration || '30' }}" + # drive write load so the sampled profile reflects real work + ./weed_bin benchmark -master=localhost:9333 -writeOnly \ + -c=16 -n=5000000 -size=1024 > benchmark-load.log 2>&1 & + echo "Sampling CPU profile for ${DURATION}s..." + curl -s "http://localhost:6060/debug/pprof/profile?seconds=${DURATION}" -o cpu.pprof + curl -s "http://localhost:6060/debug/pprof/heap" -o heap.pprof + curl -s "http://localhost:6060/debug/pprof/goroutine?debug=1" -o goroutine.txt + go tool pprof -top -nodecount=50 cpu.pprof > cpu-top.txt 2>/dev/null || true + go tool pprof -top -nodecount=50 -sample_index=inuse_space heap.pprof > heap-top.txt 2>/dev/null || true + + - name: Record memory usage + if: always() + run: | + kill -TERM "${SAMPLER_PID}" 2>/dev/null || true + sleep 2 + { + echo "## Memory usage (peak RSS)" + echo '```' + if [ -f mem-profile.csv.peak ]; then + awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-profile.csv.peak + else + echo "no memory samples captured" + fi + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Profile summary + if: always() + run: | + { + echo "## CPU profile (top functions)" + echo '```' + head -45 cpu-top.txt 2>/dev/null || echo "no cpu profile captured" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Stop server + if: always() + run: kill "${WEED_PID}" 2>/dev/null || true + + - name: Show server log on failure + if: failure() + run: tail -200 weed.log || true + + - name: Upload profiles + if: always() + uses: actions/upload-artifact@v7 + with: + name: performance-profile-${{ github.run_number }} + path: | + cpu.pprof + heap.pprof + cpu-top.txt + heap-top.txt + goroutine.txt + mem-profile.csv + mem-profile.csv.peak + weed.log + retention-days: 30 + + benchmark: + name: Throughput Benchmark (${{ matrix.impl }} volume) + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + impl: [go, rust] + env: + IMPL: ${{ matrix.impl }} + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Install Rust toolchain + if: matrix.impl == 'rust' + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry and target + if: matrix.impl == 'rust' + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust- + + - name: Build weed + run: go build -o weed_bin ./weed + + - name: Build Rust volume server + if: matrix.impl == 'rust' + run: cd seaweed-volume && cargo build --release + + - name: Start master and volume server + run: | + mkdir -p ./perfdata/master ./perfdata/vol + ./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \ + -mdir=./perfdata/master -peers=none \ + -volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \ + > master.log 2>&1 & + echo "MASTER_PID=$!" >> "$GITHUB_ENV" + for i in $(seq 1 60); do + if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then + echo "master is ready" + break + fi + sleep 1 + done + if [ "${IMPL}" = "rust" ]; then + ./seaweed-volume/target/release/weed-volume \ + --master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \ + --port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \ + > volume.log 2>&1 & + else + ./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \ + -port=8080 -dir=./perfdata/vol -max=100 \ + > volume.log 2>&1 & + fi + echo "VOLUME_PID=$!" >> "$GITHUB_ENV" + for i in $(seq 1 60); do + if curl -sf http://localhost:8080/status >/dev/null 2>&1; then + echo "volume server is ready" + break + fi + sleep 1 + done + # let the volume server register with the master via heartbeat + sleep 3 + + - name: Start memory sampler + run: | + bash test/perf/mem_sample.sh mem-benchmark.csv \ + "master=${MASTER_PID}" "volume=${VOLUME_PID}" & + echo "SAMPLER_PID=$!" >> "$GITHUB_ENV" + + - name: Run throughput benchmark + run: | + N="${{ github.event.inputs.benchmark_files || '100000' }}" + C="${{ github.event.inputs.benchmark_concurrency || '16' }}" + SIZE="${{ github.event.inputs.benchmark_size || '1024' }}" + ./weed_bin benchmark -master=localhost:9333 \ + -c="${C}" -n="${N}" -size="${SIZE}" 2>&1 | tee benchmark-results.txt + + - name: Run Go micro-benchmarks + if: matrix.impl == 'go' + continue-on-error: true + run: | + go test -run='^$' -bench=. -benchmem -benchtime=10x \ + ./weed/topology/... ./weed/util/log_buffer/... ./weed/util/buffered_queue/... \ + 2>&1 | tee go-benchmarks.txt + + - name: Record memory usage + if: always() + run: | + kill -TERM "${SAMPLER_PID}" 2>/dev/null || true + sleep 2 + { + echo "## Memory usage (peak RSS, ${IMPL} volume)" + echo '```' + if [ -f mem-benchmark.csv.peak ]; then + awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-benchmark.csv.peak + else + echo "no memory samples captured" + fi + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Benchmark summary + if: always() + run: | + { + echo "## Throughput benchmark (${IMPL} volume)" + echo '```' + grep -E "Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate" \ + benchmark-results.txt 2>/dev/null || echo "no benchmark results captured" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Stop processes + if: always() + run: | + kill "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true + + - name: Show logs on failure + if: failure() + run: | + echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true + echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v7 + with: + name: benchmark-results-${{ matrix.impl }}-${{ github.run_number }} + path: | + benchmark-results.txt + go-benchmarks.txt + mem-benchmark.csv + mem-benchmark.csv.peak + master.log + volume.log + retention-days: 7 + + s3-benchmark: + name: S3 Read/Write Benchmark (${{ matrix.impl }} volume) + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + impl: [go, rust] + env: + IMPL: ${{ matrix.impl }} + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Install Rust toolchain + if: matrix.impl == 'rust' + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry and target + if: matrix.impl == 'rust' + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust- + + - name: Build weed and S3 load tool + run: | + go build -o weed_bin ./weed + go build -o s3bench ./test/s3/benchmark + + - name: Build Rust volume server + if: matrix.impl == 'rust' + run: cd seaweed-volume && cargo build --release + + - name: Start cluster with S3 gateway + run: | + mkdir -p ./perfdata/master ./perfdata/vol ./perfdata/filer + ./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \ + -mdir=./perfdata/master -peers=none \ + -volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \ + > master.log 2>&1 & + echo "MASTER_PID=$!" >> "$GITHUB_ENV" + for i in $(seq 1 60); do + if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then + echo "master is ready" + break + fi + sleep 1 + done + if [ "${IMPL}" = "rust" ]; then + ./seaweed-volume/target/release/weed-volume \ + --master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \ + --port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \ + > volume.log 2>&1 & + else + ./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \ + -port=8080 -dir=./perfdata/vol -max=100 \ + > volume.log 2>&1 & + fi + echo "VOLUME_PID=$!" >> "$GITHUB_ENV" + for i in $(seq 1 60); do + if curl -sf http://localhost:8080/status >/dev/null 2>&1; then + echo "volume server is ready" + break + fi + sleep 1 + done + sleep 3 + ./weed_bin -v=1 filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888 \ + -s3 -s3.port=8000 -s3.config=./docker/compose/s3.json \ + > filer.log 2>&1 & + echo "FILER_PID=$!" >> "$GITHUB_ENV" + for i in $(seq 1 30); do + if nc -z localhost 8000 2>/dev/null; then + echo "s3 gateway is ready" + break + fi + sleep 1 + done + sleep 2 + + - name: Start memory sampler + run: | + bash test/perf/mem_sample.sh mem-s3.csv \ + "master=${MASTER_PID}" "volume=${VOLUME_PID}" "filer=${FILER_PID}" & + echo "SAMPLER_PID=$!" >> "$GITHUB_ENV" + + - name: Run S3 read/write benchmark + run: | + OBJECTS="${{ github.event.inputs.s3_objects || '20000' }}" + C="${{ github.event.inputs.benchmark_concurrency || '16' }}" + SIZE="${{ github.event.inputs.s3_size || '4096' }}" + ./s3bench -endpoint=http://localhost:8000 \ + -access-key=some_access_key1 -secret-key=some_secret_key1 \ + -objects="${OBJECTS}" -size="${SIZE}" -concurrency="${C}" -mode=both \ + 2>&1 | tee s3-benchmark-results.txt + + - name: Record memory usage + if: always() + run: | + kill -TERM "${SAMPLER_PID}" 2>/dev/null || true + sleep 2 + { + echo "## Memory usage (peak RSS, ${IMPL} volume)" + echo '```' + if [ -f mem-s3.csv.peak ]; then + awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-s3.csv.peak + else + echo "no memory samples captured" + fi + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: S3 benchmark summary + if: always() + run: | + { + echo "## S3 read/write benchmark (${IMPL} volume)" + echo '```' + grep -E "results:|Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate|Latency" \ + s3-benchmark-results.txt 2>/dev/null || echo "no S3 benchmark results captured" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Stop processes + if: always() + run: | + kill "${FILER_PID}" "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true + + - name: Show logs on failure + if: failure() + run: | + echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true + echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true + echo "=== filer.log ==="; tail -200 filer.log 2>/dev/null || true + + - name: Upload S3 benchmark results + if: always() + uses: actions/upload-artifact@v7 + with: + name: s3-benchmark-results-${{ matrix.impl }}-${{ github.run_number }} + path: | + s3-benchmark-results.txt + mem-s3.csv + mem-s3.csv.peak + master.log + volume.log + filer.log + retention-days: 7 diff --git a/.github/workflows/pjdfstest.yml b/.github/workflows/pjdfstest.yml new file mode 100644 index 0000000..8084538 --- /dev/null +++ b/.github/workflows/pjdfstest.yml @@ -0,0 +1,118 @@ +name: "pjdfstest POSIX Compliance" + +on: + push: + branches: [ master, main ] + paths: + - 'weed/mount/**' + - 'weed/filer/**' + - 'test/pjdfstest/**' + - '.github/workflows/pjdfstest.yml' + pull_request: + branches: [ master, main ] + paths: + - 'weed/mount/**' + - 'weed/filer/**' + - 'test/pjdfstest/**' + - '.github/workflows/pjdfstest.yml' + workflow_dispatch: + +concurrency: + group: pjdfstest/${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + pjdfstest: + name: pjdfstest + runs-on: ubuntu-22.04 + timeout-minutes: 60 + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Start local Docker registry + run: docker run -d --restart=always -p 5000:5000 --name registry registry:2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + with: + driver-opts: network=host + + - name: Build weed race binary + run: | + cd docker + make binary_race + + - name: Build SeaweedFS e2e image + uses: docker/build-push-action@v7 + with: + context: docker + file: docker/Dockerfile.e2e + tags: localhost:5000/chrislusf/seaweedfs:e2e + push: true + cache-from: type=gha,scope=pjdfstest-e2e + cache-to: type=gha,mode=max,scope=pjdfstest-e2e + + - name: Tag e2e image for docker compose + run: | + docker pull localhost:5000/chrislusf/seaweedfs:e2e + docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e + + - name: Build pjdfstest image + uses: docker/build-push-action@v7 + with: + context: test/pjdfstest + build-contexts: | + chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e + tags: localhost:5000/chrislusf/seaweedfs:pjdfstest + push: true + cache-from: type=gha,scope=pjdfstest-harness + cache-to: type=gha,mode=max,scope=pjdfstest-harness + + - name: Tag pjdfstest image for docker compose + run: | + docker pull localhost:5000/chrislusf/seaweedfs:pjdfstest + docker tag localhost:5000/chrislusf/seaweedfs:pjdfstest chrislusf/seaweedfs:pjdfstest + + - name: Start SeaweedFS cluster + run: | + docker compose -f test/pjdfstest/docker-compose.yml up --wait + + - name: Run pjdfstest + run: | + set -o pipefail + docker compose -f test/pjdfstest/docker-compose.yml exec -T mount \ + /run.sh 2>&1 | tee /tmp/pjdfstest-output.log + + - name: Collect logs + if: always() + run: | + mkdir -p /tmp/pjdfstest-docker-logs + for svc in master volume filer mount; do + docker compose -f test/pjdfstest/docker-compose.yml logs "$svc" \ + > "/tmp/pjdfstest-docker-logs/${svc}.log" 2>&1 || true + done + + - name: Tear down + if: always() + run: | + docker compose -f test/pjdfstest/docker-compose.yml down -v + + - name: Upload logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: pjdfstest-results + path: | + /tmp/pjdfstest-output.log + /tmp/pjdfstest-docker-logs/ + retention-days: 7 diff --git a/.github/workflows/plugin-workers.yml b/.github/workflows/plugin-workers.yml new file mode 100644 index 0000000..b3f1055 --- /dev/null +++ b/.github/workflows/plugin-workers.yml @@ -0,0 +1,51 @@ +name: "Plugin Worker Integration Tests" + +on: + push: + branches: [ master ] + paths: + - 'weed/**' + - 'test/plugin_workers/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/plugin-workers.yml' + pull_request: + branches: [ master ] + paths: + - 'weed/**' + - 'test/plugin_workers/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/plugin-workers.yml' + +permissions: + contents: read + +jobs: + plugin-worker: + name: "Plugin Worker: ${{ matrix.worker }}" + runs-on: ubuntu-22.04 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + include: + - worker: erasure_coding + path: test/plugin_workers/erasure_coding + - worker: vacuum + path: test/plugin_workers/vacuum + - worker: volume_balance + path: test/plugin_workers/volume_balance + + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version: ^1.26 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + + - name: Run plugin worker tests + run: go test -v ./${{ matrix.path }} diff --git a/.github/workflows/postgres-tests.yml b/.github/workflows/postgres-tests.yml new file mode 100644 index 0000000..a0369b1 --- /dev/null +++ b/.github/workflows/postgres-tests.yml @@ -0,0 +1,94 @@ +name: "PostgreSQL Gateway Tests" + +on: + push: + branches: [ master ] + paths: + - 'weed/server/postgres/**' + - 'weed/query/**' + - 'weed/mq/**' + - 'test/postgres/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/postgres-tests.yml' + pull_request: + branches: [ master ] + paths: + - 'weed/server/postgres/**' + - 'weed/query/**' + - 'weed/mq/**' + - 'test/postgres/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/postgres-tests.yml' + +concurrency: + group: ${{ github.head_ref }}/postgres-tests + cancel-in-progress: true + +permissions: + contents: read + +jobs: + postgres-basic-tests: + name: PostgreSQL Basic Tests + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: test/postgres + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version: ^1.26 + id: go + + - name: Check out code + uses: actions/checkout@v7 + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Cache Docker layers + uses: actions/cache@v6 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-postgres-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-postgres- + + - name: Start PostgreSQL Gateway Services + run: | + make dev-start + sleep 10 + + - name: Run Basic Connectivity Test + run: | + make test-basic + + - name: Run PostgreSQL Client Tests + run: | + make test-client + + - name: Save logs + if: always() + run: | + docker compose logs > postgres-output.log || true + + - name: Archive logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: postgres-logs + path: test/postgres/postgres-output.log + + - name: Cleanup + if: always() + run: | + make clean || true diff --git a/.github/workflows/release_version_bump.yml b/.github/workflows/release_version_bump.yml new file mode 100644 index 0000000..3ad9633 --- /dev/null +++ b/.github/workflows/release_version_bump.yml @@ -0,0 +1,278 @@ +name: "release: bump version and cut the release" + +# One entry point for a SeaweedFS release: +# 1. bump MAJOR/MINOR in constants.go and the Helm Chart.yaml, commit to master +# 2. push the tag, which fans out to the workflows that trigger on +# `push: tags` (binaries_release*, container_release_unified, helm_manual_release) +# 3. create the GitHub release, with GitHub's generated notes +# 4. dispatch "Prepare release" in seaweedfs-csi-driver and seaweedfs-operator, +# which pick up the new master through `go get -u`, and wait for both +# +# Events raised by the default GITHUB_TOKEN do not start other workflows, and it +# cannot reach the other two repositories at all. Add a repo secret RELEASE_PAT +# with `contents: write` here and `actions: write` on the csi-driver and operator +# repos. Without it the tag is still pushed, but nothing downstream of it runs. + +on: + workflow_dispatch: + inputs: + bump: + description: "Which part to increment (ignored when 'version' is set)" + type: choice + options: + - minor + - major + default: minor + version: + description: "Explicit MAJOR.MINOR to set, e.g. 4.36 (overrides 'bump')" + type: string + required: false + downstream: + description: "Also release the CSI driver and the operator" + type: boolean + default: true + dry_run: + description: "Show the version bump, but change nothing" + type: boolean + default: false + +jobs: + release: + runs-on: ubuntu-latest + permissions: + contents: write + outputs: + app_version: ${{ steps.compute.outputs.app_version }} + sha: ${{ steps.tag.outputs.sha }} + steps: + - uses: actions/checkout@v7 + with: + ref: master + fetch-depth: 0 + token: ${{ secrets.RELEASE_PAT || secrets.GITHUB_TOKEN }} + + - name: Check the release token + env: + HAS_PAT: ${{ secrets.RELEASE_PAT != '' }} + run: | + if [ "$HAS_PAT" != "true" ]; then + echo "::warning::RELEASE_PAT is not set. The tag will be pushed with GITHUB_TOKEN, so the binary, container and helm workflows will not start on their own." + fi + + - name: Compute new version + id: compute + env: + BUMP: ${{ inputs.bump }} + INPUT_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + CONST=weed/util/version/constants.go + + MAJOR=$(grep -oP 'MAJOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST") + MINOR=$(grep -oP 'MINOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST") + echo "current: ${MAJOR}.${MINOR}" + + if [ -n "$INPUT_VERSION" ]; then + if ! [[ "$INPUT_VERSION" =~ ^[0-9]+\.[0-9]+$ ]]; then + echo "::error::version must be MAJOR.MINOR, e.g. 5.01 (got '$INPUT_VERSION')" + exit 1 + fi + # 10# forces base 10 so 08/09 are not parsed as octal. + MAJOR=$((10#${INPUT_VERSION%%.*})) + MINOR=$((10#${INPUT_VERSION##*.})) + if [ "$MINOR" -gt 99 ]; then + echo "::error::minor must be 0-99 (got $MINOR); it rolls into the next major at 99" + exit 1 + fi + else + case "$BUMP" in + # Minor is a 2-digit field: 4.99 -> 5.00 -> 5.01. + major) MAJOR=$((MAJOR + 1)); MINOR=0 ;; + minor) + if [ "$MINOR" -ge 99 ]; then + MAJOR=$((MAJOR + 1)); MINOR=0 + else + MINOR=$((MINOR + 1)) + fi + ;; + *) echo "::error::unknown bump '$BUMP'"; exit 1 ;; + esac + fi + + # appVersion mirrors the Go VERSION_NUMBER (zero-padded minor); + # chart version is plain SemVer (no leading zeros). + APP_VERSION=$(printf '%d.%02d' "$MAJOR" "$MINOR") + CHART_VERSION="${MAJOR}.${MINOR}.0" + echo "new: app=${APP_VERSION} chart=${CHART_VERSION}" + + { + echo "major=${MAJOR}" + echo "minor=${MINOR}" + echo "app_version=${APP_VERSION}" + echo "chart_version=${CHART_VERSION}" + } >> "$GITHUB_OUTPUT" + + - name: Apply version to constants.go + env: + MAJOR: ${{ steps.compute.outputs.major }} + MINOR: ${{ steps.compute.outputs.minor }} + run: | + set -euo pipefail + CONST=weed/util/version/constants.go + sed -i -E "s/(MAJOR_VERSION[[:space:]]*=[[:space:]]*int32\()[0-9]+(\))/\1${MAJOR}\2/" "$CONST" + sed -i -E "s/(MINOR_VERSION[[:space:]]*=[[:space:]]*int32\()[0-9]+(\))/\1${MINOR}\2/" "$CONST" + grep -E 'MAJOR_VERSION|MINOR_VERSION' "$CONST" + + - name: Apply version to Chart.yaml + env: + APP_VERSION: ${{ steps.compute.outputs.app_version }} + CHART_VERSION: ${{ steps.compute.outputs.chart_version }} + run: | + set -euo pipefail + CHART=k8s/charts/seaweedfs/Chart.yaml + sed -i -E "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/" "$CHART" + sed -i -E "s/^version:.*/version: ${CHART_VERSION}/" "$CHART" + cat "$CHART" + + - name: Commit, and push the tag + id: tag + env: + TAG: ${{ steps.compute.outputs.app_version }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + set -euo pipefail + if git ls-remote --exit-code --tags origin "refs/tags/${TAG}" >/dev/null 2>&1; then + echo "::error::Tag ${TAG} already exists." + exit 1 + fi + if [ "$DRY_RUN" = "true" ]; then + git --no-pager diff --stat + echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + if git diff --quiet; then + echo "::warning::Version files are already at ${TAG}; tagging the current HEAD." + else + git add weed/util/version/constants.go k8s/charts/seaweedfs/Chart.yaml + git commit -m "${TAG}" + git push + fi + + # Push the tag with git so the `push: tags` triggers fire. Creating the + # tag through the release API alone would only emit a `create` event. + git tag "$TAG" + git push origin "$TAG" + echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Create the release + if: ${{ !inputs.dry_run }} + env: + GH_TOKEN: ${{ secrets.RELEASE_PAT || secrets.GITHUB_TOKEN }} + TAG: ${{ steps.compute.outputs.app_version }} + run: gh release create "$TAG" --title "$TAG" --generate-notes --verify-tag + + downstream: + needs: release + if: ${{ inputs.downstream && !inputs.dry_run }} + runs-on: ubuntu-latest + # The wait below puts no bound of its own on runner-queue time; this does. + timeout-minutes: 120 + permissions: {} + strategy: + fail-fast: false + matrix: + include: + - repo: seaweedfs/seaweedfs-csi-driver + workflow: prepare_release.yaml + - repo: seaweedfs/seaweedfs-operator + workflow: prepare_release.yml + steps: + - name: Release ${{ matrix.repo }} + env: + GH_TOKEN: ${{ secrets.RELEASE_PAT }} + REPO: ${{ matrix.repo }} + WORKFLOW: ${{ matrix.workflow }} + SHA: ${{ needs.release.outputs.sha }} + MODULE: github.com/seaweedfs/seaweedfs + run: | + set -euo pipefail + if [ -z "${GH_TOKEN}" ]; then + echo "::error::RELEASE_PAT with actions:write on ${REPO} is required to release it" + exit 1 + fi + + # The dispatched workflow pins seaweedfs with `go get -u ...@latest`, so + # wait until the proxy serves the release commit as the tip. Asking for + # the commit by name is what makes the proxy fetch it. The proxy can + # take longer than five minutes to refresh @latest after a new tag. + for _ in $(seq 120); do + curl -sf "https://proxy.golang.org/${MODULE}/@v/${SHA}.info" >/dev/null || true + TIP=$(curl -sf "https://proxy.golang.org/${MODULE}/@latest" | jq -r '.Origin.Hash // ""' || true) + [ "$TIP" = "$SHA" ] && break + sleep 10 + done + if [ "$TIP" != "$SHA" ]; then + echo "::error::the module proxy still serves ${TIP} as the tip, so ${REPO} would pin a pre-release commit. Run ${WORKFLOW} there once it catches up." + exit 1 + fi + + # The dispatched workflow publishes the release as its last step, so + # its conclusion decides success. Wait on the run, not on a wall + # clock: time it spends queued for a runner must not count against + # the budget. A dispatch does not return its run id, so take the + # newest workflow_dispatch run created since ours; a concurrent + # dispatch would be performing this same release, and waiting on it + # is just as good. + released() { gh api "repos/${REPO}/releases?per_page=30" --jq '[.[].tag_name]'; } + + BEFORE=$(released) + # A minute early, so runner clock skew cannot hide the run. + DISPATCHED_AT=$(date -u -d '1 minute ago' '+%Y-%m-%dT%H:%M:%SZ') + gh workflow run -R "$REPO" "$WORKFLOW" --ref master -f bump=patch -f update_seaweedfs=true + + RUN_ID="" + for _ in $(seq 12); do + sleep 10 + RUN_ID=$(gh api -X GET "repos/${REPO}/actions/workflows/${WORKFLOW}/runs" \ + -f event=workflow_dispatch -f "created=>=${DISPATCHED_AT}" \ + --jq '(.workflow_runs | sort_by(.created_at) | last | .id) // empty' || true) + [ -n "$RUN_ID" ] && break + done + if [ -z "$RUN_ID" ]; then + echo "::error::the dispatch created no ${WORKFLOW} run in ${REPO}; see https://github.com/${REPO}/actions/workflows/${WORKFLOW}" + exit 1 + fi + RUN_URL="https://github.com/${REPO}/actions/runs/${RUN_ID}" + echo "waiting on ${RUN_URL}" + + # 20 minutes of execution; polls that find the run still queued do + # not consume it. + RUNNING=0 + STATE="" + while :; do + sleep 15 + STATE=$(gh api "repos/${REPO}/actions/runs/${RUN_ID}" \ + --jq '.status + "/" + (.conclusion // "")' || true) + case "$STATE" in + completed/*) break ;; + in_progress/*) RUNNING=$((RUNNING + 1)) ;; + esac + if [ "$RUNNING" -gt 80 ]; then + echo "::error::${RUN_URL} has been executing for over 20 minutes; giving up on it" + exit 1 + fi + done + if [ "$STATE" != "completed/success" ]; then + echo "::error::${RUN_URL} concluded '${STATE#completed/}'" + exit 1 + fi + + NEW=$(released | jq -c --argjson before "$BEFORE" '. - $before') + if [ "$(jq length <<<"$NEW")" -eq 0 ]; then + echo "::error::${RUN_URL} succeeded but ${REPO} shows no new release" + exit 1 + fi + echo "${REPO} released $(jq -r 'join(", ")' <<<"$NEW")" diff --git a/.github/workflows/rust-volume-server-tests.yml b/.github/workflows/rust-volume-server-tests.yml new file mode 100644 index 0000000..93ff56a --- /dev/null +++ b/.github/workflows/rust-volume-server-tests.yml @@ -0,0 +1,249 @@ +name: "Rust Volume Server Tests" + +on: + pull_request: + branches: [ master ] + paths: + - 'seaweed-volume/**' + - 'test/volume_server/**' + - 'weed/pb/volume_server.proto' + - 'weed/pb/volume_server_pb/**' + - '.github/workflows/rust-volume-server-tests.yml' + push: + branches: [ master, main ] + paths: + - 'seaweed-volume/**' + - 'test/volume_server/**' + - 'weed/pb/volume_server.proto' + - 'weed/pb/volume_server_pb/**' + - '.github/workflows/rust-volume-server-tests.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + + +jobs: + rust-unit-tests: + name: Rust Unit Tests + runs-on: ubuntu-22.04 + timeout-minutes: 15 + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + # cargo tracks its own inputs but not the runner's C toolchain, so a cached + # target/ can carry C objects built against a different glibc than we link against. + - name: Fingerprint build toolchain + id: toolchain + run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust-${{ steps.toolchain.outputs.fingerprint }}- + + - name: Build Rust volume server + run: cd seaweed-volume && cargo build --release + + # The crate is warning-free under clippy as of the sweep that added + # this step. Uncomment to make that a gate; `[lints.clippy]` in + # seaweed-volume/Cargo.toml is where crate-wide exceptions live. + # - name: Clippy + # run: cd seaweed-volume && cargo clippy --all-targets -- -D warnings + + # The crate is rustfmt-clean as of the PR that added this step. + # Uncomment to keep it that way. + # - name: Check formatting + # run: cd seaweed-volume && cargo fmt --check + + - name: Run Rust unit tests + run: cd seaweed-volume && cargo test + + - name: Run Rust unit tests (redb experimental cursor) + run: cd seaweed-volume && cargo test --features redb-experimental-cursor --lib storage::needle_map + + rust-integration-tests: + name: Rust Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + # cargo tracks its own inputs but not the runner's C toolchain, so a cached + # target/ can carry C objects built against a different glibc than we link against. + - name: Fingerprint build toolchain + id: toolchain + run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust-${{ steps.toolchain.outputs.fingerprint }}- + + - name: Build Go weed binary + run: | + cd weed + go build -tags 5BytesOffset -o weed . + chmod +x weed + ./weed version + + - name: Build Rust volume binary + run: cd seaweed-volume && cargo build --release + + - name: Run integration tests + env: + WEED_BINARY: ${{ github.workspace }}/weed/weed + RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume + run: | + echo "Running Rust volume server integration tests..." + go test -v -count=1 -timeout=15m ./test/volume_server/rust/... + + - name: Collect logs on failure + if: failure() + run: | + mkdir -p /tmp/rust-volume-server-it-logs + find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-server-it-logs/ \; || true + + - name: Archive logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: rust-volume-server-integration-test-logs + path: /tmp/rust-volume-server-it-logs/ + if-no-files-found: warn + retention-days: 7 + + - name: Test summary + if: always() + run: | + echo "## Rust Volume Server Integration Test Summary" >> "$GITHUB_STEP_SUMMARY" + echo "- Suite: test/volume_server/rust" >> "$GITHUB_STEP_SUMMARY" + echo "- Command: go test -v -count=1 -timeout=15m ./test/volume_server/rust/..." >> "$GITHUB_STEP_SUMMARY" + + rust-volume-go-tests: + name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }}) + runs-on: ubuntu-22.04 + timeout-minutes: 45 + env: + # Keep in step with the length of matrix.shard below. + SHARD_COUNT: 3 + strategy: + fail-fast: false + matrix: + test-type: [grpc, http] + shard: [1, 2, 3] + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + # cargo tracks its own inputs but not the runner's C toolchain, so a cached + # target/ can carry C objects built against a different glibc than we link against. + - name: Fingerprint build toolchain + id: toolchain + run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust-${{ steps.toolchain.outputs.fingerprint }}- + + - name: Build Go weed binary + run: | + cd weed + go build -tags 5BytesOffset -o weed . + chmod +x weed + ./weed version + + - name: Build Rust volume binary + run: cd seaweed-volume && cargo build --release + + # Dealing the listed tests out one by one keeps the shards even. Bucketing + # them by first letter did not: names cluster, so ^Test[I-S] drew 50 of + # the 114 grpc tests and ran nearly twice as long as the other two shards. + - name: Select this shard's tests + env: + TEST_TYPE: ${{ matrix.test-type }} + SHARD: ${{ matrix.shard }} + run: | + tests=$(go test -tags 5BytesOffset ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u) + # An empty list would make -run match nothing and the shard pass vacuously. + [ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; } + selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1') + echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests" + echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV" + + - name: Run volume server integration tests with Rust volume + env: + WEED_BINARY: ${{ github.workspace }}/weed/weed + RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume + VOLUME_SERVER_IMPL: rust + run: | + echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..." + go test -v -count=1 -tags 5BytesOffset -timeout=30m ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}" + + - name: Collect logs on failure + if: failure() + run: | + mkdir -p /tmp/rust-volume-go-test-logs + find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-go-test-logs/ \; || true + + - name: Archive logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: rust-volume-go-test-logs-${{ matrix.test-type }}-shard${{ matrix.shard }} + path: /tmp/rust-volume-go-test-logs/ + if-no-files-found: warn + retention-days: 7 + + - name: Test summary + if: always() + run: | + echo "## Rust Volume - Go Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY" + echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY" + echo "- Volume server: Rust (VOLUME_SERVER_IMPL=rust)" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/rust-worker-tests.yml b/.github/workflows/rust-worker-tests.yml new file mode 100644 index 0000000..ae08c06 --- /dev/null +++ b/.github/workflows/rust-worker-tests.yml @@ -0,0 +1,93 @@ +name: "Rust Plugin Worker Tests" + +on: + pull_request: + branches: [ master ] + paths: + - 'seaweed-worker/**' + - 'weed/pb/plugin.proto' + - '.github/workflows/rust-worker-tests.yml' + push: + branches: [ master, main ] + paths: + - 'seaweed-worker/**' + - 'weed/pb/plugin.proto' + - '.github/workflows/rust-worker-tests.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + rust-worker-build: + name: Rust Plugin Worker Build and Unit Tests + runs-on: ubuntu-22.04 + timeout-minutes: 45 + + steps: + - name: Checkout code + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + # cargo tracks its own inputs but not the runner's C toolchain, so a cached + # target/ can carry C objects built against a different glibc than we link against. + - name: Fingerprint build toolchain + id: toolchain + run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-worker/target/release + key: rust-worker-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-worker/Cargo.lock') }} + restore-keys: | + rust-worker-${{ steps.toolchain.outputs.fingerprint }}- + + # lance's build scripts compile their own protos and look for a protoc. + # Point them at the one protoc-bin-vendored ships, which seaweed-worker's + # own build already uses, so no job depends on a system package and every + # build sees the same version. + - name: Use the vendored protoc + run: | + cd seaweed-worker + cargo fetch + # The version from the lock, not whatever else a restored cache holds. + version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock) + test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; } + protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1) + test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; } + echo "PROTOC=$protoc" >> "$GITHUB_ENV" + + # The release profile is what ships, and it is where the release and the + # container builds would otherwise discover a break for the first time. + - name: Build the plugin workers + run: cd seaweed-worker && cargo build --release + + # The workspace is warning-free under clippy as of the sweep that added + # this step. Uncomment to make that a gate; `[workspace.lints.clippy]` + # in seaweed-worker/Cargo.toml is where crate-wide exceptions live. + # - name: Clippy + # run: cd seaweed-worker && cargo clippy --workspace --all-targets -- -D warnings + + # The workspace is rustfmt-clean as of the PR that added this step. + # Uncomment to keep it that way. + # - name: Check formatting + # run: cd seaweed-worker && cargo fmt --all --check + + # The tests that need a live gateway skip themselves without one, the way + # the Go integration tests skip without Docker; the lifecycle suite in + # test/s3tables/lifecycle is what runs them against a real cluster. + # Release, so this reuses the build above rather than compiling lance, + # arrow and datafusion a second time in another profile. + - name: Run unit tests + run: cd seaweed-worker && cargo test --release --workspace diff --git a/.github/workflows/rust_binaries_dev.yml b/.github/workflows/rust_binaries_dev.yml new file mode 100644 index 0000000..c8cb4f0 --- /dev/null +++ b/.github/workflows/rust_binaries_dev.yml @@ -0,0 +1,160 @@ +name: "rust: build dev volume server binaries" + +on: + push: + branches: [ master ] + paths: + - 'seaweed-volume/**' + - '.github/workflows/rust_binaries_dev.yml' + +permissions: + contents: read + +jobs: + + cleanup: + permissions: + contents: write + runs-on: ubuntu-latest + steps: + - name: Delete old Rust volume dev assets + uses: mknejp/delete-release-assets@v1 + continue-on-error: true + with: + token: ${{ github.token }} + tag: dev + fail-if-no-assets: false + assets: | + weed-volume-* + + build-rust-volume-dev-linux: + permissions: + contents: write + needs: cleanup + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - target: x86_64-unknown-linux-gnu + asset_suffix: linux-amd64 + + steps: + - uses: actions/checkout@v7 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust-dev-${{ matrix.target }}- + + - name: Set BUILD_TIME + run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV" + + - name: Build Rust volume server (large disk) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: cd seaweed-volume && cargo build --release + + - name: Package large disk binary + run: | + cp seaweed-volume/target/release/weed-volume weed-volume-large-disk + tar czf "weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-large-disk + rm weed-volume-large-disk + + - name: Build Rust volume server (normal) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: cd seaweed-volume && cargo build --release --no-default-features + + - name: Package normal binary + run: | + cp seaweed-volume/target/release/weed-volume weed-volume-normal + tar czf "weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-normal + rm weed-volume-normal + + - name: Upload dev release assets + uses: softprops/action-gh-release@v3 + with: + tag_name: dev + prerelease: true + files: | + weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz + weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + build-rust-volume-dev-darwin: + permissions: + contents: write + needs: build-rust-volume-dev-linux + runs-on: macos-latest + strategy: + matrix: + include: + - target: aarch64-apple-darwin + asset_suffix: darwin-arm64 + - target: x86_64-apple-darwin + asset_suffix: darwin-amd64 + + steps: + - uses: actions/checkout@v7 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust-dev-${{ matrix.target }}- + + - name: Set BUILD_TIME + run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV" + + - name: Build Rust volume server (large disk) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }} + + - name: Package large disk binary + run: | + cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk + tar czf "weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-large-disk + rm weed-volume-large-disk + + - name: Build Rust volume server (normal) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }} --no-default-features + + - name: Package normal binary + run: | + cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal + tar czf "weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-normal + rm weed-volume-normal + + - name: Upload dev release assets + uses: softprops/action-gh-release@v3 + with: + tag_name: dev + prerelease: true + files: | + weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz + weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/rust_binaries_release.yml b/.github/workflows/rust_binaries_release.yml new file mode 100644 index 0000000..a5a8b61 --- /dev/null +++ b/.github/workflows/rust_binaries_release.yml @@ -0,0 +1,371 @@ +name: "rust: build versioned binaries" + +on: + push: + tags: + - '*' + + workflow_dispatch: + +permissions: + contents: read + +jobs: + + build-rust-volume-linux: + permissions: + contents: write + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - target: x86_64-unknown-linux-gnu + asset_suffix: linux_amd64 + - target: aarch64-unknown-linux-gnu + asset_suffix: linux_arm64 + cross: true + + steps: + - uses: actions/checkout@v7 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install cross-compilation tools + if: matrix.cross + run: | + sudo dpkg --add-architecture arm64 + sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list + echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list + echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list + sudo apt-get update + sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64 + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=/usr" >> "$GITHUB_ENV" + echo "OPENSSL_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV" + echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV" + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust-release-${{ matrix.target }}- + + - name: Build Rust volume server (large disk) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} --target-dir target/large-disk + + - name: Build Rust volume server (normal) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} --no-default-features --target-dir target/normal + + - name: Package binaries + run: | + # Large disk (default, 5bytes feature) + cp seaweed-volume/target/large-disk/${{ matrix.target }}/release/weed-volume weed-volume-large-disk + tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk + rm weed-volume-large-disk + + # Normal volume size + cp seaweed-volume/target/normal/${{ matrix.target }}/release/weed-volume weed-volume-normal + tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal + rm weed-volume-normal + + - name: Generate md5 checksums + run: | + for f in weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz; do + md5sum "$f" > "$f.md5" + done + + - name: Upload release assets + if: startsWith(github.ref, 'refs/tags/') + uses: softprops/action-gh-release@v3 + with: + files: | + weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz + weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5 + weed-volume_${{ matrix.asset_suffix }}.tar.gz + weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload artifacts + if: ${{ !startsWith(github.ref, 'refs/tags/') }} + uses: actions/upload-artifact@v7 + with: + name: rust-volume-${{ matrix.asset_suffix }} + path: | + weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz + weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5 + weed-volume_${{ matrix.asset_suffix }}.tar.gz + weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5 + + # The Rust maintenance worker: Linux only, because it runs beside the cluster + # it maintains rather than on a laptop, and its dependency tree (lance, arrow, + # datafusion) makes every extra target an expensive build. + build-rust-worker-linux: + permissions: + contents: write + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - target: x86_64-unknown-linux-gnu + asset_suffix: linux_amd64 + - target: aarch64-unknown-linux-gnu + asset_suffix: linux_arm64 + cross: true + + steps: + - uses: actions/checkout@v7 + with: + # The upload step is handed a token explicitly; a cargo build script + # should not find another one sitting in the checkout's git config. + persist-credentials: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install cross-compilation tools + if: matrix.cross + run: | + sudo dpkg --add-architecture arm64 + sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list + echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list + echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list + sudo apt-get update + sudo apt-get install -y gcc-aarch64-linux-gnu + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV" + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-worker/target/${{ matrix.target }}/release + key: rust-worker-release-${{ matrix.target }}-${{ hashFiles('seaweed-worker/Cargo.lock') }} + restore-keys: | + rust-worker-release-${{ matrix.target }}- + + # lance's build scripts compile their own protos and look for a protoc. + # Point them at the one protoc-bin-vendored ships, which seaweed-worker's + # own build already uses, so no job depends on a system package and every + # build sees the same version. + - name: Use the vendored protoc + run: | + cd seaweed-worker + cargo fetch + # The version from the lock, not whatever else a restored cache holds. + version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock) + test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; } + protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1) + test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; } + echo "PROTOC=$protoc" >> "$GITHUB_ENV" + + - name: Build the Rust maintenance worker + run: | + cd seaweed-worker + cargo build --release -p weed-lance-worker --target ${{ matrix.target }} + + - name: Package binary + run: | + cp seaweed-worker/target/${{ matrix.target }}/release/weed-worker weed-worker + tar czf weed-worker_${{ matrix.asset_suffix }}.tar.gz weed-worker + rm weed-worker + md5sum weed-worker_${{ matrix.asset_suffix }}.tar.gz > weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5 + + - name: Upload release assets + if: startsWith(github.ref, 'refs/tags/') + uses: softprops/action-gh-release@v3 + with: + files: | + weed-worker_${{ matrix.asset_suffix }}.tar.gz + weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload artifacts + if: ${{ !startsWith(github.ref, 'refs/tags/') }} + uses: actions/upload-artifact@v7 + with: + name: rust-worker-${{ matrix.asset_suffix }} + path: | + weed-worker_${{ matrix.asset_suffix }}.tar.gz + weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5 + + build-rust-volume-darwin: + permissions: + contents: write + runs-on: macos-latest + strategy: + matrix: + include: + - target: x86_64-apple-darwin + asset_suffix: darwin_amd64 + - target: aarch64-apple-darwin + asset_suffix: darwin_arm64 + + steps: + - uses: actions/checkout@v7 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust-release-${{ matrix.target }}- + + - name: Build Rust volume server (large disk) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} --target-dir target/large-disk + + - name: Build Rust volume server (normal) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} --no-default-features --target-dir target/normal + + - name: Package binaries + run: | + cp seaweed-volume/target/large-disk/${{ matrix.target }}/release/weed-volume weed-volume-large-disk + tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk + rm weed-volume-large-disk + + cp seaweed-volume/target/normal/${{ matrix.target }}/release/weed-volume weed-volume-normal + tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal + rm weed-volume-normal + + - name: Generate md5 checksums + run: | + for f in weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz; do + md5 -r "$f" > "$f.md5" + done + + - name: Upload release assets + if: startsWith(github.ref, 'refs/tags/') + uses: softprops/action-gh-release@v3 + with: + files: | + weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz + weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5 + weed-volume_${{ matrix.asset_suffix }}.tar.gz + weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload artifacts + if: ${{ !startsWith(github.ref, 'refs/tags/') }} + uses: actions/upload-artifact@v7 + with: + name: rust-volume-${{ matrix.asset_suffix }} + path: | + weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz + weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5 + weed-volume_${{ matrix.asset_suffix }}.tar.gz + weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5 + + build-rust-volume-windows: + permissions: + contents: write + runs-on: windows-latest + + steps: + - uses: actions/checkout@v7 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-release-windows-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust-release-windows- + + - name: Build Rust volume server (large disk) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target-dir target/large-disk + + - name: Build Rust volume server (normal) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --no-default-features --target-dir target/normal + + - name: Package binaries + shell: bash + run: | + cp seaweed-volume/target/large-disk/release/weed-volume.exe weed-volume-large-disk.exe + 7z a weed-volume_large_disk_windows_amd64.zip weed-volume-large-disk.exe + rm weed-volume-large-disk.exe + + cp seaweed-volume/target/normal/release/weed-volume.exe weed-volume-normal.exe + 7z a weed-volume_windows_amd64.zip weed-volume-normal.exe + rm weed-volume-normal.exe + + - name: Generate md5 checksums + shell: bash + run: | + for f in weed-volume_large_disk_windows_amd64.zip weed-volume_windows_amd64.zip; do + md5sum "$f" > "$f.md5" + done + + - name: Upload release assets + if: startsWith(github.ref, 'refs/tags/') + uses: softprops/action-gh-release@v3 + with: + files: | + weed-volume_large_disk_windows_amd64.zip + weed-volume_large_disk_windows_amd64.zip.md5 + weed-volume_windows_amd64.zip + weed-volume_windows_amd64.zip.md5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload artifacts + if: ${{ !startsWith(github.ref, 'refs/tags/') }} + uses: actions/upload-artifact@v7 + with: + name: rust-volume-windows_amd64 + path: | + weed-volume_large_disk_windows_amd64.zip + weed-volume_large_disk_windows_amd64.zip.md5 + weed-volume_windows_amd64.zip + weed-volume_windows_amd64.zip.md5 diff --git a/.github/workflows/s3-etag-acl-tests.yml b/.github/workflows/s3-etag-acl-tests.yml new file mode 100644 index 0000000..12425f6 --- /dev/null +++ b/.github/workflows/s3-etag-acl-tests.yml @@ -0,0 +1,129 @@ +name: "S3 ETag and ACL Tests" + +on: + push: + branches: [ master ] + paths: + - 'weed/s3api/**' + - 'weed/filer/etag*.go' + - 'weed/server/filer_server_handlers_*.go' + - 'test/s3/etag/**' + - 'test/s3/acl/**' + - '.github/workflows/s3-etag-acl-tests.yml' + pull_request: + branches: [ master ] + paths: + - 'weed/s3api/**' + - 'weed/filer/etag*.go' + - 'weed/server/filer_server_handlers_*.go' + - 'test/s3/etag/**' + - 'test/s3/acl/**' + - '.github/workflows/s3-etag-acl-tests.yml' + +concurrency: + group: ${{ github.head_ref || github.ref }}/s3-etag-acl-tests + cancel-in-progress: true + +permissions: + contents: read + +jobs: + s3-etag-acl-tests: + name: S3 ETag + ACL Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 15 + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Install SeaweedFS + run: | + cd weed && go install -buildvcs=false + + - name: Start weed mini (S3 on :8333) + run: | + mkdir -p /tmp/seaweedfs-etag-acl + # Minimal identity config so SSE-aware tests under acl/ can authenticate. + cat > /tmp/seaweedfs-etag-acl-s3.json <<'JSON' + { + "identities": [ + { + "name": "admin", + "credentials": [ + {"accessKey": "some_access_key1", "secretKey": "some_secret_key1"} + ], + "actions": ["Admin", "Read", "Write"] + } + ] + } + JSON + AWS_ACCESS_KEY_ID=some_access_key1 \ + AWS_SECRET_ACCESS_KEY=some_secret_key1 \ + weed mini \ + -dir=/tmp/seaweedfs-etag-acl \ + -s3.port=8333 \ + -s3.config=/tmp/seaweedfs-etag-acl-s3.json \ + -ip=127.0.0.1 \ + > /tmp/weed-mini.log 2>&1 & + echo $! > /tmp/weed-mini.pid + + # Wait for the S3 endpoint to come up (returns 403 unauth before any + # request is signed; that's fine — it means the server is listening). + for i in $(seq 1 30); do + if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then + echo "weed mini is ready" + exit 0 + fi + sleep 1 + done + echo "weed mini failed to start within 30s" + tail -50 /tmp/weed-mini.log + exit 1 + + - name: Run ETag tests + # Pins the regression for #7768: PutObject of an auto-chunked file (>8MB) + # must return a pure MD5 hex ETag, not a `-N` composite — the AWS + # SDK for Java v2 rejects the latter on the PutObject path. + env: + S3_ENDPOINT: http://127.0.0.1:8333 + AWS_ACCESS_KEY_ID: some_access_key1 + AWS_SECRET_ACCESS_KEY: some_secret_key1 + AWS_REGION: us-east-1 + run: go test -v -timeout=5m ./test/s3/etag/... + + - name: Run ACL versioning tests + # Pins object-ACL behavior on a versioned bucket: GetObjectAcl / + # PutObjectAcl with and without versionId, modifying ACLs on different + # versions independently. + env: + S3_ENDPOINT: http://127.0.0.1:8333 + AWS_ACCESS_KEY_ID: some_access_key1 + AWS_SECRET_ACCESS_KEY: some_secret_key1 + AWS_REGION: us-east-1 + run: go test -v -timeout=5m ./test/s3/acl/... + + - name: Stop weed mini + if: always() + run: | + if [ -f /tmp/weed-mini.pid ]; then + kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true + fi + + - name: Show server log on failure + if: failure() + run: | + echo "=== weed mini log (last 200 lines) ===" + tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available" + + - name: Archive log + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-etag-acl-server-log + path: /tmp/weed-mini.log + retention-days: 3 diff --git a/.github/workflows/s3-example-integration-tests.yml b/.github/workflows/s3-example-integration-tests.yml new file mode 100644 index 0000000..4e39c3b --- /dev/null +++ b/.github/workflows/s3-example-integration-tests.yml @@ -0,0 +1,89 @@ +name: "S3 Authenticated Integration Tests" + +on: + pull_request: + paths: + - 'weed/s3api/**' + - 'weed/iam/**' + - 'weed/filer/**' + - 'weed/server/**' + - 'test/s3/normal/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/s3-example-integration-tests.yml' + +concurrency: + group: ${{ github.head_ref }}/s3-integration-tests + cancel-in-progress: true + +permissions: + contents: read + +jobs: + tests: + name: S3 Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Build SeaweedFS + run: | + cd weed + go build -o weed -buildvcs=false + + - name: Run S3 Integration Tests + timeout-minutes: 15 + working-directory: test/s3/normal + run: | + set -x + echo "=== Running S3 Integration Tests ===" + go test -v -timeout=60s -run TestS3Integration ./... + + - name: Run S3 DeleteBucketNotEmpty Tests + timeout-minutes: 15 + working-directory: test/s3/normal + run: | + set -x + echo "=== Running S3 DeleteBucketNotEmpty Tests ===" + go test -v -timeout=60s -run TestS3DeleteBucketNotEmpty ./... + + - name: Run S3 Empty Directory Marker Tests + timeout-minutes: 15 + working-directory: test/s3/normal + run: | + set -x + echo "=== Running S3 Empty Directory Marker Tests ===" + go test -v -timeout=180s -run TestS3ListObjectsEmptyDirectoryMarkers ./... + + - name: Run S3 Prefix Object Tests + timeout-minutes: 15 + working-directory: test/s3/normal + run: | + set -x + echo "=== Running S3 Prefix Object Tests ===" + go test -v -timeout=180s -run TestS3PrefixObjectKeys ./... + + - name: Run IAM Integration Tests + timeout-minutes: 15 + working-directory: test/s3/normal + run: | + set -x + echo "=== Running IAM Integration Tests ===" + go test -v -timeout=60s -run TestIAMOperations ./... + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: integration-test-logs + path: test/s3/normal/*.log + retention-days: 3 diff --git a/.github/workflows/s3-filer-group-tests.yml b/.github/workflows/s3-filer-group-tests.yml new file mode 100644 index 0000000..bda1ac2 --- /dev/null +++ b/.github/workflows/s3-filer-group-tests.yml @@ -0,0 +1,94 @@ +name: "S3 Filer Group Tests" + +on: + pull_request: + paths: + - 'weed/s3api/**' + - 'weed/filer/**' + - 'weed/server/**' + - 'test/s3/filer_group/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/s3-filer-group-tests.yml' + +concurrency: + group: ${{ github.head_ref }}/s3-filer-group-tests + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + working-directory: weed + +jobs: + s3-filer-group-tests: + name: S3 Filer Group Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run S3 Filer Group Tests + timeout-minutes: 15 + working-directory: test/s3/filer_group + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting Filer Group Tests ===" + + # Run tests with automatic server management + # The test-with-server target handles server startup/shutdown automatically + # Server is started with -filer.group=testgroup + make test-with-server || { + echo "❌ Filer group tests failed, checking logs..." + if [ -f weed-test.log ]; then + echo "=== Server logs ===" + tail -100 weed-test.log + fi + echo "=== Process information ===" + ps aux | grep -E "(weed|test)" || true + exit 1 + } + + - name: Show server logs on failure + if: failure() + working-directory: test/s3/filer_group + run: | + echo "=== Server Logs ===" + if [ -f weed-test.log ]; then + echo "Last 100 lines of server logs:" + tail -100 weed-test.log + else + echo "No server log file found" + fi + + echo "=== Test Environment ===" + ps aux | grep -E "(weed|test)" || true + netstat -tlnp | grep -E "(8333|9333|8080)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-filer-group-test-logs + path: test/s3/filer_group/weed-test*.log + retention-days: 3 + + diff --git a/.github/workflows/s3-go-tests.yml b/.github/workflows/s3-go-tests.yml new file mode 100644 index 0000000..19944e9 --- /dev/null +++ b/.github/workflows/s3-go-tests.yml @@ -0,0 +1,676 @@ +name: "S3 Go Tests" + +on: + pull_request: + paths: + - 'weed/s3api/**' + - 'weed/filer/**' + - 'weed/server/**' + - 'test/s3/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/s3-go-tests.yml' + +concurrency: + group: ${{ github.head_ref }}/s3-go-tests + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + working-directory: weed + +jobs: + s3-versioning-tests: + name: S3 Versioning Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + strategy: + matrix: + test-type: ["quick", "comprehensive"] + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run S3 Versioning Tests - ${{ matrix.test-type }} + timeout-minutes: 25 + working-directory: test/s3/versioning + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting Tests ===" + + # Run tests with automatic server management + # The test-with-server target handles server startup/shutdown automatically + if [ "${{ matrix.test-type }}" = "quick" ]; then + # Override TEST_PATTERN for quick tests only + make test-with-server TEST_PATTERN="TestBucketListReturnDataVersioning|TestVersioningBasicWorkflow|TestVersioningDeleteMarkers" + else + # Run all versioning tests + make test-with-server + fi + + - name: Show server logs on failure + if: failure() + working-directory: test/s3/versioning + run: | + echo "=== Server Logs ===" + if [ -f weed-test.log ]; then + echo "Last 100 lines of server logs:" + tail -100 weed-test.log + else + echo "No server log file found" + fi + + echo "=== Test Environment ===" + ps aux | grep -E "(weed|test)" || true + netstat -tlnp | grep -E "(8333|9333|8080)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-versioning-test-logs-${{ matrix.test-type }} + path: test/s3/versioning/weed-test*.log + retention-days: 3 + + s3-versioning-compatibility: + name: S3 Versioning Compatibility Test + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run Core Versioning Test (Python s3tests equivalent) + timeout-minutes: 15 + working-directory: test/s3/versioning + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + + # Run the specific test that is equivalent to the Python s3tests + make test-with-server || { + echo "❌ Test failed, checking logs..." + if [ -f weed-test.log ]; then + echo "=== Server logs ===" + tail -100 weed-test.log + fi + echo "=== Process information ===" + ps aux | grep -E "(weed|test)" || true + exit 1 + } + + - name: Upload server logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-versioning-compatibility-logs + path: test/s3/versioning/weed-test*.log + retention-days: 3 + + s3-cors-compatibility: + name: S3 CORS Compatibility Test + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run Core CORS Test (AWS S3 compatible) + timeout-minutes: 15 + working-directory: test/s3/cors + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + + # Run the specific test that is equivalent to AWS S3 CORS behavior + make test-with-server || { + echo "❌ Test failed, checking logs..." + if [ -f weed-test.log ]; then + echo "=== Server logs ===" + tail -100 weed-test.log + fi + echo "=== Process information ===" + ps aux | grep -E "(weed|test)" || true + exit 1 + } + + - name: Upload server logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-cors-compatibility-logs + path: test/s3/cors/weed-test*.log + retention-days: 3 + + s3-retention-tests: + name: S3 Retention Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + strategy: + matrix: + test-type: ["quick", "comprehensive"] + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run S3 Retention Tests - ${{ matrix.test-type }} + timeout-minutes: 25 + working-directory: test/s3/retention + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting Tests ===" + + # Run tests with automatic server management + # The test-with-server target handles server startup/shutdown automatically + if [ "${{ matrix.test-type }}" = "quick" ]; then + # Override TEST_PATTERN for quick tests only + make test-with-server TEST_PATTERN="TestBasicRetentionWorkflow|TestRetentionModeCompliance|TestLegalHoldWorkflow" + else + # Run all retention tests + make test-with-server + fi + + - name: Show server logs on failure + if: failure() + working-directory: test/s3/retention + run: | + echo "=== Server Logs ===" + if [ -f weed-test.log ]; then + echo "Last 100 lines of server logs:" + tail -100 weed-test.log + else + echo "No server log file found" + fi + + echo "=== Test Environment ===" + ps aux | grep -E "(weed|test)" || true + netstat -tlnp | grep -E "(8333|9333|8080)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-retention-test-logs-${{ matrix.test-type }} + path: test/s3/retention/weed-test*.log + retention-days: 3 + + s3-lifecycle-tests: + name: S3 Lifecycle Tests + runs-on: ubuntu-22.04 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + # One job per test so each gets a fresh `weed mini` server, avoiding + # the cross-test volume-pool exhaustion that surfaced when several + # TTL-pinned bucket collections piled up in a single run. + test: + - TestLifecycleAbortIncompleteMultipartUpload + - TestLifecycleAdminDispatchSucceedsWithCustomFilerGrpcPort + - TestLifecycleBootstrapWalkOnExistingObjects + - TestLifecycleConfigUpdateBetweenSweeps + - TestLifecycleDeleteBucketLifecycleStopsDispatching + - TestLifecycleDisabledRuleSkipsObject + - TestLifecycleEmptyBucketSweepIsNoOp + - TestLifecycleExpirationDateInThePast + - TestLifecycleExpirationFiresOnBackdatedObject + - TestLifecycleExpiredDeleteMarkerCleanup + - TestLifecycleMultipleBucketsInOneSweep + - TestLifecycleMultipleRulesInOneBucket + - TestLifecycleNewerNoncurrentVersions + - TestLifecycleNoncurrentVersionExpiration + - TestLifecycleSizeFilterGreaterThan + - TestLifecycleSkipsObjectLockedObjects + - TestLifecycleSuspendedVersioningExpiration + - TestLifecycleTagFilter + - TestLifecycleVersionedBucketCreatesDeleteMarker + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run ${{ matrix.test }} + timeout-minutes: 8 + working-directory: test/s3/lifecycle + run: | + set -x + make test-with-server TEST_PATTERN='^${{ matrix.test }}$$' + + - name: Show server logs on failure + if: failure() + working-directory: test/s3/lifecycle + run: | + if [ -f weed-test.log ]; then + echo "=== Last 200 lines of server logs ===" + tail -200 weed-test.log + fi + ps aux | grep -E "(weed|test)" || true + netstat -tlnp 2>/dev/null | grep -E "(8333|9333|8080|8888)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-lifecycle-test-logs-${{ matrix.test }} + path: test/s3/lifecycle/weed-test*.log + retention-days: 3 + + s3-checksum-tests: + name: S3 Checksum Tests + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run S3 Checksum Tests + timeout-minutes: 16 + working-directory: test/s3/checksum + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting Tests ===" + make test-with-server + + - name: Show server logs on failure + if: failure() + working-directory: test/s3/checksum + run: | + echo "=== Server Logs ===" + if [ -f weed-test.log ]; then + echo "Last 100 lines of server logs:" + tail -100 weed-test.log + else + echo "No server log file found" + fi + + echo "=== Test Environment ===" + ps aux | grep -E "(weed|test)" || true + netstat -tlnp | grep -E "(8333|9333|8080)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-checksum-test-logs + path: test/s3/checksum/weed-test*.log + retention-days: 3 + + s3-cors-tests: + name: S3 CORS Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + strategy: + matrix: + test-type: ["quick", "comprehensive"] + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run S3 CORS Tests - ${{ matrix.test-type }} + timeout-minutes: 25 + working-directory: test/s3/cors + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting Tests ===" + + # Run tests with automatic server management + # The test-with-server target handles server startup/shutdown automatically + if [ "${{ matrix.test-type }}" = "quick" ]; then + # Override TEST_PATTERN for quick tests only + make test-with-server TEST_PATTERN="TestCORSConfigurationManagement|TestServiceLevelCORS|TestCORSBasicWorkflow" + else + # Run all CORS tests + make test-with-server + fi + + - name: Show server logs on failure + if: failure() + working-directory: test/s3/cors + run: | + echo "=== Server Logs ===" + if [ -f weed-test.log ]; then + echo "Last 100 lines of server logs:" + tail -100 weed-test.log + else + echo "No server log file found" + fi + + echo "=== Test Environment ===" + ps aux | grep -E "(weed|test)" || true + netstat -tlnp | grep -E "(8333|9333|8080)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-cors-test-logs-${{ matrix.test-type }} + path: test/s3/cors/weed-test*.log + retention-days: 3 + + s3-retention-worm: + name: S3 Retention WORM Integration Test + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run WORM Integration Tests + timeout-minutes: 15 + working-directory: test/s3/retention + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + + # Run the WORM integration tests with automatic server management + # The test-with-server target handles server startup/shutdown automatically + make test-with-server TEST_PATTERN="TestWORM|TestRetentionExtendedAttributes|TestRetentionConcurrentOperations" || { + echo "❌ WORM integration test failed, checking logs..." + if [ -f weed-test.log ]; then + echo "=== Server logs ===" + tail -100 weed-test.log + fi + echo "=== Process information ===" + ps aux | grep -E "(weed|test)" || true + exit 1 + } + + - name: Upload server logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-retention-worm-logs + path: test/s3/retention/weed-test*.log + retention-days: 3 + + s3-versioning-stress: + name: S3 Versioning Stress Test + runs-on: ubuntu-22.04 + timeout-minutes: 35 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run S3 Versioning Stress Tests + timeout-minutes: 30 + working-directory: test/s3/versioning + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + + # Start server for stress tests + make start-server + + # Run stress tests (concurrent operations) + make test-versioning-stress || { + echo "❌ Stress test failed, checking logs..." + if [ -f weed-test.log ]; then + echo "=== Server logs ===" + tail -200 weed-test.log + fi + make stop-server + make clean + exit 1 + } + + # Run pagination stress tests (>1000 versions) + echo "=== Running pagination stress tests ===" + make test-versioning-pagination-stress || { + echo "❌ Pagination stress test failed, checking logs..." + if [ -f weed-test.log ]; then + echo "=== Server logs ===" + tail -200 weed-test.log + fi + make stop-server + make clean + exit 1 + } + + make stop-server + make clean + + - name: Upload stress test logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: s3-versioning-stress-logs + path: test/s3/versioning/weed-test*.log + retention-days: 7 + + s3-tagging-tests: + # CI job for S3 object tagging tests + name: S3 Tagging Tests + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run S3 Tagging Tests + timeout-minutes: 15 + working-directory: test/s3/tagging + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + + # Set environment variables for the test + export S3_ENDPOINT="http://localhost:8006" + export S3_ACCESS_KEY="0555b35654ad1656d804" + export S3_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==" + export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY" + export AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY" + + # Run the specific test that is equivalent to AWS S3 tagging behavior + make test-with-server || { + echo "❌ Test failed, checking logs..." + if [ -f weed-test.log ]; then + echo "=== Server logs ===" + tail -100 weed-test.log + fi + echo "=== Process information ===" + ps aux | grep -E "(weed|test)" || true + exit 1 + } + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-tagging-test-logs + path: test/s3/tagging/weed-test*.log + retention-days: 3 + + s3-remote-cache-tests: + name: S3 Remote Cache Tests + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run S3 Remote Cache Tests + timeout-minutes: 15 + working-directory: test/s3/remote_cache + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + + # Run the remote cache integration tests + # Tests singleflight deduplication for caching remote objects + make test-with-server || { + echo "❌ Test failed, checking logs..." + if [ -f primary-weed.log ]; then + echo "=== Primary server logs ===" + tail -100 primary-weed.log + fi + if [ -f remote-weed.log ]; then + echo "=== Remote server logs ===" + tail -100 remote-weed.log + fi + echo "=== Process information ===" + ps aux | grep -E "(weed|test)" || true + exit 1 + } + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-remote-cache-test-logs + path: | + test/s3/remote_cache/primary-weed.log + test/s3/remote_cache/remote-weed.log + retention-days: 3 + + # Removed SSE-C integration tests and compatibility job \ No newline at end of file diff --git a/.github/workflows/s3-iam-tests.yml b/.github/workflows/s3-iam-tests.yml new file mode 100644 index 0000000..121f835 --- /dev/null +++ b/.github/workflows/s3-iam-tests.yml @@ -0,0 +1,309 @@ +name: "S3 IAM Integration Tests" + +on: + pull_request: + paths: + - 'weed/iam/**' + - 'weed/s3api/**' + - 'weed/credential/**' + - 'weed/pb/**' + - 'test/s3/iam/**' + - '.github/workflows/s3-iam-tests.yml' + push: + branches: [ master ] + paths: + - 'weed/iam/**' + - 'weed/s3api/**' + - 'weed/credential/**' + - 'weed/pb/**' + - 'test/s3/iam/**' + - '.github/workflows/s3-iam-tests.yml' + +concurrency: + group: ${{ github.head_ref }}/s3-iam-tests + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + working-directory: weed + +jobs: + # Unit tests for IAM components + iam-unit-tests: + name: IAM Unit Tests + runs-on: ubuntu-22.04 + timeout-minutes: 15 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Get dependencies + run: | + go mod download + + - name: Run IAM Unit Tests + timeout-minutes: 10 + run: | + set -x + echo "=== Running IAM STS Tests ===" + go test -v -timeout 5m ./iam/sts/... + + echo "=== Running IAM Policy Tests ===" + go test -v -timeout 5m ./iam/policy/... + + echo "=== Running IAM Integration Tests ===" + go test -v -timeout 5m ./iam/integration/... + + echo "=== Running S3 API IAM Tests ===" + go test -v -timeout 5m ./s3api/... -run ".*IAM.*|.*JWT.*|.*Auth.*" + + - name: Upload test results on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: iam-unit-test-results + path: | + weed/testdata/ + weed/**/testdata/ + retention-days: 3 + + # S3 IAM integration tests with SeaweedFS services + s3-iam-integration-tests: + name: S3 IAM Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 25 + strategy: + matrix: + test-type: ["basic", "advanced", "policy-enforcement", "group", "sts"] + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + working-directory: weed + run: | + go install -buildvcs=false + + - name: Run S3 IAM Integration Tests - ${{ matrix.test-type }} + timeout-minutes: 20 + working-directory: test/s3/iam + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting S3 IAM Integration Tests (${{ matrix.test-type }}) ===" + + # Set WEED_BINARY to use the installed version + export WEED_BINARY=$(which weed) + export TEST_TIMEOUT=15m + + # Run tests based on type + case "${{ matrix.test-type }}" in + "basic") + echo "Running basic IAM functionality tests..." + make clean setup start-services wait-for-services + go test -v -timeout 15m -run "TestS3IAMAuthentication|TestS3IAMBasicWorkflow|TestS3IAMTokenValidation|TestIAMUserManagement|TestIAMAccessKeyManagement|TestIAMPolicyManagement" ./... + ;; + "advanced") + echo "Running advanced IAM feature tests..." + make clean setup start-services wait-for-services + go test -v -timeout 15m -run "TestS3IAMSessionExpiration|TestS3IAMMultipart|TestS3IAMPresigned" ./... + ;; + "policy-enforcement") + echo "Running policy enforcement tests..." + make clean setup start-services wait-for-services + go test -v -timeout 15m -run "TestS3IAMPolicyEnforcement|TestS3IAMBucketPolicy|TestS3IAMContextual" ./... + ;; + "group") + echo "Running IAM group management tests..." + make clean setup start-services wait-for-services + go test -v -timeout 15m -run "TestIAMGroup" ./... + ;; + "sts") + echo "Running STS and service account tests..." + make clean setup start-services wait-for-services + # SigV4-signed STS calls need admin credentials matching test_config.json. + # Tests default to "admin"/"admin" when env vars are unset, which don't exist. + export STS_TEST_ACCESS_KEY=test-access-key + export STS_TEST_SECRET_KEY=test-secret-key + # The use_service_account_credentials subtest is excluded because + # newly-created service-account access keys are not currently + # persisted to the filer after CreateServiceAccount — a + # pre-existing sync issue tracked separately from the + # GetFederationToken routing fix this PR addresses. + go test -v -timeout 15m \ + -run "TestSTS|TestAssumeRoleWithWebIdentity|TestServiceAccount" \ + -skip "TestServiceAccountLifecycle/use_service_account_credentials" \ + ./... + ;; + *) + echo "Unknown test type: ${{ matrix.test-type }}" + exit 1 + ;; + esac + + # Always cleanup + make stop-services + + - name: Show service logs on failure + if: failure() + working-directory: test/s3/iam + run: | + echo "=== Service Logs ===" + echo "--- Master Log ---" + tail -50 weed-master.log 2>/dev/null || echo "No master log found" + echo "" + echo "--- Filer Log ---" + tail -50 weed-filer.log 2>/dev/null || echo "No filer log found" + echo "" + echo "--- Volume Log ---" + tail -50 weed-volume.log 2>/dev/null || echo "No volume log found" + echo "" + echo "--- S3 API Log ---" + tail -50 weed-s3.log 2>/dev/null || echo "No S3 log found" + echo "" + + echo "=== Process Information ===" + ps aux | grep -E "(weed|test)" || true + netstat -tlnp | grep -E "(8333|8888|9333|8080)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-iam-integration-logs-${{ matrix.test-type }} + path: test/s3/iam/weed-*.log + retention-days: 5 + + # Distributed IAM tests + s3-iam-distributed-tests: + name: S3 IAM Distributed Tests + runs-on: ubuntu-22.04 + timeout-minutes: 25 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + working-directory: weed + run: | + go install -buildvcs=false + + - name: Run Distributed IAM Tests + timeout-minutes: 20 + working-directory: test/s3/iam + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + + export WEED_BINARY=$(which weed) + export TEST_TIMEOUT=15m + + # Test distributed configuration + echo "Testing distributed IAM configuration..." + make clean setup + + # Start services with distributed IAM config + echo "Starting services with distributed configuration..." + make start-services + make wait-for-services + + # Run distributed-specific tests + export ENABLE_DISTRIBUTED_TESTS=true + go test -v -timeout 15m -run "TestS3IAMDistributedTests" ./... || { + echo "❌ Distributed tests failed, checking logs..." + make logs + exit 1 + } + + make stop-services + + - name: Upload distributed test logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: s3-iam-distributed-logs + path: test/s3/iam/weed-*.log + retention-days: 7 + + # Performance and stress tests + s3-iam-performance-tests: + name: S3 IAM Performance Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + working-directory: weed + run: | + go install -buildvcs=false + + - name: Run IAM Performance Benchmarks + timeout-minutes: 25 + working-directory: test/s3/iam + run: | + set -x + echo "=== Running IAM Performance Tests ===" + + export WEED_BINARY=$(which weed) + export TEST_TIMEOUT=20m + + make clean setup start-services wait-for-services + + # Run performance tests (benchmarks disabled for CI) + echo "Running performance tests..." + export ENABLE_PERFORMANCE_TESTS=true + go test -v -timeout 15m -run "TestS3IAMPerformanceTests" ./... || { + echo "❌ Performance tests failed" + make logs + exit 1 + } + + make stop-services + + - name: Upload performance test results + if: always() + uses: actions/upload-artifact@v7 + with: + name: s3-iam-performance-results + path: | + test/s3/iam/weed-*.log + test/s3/iam/*.test + retention-days: 7 diff --git a/.github/workflows/s3-keycloak-tests.yml b/.github/workflows/s3-keycloak-tests.yml new file mode 100644 index 0000000..d267743 --- /dev/null +++ b/.github/workflows/s3-keycloak-tests.yml @@ -0,0 +1,161 @@ +name: "S3 Keycloak Integration Tests" + +on: + pull_request: + paths: + - 'weed/iam/**' + - 'weed/s3api/**' + - 'test/s3/iam/**' + - '.github/workflows/s3-keycloak-tests.yml' + push: + branches: [ master ] + paths: + - 'weed/iam/**' + - 'weed/s3api/**' + - 'test/s3/iam/**' + - '.github/workflows/s3-keycloak-tests.yml' + +concurrency: + group: ${{ github.head_ref }}/s3-keycloak-tests + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + working-directory: weed + +jobs: + # Dedicated job for Keycloak integration tests + s3-keycloak-integration-tests: + name: S3 Keycloak Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + working-directory: weed + run: | + go install -buildvcs=false + + - name: Run Keycloak Integration Tests + timeout-minutes: 25 + working-directory: test/s3/iam + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting S3 Keycloak Integration Tests ===" + + # Set WEED_BINARY to use the installed version + export WEED_BINARY=$(which weed) + export TEST_TIMEOUT=20m + + echo "Running Keycloak integration tests..." + # Start Keycloak container first + docker run -d \ + --name keycloak \ + -p 8080:8080 \ + -e KC_BOOTSTRAP_ADMIN_USERNAME=admin \ + -e KC_BOOTSTRAP_ADMIN_PASSWORD=admin \ + -e KC_HTTP_ENABLED=true \ + -e KC_HOSTNAME_STRICT=false \ + -e KC_HOSTNAME_STRICT_HTTPS=false \ + quay.io/keycloak/keycloak:26.0 \ + start-dev + + # Wait for Keycloak with better health checking + timeout 300 bash -c ' + while true; do + if curl -s http://localhost:8080/health/ready > /dev/null 2>&1; then + echo "✅ Keycloak health check passed" + break + fi + echo "... waiting for Keycloak to be ready" + sleep 5 + done + ' + + # Setup Keycloak configuration + ./setup_keycloak.sh + + # Start SeaweedFS services + make clean setup start-services wait-for-services + + # Verify service accessibility + echo "=== Verifying Service Accessibility ===" + curl -f http://localhost:8080/realms/master + curl -s http://localhost:8333/healthz + echo "✅ SeaweedFS S3 API is responding (IAM-protected endpoint)" + + # Run Keycloak-specific tests + echo "=== Running Keycloak Tests ===" + export KEYCLOAK_URL=http://localhost:8080 + export S3_ENDPOINT=http://localhost:8333 + + # Wait for realm to be properly configured + timeout 120 bash -c 'until curl -fs http://localhost:8080/realms/seaweedfs-test/.well-known/openid-configuration > /dev/null; do echo "... waiting for realm"; sleep 3; done' + + # Run the Keycloak integration tests + go test -v -timeout 20m -run "TestKeycloak" ./... + + - name: Show server logs on failure + if: failure() + working-directory: test/s3/iam + run: | + echo "=== Service Logs ===" + echo "--- Keycloak logs ---" + docker logs keycloak --tail=100 || echo "No Keycloak container logs" + + echo "--- SeaweedFS Master logs ---" + if [ -f weed-master.log ]; then + tail -100 weed-master.log + fi + + echo "--- SeaweedFS S3 logs ---" + if [ -f weed-s3.log ]; then + tail -100 weed-s3.log + fi + + echo "--- SeaweedFS Filer logs ---" + if [ -f weed-filer.log ]; then + tail -100 weed-filer.log + fi + + echo "=== System Status ===" + ps aux | grep -E "(weed|keycloak)" || true + netstat -tlnp | grep -E "(8333|9333|8080|8888)" || true + docker ps -a || true + + - name: Cleanup + if: always() + working-directory: test/s3/iam + run: | + # Stop Keycloak container + docker stop keycloak || true + docker rm keycloak || true + + # Stop SeaweedFS services + make clean || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-keycloak-test-logs + path: | + test/s3/iam/*.log + test/s3/iam/test-volume-data/ + retention-days: 3 diff --git a/.github/workflows/s3-mutation-regression-tests.yml b/.github/workflows/s3-mutation-regression-tests.yml new file mode 100644 index 0000000..acf09a8 --- /dev/null +++ b/.github/workflows/s3-mutation-regression-tests.yml @@ -0,0 +1,148 @@ +name: "S3 Mutation Regression Tests" + +on: + pull_request: + paths: + - 'weed/s3api/**' + - 'test/s3/delete/**' + - 'test/s3/distributed_lock/**' + - 'test/s3/versioning/**' + - 'test/volume_server/framework/**' + - 'docker/compose/s3.json' + - '.github/workflows/s3-mutation-regression-tests.yml' + +concurrency: + group: ${{ github.head_ref }}/s3-mutation-regression-tests + cancel-in-progress: true + +permissions: + contents: read + +jobs: + s3-versioning-regressions: + name: S3 Versioning Regression Tests + runs-on: ubuntu-22.04 + timeout-minutes: 25 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Run S3 versioning regression tests + timeout-minutes: 20 + working-directory: test/s3/versioning + run: | + set -x + # Run every versioning test, so a regression test lands covered instead + # of waiting for someone to remember this file. Name a test in EXCLUDE, + # with the reason, to keep it out. + # + # TestVersioningPagination*: opt-in stress tests that build 1500+ + # versions. They self-skip without ENABLE_STRESS_TESTS and have their + # own make target, so this gate should not carry them. + EXCLUDE='TestVersioningPagination.*' + tests=$(go test . -list '.*' | grep '^Test' | sort -u) + # An empty list would make -run match nothing and pass this job vacuously. + [ -n "$tests" ] || { echo "listed no versioning tests"; exit 1; } + selected=$(echo "$tests" | grep -vE "^($EXCLUDE)$") + [ -n "$selected" ] || { echo "EXCLUDE matched every test"; exit 1; } + echo "running $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) versioning tests" + # make swallows a lone trailing $, taking the anchor with it, so escape it. + make test-with-server TEST_PATTERN="^($(echo "$selected" | paste -sd'|' -))"'$$' + + - name: Show server logs on failure + if: failure() + working-directory: test/s3/versioning + run: | + echo "=== Server Logs ===" + if [ -f weed-test.log ]; then + tail -100 weed-test.log + fi + + - name: Upload versioning logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-versioning-regression-logs + path: test/s3/versioning/weed-test*.log + retention-days: 3 + + s3-delete-regressions: + name: S3 Delete Regression Tests + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Run S3 delete regression tests + timeout-minutes: 15 + working-directory: test/s3/delete + run: | + set -x + make test-with-server + + - name: Show server logs on failure + if: failure() + working-directory: test/s3/delete + run: | + echo "=== Server Logs ===" + if [ -f weed-test.log ]; then + tail -100 weed-test.log + fi + + - name: Upload delete logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-delete-regression-logs + path: test/s3/delete/weed-test*.log + retention-days: 3 + + s3-distributed-lock-regressions: + name: S3 Distributed Lock Regression Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Build SeaweedFS + run: | + go build -o weed/weed -buildvcs=false ./weed + + - name: Run distributed lock regressions + timeout-minutes: 25 + env: + TMPDIR: ${{ github.workspace }}/test/s3/distributed_lock/tmp + S3_DISTRIBUTED_LOCK_KEEP_LOGS: "1" + WEED_BINARY: ${{ github.workspace }}/weed/weed + run: | + set -x + mkdir -p "$TMPDIR" + go test -v -count=1 -timeout=20m ./test/s3/distributed_lock + + - name: Upload distributed lock logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-distributed-lock-regression-logs + path: test/s3/distributed_lock/tmp/seaweedfs_s3_distributed_lock_* + retention-days: 3 diff --git a/.github/workflows/s3-parquet-tests.yml b/.github/workflows/s3-parquet-tests.yml new file mode 100644 index 0000000..52e602a --- /dev/null +++ b/.github/workflows/s3-parquet-tests.yml @@ -0,0 +1,163 @@ +name: "S3 PyArrow Parquet Tests" + +on: + push: + branches: [master] + paths: + - 'weed/s3api/**' + - 'weed/filer/**' + - 'test/s3/parquet/**' + - '.github/workflows/s3-parquet-tests.yml' + pull_request: + branches: [master] + paths: + - 'weed/s3api/**' + - 'weed/filer/**' + - 'test/s3/parquet/**' + - '.github/workflows/s3-parquet-tests.yml' + workflow_dispatch: + +env: + S3_ACCESS_KEY: some_access_key1 + S3_SECRET_KEY: some_secret_key1 + S3_ENDPOINT_URL: http://localhost:8333 + BUCKET_NAME: test-parquet-bucket + +jobs: + parquet-integration-tests: + name: PyArrow Parquet Tests (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + timeout-minutes: 20 + + strategy: + fail-fast: false + matrix: + python-version: ['3.9', '3.11', '3.12'] + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version: ^1.26 + cache: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: 'test/s3/parquet/requirements.txt' + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y lsof netcat-openbsd + + - name: Build SeaweedFS + run: | + cd weed + go build -v + sudo cp weed /usr/local/bin/ + weed version + + - name: Run PyArrow Parquet integration tests + run: | + cd test/s3/parquet + make test-with-server + env: + SEAWEEDFS_BINARY: weed + S3_PORT: 8333 + FILER_PORT: 8888 + VOLUME_PORT: 8080 + MASTER_PORT: 9333 + VOLUME_MAX_SIZE_MB: 50 + + - name: Run implicit directory fix tests + run: | + cd test/s3/parquet + make test-implicit-dir-with-server + env: + SEAWEEDFS_BINARY: weed + S3_PORT: 8333 + FILER_PORT: 8888 + VOLUME_PORT: 8080 + MASTER_PORT: 9333 + + - name: Run PyArrow native S3 filesystem tests + run: | + cd test/s3/parquet + make test-native-s3-with-server + env: + SEAWEEDFS_BINARY: weed + S3_PORT: 8333 + FILER_PORT: 8888 + VOLUME_PORT: 8080 + MASTER_PORT: 9333 + + - name: Run cross-filesystem compatibility tests + run: | + cd test/s3/parquet + TEST_QUICK=1 make test-cross-fs-with-server + env: + SEAWEEDFS_BINARY: weed + S3_PORT: 8333 + FILER_PORT: 8888 + VOLUME_PORT: 8080 + MASTER_PORT: 9333 + + - name: Run SSE-S3 encryption compatibility tests + run: | + cd test/s3/parquet + make test-sse-s3-compat + env: + SEAWEEDFS_BINARY: weed + S3_PORT: 8333 + FILER_PORT: 8888 + VOLUME_PORT: 8080 + MASTER_PORT: 9333 + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: test-logs-python-${{ matrix.python-version }} + path: | + /tmp/seaweedfs-parquet-*.log + test/s3/parquet/*.log + retention-days: 7 + + - name: Cleanup + if: always() + run: | + cd test/s3/parquet + make stop-seaweedfs-safe || true + make clean || true + + unit-tests: + name: Go Unit Tests (Implicit Directory) + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version: ^1.26 + cache: true + + - name: Run Go unit tests + run: | + cd weed/s3api + go test -v -run TestImplicitDirectory + + - name: Run all S3 API tests + run: | + cd weed/s3api + go test -v -timeout 5m + diff --git a/.github/workflows/s3-policy-tests.yml b/.github/workflows/s3-policy-tests.yml new file mode 100644 index 0000000..7bb900c --- /dev/null +++ b/.github/workflows/s3-policy-tests.yml @@ -0,0 +1,426 @@ +name: "S3 Policy Integration Tests" + +on: + pull_request: + paths: + - 'weed/s3api/s3_iam_middleware.go' + - 'weed/s3api/s3api_bucket_policy*.go' + - 'weed/s3api/s3_action_resolver.go' + - 'weed/s3api/policy/**' + - 'weed/iam/**' + - 'test/s3/iam/**' + - 'test/s3/policy/**' + - '.github/workflows/s3-policy-tests.yml' + push: + branches: [ master, main ] + paths: + - 'weed/s3api/s3_iam_middleware.go' + - 'weed/s3api/s3api_bucket_policy*.go' + - 'weed/s3api/s3_action_resolver.go' + - 'weed/s3api/policy/**' + - 'weed/iam/**' + - 'test/s3/iam/**' + - 'test/s3/policy/**' + - '.github/workflows/s3-policy-tests.yml' + +concurrency: + group: ${{ github.head_ref }}/s3-policy-tests + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + working-directory: weed + +jobs: + # Unit tests for policy components + policy-unit-tests: + name: S3 Policy Unit Tests + runs-on: ubuntu-22.04 + timeout-minutes: 15 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Get dependencies + run: | + go mod download + + - name: Run S3 Policy Unit Tests + timeout-minutes: 10 + run: | + set -x + echo "=== Running S3 Action Resolver Tests ===" + go test -v -timeout 5m ./s3api/... -run ".*ActionResolver.*" + + echo "=== Running S3 Bucket Policy Engine Tests ===" + go test -v -timeout 5m ./s3api/... -run ".*BucketPolicy.*|.*PolicyEngine.*" + + echo "=== Running IAM Policy Tests ===" + go test -v -timeout 5m ./iam/policy/... + + - name: Upload test results on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: policy-unit-test-results + path: | + weed/testdata/ + weed/**/testdata/ + retention-days: 3 + + # S3 Policy Variables Integration Tests + s3-policy-variables-tests: + name: S3 Policy Variables Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 25 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run S3 Policy Variables Integration Tests + timeout-minutes: 20 + working-directory: test/s3/iam + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + df -h + + echo "=== Starting S3 Policy Variables Integration Tests ===" + + # Set WEED_BINARY to use the installed version + export WEED_BINARY=$(which weed) + export TEST_TIMEOUT=15m + + # Run policy variables tests + echo "Running policy variables tests..." + + # Kill any existing weed server on port 8333 + if lsof -Pi :8333 -sTCP:LISTEN -t >/dev/null 2>&1 ; then + kill $(lsof -t -i:8333) 2>/dev/null || true + sleep 2 + fi + + # Start weed server with IAM configuration + echo "Starting weed server with IAM configuration..." + $WEED_BINARY server \ + -s3 \ + -s3.port=8333 \ + -s3.iam.config="$(pwd)/test_iam_config.json" \ + -filer \ + -volume.max=0 \ + -master.volumeSizeLimitMB=100 \ + -s3.allowDeleteBucketNotEmpty=true \ + > /tmp/weed_policy_test_server.log 2>&1 & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + + # Wait for server to be ready + echo "Waiting for server to be ready..." + MAX_WAIT=30 + COUNTER=0 + while ! curl -s http://localhost:8333/status > /dev/null 2>&1; do + sleep 1 + COUNTER=$((COUNTER + 1)) + if [ $COUNTER -ge $MAX_WAIT ]; then + echo "Server failed to start within ${MAX_WAIT} seconds" + echo "Server log:" + cat /tmp/weed_policy_test_server.log + kill $SERVER_PID 2>/dev/null || true + exit 1 + fi + done + + echo "Server is ready!" + + # Trap to ensure server is killed on exit + trap "kill $SERVER_PID 2>/dev/null || true" EXIT + + # Run the tests + go test -v -timeout 15m -run TestS3PolicyVariables ./... + + - name: Show service logs on failure + if: failure() + working-directory: test/s3/iam + run: | + echo "=== Service Logs ===" + if [ -f /tmp/weed_policy_test_server.log ]; then + echo "--- Last 100 lines of Server Log ---" + tail -100 /tmp/weed_policy_test_server.log + fi + echo "" + echo "=== Process Information ===" + ps aux | grep -E "(weed|test)" || true + netstat -tlnp | grep -E "(8333|8888|9333|8080)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-policy-variables-test-logs + path: /tmp/weed_policy_test_server.log + retention-days: 5 + + # S3 Policy Enforcement Integration Tests + s3-policy-enforcement-tests: + name: S3 Policy Enforcement Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + strategy: + matrix: + test-case: ["basic-policy", "contextual-policy", "advanced-policy"] + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run S3 Policy Enforcement Tests - ${{ matrix.test-case }} + timeout-minutes: 25 + working-directory: test/s3/iam + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + df -h + + echo "=== Starting S3 Policy Enforcement Tests (${{ matrix.test-case }}) ===" + + export WEED_BINARY=$(which weed) + export TEST_TIMEOUT=20m + + # Kill any existing weed server on port 8333 + if lsof -Pi :8333 -sTCP:LISTEN -t >/dev/null 2>&1 ; then + kill $(lsof -t -i:8333) 2>/dev/null || true + sleep 2 + fi + + # Start weed server with IAM configuration + echo "Starting weed server with IAM configuration..." + $WEED_BINARY server \ + -s3 \ + -s3.port=8333 \ + -s3.iam.config="$(pwd)/test_iam_config.json" \ + -filer \ + -volume.max=0 \ + -master.volumeSizeLimitMB=100 \ + -s3.allowDeleteBucketNotEmpty=true \ + > /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log 2>&1 & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + + # Wait for server to be ready + echo "Waiting for server to be ready..." + MAX_WAIT=30 + COUNTER=0 + while ! curl -s http://localhost:8333/status > /dev/null 2>&1; do + sleep 1 + COUNTER=$((COUNTER + 1)) + if [ $COUNTER -ge $MAX_WAIT ]; then + echo "Server failed to start within ${MAX_WAIT} seconds" + cat /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log + kill $SERVER_PID 2>/dev/null || true + exit 1 + fi + done + + echo "Server is ready!" + + # Trap to ensure server is killed on exit + trap "kill $SERVER_PID 2>/dev/null || true" EXIT + + # Run tests based on test case + case "${{ matrix.test-case }}" in + "basic-policy") + echo "Running basic policy enforcement tests..." + go test -v -timeout 20m -run "TestS3IAMBucketPolicy|TestS3IAMPolicyEnforcement" ./... + ;; + "contextual-policy") + echo "Running contextual policy tests..." + go test -v -timeout 20m -run "TestS3PolicyVariables|TestS3IAMContextual" ./... + ;; + "advanced-policy") + echo "Running advanced policy tests..." + go test -v -timeout 20m -run "TestS3IAMMultipart|TestS3IAMPresigned" ./... + ;; + *) + echo "Unknown test case: ${{ matrix.test-case }}" + exit 1 + ;; + esac + + - name: Show service logs on failure + if: failure() + working-directory: test/s3/iam + run: | + echo "=== Service Logs ===" + if [ -f /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log ]; then + echo "--- Last 100 lines of Server Log ---" + tail -100 /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log + fi + echo "" + echo "=== Process Information ===" + ps aux | grep -E "(weed|test)" || true + netstat -tlnp | grep -E "(8333|8888|9333|8080)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-policy-enforcement-logs-${{ matrix.test-case }} + path: /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log + retention-days: 5 + + # Trusted Proxy Detection Tests + trusted-proxy-tests: + name: Trusted Proxy Detection Tests + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run Trusted Proxy Tests + timeout-minutes: 15 + working-directory: test/s3/iam + run: | + set -x + echo "=== Running Trusted Proxy Detection Tests ===" + + export WEED_BINARY=$(which weed) + + # Kill any existing weed server on port 8333 + if lsof -Pi :8333 -sTCP:LISTEN -t >/dev/null 2>&1 ; then + kill $(lsof -t -i:8333) 2>/dev/null || true + sleep 2 + fi + + # Start weed server + echo "Starting weed server..." + $WEED_BINARY server \ + -s3 \ + -s3.port=8333 \ + -s3.iam.config="$(pwd)/test_iam_config.json" \ + -filer \ + -volume.max=0 \ + -master.volumeSizeLimitMB=100 \ + -s3.allowDeleteBucketNotEmpty=true \ + > /tmp/weed_proxy_test.log 2>&1 & + + SERVER_PID=$! + echo "Server started with PID: $SERVER_PID" + + # Wait for server to be ready + echo "Waiting for server to be ready..." + MAX_WAIT=30 + COUNTER=0 + while ! curl -s http://localhost:8333/status > /dev/null 2>&1; do + sleep 1 + COUNTER=$((COUNTER + 1)) + if [ $COUNTER -ge $MAX_WAIT ]; then + echo "Server failed to start within ${MAX_WAIT} seconds" + kill $SERVER_PID 2>/dev/null || true + exit 1 + fi + done + + # Trap to ensure server is killed on exit + trap "kill $SERVER_PID 2>/dev/null || true" EXIT + + # Run proxy tests + go test -v -timeout 10m -run "TestTrustedProxy|TestPrivateIP" ./... + + - name: Show service logs on failure + if: failure() + run: | + echo "=== Service Logs ===" + if [ -f /tmp/weed_proxy_test.log ]; then + echo "--- Last 100 lines of Server Log ---" + tail -100 /tmp/weed_proxy_test.log + fi + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: trusted-proxy-test-logs + path: /tmp/weed_proxy_test.log + retention-days: 3 + + # S3 Policy Shell Integration Tests + s3-policy-shell-tests: + name: S3 Policy Shell Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 15 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run S3 Policy Shell Tests + timeout-minutes: 10 + working-directory: test/s3/policy + run: | + set -x + echo "=== Running S3 Policy Shell Tests ===" + + # Set WEED_BINARY to use the installed version (though test uses 'weed' command) + export WEED_BINARY=$(which weed) + export PATH=$PATH:$(dirname $WEED_BINARY) + + go test -v -timeout 10m ./... diff --git a/.github/workflows/s3-proxy-signature-tests.yml b/.github/workflows/s3-proxy-signature-tests.yml new file mode 100644 index 0000000..a8cc6c3 --- /dev/null +++ b/.github/workflows/s3-proxy-signature-tests.yml @@ -0,0 +1,143 @@ +name: "S3 Proxy Signature Tests" + +on: + push: + branches: [ master ] + paths: + - 'weed/s3api/**' + - 'weed/server/**' + - 'test/s3/proxy_signature/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/s3-proxy-signature-tests.yml' + pull_request: + branches: [ master ] + paths: + - 'weed/s3api/**' + - 'weed/server/**' + - 'test/s3/proxy_signature/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/s3-proxy-signature-tests.yml' + +concurrency: + group: ${{ github.head_ref || github.ref }}/s3-proxy-signature-tests + cancel-in-progress: true + +permissions: + contents: read + +jobs: + proxy-signature-tests: + name: S3 Proxy Signature Verification Tests + runs-on: ubuntu-22.04 + timeout-minutes: 15 + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Build SeaweedFS binary for Linux + run: | + set -x + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -buildvcs=false -v -o test/s3/proxy_signature/weed ./weed + + - name: Run S3 Proxy Signature Tests + timeout-minutes: 10 + working-directory: test/s3/proxy_signature + run: | + set -x + echo "Starting Docker Compose services..." + docker compose up -d --build + + # Check if containers are running + echo "Checking container status..." + docker compose ps + + # Wait for services to be ready + echo "Waiting for nginx proxy to be ready..." + PROXY_READY=0 + for i in $(seq 1 30); do + if curl -s http://localhost:9000/ > /dev/null 2>&1; then + echo "Proxy is ready" + PROXY_READY=1 + break + fi + echo "Waiting for proxy... ($i/30)" + sleep 1 + done + if [ $PROXY_READY -eq 0 ]; then + echo "ERROR: Proxy failed to become ready after 30 seconds" + echo "Docker compose logs:" + docker compose logs --no-color || true + exit 1 + fi + + # Wait for SeaweedFS to be ready + echo "Waiting for SeaweedFS S3 gateway to be ready via proxy..." + S3_READY=0 + for i in $(seq 1 30); do + # Check logs first for the readiness line. weed mini's progress + # board prints " S3 ready (Xs)"; older builds and the + # standalone S3 binary log "S3 (gateway|service) ... ready". + if docker compose logs seaweedfs 2>&1 | grep -qE "S3 (gateway|service).*(started|ready)|S3[[:space:]]+ready"; then + echo "SeaweedFS S3 gateway is ready" + S3_READY=1 + break + fi + # Fallback: check headers via proxy (which is already ready) + if curl -s -I http://localhost:9000/ | grep -qi "SeaweedFS"; then + echo "SeaweedFS S3 gateway is responding via proxy" + S3_READY=1 + break + fi + echo "Waiting for S3 gateway... ($i/30)" + sleep 1 + done + if [ $S3_READY -eq 0 ]; then + echo "ERROR: SeaweedFS S3 gateway failed to become ready after 30 seconds" + echo "Latest seaweedfs logs:" + docker compose logs --no-color --tail 20 seaweedfs || true + exit 1 + fi + + # Run the test script inside AWS CLI container + echo "Running test script..." + docker run --rm --network host \ + --entrypoint bash \ + amazon/aws-cli:latest \ + -c "$(cat test.sh)" + + TEST_RESULT=$? + + # Cleanup + docker compose down + + exit $TEST_RESULT + + - name: Cleanup on failure + if: failure() + working-directory: test/s3/proxy_signature + run: | + echo "Cleaning up Docker containers..." + ls -al weed || true + ldd weed || true + echo "Docker compose logs:" + docker compose logs --no-color || true + echo "Container status before cleanup:" + docker ps -a + echo "Stopping services..." + docker compose down || true diff --git a/.github/workflows/s3-sdk-v2-routing-tests.yml b/.github/workflows/s3-sdk-v2-routing-tests.yml new file mode 100644 index 0000000..e63528e --- /dev/null +++ b/.github/workflows/s3-sdk-v2-routing-tests.yml @@ -0,0 +1,110 @@ +name: "S3 SDK V2 Route Disambiguation Tests" + +on: + push: + branches: [ master ] + paths: + - 'weed/s3api/**' + - 'test/s3/sdk_v2_routing/**' + - '.github/workflows/s3-sdk-v2-routing-tests.yml' + pull_request: + branches: [ master ] + paths: + - 'weed/s3api/**' + - 'test/s3/sdk_v2_routing/**' + - '.github/workflows/s3-sdk-v2-routing-tests.yml' + +concurrency: + group: ${{ github.head_ref || github.ref }}/s3-sdk-v2-routing-tests + cancel-in-progress: true + +permissions: + contents: read + +jobs: + s3-sdk-v2-routing-tests: + name: S3 SDK V2 Routing Tests + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Install SeaweedFS + run: | + cd weed && go install -buildvcs=false + + - name: Start weed mini (S3 on :8333) + # Pins the regression for issue #9559: AWS SDK V2 / Hadoop s3a + # listing a bucket literally named "buckets" must get an XML + # ListObjectsV2 response, not the JSON ListTableBuckets body + # served by the S3 Tables REST endpoint on the same path. + run: | + mkdir -p /tmp/seaweedfs-sdk-v2-routing + cat > /tmp/seaweedfs-sdk-v2-routing-s3.json <<'JSON' + { + "identities": [ + { + "name": "admin", + "credentials": [ + {"accessKey": "some_access_key1", "secretKey": "some_secret_key1"} + ], + "actions": ["Admin", "Read", "Write"] + } + ] + } + JSON + AWS_ACCESS_KEY_ID=some_access_key1 \ + AWS_SECRET_ACCESS_KEY=some_secret_key1 \ + weed mini \ + -dir=/tmp/seaweedfs-sdk-v2-routing \ + -s3.port=8333 \ + -s3.config=/tmp/seaweedfs-sdk-v2-routing-s3.json \ + -ip=127.0.0.1 \ + > /tmp/weed-mini.log 2>&1 & + echo $! > /tmp/weed-mini.pid + + for i in $(seq 1 30); do + if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then + echo "weed mini is ready" + exit 0 + fi + sleep 1 + done + echo "weed mini failed to start within 30s" + tail -50 /tmp/weed-mini.log + exit 1 + + - name: Run SDK V2 routing tests + env: + S3_ENDPOINT: http://127.0.0.1:8333 + AWS_ACCESS_KEY_ID: some_access_key1 + AWS_SECRET_ACCESS_KEY: some_secret_key1 + AWS_REGION: us-east-1 + run: go test -v -timeout=5m ./test/s3/sdk_v2_routing/... + + - name: Stop weed mini + if: always() + run: | + if [ -f /tmp/weed-mini.pid ]; then + kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true + fi + + - name: Show server log on failure + if: failure() + run: | + echo "=== weed mini log (last 200 lines) ===" + tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available" + + - name: Archive log + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-sdk-v2-routing-server-log + path: /tmp/weed-mini.log + retention-days: 3 diff --git a/.github/workflows/s3-spark-tests.yml b/.github/workflows/s3-spark-tests.yml new file mode 100644 index 0000000..c868299 --- /dev/null +++ b/.github/workflows/s3-spark-tests.yml @@ -0,0 +1,85 @@ +name: "S3 Spark Integration Tests" + +on: + pull_request: + paths: + - 'weed/s3api/**' + - 'weed/filer/**' + - 'test/s3/spark/**' + - 'test/s3tables/testutil/**' + - '.github/workflows/s3-spark-tests.yml' + workflow_dispatch: + +concurrency: + group: ${{ github.head_ref }}/s3-spark-tests + cancel-in-progress: true + +permissions: + contents: read + +jobs: + s3-spark-issue-repro-tests: + name: S3 Spark Issue Reproduction Tests + runs-on: ubuntu-22.04 + timeout-minutes: 45 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Install SeaweedFS + run: | + go install -buildvcs=false ./weed + + - name: Pre-pull Spark image + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull apache/spark:3.5.1 + + - name: Run S3 Spark integration tests + working-directory: test/s3/spark + timeout-minutes: 35 + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting S3 Spark Integration Tests ===" + + go test -v -timeout 30m . 2>&1 | tee test-output.log || { + echo "S3 Spark integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3/spark + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker|spark)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-spark-test-logs + path: test/s3/spark/test-output.log + retention-days: 3 diff --git a/.github/workflows/s3-sse-tests.yml b/.github/workflows/s3-sse-tests.yml new file mode 100644 index 0000000..d1e2102 --- /dev/null +++ b/.github/workflows/s3-sse-tests.yml @@ -0,0 +1,401 @@ +name: "S3 SSE Tests" + +on: + pull_request: + paths: + - 'weed/s3api/s3_sse_*.go' + - 'weed/s3api/s3api_object_handlers.go' + - 'weed/s3api/s3api_object_handlers_put.go' + - 'weed/s3api/s3api_object_handlers_copy*.go' + - 'weed/server/filer_server_handlers_*.go' + - 'weed/kms/**' + - 'test/s3/sse/**' + - '.github/workflows/s3-sse-tests.yml' + push: + branches: [ master, main ] + paths: + - 'weed/s3api/s3_sse_*.go' + - 'weed/s3api/s3api_object_handlers.go' + - 'weed/s3api/s3api_object_handlers_put.go' + - 'weed/s3api/s3api_object_handlers_copy*.go' + - 'weed/server/filer_server_handlers_*.go' + - 'weed/kms/**' + - 'test/s3/sse/**' + +concurrency: + group: ${{ github.head_ref }}/s3-sse-tests + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + working-directory: weed + +jobs: + s3-sse-integration-tests: + name: S3 SSE Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + strategy: + matrix: + test-type: ["quick", "comprehensive"] + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run S3 SSE Integration Tests - ${{ matrix.test-type }} + timeout-minutes: 25 + working-directory: test/s3/sse + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting SSE Tests ===" + + # Run tests with automatic server management + # The test-with-server target handles server startup/shutdown automatically + if [ "${{ matrix.test-type }}" = "quick" ]; then + # Quick tests - basic SSE-C and SSE-KMS functionality + Range requests + make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior" + else + # Comprehensive tests - SSE-C/KMS functionality plus cross-SSE copy. + # The copy-operation tests (`.*ObjectCopyIntegration`, `TestCrossSSECopy`, + # `TestSSEMultipartCopy`) were excluded for a long time as "pre-existing + # SSE-C issues" (#9281); fixed and brought back into CI as part of the + # same change that fixed them. + make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior|.*ObjectCopyIntegration|TestCrossSSECopy|TestSSEMultipartCopy" + fi + + - name: Show server logs on failure + if: failure() + working-directory: test/s3/sse + run: | + echo "=== Server Logs ===" + if [ -f weed-test.log ]; then + echo "Last 100 lines of server logs:" + tail -100 weed-test.log + else + echo "No server log file found" + fi + + echo "=== Test Environment ===" + ps aux | grep -E "(weed|test)" || true + netstat -tlnp | grep -E "(8333|9333|8080|8888)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-sse-test-logs-${{ matrix.test-type }} + path: test/s3/sse/weed-test*.log + retention-days: 3 + + s3-sse-compatibility: + name: S3 SSE Compatibility Test + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run Core SSE Compatibility Test (AWS S3 equivalent) + timeout-minutes: 15 + working-directory: test/s3/sse + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + + # Run the specific tests that validate AWS S3 SSE compatibility - both SSE-C and SSE-KMS basic functionality plus Range requests + make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|.*RangeRequestsServerBehavior" || { + echo "❌ SSE compatibility test failed, checking logs..." + if [ -f weed-test.log ]; then + echo "=== Server logs ===" + tail -100 weed-test.log + fi + echo "=== Process information ===" + ps aux | grep -E "(weed|test)" || true + exit 1 + } + + - name: Upload server logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-sse-compatibility-logs + path: test/s3/sse/weed-test*.log + retention-days: 3 + + s3-sse-metadata-persistence: + name: S3 SSE Metadata Persistence Test + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run SSE Metadata Persistence Test + timeout-minutes: 15 + working-directory: test/s3/sse + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + + # Run the specific test that would catch filer metadata storage bugs + # This test validates that encryption metadata survives the full PUT/GET cycle + make test-metadata-persistence || { + echo "❌ SSE metadata persistence test failed, checking logs..." + if [ -f weed-test.log ]; then + echo "=== Server logs ===" + tail -100 weed-test.log + fi + echo "=== Process information ===" + ps aux | grep -E "(weed|test)" || true + exit 1 + } + + - name: Upload server logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-sse-metadata-persistence-logs + path: test/s3/sse/weed-test*.log + retention-days: 3 + + s3-sse-copy-operations: + name: S3 SSE Copy Operations Test + runs-on: ubuntu-22.04 + timeout-minutes: 25 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run SSE Copy Operations Tests + timeout-minutes: 20 + working-directory: test/s3/sse + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + + # Run tests that validate SSE copy operations and cross-encryption scenarios + echo "🚀 Running SSE copy operations tests..." + echo "📋 Note: SSE-C copy operations have pre-existing functionality gaps" + echo " Cross-encryption copy security fix has been implemented and maintained" + + # Skip SSE-C copy operations due to pre-existing HTTP 500 errors + # The critical security fix for cross-encryption (SSE-C → SSE-KMS) has been preserved + echo "⏭ī¸ Skipping SSE copy operations tests due to known limitations:" + echo " - SSE-C copy operations: HTTP 500 errors (pre-existing functionality gap)" + echo " - Cross-encryption security fix: ✅ Implemented and tested (forces streaming copy)" + echo " - These limitations are documented as pre-existing issues" + exit 0 # Job succeeds with security fix preserved and limitations documented + + - name: Upload server logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-sse-copy-operations-logs + path: test/s3/sse/weed-test*.log + retention-days: 3 + + s3-sse-multipart: + name: S3 SSE Multipart Upload Test + runs-on: ubuntu-22.04 + timeout-minutes: 25 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run SSE Multipart Upload Tests + timeout-minutes: 20 + working-directory: test/s3/sse + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + + # Multipart tests - Document known architectural limitations + echo "🚀 Running multipart upload tests..." + echo "📋 Note: SSE-KMS multipart upload has known architectural limitation requiring per-chunk metadata storage" + echo " SSE-C multipart tests will be skipped due to pre-existing functionality gaps" + + # Test SSE-C basic multipart (skip advanced multipart that fails with HTTP 500) + # Skip SSE-KMS multipart due to architectural limitation (each chunk needs independent metadata) + echo "⏭ī¸ Skipping multipart upload tests due to known limitations:" + echo " - SSE-C multipart GET operations: HTTP 500 errors (pre-existing functionality gap)" + echo " - SSE-KMS multipart decryption: Requires per-chunk SSE metadata architecture changes" + echo " - These limitations are documented and require future architectural work" + exit 0 # Job succeeds with clear documentation of known limitations + + - name: Upload server logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-sse-multipart-logs + path: test/s3/sse/weed-test*.log + retention-days: 3 + + s3-sse-performance: + name: S3 SSE Performance Test + runs-on: ubuntu-22.04 + timeout-minutes: 35 + # Only run performance tests on master branch pushes to avoid overloading PR testing + if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main') + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run S3 SSE Performance Tests + timeout-minutes: 30 + working-directory: test/s3/sse + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + + # Run performance tests with various data sizes + make perf || { + echo "❌ SSE performance test failed, checking logs..." + if [ -f weed-test.log ]; then + echo "=== Server logs ===" + tail -200 weed-test.log + fi + make clean + exit 1 + } + make clean + + - name: Upload performance test logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: s3-sse-performance-logs + path: test/s3/sse/weed-test*.log + retention-days: 7 + + s3-volume-encryption: + name: S3 Volume Encryption Test + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run S3 Volume Encryption Integration Tests + timeout-minutes: 15 + working-directory: test/s3/sse + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + + # Run volume encryption tests with -s3.encryptVolumeData flag + echo "🚀 Running S3 volume encryption integration tests..." + make test-volume-encryption || { + echo "❌ Volume encryption tests failed, checking logs..." + if [ -f /tmp/seaweedfs-sse-mini.log ]; then + echo "=== Server logs ===" + tail -100 /tmp/seaweedfs-sse-mini.log + fi + echo "=== Process information ===" + ps aux | grep -E "(weed|test)" || true + exit 1 + } + + - name: Upload server logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-volume-encryption-logs + path: /tmp/seaweedfs-sse-*.log + retention-days: 3 + diff --git a/.github/workflows/s3-tables-tests.yml b/.github/workflows/s3-tables-tests.yml new file mode 100644 index 0000000..3a2667a --- /dev/null +++ b/.github/workflows/s3-tables-tests.yml @@ -0,0 +1,1324 @@ +name: "S3 Tables Integration Tests" + +on: + pull_request: + paths: + - 'weed/s3api/**' + - 'weed/filer/**' + - 'weed/server/**' + - 'weed/worker/tasks/iceberg/**' + - 'test/s3tables/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/s3-tables-tests.yml' + +permissions: + contents: read + + +jobs: + s3-tables-tests: + name: S3 Tables Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Run go mod tidy + run: go mod tidy + + - name: Install SeaweedFS + run: | + go install -buildvcs=false ./weed + + - name: Run S3 Tables Integration Tests + timeout-minutes: 25 + working-directory: test/s3tables/table-buckets + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting S3 Tables Tests ===" + + # Run S3 Tables integration tests + go test -v -timeout 20m . 2>&1 | tee test-output.log || { + echo "S3 Tables integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/table-buckets + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-tables-test-logs + path: test/s3tables/table-buckets/test-output.log + retention-days: 3 + + iceberg-catalog-tests: + name: Iceberg Catalog Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Run go mod tidy + run: go mod tidy + + - name: Run Iceberg Catalog Integration Tests + timeout-minutes: 25 + working-directory: test/s3tables/catalog + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting Iceberg Catalog Tests ===" + + # Run Iceberg catalog integration tests using Makefile (handles build) + make test 2>&1 | tee test-output.log || { + echo "Iceberg catalog integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/catalog + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: iceberg-catalog-test-logs + path: test/s3tables/catalog/test-output.log + retention-days: 3 + + trino-iceberg-catalog-tests: + name: Trino Iceberg Catalog Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Pre-pull Trino image + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull trinodb/trino:479 + + - name: Run go mod tidy + run: go mod tidy + + - name: Install SeaweedFS + run: | + go install -buildvcs=false ./weed + + - name: Run Trino Iceberg Catalog Integration Tests + timeout-minutes: 25 + working-directory: test/s3tables/catalog_trino + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting Trino Iceberg Catalog Tests ===" + + # Run Trino + Iceberg catalog integration tests + go test -v -timeout 20m . 2>&1 | tee test-output.log || { + echo "Trino Iceberg catalog integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/catalog_trino + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: trino-iceberg-catalog-test-logs + path: test/s3tables/catalog_trino/test-output.log + retention-days: 3 + + dremio-iceberg-catalog-tests: + name: Dremio Iceberg Catalog Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Pre-pull images + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull dremio/dremio-oss:25.2.0 + pull python:3.11-slim + + - name: Run go mod tidy + run: go mod tidy + + - name: Install SeaweedFS + run: | + go install -buildvcs=false ./weed + + - name: Run Dremio Iceberg Catalog Integration Tests + timeout-minutes: 25 + working-directory: test/s3tables/catalog_dremio + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + docker info + echo "=== Starting Dremio Iceberg Catalog Tests ===" + + go test -v -timeout 20m . 2>&1 | tee test-output.log || { + echo "Dremio Iceberg catalog integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/catalog_dremio + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker|dremio)" || true + echo "=== Dremio containers ===" + docker ps -a --filter "name=seaweed-dremio" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: dremio-iceberg-catalog-test-logs + path: test/s3tables/catalog_dremio/test-output.log + retention-days: 3 + + doris-iceberg-catalog-tests: + name: Doris Iceberg Catalog Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 35 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Pre-pull images + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull apache/doris:doris-all-in-one-2.1.0 + pull python:3.11-slim + + - name: Run go mod tidy + run: go mod tidy + + - name: Install SeaweedFS + run: | + go install -buildvcs=false ./weed + + - name: Run Doris Iceberg Catalog Integration Tests + timeout-minutes: 30 + working-directory: test/s3tables/catalog_doris + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + docker info + echo "=== Starting Doris Iceberg Catalog Tests ===" + + go test -v -timeout 25m . 2>&1 | tee test-output.log || { + echo "Doris Iceberg catalog integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/catalog_doris + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker|doris)" || true + echo "=== Doris containers ===" + docker ps -a --filter "name=seaweed-doris" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: doris-iceberg-catalog-test-logs + path: test/s3tables/catalog_doris/test-output.log + retention-days: 3 + + clickhouse-iceberg-catalog-tests: + name: ClickHouse Iceberg Catalog Integration Tests (${{ matrix.tag }}) + runs-on: ubuntu-22.04 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + # Pinned baseline, and latest so new ClickHouse releases are + # exercised without a code change. + - clickhouse-image: clickhouse/clickhouse-server:25.8 + tag: "25.8" + - clickhouse-image: clickhouse/clickhouse-server:latest + tag: latest + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Pre-pull images + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull ${{ matrix.clickhouse-image }} + pull python:3.11-slim + + - name: Run go mod tidy + run: go mod tidy + + - name: Install SeaweedFS + run: | + go install -buildvcs=false ./weed + + - name: Run ClickHouse Iceberg Catalog Integration Tests + timeout-minutes: 25 + working-directory: test/s3tables/catalog_clickhouse + env: + CLICKHOUSE_IMAGE: ${{ matrix.clickhouse-image }} + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + docker info + echo "=== Starting ClickHouse Iceberg Catalog Tests ===" + + go test -v -timeout 20m . 2>&1 | tee test-output.log || { + echo "ClickHouse Iceberg catalog integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/catalog_clickhouse + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker|clickhouse)" || true + echo "=== ClickHouse containers ===" + docker ps -a --filter "name=seaweed-clickhouse" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: clickhouse-iceberg-catalog-test-logs-${{ matrix.tag }} + path: test/s3tables/catalog_clickhouse/test-output.log + retention-days: 3 + + polaris-integration-tests: + name: Polaris Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Run go mod tidy + run: go mod tidy + + - name: Install SeaweedFS + run: | + go install -buildvcs=false ./weed + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Pre-pull Polaris image + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull apache/polaris:latest + + - name: Run Polaris Integration Tests + timeout-minutes: 25 + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting Polaris Tests ===" + + go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || { + echo "Polaris integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/polaris + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: polaris-test-logs + path: test/s3tables/polaris/test-output.log + retention-days: 3 + + spark-iceberg-catalog-tests: + name: Spark Iceberg Catalog Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Pre-pull Spark image + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull apache/spark:3.5.1 + + - name: Run go mod tidy + run: go mod tidy + + - name: Install SeaweedFS + run: | + go install -buildvcs=false ./weed + + - name: Run Spark Iceberg Catalog Integration Tests + timeout-minutes: 25 + working-directory: test/s3tables/catalog_spark + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting Spark Iceberg Catalog Tests ===" + + # Run Spark + Iceberg catalog integration tests + go test -v -timeout 20m . 2>&1 | tee test-output.log || { + echo "Spark Iceberg catalog integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/catalog_spark + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: spark-iceberg-catalog-test-logs + path: test/s3tables/catalog_spark/test-output.log + retention-days: 3 + + risingwave-catalog-tests: + name: RisingWave Catalog Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Pre-pull RisingWave image + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull risingwavelabs/risingwave:v2.5.0 + pull postgres:16-alpine + + - name: Run go mod tidy + run: go mod tidy + + - name: Install SeaweedFS + run: | + go install -buildvcs=false ./weed + + - name: Run RisingWave Catalog Integration Tests + timeout-minutes: 25 + working-directory: test/s3tables/catalog_risingwave + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting RisingWave Catalog Tests ===" + + # Run RisingWave catalog integration tests + go test -v -timeout 20m . 2>&1 | tee test-output.log || { + echo "RisingWave catalog integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/catalog_risingwave + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: risingwave-catalog-test-logs + path: test/s3tables/catalog_risingwave/test-output.log + retention-days: 3 + + sts-integration-tests: + name: STS Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Week stamp for image cache key + id: week + run: echo "week=$(date -u +%G-%V)" >> "$GITHUB_OUTPUT" + + - name: Restore python:3 image cache + id: python-image + uses: actions/cache@v6 + with: + path: /tmp/python3-image.tar + key: python3-image-${{ steps.week.outputs.week }} + restore-keys: | + python3-image- + + - name: Load or pull python:3 + run: | + if [ "${{ steps.python-image.outputs.cache-hit }}" = "true" ]; then + docker load -i /tmp/python3-image.tar + exit 0 + fi + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + if pull python:3; then + docker save -o /tmp/python3-image.tar python:3 + elif [ -f /tmp/python3-image.tar ]; then + # Docker Hub unreachable; fall back to last week's cached image + docker load -i /tmp/python3-image.tar + else + exit 1 + fi + + - name: Run go mod tidy + run: go mod tidy + + - name: Install SeaweedFS + run: | + go install -buildvcs=false ./weed + + - name: Run STS Integration Tests + timeout-minutes: 25 + working-directory: test/s3tables/sts_integration + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting STS Integration Tests ===" + + # Run STS integration tests + go test -v -timeout 20m . 2>&1 | tee test-output.log || { + echo "STS integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/sts_integration + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: sts-integration-test-logs + path: test/s3tables/sts_integration/test-output.log + retention-days: 3 + + lakekeeper-integration-tests: + name: Lakekeeper Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Run go mod tidy + run: go mod tidy + + - name: Install SeaweedFS + run: | + go install -buildvcs=false ./weed + + - name: Run Lakekeeper Integration Tests + timeout-minutes: 25 + working-directory: test/s3tables/lakekeeper + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting Lakekeeper Integration Tests ===" + + # Run Lakekeeper integration tests + go test -v -timeout 20m . 2>&1 | tee test-output.log || { + echo "Lakekeeper integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/lakekeeper + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: lakekeeper-integration-test-logs + path: test/s3tables/lakekeeper/test-output.log + retention-days: 3 + + unity-catalog-integration-tests: + name: Unity Catalog Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Pre-pull images + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull unitycatalog/unitycatalog:v0.4.0 + pull python:3.11-slim + + - name: Run go mod tidy + run: go mod tidy + + - name: Install SeaweedFS + run: | + go install -buildvcs=false ./weed + + - name: Run Unity Catalog Integration Tests + timeout-minutes: 25 + working-directory: test/s3tables/unity_catalog + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + docker info + echo "=== Starting Unity Catalog Tests ===" + + go test -v -timeout 20m . 2>&1 | tee test-output.log || { + echo "Unity Catalog integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/unity_catalog + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker|unitycatalog)" || true + echo "=== Unity Catalog containers ===" + docker ps -a --filter "name=seaweed-unity-catalog" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: unity-catalog-integration-test-logs + path: test/s3tables/unity_catalog/test-output.log + retention-days: 3 + + lancedb-namespace-tests: + name: LanceDB Namespace Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Pre-pull images + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull python:3.11-slim + + - name: Run go mod tidy + run: go mod tidy + + - name: Build SeaweedFS + run: | + cd weed && go build -buildvcs=false . + + - name: Run LanceDB Namespace Integration Tests + timeout-minutes: 25 + working-directory: test/s3tables/catalog_lancedb + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + docker info + echo "=== Starting LanceDB Namespace Tests ===" + + go test -v -timeout 20m . 2>&1 | tee test-output.log || { + echo "LanceDB namespace integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/catalog_lancedb + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: lancedb-namespace-test-logs + path: test/s3tables/catalog_lancedb/test-output.log + retention-days: 3 + + table-lifecycle-tests: + name: Table Lifecycle Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 40 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Pre-pull images + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull python:3.11-slim + pull duckdb/duckdb:latest + + - name: Run go mod tidy + run: go mod tidy + + - name: Build SeaweedFS + run: | + cd weed && go build -buildvcs=false . + + - name: Run Table Lifecycle Integration Tests + timeout-minutes: 35 + working-directory: test/s3tables/lifecycle + env: + # The Rust worker's own tests cover its handlers; a cold build of the + # lance crate costs more here than the layer it would be checking. + WEED_LANCE_MAINTENANCE: library + run: | + set -x + set -o pipefail + go test -v -timeout 30m . 2>&1 | tee test-output.log || { + echo "Table lifecycle integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/lifecycle + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: table-lifecycle-test-logs + path: test/s3tables/lifecycle/test-output.log + retention-days: 3 + + duckdb-lance-tests: + name: DuckDB Lance Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v7 + with: + # The job uploads a test log on failure; nothing here needs to push, + # so do not leave a token in the checkout for it to pick up. + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Pre-pull images + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull duckdb/duckdb:latest + pull python:3.11-slim + + - name: Run go mod tidy + run: go mod tidy + + - name: Build SeaweedFS + run: | + cd weed && go build -buildvcs=false . + + - name: Run DuckDB Lance Integration Tests + timeout-minutes: 25 + working-directory: test/s3tables/catalog_duckdb_lance + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + docker info + echo "=== Starting DuckDB Lance Tests ===" + + go test -v -timeout 20m . 2>&1 | tee test-output.log || { + echo "DuckDB Lance integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/catalog_duckdb_lance + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker|duckdb)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: duckdb-lance-test-logs + path: test/s3tables/catalog_duckdb_lance/test-output.log + retention-days: 3 + + spark-lance-namespace-tests: + name: Spark Lance Namespace Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 40 + + steps: + - name: Check out code + uses: actions/checkout@v7 + with: + # The job uploads a test log on failure; nothing here needs to push, + # so do not leave a token in the checkout for it to pick up. + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Configure Docker Hub mirror + run: | + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Pre-pull images + run: | + pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } + pull apache/spark:3.5.1 + + - name: Run go mod tidy + run: go mod tidy + + - name: Build SeaweedFS + run: | + cd weed && go build -buildvcs=false . + + - name: Run Spark Lance Namespace Integration Tests + timeout-minutes: 35 + working-directory: test/s3tables/catalog_spark_lance + run: | + set -x + set -o pipefail + echo "=== System Information ===" + uname -a + free -h + df -h + docker info + echo "=== Starting Spark Lance Namespace Tests ===" + + go test -v -timeout 30m . 2>&1 | tee test-output.log || { + echo "Spark Lance namespace integration tests failed" + exit 1 + } + + - name: Show test output on failure + if: failure() + working-directory: test/s3tables/catalog_spark_lance + run: | + echo "=== Test Output ===" + if [ -f test-output.log ]; then + tail -200 test-output.log + fi + + echo "=== Process information ===" + ps aux | grep -E "(weed|test|docker|spark)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: spark-lance-namespace-test-logs + path: test/s3tables/catalog_spark_lance/test-output.log + retention-days: 3 + + s3-tables-build-verification: + name: S3 Tables Build Verification + runs-on: ubuntu-22.04 + timeout-minutes: 15 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Verify S3 Tables Package Builds + run: | + set -x + echo "=== Building S3 Tables package ===" + go build ./weed/s3api/s3tables || { + echo "S3 Tables package build failed" + exit 1 + } + echo "S3 Tables package built successfully" + + - name: Verify S3 API Integration Builds + run: | + set -x + echo "=== Building S3 API with S3 Tables integration ===" + go build ./weed/s3api || { + echo "S3 API build with S3 Tables failed" + exit 1 + } + echo "S3 API with S3 Tables integration built successfully" + + - name: Run Go Tests for S3 Tables Package + run: | + set -x + echo "=== Running Go unit tests for S3 Tables ===" + go test -v -race -timeout 5m ./weed/s3api/s3tables/... || { + echo "S3 Tables unit tests failed" + exit 1 + } + echo "S3 Tables unit tests passed" + + - name: Verify Iceberg Package Builds + run: | + set -x + echo "=== Building Iceberg package ===" + go build ./weed/s3api/iceberg || { + echo "Iceberg package build failed" + exit 1 + } + echo "Iceberg package built successfully" + + - name: Run Go Tests for Iceberg Package + run: | + set -x + echo "=== Running Go unit tests for Iceberg ===" + go test -v -race -timeout 5m ./weed/s3api/iceberg/... || { + echo "Iceberg unit tests failed" + exit 1 + } + echo "Iceberg unit tests passed" + + s3-tables-fmt-check: + name: S3 Tables Format Check + runs-on: ubuntu-22.04 + timeout-minutes: 10 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Check Go Format + run: | + set -x + echo "=== Checking S3 Tables Go format ===" + unformatted=$(gofmt -l ./weed/s3api/s3tables) + if [ -n "$unformatted" ]; then + echo "Go format check failed - files need formatting" + echo "$unformatted" + exit 1 + fi + echo "All S3 Tables files are properly formatted" + + - name: Check S3 Tables Test Format + run: | + set -x + echo "=== Checking S3 Tables test format ===" + unformatted=$(gofmt -l ./test/s3tables) + if [ -n "$unformatted" ]; then + echo "Go format check failed for tests" + echo "$unformatted" + exit 1 + fi + echo "All S3 Tables test files are properly formatted" + + - name: Check Iceberg Format + run: | + set -x + echo "=== Checking Iceberg Go format ===" + unformatted=$(gofmt -l ./weed/s3api/iceberg) + if [ -n "$unformatted" ]; then + echo "Go format check failed for Iceberg - files need formatting" + echo "$unformatted" + exit 1 + fi + echo "All Iceberg files are properly formatted" + + s3-tables-vet: + name: S3 Tables Go Vet Check + runs-on: ubuntu-22.04 + timeout-minutes: 10 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Run Go Vet + run: | + set -x + echo "=== Running go vet on S3 Tables package ===" + go vet ./weed/s3api/s3tables/... || { + echo "go vet check failed" + exit 1 + } + echo "go vet checks passed" + + - name: Run Go Vet on Tests + run: | + set -x + echo "=== Running go vet on S3 Tables tests ===" + go vet ./test/s3tables/... || { + echo "go vet check failed for tests" + exit 1 + } + echo "go vet checks passed for tests" + + - name: Run Go Vet on Iceberg + run: | + set -x + echo "=== Running go vet on Iceberg package ===" + go vet ./weed/s3api/iceberg/... || { + echo "go vet check failed for Iceberg" + exit 1 + } + echo "go vet checks passed for Iceberg" diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml new file mode 100644 index 0000000..980b921 --- /dev/null +++ b/.github/workflows/s3tests.yml @@ -0,0 +1,1241 @@ +name: "Ceph S3 tests" + +on: + push: + branches: [ master ] + paths: + - 'weed/s3api/**' + - 'weed/filer/**' + - 'weed/server/**' + - 'weed/iam/**' + - 'test/s3/compatibility/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/s3tests.yml' + pull_request: + branches: [ master ] + paths: + - 'weed/s3api/**' + - 'weed/filer/**' + - 'weed/server/**' + - 'weed/iam/**' + - 'test/s3/compatibility/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/s3tests.yml' + +concurrency: + group: ${{ github.head_ref }}/s3tests + cancel-in-progress: true + +permissions: + contents: read + +jobs: + basic-s3-tests: + name: Basic S3 tests (KV store) + runs-on: ubuntu-22.04 + timeout-minutes: 15 + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: '3.9' + + - name: Clone s3-tests + run: | + git clone https://github.com/ceph/s3-tests.git + cd s3-tests + sudo apt-get update -qq + sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev + pip install -r requirements.txt + pip install tox + pip install -e . + + - name: Fix S3 tests bucket creation conflicts + run: | + python3 test/s3/compatibility/fix_s3_tests_bucket_conflicts.py + env: + S3_TESTS_PATH: s3-tests + + - name: Run Basic S3 tests + timeout-minutes: 15 + env: + S3TEST_CONF: ../docker/compose/s3tests.conf + shell: bash + run: | + cd weed + go install -tags s3tests -buildvcs=false + set -x + # Create clean data directory for this test run + export WEED_DATA_DIR="/tmp/seaweedfs-s3tests-$(date +%s)" + mkdir -p "$WEED_DATA_DIR" + weed -v 3 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \ + -dir="$WEED_DATA_DIR" \ + -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \ + -volume.max=100 -volume.preStopSeconds=1 \ + -master.port=9333 -volume.port=8080 -filer.port=8888 -s3.port=8000 -metricsPort=9324 \ + -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none & + pid=$! + + # Wait for all SeaweedFS components to be ready + echo "Waiting for SeaweedFS components to start..." + for i in {1..30}; do + if curl -s http://localhost:9333/cluster/status > /dev/null 2>&1; then + echo "Master server is ready" + break + fi + echo "Waiting for master server... ($i/30)" + sleep 2 + done + + for i in {1..30}; do + if curl -s http://localhost:8080/status > /dev/null 2>&1; then + echo "Volume server is ready" + break + fi + echo "Waiting for volume server... ($i/30)" + sleep 2 + done + + for i in {1..30}; do + if curl -s http://localhost:8888/ > /dev/null 2>&1; then + echo "Filer is ready" + break + fi + echo "Waiting for filer... ($i/30)" + sleep 2 + done + + for i in {1..30}; do + if curl -s http://localhost:8000/ > /dev/null 2>&1; then + echo "S3 server is ready" + break + fi + echo "Waiting for S3 server... ($i/30)" + sleep 2 + done + + echo "All SeaweedFS components are ready!" + cd ../s3-tests + sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests/functional/test_s3.py + # The suite expects RGW's 400 InvalidPart for a partNumber past the last + # part; AWS answers 416 InvalidPartNumber, which is what we return. + sed -i "/# request PartNumber out of range/,+4{s/assert status == 400/assert status == 416/; s/assert error_code == 'InvalidPart'/assert error_code == 'InvalidPartNumber'/}" s3tests/functional/test_s3.py + + # Debug: Show the config file contents + echo "=== S3 Config File Contents ===" + cat ../docker/compose/s3tests.conf + echo "=== End Config ===" + + # Additional wait for S3-Filer integration to be fully ready + echo "Waiting additional 10 seconds for S3-Filer integration..." + sleep 10 + + # Test S3 connection before running tests + echo "Testing S3 connection..." + for i in {1..10}; do + if curl -s -f http://localhost:8000/ > /dev/null 2>&1; then + echo "S3 connection test successful" + break + fi + echo "S3 connection test failed, retrying... ($i/10)" + sleep 2 + done + + echo "✅ S3 server is responding, starting tests..." + + # Spawn the lifecycle worker so test_lifecycle_expiration etc. have + # something driving deletions. The s3tests build tag rescales one + # day to LifeCycleInterval=10s, so a 1d rule fires within ~10s of + # the upload's mtime; -dispatch / -checkpoint defaults are already + # tightened under the same build tag. + LC_LOG=/tmp/lifecycle-worker.log + # -debug routes glog to stderr so the bootstrap walker's progress + # shows up in $LC_LOG; without it weed shell silences glog. + (echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18000 -events 0 -runtime 1800s -refresh 2s" && echo exit) \ + | weed shell -debug -master=localhost:9333 \ + > "$LC_LOG" 2>&1 & + lc_pid=$! + # Aliveness check: a bad shell command exits in <1s and the suite + # would otherwise just timeout the expiration tests with no signal. + sleep 2 + if ! kill -0 "$lc_pid" 2>/dev/null; then + echo "lifecycle worker died on startup" + tail -50 "$LC_LOG" 2>/dev/null || true + exit 1 + fi + echo "lifecycle worker pid=$lc_pid" + + # bash -e exits the step on the first tox failure, so move teardown + # into a trap to guarantee the worker log + data dir reach the runner. + cleanup() { + status=$? + # SIGTERM first so the worker's stdout flushes; SIGKILL is the + # bash fallback if it ignores TERM. Reading the log AFTER the + # graceful-stop window catches the bootstrap walker's progress. + kill -TERM "$lc_pid" 2>/dev/null || true + kill -TERM "$pid" 2>/dev/null || true + sleep 1 + if [ "$status" -ne 0 ]; then + echo "=== lifecycle worker log (tail) ===" + tail -200 "$LC_LOG" 2>/dev/null || true + fi + kill -9 "$lc_pid" 2>/dev/null || true + kill -9 "$pid" 2>/dev/null || true + rm -rf "$WEED_DATA_DIR" 2>/dev/null || true + } + trap cleanup EXIT + + tox -- \ + s3tests/functional/test_s3.py::test_bucket_list_empty \ + s3tests/functional/test_s3.py::test_bucket_list_distinct \ + s3tests/functional/test_s3.py::test_bucket_list_many \ + s3tests/functional/test_s3.py::test_bucket_listv2_many \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_basic \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_basic \ + s3tests/functional/test_s3.py::test_bucket_listv2_encoding_basic \ + s3tests/functional/test_s3.py::test_bucket_list_encoding_basic \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix_ends_with_delimiter \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_alt \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_alt \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_underscore \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix_underscore \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_percentage \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_percentage \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_whitespace \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_whitespace \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_dot \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_dot \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_unreadable \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_unreadable \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_empty \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_empty \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_none \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_none \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_not_exist \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_not_exist \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_not_skip_special \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_basic \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_basic \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_alt \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_alt \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_not_exist \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_not_exist \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_delimiter_not_exist \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_delimiter_not_exist \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist \ + s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_notempty \ + s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_defaultempty \ + s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_empty \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_basic \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_basic \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_alt \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_alt \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_empty \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_empty \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_none \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_none \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_not_exist \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_not_exist \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_unreadable \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_unreadable \ + s3tests/functional/test_s3.py::test_bucket_list_maxkeys_one \ + s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_one \ + s3tests/functional/test_s3.py::test_bucket_list_maxkeys_zero \ + s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \ + s3tests/functional/test_s3.py::test_bucket_list_maxkeys_none \ + s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_none \ + s3tests/functional/test_s3.py::test_bucket_list_unordered \ + s3tests/functional/test_s3.py::test_bucket_listv2_unordered \ + s3tests/functional/test_s3.py::test_bucket_list_maxkeys_invalid \ + s3tests/functional/test_s3.py::test_bucket_list_marker_none \ + s3tests/functional/test_s3.py::test_bucket_list_marker_empty \ + s3tests/functional/test_s3.py::test_bucket_listv2_continuationtoken_empty \ + s3tests/functional/test_s3.py::test_bucket_listv2_continuationtoken \ + s3tests/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \ + s3tests/functional/test_s3.py::test_bucket_list_marker_unreadable \ + s3tests/functional/test_s3.py::test_bucket_listv2_startafter_unreadable \ + s3tests/functional/test_s3.py::test_bucket_list_marker_not_in_list \ + s3tests/functional/test_s3.py::test_bucket_listv2_startafter_not_in_list \ + s3tests/functional/test_s3.py::test_bucket_list_marker_after_list \ + s3tests/functional/test_s3.py::test_bucket_listv2_startafter_after_list \ + s3tests/functional/test_s3.py::test_bucket_list_return_data \ + s3tests/functional/test_s3.py::test_bucket_list_objects_anonymous \ + s3tests/functional/test_s3.py::test_bucket_listv2_objects_anonymous \ + s3tests/functional/test_s3.py::test_bucket_list_objects_anonymous_fail \ + s3tests/functional/test_s3.py::test_bucket_listv2_objects_anonymous_fail \ + s3tests/functional/test_s3.py::test_bucket_list_long_name \ + s3tests/functional/test_s3.py::test_bucket_list_special_prefix \ + s3tests/functional/test_s3.py::test_bucket_delete_notexist \ + s3tests/functional/test_s3.py::test_bucket_create_delete \ + s3tests/functional/test_s3.py::test_object_read_not_exist \ + s3tests/functional/test_s3.py::test_multi_object_delete \ + s3tests/functional/test_s3.py::test_multi_objectv2_delete \ + s3tests/functional/test_s3.py::test_object_head_zero_bytes \ + s3tests/functional/test_s3.py::test_object_write_check_etag \ + s3tests/functional/test_s3.py::test_object_write_cache_control \ + s3tests/functional/test_s3.py::test_object_write_expires \ + s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \ + s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \ + s3tests/functional/test_s3.py::test_object_write_file \ + s3tests/functional/test_s3.py::test_post_object_invalid_date_format \ + s3tests/functional/test_s3.py::test_post_object_no_key_specified \ + s3tests/functional/test_s3.py::test_post_object_missing_signature \ + s3tests/functional/test_s3.py::test_post_object_condition_is_case_sensitive \ + s3tests/functional/test_s3.py::test_post_object_expires_is_case_sensitive \ + s3tests/functional/test_s3.py::test_post_object_missing_expires_condition \ + s3tests/functional/test_s3.py::test_post_object_missing_conditions_list \ + s3tests/functional/test_s3.py::test_post_object_upload_size_limit_exceeded \ + s3tests/functional/test_s3.py::test_post_object_missing_content_length_argument \ + s3tests/functional/test_s3.py::test_post_object_invalid_content_length_argument \ + s3tests/functional/test_s3.py::test_post_object_upload_size_below_minimum \ + s3tests/functional/test_s3.py::test_post_object_empty_conditions \ + s3tests/functional/test_s3.py::test_get_object_ifmatch_good \ + s3tests/functional/test_s3.py::test_get_object_ifnonematch_good \ + s3tests/functional/test_s3.py::test_get_object_ifmatch_failed \ + s3tests/functional/test_s3.py::test_get_object_ifnonematch_failed \ + s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \ + s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \ + s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \ + s3tests/functional/test_s3.py::test_bucket_head \ + s3tests/functional/test_s3.py::test_bucket_head_notexist \ + s3tests/functional/test_s3.py::test_object_raw_authenticated \ + s3tests/functional/test_s3.py::test_object_raw_authenticated_bucket_acl \ + s3tests/functional/test_s3.py::test_object_raw_authenticated_object_acl \ + s3tests/functional/test_s3.py::test_object_raw_authenticated_object_gone \ + s3tests/functional/test_s3.py::test_object_raw_get_x_amz_expires_out_range_zero \ + s3tests/functional/test_s3.py::test_object_anon_put \ + s3tests/functional/test_s3.py::test_object_put_authenticated \ + s3tests/functional/test_s3.py::test_bucket_recreate_overwrite_acl \ + s3tests/functional/test_s3.py::test_bucket_recreate_new_acl \ + s3tests/functional/test_s3.py::test_buckets_create_then_list \ + s3tests/functional/test_s3.py::test_buckets_list_ctime \ + s3tests/functional/test_s3.py::test_list_buckets_invalid_auth \ + s3tests/functional/test_s3.py::test_list_buckets_bad_auth \ + s3tests/functional/test_s3.py::test_bucket_create_naming_good_contains_period \ + s3tests/functional/test_s3.py::test_bucket_create_naming_good_contains_hyphen \ + s3tests/functional/test_s3.py::test_bucket_list_special_prefix \ + s3tests/functional/test_s3.py::test_object_copy_zero_size \ + s3tests/functional/test_s3.py::test_object_copy_same_bucket \ + s3tests/functional/test_s3.py::test_object_copy_to_itself \ + s3tests/functional/test_s3.py::test_object_copy_diff_bucket \ + s3tests/functional/test_s3.py::test_object_copy_canned_acl \ + s3tests/functional/test_s3.py::test_object_copy_bucket_not_found \ + s3tests/functional/test_s3.py::test_object_copy_key_not_found \ + s3tests/functional/test_s3.py::test_multipart_copy_small \ + s3tests/functional/test_s3.py::test_multipart_copy_without_range \ + s3tests/functional/test_s3.py::test_multipart_copy_special_names \ + s3tests/functional/test_s3.py::test_multipart_copy_multiple_sizes \ + s3tests/functional/test_s3.py::test_multipart_get_part \ + s3tests/functional/test_s3.py::test_multipart_upload \ + s3tests/functional/test_s3.py::test_multipart_upload_empty \ + s3tests/functional/test_s3.py::test_multipart_upload_multiple_sizes \ + s3tests/functional/test_s3.py::test_multipart_upload_contents \ + s3tests/functional/test_s3.py::test_multipart_upload_overwrite_existing_object \ + s3tests/functional/test_s3.py::test_multipart_upload_size_too_small \ + s3tests/functional/test_s3.py::test_multipart_resend_first_finishes_last \ + s3tests/functional/test_s3.py::test_multipart_upload_resend_part \ + s3tests/functional/test_s3.py::test_multipart_upload_missing_part \ + s3tests/functional/test_s3.py::test_multipart_upload_incorrect_etag \ + s3tests/functional/test_s3.py::test_abort_multipart_upload \ + s3tests/functional/test_s3.py::test_list_multipart_upload \ + s3tests/functional/test_s3.py::test_atomic_read_1mb \ + s3tests/functional/test_s3.py::test_atomic_read_4mb \ + s3tests/functional/test_s3.py::test_atomic_read_8mb \ + s3tests/functional/test_s3.py::test_atomic_write_1mb \ + s3tests/functional/test_s3.py::test_atomic_write_4mb \ + s3tests/functional/test_s3.py::test_atomic_write_8mb \ + s3tests/functional/test_s3.py::test_atomic_dual_write_1mb \ + s3tests/functional/test_s3.py::test_atomic_dual_write_4mb \ + s3tests/functional/test_s3.py::test_atomic_dual_write_8mb \ + s3tests/functional/test_s3.py::test_atomic_multipart_upload_write \ + s3tests/functional/test_s3.py::test_ranged_request_response_code \ + s3tests/functional/test_s3.py::test_ranged_big_request_response_code \ + s3tests/functional/test_s3.py::test_ranged_request_skip_leading_bytes_response_code \ + s3tests/functional/test_s3.py::test_ranged_request_return_trailing_bytes_response_code \ + s3tests/functional/test_s3.py::test_copy_object_ifmatch_good \ + s3tests/functional/test_s3.py::test_copy_object_ifnonematch_failed \ + s3tests/functional/test_s3.py::test_copy_object_ifmatch_failed \ + s3tests/functional/test_s3.py::test_copy_object_ifnonematch_good \ + s3tests/functional/test_s3.py::test_lifecycle_set \ + s3tests/functional/test_s3.py::test_lifecycle_get \ + s3tests/functional/test_s3.py::test_lifecycle_set_filter \ + s3tests/functional/test_s3.py::test_lifecycle_expiration \ + s3tests/functional/test_s3.py::test_lifecyclev2_expiration + # cleanup() trap handles worker/server kill + data dir wipe. + + versioning-tests: + name: S3 Versioning & Object Lock tests + runs-on: ubuntu-22.04 + timeout-minutes: 15 + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: '3.9' + + - name: Clone s3-tests + run: | + git clone https://github.com/ceph/s3-tests.git + cd s3-tests + sudo apt-get update -qq + sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev + pip install -r requirements.txt + pip install tox + pip install -e . + + - name: Fix S3 tests bucket creation conflicts + run: | + python3 test/s3/compatibility/fix_s3_tests_bucket_conflicts.py + env: + S3_TESTS_PATH: s3-tests + + - name: Run S3 Object Lock, Retention, and Versioning tests + timeout-minutes: 15 + shell: bash + run: | + cd weed + go install -buildvcs=false + set -x + # Create clean data directory for this test run + export WEED_DATA_DIR="/tmp/seaweedfs-objectlock-versioning-$(date +%s)" + mkdir -p "$WEED_DATA_DIR" + + # Verify S3 config file exists + echo "Checking S3 config file: $GITHUB_WORKSPACE/docker/compose/s3.json" + ls -la "$GITHUB_WORKSPACE/docker/compose/s3.json" + weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \ + -dir="$WEED_DATA_DIR" \ + -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \ + -volume.max=100 -volume.preStopSeconds=1 \ + -master.port=9334 -volume.port=8081 -filer.port=8889 -s3.port=8001 -metricsPort=9325 \ + -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none & + pid=$! + + # Wait for all SeaweedFS components to be ready + echo "Waiting for SeaweedFS components to start..." + for i in {1..30}; do + if curl -s http://localhost:9334/cluster/status > /dev/null 2>&1; then + echo "Master server is ready" + break + fi + echo "Waiting for master server... ($i/30)" + sleep 2 + done + + for i in {1..30}; do + if curl -s http://localhost:8081/status > /dev/null 2>&1; then + echo "Volume server is ready" + break + fi + echo "Waiting for volume server... ($i/30)" + sleep 2 + done + + for i in {1..30}; do + if curl -s http://localhost:8889/ > /dev/null 2>&1; then + echo "Filer is ready" + break + fi + echo "Waiting for filer... ($i/30)" + sleep 2 + done + + for i in {1..30}; do + if curl -s http://localhost:8001/ > /dev/null 2>&1; then + echo "S3 server is ready" + break + fi + echo "Waiting for S3 server... ($i/30)" + sleep 2 + done + + echo "All SeaweedFS components are ready!" + cd ../s3-tests + sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests/functional/test_s3.py + # Create and update s3tests.conf to use port 8001 + cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-versioning.conf + sed -i 's/port = 8000/port = 8001/g' ../docker/compose/s3tests-versioning.conf + sed -i 's/:8000/:8001/g' ../docker/compose/s3tests-versioning.conf + sed -i 's/localhost:8000/localhost:8001/g' ../docker/compose/s3tests-versioning.conf + sed -i 's/127\.0\.0\.1:8000/127.0.0.1:8001/g' ../docker/compose/s3tests-versioning.conf + # Use the configured bucket prefix from config and do not override with unique prefixes + # This avoids mismatch in tests that rely on a fixed provided name + export S3TEST_CONF=../docker/compose/s3tests-versioning.conf + + # Debug: Show the config file contents + echo "=== S3 Config File Contents ===" + cat ../docker/compose/s3tests-versioning.conf + echo "=== End Config ===" + + # Additional wait for S3-Filer integration to be fully ready + echo "Waiting additional 10 seconds for S3-Filer integration..." + sleep 10 + + # Test S3 connection before running tests + echo "Testing S3 connection..." + for i in {1..10}; do + if curl -s -f http://localhost:8001/ > /dev/null 2>&1; then + echo "S3 connection test successful" + break + fi + echo "S3 connection test failed, retrying... ($i/10)" + sleep 2 + done + + # Force cleanup any existing buckets to avoid conflicts + echo "Cleaning up any existing buckets..." + python3 -c " + import boto3 + from botocore.exceptions import ClientError + try: + s3 = boto3.client('s3', + endpoint_url='http://localhost:8001', + aws_access_key_id='0555b35654ad1656d804', + aws_secret_access_key='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==') + buckets = s3.list_buckets()['Buckets'] + for bucket in buckets: + bucket_name = bucket['Name'] + print(f'Deleting bucket: {bucket_name}') + try: + # Delete all objects first + objects = s3.list_objects_v2(Bucket=bucket_name) + if 'Contents' in objects: + for obj in objects['Contents']: + s3.delete_object(Bucket=bucket_name, Key=obj['Key']) + # Delete all versions if versioning enabled + versions = s3.list_object_versions(Bucket=bucket_name) + if 'Versions' in versions: + for version in versions['Versions']: + s3.delete_object(Bucket=bucket_name, Key=version['Key'], VersionId=version['VersionId']) + if 'DeleteMarkers' in versions: + for marker in versions['DeleteMarkers']: + s3.delete_object(Bucket=bucket_name, Key=marker['Key'], VersionId=marker['VersionId']) + # Delete bucket + s3.delete_bucket(Bucket=bucket_name) + except ClientError as e: + print(f'Error deleting bucket {bucket_name}: {e}') + except Exception as e: + print(f'Cleanup failed: {e}') + " || echo "Cleanup completed with some errors (expected)" + + # Run versioning and object lock tests once (avoid duplicates) + tox -- s3tests/functional/test_s3.py -k "object_lock or versioning" --tb=short + kill -9 $pid || true + # Clean up data directory + rm -rf "$WEED_DATA_DIR" || true + + cors-tests: + name: S3 CORS tests + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: '3.9' + + - name: Clone s3-tests + run: | + git clone https://github.com/ceph/s3-tests.git + cd s3-tests + sudo apt-get update -qq + sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev + pip install -r requirements.txt + pip install tox + pip install -e . + + - name: Run S3 CORS tests + timeout-minutes: 10 + shell: bash + run: | + cd weed + go install -buildvcs=false + set -x + # Create clean data directory for this test run + export WEED_DATA_DIR="/tmp/seaweedfs-cors-test-$(date +%s)" + mkdir -p "$WEED_DATA_DIR" + weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \ + -dir="$WEED_DATA_DIR" \ + -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \ + -volume.max=100 -volume.preStopSeconds=1 \ + -master.port=9335 -volume.port=8082 -filer.port=8890 -s3.port=8002 -metricsPort=9326 \ + -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none & + pid=$! + + # Wait for all SeaweedFS components to be ready + echo "Waiting for SeaweedFS components to start..." + for i in {1..30}; do + if curl -s http://localhost:9335/cluster/status > /dev/null 2>&1; then + echo "Master server is ready" + break + fi + echo "Waiting for master server... ($i/30)" + sleep 2 + done + + for i in {1..30}; do + if curl -s http://localhost:8082/status > /dev/null 2>&1; then + echo "Volume server is ready" + break + fi + echo "Waiting for volume server... ($i/30)" + sleep 2 + done + + for i in {1..30}; do + if curl -s http://localhost:8890/ > /dev/null 2>&1; then + echo "Filer is ready" + break + fi + echo "Waiting for filer... ($i/30)" + sleep 2 + done + + for i in {1..30}; do + if curl -s http://localhost:8002/ > /dev/null 2>&1; then + echo "S3 server is ready" + break + fi + echo "Waiting for S3 server... ($i/30)" + sleep 2 + done + + echo "All SeaweedFS components are ready!" + cd ../s3-tests + sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests/functional/test_s3.py + # Create and update s3tests.conf to use port 8002 + cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-cors.conf + sed -i 's/port = 8000/port = 8002/g' ../docker/compose/s3tests-cors.conf + sed -i 's/:8000/:8002/g' ../docker/compose/s3tests-cors.conf + sed -i 's/localhost:8000/localhost:8002/g' ../docker/compose/s3tests-cors.conf + sed -i 's/127\.0\.0\.1:8000/127.0.0.1:8002/g' ../docker/compose/s3tests-cors.conf + export S3TEST_CONF=../docker/compose/s3tests-cors.conf + + # Debug: Show the config file contents + echo "=== S3 Config File Contents ===" + cat ../docker/compose/s3tests-cors.conf + echo "=== End Config ===" + + # Additional wait for S3-Filer integration to be fully ready + echo "Waiting additional 10 seconds for S3-Filer integration..." + sleep 10 + + # Test S3 connection before running tests + echo "Testing S3 connection..." + for i in {1..10}; do + if curl -s -f http://localhost:8002/ > /dev/null 2>&1; then + echo "S3 connection test successful" + break + fi + echo "S3 connection test failed, retrying... ($i/10)" + sleep 2 + done + # Run CORS-specific tests from s3-tests suite + tox -- s3tests/functional/test_s3.py -k "cors" --tb=short || echo "No CORS tests found in s3-tests suite" + # If no specific CORS tests exist, run bucket configuration tests that include CORS + tox -- s3tests/functional/test_s3.py::test_put_bucket_cors || echo "No put_bucket_cors test found" + tox -- s3tests/functional/test_s3.py::test_get_bucket_cors || echo "No get_bucket_cors test found" + tox -- s3tests/functional/test_s3.py::test_delete_bucket_cors || echo "No delete_bucket_cors test found" + kill -9 $pid || true + # Clean up data directory + rm -rf "$WEED_DATA_DIR" || true + + copy-tests: + name: SeaweedFS Custom S3 Copy tests + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Run SeaweedFS Custom S3 Copy tests + timeout-minutes: 10 + shell: bash + run: | + cd weed + go install -buildvcs=false + # Create clean data directory for this test run + export WEED_DATA_DIR="/tmp/seaweedfs-copy-test-$(date +%s)" + mkdir -p "$WEED_DATA_DIR" + set -x + # Each test bucket is its own collection and grows 7 volumes; the suite runs + # faster than the heartbeat that returns slots from the deleted collections. + weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \ + -dir="$WEED_DATA_DIR" \ + -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \ + -volume.max=300 -volume.preStopSeconds=1 \ + -master.port=9336 -volume.port=8083 -filer.port=8891 -s3.port=8003 -metricsPort=9327 \ + -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none & + pid=$! + + # Wait for all SeaweedFS components to be ready + echo "Waiting for SeaweedFS components to start..." + for i in {1..30}; do + if curl -s http://localhost:9336/cluster/status > /dev/null 2>&1; then + echo "Master server is ready" + break + fi + echo "Waiting for master server... ($i/30)" + sleep 2 + done + + for i in {1..30}; do + if curl -s http://localhost:8083/status > /dev/null 2>&1; then + echo "Volume server is ready" + break + fi + echo "Waiting for volume server... ($i/30)" + sleep 2 + done + + for i in {1..30}; do + if curl -s http://localhost:8891/ > /dev/null 2>&1; then + echo "Filer is ready" + break + fi + echo "Waiting for filer... ($i/30)" + sleep 2 + done + + for i in {1..30}; do + if curl -s http://localhost:8003/ > /dev/null 2>&1; then + echo "S3 server is ready" + break + fi + echo "Waiting for S3 server... ($i/30)" + sleep 2 + done + + echo "All SeaweedFS components are ready!" + cd ../test/s3/copying + # Patch Go tests to use the correct S3 endpoint (port 8003) + sed -i 's/http:\/\/127\.0\.0\.1:8000/http:\/\/127.0.0.1:8003/g' s3_copying_test.go + + # Debug: Show what endpoint the Go tests will use + echo "=== Go Test Configuration ===" + grep -n "127.0.0.1" s3_copying_test.go || echo "No IP configuration found" + echo "=== End Configuration ===" + + # Additional wait for S3-Filer integration to be fully ready + echo "Waiting additional 10 seconds for S3-Filer integration..." + sleep 10 + + # Test S3 connection before running tests + echo "Testing S3 connection..." + for i in {1..10}; do + if curl -s -f http://localhost:8003/ > /dev/null 2>&1; then + echo "S3 connection test successful" + break + fi + echo "S3 connection test failed, retrying... ($i/10)" + sleep 2 + done + + MASTER_ENDPOINT="http://127.0.0.1:9336" go test -v + kill -9 $pid || true + # Clean up data directory + rm -rf "$WEED_DATA_DIR" || true + + sql-store-tests: + name: Basic S3 tests (SQL store) + runs-on: ubuntu-22.04 + timeout-minutes: 15 + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: '3.9' + + - name: Clone s3-tests + run: | + git clone https://github.com/ceph/s3-tests.git + cd s3-tests + sudo apt-get update -qq + sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev + pip install -r requirements.txt + pip install tox + pip install -e . + + - name: Run Ceph S3 tests with SQL store + timeout-minutes: 15 + shell: bash + run: | + cd weed + + # Debug: Check for port conflicts before starting + echo "=== Pre-start Port Check ===" + netstat -tulpn | grep -E "(9337|8085|8892|8004|9328)" || echo "Ports are free" + + # Kill any existing weed processes that might interfere + echo "=== Cleanup existing processes ===" + pkill -f weed || echo "No weed processes found" + + # More aggressive port cleanup using multiple methods + for port in 9337 8085 8892 8004 9328; do + echo "Cleaning port $port..." + + # Method 1: lsof + pid=$(lsof -ti :$port 2>/dev/null || echo "") + if [ -n "$pid" ]; then + echo "Found process $pid using port $port (via lsof)" + kill -9 $pid 2>/dev/null || echo "Failed to kill $pid" + fi + + # Method 2: netstat + ps (for cases where lsof fails) + netstat_pids=$(netstat -tlnp 2>/dev/null | grep ":$port " | awk '{print $7}' | cut -d'/' -f1 | grep -v '^-$' || echo "") + for npid in $netstat_pids; do + if [ -n "$npid" ] && [ "$npid" != "-" ]; then + echo "Found process $npid using port $port (via netstat)" + kill -9 $npid 2>/dev/null || echo "Failed to kill $npid" + fi + done + + # Method 3: fuser (if available) + if command -v fuser >/dev/null 2>&1; then + fuser -k ${port}/tcp 2>/dev/null || echo "No process found via fuser for port $port" + fi + + sleep 1 + done + + # Wait for ports to be released + sleep 5 + + echo "=== Post-cleanup Port Check ===" + netstat -tulpn | grep -E "(9337|8085|8892|8004|9328)" || echo "All ports are now free" + + # If any ports are still in use, fail fast + if netstat -tulpn | grep -E "(9337|8085|8892|8004|9328)" >/dev/null 2>&1; then + echo "❌ ERROR: Some ports are still in use after aggressive cleanup!" + echo "=== Detailed Port Analysis ===" + for port in 9337 8085 8892 8004 9328; do + echo "Port $port:" + netstat -tlnp 2>/dev/null | grep ":$port " || echo " Not in use" + lsof -i :$port 2>/dev/null || echo " No lsof info" + done + exit 1 + fi + + go install -tags "sqlite s3tests" -buildvcs=false + # Create clean data directory for this test run with unique timestamp and process ID + export WEED_DATA_DIR="/tmp/seaweedfs-sql-test-$(date +%s)-$$" + mkdir -p "$WEED_DATA_DIR" + chmod 777 "$WEED_DATA_DIR" + + # SQLite-specific configuration + export WEED_LEVELDB2_ENABLED="false" + export WEED_SQLITE_ENABLED="true" + export WEED_SQLITE_DBFILE="$WEED_DATA_DIR/filer.db" + + echo "=== SQL Store Configuration ===" + echo "Data Dir: $WEED_DATA_DIR" + echo "SQLite DB: $WEED_SQLITE_DBFILE" + echo "LEVELDB2_ENABLED: $WEED_LEVELDB2_ENABLED" + echo "SQLITE_ENABLED: $WEED_SQLITE_ENABLED" + + set -x + weed -v 1 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \ + -dir="$WEED_DATA_DIR" \ + -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \ + -volume.max=100 -volume.preStopSeconds=1 \ + -master.port=9337 -volume.port=8085 -filer.port=8892 -s3.port=8004 -metricsPort=9328 \ + -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" \ + -master.peers=none \ + > /tmp/seaweedfs-sql-server.log 2>&1 & + pid=$! + + echo "=== Server started with PID: $pid ===" + + # Wait for all SeaweedFS components to be ready + echo "Waiting for SeaweedFS components to start..." + + # Check if server process is still alive before waiting + if ! kill -0 $pid 2>/dev/null; then + echo "❌ Server process died immediately after start" + echo "=== Immediate Log Check ===" + tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null || echo "No log available" + exit 1 + fi + + sleep 5 # Give SQLite more time to initialize + + for i in {1..30}; do + if curl -s http://localhost:9337/cluster/status > /dev/null 2>&1; then + echo "Master server is ready" + break + fi + echo "Waiting for master server... ($i/30)" + # Check if server process is still alive + if ! kill -0 $pid 2>/dev/null; then + echo "❌ Server process died while waiting for master" + tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null + exit 1 + fi + sleep 2 + done + + for i in {1..30}; do + if curl -s http://localhost:8085/status > /dev/null 2>&1; then + echo "Volume server is ready" + break + fi + echo "Waiting for volume server... ($i/30)" + if ! kill -0 $pid 2>/dev/null; then + echo "❌ Server process died while waiting for volume" + tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null + exit 1 + fi + sleep 2 + done + + for i in {1..30}; do + if curl -s http://localhost:8892/ > /dev/null 2>&1; then + echo "Filer (SQLite) is ready" + break + fi + echo "Waiting for filer (SQLite)... ($i/30)" + if ! kill -0 $pid 2>/dev/null; then + echo "❌ Server process died while waiting for filer" + tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null + exit 1 + fi + sleep 2 + done + + # Extra wait for SQLite filer to fully initialize + echo "Giving SQLite filer extra time to initialize..." + sleep 5 + + for i in {1..30}; do + if curl -s http://localhost:8004/ > /dev/null 2>&1; then + echo "S3 server is ready" + break + fi + echo "Waiting for S3 server... ($i/30)" + if ! kill -0 $pid 2>/dev/null; then + echo "❌ Server process died while waiting for S3" + tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null + exit 1 + fi + sleep 2 + done + + echo "All SeaweedFS components are ready!" + cd ../s3-tests + sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests/functional/test_s3.py + # The suite expects RGW's 400 InvalidPart for a partNumber past the last + # part; AWS answers 416 InvalidPartNumber, which is what we return. + sed -i "/# request PartNumber out of range/,+4{s/assert status == 400/assert status == 416/; s/assert error_code == 'InvalidPart'/assert error_code == 'InvalidPartNumber'/}" s3tests/functional/test_s3.py + # Create and update s3tests.conf to use port 8004 + cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-sql.conf + sed -i 's/port = 8000/port = 8004/g' ../docker/compose/s3tests-sql.conf + sed -i 's/:8000/:8004/g' ../docker/compose/s3tests-sql.conf + sed -i 's/localhost:8000/localhost:8004/g' ../docker/compose/s3tests-sql.conf + sed -i 's/127\.0\.0\.1:8000/127.0.0.1:8004/g' ../docker/compose/s3tests-sql.conf + export S3TEST_CONF=../docker/compose/s3tests-sql.conf + + # Debug: Show the config file contents + echo "=== S3 Config File Contents ===" + cat ../docker/compose/s3tests-sql.conf + echo "=== End Config ===" + + # Additional wait for S3-Filer integration to be fully ready + echo "Waiting additional 10 seconds for S3-Filer integration..." + sleep 10 + + # Test S3 connection before running tests + echo "Testing S3 connection..." + + # Debug: Check if SeaweedFS processes are running + echo "=== Process Status ===" + ps aux | grep -E "(weed|seaweedfs)" | grep -v grep || echo "No SeaweedFS processes found" + + # Debug: Check port status + echo "=== Port Status ===" + netstat -tulpn | grep -E "(8004|9337|8085|8892)" || echo "Ports not found" + + # Debug: Check server logs + echo "=== Recent Server Logs ===" + echo "--- SQL Server Log ---" + tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null || echo "No SQL server log found" + echo "--- Other Logs ---" + ls -la /tmp/seaweedfs-*.log 2>/dev/null || echo "No other log files found" + + for i in {1..10}; do + if curl -s -f http://localhost:8004/ > /dev/null 2>&1; then + echo "S3 connection test successful" + break + fi + echo "S3 connection test failed, retrying... ($i/10)" + + # Debug: Try different HTTP methods + echo "Debug: Testing different endpoints..." + curl -s -I http://localhost:8004/ || echo "HEAD request failed" + curl -s http://localhost:8004/status || echo "Status endpoint failed" + + sleep 2 + done + + # Spawn the lifecycle worker (see basic-tests block for context). + LC_LOG=/tmp/lifecycle-worker-sql.log + (echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18004 -events 0 -runtime 1800s -refresh 2s" && echo exit) \ + | weed shell -debug -master=localhost:9337 \ + > "$LC_LOG" 2>&1 & + lc_pid=$! + sleep 2 + if ! kill -0 "$lc_pid" 2>/dev/null; then + echo "lifecycle worker died on startup" + tail -50 "$LC_LOG" 2>/dev/null || true + exit 1 + fi + echo "lifecycle worker pid=$lc_pid" + + cleanup() { + status=$? + # SIGTERM first so the worker's stdout flushes; SIGKILL is the + # bash fallback if it ignores TERM. Reading the log AFTER the + # graceful-stop window catches the bootstrap walker's progress. + kill -TERM "$lc_pid" 2>/dev/null || true + kill -TERM "$pid" 2>/dev/null || true + sleep 1 + if [ "$status" -ne 0 ]; then + echo "=== lifecycle worker log (tail) ===" + tail -200 "$LC_LOG" 2>/dev/null || true + fi + kill -9 "$lc_pid" 2>/dev/null || true + kill -9 "$pid" 2>/dev/null || true + rm -rf "$WEED_DATA_DIR" 2>/dev/null || true + } + trap cleanup EXIT + + tox -- \ + s3tests/functional/test_s3.py::test_bucket_list_empty \ + s3tests/functional/test_s3.py::test_bucket_list_distinct \ + s3tests/functional/test_s3.py::test_bucket_list_many \ + s3tests/functional/test_s3.py::test_bucket_listv2_many \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_basic \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_basic \ + s3tests/functional/test_s3.py::test_bucket_listv2_encoding_basic \ + s3tests/functional/test_s3.py::test_bucket_list_encoding_basic \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix_ends_with_delimiter \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_alt \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_alt \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_underscore \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix_underscore \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_percentage \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_percentage \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_whitespace \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_whitespace \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_dot \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_dot \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_unreadable \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_unreadable \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_empty \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_empty \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_none \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_none \ + s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_not_exist \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_not_exist \ + s3tests/functional/test_s3.py::test_bucket_list_delimiter_not_skip_special \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_basic \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_basic \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_alt \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_alt \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_not_exist \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_not_exist \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_delimiter_not_exist \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_delimiter_not_exist \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist \ + s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_notempty \ + s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_defaultempty \ + s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_empty \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_basic \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_basic \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_alt \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_alt \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_empty \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_empty \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_none \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_none \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_not_exist \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_not_exist \ + s3tests/functional/test_s3.py::test_bucket_list_prefix_unreadable \ + s3tests/functional/test_s3.py::test_bucket_listv2_prefix_unreadable \ + s3tests/functional/test_s3.py::test_bucket_list_maxkeys_one \ + s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_one \ + s3tests/functional/test_s3.py::test_bucket_list_maxkeys_zero \ + s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \ + s3tests/functional/test_s3.py::test_bucket_list_maxkeys_none \ + s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_none \ + s3tests/functional/test_s3.py::test_bucket_list_unordered \ + s3tests/functional/test_s3.py::test_bucket_listv2_unordered \ + s3tests/functional/test_s3.py::test_bucket_list_maxkeys_invalid \ + s3tests/functional/test_s3.py::test_bucket_list_marker_none \ + s3tests/functional/test_s3.py::test_bucket_list_marker_empty \ + s3tests/functional/test_s3.py::test_bucket_listv2_continuationtoken_empty \ + s3tests/functional/test_s3.py::test_bucket_listv2_continuationtoken \ + s3tests/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \ + s3tests/functional/test_s3.py::test_bucket_list_marker_unreadable \ + s3tests/functional/test_s3.py::test_bucket_listv2_startafter_unreadable \ + s3tests/functional/test_s3.py::test_bucket_list_marker_not_in_list \ + s3tests/functional/test_s3.py::test_bucket_listv2_startafter_not_in_list \ + s3tests/functional/test_s3.py::test_bucket_list_marker_after_list \ + s3tests/functional/test_s3.py::test_bucket_listv2_startafter_after_list \ + s3tests/functional/test_s3.py::test_bucket_list_return_data \ + s3tests/functional/test_s3.py::test_bucket_list_objects_anonymous \ + s3tests/functional/test_s3.py::test_bucket_listv2_objects_anonymous \ + s3tests/functional/test_s3.py::test_bucket_list_objects_anonymous_fail \ + s3tests/functional/test_s3.py::test_bucket_listv2_objects_anonymous_fail \ + s3tests/functional/test_s3.py::test_bucket_list_long_name \ + s3tests/functional/test_s3.py::test_bucket_list_special_prefix \ + s3tests/functional/test_s3.py::test_bucket_delete_notexist \ + s3tests/functional/test_s3.py::test_bucket_create_delete \ + s3tests/functional/test_s3.py::test_object_read_not_exist \ + s3tests/functional/test_s3.py::test_multi_object_delete \ + s3tests/functional/test_s3.py::test_multi_objectv2_delete \ + s3tests/functional/test_s3.py::test_object_head_zero_bytes \ + s3tests/functional/test_s3.py::test_object_write_check_etag \ + s3tests/functional/test_s3.py::test_object_write_cache_control \ + s3tests/functional/test_s3.py::test_object_write_expires \ + s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \ + s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \ + s3tests/functional/test_s3.py::test_object_write_file \ + s3tests/functional/test_s3.py::test_post_object_invalid_date_format \ + s3tests/functional/test_s3.py::test_post_object_no_key_specified \ + s3tests/functional/test_s3.py::test_post_object_missing_signature \ + s3tests/functional/test_s3.py::test_post_object_condition_is_case_sensitive \ + s3tests/functional/test_s3.py::test_post_object_expires_is_case_sensitive \ + s3tests/functional/test_s3.py::test_post_object_missing_expires_condition \ + s3tests/functional/test_s3.py::test_post_object_missing_conditions_list \ + s3tests/functional/test_s3.py::test_post_object_upload_size_limit_exceeded \ + s3tests/functional/test_s3.py::test_post_object_missing_content_length_argument \ + s3tests/functional/test_s3.py::test_post_object_invalid_content_length_argument \ + s3tests/functional/test_s3.py::test_post_object_upload_size_below_minimum \ + s3tests/functional/test_s3.py::test_post_object_empty_conditions \ + s3tests/functional/test_s3.py::test_get_object_ifmatch_good \ + s3tests/functional/test_s3.py::test_get_object_ifnonematch_good \ + s3tests/functional/test_s3.py::test_get_object_ifmatch_failed \ + s3tests/functional/test_s3.py::test_get_object_ifnonematch_failed \ + s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \ + s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \ + s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \ + s3tests/functional/test_s3.py::test_bucket_head \ + s3tests/functional/test_s3.py::test_bucket_head_notexist \ + s3tests/functional/test_s3.py::test_object_raw_authenticated \ + s3tests/functional/test_s3.py::test_object_raw_authenticated_bucket_acl \ + s3tests/functional/test_s3.py::test_object_raw_authenticated_object_acl \ + s3tests/functional/test_s3.py::test_object_raw_authenticated_object_gone \ + s3tests/functional/test_s3.py::test_object_raw_get_x_amz_expires_out_range_zero \ + s3tests/functional/test_s3.py::test_object_anon_put \ + s3tests/functional/test_s3.py::test_object_put_authenticated \ + s3tests/functional/test_s3.py::test_bucket_recreate_overwrite_acl \ + s3tests/functional/test_s3.py::test_bucket_recreate_new_acl \ + s3tests/functional/test_s3.py::test_buckets_create_then_list \ + s3tests/functional/test_s3.py::test_buckets_list_ctime \ + s3tests/functional/test_s3.py::test_list_buckets_invalid_auth \ + s3tests/functional/test_s3.py::test_list_buckets_bad_auth \ + s3tests/functional/test_s3.py::test_bucket_create_naming_good_contains_period \ + s3tests/functional/test_s3.py::test_bucket_create_naming_good_contains_hyphen \ + s3tests/functional/test_s3.py::test_bucket_list_special_prefix \ + s3tests/functional/test_s3.py::test_object_copy_zero_size \ + s3tests/functional/test_s3.py::test_object_copy_same_bucket \ + s3tests/functional/test_s3.py::test_object_copy_to_itself \ + s3tests/functional/test_s3.py::test_object_copy_diff_bucket \ + s3tests/functional/test_s3.py::test_object_copy_canned_acl \ + s3tests/functional/test_s3.py::test_object_copy_bucket_not_found \ + s3tests/functional/test_s3.py::test_object_copy_key_not_found \ + s3tests/functional/test_s3.py::test_multipart_copy_small \ + s3tests/functional/test_s3.py::test_multipart_copy_without_range \ + s3tests/functional/test_s3.py::test_multipart_copy_special_names \ + s3tests/functional/test_s3.py::test_multipart_copy_multiple_sizes \ + s3tests/functional/test_s3.py::test_multipart_get_part \ + s3tests/functional/test_s3.py::test_multipart_upload \ + s3tests/functional/test_s3.py::test_multipart_upload_empty \ + s3tests/functional/test_s3.py::test_multipart_upload_multiple_sizes \ + s3tests/functional/test_s3.py::test_multipart_upload_contents \ + s3tests/functional/test_s3.py::test_multipart_upload_overwrite_existing_object \ + s3tests/functional/test_s3.py::test_multipart_upload_size_too_small \ + s3tests/functional/test_s3.py::test_multipart_resend_first_finishes_last \ + s3tests/functional/test_s3.py::test_multipart_upload_resend_part \ + s3tests/functional/test_s3.py::test_multipart_upload_missing_part \ + s3tests/functional/test_s3.py::test_multipart_upload_incorrect_etag \ + s3tests/functional/test_s3.py::test_abort_multipart_upload \ + s3tests/functional/test_s3.py::test_list_multipart_upload \ + s3tests/functional/test_s3.py::test_atomic_read_1mb \ + s3tests/functional/test_s3.py::test_atomic_read_4mb \ + s3tests/functional/test_s3.py::test_atomic_read_8mb \ + s3tests/functional/test_s3.py::test_atomic_write_1mb \ + s3tests/functional/test_s3.py::test_atomic_write_4mb \ + s3tests/functional/test_s3.py::test_atomic_write_8mb \ + s3tests/functional/test_s3.py::test_atomic_dual_write_1mb \ + s3tests/functional/test_s3.py::test_atomic_dual_write_4mb \ + s3tests/functional/test_s3.py::test_atomic_dual_write_8mb \ + s3tests/functional/test_s3.py::test_atomic_multipart_upload_write \ + s3tests/functional/test_s3.py::test_ranged_request_response_code \ + s3tests/functional/test_s3.py::test_ranged_big_request_response_code \ + s3tests/functional/test_s3.py::test_ranged_request_skip_leading_bytes_response_code \ + s3tests/functional/test_s3.py::test_ranged_request_return_trailing_bytes_response_code \ + s3tests/functional/test_s3.py::test_copy_object_ifmatch_good \ + s3tests/functional/test_s3.py::test_copy_object_ifnonematch_failed \ + s3tests/functional/test_s3.py::test_copy_object_ifmatch_failed \ + s3tests/functional/test_s3.py::test_copy_object_ifnonematch_good \ + s3tests/functional/test_s3.py::test_lifecycle_set \ + s3tests/functional/test_s3.py::test_lifecycle_get \ + s3tests/functional/test_s3.py::test_lifecycle_set_filter \ + s3tests/functional/test_s3.py::test_lifecycle_expiration \ + s3tests/functional/test_s3.py::test_lifecyclev2_expiration + # cleanup() trap handles worker/server kill + data dir wipe. + + diff --git a/.github/workflows/samba-integration.yml b/.github/workflows/samba-integration.yml new file mode 100644 index 0000000..082ec5a --- /dev/null +++ b/.github/workflows/samba-integration.yml @@ -0,0 +1,120 @@ +name: "Samba on FUSE Integration" + +on: + push: + branches: [ master, main ] + paths: + - 'weed/mount/**' + - 'weed/filer/**' + - 'weed/cluster/**' + - 'test/samba/**' + - '.github/workflows/samba-integration.yml' + pull_request: + branches: [ master, main ] + paths: + - 'weed/mount/**' + - 'weed/filer/**' + - 'weed/cluster/**' + - 'test/samba/**' + - '.github/workflows/samba-integration.yml' + workflow_dispatch: + +concurrency: + group: samba-integration/${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + samba-integration: + name: samba-integration + runs-on: ubuntu-22.04 + timeout-minutes: 45 + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Start local Docker registry + run: docker run -d --restart=always -p 5000:5000 --name registry registry:2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + with: + driver-opts: network=host + + - name: Build weed race binary + run: | + cd docker + make binary_race + + - name: Build SeaweedFS e2e image + uses: docker/build-push-action@v7 + with: + context: docker + file: docker/Dockerfile.e2e + tags: localhost:5000/chrislusf/seaweedfs:e2e + push: true + cache-from: type=gha,scope=samba-e2e + cache-to: type=gha,mode=max,scope=samba-e2e + + - name: Tag e2e image for docker compose + run: | + docker pull localhost:5000/chrislusf/seaweedfs:e2e + docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e + + - name: Build samba image + uses: docker/build-push-action@v7 + with: + context: test/samba + build-contexts: | + chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e + tags: localhost:5000/chrislusf/seaweedfs:samba + push: true + cache-from: type=gha,scope=samba-harness + cache-to: type=gha,mode=max,scope=samba-harness + + - name: Tag samba image for docker compose + run: | + docker pull localhost:5000/chrislusf/seaweedfs:samba + docker tag localhost:5000/chrislusf/seaweedfs:samba chrislusf/seaweedfs:samba + + - name: Start SeaweedFS cluster and Samba + run: | + docker compose -f test/samba/docker-compose.yml up --wait + + - name: Run Samba test battery + run: | + set -o pipefail + docker compose -f test/samba/docker-compose.yml exec -T samba \ + /run_inside_container.sh 2>&1 | tee /tmp/samba-output.log + + - name: Collect logs + if: always() + run: | + mkdir -p /tmp/samba-docker-logs + for svc in master volume filer samba; do + docker compose -f test/samba/docker-compose.yml logs "$svc" \ + > "/tmp/samba-docker-logs/${svc}.log" 2>&1 || true + done + + - name: Tear down + if: always() + run: | + docker compose -f test/samba/docker-compose.yml down -v + + - name: Upload logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: samba-integration-results + path: | + /tmp/samba-output.log + /tmp/samba-docker-logs/ + retention-days: 7 diff --git a/.github/workflows/sftp-tests.yml b/.github/workflows/sftp-tests.yml new file mode 100644 index 0000000..377608d --- /dev/null +++ b/.github/workflows/sftp-tests.yml @@ -0,0 +1,93 @@ +name: "SFTP Integration Tests" + +on: + push: + branches: [ master, main ] + paths: + - 'weed/sftpd/**' + - 'weed/command/sftp.go' + - 'test/sftp/**' + - '.github/workflows/sftp-tests.yml' + pull_request: + branches: [ master, main ] + paths: + - 'weed/sftpd/**' + - 'weed/command/sftp.go' + - 'test/sftp/**' + - '.github/workflows/sftp-tests.yml' + +concurrency: + group: ${{ github.head_ref }}/sftp-tests + cancel-in-progress: true + +permissions: + contents: read + +env: + TEST_TIMEOUT: '15m' + +jobs: + sftp-integration: + name: SFTP Integration Testing + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y openssh-client + + - name: Build SeaweedFS + run: | + cd weed + go build -o weed . + chmod +x weed + ./weed version + + - name: Run SFTP Integration Tests + run: | + cd test/sftp + + echo "đŸ§Ē Running SFTP integration tests..." + echo "============================================" + + # Install test dependencies + go mod download + + # Run all SFTP tests + go test -v -timeout=${{ env.TEST_TIMEOUT }} ./... + + echo "============================================" + echo "✅ SFTP integration tests completed" + + - name: Test Summary + if: always() + run: | + echo "## 🔐 SFTP Integration Test Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY + echo "- ✅ **HomeDir Path Translation**: User home directory mapping (fixes #7470)" >> $GITHUB_STEP_SUMMARY + echo "- ✅ **File Operations**: Upload, download, delete" >> $GITHUB_STEP_SUMMARY + echo "- ✅ **Directory Operations**: Create, list, remove" >> $GITHUB_STEP_SUMMARY + echo "- ✅ **Large File Handling**: 1MB+ file support" >> $GITHUB_STEP_SUMMARY + echo "- ✅ **Path Edge Cases**: Unicode, trailing slashes, .. paths" >> $GITHUB_STEP_SUMMARY + echo "- ✅ **Admin Access**: Root user verification" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Test Configuration" >> $GITHUB_STEP_SUMMARY + echo "| User | HomeDir | Permissions |" >> $GITHUB_STEP_SUMMARY + echo "|------|---------|-------------|" >> $GITHUB_STEP_SUMMARY + echo "| admin | / | Full access |" >> $GITHUB_STEP_SUMMARY + echo "| testuser | /sftp/testuser | Home directory only |" >> $GITHUB_STEP_SUMMARY + echo "| readonly | /public | Read-only |" >> $GITHUB_STEP_SUMMARY + + + diff --git a/.github/workflows/spark-integration-tests.yml b/.github/workflows/spark-integration-tests.yml new file mode 100644 index 0000000..9a18898 --- /dev/null +++ b/.github/workflows/spark-integration-tests.yml @@ -0,0 +1,263 @@ +name: Spark Integration Tests + +on: + push: + paths: + - 'test/java/spark/**' + - 'other/java/hdfs3/**' + - 'other/java/client/**' + - '.github/workflows/spark-integration-tests.yml' + pull_request: + paths: + - 'test/java/spark/**' + - 'other/java/hdfs3/**' + - 'other/java/client/**' + - '.github/workflows/spark-integration-tests.yml' + workflow_dispatch: + +permissions: + contents: read + checks: write + pull-requests: write + +jobs: + spark-integration-tests: + name: Spark Integration Tests + runs-on: ubuntu-latest + timeout-minutes: 45 + + steps: + # ======================================== + # SETUP & BUILD + # ======================================== + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up JDK 11 + uses: actions/setup-java@v6 + with: + java-version: '11' + distribution: 'temurin' + cache: maven + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Build SeaweedFS binary + run: | + echo "Building SeaweedFS binary (statically linked for Alpine)..." + cd weed + CGO_ENABLED=0 go build -o ../docker/weed + cd ../docker + ls -la weed filer.toml entrypoint.sh + file weed + echo "OK SeaweedFS binary built" + + - name: Build SeaweedFS Java dependencies + run: | + echo "Building Java client..." + cd other/java/client + mvn clean install -U -DskipTests -Dgpg.skip=true -Dcentral.publishing.skip=true + echo "OK Java client built" + cd ../../.. + + echo "Building HDFS3 client..." + cd other/java/hdfs3 + mvn clean install -U -DskipTests -Dgpg.skip=true -Dcentral.publishing.skip=true + echo "OK HDFS3 client built" + echo "" + echo "All Java dependencies installed to ~/.m2/repository" + + # ======================================== + # SPARK INTEGRATION TESTS (DOCKER) + # ======================================== + - name: Start SeaweedFS services for tests + working-directory: test/java/spark + run: | + echo "=== Starting SeaweedFS Services for Tests ===" + docker compose down -v || true + docker compose build --no-cache + docker compose up -d seaweedfs-master seaweedfs-volume seaweedfs-filer + + echo "Waiting for services..." + for i in {1..30}; do + if curl -f http://localhost:8888/ > /dev/null 2>&1; then + echo "OK SeaweedFS filer is ready!" + break + fi + if [ $i -eq 30 ]; then + echo "FAILED Services failed to start" + docker compose ps -a + docker compose logs + exit 1 + fi + echo "Waiting... ($i/30)" + sleep 2 + done + + curl -f http://localhost:9333/cluster/status || exit 1 + echo "OK All services healthy" + + - name: Prepare Maven repository for Docker + working-directory: test/java/spark + run: | + echo "Copying Maven artifacts for Docker container..." + mkdir -p .m2/repository/com + cp -r ~/.m2/repository/com/seaweedfs .m2/repository/com/ + echo "OK Maven artifacts copied" + + - name: Run Spark integration tests + working-directory: test/java/spark + run: | + echo "=== Running Spark Integration Tests ===" + docker compose up --abort-on-container-exit spark-tests + + echo "" + echo "=== Test Logs ===" + docker compose logs spark-tests | tail -100 + + - name: Stop test services + if: always() + working-directory: test/java/spark + run: docker compose down -v + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v7 + with: + name: spark-test-results + path: test/java/spark/target/surefire-reports/ + retention-days: 30 + + - name: Publish test report + if: always() + uses: dorny/test-reporter@v3 + with: + name: Spark Test Results + path: test/java/spark/target/surefire-reports/*.xml + reporter: java-junit + fail-on-error: true + + # ======================================== + # SPARK EXAMPLE (HOST-BASED) - DISABLED + # Note: Host-based example doesn't work with Docker networking + # because master returns Docker hostnames (seaweedfs-volume) + # which are not resolvable from the host. + # The Docker-based tests above are sufficient. + # ======================================== + - name: Cache Apache Spark + if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + id: cache-spark + uses: actions/cache@v6 + with: + path: spark-3.5.0-bin-hadoop3 + key: spark-3.5.0-hadoop3 + + - name: Download Apache Spark + if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && steps.cache-spark.outputs.cache-hit != 'true' + run: | + echo "Downloading Apache Spark 3.5.0..." + wget -q https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz + tar xzf spark-3.5.0-bin-hadoop3.tgz + echo "OK Spark downloaded" + + - name: Start SeaweedFS services for example + if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + working-directory: test/java/spark + run: | + echo "=== Starting SeaweedFS Services for Example ===" + docker compose down -v || true + docker compose build --no-cache + docker compose up -d seaweedfs-master seaweedfs-volume seaweedfs-filer + + echo "Waiting for services..." + for i in {1..30}; do + if curl -f http://localhost:8888/ > /dev/null 2>&1; then + echo "OK SeaweedFS filer is ready!" + break + fi + if [ $i -eq 30 ]; then + echo "FAILED Services failed to start" + docker compose ps -a + docker compose logs + exit 1 + fi + echo "Waiting... ($i/30)" + sleep 2 + done + + curl -f http://localhost:9333/cluster/status || exit 1 + echo "OK All services healthy" + + - name: Clean target directory + if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + working-directory: test/java/spark + run: | + # Force remove target directory to avoid permission issues + sudo rm -rf target || rm -rf target || true + echo "OK Target directory cleaned" + + - name: Build project for example + if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + working-directory: test/java/spark + run: mvn package -DskipTests + + - name: Run Spark example application + if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + working-directory: test/java/spark + run: | + echo "=== Running Spark Example Application ===" + export SPARK_HOME=$(pwd)/../../../spark-3.5.0-bin-hadoop3 + $SPARK_HOME/bin/spark-submit \ + --class seaweed.spark.SparkSeaweedFSExample \ + --master local[2] \ + --conf spark.hadoop.fs.seaweedfs.impl=seaweed.hdfs.SeaweedFileSystem \ + --conf spark.hadoop.fs.seaweed.filer.host=localhost \ + --conf spark.hadoop.fs.seaweed.filer.port=8888 \ + --conf spark.hadoop.fs.seaweed.filer.port.grpc=18888 \ + --conf spark.hadoop.fs.seaweed.replication="" \ + target/seaweedfs-spark-integration-tests-1.0-SNAPSHOT.jar \ + seaweedfs://localhost:8888/ci-spark-output + echo "OK Example completed" + + - name: Verify example output + if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + run: | + echo "Verifying output..." + curl -s http://localhost:8888/ci-spark-output/ || echo "Output listing unavailable" + + - name: Stop example services + if: false && always() && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + working-directory: test/java/spark + run: docker compose down -v + + # ======================================== + # DIAGNOSTICS + # ======================================== + - name: Display diagnostics on failure + if: failure() + working-directory: test/java/spark + run: | + echo "=== Container Status ===" + docker compose ps -a + echo "" + echo "=== Master Logs ===" + docker compose logs seaweedfs-master + echo "" + echo "=== Volume Logs ===" + docker compose logs seaweedfs-volume + echo "" + echo "=== Filer Logs ===" + docker compose logs seaweedfs-filer + echo "" + echo "=== Volume List ===" + docker compose exec -T seaweedfs-master weed shell < weed.log 2>&1 & + until curl -s http://localhost:8333/healthz > /dev/null; do sleep 1; done + + - name: Setup Caddy + run: | + curl -fsSL "https://caddyserver.com/api/download?os=linux&arch=amd64" -o caddy + chmod +x caddy + ./caddy version + echo "{ + auto_https disable_redirects + local_certs + } + localhost:8443 { + tls internal + reverse_proxy localhost:8333 + }" > Caddyfile + + - name: Start Caddy + run: | + ./caddy start + until curl -fsS --insecure https://localhost:8443/healthz > /dev/null; do sleep 1; done + + - name: Create Bucket + run: | + aws --no-verify-ssl s3api create-bucket --bucket bucket + + - name: Test PutObject + run: | + set -e + dd if=/dev/urandom of=generated bs=1M count=2 + aws --no-verify-ssl s3api put-object --bucket bucket --key test-putobject --body generated + aws --no-verify-ssl s3api get-object --bucket bucket --key test-putobject downloaded + diff -q generated downloaded + rm -f generated downloaded + + - name: Test Multi-part Upload + run: | + set -e + dd if=/dev/urandom of=generated bs=1M count=32 + aws --no-verify-ssl s3 cp --no-progress generated s3://bucket/test-multipart + aws --no-verify-ssl s3 cp --no-progress s3://bucket/test-multipart downloaded + diff -q generated downloaded + rm -f generated downloaded + + - name: Test GetObject with If-Match + run: | + set -e + dd if=/dev/urandom of=generated bs=1M count=32 + ETAG=$(aws --no-verify-ssl s3api put-object --bucket bucket --key test-get-obj --body generated | jq -r .ETag) + # jq -r already removes quotes, so use ETAG directly (handles both simple and multipart ETags) + aws --no-verify-ssl s3api get-object --bucket bucket --key test-get-obj --if-match "$ETAG" downloaded + diff -q generated downloaded + rm -f generated downloaded + + - name: Show server logs on failure + if: failure() + run: | + echo "=========================================" + echo "SeaweedFS Server Logs" + echo "=========================================" + # Note: weed.log is relative to working-directory (weed/) + if [ -f weed.log ]; then + cat weed.log + else + echo "No weed.log file found" + fi + + - name: Upload server logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: seaweedfs-logs + # Note: actions don't use defaults.run.working-directory, so path is relative to workspace root + path: weed/weed.log + retention-days: 3 diff --git a/.github/workflows/tls-rotation-tests.yml b/.github/workflows/tls-rotation-tests.yml new file mode 100644 index 0000000..ce4ecc9 --- /dev/null +++ b/.github/workflows/tls-rotation-tests.yml @@ -0,0 +1,68 @@ +name: "TLS Rotation Integration Tests" + +on: + push: + branches: [ master ] + paths: + - 'weed/**' + - 'test/tls_rotation/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/tls-rotation-tests.yml' + pull_request: + branches: [ master ] + paths: + - 'weed/**' + - 'test/tls_rotation/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/tls-rotation-tests.yml' + +permissions: + contents: read + +jobs: + tls-rotation-tests: + name: TLS Rotation Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version: ^1.26 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + + - name: Build weed binary + run: | + cd weed && go build -o weed . + + - name: Run TLS Rotation Integration Tests + working-directory: test/tls_rotation + run: | + go test -v -count=1 -timeout 5m + + - name: Collect server logs on failure + if: failure() + run: | + echo "Collecting master logs from temp directories..." + mkdir -p /tmp/tls-rotation-test-logs + find /tmp -maxdepth 1 -type d -name "TestMasterHTTPS*" 2>/dev/null | while read dir; do + if [ -d "$dir" ]; then + echo "Found test directory: $dir" + cp -r "$dir" /tmp/tls-rotation-test-logs/ 2>/dev/null || true + fi + done + echo "Collected logs:" + find /tmp/tls-rotation-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found" + + - name: Archive logs + if: failure() + uses: actions/upload-artifact@v7 + with: + name: tls-rotation-test-logs + path: /tmp/tls-rotation-test-logs/ + retention-days: 14 diff --git a/.github/workflows/tus-tests.yml b/.github/workflows/tus-tests.yml new file mode 100644 index 0000000..9e7f4e1 --- /dev/null +++ b/.github/workflows/tus-tests.yml @@ -0,0 +1,110 @@ +name: "TUS Protocol Tests" + +on: + pull_request: + paths: + - 'weed/server/**' + - 'weed/filer/**' + - 'test/tus/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/tus-tests.yml' + +concurrency: + group: ${{ github.head_ref || github.ref }}/tus-tests + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + working-directory: weed + +jobs: + tus-integration-tests: + name: TUS Protocol Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run TUS Integration Tests + timeout-minutes: 15 + working-directory: test/tus + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + df -h + echo "=== Starting TUS Tests ===" + + # Run tests with automatic server management + make test-with-server || { + echo "TUS integration tests failed, checking logs..." + if [ -f /tmp/seaweedfs-tus-filer.log ]; then + echo "=== Filer logs ===" + tail -100 /tmp/seaweedfs-tus-filer.log + fi + if [ -f /tmp/seaweedfs-tus-master.log ]; then + echo "=== Master logs ===" + tail -50 /tmp/seaweedfs-tus-master.log + fi + if [ -f /tmp/seaweedfs-tus-volume.log ]; then + echo "=== Volume logs ===" + tail -50 /tmp/seaweedfs-tus-volume.log + fi + exit 1 + } + + - name: Show server logs on failure + if: failure() + working-directory: test/tus + run: | + echo "=== Filer Server Logs ===" + if [ -f /tmp/seaweedfs-tus-filer.log ]; then + echo "Last 100 lines of filer logs:" + tail -100 /tmp/seaweedfs-tus-filer.log + else + echo "No filer log file found" + fi + + echo "=== Master Server Logs ===" + if [ -f /tmp/seaweedfs-tus-master.log ]; then + tail -50 /tmp/seaweedfs-tus-master.log + else + echo "No master log file found" + fi + + echo "=== Volume Server Logs ===" + if [ -f /tmp/seaweedfs-tus-volume.log ]; then + tail -50 /tmp/seaweedfs-tus-volume.log + else + echo "No volume log file found" + fi + + echo "=== Test Environment ===" + ps aux | grep -E "(weed|test)" || true + netstat -tlnp 2>/dev/null | grep -E "(18888|19333|18080)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: tus-test-logs + path: | + /tmp/seaweedfs-tus-*.log + retention-days: 3 diff --git a/.github/workflows/vacuum-integration-tests.yml b/.github/workflows/vacuum-integration-tests.yml new file mode 100644 index 0000000..e9f2a4c --- /dev/null +++ b/.github/workflows/vacuum-integration-tests.yml @@ -0,0 +1,68 @@ +name: "Vacuum Integration Tests" + +on: + push: + branches: [ master ] + paths: + - 'weed/**' + - 'test/vacuum/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/vacuum-integration-tests.yml' + pull_request: + branches: [ master ] + paths: + - 'weed/**' + - 'test/vacuum/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/vacuum-integration-tests.yml' + +permissions: + contents: read + +jobs: + vacuum-integration-tests: + name: Vacuum Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 15 + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v7 + with: + go-version: ^1.26 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v7 + + - name: Build weed binary + run: | + cd weed && go build -o weed . + + - name: Run Vacuum Integration Tests + working-directory: test/vacuum + run: | + go test -v -timeout 10m + + - name: Collect server logs on failure + if: failure() + run: | + echo "Collecting server logs from temp directories..." + mkdir -p /tmp/vacuum-test-logs + find /tmp -maxdepth 1 -type d -name "TestVacuum*" 2>/dev/null | while read dir; do + if [ -d "$dir" ]; then + echo "Found test directory: $dir" + cp -r "$dir" /tmp/vacuum-test-logs/ 2>/dev/null || true + fi + done + echo "Collected logs:" + find /tmp/vacuum-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found" + + - name: Archive logs + if: failure() + uses: actions/upload-artifact@v7 + with: + name: vacuum-integration-test-logs + path: /tmp/vacuum-test-logs/ + retention-days: 14 diff --git a/.github/workflows/volume-server-integration-tests.yml b/.github/workflows/volume-server-integration-tests.yml new file mode 100644 index 0000000..c1874fa --- /dev/null +++ b/.github/workflows/volume-server-integration-tests.yml @@ -0,0 +1,104 @@ +name: "Volume Server Integration Tests" + +on: + pull_request: + branches: [ master ] + paths: + - 'test/volume_server/**' + - 'weed/server/**' + - 'weed/storage/**' + - 'weed/pb/volume_server.proto' + - 'weed/pb/volume_server_pb/**' + - '.github/workflows/volume-server-integration-tests.yml' + push: + branches: [ master, main ] + paths: + - 'test/volume_server/**' + - 'weed/server/**' + - 'weed/storage/**' + - 'weed/pb/volume_server.proto' + - 'weed/pb/volume_server_pb/**' + - '.github/workflows/volume-server-integration-tests.yml' + +concurrency: + group: ${{ github.head_ref || github.ref }}/volume-server-integration-tests + cancel-in-progress: true + +permissions: + contents: read + +env: + TEST_TIMEOUT: '30m' + # Keep in step with the length of matrix.shard below. + SHARD_COUNT: 3 + +jobs: + volume-server-integration-tests: + name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }}) + runs-on: ubuntu-22.04 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + test-type: [grpc, http] + shard: [1, 2, 3] + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Build SeaweedFS binary + run: | + cd weed + go build -o weed . + chmod +x weed + ./weed version + + # Dealing the listed tests out one by one keeps the shards even. Bucketing + # them by first letter did not: names cluster, so ^Test[I-S] drew 50 of + # the 114 grpc tests and ran nearly twice as long as the other two shards. + - name: Select this shard's tests + env: + TEST_TYPE: ${{ matrix.test-type }} + SHARD: ${{ matrix.shard }} + run: | + tests=$(go test ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u) + # An empty list would make -run match nothing and the shard pass vacuously. + [ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; } + selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1') + echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests" + echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV" + + - name: Run volume server integration tests + env: + WEED_BINARY: ${{ github.workspace }}/weed/weed + run: | + echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..." + go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}" + + - name: Collect logs on failure + if: failure() + run: | + mkdir -p /tmp/volume-server-it-logs + find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/volume-server-it-logs/ \; || true + + - name: Archive logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: volume-server-integration-test-logs + path: /tmp/volume-server-it-logs/ + if-no-files-found: warn + retention-days: 7 + + - name: Test summary + if: always() + run: | + echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY" + echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY" + echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"\${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9a3bf80 --- /dev/null +++ b/.gitignore @@ -0,0 +1,146 @@ +.goxc* +vendor +tags +*.swp +.claude/ +### OSX template +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio + +*.iml + +## Directory-based project format: +.idea/ +# if you remove the above rule, at least ignore the following: + +# User-specific stuff: +# .idea/workspace.xml +# .idea/tasks.xml +# .idea/dictionaries + +# Sensitive or high-churn files: +# .idea/dataSources.ids +# .idea/dataSources.xml +# .idea/sqlDataSources.xml +# .idea/dynamic.xml +# .idea/uiDesigner.xml + +# Gradle: +# .idea/gradle.xml +# .idea/libraries + +# Mongo Explorer plugin: +# .idea/mongoSettings.xml + +## vscode +.vscode +## File-based project format: +*.ipr +*.iws + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties + +workspace/ + +test_data +build +target +*.class +other/java/hdfs/dependency-reduced-pom.xml + +# binary file +weed/weed +docker/weed + +# test generated files +weed/*/*.jpg +docker/weed_sub +docker/weed_pub +weed/mq/schema/example.parquet +docker/agent_sub_record +test/mq/bin/consumer +test/mq/bin/producer +test/producer +bin/weed +weed_binary +/test/s3/copying/filerldb2 +/filerldb2 +/test/s3/retention/test-volume-data +test/s3/cors/weed-test.log +test/s3/cors/weed-server.pid +/test/s3/cors/test-volume-data +test/s3/cors/cors.test +/test/s3/retention/filerldb2 +test/s3/retention/weed-server.pid +test/s3/retention/weed-test.log +/test/s3/versioning/test-volume-data +test/s3/versioning/weed-test.log +/docker/admin_integration/data +docker/agent_pub_record +docker/admin_integration/weed-local +/test/s3/encryption/filerldb2 +/test/s3/sse/filerldb2 +test/s3/sse/weed-test.log +ADVANCED_IAM_DEVELOPMENT_PLAN.md +/test/s3/iam/test-volume-data +*.log +weed-iam +test/kafka/kafka-client-loadtest/weed-linux-arm64 +/test/tus/filerldb2 +coverage.out +/test/s3/remote_cache/test-primary-data +/test/s3/remote_cache/test-remote-data +test/s3/remote_cache/remote-server.pid +test/s3/remote_cache/primary-server.pid +/test/erasure_coding/filerldb2 +/test/s3/cors/test-mini-data +/test/s3/filer_group/test-volume-data + +# ID and PID files +*.id +*.pid +test/s3/iam/.test_env +/test/erasure_coding/admin_dockertest/tmp +/test/erasure_coding/admin_dockertest/task_logs +weed_bin +telemetry/server/telemetry-server +.aider* +/seaweed-volume/docs diff --git a/.superset/config.json b/.superset/config.json new file mode 100644 index 0000000..f806b52 --- /dev/null +++ b/.superset/config.json @@ -0,0 +1,5 @@ +{ + "setup": [], + "teardown": [], + "run": [] +} diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..c561b2f --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,74 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to make participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual attention or + advances +- Trolling, insulting/derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or electronic + address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at . All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c9c58bb --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 Chris Lu + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..71fd05f --- /dev/null +++ b/Makefile @@ -0,0 +1,73 @@ +.PHONY: test admin-generate admin-build admin-clean admin-dev admin-run admin-test admin-fmt admin-help weed-commands + +BINARY = weed +ADMIN_DIR = weed/admin + +SOURCE_DIR = . +debug ?= 0 + +all: install + +install: admin-generate + cd weed; go install + +weed-commands: + cd weed && $(MAKE) weed-db weed-sql + +warp_install: + go install github.com/minio/warp@v0.7.6 + +full_install: admin-generate + cd weed; go install -tags "elastic gocdk sqlite ydb tarantool tikv rclone" + +server: install + weed -v 0 server -s3 -filer -filer.maxMB=64 -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowDeleteBucketNotEmpty=true -s3.config=./docker/compose/s3.json -metricsPort=9324 + +benchmark: install warp_install + pkill weed || true + pkill warp || true + weed server -debug=$(debug) -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json & + warp client & + while ! nc -z localhost 8000 ; do sleep 1 ; done + warp mixed --host=127.0.0.1:8000 --access-key=some_access_key1 --secret-key=some_secret_key1 --autoterm + pkill warp + pkill weed + +# curl -o profile "http://127.0.0.1:6060/debug/pprof/profile?debug=1" +benchmark_with_pprof: debug = 1 +benchmark_with_pprof: benchmark + +test: admin-generate + cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./... + +# Admin component targets +admin-generate: + @cd $(ADMIN_DIR) && $(MAKE) generate + +admin-build: admin-generate + @echo "Building admin component..." + @cd $(ADMIN_DIR) && $(MAKE) build + +admin-clean: + @echo "Cleaning admin component..." + @cd $(ADMIN_DIR) && $(MAKE) clean + +admin-dev: + @echo "Starting admin development server..." + @cd $(ADMIN_DIR) && $(MAKE) dev + +admin-run: + @echo "Running admin server..." + @cd $(ADMIN_DIR) && $(MAKE) run + +admin-test: + @echo "Testing admin component..." + @cd $(ADMIN_DIR) && $(MAKE) test + +admin-fmt: + @echo "Formatting admin component..." + @cd $(ADMIN_DIR) && $(MAKE) fmt + +admin-help: + @echo "Admin component help..." + @cd $(ADMIN_DIR) && $(MAKE) help diff --git a/README.md b/README.md new file mode 100644 index 0000000..82ae132 --- /dev/null +++ b/README.md @@ -0,0 +1,515 @@ +# SeaweedFS + + +[![Slack](https://img.shields.io/badge/slack-purple)](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY) +[![Twitter](https://img.shields.io/twitter/follow/seaweedfs.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=seaweedfs) +[![Build Status](https://img.shields.io/github/actions/workflow/status/seaweedfs/seaweedfs/go.yml)](https://github.com/seaweedfs/seaweedfs/actions/workflows/go.yml) +[![GoDoc](https://godoc.org/github.com/seaweedfs/seaweedfs/weed?status.svg)](https://godoc.org/github.com/seaweedfs/seaweedfs/weed) +[![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/seaweedfs/seaweedfs/wiki) +[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/) +[![SeaweedFS on Maven Central](https://img.shields.io/maven-central/v/com.github.chrislusf/seaweedfs-client)](https://search.maven.org/search?q=g:com.github.chrislusf) +[![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/seaweedfs)](https://artifacthub.io/packages/search?repo=seaweedfs) + +![SeaweedFS Logo](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/seaweedfs.png) + +SeaweedFS is a simple and highly scalable distributed file system. There are two objectives: + +1. to store billions of files! +2. to serve the files fast! + +One `weed` binary serves an S3 object store, a POSIX file system, and a lakehouse with S3 Tables, all over the same data. Each blob is one disk read away, capacity grows by starting another volume server, and cloud storage can be cached or tiered transparently. + +- [Download Binaries for different platforms](https://github.com/seaweedfs/seaweedfs/releases/latest) +- [Wiki Documentation](https://github.com/seaweedfs/seaweedfs/wiki) +- Community: [Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY), [Twitter](https://twitter.com/SeaweedFS), [Telegram](https://t.me/Seaweedfs), [Reddit](https://www.reddit.com/r/SeaweedFS/), [Mailing List](https://groups.google.com/d/forum/seaweedfs) +- [SeaweedFS White Paper](https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS_Architecture.pdf) and introduction slides: [2025.5](https://docs.google.com/presentation/d/1tdkp45J01oRV68dIm4yoTXKJDof-EhainlA0LMXexQE/edit?usp=sharing), [2021.5](https://docs.google.com/presentation/d/1DcxKWlINc-HNCjhYeERkpGXXm6nTCES8mi2W5G0Z4Ts/edit?usp=sharing), [2019.3](https://www.slideshare.net/chrislusf/seaweedfs-introduction) + +Table of Contents +================= + +* [Quick Start](#quick-start) + * [One command](#one-command) + * [Docker](#docker) + * [Docker Compose](#docker-compose) + * [Kubernetes with Helm](#kubernetes-with-helm) + * [Build from source](#build-from-source) + * [Scale out](#scale-out) +* [Why SeaweedFS](#why-seaweedfs) + * [Fast](#fast) + * [Scalable](#scalable) + * [The most complete S3 API](#the-most-complete-s3-api) + * [A data warehouse with S3 Tables](#a-data-warehouse-with-s3-tables) + * [A fast cache for cloud storage](#a-fast-cache-for-cloud-storage) + * [Active-active replication and more](#active-active-replication-and-more) +* [Architecture](#architecture) +* [Compared to Other Systems](#compared-to-other-systems) + * [Compared to HDFS](#compared-to-hdfs) + * [Compared to GlusterFS, Ceph](#compared-to-glusterfs-ceph) + * [Compared to MooseFS](#compared-to-moosefs) + * [Compared to Ceph](#compared-to-ceph) + * [Compared to MinIO, RustFS](#compared-to-minio-rustfs) +* [Benchmark](#benchmark) +* [Enterprise](#enterprise) +* [License](#license) +* [Sponsors](#sponsors) + +# Quick Start # + +## One command ## + +Download the latest binary from the [releases](https://github.com/seaweedfs/seaweedfs/releases/latest) page and unzip the single `weed` (or `weed.exe`) file, or let the install script put it in `/usr/local/bin`: + +```bash +curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash +``` + +Then start a ready-to-use S3 object store: + +```bash +AWS_ACCESS_KEY_ID=admin \ +AWS_SECRET_ACCESS_KEY=secret \ +S3_BUCKET=my-bucket \ +./weed mini -dir=./data +``` + +That's it. The S3 endpoint is at http://localhost:8333, `my-bucket` exists, and `admin`/`secret` are valid credentials: + +```bash +AWS_ACCESS_KEY_ID=admin AWS_SECRET_ACCESS_KEY=secret \ + aws --endpoint-url http://localhost:8333 s3 cp README.md s3://my-bucket/ +``` + +The same process also runs the master, a volume server, the filer, WebDAV, the Iceberg REST catalog, and the Admin UI. Add `S3_TABLE_BUCKET=warehouse` to also create an Iceberg table bucket, or `warehouse:LANCE` for a Lance one. Drop the AWS keys to run without authentication for development. + +> macOS: if the binary is quarantined, run `xattr -d com.apple.quarantine ./weed` first. + +`weed mini` is auto-tuned for one node and is fine for single-node production, such as an S3 gateway that issues presigned URLs. See [Quick Start with weed mini][WeedMini]. + +## Docker ## + +```bash +docker run -p 8333:8333 -v weed-data:/data \ + -e AWS_ACCESS_KEY_ID=admin \ + -e AWS_SECRET_ACCESS_KEY=secret \ + -e S3_BUCKET=my-bucket \ + chrislusf/seaweedfs +``` + +Same behavior as the `weed mini` command above. + +## Docker Compose ## + +To run master, volume server, filer, S3, and WebDAV as separate services: + +```bash +wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-compose.yml +wget -P prometheus https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/prometheus/prometheus.yml +docker compose -f seaweedfs-compose.yml -p seaweedfs up +``` + +[Docker Compose for S3][DockerComposeS3] adds credentials, and the [docker/compose](docker/compose) folder has variants for replication, mounts, message queues, and more. + +## Kubernetes with Helm ## + +```bash +helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm +helm install seaweedfs seaweedfs/seaweedfs -n seaweedfs --create-namespace -f values.yaml +``` + +A production-shaped `values.yaml` for a three-node cluster: two copies of every write, three masters, and an S3 endpoint with credentials and a bucket. + +```yaml +global: + seaweedfs: + enableReplication: true + replicationPlacement: "001" # one extra copy on another server; "002" for two + +master: + replicas: 3 + data: + type: persistentVolumeClaim # the cluster's default storage class; add storageClass to pick one + size: 1Gi + +volume: + replicas: 3 # at least 1 + the sum of the replication digits + dataDirs: + - name: data + type: persistentVolumeClaim + size: 500Gi + maxVolumes: 0 # size the volume count from the disk + +filer: + replicas: 2 + data: + type: persistentVolumeClaim + size: 20Gi + +s3: + enabled: true + replicas: 2 + enableAuth: true + credentials: + admin: + accessKey: admin + secretKey: change-me + createBuckets: + - name: app-storage +``` + +The S3 endpoint is the `seaweedfs-s3` service on port 8333. [Helm Chart Recipes][HelmRecipes] has values for a development cluster, a lakehouse with the Iceberg catalog exposed, filer metadata on PostgreSQL, and node-local disks. The [SeaweedFS Operator][Operator] and the [CSI driver][SeaweedFsCsiDriver] are the other Kubernetes paths. + +## Build from source ## + +```bash +git clone https://github.com/seaweedfs/seaweedfs.git +cd seaweedfs/weed && make install +``` + +`weed` lands in `$GOPATH/bin`. [Getting Started][GettingStarted] covers running master, volume, filer, and S3 as separate processes. + +## Scale out ## + +Capacity is a volume server. Start one on any machine with disk and point it at the master: + +```bash +weed volume -dir=/data -master=:9333 +``` + +Nothing rebalances until you ask it to. Throughput is a filer or S3 gateway; they are stateless, so run as many as you need behind a load balancer. [Production Setup][ProductionSetup] walks through a multi-node cluster. + +[Back to TOC](#table-of-contents) + +# Why SeaweedFS # + +## Fast ## + +* One disk read per blob. A small file is one blob; a large file is split into chunks of a few MB, each its own blob. A volume server keeps a 16-byte index entry per blob in memory and reads it in a single seek, also for erasure-coded data. +* The master is not in the read path. Clients cache the volume-to-server mapping and talk to volume servers directly. +* 40 bytes of metadata per file on disk. Small files are packed into append-only volume files, so there is no per-file inode, no per-file metadata file, no fragmentation, and writes are SSD friendly. +* Hot data is replicated; [erasure coding][ErasureCoding] is applied to warm data in the background, so writes never pay the encoding cost. +* The [Rust volume server][RustVolume] is a drop-in for higher throughput and lower tail latency on the same on-disk format. + +On one laptop, [`weed benchmark`][Benchmarks] writes 1KB files at 15,700 per second and reads them back at 47,000 per second, and a mixed S3 [warp][S3Benchmark] run totals 3.2 GiB/s. Numbers are in the [Benchmark](#benchmark) section; throughput grows with volume servers and gateways. + +## Scalable ## + +* The master tracks volumes, not files. A cluster with billions of files has a few thousand volumes, so the master stays small. One master is enough for most clusters; run three for [Raft failover][FailoverMaster]. +* Adding a server adds capacity with no data reshuffle. Balancing, vacuum, erasure coding, and repair run on demand from [`weed shell`][WeedShell] or the [maintenance worker][Worker]. +* Filer and S3 gateways are stateless and scale linearly. Directory metadata lives in a [store you already run][FilerStores]: LevelDB, RocksDB, SQLite, MySQL, PostgreSQL, Cassandra, HBase, MongoDB, Redis, Elasticsearch, etcd, TiKV, FoundationDB, YDB, ArangoDB, Tarantool, and MySQL or PostgreSQL compatible databases such as TiDB, CockroachDB, and MemSQL. +* Rack and data center aware [replication][Replication], [tiered storage][TieredStorage] across disk types, and [transparent cloud tiering][CloudTier] for unlimited capacity. +* Files from a byte to [tens of TB][SuperLargeFiles]. Volumes up to 8TB with the large-disk build. + +## The most complete S3 API ## + +The S3 gateway implements the object, bucket, S3 Tables, IAM, and STS APIs on one endpoint, so the AWS SDKs and CLI, rclone, restic, Spark, and Trino work unchanged. + +| API | Operations | +| --- | --- | +| S3 bucket and object | 73 | +| S3 Tables | 36 | +| IAM | 39 | +| STS | 5 | + +* [Versioning][Versioning], [Object Lock][ObjectLock] with retention and legal hold, [lifecycle][Lifecycle] rules, tagging, [CORS][CORS], [conditional reads and writes][ConditionalOps], checksums, presigned URLs, browser POST uploads, multipart uploads, and an atomic [RenameObject][RenameObject]. +* [Bucket policies][BucketPolicies] with [conditions][PolicyConditions] and [variables][PolicyVariables]; IAM users, groups, and policies; STS with [OIDC][OIDC], LDAP, and [Kubernetes service accounts][K8sSA]. +* [SSE-S3, SSE-KMS, and SSE-C][SSE] server-side encryption, with OpenBao and Vault, AWS KMS, Azure Key Vault, and GCP KMS as key providers. +* [Audit log][AuditLog], [bucket quota][BucketQuota], and [rate limiting][RateLimiting]. +* Each bucket is its own collection, so deleting a bucket is instant. + +The full operation list is in [Amazon S3 API][AmazonS3API], and [Supported APIs vs MinIO][S3vsMinio] compares. The S3 compatibility suite and the SDK, IAM, SSE, policy, and Spark integration tests run in CI on every change. + +## A data warehouse with S3 Tables ## + +SeaweedFS is a lakehouse in one system. [S3 Table Buckets][S3TableBucket] hold Apache Iceberg tables by default, or [Lance][LanceCatalog] tables for vectors and multimodal data, and the built-in [Iceberg REST Catalog][IcebergCatalog] and Lance namespace serve them directly. There is no Hive Metastore, Glue, or separate catalog service to deploy, secure, and back up. + +* Query engines operate on the same tables at the same time: [Spark][SparkIceberg], [Trino][TrinoIceberg], [Dremio][DremioIceberg], [DuckDB][DuckDBIceberg], [Apache Doris][DorisIceberg], [RisingWave][RisingWaveIceberg], ClickHouse, and [LanceDB][LanceDB]. Catalog commits are atomic compare-and-swap, so concurrent writers are safe. [Lakekeeper][Lakekeeper] can front the same storage with STS-vended credentials. +* [Automated table maintenance][IcebergMaintenance]: compaction, snapshot expiration, orphan file removal, and manifest rewriting, configured per bucket or table through the S3 Tables maintenance APIs, and the same for [Lance][LanceMaintenance]. +* IAM at the bucket, namespace, and table level with standard bucket policies, see [S3 Tables Security][S3TablesSecurity]. +* A [Hadoop compatible file system][Hadoop] for Spark, Flink, and HBase. + +`S3_TABLE_BUCKET=warehouse ./weed mini -dir=./data` brings the whole stack up on a laptop. + +## A fast cache for cloud storage ## + +[Cloud Drive][CloudDrive] mounts a bucket from S3, Google Cloud Storage, Azure, Backblaze B2, Wasabi, Storj, or any S3-compatible store into SeaweedFS and serves it at local speed: + +* Metadata is pulled once, so listing, stat, and directory walks cost no cloud API calls. +* File content is downloaded once, on first read or [warmed][CacheRemote] by folder, name pattern, size, or age, and cached with the capacity of the whole cluster: cache everything, no churn. +* Local writes complete at local latency and are written back to the cloud asynchronously in the cloud's native layout, so other tools keep reading the bucket directly. +* Uncache by the same rules to free local disk while keeping the metadata. + +[Cloud Tier][CloudTier] goes the other direction, moving whole warm volumes to cloud storage while keeping one-read access, and the [Gateway to Remote Object Storage][GatewayToRemoteObjectStore] mirrors every bucket to a remote store. Faster and cheaper than reading the cloud directly. + +## Active-active replication and more ## + +* [Active-active or active-passive replication][ActiveActiveAsyncReplication] between clusters, continuous and resumable, for the whole tree or chosen folders, across data centers. +* [Filer store replication][FilerStoreReplication] for metadata HA, [async backup][AsyncBackup] to cloud storage, [metadata backup][MetaBackup], and [change data capture][CDC] with [webhooks][Webhook] on every metadata event. +* The same data as a [FUSE mount][Mount] on Linux, macOS, and [Windows][MountWindows], over [WebDAV][WebDAV], [SFTP][SFTP], HDFS, HTTP, and [TUS resumable uploads][TUS]; on Kubernetes through the [CSI driver][SeaweedFsCsiDriver] and [Operator][Operator]. +* [AES256-GCM encryption at rest][FilerDataEncryption], TLS and mTLS between components, JWT-signed volume access, and [FIPS][FIPS] builds. +* [Admin UI][AdminUI], Prometheus [metrics][Metrics], [TTL][VolumeServerTTL] per file or volume, automatic compression and compaction, and [seaweed-up][SeaweedUp] for bare-metal clusters. + +[Back to TOC](#table-of-contents) + +# Architecture # + +![SeaweedFS Architecture](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/SeaweedFS_Architecture.png) + +* **Master** servers, one or a Raft group of three, track which volume lives on which volume server and hand out file ids. They are not in the read path. +* **Volume** servers store blobs in append-only volume files, keep a 16-byte in-memory index per blob, and replicate or erasure-code at the volume level. +* **Filer** servers add directories and files on top, with metadata in a store of your choice, and expose HTTP, S3, WebDAV, SFTP, FUSE, and the table catalogs. + +The blob store started from [Facebook's Haystack](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf), erasure coding takes ideas from [f4](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf), and the whole has a lot in common with [Tectonic](https://www.usenix.org/system/files/fast21-pan.pdf) and [Colossus](https://cloud.google.com/blog/products/storage-data-transfer/a-peek-behind-colossus-googles-file-system). How file ids are assigned, written, and looked up, and why a master that tracks volumes scales, is in [Blob Store Architecture][BlobStoreArchitecture]; the services are in [Components][Components] and the [white paper][WhitePaper]. + +[Back to TOC](#table-of-contents) + +# Compared to Other Systems # + +Most other distributed file systems seem more complicated than necessary. + +SeaweedFS is meant to be fast and simple, in both setup and operation. If you do not understand how it works when you reach here, we've failed! Please raise an issue with any questions or update this file with clarifications. + +SeaweedFS is constantly moving forward. Same with other systems. These comparisons can be outdated quickly. Please help to keep them updated. + +## Compared to HDFS ## + +HDFS uses the chunk approach for each file, and is ideal for storing large files. + +SeaweedFS is ideal for serving relatively smaller files quickly and concurrently. + +SeaweedFS can also store extra large files by splitting them into manageable data chunks, and store the file ids of the data chunks into a meta chunk. This is managed by "weed upload/download" tool, and the weed master or volume servers are agnostic about it. + +## Compared to GlusterFS, Ceph ## + +The architectures are mostly the same. SeaweedFS aims to store and read files fast, with a simple and flat architecture. The main differences are + +* SeaweedFS optimizes for small files, ensuring O(1) disk seek operation, and can also handle large files. +* SeaweedFS statically assigns a volume id for a file. Locating file content becomes just a lookup of the volume id, which can be easily cached. +* SeaweedFS Filer metadata store can be any well-known and proven data store, e.g., Redis, Cassandra, HBase, Mongodb, Elastic Search, MySql, Postgres, Sqlite, MemSql, TiDB, CockroachDB, Etcd, YDB etc, and is easy to customize. +* SeaweedFS Volume server also communicates directly with clients via HTTP, supporting range queries, direct uploads, etc. + +| System | File Metadata | File Content Read| POSIX | REST API | Optimized for large number of small files | +| ------------- | ------------------------------- | ---------------- | ------ | -------- | ------------------------- | +| SeaweedFS | lookup volume id, cacheable | O(1) disk seek | | Yes | Yes | +| SeaweedFS Filer| Linearly Scalable, Customizable | O(1) disk seek | FUSE | Yes | Yes | +| GlusterFS | hashing | | FUSE, NFS | | | +| Ceph | hashing + rules | | FUSE | Yes | | +| MooseFS | in memory | | FUSE | | No | +| MinIO | separate meta file per drive for each file | | | Yes | No | +| RustFS | separate meta file per drive for each file | | | Yes | No | + +GlusterFS stores files, both directories and content, in configurable volumes called "bricks". It hashes the path and filename into ids, and assigned to virtual volumes, and then mapped to "bricks". + +## Compared to MooseFS ## + +MooseFS chooses to neglect small file issue. From moosefs 3.0 manual, "even a small file will occupy 64KiB plus additionally 4KiB of checksums and 1KiB for the header", because it "was initially designed for keeping large amounts (like several thousands) of very big files" + +MooseFS Master Server keeps all meta data in memory. Same issue as HDFS namenode. + +## Compared to Ceph ## + +Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more complicated, with the need to support layers on top of it. [Here is a more detailed comparison](https://github.com/seaweedfs/seaweedfs/issues/120) + +SeaweedFS has a centralized master group to look up free volumes, while Ceph uses hashing and metadata servers to locate its objects. Having a centralized master makes it easy to code and manage. + +Ceph, like SeaweedFS, is based on the object store RADOS. Ceph is rather complicated with mixed reviews. + +Ceph uses CRUSH hashing to automatically manage data placement, which is efficient to locate the data. But the data has to be placed according to the CRUSH algorithm. Any wrong configuration would cause data loss. Topology changes, such as adding new servers to increase capacity, will cause data migration with high IO cost to fit the CRUSH algorithm. SeaweedFS places data by assigning them to any writable volumes. If writes to one volume failed, just pick another volume to write. Adding more volumes is also as simple as it can be. + +SeaweedFS is optimized for small files. Small files are stored as one continuous block of content, with at most 8 unused bytes between files. Small file access is O(1) disk read. + +SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Sqlite, Mongodb, Redis, Elastic Search, Cassandra, HBase, MemSql, TiDB, CockroachCB, Etcd, YDB, to manage file directories. These stores are proven, scalable, and easier to manage. + +| SeaweedFS | comparable to Ceph | advantage | +| ------------- | ------------- | ---------------- | +| Master | MDS | simpler | +| Volume | OSD | optimized for small files | +| Filer | Ceph FS | linearly scalable, Customizable, O(1) or O(logN) | + +## Compared to MinIO, RustFS ## + +Please note, as Apr 25, 2026 MinIO ceased development. It's strongly discouraged to use that unmaintained software with multiple security bugs. RustFS is a MinIO reimplementation in Rust, Apache 2.0 licensed and still developed, keeping MinIO's storage model down to a byte-compatible on-disk format. So the points below apply to both. + +MinIO followed AWS S3 closely and was ideal for testing for S3 API. It had good UI, policies, versionings, etc. SeaweedFS is trying to catch up here. + +The metadata are in simple files. Each file write incurs extra writes to the corresponding meta file, on every drive of the erasure set. Changing only tags or retention rewrites that meta file on all of them, so the write amplification does not shrink with object size. + +There is no optimization for lots of small files. The files are simply stored as is to local disks. +Plus the extra meta file and shards for erasure coding, it only amplifies the LOSF problem. + +Multiple disk IO are needed to read one file. SeaweedFS has O(1) disk reads, even for erasure coded files. + +Erasure coding is full-time. SeaweedFS uses replication on hot data for faster speed and optionally applies erasure coding on warm data. + +No POSIX-like API support. + +There are specific requirements on storage layout, which makes it hard to scale out and to maintain. An erasure set must be 2 to 16 drives and must divide the drive list symmetrically, and capacity grows or shrinks a whole pool at a time. In SeaweedFS, just start one volume server pointing to the master. That's all. + +[Back to TOC](#table-of-contents) + +# Benchmark # + +Unscientific single-machine numbers from a MacBook with an SSD. [`weed benchmark`][Benchmarks], 1 million 1KB files, concurrency 16: + +| | Requests per second | p50 | p99 | +| --- | --- | --- | --- | +| Write | 15,708 | 0.8 ms | 2.6 ms | +| Random read | 47,019 | 0.3 ms | 0.7 ms | + +`make benchmark` runs [warp][S3Benchmark] mixed S3 traffic against a local `weed server`: + +``` +Mixed operations. +Operation: DELETE, 10%, Concurrency: 20, Ran 42s. + * Throughput: 55.13 obj/s + +Operation: GET, 45%, Concurrency: 20, Ran 42s. + * Throughput: 2477.45 MiB/s, 247.75 obj/s + +Operation: PUT, 15%, Concurrency: 20, Ran 42s. + * Throughput: 825.85 MiB/s, 82.59 obj/s + +Operation: STAT, 30%, Concurrency: 20, Ran 42s. + * Throughput: 165.27 obj/s + +Cluster Total: 3302.88 MiB/s, 550.51 obj/s over 43s. +``` + +Read throughput is bounded by the random read speed of the disks, and grows with every volume server added. More numbers, including multi-node, FUSE, and Hadoop, are in [Benchmarks][Benchmarks], [S3 API Benchmark][S3Benchmark], [FIO benchmark][FIO], and [Independent Benchmarks][IndependentBenchmarks]. + +[Back to TOC](#table-of-contents) + +# Enterprise # + +For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition, +which has advanced features, including data recovery, self-healing storage, +customizable erasure coding, EC vacuum and repair, etc. + +[Back to TOC](#table-of-contents) + +# License # + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +The text of this page is available for modification and reuse under the terms of the Creative Commons Attribution-Sharealike 3.0 Unported License and the GNU Free Documentation License (unversioned, with no invariant sections, front-cover texts, or back-cover texts). + +[Back to TOC](#table-of-contents) + +# Sponsors # + +

Sponsor SeaweedFS via Patreon

+ +SeaweedFS is an independent Apache-licensed open source project with its ongoing development made +possible entirely thanks to the support of these awesome [backers](https://github.com/seaweedfs/seaweedfs/blob/master/backers.md). +If you'd like to grow SeaweedFS even stronger, please consider joining our +sponsors on Patreon. + +Your support will be really appreciated by me and other supporters! + + + +### Gold Sponsors +[![nodion](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_nodion.png)](https://www.nodion.com) +[![piknik](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/piknik.png)](https://www.piknik.com) +[![keepsec](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/keepsec.png)](https://www.keepsec.ca) +[![zyner](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_zyner.png)](https://zyner.org) + +[Back to TOC](#table-of-contents) + +## Star History + +![Star History](note/star_history.svg) + +[WeedMini]: https://github.com/seaweedfs/seaweedfs/wiki/Quick-Start-with-weed-mini +[DockerComposeS3]: https://github.com/seaweedfs/seaweedfs/wiki/Docker-Compose-for-S3 +[HelmRecipes]: https://github.com/seaweedfs/seaweedfs/wiki/Helm-Chart-Recipes +[Operator]: https://github.com/seaweedfs/seaweedfs-operator +[SeaweedFsCsiDriver]: https://github.com/seaweedfs/seaweedfs-csi-driver +[GettingStarted]: https://github.com/seaweedfs/seaweedfs/wiki/Getting-Started +[ProductionSetup]: https://github.com/seaweedfs/seaweedfs/wiki/Production-Setup +[ErasureCoding]: https://github.com/seaweedfs/seaweedfs/wiki/Erasure-Coding-for-warm-storage +[RustVolume]: https://github.com/seaweedfs/seaweedfs/wiki/Rust-Volume-Server +[Benchmarks]: https://github.com/seaweedfs/seaweedfs/wiki/Benchmarks +[S3Benchmark]: https://github.com/seaweedfs/seaweedfs/wiki/S3-API-Benchmark +[FIO]: https://github.com/seaweedfs/seaweedfs/wiki/FIO-benchmark +[IndependentBenchmarks]: https://github.com/seaweedfs/seaweedfs/wiki/Independent-Benchmarks +[FailoverMaster]: https://github.com/seaweedfs/seaweedfs/wiki/Failover-Master-Server +[WeedShell]: https://github.com/seaweedfs/seaweedfs/wiki/weed-shell +[Worker]: https://github.com/seaweedfs/seaweedfs/wiki/Worker +[FilerStores]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Stores +[Replication]: https://github.com/seaweedfs/seaweedfs/wiki/Replication +[TieredStorage]: https://github.com/seaweedfs/seaweedfs/wiki/Tiered-Storage +[CloudTier]: https://github.com/seaweedfs/seaweedfs/wiki/Cloud-Tier +[SuperLargeFiles]: https://github.com/seaweedfs/seaweedfs/wiki/Data-Structure-for-Large-Files +[Versioning]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Object-Versioning +[ObjectLock]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Object-Lock-and-Retention +[Lifecycle]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Lifecycle +[CORS]: https://github.com/seaweedfs/seaweedfs/wiki/S3-CORS +[ConditionalOps]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Conditional-Operations +[RenameObject]: https://github.com/seaweedfs/seaweedfs/wiki/S3-RenameObject +[BucketPolicies]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Bucket-Policies +[PolicyConditions]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Policy-Conditions +[PolicyVariables]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Policy-Variables +[OIDC]: https://github.com/seaweedfs/seaweedfs/wiki/OIDC-Integration +[K8sSA]: https://github.com/seaweedfs/seaweedfs/wiki/Kubernetes-ServiceAccount-Authentication +[SSE]: https://github.com/seaweedfs/seaweedfs/wiki/Server-Side-Encryption +[AuditLog]: https://github.com/seaweedfs/seaweedfs/wiki/S3-API-Audit-log +[BucketQuota]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Bucket-Quota +[RateLimiting]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Rate-Limiting +[AmazonS3API]: https://github.com/seaweedfs/seaweedfs/wiki/Amazon-S3-API +[S3vsMinio]: https://github.com/seaweedfs/seaweedfs/wiki/Supported-APIs-vs-Minio +[S3TableBucket]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Table-Bucket +[LanceCatalog]: https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS-Lance-Catalog +[IcebergCatalog]: https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS-Iceberg-Catalog +[SparkIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Spark-Iceberg-Integration +[TrinoIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Trino-Iceberg-Integration +[DremioIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Dremio-Iceberg-Integration +[DuckDBIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/DuckDB-Iceberg-Integration +[DorisIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Doris-Iceberg-Integration +[RisingWaveIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/RisingWave-Iceberg-Integration +[LanceDB]: https://github.com/seaweedfs/seaweedfs/wiki/LanceDB-Integration +[Lakekeeper]: https://github.com/seaweedfs/seaweedfs/wiki/Lakekeeper-Iceberg-Integration +[IcebergMaintenance]: https://github.com/seaweedfs/seaweedfs/wiki/Iceberg-Table-Maintenance +[LanceMaintenance]: https://github.com/seaweedfs/seaweedfs/wiki/Lance-Maintenance-Worker +[S3TablesSecurity]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Tables-Security +[Hadoop]: https://github.com/seaweedfs/seaweedfs/wiki/Hadoop-Compatible-File-System +[CloudDrive]: https://github.com/seaweedfs/seaweedfs/wiki/Cloud-Drive-Architecture +[CacheRemote]: https://github.com/seaweedfs/seaweedfs/wiki/Cache-Remote-Storage +[GatewayToRemoteObjectStore]: https://github.com/seaweedfs/seaweedfs/wiki/Gateway-to-Remote-Object-Storage +[ActiveActiveAsyncReplication]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Active-Active-cross-cluster-continuous-synchronization +[FilerStoreReplication]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication +[AsyncBackup]: https://github.com/seaweedfs/seaweedfs/wiki/Async-Backup +[MetaBackup]: https://github.com/seaweedfs/seaweedfs/wiki/Async-Filer-Metadata-Backup +[CDC]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Change-Data-Capture +[Webhook]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Notification-Webhook +[Mount]: https://github.com/seaweedfs/seaweedfs/wiki/FUSE-Mount +[MountWindows]: https://github.com/seaweedfs/seaweedfs/wiki/Mount-on-Windows +[WebDAV]: https://github.com/seaweedfs/seaweedfs/wiki/WebDAV +[SFTP]: https://github.com/seaweedfs/seaweedfs/wiki/SFTP-Server +[TUS]: https://github.com/seaweedfs/seaweedfs/wiki/TUS-Resumable-Uploads +[FilerDataEncryption]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Data-Encryption +[FIPS]: https://github.com/seaweedfs/seaweedfs/wiki/Cryptography-and-FIPS-Compliance +[AdminUI]: https://github.com/seaweedfs/seaweedfs/wiki/Admin-UI +[Metrics]: https://github.com/seaweedfs/seaweedfs/wiki/System-Metrics +[VolumeServerTTL]: https://github.com/seaweedfs/seaweedfs/wiki/Store-file-with-a-Time-To-Live +[SeaweedUp]: https://github.com/seaweedfs/seaweedfs/wiki/Deployment-with-seaweed-up +[BlobStoreArchitecture]: https://github.com/seaweedfs/seaweedfs/wiki/Blob-Store-Architecture +[Components]: https://github.com/seaweedfs/seaweedfs/wiki/Components +[WhitePaper]: https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS_Architecture.pdf diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..cc3dcbc --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,78 @@ +# Security Policy + +## Supported versions + +Security fixes land in the latest release. Please reproduce against a recent +release or `master` before reporting; issues that only reproduce on old, +unsupported versions are not eligible for a fix or an advisory. + +## Reporting a vulnerability + +Report privately through GitHub private vulnerability reporting (the "Report a +vulnerability" button under the repository's Security tab). This keeps the +report, the fix, and any CVE in one place. Do not open a public issue. + +### What a report must include + +We can only act on reports that show real impact. Please include: + +- Affected version (a release tag or `master` commit you reproduced on) +- The exact deployment and configuration: which components are running + (master, volume, filer, S3, admin), which ports are reachable by the + attacker, and what authentication is enabled +- The attacker's starting position: unauthenticated, a valid S3 user, an admin, + or someone with access to the internal cluster network +- The trust boundary that is crossed (e.g. an unauthenticated client reading + another tenant's data, an S3 user escalating to admin) +- A minimal, working reproduction or proof of concept +- Expected vs. actual behavior + +A report without a working reproduction and a clear trust boundary is a +hardening suggestion, not a vulnerability. We are glad to receive those, but +they are handled on the normal issue tracker, not as security advisories. + +### Automated and AI-assisted reports + +Output from static analysis, dependency scanners, fuzzers, or LLMs is welcome +only when you have manually validated it and can supply a working reproduction +against a supported version, per the requirements above. Raw tool output, +speculative findings, or generated reports without a demonstrated exploit will +be closed as hardening suggestions. + +## Trust model + +SeaweedFS is built to run with its cluster components (master, volume servers, +and the raw filer API) on a trusted network. Those internal APIs are not an +authentication boundary unless you explicitly enable a control (for example +volume JWT or filer authentication) and that control is bypassed. Exposing an +internal port directly to untrusted clients is a deployment mistake, not a +vulnerability in SeaweedFS. + +Reports are in scope when they cross a boundary SeaweedFS is meant to enforce, +for example: + +- Unauthenticated access to data or operations that require authentication +- One S3 identity reading, writing, or deleting another identity's data +- Privilege escalation from a normal S3 user to administrative capability +- Bypass of Object Lock / retention where it is configured +- Remotely triggered data corruption or loss + +Reports are generally out of scope when they require: + +- Direct access to an internal cluster port that is meant to be private +- Full master, filer, or volume server access (already a full compromise) +- An insecure example configuration rather than a documented secure setup +- Local-only impact on a host the attacker already controls + +## CVE assignment + +When a report is confirmed, we publish an advisory and request the CVE through +GitHub. CVEs assigned by third parties without coordinating with us, or for +issues that do not cross a boundary described above, may be disputed. + +## Response and disclosure + +- We aim to acknowledge a valid report within a few business days. +- We will investigate, work on a fix, and coordinate a disclosure timeline + with you. +- Please allow time for a fix before any public disclosure. diff --git a/VOLUME_SERVER_RUST_PLAN.md b/VOLUME_SERVER_RUST_PLAN.md new file mode 100644 index 0000000..1c40233 --- /dev/null +++ b/VOLUME_SERVER_RUST_PLAN.md @@ -0,0 +1,790 @@ +# Execution Plan: SeaweedFS Volume Server — Go to Rust Port + +## Scope Summary + +| Component | Go Source | Lines (non-test) | Description | +|---|---|---|---| +| CLI & startup | `weed/command/volume.go` | 476 | ~40 CLI flags, server bootstrap | +| HTTP server + handlers | `weed/server/volume_server*.go` | 1,517 | Struct, routes, read/write/delete handlers | +| gRPC handlers | `weed/server/volume_grpc_*.go` | 3,073 | 40 RPC method implementations | +| Storage engine | `weed/storage/` | 15,271 | Volumes, needles, index, compaction, EC, backend | +| Protobuf definitions | `weed/pb/volume_server.proto` | 759 | Service + message definitions | +| Shared utilities | `weed/security/`, `weed/stats/`, `weed/util/` | ~2,000+ | JWT, TLS, metrics, helpers | +| **Total** | | **~23,000+** | | + +## Rust Crate & Dependency Strategy + +``` +seaweed-volume/ +├── Cargo.toml +├── build.rs # protobuf codegen +├── proto/ +│ ├── volume_server.proto # copied from Go, adapted +│ └── remote.proto +├── src/ +│ ├── main.rs # CLI entry point +│ ├── config.rs # CLI flags + config +│ ├── server/ +│ │ ├── mod.rs +│ │ ├── volume_server.rs # VolumeServer struct + lifecycle +│ │ ├── http_handlers.rs # HTTP route dispatch +│ │ ├── http_read.rs # GET/HEAD handlers +│ │ ├── http_write.rs # POST/PUT handlers +│ │ ├── http_delete.rs # DELETE handler +│ │ ├── http_admin.rs # /status, /healthz, /ui +│ │ ├── grpc_service.rs # gRPC trait impl dispatch +│ │ ├── grpc_vacuum.rs +│ │ ├── grpc_copy.rs +│ │ ├── grpc_erasure_coding.rs +│ │ ├── grpc_tail.rs +│ │ ├── grpc_admin.rs +│ │ ├── grpc_read_write.rs +│ │ ├── grpc_batch_delete.rs +│ │ ├── grpc_scrub.rs +│ │ ├── grpc_tier.rs +│ │ ├── grpc_remote.rs +│ │ ├── grpc_query.rs +│ │ ├── grpc_state.rs +│ │ └── grpc_client_to_master.rs # heartbeat +│ ├── storage/ +│ │ ├── mod.rs +│ │ ├── store.rs # Store (multi-disk manager) +│ │ ├── volume.rs # Volume struct + lifecycle +│ │ ├── volume_read.rs +│ │ ├── volume_write.rs +│ │ ├── volume_compact.rs +│ │ ├── volume_info.rs +│ │ ├── needle/ +│ │ │ ├── mod.rs +│ │ │ ├── needle.rs # Needle struct + serialization +│ │ │ ├── needle_read.rs +│ │ │ ├── needle_write.rs +│ │ │ ├── needle_map.rs # in-memory NeedleMap +│ │ │ ├── needle_value.rs +│ │ │ └── crc.rs +│ │ ├── super_block.rs +│ │ ├── idx/ +│ │ │ ├── mod.rs +│ │ │ └── idx.rs # .idx file format read/write +│ │ ├── needle_map_leveldb.rs +│ │ ├── types.rs # NeedleId, Offset, Size, DiskType +│ │ ├── disk_location.rs # DiskLocation per-directory +│ │ ├── erasure_coding/ +│ │ │ ├── mod.rs +│ │ │ ├── ec_volume.rs +│ │ │ ├── ec_shard.rs +│ │ │ ├── ec_encoder.rs # Reed-Solomon encoding +│ │ │ └── ec_decoder.rs +│ │ └── backend/ +│ │ ├── mod.rs +│ │ ├── disk.rs +│ │ └── s3_backend.rs # tiered storage to S3 +│ ├── topology/ +│ │ └── volume_layout.rs # replication placement +│ ├── security/ +│ │ ├── mod.rs +│ │ ├── guard.rs # whitelist + JWT gate +│ │ ├── jwt.rs +│ │ └── tls.rs +│ ├── stats/ +│ │ ├── mod.rs +│ │ └── metrics.rs # Prometheus counters/gauges +│ └── util/ +│ ├── mod.rs +│ ├── grpc.rs +│ ├── http.rs +│ └── file.rs +└── tests/ + ├── integration/ + │ ├── http_read_test.rs + │ ├── http_write_test.rs + │ ├── grpc_test.rs + │ └── storage_test.rs + └── unit/ + ├── needle_test.rs + ├── idx_test.rs + ├── super_block_test.rs + └── ec_test.rs +``` + +### Key Rust dependencies + +| Purpose | Crate | +|---|---| +| Async runtime | `tokio` | +| gRPC | `tonic` + `prost` | +| HTTP server | `hyper` + `axum` | +| CLI parsing | `clap` (derive) | +| Prometheus metrics | `prometheus` | +| JWT | `jsonwebtoken` | +| TLS | `rustls` + `tokio-rustls` | +| LevelDB | `rusty-leveldb` or `rocksdb` | +| Reed-Solomon EC | `reed-solomon-erasure` | +| Logging | `tracing` + `tracing-subscriber` | +| Config (security.toml) | `toml` + `serde` | +| CRC32 | `crc32fast` | +| Memory-mapped files | `memmap2` | + +--- + +## Phased Execution Plan + +### Phase 1: Project Skeleton & Protobuf Codegen +**Goal:** Cargo project compiles, proto codegen works, CLI parses all flags. + +**Steps:** + +1.1. Create `seaweed-volume/Cargo.toml` with all dependencies listed above. + +1.2. Copy `volume_server.proto` and `remote.proto` into `proto/`. Adjust package paths for Rust codegen. + +1.3. Create `build.rs` using `tonic-build` to compile `.proto` files into Rust types. + +1.4. Create `src/main.rs` with `clap` derive structs mirroring all 40 CLI flags from `weed/command/volume.go`: + - `--port` (default 8080) + - `--port.grpc` (default 0 → 10000+port) + - `--port.public` (default 0 → same as port) + - `--ip` (auto-detect) + - `--id` (default empty → ip:port) + - `--publicUrl` + - `--ip.bind` + - `--master` (default "localhost:9333") + - `--mserver` (deprecated compat) + - `--preStopSeconds` (default 10) + - `--idleTimeout` (default 30) + - `--dataCenter` + - `--rack` + - `--index` [memory|leveldb|leveldbMedium|leveldbLarge] + - `--disk` [hdd|ssd|] + - `--tags` + - `--dir` (default temp dir) + - `--dir.idx` + - `--max` (default "8") + - `--whiteList` + - `--minFreeSpacePercent` (default "1") + - `--minFreeSpace` + - `--images.fix.orientation` (default false) + - `--readMode` [local|proxy|redirect] (default "proxy") + - `--cpuprofile` + - `--memprofile` + - `--compactionMBps` (default 0) + - `--maintenanceMBps` (default 0) + - `--fileSizeLimitMB` (default 256) + - `--concurrentUploadLimitMB` (default 0) + - `--concurrentDownloadLimitMB` (default 0) + - `--pprof` (default false) + - `--metricsPort` (default 0) + - `--metricsIp` + - `--inflightUploadDataTimeout` (default 60s) + - `--inflightDownloadDataTimeout` (default 60s) + - `--hasSlowRead` (default true) + - `--readBufferSizeMB` (default 4) + - `--index.leveldbTimeout` (default 0) + - `--debug` (default false) + - `--debug.port` (default 6060) + +1.5. Implement the same flag validation logic from `startVolumeServer()`: + - Parse comma-separated `--dir`, `--max`, `--minFreeSpace`, `--disk`, `--tags` + - Replicate single-value-to-all-dirs expansion + - Validate count matches between dirs and limits + - `--mserver` backward compat + +1.6. **Test:** `cargo build` succeeds. `cargo run -- --help` shows all flags. Proto types generated. + +**Verification:** Run with `--port 8080 --dir /tmp --master localhost:9333` — should parse without error and print config. + +--- + +### Phase 2: Core Storage Types & On-Disk Format +**Goal:** Read and write the SeaweedFS needle/volume binary format bit-for-bit compatible with Go. + +**Source files to port:** +- `weed/storage/types/needle_types.go` → `src/storage/types.rs` +- `weed/storage/needle/needle.go` → `src/storage/needle/needle.rs` +- `weed/storage/needle/needle_read.go` → `src/storage/needle/needle_read.rs` +- `weed/storage/needle/needle_write.go` (partial) → `src/storage/needle/needle_write.rs` +- `weed/storage/needle/crc.go` → `src/storage/needle/crc.rs` +- `weed/storage/needle/needle_value_map.go` → `src/storage/needle/needle_value.rs` +- `weed/storage/super_block/super_block.go` → `src/storage/super_block.rs` +- `weed/storage/idx/` → `src/storage/idx/` + +**Steps:** + +2.1. **Fundamental types** (`types.rs`): + - `NeedleId` (u64), `Offset` (u32 or u64 depending on version), `Size` (i32, negative = deleted) + - `Cookie` (u32) + - `DiskType` enum (HDD, SSD, Custom) + - Version constants (Version1=1, Version2=2, Version3=3, CurrentVersion=3) + - Byte serialization matching Go's `binary.BigEndian` encoding + +2.2. **SuperBlock** (`super_block.rs`): + - 8-byte header: Version(1) + ReplicaPlacement(1) + TTL(2) + CompactRevision(2) + Reserved(2) + - `ReplicaPlacement` struct with same/diff rack/dc counts + - `TTL` struct with count + unit + - Read/write from first 8 bytes of `.dat` file + - Match exact byte layout from `super_block.go` + +2.3. **Needle binary format** (`needle.rs`, `needle_read.rs`): + - Version 2/3 header: Cookie(4) + NeedleId(8) + Size(4) + - Body: Data, Flags, Name, Mime, PairsSize, Pairs, LastModified, TTL, Checksum, AppendAtNs, Padding + - CRC32 checksum (matching Go's `crc32.ChecksumIEEE`) + - Padding to 8-byte alignment + - Read path: read header → compute body length → read body → verify CRC + +2.4. **Idx file format** (`idx/`): + - Fixed 16-byte records: NeedleId(8) + Offset(4) + Size(4) + - Sequential append-only file + - Walk/iterate all entries + - Binary search not used (loaded into memory map) + +2.5. **NeedleMap (in-memory)** (`needle_map.rs`): + - HashMap where NeedleValue = {Offset, Size} + - Load from `.idx` file on volume mount + - Support Get, Set, Delete operations + - Track file count, deleted count, deleted byte count + +2.6. **Tests:** + - Unit test: write a needle to bytes → read it back → verify fields match + - Unit test: write/read SuperBlock round-trip + - Unit test: write/read idx entries round-trip + - **Cross-compat test:** Use Go volume server to create a small volume with known data. Read it from Rust and verify all needles decoded correctly. (Keep test fixture `.dat`/`.idx` files in `tests/fixtures/`) + +--- + +### Phase 3: Volume Struct & Lifecycle +**Goal:** Mount, read from, write to, and unmount a volume. + +**Source files to port:** +- `weed/storage/volume.go` → `src/storage/volume.rs` +- `weed/storage/volume_read.go` → `src/storage/volume_read.rs` +- `weed/storage/volume_write.go` → `src/storage/volume_write.rs` +- `weed/storage/volume_loading.go` +- `weed/storage/volume_vacuum.go` → `src/storage/volume_compact.rs` +- `weed/storage/volume_info/volume_info.go` → `src/storage/volume_info.rs` +- `weed/storage/volume_super_block.go` + +**Steps:** + +3.1. **Volume struct** (`volume.rs`): + - Fields: Id, dir, dataFile, nm (NeedleMap), SuperBlock, readOnly, lastModifiedTs, lastCompactIndexOffset, lastCompactRevision + - `noWriteOrDelete` / `noWriteCanDelete` / `readOnly` state flags + - File handles for `.dat` file (read + append) + - Lock strategy: `RwLock` for concurrent reads, exclusive writes + +3.2. **Volume loading** — exact logic from `volume_loading.go`: + - Open `.dat` file, read SuperBlock from first 8 bytes + - Load `.idx` file into NeedleMap + - Handle `.vif` (VolumeInfo) JSON sidecar file + - Set volume state based on SuperBlock + VolumeInfo + +3.3. **Volume read** (`volume_read.rs`) — from `volume_read.go`: + - `ReadNeedle(needleId, cookie)`: lookup in NeedleMap → seek in .dat → read needle bytes → verify cookie + CRC → return data + - Handle deleted needles (Size < 0) + - `ReadNeedleBlob(offset, size)`: raw blob read + - `ReadNeedleMeta(needleId, offset, size)`: read metadata only + +3.4. **Volume write** (`volume_write.rs`) — from `volume_write.go`: + - `WriteNeedle(needle)`: serialize needle → append to .dat → update .idx → update NeedleMap + - `DeleteNeedle(needleId)`: mark as deleted in NeedleMap + append tombstone to .idx + - File size limit check + - Concurrent write serialization (mutex on write path) + +3.5. **Volume compaction** (`volume_compact.rs`) — from `volume_vacuum.go`: + - `CheckCompact()`: compute garbage ratio + - `Compact()`: create new .dat/.idx, copy only live needles, update compact revision + - `CommitCompact()`: rename compacted files over originals + - `CleanupCompact()`: remove temp files + - Throttle by `compactionBytePerSecond` + +3.6. **Volume info** (`volume_info.rs`): + - Read/write `.vif` JSON sidecar + - VolumeInfo protobuf struct mapping + - Remote file references for tiered storage + +3.7. **Tests:** + - Mount a volume, write 100 needles, read them all back, verify content + - Delete 50 needles, verify they return "deleted" + - Compact, verify only 50 remain, verify content + - Read Go-created volume fixtures + +--- + +### Phase 4: Store (Multi-Volume, Multi-Disk Manager) +**Goal:** Manage multiple volumes across multiple disk directories. + +**Source files to port:** +- `weed/storage/store.go` → `src/storage/store.rs` +- `weed/storage/disk_location.go` → `src/storage/disk_location.rs` +- `weed/storage/store_ec.go` +- `weed/storage/store_state.go` + +**Steps:** + +4.1. **DiskLocation** (`disk_location.rs`): + - Directory path, max volume count, min free space, disk type, tags + - Load all volumes from directory on startup + - Track free space, check writable + +4.2. **Store** (`store.rs`): + - Vector of `DiskLocation`s + - `GetVolume(volumeId)` → lookup across all locations + - `HasVolume(volumeId)` check + - `AllocateVolume(...)` — create new volume in appropriate location + - `DeleteVolume(...)`, `MountVolume(...)`, `UnmountVolume(...)` + - `DeleteCollection(collection)` — delete all volumes of a collection + - Collect volume status for heartbeat + - `SetStopping()`, `Close()` + - Persistent state (maintenance mode) via `store_state.go` + +4.3. **Store state** — `VolumeServerState` protobuf with maintenance flag, persisted to disk. + +4.4. **Tests:** + - Create store with 2 dirs, allocate volumes in each, verify load balancing + - Mount/unmount/delete lifecycle + - State persistence across restart + +--- + +### Phase 5: Erasure Coding +**Goal:** Full EC shard encode/decode/read/write/rebuild. + +**Source files to port:** +- `weed/storage/erasure_coding/` (3,599 lines) + +**Steps:** + +5.1. **EC volume + shard structs** — `EcVolume`, `EcShard` with file handles for `.ec00`–`.ec13` shard files + `.ecx` index + `.ecj` journal. + +5.2. **EC encoder** — Reed-Solomon 10+4 (configurable) encoding using `reed-solomon-erasure` crate: + - `VolumeEcShardsGenerate`: read .dat → split into data shards → compute parity → write .ec00-.ec13 + .ecx + +5.3. **EC decoder/reader** — reconstruct data from any 10 of 14 shards: + - `EcShardRead`: read range from a specific shard + - Locate needle in EC volume via .ecx index + - Handle cross-shard needle reads + +5.4. **EC shard operations:** + - Copy, delete, mount, unmount shards + - `VolumeEcShardsRebuild`: rebuild missing shards from remaining + - `VolumeEcShardsToVolume`: reconstruct .dat from EC shards + - `VolumeEcBlobDelete`: mark deleted in EC journal + - `VolumeEcShardsInfo`: report shard metadata + +5.5. **Tests:** + - Encode a volume → verify 14 shards created + - Delete 4 shards → rebuild → verify data intact + - Read individual needles from EC volume + - Cross-compat with Go-generated EC shards + +--- + +### Phase 6: Backend / Tiered Storage +**Goal:** Support tiered storage to remote backends (S3, etc). + +**Source files to port:** +- `weed/storage/backend/` (1,850 lines) + +**Steps:** + +6.1. **Backend trait** — abstract `BackendStorage` trait with `ReadAt`, `WriteAt`, `Truncate`, `Close`, `Name`. + +6.2. **Disk backend** — default local disk implementation. + +6.3. **S3 backend** — upload .dat to S3, read ranges via S3 range requests. + +6.4. **Tier move operations:** + - `VolumeTierMoveDatToRemote`: upload .dat to remote, optionally delete local + - `VolumeTierMoveDatFromRemote`: download .dat from remote + +6.5. **Tests:** + - Disk backend read/write round-trip + - S3 backend with mock/localstack + +--- + +### Phase 7: Security Layer +**Goal:** JWT authentication, whitelist guard, TLS configuration. + +**Source files to port:** +- `weed/security/guard.go` → `src/security/guard.rs` +- `weed/security/jwt.go` → `src/security/jwt.rs` +- `weed/security/tls.go` → `src/security/tls.rs` + +**Steps:** + +7.1. **Guard** (`guard.rs`): + - Whitelist IP check (exact match on `r.RemoteAddr`) + - Wrap handlers with whitelist enforcement + - `UpdateWhiteList()` for live reload + +7.2. **JWT** (`jwt.rs`): + - `SeaweedFileIdClaims` with `fid` field + - Sign with HMAC-SHA256 + - Verify + decode with expiry check + - Separate signing keys for read vs write + - `GetJwt(request)` — extract from `Authorization: Bearer` header or `jwt` query param + +7.3. **TLS** (`tls.rs`): + - Load server TLS cert/key for gRPC and HTTPS + - Load client TLS for mutual TLS + - Read from `security.toml` config (same format as Go's viper config) + +7.4. **Tests:** + - JWT sign → verify round-trip + - JWT with wrong key → reject + - JWT with expired token → reject + - JWT fid mismatch → reject + - Whitelist allow/deny + +--- + +### Phase 8: Prometheus Metrics +**Goal:** Export same metric names as Go for dashboard compatibility. + +**Source files to port:** +- `weed/stats/metrics.go` (volume server counters/gauges/histograms) + +**Steps:** + +8.1. Define all Prometheus metrics matching Go names: + - `VolumeServerRequestCounter` (labels: method, status) + - `VolumeServerRequestHistogram` (labels: method) + - `VolumeServerInFlightRequestsGauge` (labels: method) + - `VolumeServerInFlightUploadSize` + - `VolumeServerInFlightDownloadSize` + - `VolumeServerConcurrentUploadLimit` + - `VolumeServerConcurrentDownloadLimit` + - `VolumeServerHandlerCounter` (labels: type — UploadLimitCond, DownloadLimitCond) + - Read/Write/Delete request counters + +8.2. Metrics HTTP endpoint on `--metricsPort`. + +8.3. Optional push-based metrics loop (`LoopPushingMetric`). + +8.4. **Test:** Verify metric names and labels match Go output. + +--- + +### Phase 9: HTTP Server & Handlers +**Goal:** All HTTP endpoints with exact same behavior as Go. + +**Source files to port:** +- `weed/server/volume_server.go` → `src/server/volume_server.rs` +- `weed/server/volume_server_handlers.go` → `src/server/http_handlers.rs` +- `weed/server/volume_server_handlers_read.go` → `src/server/http_read.rs` +- `weed/server/volume_server_handlers_write.go` → `src/server/http_write.rs` +- `weed/server/volume_server_handlers_admin.go` → `src/server/http_admin.rs` +- `weed/server/volume_server_handlers_helper.go` (URL parsing, proxy, JSON responses) +- `weed/server/volume_server_handlers_ui.go` → `src/server/http_admin.rs` + +**Steps:** + +9.1. **URL path parsing** — from `handlers_helper.go`: + - Parse `/,` and `//` patterns + - Extract volume ID, file ID, filename, ext + +9.2. **Route dispatch** — from `privateStoreHandler` and `publicReadOnlyHandler`: + - `GET /` → `GetOrHeadHandler` + - `HEAD /` → `GetOrHeadHandler` + - `POST /` → `PostHandler` (whitelist gated) + - `PUT /` → `PostHandler` (whitelist gated) + - `DELETE /` → `DeleteHandler` (whitelist gated) + - `OPTIONS /` → CORS preflight + - `GET /status` → JSON status + - `GET /healthz` → health check + - `GET /ui/index.html` → HTML UI page + - Static resources (CSS/JS for UI) + +9.3. **GET/HEAD handler** (`http_read.rs`) — from `handlers_read.go` (468 lines): + - JWT read authorization check + - Lookup needle by volume ID + needle ID + cookie + - ETag / If-None-Match / If-Modified-Since conditional responses + - Content-Type from stored MIME or filename extension + - Content-Disposition header + - Content-Encoding (gzip/zstd stored data) + - Range request support (HTTP 206 Partial Content) + - JPEG orientation fix (if configured) + - Proxy to replica on local miss (readMode=proxy) + - Redirect to replica (readMode=redirect) + - Download tracking (in-flight size accounting) + +9.4. **POST/PUT handler** (`http_write.rs`) — from `handlers_write.go` (170 lines): + - JWT write authorization check + - Multipart form parsing + - Extract file data, filename, content type, TTL, last-modified + - Optional gzip/zstd compression + - Write needle to volume + - Replicate to peers (same logic as Go's `DistributedOperation`) + - Return JSON: {name, size, eTag, error} + +9.5. **DELETE handler** — already in handlers.go: + - JWT authorization + - Delete from local volume + - Replicate delete to peers + - Return JSON result + +9.6. **Admin handlers** (`http_admin.rs`): + - `/status` → JSON with volumes, version, disk status + - `/healthz` → 200 OK if serving + - `/ui/index.html` → HTML dashboard + +9.7. **Concurrency limiting** — from `handlers.go`: + - Upload concurrency limit with `sync::Condvar` + timeout + - Download concurrency limit with proxy fallback to replicas + - HTTP 429 on timeout, 499 on client cancel + - Replication traffic bypasses upload limits + +9.8. **Public port** — if configured, separate listener with read-only routes (GET/HEAD/OPTIONS only). + +9.9. **Request ID middleware** — generate unique request ID per request. + +9.10. **Tests:** + - Integration: start server → upload file via POST → GET it back → verify content + - Integration: upload → DELETE → GET returns 404 + - Integration: conditional GET with ETag → 304 + - Integration: range request → 206 with correct bytes + - Integration: exceed upload limit → 429 + - Integration: whitelist enforcement + - Integration: JWT enforcement + +--- + +### Phase 10: gRPC Service Implementation +**Goal:** All 40 gRPC methods with exact logic. + +**Source files to port:** +- `weed/server/volume_grpc_admin.go` (380 lines) +- `weed/server/volume_grpc_vacuum.go` (124 lines) +- `weed/server/volume_grpc_copy.go` (636 lines) +- `weed/server/volume_grpc_copy_incremental.go` (66 lines) +- `weed/server/volume_grpc_read_write.go` (74 lines) +- `weed/server/volume_grpc_batch_delete.go` (124 lines) +- `weed/server/volume_grpc_tail.go` (140 lines) +- `weed/server/volume_grpc_erasure_coding.go` (619 lines) +- `weed/server/volume_grpc_scrub.go` (121 lines) +- `weed/server/volume_grpc_tier_upload.go` (98 lines) +- `weed/server/volume_grpc_tier_download.go` (85 lines) +- `weed/server/volume_grpc_remote.go` (95 lines) +- `weed/server/volume_grpc_query.go` (69 lines) +- `weed/server/volume_grpc_state.go` (26 lines) +- `weed/server/volume_grpc_read_all.go` (35 lines) +- `weed/server/volume_grpc_client_to_master.go` (325 lines) + +**Steps (grouped by functional area):** + +10.1. **Implement `tonic::Service` for `VolumeServer`** — the generated trait from proto. + +10.2. **Admin RPCs** (`grpc_admin.rs`): + - `AllocateVolume` — create volume on appropriate disk location + - `VolumeMount` / `VolumeUnmount` / `VolumeDelete` + - `VolumeMarkReadonly` / `VolumeMarkWritable` + - `VolumeConfigure` — change replication + - `VolumeStatus` — return read-only, size, file counts + - `VolumeServerStatus` — disk statuses, memory, version, DC, rack + - `VolumeServerLeave` — deregister from master + - `DeleteCollection` + - `VolumeNeedleStatus` — get needle metadata by ID + - `Ping` — latency measurement + - `GetState` / `SetState` — maintenance mode + +10.3. **Vacuum RPCs** (`grpc_vacuum.rs`): + - `VacuumVolumeCheck` — return garbage ratio + - `VacuumVolumeCompact` — stream progress (streaming response) + - `VacuumVolumeCommit` — finalize compaction + - `VacuumVolumeCleanup` — remove temp files + +10.4. **Copy RPCs** (`grpc_copy.rs`): + - `VolumeCopy` — stream .dat/.idx from source to create local copy + - `VolumeSyncStatus` — return sync metadata + - `VolumeIncrementalCopy` — stream .dat delta since timestamp (streaming) + - `CopyFile` — generic file copy by extension (streaming) + - `ReceiveFile` — receive streamed file (client streaming) + - `ReadVolumeFileStatus` — return file timestamps and sizes + +10.5. **Read/Write RPCs** (`grpc_read_write.rs`): + - `ReadNeedleBlob` — raw needle blob read + - `ReadNeedleMeta` — needle metadata + - `WriteNeedleBlob` — raw needle blob write + - `ReadAllNeedles` — stream all needles from volume(s) (streaming) + +10.6. **Batch delete** (`grpc_batch_delete.rs`): + - `BatchDelete` — delete multiple file IDs, return per-ID results + +10.7. **Tail RPCs** (`grpc_tail.rs`): + - `VolumeTailSender` — stream new needles since timestamp (streaming) + - `VolumeTailReceiver` — connect to another volume server and tail its changes + +10.8. **Erasure coding RPCs** (`grpc_erasure_coding.rs`): + - `VolumeEcShardsGenerate` — generate EC shards from volume + - `VolumeEcShardsRebuild` — rebuild missing shards + - `VolumeEcShardsCopy` — copy shards from another server + - `VolumeEcShardsDelete` — delete EC shards + - `VolumeEcShardsMount` / `VolumeEcShardsUnmount` + - `VolumeEcShardRead` — read from EC shard (streaming) + - `VolumeEcBlobDelete` — mark blob deleted in EC volume + - `VolumeEcShardsToVolume` — reconstruct volume from EC shards + - `VolumeEcShardsInfo` — return shard metadata + +10.9. **Scrub RPCs** (`grpc_scrub.rs`): + - `ScrubVolume` — integrity check volumes (INDEX / FULL / LOCAL modes) + - `ScrubEcVolume` — integrity check EC volumes + +10.10. **Tier RPCs** (`grpc_tier.rs`): + - `VolumeTierMoveDatToRemote` — upload to remote backend (streaming progress) + - `VolumeTierMoveDatFromRemote` — download from remote (streaming progress) + +10.11. **Remote storage** (`grpc_remote.rs`): + - `FetchAndWriteNeedle` — fetch from remote storage, write locally, replicate + +10.12. **Query** (`grpc_query.rs`): + - `Query` — experimental CSV/JSON/Parquet select on stored data (streaming) + +10.13. **Master heartbeat** (`grpc_client_to_master.rs`): + - `heartbeat()` background task — periodic gRPC stream to master + - Send: volume info, EC shard info, disk stats, has-no-space flags, deleted volumes + - Receive: volume size limit, leader address, metrics config + - Reconnect on failure with backoff + - `StopHeartbeat()` for graceful shutdown + +10.14. **Tests:** + - Integration test per RPC: call via tonic client → verify response + - Streaming RPCs: verify all chunks received + - Error cases: invalid volume ID, non-existent volume, etc. + - Heartbeat: mock master gRPC server, verify registration + +--- + +### Phase 11: Startup, Lifecycle & Graceful Shutdown +**Goal:** Full server startup matching Go's `runVolume()` and `startVolumeServer()`. + +**Steps:** + +11.1. **Startup sequence** (match `volume.go` exactly): + 1. Load security configuration from `security.toml` + 2. Start metrics server on metrics port + 3. Parse folder/max/minFreeSpace/diskType/tags + 4. Validate all directory writable + 5. Resolve IP, bind IP, public URL, gRPC port + 6. Create `VolumeServer` struct + 7. Check with master (initial handshake) + 8. Create `Store` (loads all existing volumes from disk) + 9. Create security `Guard` + 10. Register HTTP routes on admin mux + 11. Optionally register public mux + 12. Start gRPC server on gRPC port + 13. Start public HTTP server (if separated) + 14. Start cluster HTTP server (with optional TLS) + 15. Start heartbeat background task + 16. Start metrics push loop + 17. Register SIGHUP handler for config reload + new volume loading + +11.2. **Graceful shutdown** (match Go exactly): + 1. On SIGINT/SIGTERM: + 2. Stop heartbeat (notify master we're leaving) + 3. Wait `preStopSeconds` + 4. Stop public HTTP server + 5. Stop cluster HTTP server + 6. Graceful stop gRPC server + 7. `volumeServer.Shutdown()` → `store.Close()` (flush all volumes) + +11.3. **Reload** (SIGHUP): + - Reload security config + - Update whitelist + - Load newly appeared volumes from disk + +11.4. **Tests:** + - Start server → send SIGTERM → verify clean shutdown + - Start server → SIGHUP → verify config reloaded + +--- + +### Phase 12: Integration & Cross-Compatibility Testing +**Goal:** Rust volume server is a drop-in replacement for Go volume server. + +**Steps:** + +12.1. **Binary compatibility tests:** + - Create volumes with Go volume server + - Start Rust volume server on same data directory + - Read all data → verify identical + - Write new data with Rust → read with Go → verify + +12.2. **API compatibility tests:** + - Run same HTTP requests against both Go and Rust servers + - Compare response bodies, headers, status codes + - Test all gRPC RPCs against both + +12.3. **Master interop test:** + - Start Go master server + - Register Rust volume server + - Verify heartbeat works + - Verify volume assignment works + - Upload via filer → stored on Rust volume server → read back + +12.4. **Performance benchmarks:** + - Throughput: sequential writes, sequential reads + - Latency: p50/p99 for read/write + - Concurrency: parallel reads/writes + - Compare Rust vs Go numbers + +12.5. **Edge cases:** + - Volume at max size + - Disk full handling + - Corrupt .dat file recovery + - Network partition during replication + - EC shard loss + rebuild + +--- + +## Execution Order & Dependencies + +``` +Phase 1 (Skeleton + CLI) ← no deps, start here + ↓ +Phase 2 (Storage types) ← needs Phase 1 (types used everywhere) + ↓ +Phase 3 (Volume struct) ← needs Phase 2 + ↓ +Phase 4 (Store manager) ← needs Phase 3 + ↓ +Phase 7 (Security) ← independent, can parallel with 3-4 +Phase 8 (Metrics) ← independent, can parallel with 3-4 + ↓ +Phase 9 (HTTP server) ← needs Phase 4 + 7 + 8 +Phase 10 (gRPC server) ← needs Phase 4 + 7 + 8 + ↓ +Phase 5 (Erasure coding) ← needs Phase 4, wire into Phase 10 +Phase 6 (Tiered storage) ← needs Phase 4, wire into Phase 10 + ↓ +Phase 11 (Startup + shutdown) ← needs Phase 9 + 10 + ↓ +Phase 12 (Integration tests) ← needs all above +``` + +## Estimated Scope + +| Phase | Estimated Rust Lines | Complexity | +|---|---|---| +| 1. Skeleton + CLI | ~400 | Low | +| 2. Storage types | ~2,000 | High (binary compat critical) | +| 3. Volume struct | ~2,500 | High | +| 4. Store manager | ~1,000 | Medium | +| 5. Erasure coding | ~3,000 | High | +| 6. Tiered storage | ~1,500 | Medium | +| 7. Security | ~500 | Medium | +| 8. Metrics | ~300 | Low | +| 9. HTTP server | ~2,000 | High | +| 10. gRPC server | ~3,500 | High | +| 11. Startup/shutdown | ~500 | Medium | +| 12. Integration tests | ~2,000 | Medium | +| **Total** | **~19,000** | | + +## Critical Invariants to Preserve + +1. **Binary format compatibility** — Rust must read/write `.dat`, `.idx`, `.vif`, `.ecX` files identically to Go. A single byte off = data loss. +2. **gRPC wire compatibility** — Same proto, same field semantics. Go master must talk to Rust volume server seamlessly. +3. **HTTP API compatibility** — Same URL patterns, same JSON response shapes, same headers, same status codes. +4. **Replication protocol** — Write replication between Go and Rust volume servers must work bidirectionally. +5. **Heartbeat protocol** — Rust volume server must register with Go master and maintain heartbeat. +6. **CRC32 algorithm** — Must use IEEE polynomial (same as Go's `crc32.ChecksumIEEE`). +7. **JWT compatibility** — Tokens signed by Go filer/master must be verifiable by Rust volume server and vice versa. diff --git a/backers.md b/backers.md new file mode 100644 index 0000000..a1f3aa3 --- /dev/null +++ b/backers.md @@ -0,0 +1,23 @@ + +

Sponsors & Backers

+ +- [Become a backer or sponsor on Patreon](https://www.patreon.com/seaweedfs). + +

Generous Backers ($50+)

+ +- [Evercam Camera Management Software](https://evercam.io/) +- [Spherical Elephant GmbH](https://www.sphericalelephant.com) +- [WizardTales GmbH](https://www.wizardtales.com) +- [Nimbus Web Services](https://nimbusws.com) + +-

Backers

+ +- [ColorfulClouds Tech Co. Ltd.](https://caiyunai.com/) +- [Haravan - Ecommerce Platform](https://www.haravan.com) +- PeterCxy - Creator of Shelter App +- [Hive Games](https://playhive.com/) +- Flowm +- Yoni Nakache +- Catalin Constantin +- MingLi Yuan +- Leroy van Logchem diff --git a/cmd/weed-db/dbcmd.go b/cmd/weed-db/dbcmd.go new file mode 100644 index 0000000..05fa18c --- /dev/null +++ b/cmd/weed-db/dbcmd.go @@ -0,0 +1,419 @@ +package main + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/seaweedfs/seaweedfs/weed/server/postgres" + "github.com/seaweedfs/seaweedfs/weed/util" + flag "github.com/seaweedfs/seaweedfs/weed/util/fla9" +) + +const usageLine = "weed-db -port=5432 -master=" + +const longHelp = `Start a PostgreSQL wire protocol compatible database server that provides SQL query access to SeaweedFS. + +This database server enables any PostgreSQL client, tool, or application to connect to SeaweedFS +and execute SQL queries against MQ topics. It implements the PostgreSQL wire protocol for maximum +compatibility with the existing PostgreSQL ecosystem. + +Examples: + + # Start database server on default port 5432 + weed-db + + # Start with MD5 authentication using JSON format (recommended) + weed-db -auth=md5 -users='{"admin":"secret","readonly":"view123"}' + + # Start with complex passwords using JSON format + weed-db -auth=md5 -users='{"admin":"pass;with;semicolons","user":"password:with:colons"}' + + # Start with credentials from JSON file (most secure) + weed-db -auth=md5 -users="@/etc/seaweedfs/users.json" + + # Start with custom port and master + weed-db -port=5433 -master=master1:9333 + + # Allow connections from any host + weed-db -host=0.0.0.0 -port=5432 + + # Start with TLS encryption + weed-db -tls-cert=server.crt -tls-key=server.key + +Client Connection Examples: + + # psql command line client + psql "host=localhost port=5432 dbname=default user=seaweedfs" + psql -h localhost -p 5432 -U seaweedfs -d default + + # With password + PGPASSWORD=secret psql -h localhost -p 5432 -U admin -d default + + # Connection string + psql "postgresql://admin:secret@localhost:5432/default" + +Programming Language Examples: + + # Python (psycopg2) + import psycopg2 + conn = psycopg2.connect( + host="localhost", port=5432, + user="seaweedfs", database="default" + ) + + # Java JDBC + String url = "jdbc:postgresql://localhost:5432/default"; + Connection conn = DriverManager.getConnection(url, "seaweedfs", ""); + + # Go (lib/pq) + db, err := sql.Open("postgres", "host=localhost port=5432 user=seaweedfs dbname=default sslmode=disable") + + # Node.js (pg) + const client = new Client({ + host: 'localhost', port: 5432, + user: 'seaweedfs', database: 'default' + }); + +Supported SQL Operations: + - SELECT queries on MQ topics + - DESCRIBE/DESC table_name commands + - EXPLAIN query execution plans + - SHOW DATABASES/TABLES commands + - Aggregation functions (COUNT, SUM, AVG, MIN, MAX) + - WHERE clauses with filtering + - System columns (_timestamp_ns, _key, _source) + - Basic PostgreSQL system queries (version(), current_database(), current_user) + +Authentication Methods: + - trust: No authentication required (default) + - password: Clear text password authentication + - md5: MD5 password authentication + +User Credential Formats: + - JSON format: '{"user1":"pass1","user2":"pass2"}' (supports any special characters) + - File format: "@/path/to/users.json" (JSON file) + + Note: JSON format supports passwords with semicolons, colons, and any other special characters. + File format is recommended for production to keep credentials secure. + +Compatible Tools: + - psql (PostgreSQL command line client) + - Any PostgreSQL JDBC/ODBC compatible tool + +Security Features: + - Multiple authentication methods + - TLS encryption support + - Read-only access (no data modification) + +Performance Features: + - Fast path aggregation optimization (COUNT, MIN, MAX without WHERE clauses) + - Hybrid data scanning (parquet files + live logs) + - PostgreSQL wire protocol + - Query result streaming + +` + +type Options struct { + Host string + Port int + MasterAddr string + AuthMethod string + Users string + Database string + MaxConns int + IdleTimeout string + TLSCert string + TLSKey string +} + +// Run executes the weed-db CLI. +func Run(args []string) int { + fs := flag.NewFlagSet("weed-db", flag.ContinueOnError) + usageWriter := io.Writer(os.Stderr) + fs.SetOutput(usageWriter) + + var opts Options + fs.StringVar(&opts.Host, "host", "localhost", "Database server host") + fs.IntVar(&opts.Port, "port", 5432, "Database server port") + fs.StringVar(&opts.MasterAddr, "master", "localhost:9333", "SeaweedFS master server address") + fs.StringVar(&opts.AuthMethod, "auth", "trust", "Authentication method: trust, password, md5") + fs.StringVar(&opts.Users, "users", "", "User credentials for auth (JSON format '{\"user1\":\"pass1\",\"user2\":\"pass2\"}' or file '@/path/to/users.json')") + fs.StringVar(&opts.Database, "database", "default", "Default database name") + fs.IntVar(&opts.MaxConns, "max-connections", 100, "Maximum concurrent connections per server") + fs.StringVar(&opts.IdleTimeout, "idle-timeout", "1h", "Connection idle timeout") + fs.StringVar(&opts.TLSCert, "tls-cert", "", "TLS certificate file path") + fs.StringVar(&opts.TLSKey, "tls-key", "", "TLS private key file path") + + fs.Usage = func() { + fmt.Fprintf(usageWriter, "Usage: %s\n\n%s\n", usageLine, longHelp) + fmt.Fprintln(usageWriter, "Default Parameters:") + fs.PrintDefaults() + } + + if err := fs.Parse(args); err != nil { + return 2 + } + + if !runWithOptions(&opts) { + return 1 + } + return 0 +} + +func runWithOptions(opts *Options) bool { + util.LoadConfiguration("security", false) + + // Validate options. + if opts.MasterAddr == "" { + fmt.Fprintf(os.Stderr, "Error: master address is required\n") + return false + } + + // Parse authentication method. + authMethod, err := parseAuthMethod(opts.AuthMethod) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + return false + } + + // Parse user credentials. + users, err := parseUsers(opts.Users, authMethod) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + return false + } + + // Parse idle timeout. + idleTimeout, err := time.ParseDuration(opts.IdleTimeout) + if err != nil { + fmt.Fprintf(os.Stderr, "Error parsing idle timeout: %v\n", err) + return false + } + + // Validate port number. + if err := validatePortNumber(opts.Port); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + return false + } + + // Setup TLS if requested. + var tlsConfig *tls.Config + if opts.TLSCert != "" && opts.TLSKey != "" { + cert, err := tls.LoadX509KeyPair(opts.TLSCert, opts.TLSKey) + if err != nil { + fmt.Fprintf(os.Stderr, "Error loading TLS certificates: %v\n", err) + return false + } + tlsConfig = &tls.Config{ + Certificates: []tls.Certificate{cert}, + } + } + + // Create server configuration. + config := &postgres.PostgreSQLServerConfig{ + Host: opts.Host, + Port: opts.Port, + AuthMethod: authMethod, + Users: users, + Database: opts.Database, + MaxConns: opts.MaxConns, + IdleTimeout: idleTimeout, + TLSConfig: tlsConfig, + } + + // Create database server. + dbServer, err := postgres.NewPostgreSQLServer(config, opts.MasterAddr) + if err != nil { + fmt.Fprintf(os.Stderr, "Error creating database server: %v\n", err) + return false + } + + // Print startup information. + fmt.Printf("Starting SeaweedFS Database Server...\n") + fmt.Printf("Host: %s\n", opts.Host) + fmt.Printf("Port: %d\n", opts.Port) + fmt.Printf("Master: %s\n", opts.MasterAddr) + fmt.Printf("Database: %s\n", opts.Database) + fmt.Printf("Auth Method: %s\n", opts.AuthMethod) + fmt.Printf("Max Connections: %d\n", opts.MaxConns) + fmt.Printf("Idle Timeout: %s\n", opts.IdleTimeout) + if tlsConfig != nil { + fmt.Printf("TLS: Enabled\n") + } else { + fmt.Printf("TLS: Disabled\n") + } + if len(users) > 0 { + fmt.Printf("Users: %d configured\n", len(users)) + } + + fmt.Printf("\nDatabase Connection Examples:\n") + fmt.Printf(" psql -h %s -p %d -U seaweedfs -d %s\n", opts.Host, opts.Port, opts.Database) + if len(users) > 0 { + // Show first user as example. + for username := range users { + fmt.Printf(" psql -h %s -p %d -U %s -d %s\n", opts.Host, opts.Port, username, opts.Database) + break + } + } + fmt.Printf(" postgresql://%s:%d/%s\n", opts.Host, opts.Port, opts.Database) + + fmt.Printf("\nSupported Operations:\n") + fmt.Printf(" - SELECT queries on MQ topics\n") + fmt.Printf(" - DESCRIBE/DESC table_name\n") + fmt.Printf(" - EXPLAIN query execution plans\n") + fmt.Printf(" - SHOW DATABASES/TABLES\n") + fmt.Printf(" - Aggregations: COUNT, SUM, AVG, MIN, MAX\n") + fmt.Printf(" - System columns: _timestamp_ns, _key, _source\n") + fmt.Printf(" - Basic PostgreSQL system queries\n") + + fmt.Printf("\nReady for database connections!\n\n") + + // Start the server. + err = dbServer.Start() + if err != nil { + fmt.Fprintf(os.Stderr, "Error starting database server: %v\n", err) + return false + } + + // Set up signal handling for graceful shutdown. + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + + // Wait for shutdown signal. + <-sigChan + fmt.Printf("\nReceived shutdown signal, stopping database server...\n") + + // Create context with timeout for graceful shutdown. + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Stop the server with timeout. + done := make(chan error, 1) + go func() { + done <- dbServer.Stop() + }() + + select { + case err := <-done: + if err != nil { + fmt.Fprintf(os.Stderr, "Error stopping database server: %v\n", err) + return false + } + fmt.Printf("Database server stopped successfully\n") + case <-ctx.Done(): + fmt.Fprintf(os.Stderr, "Timeout waiting for database server to stop\n") + return false + } + + return true +} + +// parseAuthMethod parses the authentication method string. +func parseAuthMethod(method string) (postgres.AuthMethod, error) { + switch strings.ToLower(method) { + case "trust": + return postgres.AuthTrust, nil + case "password": + return postgres.AuthPassword, nil + case "md5": + return postgres.AuthMD5, nil + default: + return postgres.AuthTrust, fmt.Errorf("unsupported auth method '%s'. Supported: trust, password, md5", method) + } +} + +// parseUsers parses the user credentials string with support for secure formats only. +// Supported formats: +// 1. JSON format: {"username":"password","username2":"password2"} +// 2. File format: /path/to/users.json or @/path/to/users.json +func parseUsers(usersStr string, authMethod postgres.AuthMethod) (map[string]string, error) { + users := make(map[string]string) + + if usersStr == "" { + // No users specified. + if authMethod != postgres.AuthTrust { + return nil, fmt.Errorf("users must be specified when auth method is not 'trust'") + } + return users, nil + } + + // Trim whitespace. + usersStr = strings.TrimSpace(usersStr) + + // Determine format and parse accordingly. + if strings.HasPrefix(usersStr, "{") && strings.HasSuffix(usersStr, "}") { + // JSON format. + return parseUsersJSON(usersStr, authMethod) + } + + // Check if it's a file path (with or without @ prefix) before declaring invalid format. + filePath := strings.TrimPrefix(usersStr, "@") + if _, err := os.Stat(filePath); err == nil { + // File format. + return parseUsersFile(usersStr, authMethod) // Pass original string to preserve @ handling. + } + + // Invalid format. + return nil, fmt.Errorf("invalid user credentials format. Use JSON format '{\"user\":\"pass\"}' or file format '@/path/to/users.json' or 'path/to/users.json'. Legacy semicolon-separated format is no longer supported") +} + +// parseUsersJSON parses user credentials from JSON format. +func parseUsersJSON(jsonStr string, authMethod postgres.AuthMethod) (map[string]string, error) { + var users map[string]string + if err := json.Unmarshal([]byte(jsonStr), &users); err != nil { + return nil, fmt.Errorf("invalid JSON format for users: %v", err) + } + + // Validate users. + for username, password := range users { + if username == "" { + return nil, fmt.Errorf("empty username in JSON user specification") + } + if authMethod != postgres.AuthTrust && password == "" { + return nil, fmt.Errorf("empty password for user '%s' with auth method", username) + } + } + + return users, nil +} + +// parseUsersFile parses user credentials from a JSON file. +func parseUsersFile(filePath string, authMethod postgres.AuthMethod) (map[string]string, error) { + // Remove @ prefix if present. + filePath = strings.TrimPrefix(filePath, "@") + + // Read file content. + content, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read users file '%s': %v", filePath, err) + } + + contentStr := strings.TrimSpace(string(content)) + + // File must contain JSON format. + if !strings.HasPrefix(contentStr, "{") || !strings.HasSuffix(contentStr, "}") { + return nil, fmt.Errorf("users file '%s' must contain JSON format: {\"user\":\"pass\"}. Legacy formats are no longer supported", filePath) + } + + // Parse as JSON. + return parseUsersJSON(contentStr, authMethod) +} + +// validatePortNumber validates that the port number is reasonable. +func validatePortNumber(port int) error { + if port < 1 || port > 65535 { + return fmt.Errorf("port number must be between 1 and 65535, got %d", port) + } + if port < 1024 { + fmt.Fprintf(os.Stderr, "Warning: port number %d may require root privileges\n", port) + } + return nil +} diff --git a/cmd/weed-db/main.go b/cmd/weed-db/main.go new file mode 100644 index 0000000..735dde6 --- /dev/null +++ b/cmd/weed-db/main.go @@ -0,0 +1,7 @@ +package main + +import "os" + +func main() { + os.Exit(Run(os.Args[1:])) +} diff --git a/cmd/weed-sql/main.go b/cmd/weed-sql/main.go new file mode 100644 index 0000000..735dde6 --- /dev/null +++ b/cmd/weed-sql/main.go @@ -0,0 +1,7 @@ +package main + +import "os" + +func main() { + os.Exit(Run(os.Args[1:])) +} diff --git a/cmd/weed-sql/sqlcmd.go b/cmd/weed-sql/sqlcmd.go new file mode 100644 index 0000000..c25b650 --- /dev/null +++ b/cmd/weed-sql/sqlcmd.go @@ -0,0 +1,623 @@ +package main + +import ( + "context" + "encoding/csv" + "encoding/json" + "fmt" + "io" + "os" + "path" + "strings" + "time" + + "github.com/peterh/liner" + "github.com/seaweedfs/seaweedfs/weed/query/engine" + flag "github.com/seaweedfs/seaweedfs/weed/util/fla9" + "github.com/seaweedfs/seaweedfs/weed/util/grace" + "github.com/seaweedfs/seaweedfs/weed/util/sqlutil" +) + +const usageLine = "weed-sql [-master=localhost:9333] [-interactive] [-file=query.sql] [-output=table|json|csv] [-database=dbname] [-query=\"SQL\"]" + +const longHelp = `Enhanced SQL interface for SeaweedFS Message Queue topics with multiple execution modes. + +Execution Modes: +- Interactive shell (default): weed-sql -interactive +- Single query: weed-sql -query "SELECT * FROM user_events" +- Batch from file: weed-sql -file queries.sql +- Context switching: weed-sql -database analytics -interactive + +Output Formats: +- table: ASCII table format (default for interactive) +- json: JSON format (default for non-interactive) +- csv: Comma-separated values + +Features: +- Full WHERE clause support (=, <, >, <=, >=, !=, LIKE, IN) +- Advanced pattern matching with LIKE wildcards (%, _) +- Multi-value filtering with IN operator +- Real MQ namespace and topic discovery +- Database context switching + +Examples: + weed-sql -interactive + weed-sql -query "SHOW DATABASES" -output json + weed-sql -file batch_queries.sql -output csv + weed-sql -database analytics -query "SELECT COUNT(*) FROM metrics" + weed-sql -master broker1:9333 -interactive +` + +type Options struct { + Master string + Interactive bool + File string + Output string + Database string + Query string +} + +// OutputFormat represents different output formatting options. +type OutputFormat string + +const ( + OutputTable OutputFormat = "table" + OutputJSON OutputFormat = "json" + OutputCSV OutputFormat = "csv" +) + +// SQLContext holds the execution context for SQL operations. +type SQLContext struct { + engine *engine.SQLEngine + currentDatabase string + outputFormat OutputFormat + interactive bool + master string +} + +// Run executes the weed-sql CLI. +func Run(args []string) int { + fs := flag.NewFlagSet("weed-sql", flag.ContinueOnError) + usageWriter := io.Writer(os.Stderr) + fs.SetOutput(usageWriter) + + var opts Options + fs.StringVar(&opts.Master, "master", "localhost:9333", "SeaweedFS master server HTTP address") + fs.BoolVar(&opts.Interactive, "interactive", false, "start interactive shell mode") + fs.StringVar(&opts.File, "file", "", "execute SQL queries from file") + fs.StringVar(&opts.Output, "output", "", "output format: table, json, csv (auto-detected if not specified)") + fs.StringVar(&opts.Database, "database", "", "default database context") + fs.StringVar(&opts.Query, "query", "", "execute single SQL query") + + fs.Usage = func() { + fmt.Fprintf(usageWriter, "Usage: %s\n\n%s\n", usageLine, longHelp) + fmt.Fprintln(usageWriter, "Default Parameters:") + fs.PrintDefaults() + } + + if err := fs.Parse(args); err != nil { + return 2 + } + + if !runWithOptions(&opts) { + return 1 + } + return 0 +} + +func runWithOptions(opts *Options) bool { + // Initialize SQL engine with master address for service discovery. + sqlEngine := engine.NewSQLEngine(opts.Master) + + // Determine execution mode and output format. + interactive := opts.Interactive || (opts.Query == "" && opts.File == "") + outputFormat := determineOutputFormat(opts.Output, interactive) + + // Create SQL context. + ctx := &SQLContext{ + engine: sqlEngine, + currentDatabase: opts.Database, + outputFormat: outputFormat, + interactive: interactive, + master: opts.Master, + } + + // Set current database in SQL engine if specified via command line. + if opts.Database != "" { + ctx.engine.GetCatalog().SetCurrentDatabase(opts.Database) + } + + // Execute based on mode. + switch { + case opts.Query != "": + // Single query mode. + return executeSingleQuery(ctx, opts.Query) + case opts.File != "": + // Batch file mode. + return executeFileQueries(ctx, opts.File) + default: + // Interactive mode. + return runInteractiveShell(ctx) + } +} + +// determineOutputFormat selects the appropriate output format. +func determineOutputFormat(specified string, interactive bool) OutputFormat { + switch strings.ToLower(specified) { + case "table": + return OutputTable + case "json": + return OutputJSON + case "csv": + return OutputCSV + default: + // Auto-detect based on mode. + if interactive { + return OutputTable + } + return OutputJSON + } +} + +// executeSingleQuery executes a single query and outputs the result. +func executeSingleQuery(ctx *SQLContext, query string) bool { + if ctx.outputFormat != OutputTable { + // Suppress banner for non-interactive output. + return executeAndDisplay(ctx, query, false) + } + + fmt.Printf("Executing query against %s...\n", ctx.master) + return executeAndDisplay(ctx, query, true) +} + +// executeFileQueries processes SQL queries from a file. +func executeFileQueries(ctx *SQLContext, filename string) bool { + content, err := os.ReadFile(filename) + if err != nil { + fmt.Printf("Error reading file %s: %v\n", filename, err) + return false + } + + if ctx.outputFormat == OutputTable && ctx.interactive { + fmt.Printf("Executing queries from %s against %s...\n", filename, ctx.master) + } + + // Split file content into individual queries (robust approach). + queries := sqlutil.SplitStatements(string(content)) + + for i, query := range queries { + query = strings.TrimSpace(query) + if query == "" { + continue + } + + if ctx.outputFormat == OutputTable && len(queries) > 1 { + fmt.Printf("\n--- Query %d ---\n", i+1) + } + + if !executeAndDisplay(ctx, query, ctx.outputFormat == OutputTable) { + return false + } + } + + return true +} + +// runInteractiveShell starts the enhanced interactive shell with readline support. +func runInteractiveShell(ctx *SQLContext) bool { + fmt.Println("SeaweedFS Enhanced SQL Interface") + fmt.Println("Type 'help;' for help, 'exit;' to quit") + fmt.Printf("Connected to master: %s\n", ctx.master) + if ctx.currentDatabase != "" { + fmt.Printf("Current database: %s\n", ctx.currentDatabase) + } + fmt.Println("Advanced WHERE operators supported: <=, >=, !=, LIKE, IN") + fmt.Println("Use up/down arrows for command history") + fmt.Println() + + // Initialize liner for readline functionality. + line := liner.NewLiner() + defer line.Close() + + // Handle Ctrl+C gracefully. + line.SetCtrlCAborts(true) + grace.OnInterrupt(func() { + line.Close() + }) + + // Load command history. + historyPath := path.Join(os.TempDir(), "weed-sql-history") + if f, err := os.Open(historyPath); err == nil { + line.ReadHistory(f) + f.Close() + } + + // Save history on exit. + defer func() { + if f, err := os.Create(historyPath); err == nil { + line.WriteHistory(f) + f.Close() + } + }() + + var queryBuffer strings.Builder + + for { + // Show prompt with current database context. + var prompt string + if queryBuffer.Len() == 0 { + if ctx.currentDatabase != "" { + prompt = fmt.Sprintf("seaweedfs:%s> ", ctx.currentDatabase) + } else { + prompt = "seaweedfs> " + } + } else { + prompt = " -> " // Continuation prompt. + } + + // Read line with readline support. + input, err := line.Prompt(prompt) + if err != nil { + if err == liner.ErrPromptAborted { + fmt.Println("Query cancelled") + queryBuffer.Reset() + continue + } + if err != io.EOF { + fmt.Printf("Input error: %v\n", err) + } + break + } + + lineStr := strings.TrimSpace(input) + + // Handle empty lines. + if lineStr == "" { + continue + } + + // Accumulate lines in query buffer. + if queryBuffer.Len() > 0 { + queryBuffer.WriteString(" ") + } + queryBuffer.WriteString(lineStr) + + // Check if we have a complete statement (ends with semicolon or special command). + fullQuery := strings.TrimSpace(queryBuffer.String()) + isComplete := strings.HasSuffix(lineStr, ";") || + isSpecialCommand(fullQuery) + + if !isComplete { + continue // Continue reading more lines. + } + + // Add completed command to history. + line.AppendHistory(fullQuery) + + // Handle special commands (with or without semicolon). + cleanQuery := strings.TrimSuffix(fullQuery, ";") + cleanQuery = strings.TrimSpace(cleanQuery) + + if cleanQuery == "exit" || cleanQuery == "quit" || cleanQuery == "\\q" { + fmt.Println("Goodbye!") + break + } + + if cleanQuery == "help" { + showEnhancedHelp() + queryBuffer.Reset() + continue + } + + // Handle database switching - use proper SQL parser instead of manual parsing. + if strings.HasPrefix(strings.ToUpper(cleanQuery), "USE ") { + // Execute USE statement through the SQL engine for proper parsing. + result, err := ctx.engine.ExecuteSQL(context.Background(), cleanQuery) + if err != nil { + fmt.Printf("Error: %v\n\n", err) + } else if result.Error != nil { + fmt.Printf("Error: %v\n\n", result.Error) + } else { + // Extract the database name from the result message for CLI context. + if len(result.Rows) > 0 && len(result.Rows[0]) > 0 { + message := result.Rows[0][0].ToString() + // Extract database name from "Database changed to: dbname". + if strings.HasPrefix(message, "Database changed to: ") { + ctx.currentDatabase = strings.TrimPrefix(message, "Database changed to: ") + } + fmt.Printf("%s\n\n", message) + } + } + queryBuffer.Reset() + continue + } + + // Handle output format switching. + if strings.HasPrefix(strings.ToUpper(cleanQuery), "\\FORMAT ") { + format := strings.TrimSpace(strings.TrimPrefix(strings.ToUpper(cleanQuery), "\\FORMAT ")) + switch format { + case "TABLE": + ctx.outputFormat = OutputTable + fmt.Println("Output format set to: table") + case "JSON": + ctx.outputFormat = OutputJSON + fmt.Println("Output format set to: json") + case "CSV": + ctx.outputFormat = OutputCSV + fmt.Println("Output format set to: csv") + default: + fmt.Printf("Invalid format: %s. Supported: table, json, csv\n", format) + } + queryBuffer.Reset() + continue + } + + // Execute SQL query (without semicolon). + executeAndDisplay(ctx, cleanQuery, true) + + // Reset buffer for next query. + queryBuffer.Reset() + } + + return true +} + +// isSpecialCommand checks if a command is a special command that doesn't require semicolon. +func isSpecialCommand(query string) bool { + cleanQuery := strings.TrimSuffix(strings.TrimSpace(query), ";") + cleanQuery = strings.ToLower(cleanQuery) + + // Special commands that work with or without semicolon. + specialCommands := []string{ + "exit", "quit", "\\q", "help", + } + + for _, cmd := range specialCommands { + if cleanQuery == cmd { + return true + } + } + + // Commands that are exactly specific commands (not just prefixes). + parts := strings.Fields(strings.ToUpper(cleanQuery)) + if len(parts) == 0 { + return false + } + return (parts[0] == "USE" && len(parts) >= 2) || + strings.HasPrefix(strings.ToUpper(cleanQuery), "\\FORMAT ") +} + +// executeAndDisplay executes a query and displays the result in the specified format. +func executeAndDisplay(ctx *SQLContext, query string, showTiming bool) bool { + startTime := time.Now() + + // Execute the query. + execCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + result, err := ctx.engine.ExecuteSQL(execCtx, query) + if err != nil { + if ctx.outputFormat == OutputJSON { + errorResult := map[string]interface{}{ + "error": err.Error(), + "query": query, + } + jsonBytes, _ := json.MarshalIndent(errorResult, "", " ") + fmt.Println(string(jsonBytes)) + } else { + fmt.Printf("Error: %v\n", err) + } + return false + } + + if result.Error != nil { + if ctx.outputFormat == OutputJSON { + errorResult := map[string]interface{}{ + "error": result.Error.Error(), + "query": query, + } + jsonBytes, _ := json.MarshalIndent(errorResult, "", " ") + fmt.Println(string(jsonBytes)) + } else { + fmt.Printf("Query Error: %v\n", result.Error) + } + return false + } + + // Display results in the specified format. + switch ctx.outputFormat { + case OutputTable: + displayTableResult(result) + case OutputJSON: + displayJSONResult(result) + case OutputCSV: + displayCSVResult(result) + } + + // Show execution time for interactive/table mode. + // Only show timing if there are columns or if result is truly empty. + if showTiming && ctx.outputFormat == OutputTable && (len(result.Columns) > 0 || len(result.Rows) == 0) { + elapsed := time.Since(startTime) + fmt.Printf("\n(%d rows in set, %.3f sec)\n\n", len(result.Rows), elapsed.Seconds()) + } + + return true +} + +// displayTableResult formats and displays query results in ASCII table format. +func displayTableResult(result *engine.QueryResult) { + if len(result.Columns) == 0 { + fmt.Println("Empty result set") + return + } + + // Calculate column widths for formatting. + colWidths := make([]int, len(result.Columns)) + for i, col := range result.Columns { + colWidths[i] = len(col) + } + + // Check data for wider columns. + for _, row := range result.Rows { + for i, val := range row { + if i < len(colWidths) { + valStr := val.ToString() + if len(valStr) > colWidths[i] { + colWidths[i] = len(valStr) + } + } + } + } + + // Print header separator. + fmt.Print("+") + for _, width := range colWidths { + fmt.Print(strings.Repeat("-", width+2) + "+") + } + fmt.Println() + + // Print column headers. + fmt.Print("|") + for i, col := range result.Columns { + fmt.Printf(" %-*s |", colWidths[i], col) + } + fmt.Println() + + // Print separator. + fmt.Print("+") + for _, width := range colWidths { + fmt.Print(strings.Repeat("-", width+2) + "+") + } + fmt.Println() + + // Print data rows. + for _, row := range result.Rows { + fmt.Print("|") + for i, val := range row { + if i < len(colWidths) { + fmt.Printf(" %-*s |", colWidths[i], val.ToString()) + } + } + fmt.Println() + } + + // Print bottom separator. + fmt.Print("+") + for _, width := range colWidths { + fmt.Print(strings.Repeat("-", width+2) + "+") + } + fmt.Println() +} + +// displayJSONResult outputs query results in JSON format. +func displayJSONResult(result *engine.QueryResult) { + // Convert result to JSON-friendly format. + jsonResult := map[string]interface{}{ + "columns": result.Columns, + "rows": make([]map[string]interface{}, len(result.Rows)), + "count": len(result.Rows), + } + + // Convert rows to JSON objects. + for i, row := range result.Rows { + rowObj := make(map[string]interface{}) + for j, val := range row { + if j < len(result.Columns) { + rowObj[result.Columns[j]] = val.ToString() + } + } + jsonResult["rows"].([]map[string]interface{})[i] = rowObj + } + + // Marshal and print JSON. + jsonBytes, err := json.MarshalIndent(jsonResult, "", " ") + if err != nil { + fmt.Printf("Error formatting JSON: %v\n", err) + return + } + + fmt.Println(string(jsonBytes)) +} + +// displayCSVResult outputs query results in CSV format. +func displayCSVResult(result *engine.QueryResult) { + // Handle execution plan results specially to avoid CSV quoting issues. + if len(result.Columns) == 1 && result.Columns[0] == "Query Execution Plan" { + // For execution plans, output directly without CSV encoding to avoid quotes. + for _, row := range result.Rows { + if len(row) > 0 { + fmt.Println(row[0].ToString()) + } + } + return + } + + // Standard CSV output for regular query results. + writer := csv.NewWriter(os.Stdout) + defer writer.Flush() + + // Write headers. + if err := writer.Write(result.Columns); err != nil { + fmt.Printf("Error writing CSV headers: %v\n", err) + return + } + + // Write data rows. + for _, row := range result.Rows { + csvRow := make([]string, len(row)) + for i, val := range row { + csvRow[i] = val.ToString() + } + if err := writer.Write(csvRow); err != nil { + fmt.Printf("Error writing CSV row: %v\n", err) + return + } + } +} + +func showEnhancedHelp() { + fmt.Println(`SeaweedFS Enhanced SQL Interface Help: + +METADATA OPERATIONS: + SHOW DATABASES; - List all MQ namespaces + SHOW TABLES; - List all topics in current namespace + SHOW TABLES FROM database; - List topics in specific namespace + DESCRIBE table_name; - Show table schema + +ADVANCED QUERYING: + SELECT * FROM table_name; - Query all data + SELECT col1, col2 FROM table WHERE ...; - Column projection + SELECT * FROM table WHERE id <= 100; - Range filtering + SELECT * FROM table WHERE name LIKE 'admin%'; - Pattern matching + SELECT * FROM table WHERE status IN ('active', 'pending'); - Multi-value + SELECT COUNT(*), MAX(id), MIN(id) FROM ...; - Aggregation functions + +QUERY ANALYSIS: + EXPLAIN SELECT ...; - Show hierarchical execution plan + (data sources, optimizations, timing) + +DDL OPERATIONS: + CREATE TABLE topic (field1 INT, field2 STRING); - Create topic + Note: ALTER TABLE and DROP TABLE are not supported + +SPECIAL COMMANDS: + USE database_name; - Switch database context + \format table|json|csv - Change output format + help; - Show this help + exit; or quit; or \q - Exit interface + +EXTENDED WHERE OPERATORS: + =, <, >, <=, >= - Comparison operators + !=, <> - Not equal operators + LIKE 'pattern%' - Pattern matching (% = any chars, _ = single char) + IN (value1, value2, ...) - Multi-value matching + AND, OR - Logical operators + +EXAMPLES: + SELECT * FROM user_events WHERE user_id >= 10 AND status != 'deleted'; + SELECT username FROM users WHERE email LIKE '%@company.com'; + SELECT * FROM logs WHERE level IN ('error', 'warning') AND timestamp >= '2023-01-01'; + EXPLAIN SELECT MAX(id) FROM events; -- View execution plan + +Current Status: Full WHERE clause support + Real MQ integration`) +} diff --git a/design-bucket-config-serialization.md b/design-bucket-config-serialization.md new file mode 100644 index 0000000..aee383d --- /dev/null +++ b/design-bucket-config-serialization.md @@ -0,0 +1,167 @@ +# Design: Serializing Bucket Configuration Mutations + +Issue #9651 — concurrent `PutBucketVersioning` + `PutBucketEncryption` (as Terraform +issues them in parallel) intermittently lose the encryption write. + +## Root cause + +The bucket's entire config lives in one filer entry, `/buckets/`. Every +config API does a read-modify-write of that single entry, and the writes are not +serialized: + +- `updateBucketConfig(bucket, fn)` (`s3api_bucket_config.go:468`) — sources from a + possibly-stale cached `BucketConfig`, mutates `Entry.Extended`, writes the + **whole** entry. Used by: versioning, object-lock config, lifecycle, ACL/owner. +- `UpdateBucketMetadata` → `setBucketMetadata` (`:1042`) — reads a fresh entry, + mutates `Entry.Content`, writes the **whole** entry. Used by: encryption, CORS, + tagging, ownership, policy, notification. + +Two ingredients produce the lost update: + +1. **No serialization** of the read→modify→write (the cache mutexes only guard the + in-memory map, not the RMW). +2. **Whole-entry rewrite from an independent snapshot** — `updateBucketConfig` + rebuilds from a stale cached `BucketConfig` whose `Content` predates the + concurrent encryption write, so writing the whole entry reverts `Content`. + +Sequential calls always pass (each sees the previous write), so it only surfaces +under concurrency — and CI's slower IO widens the window (the "2 of ~12 runs"). + +## Goals + +- No lost updates across concurrent bucket-config changes — for **all** config + fields, not just versioning/encryption. +- Correct for a single S3 gateway (the reported case) and for multiple gateways. +- Reuse the filer primitives just merged (per-path lock, `WriteCondition`, + `ObjectTransaction`); do not reintroduce a distributed lock. +- Minimal blast radius: the fix lands at the two chokepoint helpers. + +## Non-goals + +- Changing the one-entry-per-bucket storage model. +- Multi-filer-concurrent bucket writes (addressed only as an optional phase 3). + +## The two ingredients map to two complementary fixes + +### Fix A — serialize + read fresh (closes the window for whole-entry writers) + +Both `updateBucketConfig` and `UpdateBucketMetadata` must run their RMW under one +per-bucket critical section, and **re-read the entry fresh from the filer inside +it** — not rebuild from the cached `BucketConfig`. The lock alone is insufficient: +without the fresh read, two serialized writers still each apply a stale snapshot. + +### Fix B — field-level updates (removes the collision entirely) + +The two writers touch disjoint fields (`Extended[versioning]` vs `Content`). If +each path updated only its own field instead of rewriting the whole entry, neither +could clobber the other regardless of ordering. This is the structural fix and +makes serialization a defense-in-depth concern rather than a correctness +requirement for cross-field cases. + +## Where to serialize (layering) + +The bucket entry is a single filer entry, so unlike object writes there is no +sharding — the question is purely the scope of the lock: + +| Layer | Serializes across | Cost | Notes | +|---|---|---|---| +| 1. Gateway-local per-bucket lock | one gateway process | tiny | fixes the reported (single-gateway/CI) case | +| 2. Filer per-path lock via conditional write | all gateways on one filer | small | reuses #9640 `CreateEntry`+`WriteCondition` | +| 3. Route-by-key to bucket-key owner filer | all gateways and filers | medium | same mechanism as the object DLM-removal | + +## Recommended plan (phased) + +### Phase 1 — minimal fix for #9651 (gateway-local lock + fresh read) + +Add a bounded per-bucket lock table to `S3ApiServer`, reusing the same +`util.LockTable` the filer uses for its per-path lock: + +```go +// in S3ApiServer +bucketConfigLocks *util.LockTable[string] // serialize bucket-entry RMW + +func (s3a *S3ApiServer) withBucketConfigLock(bucket string, fn func() s3err.ErrorCode) s3err.ErrorCode { + lk := s3a.bucketConfigLocks.AcquireLock("bucketConfig", bucket, util.ExclusiveLock) + defer s3a.bucketConfigLocks.ReleaseLock(bucket, lk) + return fn() +} +``` + +Wrap the RMW in **both** chokepoints, and inside the lock read the entry fresh: + +- `updateBucketConfig`: acquire the lock; re-read `/buckets/` from the filer + (not the cache); rebuild `BucketConfig` from that fresh entry; apply `fn`; write; + invalidate cache; release. +- `UpdateBucketMetadata`/`setBucketMetadata`: same lock key; it already reads fresh, + so it just needs to share the critical section. + +Both must use the **same** lock keyed on `bucket`, so versioning and encryption +contend on one mutex. This closes the reported window. Limitation: only one +gateway; two gateways behind a load balancer still race. + +Test: parallel `PutBucketVersioning` + `PutBucketEncryption`, assert both persist +(the exact Terraform scenario), plus an N-way parallel variant over distinct +fields. + +### Phase 2 — robust across gateways (field-level + CAS via merged primitives) + +Move the writers off whole-entry rewrites: + +- **Extended-based config** (versioning, object-lock, ownership, tagging-in-Extended) + → `ObjectTransaction` `PATCH_EXTENDED` on `/buckets/`. The owner filer reads + the entry fresh under its per-path lock and merges only the named keys, so the + gateway never sends a whole-entry snapshot — this dissolves *both* ingredients for + these fields. +- **`Content`-based config** (encryption, CORS, tags blob) — **chosen and + implemented (b3): extend `PATCH_EXTENDED` with `set_content`.** Under the same + per-path lock the filer reads the entry fresh, merges extended attributes, and + replaces `Content`, preserving the rest. So a content write becomes a field-level + patch too — `setBucketMetadata` patches `Content`, `updateBucketConfig` patches + extended keys, and the two serialize on the lock instead of racing whole-entry + rewrites. This is cleaner than the alternatives below: no client-side retry, no + storage migration, and it reuses `ObjectTransaction`'s existing atomic lock. + - (b1, rejected) Conditional `CreateEntry` overwrite with `IF_ETAG_MATCH` + retry + (#9640): correct but needs client-side retry, and the bucket directory entry has + no reliable ETag to compare on. + - (b2, future) Migrate each per-feature config out of the single `Content` blob + into its own `Extended` key. Then even *intra-blob* writes (tags vs encryption) + stop racing. Larger migration; tracked separately. + +Once all paths are field-level patches, the phase-1 gateway lock is unnecessary — +the filer enforces atomicity. (This is the path taken: phase 1 was skipped.) + +### Phase 3 — multi-filer (only if needed) + +If multiple filers can write `/buckets/` concurrently, a filer-local per-path +lock no longer suffices. Route bucket-config writes to +`PrimaryForKey("/buckets/")` (the lock-ring view) and serialize on that one +owner filer — the same route-by-key design used to take object writes off the DLM. +Overkill for rare config writes; include only if multi-filer bucket writes are real. + +## Correctness summary + +- Phase 1: all RMW for a bucket serialize within a gateway; the fresh read means the + second writer observes the first's change. Closes #9651 for single-gateway. +- Phase 2: `PATCH_EXTENDED` is atomic field-level merge at the filer (no snapshot); + CAS turns a concurrent `Content` write into a retry, enforced under the filer's + per-path lock — correct for any number of gateways sharing a filer. +- Phase 3: one owner filer serializes all writers — correct across filers too. + +## Scope checklist (every path that RMWs the bucket entry) + +All of these funnel through the two chokepoints, so fixing the chokepoints covers +them — but the fix must not leave any of them on an unserialized path: + +- via `updateBucketConfig`: versioning, object-lock config, lifecycle, ACL/owner. +- via `UpdateBucketMetadata`/`setBucketMetadata`: encryption, CORS, tagging, + ownership controls, bucket policy, notification. +- bucket create/delete (`CreateEntry`/`DeleteEntry` of `/buckets/`) already + go through the filer's per-path lock on `CreateEntry`; ensure they take the same + bucket lock if they also patch config. + +## Cache rule (must document in code) + +Under the lock, **read the entry from the filer, never rebuild from the cached +`BucketConfig`**. The cache is for reads; it must be invalidated on every write and +never be the source for an RMW. This is the single most important detail — the lock +without the fresh read does not fix the bug. diff --git a/design-cloudstack-provider.md b/design-cloudstack-provider.md new file mode 100644 index 0000000..24e99db --- /dev/null +++ b/design-cloudstack-provider.md @@ -0,0 +1,418 @@ +# SeaweedFS as an Apache CloudStack Object Storage Provider + +A CloudStack ObjectStore plugin that makes SeaweedFS a first-class object storage +backend inside Apache CloudStack, alongside the existing MinIO and Ceph RGW +providers. This is a collaboration with proIO (Swen), who builds private clouds on +CloudStack and wants SeaweedFS as a storage option. + +## The request + +> We can only add MinIO and Ceph as object storage [in CloudStack] today. I want +> to get SeaweedFS into this project... What we need is to build a provider which +> does the communication between Cloudstack and SeaweedFS. + +This is **not** a SeaweedFS-side feature. The work lives in the Apache CloudStack +repo (Java): a new plugin under `plugins/storage/object/seaweedfs/` that implements +CloudStack's ObjectStore plugin framework and talks to SeaweedFS over its S3 and +IAM APIs. SeaweedFS itself needs no changes for the core to work — its S3 API +already covers every bucket operation CloudStack requires, and its IAM API covers +user/credential management. + +## How the CloudStack ObjectStore framework works + +CloudStack 4.18+ introduced an Object Storage framework. An admin registers an +object storage pool via `addObjectStoragePool` (URL + provider + credentials); +tenants then create and manage buckets on it through CloudStack APIs. CloudStack +manages pool and bucket lifecycle; the underlying provider handles the actual +object protocol. + +A provider is a plugin module implementing three interfaces: + +### 1. `ObjectStoreProvider` — registration + +`MinIOObjectStoreProviderImpl` is the reference. It is a Spring `@Component` that: +- Returns a provider name (`"MinIO"`) +- Returns `DataStoreProviderType.OBJECT` +- In `configure()`, injects the lifecycle and driver implementations and calls + `storeMgr.registerDriver(name, driver)` + +### 2. `ObjectStoreLifeCycle` — pool add/remove + +`MinIOObjectStoreLifeCycleImpl.initialize()` reads the URL, name, and +`accesskey`/`secretkey` details from the `addObjectStoragePool` call, tests the +connection by listing buckets, and persists an `ObjectStoreVO` via +`ObjectStoreHelper`. The other methods (attachCluster/Host/Zone, maintain, +deleteDataStore) are no-ops for object storage. + +### 3. `ObjectStoreDriver` — bucket + user operations + +`ObjectStoreDriver` (in `engine/storage/.../object/ObjectStoreDriver.java`) extends +`DataStoreDriver` and defines the bucket/user contract. Every provider must +implement: + +| Method | Purpose | +| --- | --- | +| `createBucket(Bucket, boolean objectLock)` | Create a bucket | +| `listBuckets(long storeId)` | List all buckets | +| `deleteBucket(BucketTO, long storeId)` | Delete a bucket | +| `createUser(long accountId, long storeId)` | Provision a user + credentials for a CloudStack account | +| `setBucketPolicy` / `getBucketPolicy` / `deleteBucketPolicy` | Bucket policy CRUD | +| `setBucketEncryption` / `deleteBucketEncryption` | SSE config | +| `setBucketVersioning` / `deleteBucketVersioning` | Versioning enable/suspend | +| `setBucketQuota(BucketTO, long storeId, long size)` | Per-bucket quota | +| `getAllBucketsUsage(long storeId)` | Usage map for billing/accounting | +| `getBucketAcl` / `setBucketAcl` | ACLs (MinIO/Ceph return null / no-op) | + +`BaseObjectStoreDriverImpl` provides no-op defaults for the `DataStoreDriver` +methods (`createAsync`, `deleteAsync`, `copyAsync`, `canCopy`, `resize`, +`getTO`, `getStoreTO`), so object-store providers only implement the bucket/user +methods above. + +## How the four existing providers differ (and where SeaweedFS lands) + +CloudStack ships four object-store providers. Three are relevant; the simulator +is a test stub. + +| Concern | MinIO | Ceph RGW | Cloudian HyperStore | SeaweedFS | +| --- | --- | --- | --- | --- | +| Bucket CRUD | `MinioClient` (S3) | `AmazonS3` (AWS SDK v1) | `AmazonS3` (AWS SDK v1) | `AmazonS3` (AWS SDK v1) | +| Bucket policy | `MinioClient` | `AmazonS3` | `AmazonS3` | `AmazonS3` | +| Versioning | `MinioClient` | `AmazonS3` | `AmazonS3` | `AmazonS3` | +| Encryption | `MinioClient` | not implemented | `AmazonS3` | `AmazonS3` | +| **User creation** | `MinioAdminClient` | `RgwAdmin` | **`AmazonIdentityManagement`** | **`AmazonIdentityManagement`** | +| **Per-bucket quota** | `MinioAdminClient` | `RgwAdmin` | **not supported** (throws) | **S3 extension** (`PUT /{bucket}?seaweedfs-quota`, SigV4, `s3:PutBucketQuota`) | +| **Usage reporting** | `MinioAdminClient` | `RgwAdmin` | Cloudian admin API | S3 `ListObjectsV2` (MVP); Prometheus / SOSAPI `capacity.xml` (recommended) | + +**Cloudian HyperStore is the direct precedent.** It is an S3-compatible store +that, like SeaweedFS, manages users via the **standard AWS IAM API** using the +AWS IAM Java SDK (`com.amazonaws.services.identitymanagement`). Its driver +(`CloudianHyperStoreObjectStoreDriverImpl`) and util +(`CloudianHyperStoreUtil`) are the template this design follows almost line for +line. Cloudian even validates the quota limitation the same way this design +proposes for the MVP: `setBucketQuota` throws for any non-zero size and only +accepts `0` (no quota). + +The SeaweedFS plugin is therefore a **simpler Cloudian** — same AWS S3 + IAM SDK +clients, same store-details keys (`s3Url`, `iamUrl`, `accesskey`, `secretkey`), +same IAM-user-with-restricted-policy pattern, but with no proprietary admin +client at all (Cloudian has its own `CloudianClient` for its admin API; SeaweedFS +needs only S3 + IAM). For quota, the plugin uses a narrow SeaweedFS S3 extension +(see below); for usage reporting, it falls back to S3 `ListObjectsV2` in the MVP +and recommends Prometheus or SOSAPI `capacity.xml` for production scale. + +### Quota via the S3 `?seaweedfs-quota` extension + +SeaweedFS supports bucket quota natively (server-side enforcement via a +read-only flag when usage exceeds the limit). Rather than exposing the broad +admin REST API (which would require a global bearer token and grant cluster-wide +admin access), the integration uses a **narrow, scoped S3 subresource**: + +- `PUT /{bucket}?seaweedfs-quota` — set bucket quota (IAM permission `s3:PutBucketQuota`) +- `GET /{bucket}?seaweedfs-quota` — get bucket quota (IAM permission `s3:GetBucketQuota`) + +**PUT request body** (JSON): +```json +{"quota_size": 100, "quota_unit": "GB", "quota_enabled": true} +``` + +**GET response body** (JSON): +```json +{"quota_size": 107374182400, "quota_unit": "B", "quota_enabled": true} +``` + +Note: GET always returns `quota_unit: "B"` and the absolute byte count, not +the original unit. A disabled-but-retained quota returns a positive +`quota_size` with `quota_enabled: false`. + +Quota is stored on the bucket's filer entry (positive = enabled, negative = +disabled but retained, zero = no quota), matching the existing admin REST API +behavior. When quota is cleared, the bucket's read-only flag is also lifted. + +**Authentication** uses the existing S3 SigV4 flow — no new global secret is +needed. The CloudStack service credential (the `accesskey`/`secretkey` on the +object store) is the admin credential used for all driver operations: bucket +CRUD, IAM user provisioning, and quota management. It must have broad S3 and +IAM permissions. The per-account IAM users created by `createUser` are the +ones with restricted permissions (full S3 access except bucket +creation/deletion). A future hardening could split quota management onto a +separate credential scoped to only `s3:PutBucketQuota`/`s3:GetBucketQuota`, +but the MVP uses the single admin credential for simplicity, matching how +the MinIO and Ceph providers work. + +The plugin's `setBucketQuota` signs and sends the `PUT /{bucket}?seaweedfs-quota` +request using the AWS SDK v1 `AWSS3V4Signer` for SigV4 signing, then sends the +signed request via `java.net.http.HttpClient` (the AWS S3 SDK doesn't natively +support custom subresources, so we sign manually and send the request +ourselves). The `seaweedfs-quota` query parameter is included in the signed +canonical query string. + +### Usage reporting + +`getAllBucketsUsage` must return a `Map` of bucket name → size. +MinIO uses `MinioAdminClient.getDataUsageInfo`; Ceph uses +`RgwAdmin.listBucketInfo`. SeaweedFS has no admin rollup endpoint, so the MVP +plugin computes it by listing buckets and summing object sizes via S3 +`ListObjectsV2` — expensive for large stores. + +For production scale, SeaweedFS already exposes per-bucket size in: +- **Prometheus metrics** (`bucket_size_bytes` gauge, refreshed every minute) +- **SOSAPI `capacity.xml`** (reports capacity, available space, and usage + through the S3 endpoint) + +Operators should consume one of those instead of S3 list-based aggregation for +large deployments. The MVP's list-based approach is correct but slow; flag it as +a known limitation. + +## SeaweedFS API surface (what the plugin relies on) + +SeaweedFS exposes two relevant APIs, both AWS-compatible: + +### S3 API (`weed s3`) +Full S3-compatible surface. Confirmed against the SeaweedFS S3 wiki and code: +- `CreateBucket`, `HeadBucket`, `ListBuckets`, `DeleteBucket` +- `PutBucketPolicy`, `GetBucketPolicy`, `DeleteBucketPolicy` +- `PutBucketVersioning` (Enabled / Suspended), `GetBucketVersioning` +- `PutBucketEncryption`, `GetBucketEncryption`, `DeleteBucketEncryption` +- `PutBucketAcl`, `GetBucketAcl` +- `ListObjectsV2`, `HeadObject`, `GetObject`, `PutObject`, `DeleteObject` +- Bucket quota via extended attributes / `s3.bucket.quota` (enforced server-side, + surfaced as a read-only state when exceeded — see PR #10224) + +### IAM API (`weed iam` / `iamapi`) +AWS IAM-compatible REST endpoints, implemented in `weed/iamapi/`. Confirmed by +the test suite which uses the **AWS IAM SDK** (`aws-sdk-go/service/iam`) against +the same handlers CloudStack would call: +- `CreateUser`, `DeleteUser`, `ListUsers`, `GetUser` +- `CreateAccessKey`, `DeleteAccessKey`, `ListAccessKeys` +- `PutUserPolicy`, `GetUserPolicy`, `DeleteUserPolicy` +- `AttachUserPolicy`, `ListAttachedUserPolicies` + +This means the CloudStack plugin can manage SeaweedFS users with the **AWS IAM +Java SDK** (`com.amazonaws.services.identitymanagement.AmazonIdentityManagement`), +exactly the way the AWS IAM Go SDK is used in SeaweedFS's own tests. No proprietary +admin client is needed. **Cloudian HyperStore already does exactly this** in the +CloudStack tree — the SeaweedFS plugin follows the same pattern. + +## Design + +### Module layout + +New CloudStack plugin module, mirroring `plugins/storage/object/cloudian/` +(the closest precedent — same AWS S3 + IAM SDK approach): + +``` +plugins/storage/object/seaweedfs/ + pom.xml + src/main/java/org/apache/cloudstack/storage/datastore/ + driver/SeaweedFSObjectStoreDriverImpl.java + lifecycle/SeaweedFSObjectStoreLifeCycleImpl.java + provider/SeaweedFSObjectStoreProviderImpl.java + util/SeaweedFSObjectStoreUtil.java + src/test/java/org/apache/cloudstack/storage/datastore/ + driver/SeaweedFSObjectStoreDriverImplTest.java + provider/SeaweedFSObjectStoreProviderImplTest.java + src/main/resources/META-INF/cloudstack/storage-object-seaweedfs/ + module.properties + spring-storage-object-seaweedfs-context.xml +``` + +### `SeaweedFSObjectStoreProviderImpl` + +Direct copy of `MinIOObjectStoreProviderImpl` with `providerName = "SeaweedFS"`, +injecting the SeaweedFS lifecycle and driver. Registers via +`storeMgr.registerDriver`. + +### `SeaweedFSObjectStoreLifeCycleImpl` + +Copy of `MinIOObjectStoreLifeCycleImpl`. `initialize()` reads `url`, `name`, +`accesskey`, `secretkey` from the `addObjectStoragePool` details map, tests the +connection by calling `AmazonS3.listBuckets()` against the SeaweedFS S3 endpoint, +and persists the `ObjectStoreVO`. No proprietary client needed — the AWS S3 SDK +is enough for the health check. + +### `SeaweedFSObjectStoreDriverImpl` + +The substantive class. Uses two AWS SDK v1 clients (same dependency Ceph already +pulls in, so no new CloudStack dependency): + +- `AmazonS3` for bucket operations (path-style, endpoint-pinned, `us-east-1` + region placeholder — same as Ceph's `getS3Client`) +- `AmazonIdentityManagement` for user/credential operations, pointed at the + SeaweedFS IAM endpoint + +#### Bucket operations — straightforward S3 + +| Interface method | Implementation | +| --- | --- | +| `createBucket` | `s3.createBucket(name)`; reject if `doesBucketExistV2`; persist access/secret key + URL on `BucketVO` (same as Ceph) | +| `listBuckets` | `s3.listBuckets()` → wrap as `BucketObject` (same as Ceph) | +| `deleteBucket` | `s3.deleteBucket(name)` (same as Ceph) | +| `setBucketPolicy` | `s3.setBucketPolicy(...)` with the same public/private JSON the MinIO/Ceph drivers build | +| `getBucketPolicy` / `deleteBucketPolicy` | `s3.getBucketPolicy` / `s3.deleteBucketPolicy` | +| `setBucketVersioning` | `s3.setBucketVersioningConfiguration(Enabled)` | +| `deleteBucketVersioning` | `s3.setBucketVersioningConfiguration(Suspended)` | +| `setBucketEncryption` | `s3.setBucketEncryptionConfiguration(SSE-S3 rule)` | +| `deleteBucketEncryption` | `s3.deleteBucketEncryptionConfiguration` | +| `getBucketAcl` / `setBucketAcl` | no-op / null (same as MinIO and Ceph) | + +#### User creation — the key difference + +MinIO calls `MinioAdminClient.addUser`; Ceph calls `RgwAdmin.createUser`. SeaweedFS +exposes the standard AWS IAM API, so the plugin calls: + +```java +AmazonIdentityManagement iam = getIamClient(storeId); +String userName = "acs-" + account.getUuid(); + +// CreateUser (idempotent — check GetUser first, like Ceph does) +iam.createUser(new CreateUserRequest(userName)); + +// CreateAccessKey → returns the access key + secret key to persist +CreateAccessKeyResult result = iam.createAccessKey( + new CreateAccessKeyRequest().withUserName(userName)); +AccessKey key = result.getAccessKey(); + +// Persist per-account, same pattern as Ceph's CEPH_ACCESS_KEY/CEPH_SECRET_KEY +details.put(SEAWEEDFS_ACCESS_KEY, key.getAccessKeyId()); +details.put(SEAWEEDFS_SECRET_KEY, key.getSecretAccessKey()); +_accountDetailsDao.persist(accountId, details); +``` + +This is the cleanest mapping of the three providers: no proprietary admin client, +just the AWS IAM SDK that CloudStack already has access to. The IAM endpoint URL +is provided as `iamUrl` in the store details. If `iamUrl` is omitted, the driver +defaults it to `s3Url` — SeaweedFS registers its embedded IAM API at `POST /` on +the same S3 endpoint (`UnifiedPostHandler` in `s3api_server.go`), so the IAM +endpoint is the same as the S3 endpoint unless the deployment runs a separate +`weed iam` server. + +#### Bucket quota — S3 `?seaweedfs-quota` extension + +This is the one genuine gap. MinIO and Ceph both have an admin API to set a +per-bucket quota that the backend enforces. SeaweedFS enforces bucket quota +server-side, but the configuration path was **not exposed over a standard S3 or +IAM API** — it was only set via the admin REST API or shell commands. + +The integration adds a **narrow S3 subresource** to SeaweedFS: +- `PUT /{bucket}?seaweedfs-quota` — set bucket quota (IAM permission `s3:PutBucketQuota`) +- `GET /{bucket}?seaweedfs-quota` — get bucket quota (IAM permission `s3:GetBucketQuota`) + +This is implemented in SeaweedFS PR #11279. It uses SigV4 authentication and +dedicated IAM permissions, so the CloudStack service credential can be scoped +to quota management only — no global admin token, no cluster-wide admin access. +The enforcement already exists (PR #10224); this PR only adds the HTTP +configuration surface. + +An earlier approach (PR #11278, closed) added bearer-token auth to the broad +admin REST API. After review, that was unnecessary for this integration — +static S3 config plus standard S3 APIs plus one scoped quota mutation API is +sufficient and far safer. + +> **Note on AWS tools compatibility.** `?seaweedfs-quota` is a SeaweedFS-specific +> S3 subresource, not part of the AWS S3 API. Standard AWS tools (`aws s3api`, +> `s3cmd`, `rclone`) cannot call it directly. This is the same limitation MinIO +> and Ceph have — MinIO quota lives behind a separate admin API (`mc admin +> bucket quota`), and Ceph quota lives behind the Admin Ops API +> (`radosgw-admin quota set`). Neither is callable via `aws s3api` either. +> SeaweedFS's approach is the closest to standard S3 because it uses the same +> endpoint and same SigV4 credentials, just with a custom query parameter. +> Interactive quota management remains available via `weed shell`; the S3 +> extension exists for programmatic integration (CloudStack) where the +> integrator can sign SigV4 requests but cannot run shell commands. + +#### Usage reporting + +`getAllBucketsUsage` must return a `Map` of bucket name → size. +MinIO uses `MinioAdminClient.getDataUsageInfo`; Ceph uses +`RgwAdmin.listBucketInfo`. SeaweedFS has no admin rollup endpoint, so the MVP +plugin computes it by listing buckets and summing object sizes via S3 +`ListObjectsV2` — expensive for large stores. Better options exist in +SeaweedFS already: +- **Prometheus metrics** (`bucket_size_bytes` gauge, refreshed every minute) +- **SOSAPI `capacity.xml`** (reports capacity, available space, and usage + through the S3 endpoint — note: the current "return zero on backend error" + behavior should be validated before using it for billing) + +For the MVP, `listBuckets` + per-bucket size via the S3 API is correct but slow; +flag it as a known limitation. Operators should consume Prometheus or SOSAPI +for production-scale usage reporting. + +### Spring wiring + +`spring-storage-object-seaweedfs-context.xml` registers the provider bean, +identical to the MinIO one. `module.properties` sets +`name=storage-object-seaweedfs`, `parent=storage`. + +### `pom.xml` + +Depends on `aws-java-sdk-s3` and `aws-java-sdk-iam` — both already in the +CloudStack dependency tree (Ceph uses the S3 SDK; the IAM SDK is the standard AWS +bundle). No new third-party dependency, unlike MinIO which pulls in the MinIO +Java client. + +## What changes on the SeaweedFS side + +**One narrow S3 extension is required for quota management.** SeaweedFS PR #11279 +adds the `?seaweedfs-quota` S3 subresource: + +- `PUT /{bucket}?seaweedfs-quota` — set bucket quota (IAM permission `s3:PutBucketQuota`) +- `GET /{bucket}?seaweedfs-quota` — get bucket quota (IAM permission `s3:GetBucketQuota`) + +This is authenticated via standard S3 SigV4 and authorized via dedicated IAM +permissions, so no global admin token is needed. The enforcement already exists +(PR #10224); this PR only adds the HTTP configuration surface. + +One follow-up improvement on the SeaweedFS side would close the usage reporting +gap: + +1. **Validate SOSAPI `capacity.xml` usage calculation** — the current "return + zero on backend error" behavior should be validated before using it for + billing. If reliable, CloudStack can consume it directly instead of + list-based aggregation. + +## Open questions for proIO / Swen + +1. **IAM endpoint path.** ~~Where does `weed iam` listen relative to the S3 + endpoint in a typical proIO deployment?~~ **Resolved.** SeaweedFS registers + its embedded IAM API at `POST /` on the same S3 endpoint + (`UnifiedPostHandler`), so the driver defaults `iamUrl` to `s3Url`. A + separate `iamUrl` is only needed if the deployment runs a standalone + `weed iam` server on a different host/port. +2. **Quota requirements.** Do proIO's customers need server-enforced per-bucket + quotas, or is CloudStack-side accounting sufficient for the first release? + The `?seaweedfs-quota` S3 extension (PR #11279) provides server-enforced + quotas via a scoped credential; this is the recommended path. +3. **Object Lock.** `createBucket` takes an `objectLock` boolean. MinIO supports + it; Ceph ignores it. SeaweedFS has Object Lock support. Should the plugin pass + it through? +4. **Contribution model.** Does proIO want to submit the PR to + `apache/cloudstack` themselves (with SeaweedFS maintainers as reviewers), or + the reverse? Apache CloudStack requires an ICLA for non-trivial contributions. + +## Files + +All in the `apache/cloudstack` repo (new module): + +| File | Purpose | +| --- | --- | +| `plugins/storage/object/seaweedfs/pom.xml` | Maven module | +| `.../datastore/util/SeaweedFSObjectStoreUtil.java` | S3 + IAM client builders, constants, URL validators | +| `.../datastore/provider/SeaweedFSObjectStoreProviderImpl.java` | Spring provider registration | +| `.../datastore/lifecycle/SeaweedFSObjectStoreLifeCycleImpl.java` | Pool add/health-check | +| `.../datastore/driver/SeaweedFSObjectStoreDriverImpl.java` | Bucket + user ops via S3 + IAM SDK | +| `.../resources/META-INF/cloudstack/storage-object-seaweedfs/module.properties` | Module name | +| `.../resources/META-INF/cloudstack/storage-object-seaweedfs/spring-storage-object-seaweedfs-context.xml` | Spring bean | +| `plugins/pom.xml` | Register `storage/object/seaweedfs` module | + +No files in `seaweedfs/seaweedfs` for the MVP. + +### SeaweedFS-side changes (PR #11279) + +| File | Purpose | +| --- | --- | +| `weed/s3api/s3_constants/s3_action_strings.go` | Add `S3_ACTION_PUT_BUCKET_QUOTA` and `S3_ACTION_GET_BUCKET_QUOTA` | +| `weed/s3api/s3_constants/s3_actions.go` | Add coarse-grained `ACTION_PUT_BUCKET_QUOTA` and `ACTION_GET_BUCKET_QUOTA` | +| `weed/s3api/s3_action_resolver.go` | Map `seaweedfs-quota` query param to fine-grained s3: actions | +| `weed/s3api/s3api_bucket_quota_handlers.go` | New — `PutBucketQuotaHandler` and `GetBucketQuotaHandler` | +| `weed/s3api/s3api_bucket_quota_handlers_test.go` | New — tests for unit conversion, validation, and error paths | +| `weed/s3api/s3api_server.go` | Register the two routes in the bucket subrouter | diff --git a/design-lance-catalog.md b/design-lance-catalog.md new file mode 100644 index 0000000..ce657b9 --- /dev/null +++ b/design-lance-catalog.md @@ -0,0 +1,696 @@ +# Lance Catalog for SeaweedFS + +A second catalog surface next to the Iceberg REST catalog, speaking the Lance Namespace +REST spec, over the same table buckets and the same filer. + +## Why + +Gravitino 1.1 added a Lance REST service and 1.3 ships it as a standalone server; Lakekeeper +added Lance in the same window by a completely different route. That is the useful signal: +two unrelated catalogs decided independently that Lance had to be first-class, not a niche. +The client side is already there — `lance-spark` (`LanceNamespaceSparkCatalog` +with `impl=rest`), `lance-ray`, and the generated Python/Java/Rust clients all talk the same +OpenAPI. Implementing the spec means those engines work against SeaweedFS with no +SeaweedFS-specific code on the client. + +The second reason is that Gravitino's own documentation names the gap it cannot close: +DuckDB, pandas and DataFusion "do not support Lance REST natively yet" and have to fetch a +location from the catalog and then open the dataset directly. Gravitino cannot help there, +because it does not own the storage. SeaweedFS does. That is the whole design opportunity +below. + +## Prior art: three families + +Upstream lists twelve catalog implementations, and they fall into three shapes. Knowing +which one we are building matters more than any individual API decision. + +**1. Storage-native, no service.** The Lance Directory Catalog. V1 is a directory listing +where every `.lance/` child of a prefix is a table; V2 adds a `__manifest` table — +itself a Lance table — holding `object_id`/`object_type`/`location` rows, with nested +namespaces, hash-prefixed table directories, and optional managed versioning. No server, no +credentials, no governance. This is the floor every other implementation has to beat. + +**2. Protocol-native server.** Someone implements the Lance Namespace REST OpenAPI and +clients connect with `impl=rest`. Gravitino is the only one of the twelve that does this, +and it is what this design proposes. + +**3. Client-side adapters onto an existing catalog.** Nine of the twelve. The Lance client +translates namespace operations into whatever the backing catalog already speaks: Apache +Polaris, Unity Catalog, AWS Glue, Hive Metastore v2 and v3, Google BigLake, Dataproc, +Microsoft OneLake — and Apache Iceberg REST. Two flavors: + +- Catalogs with a real non-Iceberg table concept mark the format directly. Polaris uses its + Generic Table API with `format = lance`; Unity uses an `EXTERNAL` table with + `table_type=lance` in properties and the path in `storage_location`; Glue uses + `EXTERNAL_TABLE` plus `table_type=lance` in `Parameters`, path in + `StorageDescriptor.Location`. +- Catalogs with no such concept fake one. The Iceberg REST adapter registers **a regular + Iceberg table with a dummy schema — a single nullable string column named `dummy`** — + carrying the property `table_type=lance`, and treats the Iceberg table location as the + Lance dataset root. + +Every adapter in family 3 lands in the same place: `DeclareTable`/`ListTables`/ +`DescribeTable`/`DeregisterTable` only, `DropNamespace` in RESTRICT mode only, +`load_detailed_metadata=false` only, and `managed_versioning=false`. They are a name-to- +location map and nothing more. + +Lakekeeper is the instructive outlier. It has the same generic-table concept Polaris has, +but no upstream adapter exists for it — there is no `lance-namespace` reference anywhere in +its repository and no page for it in the supported-catalogs list. So Polaris's generic tables +are reachable from a stock Lance client and Lakekeeper's are not, despite being the same +idea. Shipping the concept is not the same as shipping the integration. + +## Gravitino and Lakekeeper: the two opposite bets + +Both shipped Lance support in the same window and did not build the same thing. + +**Gravitino implements the protocol.** Its `lance/` module serves the Lance Namespace REST +spec on its own port (`:9101/lance`), so stock `lance-spark` and `lance-ray` connect with +`impl=rest` and no vendor-specific client. The cost is governance: storage credentials are +static properties on the catalog (`lance.storage.access_key_id`, `secret_access_key`, +`endpoint`, `region`, `allow_http`), optionally overridden per table, handed to the engine +as-is. No STS, no expiry, no per-table scoping. + +**Lakekeeper refuses the protocol and governs the object instead.** There is no +`lance-namespace` anywhere in the repository; Lance arrived in 0.13.0 (2026-06-30, issue +#1673 `Generic Table API with Lance`) as one `format` string on a Lakekeeper-native Generic +Table API: + +``` +POST/GET/DELETE /lakekeeper/v1/{prefix}/namespaces/{ns}/generic-tables[/{table}] +GET /lakekeeper/v1/{prefix}/namespaces/{ns}/generic-tables/{table}/credentials +POST /lakekeeper/v1/{prefix}/generic-tables/rename +``` + +`format` is opaque, `schema` and `statistics` are stored but never validated, and the +catalog writes no format-specific metadata — engines go straight to the location. In +exchange Lance tables get everything Iceberg tables get: STS-vended prefix-scoped +credentials, OpenFGA per-action permissions (16 actions), soft-delete with undrop, a +protection flag, rename, pagination, and name uniqueness across Iceberg tables, views and +generic tables in one namespace. The price is that no stock Lance client can talk to it — +you need `pylakekeeper`, which exists mainly to translate vended credentials into +`lance_storage_options`. + +So: protocol fidelity and weak governance, or strong governance and client lock-in. Both +documented their limit honestly, and it is the same limit. Lakekeeper's capability table +says it outright — "Commit coordination: the catalog does not arbitrate writes — engines +write directly." Gravitino does not claim it either. Neither of them coordinates a Lance +commit, which is exactly the thing a store can do and a control plane cannot. + +We do not have to choose. Serve the Lance protocol natively the way Gravitino does, over +the `s3tables` entries that already carry ARNs, policies, tags and maintenance config, and +the governance comes from the layer underneath rather than from a proprietary API on top. +That is only available to us because we are the store, which is also what makes the third +option — arbitrating the commit — available. + +## We are probably already a Lance catalog, and that is a problem + +The Iceberg REST adapter does not care whose Iceberg catalog it is talking to. It needs +`/v1/config?warehouse=`, `/v1/{prefix}/namespaces`, `/v1/{prefix}/namespaces/{ns}/tables` +and unit-separator (`\x1F`) multi-level namespaces. We serve all of those, and +`parseNamespace` in `weed/s3api/iceberg/utils.go:22` already splits on `\x1F`. So a stock +Lance client pointed at our Iceberg catalog on :8181 with the Iceberg impl should already +create, list, describe and deregister Lance tables today, with no SeaweedFS change at all. + +That is worth testing before writing a line of the design above, for two reasons. It is a +free baseline — and possibly a free announcement. And it is a data-loss hazard. + +A Lance table registered this way is an Iceberg table whose metadata references no data +files, sitting on top of a Lance dataset that uses `data/` for its fragments — the same +subdirectory name Iceberg uses. The maintenance worker's orphan cleaner walks exactly +`/metadata` and `
/data`, and deletes every file not referenced by a snapshot +and older than `orphan_older_than_hours` +(`weed/worker/tasks/iceberg/operations.go:331`, default 72). Against an adapter-registered +Lance table, every fragment is unreferenced by construction. Run maintenance and the +dataset is deleted. + +Maintenance is disabled by default (`handler.go:334`), so this is a latent hazard rather +than a live one: it needs an operator to enable Iceberg maintenance on a bucket that also +holds adapter-registered Lance tables. But it costs nothing to close — detection should +skip any table carrying a non-Iceberg format marker (`table_type` property, or +`Format != "ICEBERG"` once the format field is honest), and that guard is worth landing on +its own regardless of whether the rest of this design ever gets built. It is the same +"catalog-only, no maintenance" marker the generic-format question needs. + +## Where we differ from Gravitino + +Gravitino is a metadata service in front of somebody else's object store: + +``` + Spark / Ray Spark / Ray / pandas / duckdb + | | + Lance REST Lance REST (direct S3) + | | | + Gravitino SeaweedFS S3 gateway ----+ + | | + S3 keys handed out SeaweedFS filer + volumes + | + somebody else's S3 +``` + +It resolves a name to a location plus `lance.storage.*` credentials, and steps out of the +way. Everything a Lance table actually is — `_versions/`, `data/`, `_indices/` — is opaque +to it. + +We are the store. Three things follow that Gravitino cannot do: + +1. The catalog and a plain directory listing can be made to agree, so a client with no + catalog at all still sees the right tables. +2. `_versions/` is a filer directory listing, not an object-store `LIST`. Version history + is cheap and can back the admin UI. +3. We can offer a genuinely atomic commit reservation. Lance's commit protocol needs + put-if-not-exists; our S3 layer does not currently provide one (see + [Commit safety](#commit-safety)). The filer does. + +## Placement + +The Iceberg catalog is a thin HTTP shell over `s3tables.Manager`; the storage work lives in +`weed/s3api/s3tables`. Table buckets live under `TablesPath = s3_constants.DefaultBucketsPath`, +i.e. the same filer tree the S3 gateway serves, so `s3://bucket/ns/table/` is simultaneously +a catalog entry and an S3 prefix. Catalog entries are filer directories carrying `s3tables.*` +extended attributes. `Table.Format` already exists and is hard-checked against `"ICEBERG"` +in `weed/s3api/s3tables/handler_table.go:48`. + +So: + +``` +weed/s3api/lance/ new: HTTP surface, id codec, error model +weed/s3api/s3tables/ extended: Format "LANCE", lance state xattr, version entries +weed/command/s3.go new: -port.lance (default 9101), startLanceServer +``` + +`Format: "LANCE"` on the table entry is the whole storage-model change for phase 1. +Everything else — namespaces, ARNs, policies, tags, ownership — is shared verbatim. + +``` + s3tables.Manager (filer) + | + +------------------------+------------------------+ + | | + weed/s3api/iceberg weed/s3api/lance + Iceberg REST :8181 Lance REST :9101 + | | + Iceberg tables Lance datasets + \ / + +-------------------- s3 :8333 -----------------+ + | + SeaweedFS volumes +``` + +## Identifier mapping + +Lance identifiers are `["ns", ..., "table"]`, encoded in the URL as a single string joined +by a delimiter that defaults to `$`. The delimiter alone means the root namespace, so +`/v1/namespace/$/list` lists the root's children. + +Iceberg had to invent a warehouse selector because its identifier is flat and every table +bucket is a separate catalog. Lance does not need that — its identifier is already +hierarchical, and Gravitino uses exactly three levels (`["lance_catalog", "sales", "orders"]`). +That maps onto us without inventing anything: + +``` + $ root -> list of table buckets + $analytics level 1 -> a table bucket + $analytics$sales level 2 -> a namespace in that bucket + $analytics$sales$orders table +``` + +`spark.sql.catalog.lance.parent = analytics` then makes `sales.orders` resolve, which is the +same shape Gravitino's Spark example uses. + +Levels 2..N join into one `s3tables` namespace with `.`, matching what the Iceberg catalog +already does with `flattenNamespacePath`. The flattened form is only the directory name — +`namespaceMetadata.Namespace []string` in the xattr keeps the authoritative parts, so the +mapping stays invertible even though `.` is a legal character inside a namespace part. +Reject `$` in any name part with `InvalidInput`; our charsets already exclude it, so no +escaping scheme is needed. + +Root-level `ListNamespaces` returning table buckets means an unauthenticated or +broadly-scoped caller can enumerate buckets. Filter it through the same +`s3tables/permissions.go` check `ListTableBuckets` uses, not a separate path. + +`CreateNamespace` on a one-part identifier creates a table bucket, and it does so only if +the caller is permitted to — the namespace never creates a bucket as a side effect of +creating something inside it. A table bucket is a tenant resource with its own policy, ARN +and lifecycle, and conjuring one because a client said `CREATE SCHEMA` is a privilege +escalation dressed as a convenience. Lakekeeper draws the same line explicitly: its client +creates tables, not warehouses. + +## Storage layout + +Lay tables out as: + +``` +s3:////
/ + data/ + _versions/ + _indices/ +``` + +**Built without the `.lance` suffix this design originally proposed.** The suffix would have +made every namespace prefix a valid Lance Directory Catalog V1 root, since V1 recognises a +table by exactly that naming. It does not survive contact with the storage layer: the +catalog entry *is* the dataset directory, `validateTableName` excludes `.` from the charset, +and a suffixed entry name would leak into ARNs, policy documents and the S3 Tables API, +where the same table would answer to two different names. Making `GetTablePath` format-aware +instead spreads an "unless it is Lance" branch through code that has no business knowing — +the exact cross-cutting cost this design rejects family 3 for. + +So one name, one directory. What survives is direct access by URI, which is the larger half +of the story and needs no naming convention at all: + +```python +# with the catalog +spark.sql("SELECT * FROM lance.sales.orders") + +# without it, same bytes +lance.dataset("s3://analytics/sales/orders") +``` + +DuckDB, pandas and DataFusion still reach the data with no catalog running, which is the gap +Gravitino's documentation admits to. What they no longer get for free is *enumeration* — a +directory-catalog client pointed at the namespace prefix will not list these as tables. If +that turns out to matter, the cheapest fix is a repair-style tool that materialises `.lance` +aliases, not a rename of the catalog entry. + +Note also that the directory catalog's own V2 mode puts child-namespace tables in +`_` directories at the root and creates no physical subdirectories for +namespaces, so full directory-catalog fidelity was never on offer anyway. We are a +server-backed catalog; the human-readable prefix layout is worth more than partial V1 +lookalike behaviour. + +## The table bucket was not a neutral container + +This design assumed a table bucket is a place to put a table's files. It is +not: `validateTableBucketObjectPath` runs on every S3 write into one and +validated the path against Iceberg's layout, so a Lance client got 403 on +`data/*.lance`, on `_versions/`, and on `_transactions/` — a directory Lance +writes that neither the spec documentation nor this design anticipated. Nothing +about the catalog worked end to end until that changed. + +The layout guard now admits the union of what the supported formats write, and +treats any underscore-prefixed top-level directory as belonging to the format, +checking only that the path stays inside the table. Enumerating Lance's +internal directories by name is exactly the mistake that missed +`_transactions`. Iceberg writes none of them, so it loses nothing. + +Found by pointing the real Python client at a running gateway, not by reading +the spec. Worth remembering for the next format: the premise to check first is +whether the storage layer will accept its files at all. + +## Table lifecycle + +Lance has three table states, and the spec pins them to marker files: + +| State | Marker | Created by | Visible in ListTables | +| --- | --- | --- | --- | +| declared | `.lance-reserved` | `DeclareTable` | yes, when `include_declared=true` | +| created | `_versions/` present | client writes, or `CreateTable` | yes | +| deregistered | `.lance-deregistered` | `DeregisterTable` | no; data preserved | + +Record the state in an xattr (`s3tables.lanceState`) on the catalog entry *and* write the +marker file into the table directory. The xattr is what the catalog reads; the marker is +what keeps a directory-catalog client honest. Dual-write is the price of the interop claim +above, and it is one extra filer write on three rarely-called operations. + +`DeclareTable` is the operation `lance-spark` actually calls on `CREATE TABLE` (it replaced +the legacy `create-empty`), so it is not optional in practice even though the spec marks +only a subset as required. + +`DeregisterTable` preserving data is the same shape as our Iceberg rename, where the catalog +entry moves and the data stays put — reuse `TableDataDirFromMetadataLocation`'s idea rather +than re-deriving the data path from the catalog name. + +## Commit safety + +This is the part I got wrong, and the correction removed a feature rather than adding one. + +Lance commits a version by writing `_versions/{v}.manifest` with put-if-not-exists: exactly +one writer is supposed to win, and the loser rebases. In lance 10 that path is not optional +and needs nothing bolted on — `commit_handler_from_url` hands every `s3://` dataset a +`ConditionalPutCommitHandler`, which calls `put_opts` with `PutMode::Create`, which +object_store's S3 backend sends as `If-None-Match: *`. + +I originally read our gateway as evaluating that header check-then-act, and designed around +it. That was already out of date. `buildWriteCondition` +(`weed/s3api/s3api_object_routed_write.go`) reduces `If-None-Match: *` to a filer +`WriteCondition{IF_NOT_EXISTS}`, and `putToFiler` routes the create to the object's owner +filer, which evaluates the precondition under its per-path lock; when routing is not +available it falls back to the object write lock, which evaluates it under the lock too. +Either way it is atomic. Sixteen concurrent writers of the same fresh key get one 200 and +fifteen 412s, repeatedly. + +So the store already has the primitive Lance needs, cluster-wide, for every conditional-PUT +client and not just this one. + +### What that removed + +An earlier draft of this design offered the catalog as an **external manifest store**: +`managed_versioning: true` plus `CreateTableVersion` and friends, with the reserve step as a +filer `CreateEntry` with `o_excl`. It was implemented, tested, and shipped behind a default-off +flag — and it should not exist. + +- It solves a problem this store does not have. The spec offers that path for stores that + cannot order commits themselves. +- It moves a table's version history out of the dataset and into the catalog, so a reader + that does not go through this namespace no longer sees the whole picture. That is a real + cost paid for nothing. +- lance 10 cannot even use it past the first commit: `NamespaceManifestStore::put_if_not_exists` + answers "put_if_not_exists is not supported for namespace-backed stores", which is exactly + what a second `append` needs. + +The version operations now answer `Unsupported` alongside the other operations the catalog +does not serve, and `managed_versioning` is answered `false`. The property they were +protecting is covered instead by a test that races eight writers at the manifest key through +S3 and asserts one wins — testing the path Lance actually takes. + +## Credential vending + +Iceberg needed a header (`X-Iceberg-Access-Delegation: vended-credentials`) and a bespoke +response shape. Lance has it in the spec: `vend_credentials: true` on the request, +`storage_options` on the response, with `expires_at_millis` as the well-known expiry key. + +Reuse the existing vendor interface unchanged — `iceberg.CredentialVendor` / +`STSService.AssumeRoleForPrincipal` scoped to the table prefix (#10777) — and map its output +to the storage options Lance passes through to `object_store`: + +``` +aws_access_key_id, aws_secret_access_key, aws_session_token, +aws_region, aws_endpoint, allow_http, expires_at_millis +``` + +Those are the names `pylakekeeper` emits as `lance_storage_options`, which is the shape +Lakekeeper's tested S3 path actually feeds to Lance. `object_store` also accepts the +un-prefixed aliases (`endpoint`, `region`) that the directory catalog's `storage.` prefix +strips down to and that Gravitino's `lance.storage.endpoint` resolves to, but the `aws_` +forms are the ones with a tested integration behind them, so emit those. `aws_endpoint` +should come from `deriveS3AdvertisedEndpoint()`, the same source the Iceberg `FileIO` config +uses, and `allow_http` must be set when that endpoint is plain HTTP or every read fails with +a TLS error that looks like a credential problem — Lakekeeper vends both automatically for +exactly this reason, and calls out that there is then no per-vendor branch in client code. + +We emit this server-side, in the `storage_options` field the Lance spec already defines, +which is strictly better than Lakekeeper's arrangement: no client library has to translate +anything, so vending works from any stock Lance client rather than only from theirs. + +Guard the same way #10777 had to after review: bucket-scoped list grants need an `s3:prefix` +condition, and a location containing `*` or `?` must be refused rather than widened into a +resource pattern. + +## Auth and authorization + +Authentication reuses `S3Authenticator` and `CredentialValidator` as-is. The Lance spec maps +identity to headers — `api_key` to `x-api-key`, `auth_token` to `Authorization: Bearer` — and +SigV4 keeps working because it is the same authenticator the Iceberg catalog already fronts. + +Authorization needs nothing new. A Lance table gets the same ARN shape, +`arn:aws:s3tables:...:bucket/B/table/NS/T`, so every existing table-bucket policy covers +Lance tables with no new policy language and no second permission model. Route it through +`s3tables/permissions.go` and inherit the `DefaultAllow` semantics the Iceberg server already +mirrors from the S3 port. + +One spec quirk worth honoring: request context entries prefixed `header.` become request +headers, and every response header comes back as a `header.`-prefixed context entry. Echoing +`x-request-id` through it costs nothing and makes tracing work. + +## What to take from Lakekeeper + +Rejecting Lakekeeper's API shape does not mean rejecting what it learned building it. + +**Deregister is soft-delete, so implement it as one.** Lakekeeper gives generic tables +soft-deletion with undrop and a `protected` flag that makes a drop require `force=true`. +Lance already has the concept — `DeregisterTable` preserves the data and hides the table — +so the `.lance-deregistered` marker is a soft-delete by another name, and a re-register is +an undrop. A protection flag on table-bucket entries is worth having regardless of Lance: +it is a few lines against the existing xattrs and it applies to Iceberg tables too. + +**Enforce one identifier space across entry kinds.** Lakekeeper rejects a generic table +whose name collides with an Iceberg table or view in the same namespace. Our catalog entries +already share one filer directory and already carry `s3tables.entryType`, so this is +structurally true — but it has to be enforced deliberately on every path, or a Lance handler +happily loads an Iceberg table's directory and vice versa. That is the same crossover bug +class as the view/table rename authorization fixed in #10776; the `catalogEntryKind` pattern +from that change is the thing to reuse rather than re-derive. + +**A re-vend path matters more than it looks.** Lakekeeper exposes `/credentials` separately +from load, because STS credentials expire in the middle of long jobs and re-loading the +whole table to refresh them is wasteful. In Lance the spec's answer is another +`DescribeTable` with `vend_credentials: true`, which is fine — but it means `DescribeTable` +must stay cheap when `load_detailed_metadata` is false, which is another reason not to open +the dataset on that path. + +**Generic tables are a cheap orthogonal win.** Lakekeeper's real insight is that Delta, +Parquet, CSV, Vortex and Paimon all get governance for free once the catalog stops caring +what the format is. Our `Table.Format` field already exists and the only thing stopping it +is the hard `"ICEBERG"` check in `handler_table.go:48`. Loosening that and letting the S3 +Tables API register a table with an arbitrary format and a location — no metadata, no +commits — is a small change that makes every format cataloguable. It is independent of this +design and probably worth doing first, since `Format: "LANCE"` is then just a value rather +than a special case. + +**Skip remote signing.** It is Lakekeeper's fallback for S3-compatible stores with no STS, +and their own documentation notes that Lance will not use it — format libraries with their +own S3 client expect static credentials and do not implement the Iceberg signer protocol. We +have STS, so vended credentials are the path, and the signer is not worth building for a +client that cannot consume it. + +## Errors + +Lance uses `{code, error, detail, instance}` with numeric codes, not Iceberg's exception-type +strings. The mapping is mechanical: + +| HTTP | code | when | +| --- | --- | --- | +| 400 | 13 InvalidInput | charset violations, malformed id, route/body mismatch | +| 401 | 16 Unauthenticated | | +| 403 | 15 PermissionDenied | | +| 404 | 1 NamespaceNotFound, 4 TableNotFound, 11 TableVersionNotFound | | +| 409 | 2/5 AlreadyExists, 3 NamespaceNotEmpty, 14 ConcurrentModification | | +| 501 | 0 Unsupported | every phase-3 data operation | + +Route/body mismatch is a spec requirement, not a nicety: when the identifier appears in both +the path and the body and they disagree, the server must return 400. Cheap to get right at +the decode step, annoying to retrofit. + +## Route surface + +Phase 0 is not in this table: point a stock Lance client at the existing Iceberg catalog +with the Iceberg impl, see how far it gets, and land the maintenance guard either way. That +tells us what the native server actually has to beat. + +Phase 1, the whole `lance-spark` and `lance-ray` contract: + +``` +POST /v1/namespace/{id}/create CreateNamespace mode: Create|ExistOk|Overwrite +GET /v1/namespace/{id}/list ListNamespaces +POST /v1/namespace/{id}/describe DescribeNamespace +POST /v1/namespace/{id}/drop DropNamespace mode: Fail|Skip, behavior: Restrict|Cascade +POST /v1/namespace/{id}/exists NamespaceExists +GET /v1/namespace/{id}/table/list ListTables ?include_declared, ?page_token, ?limit +GET /v1/table ListAllTables +POST /v1/table/{id}/declare DeclareTable +POST /v1/table/{id}/describe DescribeTable ?with_table_uri, ?load_detailed_metadata, ?check_declared +POST /v1/table/{id}/exists TableExists +POST /v1/table/{id}/register RegisterTable mode: Create|Overwrite +POST /v1/table/{id}/deregister DeregisterTable +POST /v1/table/{id}/drop DropTable +POST /v1/table/{id}/rename RenameTable +``` + +`DescribeTable` with `load_detailed_metadata=false` needs only `location`, which is the +common case and which we can answer from xattrs alone. With `load_detailed_metadata=true` +the spec wants `version`, `schema` and `stats`, which means reading the Lance manifest. For +phase 1, return the fields we can derive from the filer — `version` from the highest entry in +`_versions/`, given V2 naming is `{u64::MAX - version:020}.manifest` and V1 is +`{version}.manifest` — and omit `schema`/`stats` rather than fabricating them. The spec +tolerates a partial response here; it does not tolerate a wrong one. + +Phase 2 was the five version operations plus `managed_versioning`; it was built and then +removed, for the reasons under Commit safety. + +Phase 3 is the data plane: `CreateTable`, `InsertIntoTable`, `MergeInsertIntoTable`, +`UpdateTable`, `DeleteFromTable`, `QueryTable`, `CountTableRows`, and the index and tag +operations. These exchange Arrow IPC, and more to the point they require reading and writing +the Lance file format, for which no Go implementation exists. Return `Unsupported` (code 0) +and say so in the docs. `arrow-go/v18` is already an indirect dependency, so Arrow framing is +not the blocker — Lance is. + +## Does a Lance table need maintenance? + +Yes, and one part of it has no Iceberg equivalent. The client exposes three jobs: + +- `optimize.compact_files()` — Lance writes a fragment per write batch, so a table fed by + small appends accumulates small files exactly the way an Iceberg table does. +- `optimize.optimize_indices()` — **rows written after an index was built are not covered by + it.** A vector search against a stale index silently misses recent data. That is a + correctness-shaped failure, not a slow query, and it is specific to what people use Lance + for. +- `cleanup_old_versions()` — every version is retained until something removes it. Lance can + do this itself: `optimize.enable_auto_cleanup()` sets it on the dataset, so this one need + not be an external job at all. + +None of it can run in the Go worker. All three read and rewrite Lance files, which needs +Lance format code that does not exist in Go, and there is no useful subset either: deciding +which fragments an old version still references means parsing Lance manifests. + +So the maintenance worker must not touch a Lance table, and it declines by reading the format +the catalog recorded rather than by failing to parse Iceberg metadata. + +## The worker can be Rust, and it is not a sidecar + +The Go worker is not the only worker. `weed/pb/plugin.proto` defines `PluginControlService`, +a language-agnostic gRPC stream that external maintenance workers connect on: the worker +opens `WorkerStream`, sends `WorkerHello` with the job types it can `detect` and `execute`, +answers `RequestConfigSchema` with a `JobTypeDescriptor`, replies to `RunDetectionRequest` +with `JobProposal`s and to `ExecuteJobRequest` with `JobProgressUpdate`s and `JobCompleted`. +`weed worker -admin=host:23646` is the Go reference implementation of exactly that contract, +from outside the admin process. + +Nothing in it is Go-specific, and the Rust toolchain is already in the tree. +`seaweed-volume/build.rs` compiles protos straight out of `../weed/pb/` with `tonic_build`, +including `filer.proto`, on tonic 0.12 and prost 0.13. A Lance worker is that same build +with `plugin.proto` added and the `lance` crate as a dependency — the real one, no FFI and +no Python. + +Three job types, one per real maintenance operation: + +| Job type | Calls | Detected from | +| --- | --- | --- | +| `lance_compact` | `optimize.compact_files` | fragment count and sizes | +| `lance_optimize_indices` | `optimize.optimize_indices` | rows an index does not cover | +| `lance_cleanup_versions` | `cleanup_old_versions` | version count and age | + +What the existing machinery then supplies for free is the part worth noticing. Scheduling, +retries, dedupe by `dedupe_key`, progress reporting, per-job concurrency limits and the +admin settings page all come from the protocol: a worker that answers `RequestConfigSchema` +with a descriptor gets its configuration form rendered in the admin UI without a line of Go +or templ. A Rust worker is a first-class maintenance worker, not an appendage. + +The remaining wiring is small and mostly decided already. `RunDetectionRequest` carries a +`ClusterContext` with filer and S3 addresses plus a free-form `metadata` map, which is where +the Lance namespace URL goes; the worker lists Lance tables from the namespace, which is the +catalog of record and already filters by format. It gets at the data by asking +`DescribeTable` for `storage_options` with `vend_credentials`, so the worker is just another +client of the STS path rather than a component with its own credentials. And when it commits +a compaction it goes through `CreateTableVersion` like any other writer, which is what +managed versioning was for. + +## The worker is also the only thing that can describe the table + +Admin can render an Iceberg table because it can read Iceberg metadata. It cannot read +Lance: it knows the dataset's location and its format string, and that is the whole of it. +The details page showed a location and two empty panels, which is an honest answer and a +useless one. + +The worker already knows. Detection opens every dataset to decide whether it needs +compacting, so at that moment it holds the schema, the row count, the fragment count and +the version count. It just had no way to say so — every message on the stream was about +work. + +So `WorkerObservations` is a body on `WorkerToAdminMessage`: a repeated `ObjectObservation` +of `object_id`, `object_kind`, `format`, and a `ConfigValue` map the worker fills with +whatever it can cheaply say. Admin keeps the last observation per object and serves it back +with the time it was taken and the worker that took it. Nothing schedules from it, and it is +not authoritative — it is a cache with its staleness on the label, which is why the page +badges it rather than presenting it as metadata it read itself. + +The keys are the worker's to choose, which keeps the protocol out of the business of knowing +what a Lance table is. A worker for any other format admin cannot parse describes itself the +same way. + +## A bucket declares its format + +Format was recorded per table, which is enough for the storage layer and not enough for +anything that has to answer a question about a bucket. The admin UI printed one Iceberg +endpoint for every bucket, including the ones holding Lance datasets, where that endpoint +serves nothing; an empty bucket had no format at all. + +So `CreateTableBucket` takes an optional `format`, stored with the rest of the bucket +metadata. Empty means `ICEBERG` - what AWS S3 Tables serves, and therefore what an SDK +that has never heard of the field means. `CreateTable` refuses another format, and +`CreateView` refuses outright outside an Iceberg bucket, a view being Iceberg metadata. +The Lance namespace declares `LANCE` for the buckets it creates. + +**Enforced rather than defaulted**, because the point of showing a format at all is the +endpoint that follows from it, and that endpoint is only truthful if the bucket holds one +format. **Buckets that already exist stay undeclared** and keep taking anything: nothing is +migrated, and the UI shows "unset" as a fact about the bucket's age rather than a fault. +That state is also the only way to hold both formats at once, which is what the +Iceberg-REST adapter path produces. + +## Sample rows are fetched, not cached + +The same asymmetry has a second half. Admin renders an Iceberg table's rows by +reading its Parquet files directly; for Lance it has nothing to read with, so the data +page offered a Browse Data button that led to an empty grid. + +`RequestObjectPreview` / `ObjectPreviewResponse` mirror the config-schema round trip +already on the stream: admin asks, the worker scans the dataset and hands back rows it +has already rendered as text, because it is the only side that knows the types. Admin +picks the worker from the observation store, so the one that last described a table is +the one asked to read it. + +The rows are deliberately not cached, and that is the line between the two channels. An +observation describes an object, so a copy with a timestamp on it is useful. Rows are the +object's contents: a copy held in admin would be stale, larger, and nobody's business. +The page fetches on load, bounded, or says why it cannot. + +## The sidecar question + +The data plane is a different problem, and this design previously conflated the two. +Maintenance rides the worker protocol; `QueryTable` and `InsertIntoTable` do not, because +they are synchronous REST operations on the namespace's own surface. Serving those means a +Rust process that answers HTTP, either behind the Go namespace as a proxy target or in front +of it. It would make SeaweedFS a store you can run vector search *in* rather than one you +read vectors *out of*, which is the larger prize and the reason to keep the option open. + +Neither should gate phase 1. Phases 1 and 2 are pure Go over the filer and are worth +shipping on their own — they are what makes Spark and Ray work. + +## Testing + +Mirror the Iceberg package: `httptest` plus a fake filer client for the handler tests, in +`weed/s3api/lance`. Then an integration suite under `test/s3tables/catalog/` next to the +existing `pyiceberg_test.go`, driving the generated Python `lance-namespace` client against +a live gateway. Three things that suite must cover and unit tests cannot: + +- the storage-options key names actually work, i.e. a client that gets `storage_options` from + `DescribeTable` can open the dataset; +- a table created through the catalog is visible to `lance.dataset()` by URI and to a V1 + directory-catalog client rooted at the namespace prefix; +- concurrent writers do not lose a commit, which is the phase-2 acceptance test and the + thing that justifies the external manifest store. + +Phase 1 is validated: `lance_namespace` 0.11.1 with `impl=rest` drives the namespace, +`lance.write_dataset` writes to the vended location with the vended `storage_options`, and +the rows read back. Note that this client version drops `check_declared` and +`include_declared` on the wire, so `is_only_declared` reads null through it however the +server behaves. + +The commit path is validated at both levels. The mechanism: eight writers race the same +manifest key through S3 with `If-None-Match: *`, and exactly one wins. The property that +actually matters, which single-winner exclusivity does not by itself establish: eight +writers append to one dataset concurrently through lance, and afterwards every batch is +still there — the losers saw the conflict, rebased, and committed again. That second test +is also the sequence managed versioning could not complete at all, since its store answers +"put_if_not_exists is not supported" to the second commit. + +One more that belongs in the Iceberg suite, not this one: a Lance dataset registered through +the Iceberg adapter must survive a full maintenance pass. Reading the code, that test should +fail today; it has not been run. + +## Open questions + +- Root-level `ListNamespaces` enumerating table buckets is convenient and is a listing + surface we do not have on the Iceberg side. Decide whether it is gated behind a flag. +- Whether the `.lance` directory suffix is worth the divergence from the Iceberg layout. I + think yes — it is what makes the catalog optional — but it means the two catalogs' tables + do not look alike on disk, and the admin UI has to know that. +- Names: our charsets are lowercase-only and Lance identifiers are arbitrary strings. Reject + and document, as Iceberg does, or case-fold. Rejecting is right, but see #10734 for how + case handling bites when only one side normalizes. +- Whether to land generic-format registration first. Dropping the `"ICEBERG"` check and + letting a table carry an arbitrary format plus a location is smaller than this whole + design, gets Delta and Parquet catalogued as a side effect, and turns `Format: "LANCE"` + into an ordinary value. The argument against is that it invites tables the maintenance + worker cannot service, so it needs a "catalog-only, no maintenance" marker to be honest. diff --git a/docker/Dockerfile.e2e b/docker/Dockerfile.e2e new file mode 100644 index 0000000..4ffad1c --- /dev/null +++ b/docker/Dockerfile.e2e @@ -0,0 +1,44 @@ +FROM ubuntu:22.04 + +LABEL author="Chris Lu" + +# Note: This e2e test image intentionally runs as root for simplicity and compatibility. +# Production images (Dockerfile.go_build) use proper user isolation with su-exec. +# For testing purposes, running as root avoids permission complexities and dependency +# on Alpine-specific tools like su-exec (not available in Ubuntu repos). + +# apt-install prefers Azure's mirror, which archive.ubuntu.com is slow enough from +# GitHub-hosted runners to justify, and falls through to archive.ubuntu.com when +# Azure is unreachable - which it periodically is, and Acquire::Retries against a +# single mirror just retries a dead host. Images built FROM this one install +# through it for the same reason. +COPY apt-install /usr/local/bin/apt-install +RUN chmod +x /usr/local/bin/apt-install && \ + apt-install curl fio fuse ca-certificates && \ + rm -rf /tmp/* /var/tmp/* + +RUN mkdir -p /etc/seaweedfs /data/filerldb2 + +COPY ./weed /usr/bin/ +COPY ./filer.toml /etc/seaweedfs/filer.toml +COPY ./entrypoint_e2e.sh /entrypoint.sh + +# volume server grpc port +EXPOSE 18080 +# volume server http port +EXPOSE 8080 +# filer server grpc port +EXPOSE 18888 +# filer server http port +EXPOSE 8888 +# master server shared grpc port +EXPOSE 19333 +# master server shared http port +EXPOSE 9333 + +VOLUME /data +WORKDIR /data + +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/Dockerfile.foundationdb_large b/docker/Dockerfile.foundationdb_large new file mode 100644 index 0000000..9ec3e13 --- /dev/null +++ b/docker/Dockerfile.foundationdb_large @@ -0,0 +1,132 @@ +FROM golang:1.26 AS builder + +RUN apt-get update && \ + apt-get install -y build-essential wget ca-certificates && \ + rm -rf /var/lib/apt/lists/* + +ARG FDB_VERSION=7.4.5 +ENV FDB_VERSION=${FDB_VERSION} +ARG TARGETARCH + +# Install FoundationDB client libraries with SHA256 checksum verification +# Known SHA256 checksums for FoundationDB client packages (verified 2025-01-19) +# To add checksums for new versions: run docker/get_fdb_checksum.sh +RUN cd /tmp && \ + case "${TARGETARCH}" in \ + "amd64") FDB_ARCH="amd64"; PACKAGE_ARCH="amd64" ;; \ + "arm64") FDB_ARCH="arm64"; PACKAGE_ARCH="aarch64" ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \ + esac && \ + case "${FDB_VERSION}_${FDB_ARCH}" in \ + "7.4.5_amd64") \ + EXPECTED_SHA256="eea6b98cf386a0848655b2e196d18633662a7440a7ee061c10e32153c7e7e112" ;; \ + "7.4.5_arm64") \ + EXPECTED_SHA256="f2176b86b7e1b561c3632b4e6e7efb82e3b8f57c2ff0d0ac4671e742867508aa" ;; \ + "7.3.43_amd64") \ + EXPECTED_SHA256="c3fa0a59c7355b914a1455dac909238d5ea3b6c6bc7b530af8597e6487c1651a" ;; \ + "7.3.43_arm64") \ + echo "ERROR: FoundationDB ${FDB_VERSION} does not publish arm64 client packages." >&2; \ + echo "Please upgrade to 7.4.5+ when targeting arm64." >&2; \ + exit 1 ;; \ + *) \ + echo "ERROR: No checksum available for FDB version ${FDB_VERSION} on ${FDB_ARCH}" >&2; \ + echo "This is a security requirement. To add verification:" >&2; \ + echo " 1. Run: docker/get_fdb_checksum.sh ${FDB_VERSION} ${FDB_ARCH}" >&2; \ + echo " 2. Add the checksum to this Dockerfile" >&2; \ + echo "Refusing to proceed without checksum verification." >&2; \ + exit 1 ;; \ + esac && \ + PACKAGE="foundationdb-clients_${FDB_VERSION}-1_${PACKAGE_ARCH}.deb" && \ + wget --timeout=30 --tries=3 https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/${PACKAGE} && \ + echo "${EXPECTED_SHA256} ${PACKAGE}" | sha256sum -c - || \ + (echo "ERROR: Checksum verification failed for FoundationDB ${FDB_VERSION} (${FDB_ARCH})" >&2; \ + echo "Expected: ${EXPECTED_SHA256}" >&2; \ + echo "This indicates either a corrupted download or potential tampering." >&2; \ + exit 1) && \ + dpkg -i ${PACKAGE} && \ + rm ${PACKAGE} + +# Set up FoundationDB environment variables for CGO +ENV CGO_CFLAGS="-I/usr/include/foundationdb" +ENV CGO_LDFLAGS="-lfdb_c" + +# build SeaweedFS sources; prefer local context but fall back to git clone if context only has docker files +ARG SOURCE_REF=master +WORKDIR /go/src/github.com/seaweedfs/seaweedfs +COPY . . +RUN set -euo pipefail && \ + if [ ! -d weed ]; then \ + echo "Local build context does not include SeaweedFS sources; cloning ${SOURCE_REF}" >&2; \ + mkdir -p /tmp/local-context && cp -a /go/src/github.com/seaweedfs/seaweedfs/. /tmp/local-context && \ + cd / && rm -rf /go/src/github.com/seaweedfs/seaweedfs && \ + git clone --depth 1 --branch ${SOURCE_REF} https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs && \ + cp -a /tmp/local-context/. /go/src/github.com/seaweedfs/seaweedfs/docker/ && \ + rm -rf /tmp/local-context && \ + cd /go/src/github.com/seaweedfs/seaweedfs; \ + fi && \ + cd weed \ + && COMMIT_SHA=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") \ + && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${COMMIT_SHA}" \ + && go install -tags "5BytesOffset foundationdb" -ldflags "${LDFLAGS}" + + +FROM debian:bookworm-slim AS final +LABEL author="Chris Lu" + +# Install runtime dependencies first +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + fuse \ + wget && \ + rm -rf /var/lib/apt/lists/* + +# Reuse FoundationDB artifacts installed during the build stage +COPY --from=builder /usr/lib/libfdb_c* /usr/lib/ +COPY --from=builder /usr/lib/foundationdb /usr/lib/foundationdb +COPY --from=builder /usr/bin/fdb* /usr/bin/ +RUN ldconfig + +# Copy SeaweedFS binary and configuration +COPY --from=builder /go/bin/weed /usr/bin/ +RUN mkdir -p /etc/seaweedfs +COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_foundationdb.toml /etc/seaweedfs/filer.toml +COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh + +# Create non-root user +RUN groupadd -g 1000 seaweed && \ + useradd -u 1000 -g seaweed -s /bin/bash -m seaweed + +# volume server gprc port +EXPOSE 18080 +# volume server http port +EXPOSE 8080 +# filer server gprc port +EXPOSE 18888 +# filer server http port +EXPOSE 8888 +# master server shared gprc port +EXPOSE 19333 +# master server shared http port +EXPOSE 9333 +# s3 server http port +EXPOSE 8333 +# webdav server http port +EXPOSE 7333 + +# Create data directory and set proper ownership for seaweed user +RUN mkdir -p /data && \ + chown -R seaweed:seaweed /data && \ + chown -R seaweed:seaweed /etc/seaweedfs && \ + chmod 755 /entrypoint.sh + +VOLUME /data + +WORKDIR /data + +# Switch to non-root user +USER seaweed + +ENTRYPOINT ["/entrypoint.sh"] + diff --git a/docker/Dockerfile.go_build b/docker/Dockerfile.go_build new file mode 100644 index 0000000..227f58c --- /dev/null +++ b/docker/Dockerfile.go_build @@ -0,0 +1,120 @@ +# Pin the builder to the host arch and cross-compile the (CGO-free) Go binary, +# so arm64/arm/386 targets skip QEMU emulation of the whole compile. +FROM --platform=$BUILDPLATFORM golang:1.26-alpine AS builder +RUN apk add git g++ fuse +RUN mkdir -p /go/src/github.com/seaweedfs/ +ARG BRANCH=${BRANCH:-master} +# Clone with full history and all tags to ensure all commits are available +RUN git clone --no-single-branch --tags https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs +ARG TAGS +RUN cd /go/src/github.com/seaweedfs/seaweedfs && \ + (git checkout $BRANCH || \ + (echo "Checkout failed, fetching all history..." && \ + git fetch --all --tags --prune && \ + git checkout $BRANCH) || \ + (echo "ERROR: Branch/commit $BRANCH not found in repository" && \ + echo "Available branches:" && git branch -a && exit 1)) +ARG TARGETOS TARGETARCH TARGETVARIANT +RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \ + && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \ + && export GOOS=$TARGETOS GOARCH=$TARGETARCH \ + && case "$TARGETARCH" in arm) export GOARM="${TARGETVARIANT#v}";; esac \ + && CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}" -o /go/bin/weed . + +# Rust volume server: use pre-built binary from CI when available (placed in +# weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile +# from source. Pre-building avoids a multi-hour QEMU-emulated cargo build +# for non-native architectures. +FROM alpine:3.23 as rust_builder +ARG TARGETARCH +ARG TAGS +COPY weed-volume-prebuilt/ /prebuilt/ +COPY weed-worker-prebuilt/ /prebuilt-worker/ +COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume +COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed +WORKDIR /build/seaweed-volume +RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \ + echo "Using pre-built Rust binary for ${TARGETARCH}" && \ + cp "/prebuilt/weed-volume-${TARGETARCH}" /weed-volume; \ + elif [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \ + apk add --no-cache musl-dev openssl-dev protobuf-dev git rust cargo; \ + if [ "$TAGS" = "5BytesOffset" ]; then \ + cargo build --release; \ + else \ + cargo build --release --no-default-features; \ + fi && \ + cp target/release/weed-volume /weed-volume; \ + else \ + echo "Skipping Rust build for $TARGETARCH (unsupported)" && \ + touch /weed-volume; \ + fi +# The Rust maintenance worker is taken pre-built or not at all: the lance jobs +# it carries pull in arrow and datafusion, a far larger dependency tree than +# the image build can carry, so an architecture CI did not build for gets the +# same empty placeholder the entrypoint refuses to exec. +RUN if [ -f "/prebuilt-worker/weed-worker-${TARGETARCH}" ]; then \ + echo "Using pre-built Rust worker for ${TARGETARCH}" && \ + cp "/prebuilt-worker/weed-worker-${TARGETARCH}" /weed-worker; \ + else \ + echo "No pre-built Rust worker for ${TARGETARCH}" && \ + touch /weed-worker; \ + fi + +# Pre-built binaries arrive via GitHub Actions artifacts, which drop the +# executable bit, so the copied file is 0644 and exec fails with "Permission +# denied". Restore it (no-op for the empty placeholders, which stay size 0). +RUN chmod 0755 /weed-volume /weed-worker + +FROM alpine AS final +LABEL author="Chris Lu" +COPY --from=builder /go/bin/weed /usr/bin/ +# Copy Rust volume server binary (real binary on amd64/arm64, empty placeholder on other platforms) +COPY --from=rust_builder /weed-volume /usr/bin/weed-volume +# Same for the Rust maintenance worker, which serves Lance table buckets +COPY --from=rust_builder /weed-worker /usr/bin/weed-worker +RUN mkdir -p /etc/seaweedfs +COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml +COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh + +# FIPS 140-3 mode is ON by default (Go 1.24+) +# To disable: docker run -e GODEBUG=fips140=off ... + +# Install dependencies and create non-root user +RUN apk upgrade --no-cache && \ + apk add --no-cache fuse curl su-exec libgcc libcrypto3 libssl3 && \ + addgroup -g 1000 seaweed && \ + adduser -D -u 1000 -G seaweed seaweed + +# volume server gprc port +EXPOSE 18080 +# volume server http port +EXPOSE 8080 +# filer server gprc port +EXPOSE 18888 +# filer server http port +EXPOSE 8888 +# master server shared gprc port +EXPOSE 19333 +# master server shared http port +EXPOSE 9333 +# s3 server http port +EXPOSE 8333 +# webdav server http port +EXPOSE 7333 + +# Create data directory and set proper ownership for seaweed user +RUN mkdir -p /data/filerldb2 && \ + chown -R seaweed:seaweed /data && \ + chown -R seaweed:seaweed /etc/seaweedfs && \ + chmod 755 /entrypoint.sh + +VOLUME /data +WORKDIR /data + +# Entrypoint will handle permission fixes and user switching +ENTRYPOINT ["/entrypoint.sh"] +# Default to a complete single-process cluster (master+volume+filer+S3+admin) +# so the image is usable out of the box — including in environments like +# GitHub Actions service containers that cannot pass arguments to the entrypoint. +# Override with any other subcommand at `docker run` / compose time. +CMD ["mini", "-dir=/data"] diff --git a/docker/Dockerfile.local b/docker/Dockerfile.local new file mode 100644 index 0000000..a275cc6 --- /dev/null +++ b/docker/Dockerfile.local @@ -0,0 +1,47 @@ +FROM alpine AS final +LABEL author="Chris Lu" +COPY ./weed /usr/bin/weed +RUN chmod +x /usr/bin/weed && ls -la /usr/bin/weed +RUN mkdir -p /etc/seaweedfs +COPY ./filer.toml /etc/seaweedfs/filer.toml +COPY ./entrypoint.sh /entrypoint.sh + +# Install dependencies and create non-root user +RUN apk upgrade --no-cache && \ + apk add --no-cache fuse curl su-exec && \ + addgroup -g 1000 seaweed && \ + adduser -D -u 1000 -G seaweed seaweed + +# volume server grpc port +EXPOSE 18080 +# volume server http port +EXPOSE 8080 +# filer server grpc port +EXPOSE 18888 +# filer server http port +EXPOSE 8888 +# master server shared grpc port +EXPOSE 19333 +# master server shared http port +EXPOSE 9333 +# s3 server http port +EXPOSE 8333 +# webdav server http port +EXPOSE 7333 + +# Create data directory and set proper ownership for seaweed user +RUN mkdir -p /data/filerldb2 && \ + chown -R seaweed:seaweed /data && \ + chown -R seaweed:seaweed /etc/seaweedfs && \ + chmod 755 /entrypoint.sh + +VOLUME /data +WORKDIR /data + +# Entrypoint will handle permission fixes and user switching +ENTRYPOINT ["/entrypoint.sh"] +# Default to a complete single-process cluster (master+volume+filer+S3+admin) +# so the image is usable out of the box — including in environments like +# GitHub Actions service containers that cannot pass arguments to the entrypoint. +# Override with any other subcommand at `docker run` / compose time. +CMD ["mini", "-dir=/data"] diff --git a/docker/Dockerfile.rocksdb_dev_env b/docker/Dockerfile.rocksdb_dev_env new file mode 100644 index 0000000..95154c3 --- /dev/null +++ b/docker/Dockerfile.rocksdb_dev_env @@ -0,0 +1,17 @@ +FROM golang:1.26 AS builder + +RUN apt-get update +RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev + +ARG ROCKSDB_VERSION=v10.10.1 +ENV ROCKSDB_VERSION=${ROCKSDB_VERSION} + +# build RocksDB +RUN cd /tmp && \ + git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \ + cd rocksdb && \ + PORTABLE=1 make -j"$(nproc)" static_lib && \ + make install-static + +ENV CGO_CFLAGS="-I/tmp/rocksdb/include" +ENV CGO_LDFLAGS="-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd" diff --git a/docker/Dockerfile.rocksdb_large b/docker/Dockerfile.rocksdb_large new file mode 100644 index 0000000..d902c54 --- /dev/null +++ b/docker/Dockerfile.rocksdb_large @@ -0,0 +1,75 @@ +FROM golang:1.26 AS builder + +RUN apt-get update +RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev + +ARG ROCKSDB_VERSION=v10.10.1 +ENV ROCKSDB_VERSION=${ROCKSDB_VERSION} + +# build RocksDB +RUN cd /tmp && \ + git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \ + cd rocksdb && \ + PORTABLE=1 make -j"$(nproc)" static_lib && \ + make install-static + +ENV CGO_CFLAGS="-I/tmp/rocksdb/include" +ENV CGO_LDFLAGS="-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd" + +# build SeaweedFS +RUN mkdir -p /go/src/github.com/seaweedfs/ +RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs +ARG BRANCH=master +RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH +RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \ + && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \ + && go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}" + + +FROM alpine AS final +LABEL author="Chris Lu" +COPY --from=builder /go/bin/weed /usr/bin/ +RUN mkdir -p /etc/seaweedfs +COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml +COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh + +# Install dependencies and create non-root user +RUN apk upgrade --no-cache && \ + apk add --no-cache fuse snappy gflags curl su-exec && \ + addgroup -g 1000 seaweed && \ + adduser -D -u 1000 -G seaweed seaweed + +# volume server gprc port +EXPOSE 18080 +# volume server http port +EXPOSE 8080 +# filer server gprc port +EXPOSE 18888 +# filer server http port +EXPOSE 8888 +# master server shared gprc port +EXPOSE 19333 +# master server shared http port +EXPOSE 9333 +# s3 server http port +EXPOSE 8333 +# webdav server http port +EXPOSE 7333 + +# Create data directory and set proper ownership for seaweed user +RUN mkdir -p /data/filer_rocksdb && \ + chown -R seaweed:seaweed /data && \ + chown -R seaweed:seaweed /etc/seaweedfs && \ + chmod 755 /entrypoint.sh + +VOLUME /data + +WORKDIR /data + +# Entrypoint will handle permission fixes and user switching +ENTRYPOINT ["/entrypoint.sh"] +# Default to a complete single-process cluster (master+volume+filer+S3+admin) +# so the image is usable out of the box — including in environments like +# GitHub Actions service containers that cannot pass arguments to the entrypoint. +# Override with any other subcommand at `docker run` / compose time. +CMD ["mini", "-dir=/data"] diff --git a/docker/Dockerfile.rocksdb_large_local b/docker/Dockerfile.rocksdb_large_local new file mode 100644 index 0000000..007a6a8 --- /dev/null +++ b/docker/Dockerfile.rocksdb_large_local @@ -0,0 +1,53 @@ +FROM chrislusf/rocksdb_dev_env as builder + +# build SeaweedFS +RUN mkdir -p /go/src/github.com/seaweedfs/ +ADD . /go/src/github.com/seaweedfs/seaweedfs +RUN ls -al /go/src/github.com/seaweedfs/ && \ + cd /go/src/github.com/seaweedfs/seaweedfs/weed \ + && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \ + && go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}" + + +FROM alpine AS final +LABEL author="Chris Lu" +COPY --from=builder /go/bin/weed /usr/bin/ +RUN mkdir -p /etc/seaweedfs +COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml +COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh + +# Install dependencies and create non-root user +RUN apk upgrade --no-cache && \ + apk add --no-cache fuse snappy gflags curl tmux su-exec && \ + addgroup -g 1000 seaweed && \ + adduser -D -u 1000 -G seaweed seaweed + +# volume server gprc port +EXPOSE 18080 +# volume server http port +EXPOSE 8080 +# filer server gprc port +EXPOSE 18888 +# filer server http port +EXPOSE 8888 +# master server shared gprc port +EXPOSE 19333 +# master server shared http port +EXPOSE 9333 +# s3 server http port +EXPOSE 8333 +# webdav server http port +EXPOSE 7333 + +# Create data directory and set proper ownership for seaweed user +RUN mkdir -p /data/filer_rocksdb && \ + chown -R seaweed:seaweed /data && \ + chown -R seaweed:seaweed /etc/seaweedfs && \ + chmod 755 /entrypoint.sh + +VOLUME /data + +WORKDIR /data + +# Entrypoint will handle permission fixes and user switching +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/Dockerfile.s3tests b/docker/Dockerfile.s3tests new file mode 100644 index 0000000..75c7f5f --- /dev/null +++ b/docker/Dockerfile.s3tests @@ -0,0 +1,31 @@ +FROM ubuntu:20.04 + +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + git \ + sudo \ + debianutils \ + python3-pip \ + python3-virtualenv \ + python3-dev \ + libevent-dev \ + libffi-dev \ + libxml2-dev \ + libxslt-dev \ + zlib1g-dev && \ + DEBIAN_FRONTEND=noninteractive apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + git clone https://github.com/ceph/s3-tests.git /opt/s3-tests + +WORKDIR /opt/s3-tests +RUN ./bootstrap + +ENV \ + NOSETESTS_EXCLUDE="" \ + NOSETESTS_ATTR="" \ + NOSETESTS_OPTIONS="" \ + S3TEST_CONF="/s3tests.conf" + +ENTRYPOINT ["/bin/bash", "-c"] +CMD ["sleep 30 && exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"] diff --git a/docker/Dockerfile.tarantool.dev_env b/docker/Dockerfile.tarantool.dev_env new file mode 100644 index 0000000..d9fad40 --- /dev/null +++ b/docker/Dockerfile.tarantool.dev_env @@ -0,0 +1,23 @@ +FROM tarantool/tarantool:3.3.1 AS builder + +# install dependencies +RUN apt update && \ + apt install -y git unzip cmake tt=2.7.0 + +# init tt dir structure, create dir for app, create symlink +RUN tt init && \ + mkdir app && \ + ln -sfn ${PWD}/app/ ${PWD}/instances.enabled/app + +# copy cluster configs +COPY tarantool /opt/tarantool/app + +# build app +RUN tt build app + +# the base image's default healthcheck assumes a single-instance layout and +# checks a control socket path that doesn't exist for this multi-instance +# app, so it never reports healthy; check instance status instead. +HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=6 \ + CMD tt status app | awk 'NR>2{if ($2!="RUNNING") exit 1} END{if (NR<=2) exit 1}' + diff --git a/docker/Makefile b/docker/Makefile new file mode 100644 index 0000000..d803c5a --- /dev/null +++ b/docker/Makefile @@ -0,0 +1,146 @@ +all: gen + +.PHONY : gen + +gen: dev + +cgo ?= 0 +ldflags_extra ?= -extldflags -static + +binary: + export SWCOMMIT=$(shell git rev-parse --short HEAD) + export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(SWCOMMIT)" + cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-s -w $(ldflags_extra) $(SWLDFLAGS)" -o weed_binary && mv weed_binary ../docker/weed + cd ../other/mq_client_example/agent_pub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv agent_pub_record ../../../docker/ + cd ../other/mq_client_example/agent_sub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv agent_sub_record ../../../docker/ + +# Race detector requires CGO and dynamic linking - don't use -static +binary_race: options = -race +binary_race: cgo = 1 +binary_race: ldflags_extra = +binary_race: binary + +build: binary + docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local . + +build_e2e: binary_race + docker buildx build \ + --cache-from=type=local,src=/tmp/.buildx-cache \ + --cache-to=type=local,dest=/tmp/.buildx-cache-new,mode=max \ + --load \ + -t chrislusf/seaweedfs:e2e \ + -f Dockerfile.e2e . + # Move cache to avoid growing cache size + rm -rf /tmp/.buildx-cache || true + mv /tmp/.buildx-cache-new /tmp/.buildx-cache || true + +go_build: # make go_build tags=elastic,ydb,gocdk,hdfs,5BytesOffset,tarantool + docker build --build-arg TAGS=$(tags) --no-cache -t chrislusf/seaweedfs:go_build -f Dockerfile.go_build . + +go_build_large_disk: + docker build --build-arg TAGS=large_disk --no-cache -t chrislusf/seaweedfs:large_disk -f Dockerfile.go_build . + +build_rocksdb_dev_env: + docker build --no-cache -t chrislusf/rocksdb_dev_env -f Dockerfile.rocksdb_dev_env . + +build_rocksdb_local: build_rocksdb_dev_env + cd .. ; docker build --no-cache -t chrislusf/seaweedfs:rocksdb_local -f docker/Dockerfile.rocksdb_large_local . + +build_rocksdb: + docker build --no-cache -t chrislusf/seaweedfs:rocksdb -f Dockerfile.rocksdb_large . + +build_tarantool_dev_env: + docker build --no-cache -t chrislusf/tarantool_dev_env -f Dockerfile.tarantool.dev_env . + +s3tests_build: + docker build --no-cache -t chrislusf/ceph-s3-tests:local -f Dockerfile.s3tests . + +dev: build + docker compose -f compose/local-dev-compose.yml -p seaweedfs up + +dev_race: binary_race + docker compose -f compose/local-dev-compose.yml -p seaweedfs up + +dev_tls: build certstrap + ENV_FILE="tls.env" docker compose -f compose/local-dev-compose.yml -p seaweedfs up + +dev_mount: build + docker compose -f compose/local-mount-compose.yml -p seaweedfs up + +run_image: build + docker run --rm -ti --device /dev/fuse --cap-add SYS_ADMIN --entrypoint /bin/sh chrislusf/seaweedfs:local + +profile_mount: build + docker compose -f compose/local-mount-profile-compose.yml -p seaweedfs up + +k8s: build + docker compose -f compose/local-k8s-compose.yml -p seaweedfs up + +dev_registry: build + docker compose -f compose/local-registry-compose.yml -p seaweedfs up + +dev_replicate: + docker build --build-arg TAGS=gocdk --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.go_build . + docker compose -f compose/local-replicate-compose.yml -p seaweedfs up + +dev_auditlog: build + docker compose -f compose/local-auditlog-compose.yml -p seaweedfs up + +dev_nextcloud: build + docker compose -f compose/local-nextcloud-compose.yml -p seaweedfs up + +cluster: build + docker compose -f compose/local-cluster-compose.yml -p seaweedfs up + +2clusters: build + docker compose -f compose/local-clusters-compose.yml -p seaweedfs up + +2mount: build + docker compose -f compose/local-sync-mount-compose.yml -p seaweedfs up + +filer_backup: build + docker compose -f compose/local-filer-backup-compose.yml -p seaweedfs up + +hashicorp_raft: build + docker compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up + +s3tests: build s3tests_build + docker compose -f compose/local-s3tests-compose.yml -p seaweedfs up + +brokers: build + docker compose -f compose/local-brokers-compose.yml -p seaweedfs up + +agent: build + docker compose -f compose/local-mq-test.yml -p seaweedfs up + +filer_etcd: build + docker stack deploy -c compose/swarm-etcd.yml fs + +test_etcd: build + docker compose -f compose/test-etcd-filer.yml -p seaweedfs up + +test_ydb: tags = ydb +test_ydb: build + docker compose -f compose/test-ydb-filer.yml -p seaweedfs up + +test_tarantool: tags = tarantool +test_tarantool: build_tarantool_dev_env build + docker compose -f compose/test-tarantool-filer.yml -p seaweedfs up + +test_keycloak_s3: build + docker compose -f compose/test-keycloak-s3.yml -p seaweedfs up + +clean: + rm ./weed + +certstrap: + go install -v github.com/square/certstrap@latest + certstrap --depot-path compose/tls init --curve P-256 --passphrase "" --common-name "SeaweedFS CA" || true + certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --domain localhost --common-name volume01.dev || true + certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name master01.dev || true + certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name filer01.dev || true + certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name client01.dev || true + certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" volume01.dev || true + certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" master01.dev || true + certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" filer01.dev || true + certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" client01.dev || true diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..731587c --- /dev/null +++ b/docker/README.md @@ -0,0 +1,104 @@ +# Docker + +## Compose V2 +SeaweedFS now uses the `v2` syntax `docker compose` + +If you rely on using Docker Compose as docker-compose (with a hyphen), you can set up Compose V2 to act as a drop-in replacement of the previous docker-compose. Refer to the [Installing Compose](https://docs.docker.com/compose/install/) section for detailed instructions on upgrading. + +Confirm your system has docker compose v2 with a version check +```bash +$ docker compose version +Docker Compose version v2.10.2 +``` + +## Try it out + +```bash + +wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-compose.yml + +docker compose -f seaweedfs-compose.yml -p seaweedfs up + +``` + +## Try latest tip + +```bash + +wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-dev-compose.yml + +docker compose -f seaweedfs-dev-compose.yml -p seaweedfs up + +``` + +## Verify an image signature + +Every image CI pushes to `chrislusf/seaweedfs` and `ghcr.io/chrislusf/seaweedfs` is signed with [cosign](https://docs.sigstore.dev/cosign/verifying/verify/), keyless, by the GitHub Actions workflow that built it, so there is no key to fetch or pin. The signature is attached to the image digest and covers the multi-arch index and each platform image in it; `latest` is the release image under another tag and verifies the same way. Images published before September 2026 predate signing. + +```bash +cosign verify \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity-regexp '^https://github.com/seaweedfs/seaweedfs/\.github/workflows/container_release_unified\.yml@' \ + chrislusf/seaweedfs:latest +``` + +cosign prints the digest it verified. Deploy by that digest, or let an admission controller resolve the tag, so what runs is what was checked. + +The identity is `https://github.com/seaweedfs/seaweedfs/.github/workflows/@`. The ref is `refs/tags/` for a release and `refs/heads/master` when a variant was republished by hand. The workflow is `container_release_unified.yml` for the release images, `container_dev.yml` for `dev`, `container_latest.yml` for a `latest` rebuilt by hand, `container_release_foundationdb.yml` for the `_large_disk_foundationdb` release image, and `container_foundationdb_version.yml` or `container_rocksdb_version.yml` for the per-version builds. The regexp above accepts release images only; `container_[a-z_]+\.yml@` accepts everything this repository publishes, `dev` included. + +The same check as a Kyverno policy, release images only: + +```yaml +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: verify-seaweedfs-images +spec: + validationFailureAction: Enforce + webhookTimeoutSeconds: 30 + rules: + - name: signed-by-the-release-workflow + match: + any: + - resources: + kinds: + - Pod + verifyImages: + - imageReferences: + - "docker.io/chrislusf/seaweedfs:*" + - "ghcr.io/chrislusf/seaweedfs:*" + attestors: + - entries: + - keyless: + issuer: https://token.actions.githubusercontent.com + subject: https://github.com/seaweedfs/seaweedfs/.github/workflows/container_release_unified.yml@refs/tags/* +``` + +## Local Development + +```bash +cd $GOPATH/src/github.com/seaweedfs/seaweedfs/docker +make +``` + +### S3 cmd + +list +``` +s3cmd --no-ssl --host=127.0.0.1:8333 ls s3:// +``` + +## Build and push a multiarch build + +Make sure that `docker buildx` is supported (might be an experimental docker feature) +```bash +BUILDER=$(docker buildx create --driver docker-container --use) +docker buildx build --pull --push --platform linux/386,linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 . -t chrislusf/seaweedfs +docker buildx stop $BUILDER +``` + +## Minio debugging +``` +mc config host add local http://127.0.0.1:9000 some_access_key1 some_secret_key1 +mc admin trace --all --verbose local +``` diff --git a/docker/admin_integration/Dockerfile.local b/docker/admin_integration/Dockerfile.local new file mode 100644 index 0000000..9795b6e --- /dev/null +++ b/docker/admin_integration/Dockerfile.local @@ -0,0 +1,18 @@ +FROM alpine:latest + +# Install required packages +RUN apk add --no-cache \ + ca-certificates \ + fuse \ + curl \ + jq + +# Copy our locally built binary +COPY weed-local /usr/bin/weed +RUN chmod +x /usr/bin/weed + +# Create working directory +WORKDIR /data + +# Default command +ENTRYPOINT ["/usr/bin/weed"] \ No newline at end of file diff --git a/docker/admin_integration/EC-TESTING-README.md b/docker/admin_integration/EC-TESTING-README.md new file mode 100644 index 0000000..57e0a59 --- /dev/null +++ b/docker/admin_integration/EC-TESTING-README.md @@ -0,0 +1,438 @@ +# SeaweedFS EC Worker Testing Environment + +This Docker Compose setup provides a comprehensive testing environment for SeaweedFS Erasure Coding (EC) workers using **official SeaweedFS commands**. + +## 📂 Directory Structure + +The testing environment is located in `docker/admin_integration/` and includes: + +``` +docker/admin_integration/ +├── Makefile # Main management interface +├── docker-compose-ec-test.yml # Docker compose configuration +├── EC-TESTING-README.md # This documentation +└── run-ec-test.sh # Quick start script +``` + +## 🏗ī¸ Architecture + +The testing environment uses **official SeaweedFS commands** and includes: + +- **1 Master Server** (port 9333) - Coordinates the cluster with 50MB volume size limit +- **6 Volume Servers** (ports 8080-8085) - Distributed across 2 data centers and 3 racks for diversity +- **1 Filer** (port 8888) - Provides file system interface +- **1 Admin Server** (port 23646) - Detects volumes needing EC and manages workers using official `admin` command +- **3 EC Workers** - Execute erasure coding tasks using official `worker` command with task-specific working directories +- **1 Load Generator** - Continuously writes and deletes files using SeaweedFS shell commands +- **1 Monitor** - Tracks cluster health and EC progress using shell scripts + +## ✨ New Features + +### **Task-Specific Working Directories** +Each worker now creates dedicated subdirectories for different task types: +- `/work/erasure_coding/` - For EC encoding tasks +- `/work/vacuum/` - For vacuum cleanup tasks +- `/work/balance/` - For volume balancing tasks + +This provides: +- **Organization**: Each task type gets isolated working space +- **Debugging**: Easy to find files/logs related to specific task types +- **Cleanup**: Can clean up task-specific artifacts easily +- **Concurrent Safety**: Different task types won't interfere with each other's files + +## 🚀 Quick Start + +### Prerequisites + +- Docker and Docker Compose installed +- GNU Make installed +- At least 4GB RAM available for containers +- Ports 8080-8085, 8888, 9333, 23646 available + +### Start the Environment + +```bash +# Navigate to the admin integration directory +cd docker/admin_integration/ + +# Show available commands +make help + +# Start the complete testing environment +make start +``` + +The `make start` command will: +1. Start all services using official SeaweedFS images +2. Configure workers with task-specific working directories +3. Wait for services to be ready +4. Display monitoring URLs and run health checks + +### Alternative Commands + +```bash +# Quick start aliases +make up # Same as 'make start' + +# Development mode (higher load for faster testing) +make dev-start + +# Build images without starting +make build +``` + +## 📋 Available Make Targets + +Run `make help` to see all available targets: + +### **🚀 Main Operations** +- `make start` - Start the complete EC testing environment +- `make stop` - Stop all services +- `make restart` - Restart all services +- `make clean` - Complete cleanup (containers, volumes, images) + +### **📊 Monitoring & Status** +- `make health` - Check health of all services +- `make status` - Show status of all containers +- `make urls` - Display all monitoring URLs +- `make monitor` - Open monitor dashboard in browser +- `make monitor-status` - Show monitor status via API +- `make volume-status` - Show volume status from master +- `make admin-status` - Show admin server status +- `make cluster-status` - Show complete cluster status + +### **📋 Logs Management** +- `make logs` - Show logs from all services +- `make logs-admin` - Show admin server logs +- `make logs-workers` - Show all worker logs +- `make logs-worker1/2/3` - Show specific worker logs +- `make logs-load` - Show load generator logs +- `make logs-monitor` - Show monitor logs +- `make backup-logs` - Backup all logs to files + +### **⚖ī¸ Scaling & Testing** +- `make scale-workers WORKERS=5` - Scale workers to 5 instances +- `make scale-load RATE=25` - Increase load generation rate +- `make test-ec` - Run focused EC test scenario + +### **🔧 Development & Debug** +- `make shell-admin` - Open shell in admin container +- `make shell-worker1` - Open shell in worker container +- `make debug` - Show debug information +- `make troubleshoot` - Run troubleshooting checks + +## 📊 Monitoring URLs + +| Service | URL | Description | +|---------|-----|-------------| +| Master UI | http://localhost:9333 | Cluster status and topology | +| Filer | http://localhost:8888 | File operations | +| Admin Server | http://localhost:23646/ | Task management | +| Monitor | http://localhost:9999/status | Complete cluster monitoring | +| Volume Servers | http://localhost:8080-8085/status | Individual volume server stats | + +Quick access: `make urls` or `make monitor` + +## 🔄 How EC Testing Works + +### 1. Continuous Load Generation +- **Write Rate**: 10 files/second (1-5MB each) +- **Delete Rate**: 2 files/second +- **Target**: Fill volumes to 50MB limit quickly + +### 2. Volume Detection +- Admin server scans master every 30 seconds +- Identifies volumes >40MB (80% of 50MB limit) +- Queues EC tasks for eligible volumes + +### 3. EC Worker Assignment +- **Worker 1**: EC specialist (max 2 concurrent tasks) +- **Worker 2**: EC + Vacuum hybrid (max 2 concurrent tasks) +- **Worker 3**: EC + Vacuum hybrid (max 1 concurrent task) + +### 4. Comprehensive EC Process +Each EC task follows 6 phases: +1. **Copy Volume Data** (5-15%) - Stream .dat/.idx files locally +2. **Mark Read-Only** (20-25%) - Ensure data consistency +3. **Local Encoding** (30-60%) - Create 14 shards (10+4 Reed-Solomon) +4. **Calculate Placement** (65-70%) - Smart rack-aware distribution +5. **Distribute Shards** (75-90%) - Upload to optimal servers +6. **Verify & Cleanup** (95-100%) - Validate and clean temporary files + +### 5. Real-Time Monitoring +- Volume analysis and EC candidate detection +- Worker health and task progress +- No data loss verification +- Performance metrics + +## 📋 Key Features Tested + +### ✅ EC Implementation Features +- [x] Local volume data copying with progress tracking +- [x] Local Reed-Solomon encoding (10+4 shards) +- [x] Intelligent shard placement with rack awareness +- [x] Load balancing across available servers +- [x] Backup server selection for redundancy +- [x] Detailed step-by-step progress tracking +- [x] Comprehensive error handling and recovery + +### ✅ Infrastructure Features +- [x] Multi-datacenter topology (dc1, dc2) +- [x] Rack diversity (rack1, rack2, rack3) +- [x] Volume size limits (50MB) +- [x] Worker capability matching +- [x] Health monitoring and alerting +- [x] Continuous workload simulation + +## 🛠ī¸ Common Usage Patterns + +### Basic Testing Workflow +```bash +# Start environment +make start + +# Watch progress +make monitor-status + +# Check for EC candidates +make volume-status + +# View worker activity +make logs-workers + +# Stop when done +make stop +``` + +### High-Load Testing +```bash +# Start with higher load +make dev-start + +# Scale up workers and load +make scale-workers WORKERS=5 +make scale-load RATE=50 + +# Monitor intensive EC activity +make logs-admin +``` + +### Debugging Issues +```bash +# Check port conflicts and system state +make troubleshoot + +# View specific service logs +make logs-admin +make logs-worker1 + +# Get shell access for debugging +make shell-admin +make shell-worker1 + +# Check detailed status +make debug +``` + +### Development Iteration +```bash +# Quick restart after code changes +make restart + +# Rebuild and restart +make clean +make start + +# Monitor specific components +make logs-monitor +``` + +## 📈 Expected Results + +### Successful EC Testing Shows: +1. **Volume Growth**: Steady increase in volume sizes toward 50MB limit +2. **EC Detection**: Admin server identifies volumes >40MB for EC +3. **Task Assignment**: Workers receive and execute EC tasks +4. **Shard Distribution**: 14 shards distributed across 6 volume servers +5. **No Data Loss**: All files remain accessible during and after EC +6. **Performance**: EC tasks complete within estimated timeframes + +### Sample Monitor Output: +```bash +# Check current status +make monitor-status + +# Output example: +{ + "monitor": { + "uptime": "15m30s", + "master_addr": "master:9333", + "admin_addr": "admin:9900" + }, + "stats": { + "VolumeCount": 12, + "ECTasksDetected": 3, + "WorkersActive": 3 + } +} +``` + +## 🔧 Configuration + +### Environment Variables + +You can customize the environment by setting variables: + +```bash +# High load testing +WRITE_RATE=25 DELETE_RATE=5 make start + +# Extended test duration +TEST_DURATION=7200 make start # 2 hours +``` + +### Scaling Examples + +```bash +# Scale workers +make scale-workers WORKERS=6 + +# Increase load generation +make scale-load RATE=30 + +# Combined scaling +make scale-workers WORKERS=4 +make scale-load RATE=40 +``` + +## 🧹 Cleanup Options + +```bash +# Stop services only +make stop + +# Remove containers but keep volumes +make down + +# Remove data volumes only +make clean-volumes + +# Remove built images only +make clean-images + +# Complete cleanup (everything) +make clean +``` + +## 🐛 Troubleshooting + +### Quick Diagnostics +```bash +# Run complete troubleshooting +make troubleshoot + +# Check specific components +make health +make debug +make status +``` + +### Common Issues + +**Services not starting:** +```bash +# Check port availability +make troubleshoot + +# View startup logs +make logs-master +make logs-admin +``` + +**No EC tasks being created:** +```bash +# Check volume status +make volume-status + +# Increase load to fill volumes faster +make scale-load RATE=30 + +# Check admin detection +make logs-admin +``` + +**Workers not responding:** +```bash +# Check worker registration +make admin-status + +# View worker logs +make logs-workers + +# Restart workers +make restart +``` + +### Performance Tuning + +**For faster testing:** +```bash +make dev-start # Higher default load +make scale-load RATE=50 # Very high load +``` + +**For stress testing:** +```bash +make scale-workers WORKERS=8 +make scale-load RATE=100 +``` + +## 📚 Technical Details + +### Network Architecture +- Custom bridge network (172.20.0.0/16) +- Service discovery via container names +- Health checks for all services + +### Storage Layout +- Each volume server: max 100 volumes +- Data centers: dc1, dc2 +- Racks: rack1, rack2, rack3 +- Volume limit: 50MB per volume + +### EC Algorithm +- Reed-Solomon RS(10,4) +- 10 data shards + 4 parity shards +- Rack-aware distribution +- Backup server redundancy + +### Make Integration +- Color-coded output for better readability +- Comprehensive help system (`make help`) +- Parallel execution support +- Error handling and cleanup +- Cross-platform compatibility + +## đŸŽ¯ Quick Reference + +```bash +# Essential commands +make help # Show all available targets +make start # Start complete environment +make health # Check all services +make monitor # Open dashboard +make logs-admin # View admin activity +make clean # Complete cleanup + +# Monitoring +make volume-status # Check for EC candidates +make admin-status # Check task queue +make monitor-status # Full cluster status + +# Scaling & Testing +make test-ec # Run focused EC test +make scale-load RATE=X # Increase load +make troubleshoot # Diagnose issues +``` + +This environment provides a realistic testing scenario for SeaweedFS EC workers with actual data operations, comprehensive monitoring, and easy management through Make targets. \ No newline at end of file diff --git a/docker/admin_integration/Makefile b/docker/admin_integration/Makefile new file mode 100644 index 0000000..68fb0ce --- /dev/null +++ b/docker/admin_integration/Makefile @@ -0,0 +1,346 @@ +# SeaweedFS Admin Integration Test Makefile +# Tests the admin server and worker functionality using official weed commands + +.PHONY: help build build-and-restart restart-workers start stop restart logs clean status test admin-ui worker-logs master-logs admin-logs vacuum-test vacuum-demo vacuum-status vacuum-data vacuum-data-high vacuum-data-low vacuum-continuous vacuum-clean vacuum-help +.DEFAULT_GOAL := help + +COMPOSE_FILE := docker-compose-ec-test.yml +PROJECT_NAME := admin_integration + +build: ## Build SeaweedFS with latest changes and create Docker image + @echo "🔨 Building SeaweedFS with latest changes..." + @echo "1ī¸âƒŖ Generating admin templates..." + @cd ../../ && make admin-generate + @echo "2ī¸âƒŖ Building Docker image with latest changes..." + @cd ../ && make build + @echo "3ī¸âƒŖ Copying binary for local docker-compose..." + @cp ../weed ./weed-local + @echo "✅ Build complete! Updated image: chrislusf/seaweedfs:local" + @echo "💡 Run 'make restart' to apply changes to running services" + +build-and-restart: build ## Build with latest changes and restart services + @echo "🔄 Recreating services with new image..." + @echo "1ī¸âƒŖ Recreating admin server with new image..." + @docker-compose -f $(COMPOSE_FILE) up -d admin + @sleep 5 + @echo "2ī¸âƒŖ Recreating workers to reconnect..." + @docker-compose -f $(COMPOSE_FILE) up -d worker1 worker2 worker3 + @echo "✅ All services recreated with latest changes!" + @echo "🌐 Admin UI: http://localhost:23646/" + @echo "💡 Workers will reconnect to the new admin server" + +restart-workers: ## Restart all workers to reconnect to admin server + @echo "🔄 Restarting workers to reconnect to admin server..." + @docker-compose -f $(COMPOSE_FILE) restart worker1 worker2 worker3 + @echo "✅ Workers restarted and will reconnect to admin server" + +help: ## Show this help message + @echo "SeaweedFS Admin Integration Test" + @echo "================================" + @echo "Tests admin server task distribution to workers using official weed commands" + @echo "" + @echo "🏗ī¸ Cluster Management:" + @grep -E '^(start|stop|restart|clean|status|build):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}' + @echo "" + @echo "đŸ§Ē Testing:" + @grep -E '^(test|demo|validate|quick-test):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}' + @echo "" + @echo "🗑ī¸ Vacuum Testing:" + @grep -E '^vacuum-.*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}' + @echo "" + @echo "📜 Monitoring:" + @grep -E '^(logs|admin-logs|worker-logs|master-logs|admin-ui):.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-18s %s\n", $$1, $$2}' + @echo "" + @echo "🚀 Quick Start:" + @echo " make start # Start cluster" + @echo " make vacuum-test # Test vacuum tasks" + @echo " make vacuum-help # Vacuum testing guide" + @echo "" + @echo "💡 For detailed vacuum testing: make vacuum-help" + +start: ## Start the complete SeaweedFS cluster with admin and workers + @echo "🚀 Starting SeaweedFS cluster with admin and workers..." + @docker-compose -f $(COMPOSE_FILE) up -d + @echo "✅ Cluster started!" + @echo "" + @echo "📊 Access points:" + @echo " â€ĸ Admin UI: http://localhost:23646/" + @echo " â€ĸ Master UI: http://localhost:9333/" + @echo " â€ĸ Filer: http://localhost:8888/" + @echo "" + @echo "📈 Services starting up..." + @echo " â€ĸ Master server: ✓" + @echo " â€ĸ Volume servers: Starting (6 servers)..." + @echo " â€ĸ Filer: Starting..." + @echo " â€ĸ Admin server: Starting..." + @echo " â€ĸ Workers: Starting (3 workers)..." + @echo "" + @echo "âŗ Use 'make status' to check startup progress" + @echo "💡 Use 'make logs' to watch the startup process" + +start-staged: ## Start services in proper order with delays + @echo "🚀 Starting SeaweedFS cluster in stages..." + @echo "" + @echo "Stage 1: Starting Master server..." + @docker-compose -f $(COMPOSE_FILE) up -d master + @sleep 10 + @echo "" + @echo "Stage 2: Starting Volume servers..." + @docker-compose -f $(COMPOSE_FILE) up -d volume1 volume2 volume3 volume4 volume5 volume6 + @sleep 15 + @echo "" + @echo "Stage 3: Starting Filer..." + @docker-compose -f $(COMPOSE_FILE) up -d filer + @sleep 10 + @echo "" + @echo "Stage 4: Starting Admin server..." + @docker-compose -f $(COMPOSE_FILE) up -d admin + @sleep 15 + @echo "" + @echo "Stage 5: Starting Workers..." + @docker-compose -f $(COMPOSE_FILE) up -d worker1 worker2 worker3 + @sleep 10 + @echo "" + @echo "Stage 6: Starting Load generator and Monitor..." + @docker-compose -f $(COMPOSE_FILE) up -d load_generator monitor + @echo "" + @echo "✅ All services started!" + @echo "" + @echo "📊 Access points:" + @echo " â€ĸ Admin UI: http://localhost:23646/" + @echo " â€ĸ Master UI: http://localhost:9333/" + @echo " â€ĸ Filer: http://localhost:8888/" + @echo "" + @echo "âŗ Services are initializing... Use 'make status' to check progress" + +stop: ## Stop all services + @echo "🛑 Stopping SeaweedFS cluster..." + @docker-compose -f $(COMPOSE_FILE) down + @echo "✅ Cluster stopped" + +restart: stop start ## Restart the entire cluster + +clean: ## Stop and remove all containers, networks, and volumes + @echo "🧹 Cleaning up SeaweedFS test environment..." + @docker-compose -f $(COMPOSE_FILE) down -v --remove-orphans + @docker system prune -f + @rm -rf data/ + @echo "✅ Environment cleaned" + +status: ## Check the status of all services + @echo "📊 SeaweedFS Cluster Status" + @echo "==========================" + @docker-compose -f $(COMPOSE_FILE) ps + @echo "" + @echo "📋 Service Health:" + @echo "Master:" + @curl -s http://localhost:9333/cluster/status | jq '.IsLeader' 2>/dev/null || echo " ❌ Master not ready" + @echo "Admin:" + @curl -s http://localhost:23646/ | grep -q "Admin" && echo " ✅ Admin ready" || echo " ❌ Admin not ready" + +logs: ## Show logs from all services + @echo "📜 Following logs from all services..." + @echo "💡 Press Ctrl+C to stop following logs" + @docker-compose -f $(COMPOSE_FILE) logs -f + +admin-logs: ## Show logs from admin server only + @echo "📜 Admin server logs:" + @docker-compose -f $(COMPOSE_FILE) logs -f admin + +worker-logs: ## Show logs from all workers + @echo "📜 Worker logs:" + @docker-compose -f $(COMPOSE_FILE) logs -f worker1 worker2 worker3 + +master-logs: ## Show logs from master server + @echo "📜 Master server logs:" + @docker-compose -f $(COMPOSE_FILE) logs -f master + +admin-ui: ## Open admin UI in browser (macOS) + @echo "🌐 Opening admin UI in browser..." + @open http://localhost:23646/ || echo "💡 Manually open: http://localhost:23646/" + +test: ## Run integration test to verify task assignment and completion + @echo "đŸ§Ē Running Admin-Worker Integration Test" + @echo "========================================" + @echo "" + @echo "1ī¸âƒŖ Checking cluster health..." + @sleep 5 + @curl -s http://localhost:9333/cluster/status | jq '.IsLeader' > /dev/null && echo "✅ Master healthy" || echo "❌ Master not ready" + @curl -s http://localhost:23646/ | grep -q "Admin" && echo "✅ Admin healthy" || echo "❌ Admin not ready" + @echo "" + @echo "2ī¸âƒŖ Checking worker registration..." + @sleep 10 + @echo "💡 Check admin UI for connected workers: http://localhost:23646/" + @echo "" + @echo "3ī¸âƒŖ Generating load to trigger EC tasks..." + @echo "📝 Creating test files to fill volumes..." + @echo "Creating large files with random data to trigger EC (targeting ~60MB total to exceed 50MB limit)..." + @for i in {1..12}; do \ + echo "Creating 5MB random file $$i..."; \ + docker run --rm --network admin_integration_seaweed_net -v /tmp:/tmp --entrypoint sh chrislusf/seaweedfs:local -c "dd if=/dev/urandom of=/tmp/largefile$$i.dat bs=1M count=5 2>/dev/null && weed upload -master=master:9333 /tmp/largefile$$i.dat && rm /tmp/largefile$$i.dat"; \ + sleep 3; \ + done + @echo "" + @echo "4ī¸âƒŖ Waiting for volumes to process large files and reach 50MB limit..." + @echo "This may take a few minutes as we're uploading 60MB of data..." + @sleep 60 + @echo "" + @echo "5ī¸âƒŖ Checking for EC task creation and assignment..." + @echo "💡 Monitor the admin UI to see:" + @echo " â€ĸ Tasks being created for volumes needing EC" + @echo " â€ĸ Workers picking up tasks" + @echo " â€ĸ Task progress (pending → running → completed)" + @echo " â€ĸ EC shards being distributed" + @echo "" + @echo "✅ Integration test setup complete!" + @echo "📊 Monitor progress at: http://localhost:23646/" + +quick-test: ## Quick verification that core services are running + @echo "⚡ Quick Health Check" + @echo "====================" + @echo "Master: $$(curl -s http://localhost:9333/cluster/status | jq -r '.IsLeader // "not ready"')" + @echo "Admin: $$(curl -s http://localhost:23646/ | grep -q "Admin" && echo "ready" || echo "not ready")" + @echo "Workers: $$(docker-compose -f $(COMPOSE_FILE) ps worker1 worker2 worker3 | grep -c Up) running" + +validate: ## Validate integration test configuration + @echo "🔍 Validating Integration Test Configuration" + @echo "===========================================" + @chmod +x test-integration.sh + @./test-integration.sh + +demo: start ## Start cluster and run demonstration + @echo "🎭 SeaweedFS Admin-Worker Demo" + @echo "=============================" + @echo "" + @echo "âŗ Waiting for services to start..." + @sleep 45 + @echo "" + @echo "đŸŽ¯ Demo Overview:" + @echo " â€ĸ 1 Master server (coordinates cluster)" + @echo " â€ĸ 6 Volume servers (50MB volume limit)" + @echo " â€ĸ 1 Admin server (task management)" + @echo " â€ĸ 3 Workers (execute EC tasks)" + @echo " â€ĸ Load generator (creates files continuously)" + @echo "" + @echo "📊 Watch the process:" + @echo " 1. Visit: http://localhost:23646/" + @echo " 2. Observe workers connecting" + @echo " 3. Watch tasks being created and assigned" + @echo " 4. See tasks progress from pending → completed" + @echo "" + @echo "🔄 The demo will:" + @echo " â€ĸ Fill volumes to 50MB limit" + @echo " â€ĸ Admin detects volumes needing EC" + @echo " â€ĸ Workers receive and execute EC tasks" + @echo " â€ĸ Tasks complete with shard distribution" + @echo "" + @echo "💡 Use 'make worker-logs' to see worker activity" + @echo "💡 Use 'make admin-logs' to see admin task management" + +# Vacuum Testing Targets +vacuum-test: ## Create test data with garbage and verify vacuum detection + @echo "đŸ§Ē SeaweedFS Vacuum Task Testing" + @echo "================================" + @echo "" + @echo "1ī¸âƒŖ Checking cluster health..." + @curl -s http://localhost:9333/cluster/status | jq '.IsLeader' > /dev/null && echo "✅ Master ready" || (echo "❌ Master not ready. Run 'make start' first." && exit 1) + @curl -s http://localhost:23646/ | grep -q "Admin" && echo "✅ Admin ready" || (echo "❌ Admin not ready. Run 'make start' first." && exit 1) + @echo "" + @echo "2ī¸âƒŖ Creating test data with garbage..." + @docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -files=25 -delete=0.5 -size=200 + @echo "" + @echo "3ī¸âƒŖ Configuration Instructions:" + @echo " Visit: http://localhost:23646/maintenance/config/vacuum" + @echo " Set for testing:" + @echo " â€ĸ Enable Vacuum Tasks: ✅ Checked" + @echo " â€ĸ Garbage Threshold: 0.20 (20%)" + @echo " â€ĸ Scan Interval: [30] [Seconds]" + @echo " â€ĸ Min Volume Age: [0] [Minutes]" + @echo " â€ĸ Max Concurrent: 2" + @echo "" + @echo "4ī¸âƒŖ Monitor vacuum tasks at: http://localhost:23646/maintenance" + @echo "" + @echo "💡 Use 'make vacuum-status' to check volume garbage ratios" + +vacuum-demo: ## Run automated vacuum testing demonstration + @echo "🎭 Vacuum Task Demo" + @echo "==================" + @echo "" + @echo "⚠ī¸ This demo requires user interaction for configuration" + @echo "💡 Make sure cluster is running with 'make start'" + @echo "" + @docker-compose -f $(COMPOSE_FILE) exec vacuum-tester sh -c "chmod +x demo_vacuum_testing.sh && ./demo_vacuum_testing.sh" + +vacuum-status: ## Check current volume status and garbage ratios + @echo "📊 Current Volume Status" + @echo "=======================" + @docker-compose -f $(COMPOSE_FILE) exec vacuum-tester sh -c "chmod +x check_volumes.sh && ./check_volumes.sh" + +vacuum-data: ## Create test data with configurable parameters + @echo "📁 Creating vacuum test data..." + @echo "Usage: make vacuum-data [FILES=20] [DELETE=0.4] [SIZE=100]" + @echo "" + @docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go \ + -files=$${FILES:-20} \ + -delete=$${DELETE:-0.4} \ + -size=$${SIZE:-100} + +vacuum-data-high: ## Create high garbage ratio test data (should trigger vacuum) + @echo "📁 Creating high garbage test data (70% garbage)..." + @docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -files=30 -delete=0.7 -size=150 + +vacuum-data-low: ## Create low garbage ratio test data (should NOT trigger vacuum) + @echo "📁 Creating low garbage test data (15% garbage)..." + @docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -files=30 -delete=0.15 -size=150 + +vacuum-continuous: ## Generate garbage continuously for testing + @echo "🔄 Generating continuous garbage for vacuum testing..." + @echo "Creating 5 rounds of test data with 30-second intervals..." + @for i in {1..5}; do \ + echo "Round $$i: Creating garbage..."; \ + docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -files=10 -delete=0.6 -size=100; \ + echo "Waiting 30 seconds..."; \ + sleep 30; \ + done + @echo "✅ Continuous test complete. Check vacuum task activity!" + +vacuum-clean: ## Clean up vacuum test data (removes all volumes!) + @echo "🧹 Cleaning up vacuum test data..." + @echo "⚠ī¸ WARNING: This will delete ALL volumes!" + @read -p "Are you sure? (y/N): " confirm && [ "$$confirm" = "y" ] || exit 1 + @echo "Stopping cluster..." + @docker-compose -f $(COMPOSE_FILE) down + @echo "Removing volume data..." + @rm -rf data/volume*/ + @echo "Restarting cluster..." + @docker-compose -f $(COMPOSE_FILE) up -d + @echo "✅ Clean up complete. Fresh volumes ready for testing." + +vacuum-help: ## Show vacuum testing help and examples + @echo "đŸ§Ē Vacuum Testing Commands (Docker-based)" + @echo "==========================================" + @echo "" + @echo "Quick Start:" + @echo " make start # Start SeaweedFS cluster with vacuum-tester" + @echo " make vacuum-test # Create test data and instructions" + @echo " make vacuum-status # Check volume status" + @echo "" + @echo "Data Generation:" + @echo " make vacuum-data-high # High garbage (should trigger)" + @echo " make vacuum-data-low # Low garbage (should NOT trigger)" + @echo " make vacuum-continuous # Continuous garbage generation" + @echo "" + @echo "Monitoring:" + @echo " make vacuum-status # Quick volume status check" + @echo " make vacuum-demo # Full guided demonstration" + @echo "" + @echo "Configuration:" + @echo " Visit: http://localhost:23646/maintenance/config/vacuum" + @echo " Monitor: http://localhost:23646/maintenance" + @echo "" + @echo "Custom Parameters:" + @echo " make vacuum-data FILES=50 DELETE=0.8 SIZE=200" + @echo "" + @echo "💡 All commands now run inside Docker containers" + @echo "Documentation:" + @echo " See: VACUUM_TEST_README.md for complete guide" \ No newline at end of file diff --git a/docker/admin_integration/check_volumes.sh b/docker/admin_integration/check_volumes.sh new file mode 100755 index 0000000..8cc6c14 --- /dev/null +++ b/docker/admin_integration/check_volumes.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +echo "📊 Quick Volume Status Check" +echo "============================" +echo "" + +# Check if master is running +MASTER_URL="${MASTER_HOST:-master:9333}" +if ! curl -s http://$MASTER_URL/cluster/status > /dev/null; then + echo "❌ Master server not available at $MASTER_URL" + exit 1 +fi + +echo "🔍 Fetching volume status from master..." +curl -s "http://$MASTER_URL/vol/status" | jq -r ' +if .Volumes and .Volumes.DataCenters then + .Volumes.DataCenters | to_entries[] | .value | to_entries[] | .value | to_entries[] | .value | if . then .[] else empty end | + "Volume \(.Id): + Size: \(.Size | if . < 1024 then "\(.) B" elif . < 1048576 then "\(. / 1024 | floor) KB" elif . < 1073741824 then "\(. / 1048576 * 100 | floor / 100) MB" else "\(. / 1073741824 * 100 | floor / 100) GB" end) + Files: \(.FileCount) active, \(.DeleteCount) deleted + Garbage: \(.DeletedByteCount | if . < 1024 then "\(.) B" elif . < 1048576 then "\(. / 1024 | floor) KB" elif . < 1073741824 then "\(. / 1048576 * 100 | floor / 100) MB" else "\(. / 1073741824 * 100 | floor / 100) GB" end) (\(if .Size > 0 then (.DeletedByteCount / .Size * 100 | floor) else 0 end)%) + Status: \(if (.DeletedByteCount / .Size * 100) > 30 then "đŸŽ¯ NEEDS VACUUM" else "✅ OK" end) +" +else + "No volumes found" +end' + +echo "" +echo "💡 Legend:" +echo " đŸŽ¯ NEEDS VACUUM: >30% garbage ratio" +echo " ✅ OK: <30% garbage ratio" +echo "" \ No newline at end of file diff --git a/docker/admin_integration/create_vacuum_test_data.go b/docker/admin_integration/create_vacuum_test_data.go new file mode 100644 index 0000000..46acdd4 --- /dev/null +++ b/docker/admin_integration/create_vacuum_test_data.go @@ -0,0 +1,280 @@ +package main + +import ( + "bytes" + "crypto/rand" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "time" +) + +var ( + master = flag.String("master", "master:9333", "SeaweedFS master server address") + fileCount = flag.Int("files", 20, "Number of files to create") + deleteRatio = flag.Float64("delete", 0.4, "Ratio of files to delete (0.0-1.0)") + fileSizeKB = flag.Int("size", 100, "Size of each file in KB") +) + +type AssignResult struct { + Fid string `json:"fid"` + Url string `json:"url"` + PublicUrl string `json:"publicUrl"` + Count int `json:"count"` + Error string `json:"error"` +} + +func main() { + flag.Parse() + + fmt.Println("đŸ§Ē Creating fake data for vacuum task testing...") + fmt.Printf("Master: %s\n", *master) + fmt.Printf("Files to create: %d\n", *fileCount) + fmt.Printf("Delete ratio: %.1f%%\n", *deleteRatio*100) + fmt.Printf("File size: %d KB\n", *fileSizeKB) + fmt.Println() + + if *fileCount == 0 { + // Just check volume status + fmt.Println("📊 Checking volume status...") + checkVolumeStatus() + return + } + + // Step 1: Create test files + fmt.Println("📁 Step 1: Creating test files...") + fids := createTestFiles() + + // Step 2: Delete some files to create garbage + fmt.Println("🗑ī¸ Step 2: Deleting files to create garbage...") + deleteFiles(fids) + + // Step 3: Check volume status + fmt.Println("📊 Step 3: Checking volume status...") + checkVolumeStatus() + + // Step 4: Configure vacuum for testing + fmt.Println("⚙ī¸ Step 4: Instructions for testing...") + printTestingInstructions() +} + +func createTestFiles() []string { + var fids []string + + for i := 0; i < *fileCount; i++ { + // Generate random file content + fileData := make([]byte, *fileSizeKB*1024) + rand.Read(fileData) + + // Get file ID assignment + assign, err := assignFileId() + if err != nil { + log.Printf("Failed to assign file ID for file %d: %v", i, err) + continue + } + + // Upload file + err = uploadFile(assign, fileData, fmt.Sprintf("test_file_%d.dat", i)) + if err != nil { + log.Printf("Failed to upload file %d: %v", i, err) + continue + } + + fids = append(fids, assign.Fid) + + if (i+1)%5 == 0 { + fmt.Printf(" Created %d/%d files...\n", i+1, *fileCount) + } + } + + fmt.Printf("✅ Created %d files successfully\n\n", len(fids)) + return fids +} + +func deleteFiles(fids []string) { + deleteCount := int(float64(len(fids)) * *deleteRatio) + + for i := 0; i < deleteCount; i++ { + err := deleteFile(fids[i]) + if err != nil { + log.Printf("Failed to delete file %s: %v", fids[i], err) + continue + } + + if (i+1)%5 == 0 { + fmt.Printf(" Deleted %d/%d files...\n", i+1, deleteCount) + } + } + + fmt.Printf("✅ Deleted %d files (%.1f%% of total)\n\n", deleteCount, *deleteRatio*100) +} + +func assignFileId() (*AssignResult, error) { + resp, err := http.Get(fmt.Sprintf("http://%s/dir/assign", *master)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var result AssignResult + err = json.NewDecoder(resp.Body).Decode(&result) + if err != nil { + return nil, err + } + + if result.Error != "" { + return nil, fmt.Errorf("assignment error: %s", result.Error) + } + + return &result, nil +} + +func uploadFile(assign *AssignResult, data []byte, filename string) error { + url := fmt.Sprintf("http://%s/%s", assign.Url, assign.Fid) + + body := &bytes.Buffer{} + body.Write(data) + + req, err := http.NewRequest("POST", url, body) + if err != nil { + return err + } + + req.Header.Set("Content-Type", "application/octet-stream") + if filename != "" { + req.Header.Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) + } + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +func deleteFile(fid string) error { + url := fmt.Sprintf("http://%s/%s", *master, fid) + + req, err := http.NewRequest("DELETE", url, nil) + if err != nil { + return err + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + return nil +} + +func checkVolumeStatus() { + // Get volume list from master + resp, err := http.Get(fmt.Sprintf("http://%s/vol/status", *master)) + if err != nil { + log.Printf("Failed to get volume status: %v", err) + return + } + defer resp.Body.Close() + + var volumes map[string]interface{} + err = json.NewDecoder(resp.Body).Decode(&volumes) + if err != nil { + log.Printf("Failed to decode volume status: %v", err) + return + } + + fmt.Println("📊 Volume Status Summary:") + + if vols, ok := volumes["Volumes"].([]interface{}); ok { + for _, vol := range vols { + if v, ok := vol.(map[string]interface{}); ok { + id := int(v["Id"].(float64)) + size := uint64(v["Size"].(float64)) + fileCount := int(v["FileCount"].(float64)) + deleteCount := int(v["DeleteCount"].(float64)) + deletedBytes := uint64(v["DeletedByteCount"].(float64)) + + garbageRatio := 0.0 + if size > 0 { + garbageRatio = float64(deletedBytes) / float64(size) * 100 + } + + fmt.Printf(" Volume %d:\n", id) + fmt.Printf(" Size: %s\n", formatBytes(size)) + fmt.Printf(" Files: %d (active), %d (deleted)\n", fileCount, deleteCount) + fmt.Printf(" Garbage: %s (%.1f%%)\n", formatBytes(deletedBytes), garbageRatio) + + if garbageRatio > 30 { + fmt.Printf(" đŸŽ¯ This volume should trigger vacuum (>30%% garbage)\n") + } + fmt.Println() + } + } + } +} + +func formatBytes(bytes uint64) string { + if bytes < 1024 { + return fmt.Sprintf("%d B", bytes) + } else if bytes < 1024*1024 { + return fmt.Sprintf("%.1f KB", float64(bytes)/1024) + } else if bytes < 1024*1024*1024 { + return fmt.Sprintf("%.1f MB", float64(bytes)/(1024*1024)) + } else { + return fmt.Sprintf("%.1f GB", float64(bytes)/(1024*1024*1024)) + } +} + +func printTestingInstructions() { + fmt.Println("đŸ§Ē Testing Instructions:") + fmt.Println() + fmt.Println("1. Configure Vacuum for Testing:") + fmt.Println(" Visit: http://localhost:23646/maintenance/config/vacuum") + fmt.Println(" Set:") + fmt.Printf(" - Garbage Percentage Threshold: 20 (20%% - lower than default 30)\n") + fmt.Printf(" - Scan Interval: [30] [Seconds] (faster than default)\n") + fmt.Printf(" - Min Volume Age: [0] [Minutes] (no age requirement)\n") + fmt.Printf(" - Max Concurrent: 2\n") + fmt.Printf(" - Min Interval: 1m (faster repeat)\n") + fmt.Println() + + fmt.Println("2. Monitor Vacuum Tasks:") + fmt.Println(" Visit: http://localhost:23646/maintenance") + fmt.Println(" Watch for vacuum tasks to appear in the queue") + fmt.Println() + + fmt.Println("3. Manual Vacuum (Optional):") + fmt.Println(" curl -X POST 'http://localhost:9333/vol/vacuum?garbageThreshold=0.20'") + fmt.Println(" (Note: Master API still uses 0.0-1.0 decimal format)") + fmt.Println() + + fmt.Println("4. Check Logs:") + fmt.Println(" Look for messages like:") + fmt.Println(" - 'Vacuum detector found X volumes needing vacuum'") + fmt.Println(" - 'Applied vacuum configuration'") + fmt.Println(" - 'Worker executing task: vacuum'") + fmt.Println() + + fmt.Println("5. Verify Results:") + fmt.Println(" Re-run this script with -files=0 to check volume status") + fmt.Println(" Garbage ratios should decrease after vacuum operations") + fmt.Println() + + fmt.Printf("🚀 Quick test command:\n") + fmt.Printf(" go run create_vacuum_test_data.go -files=0\n") + fmt.Println() +} diff --git a/docker/admin_integration/demo_vacuum_testing.sh b/docker/admin_integration/demo_vacuum_testing.sh new file mode 100755 index 0000000..6835e14 --- /dev/null +++ b/docker/admin_integration/demo_vacuum_testing.sh @@ -0,0 +1,105 @@ +#!/bin/sh + +echo "đŸ§Ē SeaweedFS Vacuum Task Testing Demo" +echo "======================================" +echo "" + +# Check if SeaweedFS is running +echo "📋 Checking SeaweedFS status..." +MASTER_URL="${MASTER_HOST:-master:9333}" +ADMIN_URL="${ADMIN_HOST:-admin:23646}" + +if ! curl -s http://$MASTER_URL/cluster/status > /dev/null; then + echo "❌ SeaweedFS master not running at $MASTER_URL" + echo " Please ensure Docker cluster is running: make start" + exit 1 +fi + +if ! curl -s http://volume1:8080/status > /dev/null; then + echo "❌ SeaweedFS volume servers not running" + echo " Please ensure Docker cluster is running: make start" + exit 1 +fi + +if ! curl -s http://$ADMIN_URL/ > /dev/null; then + echo "❌ SeaweedFS admin server not running at $ADMIN_URL" + echo " Please ensure Docker cluster is running: make start" + exit 1 +fi + +echo "✅ All SeaweedFS components are running" +echo "" + +# Phase 1: Create test data +echo "📁 Phase 1: Creating test data with garbage..." +go run create_vacuum_test_data.go -master=$MASTER_URL -files=15 -delete=0.5 -size=150 +echo "" + +# Phase 2: Check initial status +echo "📊 Phase 2: Checking initial volume status..." +go run create_vacuum_test_data.go -master=$MASTER_URL -files=0 +echo "" + +# Phase 3: Configure vacuum +echo "⚙ī¸ Phase 3: Vacuum configuration instructions..." +echo " 1. Visit: http://localhost:23646/maintenance/config/vacuum" +echo " 2. Set these values for testing:" +echo " - Enable Vacuum Tasks: ✅ Checked" +echo " - Garbage Threshold: 0.30" +echo " - Scan Interval: [30] [Seconds]" +echo " - Min Volume Age: [0] [Minutes]" +echo " - Max Concurrent: 2" +echo " 3. Click 'Save Configuration'" +echo "" + +read -p " Press ENTER after configuring vacuum settings..." +echo "" + +# Phase 4: Monitor tasks +echo "đŸŽ¯ Phase 4: Monitoring vacuum tasks..." +echo " Visit: http://localhost:23646/maintenance" +echo " You should see vacuum tasks appear within 30 seconds" +echo "" + +echo " Waiting 60 seconds for vacuum detection and execution..." +for i in {60..1}; do + printf "\r Countdown: %02d seconds" $i + sleep 1 +done +echo "" +echo "" + +# Phase 5: Check results +echo "📈 Phase 5: Checking results after vacuum..." +go run create_vacuum_test_data.go -master=$MASTER_URL -files=0 +echo "" + +# Phase 6: Create more garbage for continuous testing +echo "🔄 Phase 6: Creating additional garbage for continuous testing..." +echo " Running 3 rounds of garbage creation..." + +for round in {1..3}; do + echo " Round $round: Creating garbage..." + go run create_vacuum_test_data.go -master=$MASTER_URL -files=8 -delete=0.6 -size=100 + echo " Waiting 30 seconds before next round..." + sleep 30 +done + +echo "" +echo "📊 Final volume status:" +go run create_vacuum_test_data.go -master=$MASTER_URL -files=0 +echo "" + +echo "🎉 Demo Complete!" +echo "" +echo "🔍 Things to check:" +echo " 1. Maintenance Queue: http://localhost:23646/maintenance" +echo " 2. Volume Status: http://localhost:9333/vol/status" +echo " 3. Admin Dashboard: http://localhost:23646" +echo "" +echo "💡 Next Steps:" +echo " - Try different garbage thresholds (0.10, 0.50, 0.80)" +echo " - Adjust scan intervals (10s, 1m, 5m)" +echo " - Monitor logs for vacuum operations" +echo " - Test with multiple volumes" +echo "" \ No newline at end of file diff --git a/docker/admin_integration/docker-compose-ec-test.yml b/docker/admin_integration/docker-compose-ec-test.yml new file mode 100644 index 0000000..1726e61 --- /dev/null +++ b/docker/admin_integration/docker-compose-ec-test.yml @@ -0,0 +1,240 @@ +name: admin_integration + +networks: + seaweed_net: + driver: bridge + +services: + master: + image: chrislusf/seaweedfs:local + ports: + - "9333:9333" + - "19333:19333" + command: "master -ip=master -mdir=/data -volumeSizeLimitMB=50" + environment: + - WEED_MASTER_VOLUME_GROWTH_COPY_1=1 + - WEED_MASTER_VOLUME_GROWTH_COPY_2=2 + - WEED_MASTER_VOLUME_GROWTH_COPY_OTHER=1 + volumes: + - ./data/master:/data + networks: + - seaweed_net + + volume1: + image: chrislusf/seaweedfs:local + ports: + - "8080:8080" + - "18080:18080" + command: "volume -master=master:9333 -ip=volume1 -dir=/data -max=10" + depends_on: + - master + volumes: + - ./data/volume1:/data + networks: + - seaweed_net + + volume2: + image: chrislusf/seaweedfs:local + ports: + - "8081:8080" + - "18081:18080" + command: "volume -master=master:9333 -ip=volume2 -dir=/data -max=10" + depends_on: + - master + volumes: + - ./data/volume2:/data + networks: + - seaweed_net + + volume3: + image: chrislusf/seaweedfs:local + ports: + - "8082:8080" + - "18082:18080" + command: "volume -master=master:9333 -ip=volume3 -dir=/data -max=10" + depends_on: + - master + volumes: + - ./data/volume3:/data + networks: + - seaweed_net + + volume4: + image: chrislusf/seaweedfs:local + ports: + - "8083:8080" + - "18083:18080" + command: "volume -master=master:9333 -ip=volume4 -dir=/data -max=10" + depends_on: + - master + volumes: + - ./data/volume4:/data + networks: + - seaweed_net + + volume5: + image: chrislusf/seaweedfs:local + ports: + - "8084:8080" + - "18084:18080" + command: "volume -master=master:9333 -ip=volume5 -dir=/data -max=10" + depends_on: + - master + volumes: + - ./data/volume5:/data + networks: + - seaweed_net + + volume6: + image: chrislusf/seaweedfs:local + ports: + - "8085:8080" + - "18085:18080" + command: "volume -master=master:9333 -ip=volume6 -dir=/data -max=10" + depends_on: + - master + volumes: + - ./data/volume6:/data + networks: + - seaweed_net + + filer: + image: chrislusf/seaweedfs:local + ports: + - "8888:8888" + - "18888:18888" + command: "filer -master=master:9333 -ip=filer" + depends_on: + - master + volumes: + - ./data/filer:/data + networks: + - seaweed_net + + admin: + image: chrislusf/seaweedfs:local + ports: + - "23646:23646" # HTTP admin interface (default port) + - "33646:33646" # gRPC worker communication (23646 + 10000) + command: "-v=2 admin -port=23646 -masters=master:9333 -dataDir=/data" + depends_on: + - master + - filer + volumes: + - ./data/admin:/data + networks: + - seaweed_net + + worker1: + image: chrislusf/seaweedfs:local + command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2" + depends_on: + - admin + volumes: + - ./data/worker1:/data + networks: + - seaweed_net + environment: + - WORKER_ID=worker-1 + + worker2: + image: chrislusf/seaweedfs:local + command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2" + depends_on: + - admin + volumes: + - ./data/worker2:/data + networks: + - seaweed_net + environment: + - WORKER_ID=worker-2 + + worker3: + image: chrislusf/seaweedfs:local + command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2" + depends_on: + - admin + volumes: + - ./data/worker3:/data + networks: + - seaweed_net + environment: + - WORKER_ID=worker-3 + + load_generator: + image: chrislusf/seaweedfs:local + entrypoint: ["/bin/sh"] + command: > + -c " + echo 'Starting load generator...'; + sleep 30; + echo 'Generating continuous load with 50MB volume limit...'; + while true; do + echo 'Writing test files...'; + echo 'Test file content at $(date)' | /usr/bin/weed upload -server=master:9333; + sleep 5; + echo 'Deleting some files...'; + /usr/bin/weed shell -master=master:9333 <<< 'fs.rm /test_file_*' || true; + sleep 10; + done + " + depends_on: + - master + - filer + - admin + networks: + - seaweed_net + + monitor: + image: alpine:latest + entrypoint: ["/bin/sh"] + command: > + -c " + apk add --no-cache curl jq; + echo 'Starting cluster monitor...'; + sleep 30; + while true; do + echo '=== Cluster Status $(date) ==='; + echo 'Master status:'; + curl -s http://master:9333/cluster/status | jq '.IsLeader, .Peers' || echo 'Master not ready'; + echo; + echo 'Admin status:'; + curl -s http://admin:23646/ | grep -o 'Admin.*Interface' || echo 'Admin not ready'; + echo; + echo 'Volume count by server:'; + curl -s http://master:9333/vol/status | jq '.Volumes | length' || echo 'Volumes not ready'; + echo; + sleep 60; + done + " + depends_on: + - master + - admin + - filer + networks: + - seaweed_net + + vacuum-tester: + image: chrislusf/seaweedfs:local + entrypoint: ["/bin/sh"] + command: > + -c " + echo 'Installing dependencies for vacuum testing...'; + apk add --no-cache jq curl go bash; + echo 'Vacuum tester ready...'; + echo 'Use: docker-compose exec vacuum-tester sh'; + echo 'Available commands: go, weed, curl, jq, bash, sh'; + sleep infinity + " + depends_on: + - master + - admin + - filer + volumes: + - .:/testing + working_dir: /testing + networks: + - seaweed_net + environment: + - MASTER_HOST=master:9333 + - ADMIN_HOST=admin:23646 \ No newline at end of file diff --git a/docker/admin_integration/test-integration.sh b/docker/admin_integration/test-integration.sh new file mode 100755 index 0000000..b355b1d --- /dev/null +++ b/docker/admin_integration/test-integration.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +set -e + +echo "đŸ§Ē Testing SeaweedFS Admin-Worker Integration" +echo "=============================================" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +cd "$(dirname "$0")" + +echo -e "${BLUE}1. Validating docker-compose configuration...${NC}" +if docker-compose -f docker-compose-ec-test.yml config > /dev/null; then + echo -e "${GREEN}✅ Docker compose configuration is valid${NC}" +else + echo -e "${RED}❌ Docker compose configuration is invalid${NC}" + exit 1 +fi + +echo -e "${BLUE}2. Checking if required ports are available...${NC}" +for port in 9333 8080 8081 8082 8083 8084 8085 8888 23646; do + if lsof -i :$port > /dev/null 2>&1; then + echo -e "${YELLOW}⚠ī¸ Port $port is in use${NC}" + else + echo -e "${GREEN}✅ Port $port is available${NC}" + fi +done + +echo -e "${BLUE}3. Testing worker command syntax...${NC}" +# Test that the worker command in docker-compose has correct syntax +if docker-compose -f docker-compose-ec-test.yml config | grep -q "workingDir=/work"; then + echo -e "${GREEN}✅ Worker working directory option is properly configured${NC}" +else + echo -e "${RED}❌ Worker working directory option is missing${NC}" + exit 1 +fi + +echo -e "${BLUE}4. Verifying admin server configuration...${NC}" +if docker-compose -f docker-compose-ec-test.yml config | grep -q "admin:23646"; then + echo -e "${GREEN}✅ Admin server port configuration is correct${NC}" +else + echo -e "${RED}❌ Admin server port configuration is incorrect${NC}" + exit 1 +fi + +echo -e "${BLUE}5. Checking service dependencies...${NC}" +if docker-compose -f docker-compose-ec-test.yml config | grep -q "depends_on"; then + echo -e "${GREEN}✅ Service dependencies are configured${NC}" +else + echo -e "${YELLOW}⚠ī¸ Service dependencies may not be configured${NC}" +fi + +echo "" +echo -e "${GREEN}🎉 Integration test configuration is ready!${NC}" +echo "" +echo -e "${BLUE}To start the integration test:${NC}" +echo " make start # Start all services" +echo " make health # Check service health" +echo " make logs # View logs" +echo " make stop # Stop all services" +echo "" +echo -e "${BLUE}Key features verified:${NC}" +echo " ✅ Official SeaweedFS images are used" +echo " ✅ Worker working directories are configured" +echo " ✅ Admin-worker communication on correct ports" +echo " ✅ Task-specific directories will be created" +echo " ✅ Load generator will trigger EC tasks" +echo " ✅ Monitor will track progress" \ No newline at end of file diff --git a/docker/apt-install b/docker/apt-install new file mode 100755 index 0000000..8dfd999 --- /dev/null +++ b/docker/apt-install @@ -0,0 +1,25 @@ +#!/bin/sh +# Install packages, falling through to the next Ubuntu mirror when one is +# unreachable. See docker/Dockerfile.e2e. +set -e + +# Every rewrite starts from the pristine list, so a mirror that just failed does +# not become the pattern the next rewrite has to match. +[ -f /etc/apt/sources.list.orig ] || cp /etc/apt/sources.list /etc/apt/sources.list.orig + +for mirror in azure.archive.ubuntu.com archive.ubuntu.com; do + # Any archive host, so this works whether the pristine list came from the + # base image (archive.ubuntu.com) or a CI runner (azure.archive.ubuntu.com). + sed "s|http://[a-z0-9.]*archive\.ubuntu\.com/ubuntu|http://$mirror/ubuntu|g; s|http://security\.ubuntu\.com/ubuntu|http://$mirror/ubuntu|g" \ + /etc/apt/sources.list.orig > /etc/apt/sources.list + if apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=15 update && \ + DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=15 install -y \ + --no-install-recommends --no-install-suggests "$@"; then + apt-get clean + rm -rf /var/lib/apt/lists/* + exit 0 + fi + echo "apt: $mirror unreachable, trying the next mirror" >&2 +done + +exit 1 diff --git a/docker/compose/dev.env b/docker/compose/dev.env new file mode 100644 index 0000000..e69de29 diff --git a/docker/compose/e2e-mount.yml b/docker/compose/e2e-mount.yml new file mode 100644 index 0000000..eb4631f --- /dev/null +++ b/docker/compose/e2e-mount.yml @@ -0,0 +1,61 @@ +version: '3.9' + +services: + master: + image: chrislusf/seaweedfs:e2e + command: "-v=4 master -ip=master -ip.bind=0.0.0.0 -raftBootstrap" + healthcheck: + test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ] + interval: 2s + timeout: 10s + retries: 30 + start_period: 10s + + volume: + image: chrislusf/seaweedfs:e2e + command: "-v=4 volume -master=master:9333 -ip=volume -ip.bind=0.0.0.0 -preStopSeconds=1" + healthcheck: + test: [ "CMD", "curl", "--fail", "-I", "http://localhost:8080/healthz" ] + interval: 2s + timeout: 10s + retries: 15 + start_period: 5s + depends_on: + master: + condition: service_healthy + + filer: + image: chrislusf/seaweedfs:e2e + command: "-v=4 filer -master=master:9333 -ip=filer -ip.bind=0.0.0.0" + healthcheck: + test: [ "CMD", "curl", "--fail", "-I", "http://localhost:8888" ] + interval: 2s + timeout: 10s + retries: 15 + start_period: 5s + depends_on: + volume: + condition: service_healthy + + mount: + image: chrislusf/seaweedfs:e2e + command: "-v=4 mount -filer=filer:8888 -filer.path=/ -dirAutoCreate -dir=/mnt/seaweedfs" + cap_add: + - SYS_ADMIN + devices: + - /dev/fuse + security_opt: + - apparmor:unconfined + deploy: + resources: + limits: + memory: 4096m + healthcheck: + test: [ "CMD", "mountpoint", "-q", "--", "/mnt/seaweedfs" ] + interval: 2s + timeout: 10s + retries: 15 + start_period: 10s + depends_on: + filer: + condition: service_healthy diff --git a/docker/compose/fluent.conf b/docker/compose/fluent.conf new file mode 100644 index 0000000..d4396dd --- /dev/null +++ b/docker/compose/fluent.conf @@ -0,0 +1,8 @@ + + @type forward + port 24224 + + + + @type stdout # Output logs to container's stdout (visible via `docker logs`) + diff --git a/docker/compose/fluent.json b/docker/compose/fluent.json new file mode 100644 index 0000000..8b50b0f --- /dev/null +++ b/docker/compose/fluent.json @@ -0,0 +1,4 @@ +{ + "fluent_port": 24224, + "fluent_host": "fluent" +} \ No newline at end of file diff --git a/docker/compose/local-auditlog-compose.yml b/docker/compose/local-auditlog-compose.yml new file mode 100644 index 0000000..dc3fee9 --- /dev/null +++ b/docker/compose/local-auditlog-compose.yml @@ -0,0 +1,38 @@ +version: '3.9' + +services: + s3: + image: chrislusf/seaweedfs:local + ports: + - 8333:8333 + - 9333:9333 + - 19333:19333 + - 8084:8080 + - 18084:18080 + - 8888:8888 + - 18888:18888 + - 8000:8000 + command: "server -ip=s3 -filer -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.auditLogConfig=/etc/seaweedfs/fluent.json -volume.max=0 -master.volumeSizeLimitMB=8 -volume.preStopSeconds=1" + volumes: + - ./fluent.json:/etc/seaweedfs/fluent.json + - ./s3.json:/etc/seaweedfs/s3.json + depends_on: + - fluent + fluent: + image: fluent/fluentd:v1.17 + volumes: + - ./fluent.conf:/fluentd/etc/fluent.conf + ports: + - 24224:24224 + #s3tests: + # image: chrislusf/ceph-s3-tests:local + # volumes: + # - ./s3tests.conf:/opt/s3-tests/s3tests.conf + # environment: + # S3TEST_CONF: "s3tests.conf" + # NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3" + # NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy" + # NOSETESTS_EXCLUDE: "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)" + # depends_on: + # - s3 + # - fluent \ No newline at end of file diff --git a/docker/compose/local-brokers-compose.yml b/docker/compose/local-brokers-compose.yml new file mode 100644 index 0000000..c97becb --- /dev/null +++ b/docker/compose/local-brokers-compose.yml @@ -0,0 +1,127 @@ +version: '3.9' + +services: + master0: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "-v=0 master -volumeSizeLimitMB 100 -resumeState=false -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp" + environment: + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_2: 2 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + master1: + image: chrislusf/seaweedfs:local + ports: + - 9334:9334 + - 19334:19334 + command: "-v=0 master -volumeSizeLimitMB 100 -resumeState=false -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp" + environment: + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_2: 2 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + master2: + image: chrislusf/seaweedfs:local + ports: + - 9335:9335 + - 19335:19335 + command: "-v=0 master -volumeSizeLimitMB 100 -resumeState=false -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp" + environment: + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_2: 2 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + volume1: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + command: 'volume -dataCenter=dc1 -rack=v1 -master="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1' + depends_on: + - master0 + - master1 + - master2 + volume2: + image: chrislusf/seaweedfs:local + ports: + - 8082:8082 + - 18082:18082 + command: 'volume -dataCenter=dc2 -rack=v2 -master="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1' + depends_on: + - master0 + - master1 + - master2 + volume3: + image: chrislusf/seaweedfs:local + ports: + - 8083:8083 + - 18083:18083 + command: 'volume -dataCenter=dc3 -rack=v3 -master="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1' + depends_on: + - master0 + - master1 + - master2 + filer1: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335" -port=8888 -ip=filer1' + depends_on: + - master0 + - master1 + - master2 + - volume1 + - volume2 + filer2: + image: chrislusf/seaweedfs:local + ports: + - 8889:8889 + - 18889:18889 + command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335" -port=8889 -ip=filer2' + depends_on: + - master0 + - master1 + - master2 + - volume1 + - volume2 + - filer1 + broker1: + image: chrislusf/seaweedfs:local + ports: + - 17777:17777 + command: 'mq.broker -master="master0:9333,master1:9334,master2:9335" -port=17777 -ip=broker1' + depends_on: + - master0 + - master1 + - master2 + - volume1 + - volume2 + - filer1 + - filer2 + broker2: + image: chrislusf/seaweedfs:local + ports: + - 17778:17778 + command: 'mq.broker -master="master0:9333,master1:9334,master2:9335" -port=17778 -ip=broker2' + depends_on: + - master0 + - master1 + - master2 + - volume1 + - volume2 + - filer1 + - filer2 + broker3: + image: chrislusf/seaweedfs:local + ports: + - 17779:17779 + command: 'mq.broker -master="master0:9333,master1:9334,master2:9335" -port=17779 -ip=broker3' + depends_on: + - master0 + - master1 + - master2 + - volume1 + - volume2 + - filer1 + - filer2 diff --git a/docker/compose/local-cluster-compose.yml b/docker/compose/local-cluster-compose.yml new file mode 100644 index 0000000..cde87a4 --- /dev/null +++ b/docker/compose/local-cluster-compose.yml @@ -0,0 +1,88 @@ +version: '3.9' + +services: + master0: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp" + environment: + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_2: 2 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + master1: + image: chrislusf/seaweedfs:local + ports: + - 9334:9334 + - 19334:19334 + command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp" + environment: + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_2: 2 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + master2: + image: chrislusf/seaweedfs:local + ports: + - 9335:9335 + - 19335:19335 + command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp" + environment: + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_2: 2 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + volume1: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + command: 'volume -dataCenter=dc1 -rack=v1 -master="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1' + depends_on: + - master0 + - master1 + - master2 + volume2: + image: chrislusf/seaweedfs:local + ports: + - 8082:8082 + - 18082:18082 + command: 'volume -dataCenter=dc2 -rack=v2 -master="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1' + depends_on: + - master0 + - master1 + - master2 + volume3: + image: chrislusf/seaweedfs:local + ports: + - 8083:8083 + - 18083:18083 + command: 'volume -dataCenter=dc3 -rack=v3 -master="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1' + depends_on: + - master0 + - master1 + - master2 + filer: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + - 8111:8111 + command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335"' + depends_on: + - master0 + - master1 + - master2 + - volume1 + - volume2 + s3: + image: chrislusf/seaweedfs:local + ports: + - 8333:8333 + command: '-v=9 s3 -filer="filer:8888"' + depends_on: + - master0 + - master1 + - master2 + - volume1 + - volume2 + - filer diff --git a/docker/compose/local-clusters-compose.yml b/docker/compose/local-clusters-compose.yml new file mode 100644 index 0000000..62b1c5d --- /dev/null +++ b/docker/compose/local-clusters-compose.yml @@ -0,0 +1,28 @@ +version: '3.9' + +services: + server1: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + - 8084:8080 + - 18084:18080 + - 8888:8888 + - 18888:18888 + command: "server -ip=server1 -filer -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1" + volumes: + - ./master-cloud.toml:/etc/seaweedfs/master.toml + depends_on: + - server2 + server2: + image: chrislusf/seaweedfs:local + ports: + - 9334:9333 + - 19334:19333 + - 8085:8080 + - 18085:18080 + - 8889:8888 + - 18889:18888 + - 8334:8333 + command: "server -ip=server2 -filer -s3 -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1" diff --git a/docker/compose/local-dev-compose.yml b/docker/compose/local-dev-compose.yml new file mode 100644 index 0000000..e6167cb --- /dev/null +++ b/docker/compose/local-dev-compose.yml @@ -0,0 +1,80 @@ +version: '3.9' + +services: + master: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "-v=1 master -ip=master -volumeSizeLimitMB=10" + volumes: + - ./tls:/etc/seaweedfs/tls + env_file: + - ${ENV_FILE:-dev.env} + volume: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + command: "-v=1 volume -master=master:9333 -port=8080 -ip=volume -preStopSeconds=1 -max=10000" + depends_on: + - master + volumes: + - ./tls:/etc/seaweedfs/tls + env_file: + - ${ENV_FILE:-dev.env} + filer: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + command: '-v=1 filer -ip.bind=0.0.0.0 -master="master:9333"' + depends_on: + - master + - volume + volumes: + - ./tls:/etc/seaweedfs/tls + env_file: + - ${ENV_FILE:-dev.env} + + iam: + image: chrislusf/seaweedfs:local + ports: + - 8111:8111 + command: '-v=1 iam -filer="filer:8888" -master="master:9333"' + depends_on: + - master + - volume + - filer + volumes: + - ./tls:/etc/seaweedfs/tls + + s3: + image: chrislusf/seaweedfs:local + ports: + - 8333:8333 + command: '-v=1 s3 -filer="filer:8888" -ip.bind=s3' + depends_on: + - master + - volume + - filer + volumes: + - ./tls:/etc/seaweedfs/tls + env_file: + - ${ENV_FILE:-dev.env} + + mount: + image: chrislusf/seaweedfs:local + privileged: true + cap_add: + - SYS_ADMIN + mem_limit: 4096m + command: '-v=4 mount -filer="filer:8888" -dirAutoCreate -dir=/mnt/seaweedfs -cacheCapacityMB=100 -concurrentWriters=128' + volumes: + - ./tls:/etc/seaweedfs/tls + env_file: + - ${ENV_FILE:-dev.env} + depends_on: + - master + - volume + - filer diff --git a/docker/compose/local-filer-backup-compose.yml b/docker/compose/local-filer-backup-compose.yml new file mode 100644 index 0000000..3e56e62 --- /dev/null +++ b/docker/compose/local-filer-backup-compose.yml @@ -0,0 +1,54 @@ +version: '3.9' + +services: + server-left: + image: chrislusf/seaweedfs:local + command: "-v=0 server -ip=server-left -filer -filer.maxMB 5 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1" + volumes: + - ./s3.json:/etc/seaweedfs/s3.json + healthcheck: + test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ] + interval: 3s + start_period: 15s + timeout: 30s + server-right: + image: chrislusf/seaweedfs:local + command: "-v=0 server -ip=server-right -filer -filer.maxMB 64 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1" + volumes: + - ./s3.json:/etc/seaweedfs/s3.json + healthcheck: + test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ] + interval: 3s + start_period: 15s + timeout: 30s + filer-backup: + image: chrislusf/seaweedfs:local + command: "-v=0 filer.backup -debug -doDeleteFiles=False -filer server-left:8888" + volumes: + - ./replication.toml:/etc/seaweedfs/replication.toml + environment: + WEED_SINK_LOCAL_INCREMENTAL_ENABLED: "false" + WEED_SINK_S3_ENABLED: "true" + WEED_SINK_S3_BUCKET: "backup" + WEED_SINK_S3_ENDPOINT: "http://server-right:8333" + WEED_SINK_S3_DIRECTORY: "/" + WEED_SINK_S3_AWS_ACCESS_KEY_ID: "some_access_key1" + WEED_SINK_S3_AWS_SECRET_ACCESS_KEY: "some_secret_key1" + WEED_SINK_S3_S3_DISABLE_CONTENT_MD5_VALIDATION: "false" + WEED_SINK_S3_UPLOADER_PART_SIZE_MB: "5" + WEED_SINK_S3_KEEP_PART_SIZE: "false" + depends_on: + server-left: + condition: service_healthy + server-right: + condition: service_healthy + minio-warp: + image: minio/warp + command: 'mixed --duration 5s --obj.size=6mb --md5 --objects 10 --concurrent 2' + restart: on-failure + environment: + WARP_HOST: "server-left:8333" + WARP_ACCESS_KEY: "some_access_key1" + WARP_SECRET_KEY: "some_secret_key1" + depends_on: + - filer-backup \ No newline at end of file diff --git a/docker/compose/local-hashicorp-raft-compose.yml b/docker/compose/local-hashicorp-raft-compose.yml new file mode 100644 index 0000000..7a5291c --- /dev/null +++ b/docker/compose/local-hashicorp-raft-compose.yml @@ -0,0 +1,89 @@ +version: '3.9' + +services: + master0: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data" + volumes: + - ./master/0:/data + environment: + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_2: 2 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + master1: + image: chrislusf/seaweedfs:local + ports: + - 9334:9334 + - 19334:19334 + command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data" + volumes: + - ./master/1:/data + environment: + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_2: 2 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + master2: + image: chrislusf/seaweedfs:local + ports: + - 9335:9335 + - 19335:19335 + command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data" + volumes: + - ./master/2:/data + environment: + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_2: 2 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + volume1: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + command: 'volume -dataCenter=dc1 -rack=v1 -master="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1' + depends_on: + - master0 + - master1 + volume2: + image: chrislusf/seaweedfs:local + ports: + - 8082:8082 + - 18082:18082 + command: 'volume -dataCenter=dc2 -rack=v2 -master="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1' + depends_on: + - master0 + - master1 + volume3: + image: chrislusf/seaweedfs:local + ports: + - 8083:8083 + - 18083:18083 + command: 'volume -dataCenter=dc3 -rack=v3 -master="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1' + depends_on: + - master0 + - master1 + filer: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + - 8111:8111 + command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335"' + depends_on: + - master0 + - master1 + - volume1 + - volume2 + s3: + image: chrislusf/seaweedfs:local + ports: + - 8333:8333 + command: '-v=9 s3 -ip.bind="s3" -filer="filer:8888"' + depends_on: + - master0 + - master1 + - volume1 + - volume2 + - filer \ No newline at end of file diff --git a/docker/compose/local-k8s-compose.yml b/docker/compose/local-k8s-compose.yml new file mode 100644 index 0000000..71cf516 --- /dev/null +++ b/docker/compose/local-k8s-compose.yml @@ -0,0 +1,94 @@ +version: '3.9' + +services: + master: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "master -ip=master" + volume: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + command: "volume -master=master:9333 -port=8080 -ip=volume" + depends_on: + - master + mysql: + image: percona/percona-server:5.7 + ports: + - 3306:3306 + volumes: + - ./seaweedfs.sql:/docker-entrypoint-initdb.d/seaweedfs.sql + environment: + - MYSQL_ROOT_PASSWORD=secret + - MYSQL_DATABASE=seaweedfs + - MYSQL_PASSWORD=secret + - MYSQL_USER=seaweedfs + filer: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + environment: + - WEED_MYSQL_HOSTNAME=mysql + - WEED_MYSQL_PORT=3306 + - WEED_MYSQL_DATABASE=seaweedfs + - WEED_MYSQL_USERNAME=seaweedfs + - WEED_MYSQL_PASSWORD=secret + - WEED_MYSQL_ENABLED=true + - WEED_MYSQL_CONNECTION_MAX_IDLE=5 + - WEED_MYSQL_CONNECTION_MAX_OPEN=75 + # "refresh" connection every 10 minutes, eliminating mysql closing "old" connections + - WEED_MYSQL_CONNECTION_MAX_LIFETIME_SECONDS=600 + # enable usage of memsql as filer backend + - WEED_MYSQL_INTERPOLATEPARAMS=true + - WEED_LEVELDB2_ENABLED=false + command: '-v 9 filer -master="master:9333"' + depends_on: + - master + - volume + - mysql + ingress: + image: jwilder/nginx-proxy:alpine + ports: + - "80:80" + volumes: + - /var/run/docker.sock:/tmp/docker.sock:ro + - ./nginx/proxy.conf:/etc/nginx/proxy.conf + s3: + image: chrislusf/seaweedfs:local + ports: + - 8333:8333 + command: '-v 9 s3 -filer="filer:8888"' + depends_on: + - master + - volume + - filer + environment: + - VIRTUAL_HOST=ingress + - VIRTUAL_PORT=8333 + registry: + image: registry:2 + environment: + REGISTRY_HTTP_ADDR: "0.0.0.0:5001" # seaweedfs s3 + REGISTRY_LOG_LEVEL: "debug" + REGISTRY_STORAGE: "s3" + REGISTRY_STORAGE_S3_REGION: "us-east-1" + REGISTRY_STORAGE_S3_REGIONENDPOINT: "http://ingress" + REGISTRY_STORAGE_S3_BUCKET: "registry" + REGISTRY_STORAGE_S3_ACCESSKEY: "some_access_key1" + REGISTRY_STORAGE_S3_SECRETKEY: "some_secret_key1" + REGISTRY_STORAGE_S3_V4AUTH: "true" + REGISTRY_STORAGE_S3_SECURE: "false" + REGISTRY_STORAGE_S3_SKIPVERIFY: "true" + REGISTRY_STORAGE_S3_ROOTDIRECTORY: "/" + REGISTRY_STORAGE_DELETE_ENABLED: "true" + REGISTRY_STORAGE_REDIRECT_DISABLE: "true" + REGISTRY_VALIDATION_DISABLED: "true" + ports: + - 5001:5001 + depends_on: + - s3 + - ingress \ No newline at end of file diff --git a/docker/compose/local-minio-gateway-compose.yml b/docker/compose/local-minio-gateway-compose.yml new file mode 100644 index 0000000..32863af --- /dev/null +++ b/docker/compose/local-minio-gateway-compose.yml @@ -0,0 +1,50 @@ +version: '3.9' + +services: + master: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "master -ip=master -volumeSizeLimitMB=100" + volume: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + command: "volume -master=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1" + depends_on: + - master + s3: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + - 8333:8333 + command: '-v 1 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333' + volumes: + - ./s3.json:/etc/seaweedfs/s3.json + depends_on: + - master + - volume + minio-gateway-s3: + image: minio/minio + ports: + - 9000:9000 + command: 'minio gateway s3 http://s3:8333' + restart: on-failure + environment: + MINIO_ACCESS_KEY: "some_access_key1" + MINIO_SECRET_KEY: "some_secret_key1" + depends_on: + - s3 + minio-warp: + image: minio/warp + command: 'mixed --duration=5m --obj.size=3mb --autoterm' + restart: on-failure + environment: + WARP_HOST: "minio-gateway-s3:9000" + WARP_ACCESS_KEY: "some_access_key1" + WARP_SECRET_KEY: "some_secret_key1" + depends_on: + - minio-gateway-s3 \ No newline at end of file diff --git a/docker/compose/local-mount-compose.yml b/docker/compose/local-mount-compose.yml new file mode 100644 index 0000000..1baf125 --- /dev/null +++ b/docker/compose/local-mount-compose.yml @@ -0,0 +1,46 @@ +version: '3.9' + +services: + master: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "master -ip=master" + volume: + image: chrislusf/seaweedfs:local + ports: + - 7455:8080 + - 9325:9325 + command: 'volume -master="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455' + depends_on: + - master + filer: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + - 9326:9326 + command: 'filer -master="master:9333" -metricsPort=9326' + tty: true + stdin_open: true + depends_on: + - master + - volume + mount_1: + image: chrislusf/seaweedfs:local + privileged: true + entrypoint: '/bin/sh -c "mkdir -p t1 && mkdir -p cache/t1 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t1 -dir=./t1 -filer.path=/c1 -volumeServerAccess=filerProxy"' + depends_on: + - master + - volume + - filer + mount_2: + image: chrislusf/seaweedfs:local + privileged: true + entrypoint: '/bin/sh -c "mkdir -p t2 && mkdir -p cache/t2 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t2 -dir=./t2 -filer.path=/c1"' + depends_on: + - master + - volume + - filer + - mount_1 diff --git a/docker/compose/local-mount-profile-compose.yml b/docker/compose/local-mount-profile-compose.yml new file mode 100644 index 0000000..ef04d46 --- /dev/null +++ b/docker/compose/local-mount-profile-compose.yml @@ -0,0 +1,47 @@ +version: '3.9' + +services: + master: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "master -ip=master" + volume: + image: chrislusf/seaweedfs:local + ports: + - 7455:8080 + - 9325:9325 + volumes: + - /Volumes/mobile_disk/99:/data + command: 'volume -master="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455' + depends_on: + - master + filer: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + - 9326:9326 + volumes: + - /Volumes/mobile_disk/99:/data + command: 'filer -master="master:9333" -metricsPort=9326' + tty: true + stdin_open: true + depends_on: + - master + - volume + mount: + image: chrislusf/seaweedfs:local + privileged: true + cap_add: + - SYS_ADMIN + devices: + - fuse + volumes: + - /Volumes/mobile_disk/99:/data + entrypoint: '/bin/sh -c "mkdir -p t1 && weed -v=4 mount -filer=filer:8888 -dir=./t1 -cacheCapacityMB=0 -memprofile=/data/mount.mem.pprof"' + depends_on: + - master + - volume + - filer diff --git a/docker/compose/local-mq-test.yml b/docker/compose/local-mq-test.yml new file mode 100644 index 0000000..fef68ca --- /dev/null +++ b/docker/compose/local-mq-test.yml @@ -0,0 +1,32 @@ +services: + server: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + - 8888:8888 + - 18888:18888 + command: "server -ip=server -filer -volume.max=0 -master.volumeSizeLimitMB=8 -volume.preStopSeconds=1" + healthcheck: + test: curl -f http://localhost:8888/healthz + mq_broker: + image: chrislusf/seaweedfs:local + ports: + - 17777:17777 + command: "mq.broker -master=server:9333 -ip=mq_broker" + depends_on: + server: + condition: service_healthy + mq_agent: + image: chrislusf/seaweedfs:local + ports: + - 16777:16777 + command: "mq.agent -broker=mq_broker:17777 -port=16777" + depends_on: + - mq_broker + mq_client: + image: chrislusf/seaweedfs:local + # run a custom command instead of entrypoint + command: "ls -al" + depends_on: + - mq_agent diff --git a/docker/compose/local-nextcloud-compose.yml b/docker/compose/local-nextcloud-compose.yml new file mode 100644 index 0000000..52dfb53 --- /dev/null +++ b/docker/compose/local-nextcloud-compose.yml @@ -0,0 +1,44 @@ +version: '3.9' + +services: + master: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "master -ip=master" + volume: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + command: "volume -master=master:9333 -port=8080 -ip=volume" + depends_on: + - master + s3: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + - 8333:8333 + command: '-v 9 filer -master="master:9333" -s3' + depends_on: + - master + - volume + nextcloud: + image: nextcloud:23.0.5-apache + environment: + - OBJECTSTORE_S3_HOST=s3 + - OBJECTSTORE_S3_BUCKET=nextcloud + - OBJECTSTORE_S3_KEY=some_access_key1 + - OBJECTSTORE_S3_SECRET=some_secret_key1 + - OBJECTSTORE_S3_PORT=8333 + - OBJECTSTORE_S3_SSL=false + - OBJECTSTORE_S3_USEPATH_STYLE=true + - SQLITE_DATABASE=nextcloud + - NEXTCLOUD_ADMIN_USER=admin + - NEXTCLOUD_ADMIN_PASSWORD=admin + ports: + - 80:80 + depends_on: + - s3 \ No newline at end of file diff --git a/docker/compose/local-registry-compose.yml b/docker/compose/local-registry-compose.yml new file mode 100644 index 0000000..0a176c1 --- /dev/null +++ b/docker/compose/local-registry-compose.yml @@ -0,0 +1,85 @@ +version: '3.9' + +services: + master: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "master -ip=master -volumeSizeLimitMB=100" + volume: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + command: "volume -master=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1" + depends_on: + - master + s3: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + - 8333:8333 + command: '-v 9 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333' + volumes: + - ./s3.json:/etc/seaweedfs/s3.json + depends_on: + - master + - volume + minio: + image: minio/minio + ports: + - 9000:9000 + command: 'minio server /data' + environment: + MINIO_ACCESS_KEY: "some_access_key1" + MINIO_SECRET_KEY: "some_secret_key1" + depends_on: + - master + registry1: + image: registry:2 + environment: + REGISTRY_HTTP_ADDR: "0.0.0.0:5001" # seaweedfs s3 + REGISTRY_LOG_LEVEL: "debug" + REGISTRY_STORAGE: "s3" + REGISTRY_STORAGE_S3_REGION: "us-east-1" + REGISTRY_STORAGE_S3_REGIONENDPOINT: "http://s3:8333" + REGISTRY_STORAGE_S3_BUCKET: "registry" + REGISTRY_STORAGE_S3_ACCESSKEY: "some_access_key1" + REGISTRY_STORAGE_S3_SECRETKEY: "some_secret_key1" + REGISTRY_STORAGE_S3_V4AUTH: "true" + REGISTRY_STORAGE_S3_SECURE: "false" + REGISTRY_STORAGE_S3_SKIPVERIFY: "true" + REGISTRY_STORAGE_S3_ROOTDIRECTORY: "/" + REGISTRY_STORAGE_DELETE_ENABLED: "true" + REGISTRY_STORAGE_REDIRECT_DISABLE: "true" + REGISTRY_VALIDATION_DISABLED: "true" + ports: + - 5001:5001 + depends_on: + - s3 + - minio + registry2: + image: registry:2 + environment: + REGISTRY_HTTP_ADDR: "0.0.0.0:5002" # minio + REGISTRY_LOG_LEVEL: "debug" + REGISTRY_STORAGE: "s3" + REGISTRY_STORAGE_S3_REGION: "us-east-1" + REGISTRY_STORAGE_S3_REGIONENDPOINT: "http://minio:9000" + REGISTRY_STORAGE_S3_BUCKET: "registry" + REGISTRY_STORAGE_S3_ACCESSKEY: "some_access_key1" + REGISTRY_STORAGE_S3_SECRETKEY: "some_secret_key1" + REGISTRY_STORAGE_S3_V4AUTH: "true" + REGISTRY_STORAGE_S3_SECURE: "false" + REGISTRY_STORAGE_S3_SKIPVERIFY: "true" + REGISTRY_STORAGE_S3_ROOTDIRECTORY: "/" + REGISTRY_STORAGE_DELETE_ENABLED: "true" + REGISTRY_STORAGE_REDIRECT_DISABLE: "true" + REGISTRY_VALIDATION_DISABLED: "true" + ports: + - 5002:5002 + depends_on: + - s3 + - minio \ No newline at end of file diff --git a/docker/compose/local-replicate-compose.yml b/docker/compose/local-replicate-compose.yml new file mode 100644 index 0000000..2bf392a --- /dev/null +++ b/docker/compose/local-replicate-compose.yml @@ -0,0 +1,61 @@ +version: '3.9' + +services: + master: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "master -ip=master" + volume: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + command: "volume -master=master:9333 -port=8080 -ip=volume -preStopSeconds=1" + depends_on: + - master + filer: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + command: '-v=9 filer -master="master:9333"' + restart: on-failure + volumes: + - ./notification.toml:/etc/seaweedfs/notification.toml + depends_on: + - master + - volume + - rabbitmq + - replicate + environment: + RABBIT_SERVER_URL: "amqp://guest:guest@rabbitmq:5672/" + replicate: + image: chrislusf/seaweedfs:local + command: '-v=9 filer.replicate' + restart: on-failure + volumes: + - ./notification.toml:/etc/seaweedfs/notification.toml + - ./replication.toml:/etc/seaweedfs/replication.toml + depends_on: + - rabbitmq + environment: + RABBIT_SERVER_URL: "amqp://guest:guest@rabbitmq:5672/" + s3: + image: chrislusf/seaweedfs:local + ports: + - 8333:8333 + command: 's3 -filer="filer:8888"' + depends_on: + - master + - volume + - filer + rabbitmq: + image: rabbitmq:3.8.10-management-alpine + ports: + - 5672:5672 + - 15671:15671 + - 15672:15672 + environment: + RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "-rabbit log_levels [{connection,error},{queue,debug}]" \ No newline at end of file diff --git a/docker/compose/local-s3tests-compose.yml b/docker/compose/local-s3tests-compose.yml new file mode 100644 index 0000000..2a65138 --- /dev/null +++ b/docker/compose/local-s3tests-compose.yml @@ -0,0 +1,45 @@ +version: '3.9' + +services: + master: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "master -ip=master -volumeSizeLimitMB=16" + environment: + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + volume: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + command: "volume -master=master:9333 -port=8080 -ip=volume -preStopSeconds=1" + depends_on: + - master + s3: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + - 8000:8000 + command: 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowDeleteBucketNotEmpty=false' + volumes: + - ./s3.json:/etc/seaweedfs/s3.json + depends_on: + - master + - volume + s3tests: + image: chrislusf/ceph-s3-tests:local + volumes: + - ./s3tests.conf:/opt/s3-tests/s3tests.conf + environment: + S3TEST_CONF: "s3tests.conf" + NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3" + NOSETESTS_ATTR: "!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy" + NOSETESTS_EXCLUDE: "(post_object_tags_anonymous_request|get_obj_tagging|set_bucket_tagging|post_object_tags_authenticated_request|put_max_tags|put_modify_tags|test_put_obj_with_tags|get_bucket_encryption|delete_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket|list_multipart_upload_owner|multipart_upload_small)" + depends_on: + - master + - volume + - s3 \ No newline at end of file diff --git a/docker/compose/local-sync-mount-compose.yml b/docker/compose/local-sync-mount-compose.yml new file mode 100644 index 0000000..0ce1fde --- /dev/null +++ b/docker/compose/local-sync-mount-compose.yml @@ -0,0 +1,56 @@ +version: '3.9' +services: + node1: + image: chrislusf/seaweedfs:local + command: "server -master -volume -filer" + ports: + - 8888:8888 + - 18888:18888 + healthcheck: + test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ] + interval: 1s + start_period: 10s + timeout: 30s + mount1: + image: chrislusf/seaweedfs:local + privileged: true + command: "mount -filer=node1:8888 -dir=/mnt -dirAutoCreate" + healthcheck: + test: [ "CMD", "curl", "--fail", "-I", "http://node1:8888/" ] + interval: 1s + start_period: 10s + timeout: 30s + depends_on: + node1: + condition: service_healthy + node2: + image: chrislusf/seaweedfs:local + ports: + - 7888:8888 + - 17888:18888 + command: "server -master -volume -filer" + healthcheck: + test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ] + interval: 1s + start_period: 10s + timeout: 30s + mount2: + image: chrislusf/seaweedfs:local + privileged: true + command: "mount -filer=node2:8888 -dir=/mnt -dirAutoCreate" + healthcheck: + test: [ "CMD", "curl", "--fail", "-I", "http://node2:8888/" ] + interval: 1s + start_period: 10s + timeout: 30s + depends_on: + node2: + condition: service_healthy + sync: + image: chrislusf/seaweedfs:local + command: "-v=4 filer.sync -a=node1:8888 -b=node2:8888 -a.debug -b.debug" + depends_on: + mount1: + condition: service_healthy + mount2: + condition: service_healthy diff --git a/docker/compose/master-cloud.toml b/docker/compose/master-cloud.toml new file mode 100644 index 0000000..27bf3f0 --- /dev/null +++ b/docker/compose/master-cloud.toml @@ -0,0 +1,34 @@ + +# Put this file to one of the location, with descending priority +# ./master.toml +# $HOME/.seaweedfs/master.toml +# /etc/seaweedfs/master.toml +# this file is read by master + +[master.maintenance] +# periodically run these scripts are the same as running them from 'weed shell' +# Scripts are skipped while an admin server is connected. +scripts = """ + lock + ec.encode -fullPercent=95 -quietFor=1h + ec.rebuild -apply + ec.balance -apply + volume.balance -apply + volume.fix.replication -apply + unlock +""" +sleep_minutes = 17 # sleep minutes between each script execution + +# configurations for tiered cloud storage +# old volumes are transparently moved to cloud for cost efficiency +[storage.backend] + [storage.backend.s3.default] + enabled = true + aws_access_key_id = "any" # if empty, loads from the shared credentials file (~/.aws/credentials). + aws_secret_access_key = "any" # if empty, loads from the shared credentials file (~/.aws/credentials). + region = "us-east-2" + bucket = "volume_bucket" # an existing bucket + endpoint = "http://server2:8333" + storage_class = "STANDARD_IA" + # upload_concurrency = 5 # concurrent multipart part uploads per volume (volume.tier.upload -concurrent overrides) + # download_concurrency = 5 # concurrent multipart part downloads per volume (volume.tier.download -concurrent overrides) diff --git a/docker/compose/notification.toml b/docker/compose/notification.toml new file mode 100644 index 0000000..d93d2ba --- /dev/null +++ b/docker/compose/notification.toml @@ -0,0 +1,17 @@ +[notification.log] +# this is only for debugging purpose and does not work with "weed filer.replicate" +enabled = false + + +[notification.gocdk_pub_sub] +# The Go Cloud Development Kit (https://gocloud.dev). +# PubSub API (https://godoc.org/gocloud.dev/pubsub). +# Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ. +enabled = true +# This URL will Dial the RabbitMQ server at the URL in the environment +# variable RABBIT_SERVER_URL and open the exchange "myexchange". +# The exchange must have already been created by some other means, like +# the RabbitMQ management plugin. ĐĄreate myexchange of type fanout and myqueue then +# create binding myexchange => myqueue +topic_url = "rabbit://swexchange" +sub_url = "rabbit://swqueue" \ No newline at end of file diff --git a/docker/compose/replication.toml b/docker/compose/replication.toml new file mode 100644 index 0000000..833bb16 --- /dev/null +++ b/docker/compose/replication.toml @@ -0,0 +1,11 @@ +[source.filer] +enabled = true +grpcAddress = "filer:18888" +# all files under this directory tree are replicated. +# this is not a directory on your hard drive, but on your filer. +# i.e., all files with this "prefix" are sent to notification message queue. +directory = "/buckets" + +[sink.local_incremental] +enabled = true +directory = "/data" \ No newline at end of file diff --git a/docker/compose/s3.json b/docker/compose/s3.json new file mode 100644 index 0000000..ce23086 --- /dev/null +++ b/docker/compose/s3.json @@ -0,0 +1,115 @@ +{ + "identities": [ + { + "name": "anonymous", + "actions": [ + "Read" + ] + }, + { + "name": "some_admin_user", + "credentials": [ + { + "accessKey": "some_access_key1", + "secretKey": "some_secret_key1" + } + ], + "actions": [ + "Admin", + "Read", + "List", + "Tagging", + "Write" + ] + }, + { + "name": "s3_tests", + "credentials": [ + { + "accessKey": "ABCDEFGHIJKLMNOPQRST", + "secretKey": "abcdefghijklmnopqrstuvwxyzabcdefghijklmn" + }, + { + "accessKey": "0555b35654ad1656d804", + "secretKey": "h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==" + } + ], + "actions": [ + "Admin", + "Read", + "List", + "Tagging", + "Write" + ], + "account": { + "id": "testid" + } + }, + { + "name": "s3_tests_alt", + "credentials": [ + { + "accessKey": "NOPQRSTUVWXYZABCDEFG", + "secretKey": "nopqrstuvwxyzabcdefghijklmnabcdefghijklm" + } + ], + "actions": [ + "Admin", + "Read", + "List", + "Tagging", + "Write" + ] + }, + { + "name": "s3_tests_tenant", + "credentials": [ + { + "accessKey": "HIJKLMNOPQRSTUVWXYZA", + "secretKey": "opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab" + } + ], + "actions": [ + "Admin", + "Read", + "List", + "Tagging", + "Write" + ] + }, + { + "name": "some_read_only_user", + "credentials": [ + { + "accessKey": "some_access_key2", + "secretKey": "some_secret_key2" + } + ], + "actions": [ + "Read" + ] + }, + { + "name": "some_normal_user", + "credentials": [ + { + "accessKey": "some_access_key3", + "secretKey": "some_secret_key3" + } + ], + "actions": [ + "Read", + "List", + "Tagging", + "Write" + ] + } + ], + "accounts": [ + { + "id" : "testid", + "displayName": "M. Tester", + "emailAddress": "tester@ceph.com" + } + ] +} \ No newline at end of file diff --git a/docker/compose/s3_static_users_example.json b/docker/compose/s3_static_users_example.json new file mode 100644 index 0000000..d8d8734 --- /dev/null +++ b/docker/compose/s3_static_users_example.json @@ -0,0 +1,53 @@ +{ + "identities": [ + { + "name": "admin", + "credentials": [ + { + "accessKey": "AKIAIOSFODNN7EXAMPLE", + "secretKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + } + ], + "actions": [ + "Admin", + "Read", + "List", + "Tagging", + "Write" + ] + }, + { + "name": "steward", + "credentials": [ + { + "accessKey": "steward-key", + "secretKey": "steward-secret" + } + ], + "actions": [ + "Read", + "List", + "Write" + ] + }, + { + "name": "le001", + "credentials": [ + { + "accessKey": "le001-key", + "secretKey": "le001-secret" + } + ], + "actions": [ + "Read", + "List" + ] + }, + { + "name": "anonymous", + "actions": [ + "Read" + ] + } + ] +} diff --git a/docker/compose/s3tests.conf b/docker/compose/s3tests.conf new file mode 100644 index 0000000..3b0629f --- /dev/null +++ b/docker/compose/s3tests.conf @@ -0,0 +1,103 @@ +[DEFAULT] +## this section is just used for host, port and bucket_prefix + +# host set for rgw in vstart.sh +host = 127.0.0.1 + +# port set for rgw in vstart.sh +port = 8000 + +## say "False" to disable TLS +is_secure = False + +[fixtures] +## all the buckets created will start with this prefix; +## {random} will be filled with random characters to pad +## the prefix to 30 characters long, and avoid collisions +bucket prefix = yournamehere-{random}- + +[s3 main] +# main display_name set in vstart.sh +display_name = M. Tester + +# main user_idname set in vstart.sh +user_id = testid + +# main email set in vstart.sh +email = tester@ceph.com + +# zonegroup api_name for bucket location +api_name = default + +## main AWS access key +access_key = 0555b35654ad1656d804 + +## main AWS secret key +secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== + +## replace with key id obtained when secret is created, or delete if KMS not tested +#kms_keyid = 01234567-89ab-cdef-0123-456789abcdef + +[s3 alt] +# alt display_name set in vstart.sh +display_name = john.doe +## alt email set in vstart.sh +email = john.doe@example.com + +# alt user_id set in vstart.sh +user_id = 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234 + +# alt AWS access key set in vstart.sh +access_key = NOPQRSTUVWXYZABCDEFG + +# alt AWS secret key set in vstart.sh +secret_key = nopqrstuvwxyzabcdefghijklmnabcdefghijklm + +[s3 tenant] +# tenant display_name set in vstart.sh +display_name = testx$tenanteduser + +# tenant user_id set in vstart.sh +user_id = 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef + +# tenant AWS secret key set in vstart.sh +access_key = HIJKLMNOPQRSTUVWXYZA + +# tenant AWS secret key set in vstart.sh +secret_key = opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab + +# tenant email set in vstart.sh +email = tenanteduser@example.com + +# tenant name +tenant = testx + +[iam] +#used for iam operations in sts-tests +#email from vstart.sh +email = s3@example.com + +#user_id from vstart.sh +user_id = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + +#access_key from vstart.sh +access_key = ABCDEFGHIJKLMNOPQRST + +#secret_key from vstart.sh +secret_key = abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz + +#display_name from vstart.sh +display_name = youruseridhere + +[iam root] +access_key = AAAAAAAAAAAAAAAAAAaa +secret_key = aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user_id = RGW11111111111111111 +email = account1@ceph.com + +# iam account root user in a different account than [iam root] +[iam alt root] +access_key = BBBBBBBBBBBBBBBBBBbb +secret_key = bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +user_id = RGW22222222222222222 +email = account2@ceph.com \ No newline at end of file diff --git a/docker/compose/swarm-etcd.yml b/docker/compose/swarm-etcd.yml new file mode 100644 index 0000000..ba49699 --- /dev/null +++ b/docker/compose/swarm-etcd.yml @@ -0,0 +1,82 @@ +# 2021-01-30 16:25:30 +services: + + etcd: + image: gasparekatapy/etcd + networks: + - net + deploy: + mode: replicated + replicas: 3 + + master: + image: chrislusf/seaweedfs:local + environment: + WEED_MASTER_FILER_DEFAULT: "filer:8888" + WEED_MASTER_SEQUENCER_TYPE: "raft" + ports: + - "9333:9333" + - "19333:19333" + networks: + - net + command: + - 'master' + - '-resumeState=true' + - '-ip=master' + - '-port=9333' + deploy: + mode: replicated + replicas: 1 + + filer: + image: chrislusf/seaweedfs:local + environment: + WEED_LEVELDB2_ENABLED: "false" + WEED_ETCD_ENABLED: "true" + WEED_ETCD_SERVERS: "etcd:2379" + ports: + - target: 8888 + published: 8888 + protocol: tcp + mode: host + - target: 18888 + published: 18888 + protocol: tcp + mode: host + networks: + - net + command: + - 'filer' + - '-ip=filer' + - '-port=8888' + - '-port.readonly=28888' + - '-master=master:9333' + - '-disableDirListing=true' + deploy: + mode: replicated + replicas: 1 + + volume: + image: chrislusf/seaweedfs:local + ports: + - target: 8080 + published: 8080 + protocol: tcp + mode: host + - target: 18080 + published: 18080 + protocol: tcp + mode: host + networks: + - net + command: + - 'volume' + - '-master=master:9333' + - '-port=8080' + deploy: + mode: global + + ########################################################################### + +networks: + net: diff --git a/docker/compose/test-etcd-filer.yml b/docker/compose/test-etcd-filer.yml new file mode 100644 index 0000000..6d37ae5 --- /dev/null +++ b/docker/compose/test-etcd-filer.yml @@ -0,0 +1,62 @@ +version: '3.9' + +services: + etcd: + image: quay.io/coreos/etcd:v3.5.4 + command: "etcd --advertise-client-urls http://etcd:2379 --listen-client-urls http://0.0.0.0:2379" + ports: + - 2379:2379 + master: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "master -ip=master -volumeSizeLimitMB=100" + volume: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + command: "volume -master=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1" + depends_on: + - master + s3: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + - 8333:8333 + command: '-v 9 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333' + environment: + WEED_LEVELDB2_ENABLED: 'false' + WEED_ETCD_ENABLED: 'true' + WEED_ETCD_KEY_PREFIX: 'seaweedfs.' + WEED_ETCD_SERVERS: "http://etcd:2379" + volumes: + - ./s3.json:/etc/seaweedfs/s3.json + depends_on: + - etcd + - master + - volume + registry: + image: registry:2 + environment: + REGISTRY_HTTP_ADDR: "0.0.0.0:5001" # seaweedfs s3 + REGISTRY_LOG_LEVEL: "debug" + REGISTRY_STORAGE: "s3" + REGISTRY_STORAGE_S3_REGION: "us-east-1" + REGISTRY_STORAGE_S3_REGIONENDPOINT: "http://s3:8333" + REGISTRY_STORAGE_S3_BUCKET: "registry" + REGISTRY_STORAGE_S3_ACCESSKEY: "some_access_key1" + REGISTRY_STORAGE_S3_SECRETKEY: "some_secret_key1" + REGISTRY_STORAGE_S3_V4AUTH: "true" + REGISTRY_STORAGE_S3_SECURE: "false" + REGISTRY_STORAGE_S3_SKIPVERIFY: "true" + REGISTRY_STORAGE_S3_ROOTDIRECTORY: "/" + REGISTRY_STORAGE_DELETE_ENABLED: "true" + REGISTRY_STORAGE_REDIRECT_DISABLE: "true" + REGISTRY_VALIDATION_DISABLED: "true" + ports: + - 5001:5001 + depends_on: + - s3 diff --git a/docker/compose/test-git-on-mount.sh b/docker/compose/test-git-on-mount.sh new file mode 100755 index 0000000..50fe588 --- /dev/null +++ b/docker/compose/test-git-on-mount.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash +# +# Integration test: git clone & pull on a SeaweedFS FUSE mount. +# +# Verifies that the mount correctly supports git's file operations by: +# 1. Creating a bare repo on the mount (acts as a remote) +# 2. Cloning it, making commits, and pushing back to the mount +# 3. Cloning from the mount into a working directory also on the mount +# 4. Pushing additional commits to the bare repo +# 5. Checking out an older revision in the on-mount clone +# 6. Running git pull to fast-forward with real changes +# 7. Verifying file content integrity at each step +# +# Usage: +# bash test-git-on-mount.sh /path/to/mount/point +# +# The mount must already be running. All test artifacts are created under +# /git-test- and cleaned up on exit (unless TEST_KEEP=1). +# +set -euo pipefail + +MOUNT_DIR="${1:?Usage: $0 }" +TEST_DIR="$MOUNT_DIR/git-test-$$" +LOCAL_DIR=$(mktemp -d) +PASS=0 +FAIL=0 + +cleanup() { + if [[ "${TEST_KEEP:-}" == "1" ]]; then + echo "TEST_KEEP=1 — leaving artifacts:" + echo " mount: $TEST_DIR" + echo " local: $LOCAL_DIR" + else + rm -rf "$TEST_DIR" 2>/dev/null || true + rm -rf "$LOCAL_DIR" 2>/dev/null || true + fi +} +trap cleanup EXIT + +# --- helpers --------------------------------------------------------------- + +pass() { PASS=$((PASS + 1)); echo " PASS: $1"; } +fail() { FAIL=$((FAIL + 1)); echo " FAIL: $1"; } + +assert_file_contains() { + local file=$1 expected=$2 label=$3 + if [[ -f "$file" ]] && grep -qF "$expected" "$file" 2>/dev/null; then + pass "$label" + else + fail "$label (expected '$expected' in $file)" + fi +} + +assert_file_exists() { + local file=$1 label=$2 + if [[ -f "$file" ]]; then + pass "$label" + else + fail "$label ($file not found)" + fi +} + +assert_file_not_exists() { + local file=$1 label=$2 + if [[ ! -f "$file" ]]; then + pass "$label" + else + fail "$label ($file should not exist)" + fi +} + +assert_eq() { + local actual=$1 expected=$2 label=$3 + if [[ "$actual" == "$expected" ]]; then + pass "$label" + else + fail "$label (expected '$expected', got '$actual')" + fi +} + +# --- setup ----------------------------------------------------------------- + +echo "========================================" +echo " Git-on-mount integration test" +echo "========================================" +echo "Mount: $MOUNT_DIR" +echo "Test: $TEST_DIR" +echo "Local: $LOCAL_DIR" +echo "" + +if ! mountpoint -q "$MOUNT_DIR" 2>/dev/null && [[ ! -d "$MOUNT_DIR" ]]; then + echo "ERROR: $MOUNT_DIR is not a valid directory" + exit 1 +fi + +mkdir -p "$TEST_DIR" + +# --- Phase 1: Create bare repo on mount ----------------------------------- + +echo "--- Phase 1: Create bare repo on mount ---" + +BARE_REPO="$TEST_DIR/repo.git" +git init --bare "$BARE_REPO" >/dev/null 2>&1 +pass "bare repo created on mount" + +# --- Phase 2: Clone locally, make initial commits, push ------------------- + +echo "--- Phase 2: Clone locally, make initial commits, push ---" + +LOCAL_CLONE="$LOCAL_DIR/clone1" +git clone "$BARE_REPO" "$LOCAL_CLONE" >/dev/null 2>&1 +cd "$LOCAL_CLONE" +git config user.email "test@seaweedfs.test" +git config user.name "SeaweedFS Test" + +# Commit 1: initial files +echo "hello world" > README.md +mkdir -p src +echo 'package main; import "fmt"; func main() { fmt.Println("v1") }' > src/main.go +git add -A && git commit -m "initial commit" >/dev/null 2>&1 +COMMIT1=$(git rev-parse HEAD) + +# Commit 2: add more files +mkdir -p data +for i in $(seq 1 20); do + printf "file-%03d: %s\n" "$i" "$(head -c 64 /dev/urandom | base64)" > "data/file-$(printf '%03d' $i).txt" +done +git add -A && git commit -m "add data files" >/dev/null 2>&1 +COMMIT2=$(git rev-parse HEAD) + +# Commit 3: modify and add +echo 'package main; import "fmt"; func main() { fmt.Println("v2") }' > src/main.go +echo "# Updated readme" >> README.md +mkdir -p docs +echo "documentation content" > docs/guide.md +git add -A && git commit -m "update src and add docs" >/dev/null 2>&1 +COMMIT3=$(git rev-parse HEAD) + +git push origin master >/dev/null 2>&1 || git push origin main >/dev/null 2>&1 +BRANCH=$(git rev-parse --abbrev-ref HEAD) +pass "3 commits pushed to mount bare repo (branch=$BRANCH)" + +# --- Phase 3: Clone from mount bare repo to mount working dir ------------- + +echo "--- Phase 3: Clone from mount bare repo to on-mount working dir ---" + +MOUNT_CLONE="$TEST_DIR/working" +git clone "$BARE_REPO" "$MOUNT_CLONE" >/dev/null 2>&1 + +# Verify clone integrity +assert_file_exists "$MOUNT_CLONE/README.md" "README.md exists after clone" +assert_file_contains "$MOUNT_CLONE/README.md" "# Updated readme" "README.md has latest content" +assert_file_contains "$MOUNT_CLONE/src/main.go" 'v2' "src/main.go has v2" +assert_file_exists "$MOUNT_CLONE/docs/guide.md" "docs/guide.md exists" +assert_file_exists "$MOUNT_CLONE/data/file-001.txt" "data files exist" +assert_file_exists "$MOUNT_CLONE/data/file-020.txt" "data/file-020.txt exists" + +CLONE_HEAD=$(cd "$MOUNT_CLONE" && git rev-parse HEAD) +assert_eq "$CLONE_HEAD" "$COMMIT3" "on-mount clone HEAD matches commit 3" + +# Count files +FILE_COUNT=$(find "$MOUNT_CLONE/data" -name '*.txt' | wc -l | tr -d ' ') +assert_eq "$FILE_COUNT" "20" "data/ has 20 files" + +# --- Phase 4: Push more commits from local clone -------------------------- + +echo "--- Phase 4: Push more commits from local clone ---" + +cd "$LOCAL_CLONE" + +# Commit 4: larger changes +for i in $(seq 21 50); do + printf "file-%03d: %s\n" "$i" "$(head -c 128 /dev/urandom | base64)" > "data/file-$(printf '%03d' $i).txt" +done +echo 'package main; import "fmt"; func main() { fmt.Println("v3") }' > src/main.go +git add -A && git commit -m "expand data and update to v3" >/dev/null 2>&1 +COMMIT4=$(git rev-parse HEAD) + +# Commit 5: rename and delete +git mv docs/guide.md docs/manual.md +git rm data/file-001.txt >/dev/null 2>&1 +git commit -m "rename guide, remove file-001" >/dev/null 2>&1 +COMMIT5=$(git rev-parse HEAD) + +git push origin "$BRANCH" >/dev/null 2>&1 +pass "2 more commits pushed (5 total)" + +# --- Phase 5: Checkout older revision in on-mount clone ------------------- + +echo "--- Phase 5: Checkout older revision in on-mount clone ---" + +cd "$MOUNT_CLONE" +git checkout "$COMMIT2" >/dev/null 2>&1 + +# Verify we're at commit 2 state +DETACHED_HEAD=$(git rev-parse HEAD) +assert_eq "$DETACHED_HEAD" "$COMMIT2" "on-mount clone at commit 2 (detached)" +assert_file_not_exists "$MOUNT_CLONE/docs/guide.md" "docs/guide.md not in commit 2" +assert_file_contains "$MOUNT_CLONE/src/main.go" 'v1' "src/main.go has v1 at commit 2" + +# --- Phase 6: Return to branch and pull ----------------------------------- + +echo "--- Phase 6: Return to branch and pull with real changes ---" + +cd "$MOUNT_CLONE" +git checkout "$BRANCH" >/dev/null 2>&1 +# At this point the on-mount clone is at commit 3, remote is at commit 5 +OLD_HEAD=$(git rev-parse HEAD) +assert_eq "$OLD_HEAD" "$COMMIT3" "on-mount clone at commit 3 before pull" + +git pull >/dev/null 2>&1 +NEW_HEAD=$(git rev-parse HEAD) +assert_eq "$NEW_HEAD" "$COMMIT5" "HEAD matches commit 5 after pull" + +# Verify commit 5 state +assert_file_contains "$MOUNT_CLONE/src/main.go" 'v3' "src/main.go has v3 after pull" +assert_file_exists "$MOUNT_CLONE/docs/manual.md" "docs/manual.md exists (renamed)" +assert_file_not_exists "$MOUNT_CLONE/docs/guide.md" "docs/guide.md gone (renamed)" +assert_file_not_exists "$MOUNT_CLONE/data/file-001.txt" "data/file-001.txt removed" +assert_file_exists "$MOUNT_CLONE/data/file-050.txt" "data/file-050.txt exists" + +FINAL_COUNT=$(find "$MOUNT_CLONE/data" -name '*.txt' | wc -l | tr -d ' ') +assert_eq "$FINAL_COUNT" "49" "data/ has 49 files (50 added, 1 removed)" + +# --- Phase 7: Verify git log integrity ----------------------------------- + +echo "--- Phase 7: Verify git log integrity ---" + +cd "$MOUNT_CLONE" +GIT_LOG=$(git log --format=%s) +LOG_COUNT=$(echo "$GIT_LOG" | wc -l | tr -d ' ') +assert_eq "$LOG_COUNT" "5" "git log shows 5 commits" + +# Verify commit messages +echo "$GIT_LOG" | grep -qF "initial commit" && pass "commit 1 message in log" || fail "commit 1 message missing" +echo "$GIT_LOG" | grep -qF "expand data" && pass "commit 4 message in log" || fail "commit 4 message missing" +echo "$GIT_LOG" | grep -qF "rename guide" && pass "commit 5 message in log" || fail "commit 5 message missing" + +# --- Phase 8: Verify git status is clean ---------------------------------- + +echo "--- Phase 8: Verify git status is clean ---" + +cd "$MOUNT_CLONE" +STATUS=$(git status --porcelain) +if [[ -z "$STATUS" ]]; then + pass "git status is clean" +else + fail "git status has changes: $STATUS" +fi + +# --- Results --------------------------------------------------------------- + +echo "" +echo "========================================" +echo " Results: $PASS passed, $FAIL failed" +echo "========================================" + +if [[ "$FAIL" -gt 0 ]]; then + exit 1 +fi diff --git a/docker/compose/test-keycloak-s3.yml b/docker/compose/test-keycloak-s3.yml new file mode 100644 index 0000000..ec4d135 --- /dev/null +++ b/docker/compose/test-keycloak-s3.yml @@ -0,0 +1,46 @@ +services: + keycloak: + image: quay.io/keycloak/keycloak:26.0.7 + command: ["start-dev", "--import-realm"] + environment: + KC_BOOTSTRAP_ADMIN_USERNAME: admin + KC_BOOTSTRAP_ADMIN_PASSWORD: admin + KC_HTTP_ENABLED: "true" + KC_HOSTNAME: "http://keycloak:8080" + KC_HOSTNAME_STRICT: "false" + ports: + - "8080:8080" + volumes: + - ../../test/s3/iam/seaweedfs-test-realm.json:/opt/keycloak/data/import/seaweedfs-test-realm.json:ro + healthcheck: + test: ["CMD-SHELL", "exec 3<>/dev/tcp/localhost/8080"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 60s + + s3: + image: chrislusf/seaweedfs:local + command: + - -v=9 + - server + - -ip=s3 + - -filer + - -master.volumeSizeLimitMB=16 + - -volume + - -volume.max=0 + - -volume.preStopSeconds=1 + - -s3 + - -s3.port=8333 + - -s3.config=/etc/seaweedfs/s3_config.json + - -s3.iam.config=/etc/seaweedfs/iam_config.json + environment: + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + volumes: + - ../../test/s3/iam/test_config.json:/etc/seaweedfs/s3_config.json:ro + - ../../test/s3/iam/iam_config_docker.json:/etc/seaweedfs/iam_config.json:ro + ports: + - "8333:8333" + depends_on: + keycloak: + condition: service_healthy diff --git a/docker/compose/test-tarantool-filer.yml b/docker/compose/test-tarantool-filer.yml new file mode 100644 index 0000000..a0fa543 --- /dev/null +++ b/docker/compose/test-tarantool-filer.yml @@ -0,0 +1,30 @@ +version: '3.9' + +services: + tarantool: + image: chrislusf/tarantool_dev_env + entrypoint: "tt start app -i" + environment: + APP_USER_PASSWORD: "app" + CLIENT_USER_PASSWORD: "client" + REPLICATOR_USER_PASSWORD: "replicator" + STORAGE_USER_PASSWORD: "storage" + network_mode: "host" + ports: + - "3303:3303" + + s3: + image: chrislusf/seaweedfs:local + command: "server -ip=127.0.0.1 -filer -master.volumeSizeLimitMB=16 -volume.max=0 -volume -volume.preStopSeconds=1 -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowDeleteBucketNotEmpty=false" + volumes: + - ./s3.json:/etc/seaweedfs/s3.json + environment: + WEED_LEVELDB2_ENABLED: "false" + WEED_TARANTOOL_ENABLED: "true" + WEED_TARANTOOL_ADDRESS: "127.0.0.1:3303" + WEED_TARANTOOL_USER: "client" + WEED_TARANTOOL_PASSWORD: "client" + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + network_mode: "host" + depends_on: + - tarantool \ No newline at end of file diff --git a/docker/compose/test-ydb-filer.yml b/docker/compose/test-ydb-filer.yml new file mode 100644 index 0000000..1e310df --- /dev/null +++ b/docker/compose/test-ydb-filer.yml @@ -0,0 +1,35 @@ +version: '3.9' + +services: + ydb: + image: cr.yandex/yc/yandex-docker-local-ydb + ports: + - 2135:2135 + - 8765:8765 + - 2136:2136 + environment: + - YDB_DEFAULT_LOG_LEVEL=DEBUG + - GRPC_TLS_PORT=2135 + - GRPC_PORT=2136 + - MON_PORT=8765 + s3: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + - 8888:8888 + - 8000:8000 + - 18888:18888 + command: "server -ip=s3 -filer -master.volumeSizeLimitMB=16 -volume.max=0 -volume -volume.preStopSeconds=1 -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowDeleteBucketNotEmpty=false" + volumes: + - ./s3.json:/etc/seaweedfs/s3.json + environment: + WEED_LEVELDB2_ENABLED: "false" + WEED_YDB_ENABLED: "true" + WEED_YDB_DSN: "grpc://ydb:2136/?database=local" + WEED_YDB_PREFIX: "seaweedfs" + YDB_ANONYMOUS_CREDENTIALS: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + depends_on: + - ydb \ No newline at end of file diff --git a/docker/compose/tls.env b/docker/compose/tls.env new file mode 100644 index 0000000..d80a3ab --- /dev/null +++ b/docker/compose/tls.env @@ -0,0 +1,20 @@ +WEED_GRPC_CA=/etc/seaweedfs/tls/SeaweedFS_CA.crt +WEED_GRPC_ALLOWED_WILDCARD_DOMAIN=".dev" +WEED_GRPC_MASTER_CERT=/etc/seaweedfs/tls/master01.dev.crt +WEED_GRPC_MASTER_KEY=/etc/seaweedfs/tls/master01.dev.key +WEED_GRPC_VOLUME_CERT=/etc/seaweedfs/tls/volume01.dev.crt +WEED_GRPC_VOLUME_KEY=/etc/seaweedfs/tls/volume01.dev.key +WEED_GRPC_FILER_CERT=/etc/seaweedfs/tls/filer01.dev.crt +WEED_GRPC_FILER_KEY=/etc/seaweedfs/tls/filer01.dev.key +WEED_GRPC_CLIENT_CERT=/etc/seaweedfs/tls/client01.dev.crt +WEED_GRPC_CLIENT_KEY=/etc/seaweedfs/tls/client01.dev.key +WEED_GRPC_MASTER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev" +WEED_GRPC_VOLUME_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev" +WEED_GRPC_FILER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev" +WEED_GRPC_CLIENT_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev" +WEED_HTTPS_CLIENT_ENABLE=true +WEED_HTTPS_VOLUME_CERT=/etc/seaweedfs/tls/volume01.dev.crt +WEED_HTTPS_VOLUME_KEY=/etc/seaweedfs/tls/volume01.dev.key +WEED_HTTPS_VOLUME_CA=/etc/seaweedfs/tls/SeaweedFS_CA.crt +#GRPC_GO_LOG_SEVERITY_LEVEL=info +#GRPC_GO_LOG_VERBOSITY_LEVEL=2 \ No newline at end of file diff --git a/docker/compose/userstore.json b/docker/compose/userstore.json new file mode 100644 index 0000000..e9f8d8a --- /dev/null +++ b/docker/compose/userstore.json @@ -0,0 +1,37 @@ +[ + { + "Username": "admin", + "Password": "myadminpassword", + "PublicKeys": [ + ], + "HomeDir": "/", + "Permissions": { + "/": ["*"] + }, + "Uid": 0, + "Gid": 0 + }, + { + "Username": "user1", + "Password": "myuser1password", + "PublicKeys": [""], + "HomeDir": "/user1", + "Permissions": { + "/user1": ["*"], + "/public": ["read", "list","write"] + }, + "Uid": 1111, + "Gid": 1111 + }, + { + "Username": "readonly", + "Password": "myreadonlypassword", + "PublicKeys": [], + "HomeDir": "/public", + "Permissions": { + "/public": ["read", "list"] + }, + "Uid": 1112, + "Gid": 1112 + } +] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..0bbd507 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,142 @@ +#!/bin/sh + +# Enable FIPS 140-3 mode by default (Go 1.24+) +# To disable: docker run -e GODEBUG=fips140=off ... +export GODEBUG="${GODEBUG:+$GODEBUG,}fips140=on" + +# Fix permissions for mounted volumes +# If /data is mounted from host, it might have different ownership +# Fix this by ensuring seaweed user owns the directory +if [ "$(id -u)" = "0" ]; then + # Running as root, check and fix permissions if needed + SEAWEED_UID=$(id -u seaweed) + SEAWEED_GID=$(id -g seaweed) + + # Verify seaweed user and group exist + if [ -z "$SEAWEED_UID" ] || [ -z "$SEAWEED_GID" ]; then + echo "Error: 'seaweed' user or group not found. Cannot fix permissions." >&2 + exit 1 + fi + + DATA_UID=$(stat -c '%u' /data 2>/dev/null) + DATA_GID=$(stat -c '%g' /data 2>/dev/null) + + # Only run chown -R if ownership doesn't already match (avoids expensive + # recursive chown on subsequent starts, and is a no-op on OpenShift when + # fsGroup has already set correct ownership on the PVC). + if [ "$DATA_UID" != "$SEAWEED_UID" ] || [ "$DATA_GID" != "$SEAWEED_GID" ]; then + echo "Fixing /data ownership for seaweed user (uid=$SEAWEED_UID, gid=$SEAWEED_GID)" + if ! chown -R seaweed:seaweed /data; then + echo "Warning: Failed to change ownership of /data. This may cause permission errors." >&2 + echo "If /data is read-only or has mount issues, the application may fail to start." >&2 + fi + fi + + # Use su-exec to drop privileges and run as seaweed user + exec su-exec seaweed "$0" "$@" +fi + +isArgPassed() { + # Match both `-flag` and `--flag` (and their `=value` forms): the Go fla9 + # library accepts both, and users may pick either form on the CLI. + arg="$1" + argWithEqualSign="$1=" + argDouble="-$1" + argDoubleWithEqualSign="-$1=" + shift + while [ $# -gt 0 ]; do + passedArg="$1" + shift + case $passedArg in + "$arg"|"$argDouble") + return 0 + ;; + "$argWithEqualSign"*|"$argDoubleWithEqualSign"*) + return 0 + ;; + esac + done + return 1 +} + +case "$1" in + + 'master') + ARGS="-mdir=/data -volumeSizeLimitMB=1024" + shift + exec /usr/bin/weed -logtostderr=true master $ARGS $@ + ;; + + 'volume') + ARGS="-dir=/data -max=0" + if isArgPassed "-max" "$@"; then + ARGS="-dir=/data" + fi + shift + exec /usr/bin/weed -logtostderr=true volume $ARGS $@ + ;; + + 'volume-rust') + ARGS="-dir /data -max 0" + if isArgPassed "-max" "$@"; then + ARGS="-dir /data" + fi + shift + if [ ! -s /usr/bin/weed-volume ]; then + echo "Error: Rust volume server is not available on this platform ($(uname -m))." >&2 + echo "Use 'volume' for the Go volume server instead." >&2 + exit 1 + fi + exec /usr/bin/weed-volume $ARGS $@ + ;; + + 'worker-rust') + shift + if [ ! -s /usr/bin/weed-worker ]; then + echo "Error: Rust maintenance worker is not available on this platform ($(uname -m))." >&2 + echo "Use 'worker' for the Go maintenance worker instead." >&2 + exit 1 + fi + exec /usr/bin/weed-worker "$@" + ;; + + 'server') + ARGS="-dir=/data -volume.max=0 -master.volumeSizeLimitMB=1024" + if isArgPassed "-volume.max" "$@"; then + ARGS="-dir=/data -master.volumeSizeLimitMB=1024" + fi + shift + exec /usr/bin/weed -logtostderr=true server $ARGS $@ + ;; + + 'mini') + ARGS="-dir=/data" + if isArgPassed "-dir" "$@"; then + ARGS="" + fi + shift + exec /usr/bin/weed -logtostderr=true mini $ARGS $@ + ;; + + 'filer') + ARGS="" + shift + exec /usr/bin/weed -logtostderr=true filer $ARGS $@ + ;; + + 's3') + ARGS="-domainName=$S3_DOMAIN_NAME -key.file=$S3_KEY_FILE -cert.file=$S3_CERT_FILE" + shift + exec /usr/bin/weed -logtostderr=true s3 $ARGS $@ + ;; + + 'shell') + ARGS="-cluster=$SHELL_CLUSTER -filer=$SHELL_FILER -filerGroup=$SHELL_FILER_GROUP -master=$SHELL_MASTER -options=$SHELL_OPTIONS" + shift + exec echo "$@" | /usr/bin/weed -logtostderr=true shell $ARGS + ;; + + *) + exec /usr/bin/weed $@ + ;; +esac diff --git a/docker/entrypoint_e2e.sh b/docker/entrypoint_e2e.sh new file mode 100755 index 0000000..291cc04 --- /dev/null +++ b/docker/entrypoint_e2e.sh @@ -0,0 +1,86 @@ +#!/bin/bash +set -e + +# Simplified entrypoint for e2e testing +# +# This script intentionally runs as root for e2e test environments to: +# 1. Simplify test setup and avoid permission-related test failures +# 2. Eliminate dependency on Alpine-specific tools (su-exec) since we use Ubuntu base +# 3. Focus testing on application logic rather than container security +# +# IMPORTANT: Production deployments should use Dockerfile.go_build with proper +# user isolation via su-exec. This simplified approach is ONLY for testing. + +isArgPassed() { + arg="$1" + argWithEqualSign="$1=" + shift + while [ $# -gt 0 ]; do + passedArg="$1" + shift + case $passedArg in + "$arg") + return 0 + ;; + "$argWithEqualSign"*) + return 0 + ;; + esac + done + return 1 +} + +case "$1" in + + 'master') + ARGS="-mdir=/data -volumeSizeLimitMB=1024" + shift + exec /usr/bin/weed -logtostderr=true master $ARGS "$@" + ;; + + 'volume') + ARGS="-dir=/data -max=0" + if isArgPassed "-max" "$@"; then + ARGS="-dir=/data" + fi + shift + exec /usr/bin/weed -logtostderr=true volume $ARGS "$@" + ;; + + 'server') + ARGS="-dir=/data -volume.max=0 -master.volumeSizeLimitMB=1024" + if isArgPassed "-volume.max" "$@"; then + ARGS="-dir=/data -master.volumeSizeLimitMB=1024" + fi + shift + exec /usr/bin/weed -logtostderr=true server $ARGS "$@" + ;; + + 'filer') + ARGS="" + shift + exec /usr/bin/weed -logtostderr=true filer $ARGS "$@" + ;; + + 's3') + ARGS="-domainName=$S3_DOMAIN_NAME -key.file=$S3_KEY_FILE -cert.file=$S3_CERT_FILE" + shift + exec /usr/bin/weed -logtostderr=true s3 $ARGS "$@" + ;; + + 'mount') + shift + exec /usr/bin/weed -logtostderr=true mount "$@" + ;; + + 'shell') + ARGS="-cluster=$SHELL_CLUSTER -filer=$SHELL_FILER -filerGroup=$SHELL_FILER_GROUP -master=$SHELL_MASTER -options=$SHELL_OPTIONS" + shift + exec echo "$@" | /usr/bin/weed -logtostderr=true shell $ARGS + ;; + + *) + exec /usr/bin/weed "$@" + ;; +esac + diff --git a/docker/filer.toml b/docker/filer.toml new file mode 100644 index 0000000..a11e5de --- /dev/null +++ b/docker/filer.toml @@ -0,0 +1,3 @@ +[leveldb2] +enabled = true +dir = "/data/filerldb2" diff --git a/docker/filer_foundationdb.toml b/docker/filer_foundationdb.toml new file mode 100644 index 0000000..6b8a00c --- /dev/null +++ b/docker/filer_foundationdb.toml @@ -0,0 +1,19 @@ +[filer.options] +# with http DELETE, by default the filer would check whether a folder is empty. +# recursive_delete will delete all sub folders and files, similar to "rm -Rf" +recursive_delete = false + +#################################################### +# FoundationDB store +#################################################### +[foundationdb] +enabled = true +cluster_file = "/etc/foundationdb/fdb.cluster" +api_version = 740 +# Optional: timeout for FDB operations (default: 10s) +# timeout = "10s" +# Optional: max retry delay for retryable errors (default: 1s) +# max_retry_delay = "1s" +# Optional: directory prefix for storing SeaweedFS data (default: "seaweedfs") +# directory_prefix = "seaweedfs" + diff --git a/docker/filer_rocksdb.toml b/docker/filer_rocksdb.toml new file mode 100644 index 0000000..c1c74a6 --- /dev/null +++ b/docker/filer_rocksdb.toml @@ -0,0 +1,3 @@ +[rocksdb] +enabled = true +dir = "/data/filer_rocksdb" diff --git a/docker/get_fdb_checksum.sh b/docker/get_fdb_checksum.sh new file mode 100755 index 0000000..73f9755 --- /dev/null +++ b/docker/get_fdb_checksum.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Helper script to get SHA256 checksum for FoundationDB client package +# Usage: ./get_fdb_checksum.sh [arch] +# Example: ./get_fdb_checksum.sh 7.4.5 amd64 +# Example: ./get_fdb_checksum.sh 7.4.5 arm64 + +set -euo pipefail + +if [ $# -lt 1 ] || [ $# -gt 2 ]; then + echo "Usage: $0 [arch]" >&2 + echo "Example: $0 7.4.5" >&2 + echo "Example: $0 7.4.5 arm64" >&2 + exit 1 +fi + +FDB_VERSION="$1" +FDB_ARCH="${2:-amd64}" + +case "$FDB_ARCH" in + "amd64") + CANONICAL_ARCH="amd64" + PACKAGE_ARCH="amd64" + ;; + "arm64"|"aarch64") + CANONICAL_ARCH="arm64" + PACKAGE_ARCH="aarch64" + ;; + *) + echo "Error: Architecture must be 'amd64', 'arm64', or 'aarch64'" >&2 + exit 1 + ;; +esac + +PACKAGE="foundationdb-clients_${FDB_VERSION}-1_${PACKAGE_ARCH}.deb" +URL="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/${PACKAGE}" + +echo "Downloading FoundationDB ${FDB_VERSION} client package for ${FDB_ARCH}..." +echo "URL: ${URL}" +echo "" + +# Download to temp directory +TEMP_DIR=$(mktemp -d) +trap 'rm -rf "${TEMP_DIR}"' EXIT + +cd "${TEMP_DIR}" +if wget --timeout=30 --tries=3 -q "${URL}"; then + CHECKSUM=$(sha256sum "${PACKAGE}" | awk '{print $1}') + echo "✓ Download successful" + echo "" + echo "SHA256 Checksum:" + echo "${CHECKSUM}" + echo "" + echo "Add this to Dockerfile.foundationdb_large:" + echo " \"${FDB_VERSION}_${CANONICAL_ARCH}\") \\" + echo " EXPECTED_SHA256=\"${CHECKSUM}\" ;; \\" +else + echo "✗ Failed to download package from ${URL}" >&2 + echo "Please verify the version number, architecture, and URL" >&2 + exit 1 +fi + diff --git a/docker/nginx/proxy.conf b/docker/nginx/proxy.conf new file mode 100644 index 0000000..59ff30c --- /dev/null +++ b/docker/nginx/proxy.conf @@ -0,0 +1,30 @@ +# HTTP 1.1 support +proxy_http_version 1.1; +#proxy_buffering off; +proxy_set_header Host $http_host; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection $proxy_connection; +proxy_set_header X-Real-IP $remote_addr; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; +proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl; +proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port; + +# Mitigate httpoxy attack (see README for details) +proxy_set_header Proxy ""; + +# aws default max_concurrent_requests 10 +# aws default multipart_threshold 8MB +proxy_buffering on; # GET buffering or “X-Accel-Buffering” enables or disables buffering of a response; +proxy_buffers 64 1m; # buffers used for reading a response from the proxied server, for a single connection +proxy_buffer_size 8k; # maximum size of the data that nginx can receive from the server at a time is set +proxy_busy_buffers_size 2m; + +proxy_request_buffering on; # PUT buffering +client_body_buffer_size 64m; # buffer size for reading client request body +client_max_body_size 64m; + +proxy_next_upstream error timeout non_idempotent http_500; # PUT request should be passed to the next server: +proxy_connect_timeout 200ms; +proxy_read_timeout 3s; #timeout is set only between two successive read operations +proxy_send_timeout 3s; #timeout is set only between two successive write operations diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml new file mode 100644 index 0000000..3fd4c3e --- /dev/null +++ b/docker/prometheus/prometheus.yml @@ -0,0 +1,18 @@ +global: + scrape_interval: 30s + scrape_timeout: 10s + +scrape_configs: + - job_name: services + metrics_path: /metrics + static_configs: + - targets: + - 'prometheus:9090' + - 'master:9324' + - 'volume:9325' + - 'filer:9326' + - 's3:9327' + # A plugin worker publishes its own metrics when started with + # -metricsPort (Go) or --metrics-port (Rust). 9328 continues the + # series, since 9327 is already the S3 gateway's. + # - 'worker:9328' diff --git a/docker/seaweedfs-compose.yml b/docker/seaweedfs-compose.yml new file mode 100644 index 0000000..1a186b2 --- /dev/null +++ b/docker/seaweedfs-compose.yml @@ -0,0 +1,59 @@ +version: '3.9' + +services: + master: + image: chrislusf/seaweedfs # use a remote image + ports: + - 9333:9333 + - 19333:19333 + - 9324:9324 + command: 'master -ip=master -ip.bind=0.0.0.0 -metricsPort=9324' + volume: + image: chrislusf/seaweedfs # use a remote image + ports: + - 8080:8080 + - 18080:18080 + - 9325:9325 + command: 'volume -ip=volume -master="master:9333" -ip.bind=0.0.0.0 -port=8080 -metricsPort=9325' + depends_on: + - master + filer: + image: chrislusf/seaweedfs # use a remote image + ports: + - 8888:8888 + - 18888:18888 + - 9326:9326 + command: 'filer -ip=filer -master="master:9333" -ip.bind=0.0.0.0 -metricsPort=9326' + tty: true + stdin_open: true + depends_on: + - master + - volume + s3: + image: chrislusf/seaweedfs # use a remote image + ports: + - 8333:8333 + - 9327:9327 + command: 's3 -filer="filer:8888" -ip.bind=0.0.0.0 -metricsPort=9327' + depends_on: + - master + - volume + - filer + webdav: + image: chrislusf/seaweedfs # use a remote image + ports: + - 7333:7333 + command: 'webdav -filer="filer:8888"' + depends_on: + - master + - volume + - filer + prometheus: + image: prom/prometheus:v2.21.0 + ports: + - 9000:9090 + volumes: + - ./prometheus:/etc/prometheus + command: '--web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml' + depends_on: + - s3 diff --git a/docker/seaweedfs-dev-compose.yml b/docker/seaweedfs-dev-compose.yml new file mode 100644 index 0000000..47c3598 --- /dev/null +++ b/docker/seaweedfs-dev-compose.yml @@ -0,0 +1,53 @@ +version: '3.9' + +services: + master: + image: chrislusf/seaweedfs:dev # use a remote dev image + ports: + - 9333:9333 + - 19333:19333 + command: 'master -ip=master -ip.bind=0.0.0.0' + volume: + image: chrislusf/seaweedfs:dev # use a remote dev image + ports: + - 8080:8080 + - 18080:18080 + command: 'volume -ip=volume -master="master:9333" -ip.bind=0.0.0.0 -port=8080' + depends_on: + - master + filer: + image: chrislusf/seaweedfs:dev # use a remote dev image + ports: + - 8888:8888 + - 18888:18888 + command: 'filer -ip=filer -master="master:9333" -ip.bind=0.0.0.0' + depends_on: + - master + - volume + s3: + image: chrislusf/seaweedfs:dev # use a remote dev image + ports: + - 8333:8333 + command: 's3 -filer="filer:8888" -ip.bind=0.0.0.0' + depends_on: + - master + - volume + - filer + webdav: + image: chrislusf/seaweedfs:dev # use a remote dev image + ports: + - 7333:7333 + command: 'webdav -filer="filer:8888"' + depends_on: + - master + - volume + - filer + admin: + image: chrislusf/seaweedfs:dev # use a remote dev image + ports: + - 23646:23646 + command: 'admin -master=master:9333' + depends_on: + - master + - volume + - filer diff --git a/docker/seaweedfs.sql b/docker/seaweedfs.sql new file mode 100644 index 0000000..c9974e0 --- /dev/null +++ b/docker/seaweedfs.sql @@ -0,0 +1,12 @@ +CREATE DATABASE IF NOT EXISTS seaweedfs; +CREATE USER IF NOT EXISTS 'seaweedfs'@'%' IDENTIFIED BY 'secret'; +GRANT ALL PRIVILEGES ON seaweedfs.* TO 'seaweedfs'@'%'; +FLUSH PRIVILEGES; +USE seaweedfs; +CREATE TABLE IF NOT EXISTS `filemeta` ( + `dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field', + `name` VARCHAR(766) NOT NULL COMMENT 'directory or file name', + `directory` TEXT NOT NULL COMMENT 'full path to parent directory', + `meta` LONGBLOB, + PRIMARY KEY (`dirhash`, `name`) +) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; \ No newline at end of file diff --git a/docker/security.toml.example b/docker/security.toml.example new file mode 100644 index 0000000..eeaec82 --- /dev/null +++ b/docker/security.toml.example @@ -0,0 +1,34 @@ +# Put this file to one of the location, with descending priority +# ./security.toml +# $HOME/.seaweedfs/security.toml +# /etc/seaweedfs/security.toml +# this file is read by master, volume server, filer, and worker + +# comma separated origins allowed to make requests to the filer and s3 gateway. +# enter in this format: https://domain.com, or http://localhost:port +[cors.allowed_origins] +values = "*" + +# this jwt signing key is read by master and volume server, and it is used for write operations: +# - the Master server generates the JWT, which can be used to write a certain file on a volume server +# - the Volume server validates the JWT on writing +# the jwt defaults to expire after 10 seconds. +# [jwt.signing] +# key = "V1JJVEVTRUNSRVRFWEFNUExFMTIzNDU2Nzg5MDEy" # Example: WRITESECRETEXAMPLE123456789012 +# this jwt signing key is read by master and volume server, and it is used for read operations: +# - the Master server generates the JWT, which can be used to read a certain file on a volume server +# - the Volume server validates the JWT on reading +# [jwt.signing.read] +# key = "UkVBRFNFQ1JFVUVYQU1QTEUxMjM0NTY3ODkwMTI=" # Example: READSECRETEXAMPLE123456789012 +# If this JWT key is configured, Filer only accepts writes over HTTP if they are signed with this JWT: +# - f.e. the S3 API Shim generates the JWT +# - the Filer server validates the JWT on writing +# the jwt defaults to expire after 10 seconds. +# [jwt.filer_signing] +# key = "RklMRVJXUklURVNFQ1JFVEVYQU1QTEUxMjM0NTY3OA==" # Example: FILERWRITESECRETEXAMPLE12345678 +# If this JWT key is configured, Filer only accepts reads over HTTP if they are signed with this JWT: +# - f.e. the S3 API Shim generates the JWT +# - the Filer server validates the JWT on reading +# the jwt defaults to expire after 10 seconds. +# [jwt.filer_signing.read] +# key = "RklMRVJSRUFEU0VDUkVURVhBTVBMRTEyMzQ1Njc4OQ==" # Example: FILERREADSECRETEXAMPLE123456789 diff --git a/docker/tarantool/app-scm-1.rockspec b/docker/tarantool/app-scm-1.rockspec new file mode 100644 index 0000000..79eb1ca --- /dev/null +++ b/docker/tarantool/app-scm-1.rockspec @@ -0,0 +1,14 @@ +package = 'app' +version = 'scm-1' +source = { + url = '/dev/null', +} +dependencies = { + 'crud == 1.5.2-1', + 'expirationd == 1.6.0-1', + 'metrics-export-role == 0.3.0-1', + 'vshard == 0.1.32-1' +} +build = { + type = 'none'; +} \ No newline at end of file diff --git a/docker/tarantool/config.yaml b/docker/tarantool/config.yaml new file mode 100644 index 0000000..00a693a --- /dev/null +++ b/docker/tarantool/config.yaml @@ -0,0 +1,145 @@ +config: + context: + app_user_password: + from: env + env: APP_USER_PASSWORD + client_user_password: + from: env + env: CLIENT_USER_PASSWORD + replicator_user_password: + from: env + env: REPLICATOR_USER_PASSWORD + storage_user_password: + from: env + env: STORAGE_USER_PASSWORD + +credentials: + roles: + crud-role: + privileges: + - permissions: [ "execute" ] + lua_call: [ "crud.delete", "crud.get", "crud.upsert" ] + users: + app: + password: '{{ context.app_user_password }}' + roles: [ public, crud-role ] + client: + password: '{{ context.client_user_password }}' + roles: [ super ] + replicator: + password: '{{ context.replicator_user_password }}' + roles: [ replication ] + storage: + password: '{{ context.storage_user_password }}' + roles: [ sharding ] + +iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + +sharding: + bucket_count: 10000 + +metrics: + include: [ all ] + exclude: [ vinyl ] + labels: + alias: '{{ instance_name }}' + + +groups: + storages: + roles: + - roles.crud-storage + - roles.expirationd + - roles.metrics-export + roles_cfg: + roles.expirationd: + cfg: + metrics: true + filer_metadata_task: + space: filer_metadata + is_expired: filer_metadata.is_expired + options: + atomic_iteration: true + force: true + index: 'expire_at_idx' + iterator_type: GT + start_key: + - 0 + tuples_per_iteration: 10000 + app: + module: storage + sharding: + roles: [ storage ] + replication: + failover: election + database: + use_mvcc_engine: true + replicasets: + storage-001: + instances: + storage-001-a: + roles_cfg: + roles.metrics-export: + http: + - listen: '0.0.0.0:8081' + endpoints: + - path: /metrics/prometheus/ + format: prometheus + - path: /metrics/json + format: json + iproto: + listen: + - uri: 127.0.0.1:3301 + advertise: + client: 127.0.0.1:3301 + storage-001-b: + roles_cfg: + roles.metrics-export: + http: + - listen: '0.0.0.0:8082' + endpoints: + - path: /metrics/prometheus/ + format: prometheus + - path: /metrics/json + format: json + iproto: + listen: + - uri: 127.0.0.1:3302 + advertise: + client: 127.0.0.1:3302 + routers: + roles: + - roles.crud-router + - roles.metrics-export + roles_cfg: + roles.crud-router: + stats: true + stats_driver: metrics + stats_quantiles: true + app: + module: router + sharding: + roles: [ router ] + replicasets: + router-001: + instances: + router-001-a: + roles_cfg: + roles.metrics-export: + http: + - listen: '0.0.0.0:8083' + endpoints: + - path: /metrics/prometheus/ + format: prometheus + - path: /metrics/json + format: json + iproto: + listen: + - uri: 127.0.0.1:3303 + advertise: + client: 127.0.0.1:3303 \ No newline at end of file diff --git a/docker/tarantool/instances.yaml b/docker/tarantool/instances.yaml new file mode 100644 index 0000000..225b738 --- /dev/null +++ b/docker/tarantool/instances.yaml @@ -0,0 +1,7 @@ +--- +storage-001-a: + +storage-001-b: + +router-001-a: + diff --git a/docker/tarantool/router.lua b/docker/tarantool/router.lua new file mode 100644 index 0000000..359a8c4 --- /dev/null +++ b/docker/tarantool/router.lua @@ -0,0 +1,77 @@ +local vshard = require('vshard') +local log = require('log') + +-- Bootstrap the vshard router. +while true do + local ok, err = vshard.router.bootstrap({ + if_not_bootstrapped = true, + }) + if ok then + break + end + log.info(('Router bootstrap error: %s'):format(err)) +end + +-- functions for filer_metadata space +local filer_metadata = { + delete_by_directory_idx = function(directory) + -- find all storages + local storages = require('vshard').router.routeall() + -- on each storage + for _, storage in pairs(storages) do + -- call local function + local result, err = storage:callrw('filer_metadata.delete_by_directory_idx', { directory }) + -- check for error + if err then + error("Failed to call function on storage: " .. tostring(err)) + end + end + -- return + return true + end, + find_by_directory_idx_and_name = function(dirPath, startFileName, includeStartFile, limit) + -- init results + local results = {} + -- find all storages + local storages = require('vshard').router.routeall() + -- on each storage + for _, storage in pairs(storages) do + -- call local function + local result, err = storage:callro('filer_metadata.find_by_directory_idx_and_name', { + dirPath, + startFileName, + includeStartFile, + limit + }) + -- check for error + if err then + error("Failed to call function on storage: " .. tostring(err)) + end + -- add to results + for _, tuple in ipairs(result) do + table.insert(results, tuple) + end + end + -- sort + table.sort(results, function(a, b) return a[3] < b[3] end) + -- apply limit + if #results > limit then + local limitedResults = {} + for i = 1, limit do + table.insert(limitedResults, results[i]) + end + results = limitedResults + end + -- return + return results + end, +} + +rawset(_G, 'filer_metadata', filer_metadata) + +-- register functions for filer_metadata space, set grants +for name, _ in pairs(filer_metadata) do + box.schema.func.create('filer_metadata.' .. name, { if_not_exists = true }) + box.schema.user.grant('app', 'execute', 'function', 'filer_metadata.' .. name, { if_not_exists = true }) + box.schema.user.grant('client', 'execute', 'function', 'filer_metadata.' .. name, { if_not_exists = true }) +end diff --git a/docker/tarantool/storage.lua b/docker/tarantool/storage.lua new file mode 100644 index 0000000..5ab244a --- /dev/null +++ b/docker/tarantool/storage.lua @@ -0,0 +1,97 @@ +box.watch('box.status', function() + if box.info.ro then + return + end + + -- ==================================== + -- key_value space + -- ==================================== + box.schema.create_space('key_value', { + format = { + { name = 'key', type = 'string' }, + { name = 'bucket_id', type = 'unsigned' }, + { name = 'value', type = 'string' } + }, + if_not_exists = true + }) + + -- create key_value space indexes + box.space.key_value:create_index('id', {type = 'tree', parts = { 'key' }, unique = true, if_not_exists = true}) + box.space.key_value:create_index('bucket_id', { type = 'tree', parts = { 'bucket_id' }, unique = false, if_not_exists = true }) + + -- ==================================== + -- filer_metadata space + -- ==================================== + box.schema.create_space('filer_metadata', { + format = { + { name = 'directory', type = 'string' }, + { name = 'bucket_id', type = 'unsigned' }, + { name = 'name', type = 'string' }, + { name = 'expire_at', type = 'unsigned' }, + { name = 'data', type = 'string' } + }, + if_not_exists = true + }) + + -- create filer_metadata space indexes + box.space.filer_metadata:create_index('id', {type = 'tree', parts = { 'directory', 'name' }, unique = true, if_not_exists = true}) + box.space.filer_metadata:create_index('bucket_id', { type = 'tree', parts = { 'bucket_id' }, unique = false, if_not_exists = true }) + box.space.filer_metadata:create_index('directory_idx', { type = 'tree', parts = { 'directory' }, unique = false, if_not_exists = true }) + box.space.filer_metadata:create_index('name_idx', { type = 'tree', parts = { 'name' }, unique = false, if_not_exists = true }) + box.space.filer_metadata:create_index('expire_at_idx', { type = 'tree', parts = { 'expire_at' }, unique = false, if_not_exists = true}) +end) + +-- functions for filer_metadata space +local filer_metadata = { + delete_by_directory_idx = function(directory) + local space = box.space.filer_metadata + local index = space.index.directory_idx + -- for each found directory + for _, tuple in index:pairs({ directory }, { iterator = 'EQ' }) do + space:delete({ tuple[1], tuple[3] }) + end + return true + end, + find_by_directory_idx_and_name = function(dirPath, startFileName, includeStartFile, limit) + local space = box.space.filer_metadata + local directory_idx = space.index.directory_idx + -- choose filter name function + local filter_filename_func + if includeStartFile then + filter_filename_func = function(value) return value >= startFileName end + else + filter_filename_func = function(value) return value > startFileName end + end + -- init results + local results = {} + -- for each found directory + for _, tuple in directory_idx:pairs({ dirPath }, { iterator = 'EQ' }) do + -- filter by name + if filter_filename_func(tuple[3]) then + table.insert(results, tuple) + end + end + -- sort + table.sort(results, function(a, b) return a[3] < b[3] end) + -- apply limit + if #results > limit then + local limitedResults = {} + for i = 1, limit do + table.insert(limitedResults, results[i]) + end + results = limitedResults + end + -- return + return results + end, + is_expired = function(args, tuple) + return (tuple[4] > 0) and (require('fiber').time() > tuple[4]) + end +} + +-- register functions for filer_metadata space, set grants +rawset(_G, 'filer_metadata', filer_metadata) +for name, _ in pairs(filer_metadata) do + box.schema.func.create('filer_metadata.' .. name, { setuid = true, if_not_exists = true }) + box.schema.user.grant('storage', 'execute', 'function', 'filer_metadata.' .. name, { if_not_exists = true }) +end diff --git a/docker/test.py b/docker/test.py new file mode 100755 index 0000000..8ac025b --- /dev/null +++ b/docker/test.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.12" +# dependencies = [ +# "boto3", +# ] +# /// + +import argparse +import json +import random +import string +import subprocess +from enum import Enum +from pathlib import Path + +import boto3 + +REGION_NAME = "us-east-1" + + +class Actions(str, Enum): + Get = "Get" + Put = "Put" + List = "List" + + +def get_user_dir(bucket_name, user, with_bucket=True): + if with_bucket: + return f"{bucket_name}/user-id-{user}" + + return f"user-id-{user}" + + +def create_power_user(): + power_user_key = "power_user_key" + power_user_secret = "power_user_secret" + command = f"s3.configure -apply -user poweruser -access_key {power_user_key} -secret_key {power_user_secret} -actions Admin" + print("Creating Power User...") + subprocess.run( + ["docker", "exec", "-i", "seaweedfs-master-1", "weed", "shell"], + input=command, + text=True, + stdout=subprocess.PIPE, + ) + print( + f"Power User created with key: {power_user_key} and secret: {power_user_secret}" + ) + return power_user_key, power_user_secret + + +def create_bucket(s3_client, bucket_name): + print(f"Creating Bucket {bucket_name}...") + s3_client.create_bucket(Bucket=bucket_name) + print(f"Bucket {bucket_name} created.") + + +def upload_file(s3_client, bucket_name, user, file_path, custom_remote_path=None): + user_dir = get_user_dir(bucket_name, user, with_bucket=False) + if custom_remote_path: + remote_path = custom_remote_path + else: + remote_path = f"{user_dir}/{str(Path(file_path).name)}" + + print(f"Uploading {file_path} for {user}... on {user_dir}") + + s3_client.upload_file(file_path, bucket_name, remote_path) + print(f"File {file_path} uploaded for {user}.") + + +def create_user(iam_client, user): + print(f"Creating user {user}...") + response = iam_client.create_access_key(UserName=user) + print( + f"User {user} created with access key: {response['AccessKey']['AccessKeyId']}" + ) + return response + + +def list_files(s3_client, bucket_name, path=None): + if path is None: + path = "" + print(f"Listing files of s3://{bucket_name}/{path}...") + try: + response = s3_client.list_objects_v2(Bucket=bucket_name, Prefix=path) + if "Contents" in response: + for obj in response["Contents"]: + print(f"\t - {obj['Key']}") + else: + print("No files found.") + except Exception as e: + print(f"Error listing files: {e}") + + +def create_policy_for_user( + iam_client, user, bucket_name, actions=[Actions.Get, Actions.List] +): + print(f"Creating policy for {user} on {bucket_name}...") + policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [f"s3:{action.value}*" for action in actions], + "Resource": [ + f"arn:aws:s3:::{get_user_dir(bucket_name, user)}/*", + ], + } + ], + } + policy_name = f"{user}-{bucket_name}-full-access" + + policy_json = json.dumps(policy_document) + filepath = f"/tmp/{policy_name}.json" + with open(filepath, "w") as f: + f.write(json.dumps(policy_document, indent=2)) + + iam_client.put_user_policy( + PolicyName=policy_name, PolicyDocument=policy_json, UserName=user + ) + print(f"Policy for {user} on {bucket_name} created.") + + +def main(): + parser = argparse.ArgumentParser(description="SeaweedFS S3 Test Script") + parser.add_argument( + "--s3-url", default="http://127.0.0.1:8333", help="S3 endpoint URL" + ) + parser.add_argument( + "--iam-url", default="http://127.0.0.1:8111", help="IAM endpoint URL" + ) + args = parser.parse_args() + + bucket_name = ( + f"test-bucket-{''.join(random.choices(string.digits + 'abcdef', k=8))}" + ) + sentinel_file = "/tmp/SENTINEL" + with open(sentinel_file, "w") as f: + f.write("Hello World") + print(f"SENTINEL file created at {sentinel_file}") + + power_user_key, power_user_secret = create_power_user() + + admin_s3_client = get_s3_client(args, power_user_key, power_user_secret) + iam_client = get_iam_client(args, power_user_key, power_user_secret) + + create_bucket(admin_s3_client, bucket_name) + upload_file(admin_s3_client, bucket_name, "Alice", sentinel_file) + upload_file(admin_s3_client, bucket_name, "Bob", sentinel_file) + list_files(admin_s3_client, bucket_name) + + alice_user_info = create_user(iam_client, "Alice") + bob_user_info = create_user(iam_client, "Bob") + + alice_key = alice_user_info["AccessKey"]["AccessKeyId"] + alice_secret = alice_user_info["AccessKey"]["SecretAccessKey"] + bob_key = bob_user_info["AccessKey"]["AccessKeyId"] + bob_secret = bob_user_info["AccessKey"]["SecretAccessKey"] + + # Make sure Admin can read any files + list_files(admin_s3_client, bucket_name) + list_files( + admin_s3_client, + bucket_name, + get_user_dir(bucket_name, "Alice", with_bucket=False), + ) + list_files( + admin_s3_client, + bucket_name, + get_user_dir(bucket_name, "Bob", with_bucket=False), + ) + + # Create read policy for Alice and Bob + create_policy_for_user(iam_client, "Alice", bucket_name) + create_policy_for_user(iam_client, "Bob", bucket_name) + + alice_s3_client = get_s3_client(args, alice_key, alice_secret) + + # Make sure Alice can read her files + list_files( + alice_s3_client, + bucket_name, + get_user_dir(bucket_name, "Alice", with_bucket=False) + "/", + ) + + # Make sure Bob can read his files + bob_s3_client = get_s3_client(args, bob_key, bob_secret) + list_files( + bob_s3_client, + bucket_name, + get_user_dir(bucket_name, "Bob", with_bucket=False) + "/", + ) + + # Update policy to include write + create_policy_for_user(iam_client, "Alice", bucket_name, actions=[Actions.Put, Actions.Get, Actions.List]) # fmt: off + create_policy_for_user(iam_client, "Bob", bucket_name, actions=[Actions.Put, Actions.Get, Actions.List]) # fmt: off + + print("############################# Make sure Alice can write her files") + upload_file( + alice_s3_client, + bucket_name, + "Alice", + sentinel_file, + custom_remote_path=f"{get_user_dir(bucket_name, 'Alice', with_bucket=False)}/SENTINEL_by_Alice", + ) + + + print("############################# Make sure Bob can write his files") + upload_file( + bob_s3_client, + bucket_name, + "Bob", + sentinel_file, + custom_remote_path=f"{get_user_dir(bucket_name, 'Bob', with_bucket=False)}/SENTINEL_by_Bob", + ) + + + print("############################# Make sure Alice can read her new files") + list_files( + alice_s3_client, + bucket_name, + get_user_dir(bucket_name, "Alice", with_bucket=False) + "/", + ) + + + print("############################# Make sure Bob can read his new files") + list_files( + bob_s3_client, + bucket_name, + get_user_dir(bucket_name, "Bob", with_bucket=False) + "/", + ) + + + print("############################# Make sure Bob cannot read Alice's files") + list_files( + bob_s3_client, + bucket_name, + get_user_dir(bucket_name, "Alice", with_bucket=False) + "/", + ) + + print("############################# Make sure Alice cannot read Bob's files") + + list_files( + alice_s3_client, + bucket_name, + get_user_dir(bucket_name, "Bob", with_bucket=False) + "/", + ) + + + +def get_iam_client(args, access_key, secret_key): + iam_client = boto3.client( + "iam", + endpoint_url=args.iam_url, + region_name=REGION_NAME, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + ) + return iam_client + + +def get_s3_client(args, access_key, secret_key): + s3_client = boto3.client( + "s3", + endpoint_url=args.s3_url, + region_name=REGION_NAME, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + ) + return s3_client + + +if __name__ == "__main__": + main() diff --git a/docker/weed-volume-prebuilt/.gitkeep b/docker/weed-volume-prebuilt/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docker/weed-worker-prebuilt/.gitkeep b/docker/weed-worker-prebuilt/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..0fe155f --- /dev/null +++ b/go.mod @@ -0,0 +1,527 @@ +module github.com/seaweedfs/seaweedfs + +go 1.26.0 + +require ( + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/pubsub v1.51.1 + cloud.google.com/go/storage v1.67.0 + github.com/Shopify/sarama v1.38.1 + github.com/aws/aws-sdk-go v1.55.8 + github.com/beorn7/perks v1.0.1 // indirect + github.com/bwmarrin/snowflake v0.3.0 + github.com/cenkalti/backoff/v4 v4.3.0 + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.7.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dustin/go-humanize v1.0.1 + github.com/eapache/go-resiliency v1.6.0 // indirect + github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect + github.com/eapache/queue v1.1.0 // indirect + github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a + github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect + github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect + github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4 + github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-redsync/redsync/v4 v4.17.0 + github.com/go-sql-driver/mysql v1.10.1 + github.com/go-zookeeper/zk v1.0.4 // indirect + github.com/golang/protobuf v1.5.4 + github.com/golang/snappy v1.0.0 + github.com/google/btree v1.1.3 + github.com/google/uuid v1.6.0 + github.com/google/wire v0.7.0 // indirect + github.com/googleapis/gax-go/v2 v2.24.0 // indirect + github.com/gorilla/mux v1.8.1 + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/jackc/pgx/v5 v5.10.0 + github.com/jcmturner/gofork v1.7.6 // indirect + github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect + github.com/jinzhu/copier v0.4.0 + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/json-iterator/go v1.1.12 + github.com/karlseguin/ccache/v2 v2.0.8 + github.com/klauspost/compress v1.19.2 + github.com/klauspost/reedsolomon v1.14.2 + github.com/kurin/blazer v0.5.3 + github.com/linxGnu/grocksdb v1.10.8 + github.com/mailru/easyjson v0.9.2 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/olivere/elastic/v7 v7.0.32 + github.com/peterh/liner v1.2.2 + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/posener/complete v1.2.3 + github.com/pquerna/cachecontrol v0.2.0 + github.com/prometheus/client_golang v1.24.1 + github.com/prometheus/client_model v0.6.3 + github.com/prometheus/common v0.70.1 // indirect + github.com/prometheus/procfs v0.22.0 + github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/seaweedfs/goexif v2.0.0+incompatible + github.com/seaweedfs/raft v1.2.1 + github.com/sirupsen/logrus v1.9.4 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/viper v1.21.0 + github.com/stretchr/testify v1.11.1 + github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 + github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 + github.com/tidwall/gjson v1.18.0 + github.com/tidwall/match v1.2.0 + github.com/tidwall/pretty v1.2.0 // indirect + github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 + github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 + github.com/valyala/bytebufferpool v1.0.0 + github.com/viant/ptrie v1.0.1 + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.2.0 + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + go.etcd.io/etcd/client/v3 v3.7.1 + go.mongodb.org/mongo-driver v1.17.9 + go.opencensus.io v0.24.0 // indirect + gocloud.dev v0.46.0 + gocloud.dev/pubsub/natspubsub v0.46.0 + gocloud.dev/pubsub/rabbitpubsub v0.46.0 + golang.org/x/crypto v0.56.0 + golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 + golang.org/x/image v0.46.0 + golang.org/x/net v0.58.0 + golang.org/x/oauth2 v0.36.0 + golang.org/x/sys v0.48.0 + golang.org/x/text v0.42.0 // indirect + golang.org/x/tools v0.49.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect + google.golang.org/api v0.297.0 + google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d // indirect + google.golang.org/grpc v1.85.0-dev + google.golang.org/protobuf v1.36.12 + gopkg.in/inf.v0 v0.9.1 // indirect + modernc.org/b v1.0.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.57.0 +) + +require ( + cloud.google.com/go/kms v1.33.0 + github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 + github.com/DATA-DOG/go-sqlmock v1.5.2 + github.com/Jille/raft-grpc-transport v1.6.1 + github.com/ThreeDotsLabs/watermill v1.5.2 + github.com/a-h/templ v0.3.1020 + github.com/apache/cassandra-gocql-driver/v2 v2.1.2 + github.com/apache/iceberg-go v0.6.1-0.20260817192109-c2105090c9e2 + github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3 + github.com/arangodb/go-driver v1.6.9 + github.com/armon/go-metrics v0.4.1 + github.com/aws/aws-sdk-go-v2 v1.47.0 + github.com/aws/aws-sdk-go-v2/config v1.32.35 + github.com/aws/aws-sdk-go-v2/credentials v1.20.4 + github.com/aws/aws-sdk-go-v2/service/s3 v1.107.3 + github.com/cespare/xxhash/v2 v2.3.0 + github.com/cognusion/imaging v1.0.4 + github.com/fluent/fluent-logger-golang v1.10.1 + github.com/getsentry/sentry-go v0.49.0 + github.com/go-ldap/ldap/v3 v3.4.14 + github.com/golang-jwt/jwt/v5 v5.3.1 + github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50 + github.com/hashicorp/golang-lru/v2 v2.0.7 + github.com/hashicorp/raft v1.7.3 + github.com/hashicorp/raft-boltdb/v2 v2.3.1 + github.com/hashicorp/vault/api v1.23.0 + github.com/jhump/protoreflect v1.18.0 + github.com/linkedin/goavro/v2 v2.15.0 + github.com/minio/crc64nvme v1.1.1 + github.com/orcaman/concurrent-map/v2 v2.0.1 + github.com/parquet-go/parquet-go v0.32.0 + github.com/pkg/sftp v1.13.11 + github.com/rabbitmq/amqp091-go v1.14.0 + github.com/rclone/rclone v1.75.1 + github.com/rdleal/intervalst v1.5.0 + github.com/redis/go-redis/v9 v9.22.0 + github.com/schollz/progressbar/v3 v3.19.1 + github.com/seaweedfs/go-fuse/v2 v2.9.4 + github.com/shirou/gopsutil/v4 v4.26.7 + github.com/tarantool/go-option v1.1.0 + github.com/tarantool/go-tarantool/v3 v3.0.1 + github.com/testcontainers/testcontainers-go v0.44.0 + github.com/tikv/client-go/v2 v2.0.7 + github.com/twmb/avro v1.8.0 + github.com/xeipuuv/gojsonschema v1.2.0 + github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.2 + github.com/ydb-platform/ydb-go-sdk/v3 v3.151.1 + go.etcd.io/etcd/client/pkg/v3 v3.7.1 + go.uber.org/atomic v1.11.0 + golang.org/x/sync v0.23.0 + golang.org/x/tools/godoc v0.1.0-deprecated + google.golang.org/grpc/security/advancedtls v1.0.0 +) + +require github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect + +require ( + atomicgo.dev/cursor v0.2.0 // indirect + atomicgo.dev/keyboard v0.2.9 // indirect + atomicgo.dev/schedule v0.1.0 // indirect + cloud.google.com/go/longrunning v1.2.0 // indirect + cloud.google.com/go/pubsub/v2 v2.6.0 // indirect + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/FilenCloudDienste/filen-sdk-go v0.0.39 // indirect + github.com/ProtonMail/gopenpgp/v3 v3.4.1 // indirect + github.com/RoaringBitmap/roaring/v2 v2.24.0 // indirect + github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 // indirect + github.com/adrg/xdg v0.5.3 // indirect + github.com/anchore/go-lzo v0.1.1 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/apache/arrow-go/v18 v18.7.0 // indirect + github.com/apache/thrift v0.24.0 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/bazelbuild/rules_go v0.46.0 // indirect + github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect + github.com/bits-and-blooms/bitset v1.24.4 // indirect + github.com/blevesearch/snowballstem v0.9.0 // indirect + github.com/boombuler/barcode v1.1.0 // indirect + github.com/buger/jsonparser v1.2.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/cockroachdb/apd/v3 v3.2.1 // indirect + github.com/cockroachdb/errors v1.14.0 // indirect + github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect + github.com/cockroachdb/redact v1.1.5 // indirect + github.com/cockroachdb/version v0.0.0-20250314144055-3860cd14adf2 // indirect + github.com/containerd/console v1.0.5 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v1.0.0-rc.4 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/dave/dst v0.27.2 // indirect + github.com/diskfs/go-diskfs v1.9.4 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.7.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/dromara/dongle v1.0.1 // indirect + github.com/geoarrow/geoarrow-go v0.0.0-20260403143023-f54751c3e3a1 // indirect + github.com/gin-gonic/gin v1.11.0 // indirect + github.com/go-asn1-ber/asn1-ber v1.5.8 // indirect + github.com/go-git/go-billy/v5 v5.9.0 // indirect + github.com/goccy/go-yaml v1.18.0 // indirect + github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/gookit/color v1.6.0 // indirect + github.com/gopherjs/gopherjs v1.17.2 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.7 // indirect + github.com/hashicorp/hcl v1.0.1-vault-7 // indirect + github.com/internxt/rclone-adapter v0.0.0-20260708165336-dd6561bacfa2 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jaegertracing/jaeger v1.47.0 // indirect + github.com/jhump/protoreflect/v2 v2.0.0-beta.1 // indirect + github.com/jtolds/gls v4.20.0+incompatible // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/lib/pq v1.12.0 // indirect + github.com/lithammer/fuzzysearch v1.1.8 // indirect + github.com/lithammer/shortuuid/v3 v3.0.7 // indirect + github.com/lpar/calendar v0.2.0 // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.3.0 // indirect + github.com/moby/moby/api v1.55.0 // indirect + github.com/moby/moby/client v0.5.0 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect + github.com/moby/sys/sequential v0.7.0 // indirect + github.com/moby/sys/user v0.4.1 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.2 // indirect + github.com/mschoch/smat v0.2.0 // indirect + github.com/oklog/ulid/v2 v2.1.1 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/openzipkin/zipkin-go v0.4.3 // indirect + github.com/parquet-go/bitpack v1.0.0 // indirect + github.com/parquet-go/jsonlite v1.0.0 // indirect + github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 // indirect + github.com/pierrre/geohash v1.0.0 // indirect + github.com/pquerna/otp v1.5.0 // indirect + github.com/pterm/pterm v0.12.83 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/rclone/Proton-API-Bridge v1.0.5 // indirect + github.com/rclone/go-proton-api v1.0.4 // indirect + github.com/rogpeppe/go-internal v1.15.0 // indirect + github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/sasha-s/go-deadlock v0.3.1 // indirect + github.com/smarty/assertions v1.15.0 // indirect + github.com/sony/gobreaker/v2 v2.4.0 // indirect + github.com/stretchr/objx v0.5.3 // indirect + github.com/substrait-io/substrait v0.87.0 // indirect + github.com/substrait-io/substrait-go/v8 v8.1.1 // indirect + github.com/substrait-io/substrait-protobuf/go v0.85.0 // indirect + github.com/twpayne/go-geom v1.6.1 // indirect + github.com/twpayne/go-kml/v3 v3.2.1 // indirect + github.com/tyler-smith/go-bip39 v1.1.0 // indirect + github.com/ulikunitz/xz v0.5.15 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.uber.org/mock v0.5.2 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/mod v0.41.0 // indirect + gonum.org/v1/gonum v0.17.0 // indirect +) + +require ( + cel.dev/expr v0.25.2 // indirect + cloud.google.com/go/auth v0.23.2 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.12.0 // indirect + cloud.google.com/go/monitoring v1.30.0 // indirect + filippo.io/edwards25519 v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 + github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 // indirect + github.com/Azure/go-ntlmssp v0.1.1 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect + github.com/Files-com/files-sdk-go/v3 v3.3.194 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect + github.com/IBM/go-sdk-core/v5 v5.23.1 // indirect + github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect + github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect + github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect + github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect + github.com/ProtonMail/go-crypto v1.4.1 // indirect + github.com/ProtonMail/go-srp v0.0.7 // indirect + github.com/PuerkitoBio/goquery v1.12.0 // indirect + github.com/abbot/go-http-auth v0.4.0 // indirect + github.com/andybalholm/brotli v1.2.2 // indirect + github.com/andybalholm/cascadia v1.3.4 // indirect + github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect + github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.31 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.39 // indirect + github.com/aws/aws-sdk-go-v2/service/sns v1.39.14 // indirect + github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.50.0 + github.com/aws/smithy-go v1.28.1 + github.com/boltdb/bolt v1.3.1 // indirect + github.com/bradenaw/juniper v0.15.3 // indirect + github.com/buengese/sgzip v0.1.1 // indirect + github.com/calebcase/tmpfile v1.0.3 // indirect + github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect + github.com/cloudflare/circl v1.6.4 // indirect + github.com/cloudinary/cloudinary-go/v2 v2.16.0 // indirect + github.com/cloudsoda/go-smb2 v0.0.0-20260701064823-d8c5600d73b8 // indirect + github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/colinmarc/hdfs/v2 v2.4.0 // indirect + github.com/creasty/defaults v1.8.0 // indirect + github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 // indirect + github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect + github.com/d4l3k/messagediff v1.2.1 // indirect + github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect + github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.4.0 // indirect + github.com/ebitengine/purego v0.10.2 // indirect + github.com/elastic/gosigar v0.14.3 // indirect + github.com/emersion/go-message v0.18.2 // indirect + github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/flynn/noise v1.1.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.13 // indirect + github.com/geoffgarside/ber v1.2.0 // indirect + github.com/go-chi/chi/v5 v5.3.1 // indirect + github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-openapi/errors v0.22.8 // indirect + github.com/go-openapi/strfmt v0.27.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.30.3 // indirect + github.com/go-resty/resty/v2 v2.17.2 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/goccy/go-json v0.10.6 // indirect + github.com/gofrs/flock v0.13.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.20 // indirect + github.com/gorilla/schema v1.4.1 // indirect + github.com/gorilla/securecookie v1.1.2 // indirect + github.com/gorilla/sessions v1.4.0 + github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-metrics v0.5.4 // indirect + github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect + github.com/hashicorp/golang-lru v0.6.0 // indirect + github.com/jcmturner/aescts/v2 v2.0.0 // indirect + github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect + github.com/jcmturner/goidentity/v6 v6.0.1 // indirect + github.com/jcmturner/rpc/v2 v2.0.3 // indirect + github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb // indirect + github.com/jonboulle/clockwork v0.5.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect + github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect + github.com/k0kubun/pp v3.0.1+incompatible + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect + github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect + github.com/kr/fs v0.1.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/lanrat/extsort v1.4.2 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect + github.com/mattn/go-runewidth v0.0.24 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 + github.com/montanaflynn/stats v0.7.1 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nats-io/nats.go v1.49.0 // indirect + github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/ncw/swift/v2 v2.0.5 // indirect + github.com/nxadm/tail v1.4.11 // indirect + github.com/oklog/ulid v1.3.1 // indirect + github.com/onsi/ginkgo/v2 v2.23.3 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/oracle/oci-go-sdk/v65 v65.121.0 // indirect + github.com/panjf2000/ants/v2 v2.12.1 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/pelletier/go-toml/v2 v2.4.1 // indirect + github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/pierrec/lz4/v4 v4.1.29 + github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect + github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect + github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect + github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pkg/xattr v0.4.12 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect + github.com/relvacode/iso8601 v1.7.0 // indirect + github.com/rfjakob/eme v1.2.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/samber/lo v1.53.0 // indirect + github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564 + github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect + github.com/sony/gobreaker v1.0.0 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spacemonkeygo/monkit/v3 v3.0.25-0.20251022131615-eb24eb109368 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04 // indirect + github.com/tarantool/go-iproto v1.1.0 // indirect + github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect + github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect + github.com/tinylib/msgp v1.6.4 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect + github.com/unknwon/goconfig v1.0.0 // indirect + github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/winfsp/cgofuse v1.6.1-0.20260126094232-f2c4fccdb286 + github.com/xanzy/ssh-agent v0.3.3 // indirect + github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect + github.com/ydb-platform/ydb-go-genproto v0.0.0-20260810122915-65bfd5c4b705 // indirect + github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect + github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect + github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/zeebo/blake3 v0.2.4 // indirect + github.com/zeebo/errs v1.4.0 // indirect + go.etcd.io/bbolt v1.5.0 // indirect + go.etcd.io/etcd/api/v3 v3.7.1 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/sdk v1.45.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.1 // indirect + golang.org/x/term v0.45.0 + golang.org/x/time v0.15.0 + google.golang.org/genproto/googleapis/api v0.0.0-20260715232425-e75dac1f907d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/validator.v2 v2.0.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/libc v1.74.4 // indirect + moul.io/http2curl/v2 v2.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect + storj.io/common v0.0.0-20260629224719-ba1bff0a7846 // indirect + storj.io/drpc v1.0.0 // indirect + storj.io/eventkit v0.0.0-20260716074419-6861a92e2aa5 // indirect + storj.io/infectious v0.0.2 // indirect + storj.io/picobuf v0.0.4 // indirect + storj.io/uplink v1.14.3 // indirect +) + +// replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft + +// tyler-smith/go-bip39 was deleted from GitHub, so `go mod download` fails for +// anyone resolving it directly (GOPROXY=direct). It only reaches us transitively +// through rclone's internxt backend, which calls IsMnemonicValid and NewSeed. +// cosmos/go-bip39 is a maintained, API-compatible fork. +replace github.com/tyler-smith/go-bip39 => github.com/cosmos/go-bip39 v1.0.0 diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..b22acf0 --- /dev/null +++ b/install.sh @@ -0,0 +1,357 @@ +#!/bin/bash +# +# SeaweedFS Installer +# Downloads Go and/or Rust binaries from GitHub releases. +# +# Usage: +# curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash +# curl -fsSL ... | bash -s -- --component volume-rust --large-disk +# curl -fsSL ... | bash -s -- --version 4.34 --dir /usr/local/bin +# +# Options: +# --component COMP Which binary to install: weed, volume-rust, worker-rust, all (default: weed) +# --version VER Release version tag (default: latest) +# --large-disk Use large disk variant (5-byte offset, 8TB max volume) +# --dir DIR Installation directory (default: /usr/local/bin) +# --help Show this help message + +set -euo pipefail + +REPO="seaweedfs/seaweedfs" +COMPONENT="weed" +VERSION="" +LARGE_DISK=false +INSTALL_DIR="/usr/local/bin" +WORKER_INSTALLED=false + +# Colors (if terminal supports them) +if [ -t 1 ]; then + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[0;33m' + BLUE='\033[0;34m' + NC='\033[0m' +else + RED='' GREEN='' YELLOW='' BLUE='' NC='' +fi + +info() { echo -e "${BLUE}[info]${NC} $*"; } +ok() { echo -e "${GREEN}[ok]${NC} $*"; } +warn() { echo -e "${YELLOW}[warn]${NC} $*"; } +error() { echo -e "${RED}[error]${NC} $*" >&2; exit 1; } + +usage() { + sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \?//' + exit 0 +} + +# Parse arguments +while [ $# -gt 0 ]; do + case "$1" in + --component) COMPONENT="$2"; shift 2 ;; + --version) VERSION="$2"; shift 2 ;; + --large-disk) LARGE_DISK=true; shift ;; + --dir) INSTALL_DIR="$2"; shift 2 ;; + --help|-h) usage ;; + *) error "Unknown option: $1. Use --help for usage." ;; + esac +done + +# Detect OS and architecture +detect_platform() { + local os arch + + case "$(uname -s)" in + Linux*) os="linux" ;; + Darwin*) os="darwin" ;; + MINGW*|MSYS*|CYGWIN*) os="windows" ;; + FreeBSD*) os="freebsd" ;; + *) error "Unsupported OS: $(uname -s)" ;; + esac + + case "$(uname -m)" in + x86_64|amd64) arch="amd64" ;; + aarch64|arm64) arch="arm64" ;; + armv7l|armv6l) arch="arm" ;; + *) error "Unsupported architecture: $(uname -m)" ;; + esac + + echo "${os}" "${arch}" +} + +# Get latest release tag from GitHub API +get_latest_version() { + local url="https://api.github.com/repos/${REPO}/releases/latest" + if command -v curl &>/dev/null; then + curl -fsSL "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/' + elif command -v wget &>/dev/null; then + wget -qO- "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/' + else + error "Neither curl nor wget found. Please install one." + fi +} + +# Download a file +download() { + local url="$1" dest="$2" + info "Downloading ${url}" + if command -v curl &>/dev/null; then + curl -fsSL -o "$dest" "$url" + elif command -v wget &>/dev/null; then + wget -qO "$dest" "$url" + fi +} + +# Build Go weed binary asset name +go_asset_name() { + local os="$1" arch="$2" + local suffix="${os}_${arch}" + if [ "$LARGE_DISK" = true ]; then + suffix="${suffix}_large_disk" + fi + echo "${suffix}.tar.gz" +} + +# Build Rust volume server asset name +rust_asset_name() { + local os="$1" arch="$2" + local prefix="weed-volume" + if [ "$LARGE_DISK" = true ]; then + prefix="weed-volume_large_disk" + else + prefix="weed-volume" + fi + local suffix="${os}_${arch}" + if [ "$os" = "windows" ]; then + echo "${prefix}_${suffix}.zip" + else + echo "${prefix}_${suffix}.tar.gz" + fi +} + +# Does a release carry this asset? Answers 0 for yes and 1 for a 404, and stops +# the installer on anything else: a rate limit or a network blip must not read +# as "this release predates the binary" and quietly skip it. +asset_exists() { + local url="$1" code="" + if command -v curl &>/dev/null; then + code="$(curl -sL -o /dev/null -I -w '%{http_code}' "$url" || true)" + elif command -v wget &>/dev/null; then + # --spider's exit status folds 404 in with every other server error, so + # read the status line itself; -S prints one per redirect hop. + code="$(wget -S --spider -q -O /dev/null "$url" 2>&1 | awk '/^ *HTTP\// {c=$2} END {print c}')" + fi + + case "$code" in + 200) return 0 ;; + 404) return 1 ;; + *) error "Could not check ${url} (HTTP ${code:-none}). Retry, or install components one at a time." ;; + esac +} + +# Build Rust maintenance worker asset name. No large-disk variant: the worker +# maintains tables through the namespace and never opens a volume file. +worker_asset_name() { + local os="$1" arch="$2" + echo "weed-worker_${os}_${arch}.tar.gz" +} + +# Install a single component +install_component() { + local component="$1" os="$2" arch="$3" + local asset_name download_url tmpdir + + tmpdir="$(mktemp -d)" + trap "rm -rf '$tmpdir'" EXIT + + case "$component" in + weed) + asset_name="$(go_asset_name "$os" "$arch")" + download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}" + download "$download_url" "${tmpdir}/${asset_name}" + + info "Extracting ${asset_name}..." + tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir" + + # The Go release action puts the binary inside a directory + local weed_bin + weed_bin="$(find "$tmpdir" -name 'weed' -type f | head -1)" + if [ -z "$weed_bin" ]; then + weed_bin="$(find "$tmpdir" -name 'weed.exe' -type f | head -1)" + fi + if [ -z "$weed_bin" ]; then + error "Could not find weed binary in archive" + fi + + chmod +x "$weed_bin" + install_binary "$weed_bin" "weed" + ok "Installed weed to ${INSTALL_DIR}/weed" + ;; + + volume-rust) + # Check platform support for Rust volume server + case "$os" in + linux|darwin|windows) ;; + *) error "Rust volume server is not available for ${os}. Supported: linux, darwin, windows" ;; + esac + case "$arch" in + amd64|arm64) ;; + *) error "Rust volume server is not available for ${arch}. Supported: amd64, arm64" ;; + esac + + asset_name="$(rust_asset_name "$os" "$arch")" + download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}" + download "$download_url" "${tmpdir}/${asset_name}" + + info "Extracting ${asset_name}..." + if [ "$os" = "windows" ]; then + unzip -q "${tmpdir}/${asset_name}" -d "$tmpdir" + else + tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir" + fi + + local rust_bin + if [ "$LARGE_DISK" = true ]; then + rust_bin="$(find "$tmpdir" -name 'weed-volume-large-disk*' -type f | head -1)" + else + rust_bin="$(find "$tmpdir" -name 'weed-volume-normal*' -type f | head -1)" + fi + if [ -z "$rust_bin" ]; then + rust_bin="$(find "$tmpdir" -name 'weed-volume*' -type f | head -1)" + fi + if [ -z "$rust_bin" ]; then + error "Could not find weed-volume binary in archive" + fi + + chmod +x "$rust_bin" + local dest_name="weed-volume" + if [ "$os" = "windows" ]; then + dest_name="weed-volume.exe" + fi + install_binary "$rust_bin" "$dest_name" + ok "Installed weed-volume to ${INSTALL_DIR}/${dest_name}" + ;; + + worker-rust) + # Published for linux only: the worker runs beside the cluster it + # maintains, and its dependency tree makes every extra target an + # expensive build. + case "$os" in + linux) ;; + *) error "Rust maintenance worker is not available for ${os}. Supported: linux" ;; + esac + case "$arch" in + amd64|arm64) ;; + *) error "Rust maintenance worker is not available for ${arch}. Supported: amd64, arm64" ;; + esac + + asset_name="$(worker_asset_name "$os" "$arch")" + download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}" + download "$download_url" "${tmpdir}/${asset_name}" + + info "Extracting ${asset_name}..." + tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir" + + local worker_bin + worker_bin="$(find "$tmpdir" -name 'weed-worker' -type f | head -1)" + if [ -z "$worker_bin" ]; then + error "Could not find weed-worker binary in archive" + fi + + chmod +x "$worker_bin" + install_binary "$worker_bin" "weed-worker" + WORKER_INSTALLED=true + ok "Installed weed-worker to ${INSTALL_DIR}/weed-worker" + ;; + + *) + error "Unknown component: ${component}. Use: weed, volume-rust, worker-rust, all" + ;; + esac + + # The trap is per-process, so a later component's would replace this one and + # leave the earlier extraction behind. Clean up here and hand the trap back; + # the error paths above exit, which still fires it. + rm -rf "$tmpdir" + trap - EXIT +} + +# Copy binary to install dir, using sudo if needed +install_binary() { + local src="$1" name="$2" + local dest="${INSTALL_DIR}/${name}" + + mkdir -p "$INSTALL_DIR" 2>/dev/null || true + + if [ -w "$INSTALL_DIR" ]; then + cp "$src" "$dest" + else + info "Need elevated permissions to write to ${INSTALL_DIR}" + sudo cp "$src" "$dest" + fi + chmod +x "$dest" 2>/dev/null || sudo chmod +x "$dest" +} + +main() { + info "SeaweedFS Installer" + + read -r os arch <<< "$(detect_platform)" + info "Detected platform: ${os}/${arch}" + + if [ -z "$VERSION" ]; then + info "Resolving latest release..." + VERSION="$(get_latest_version)" + if [ -z "$VERSION" ]; then + error "Could not determine latest version. Specify with --version" + fi + fi + info "Version: ${VERSION}" + + if [ "$LARGE_DISK" = true ]; then + info "Variant: large disk (8TB max volume)" + else + info "Variant: normal (32GB max volume)" + fi + + case "$COMPONENT" in + all) + install_component "weed" "$os" "$arch" + install_component "volume-rust" "$os" "$arch" + # The worker is published for linux amd64/arm64 only, and only by + # releases new enough to carry it; skip either case rather than fail + # an install that has already put two binaries in place. + if [ "$os" != "linux" ] || { [ "$arch" != "amd64" ] && [ "$arch" != "arm64" ]; }; then + warn "Skipping the Rust maintenance worker: no build for ${os}/${arch}" + elif ! asset_exists "https://github.com/${REPO}/releases/download/${VERSION}/$(worker_asset_name "$os" "$arch")"; then + warn "Skipping the Rust maintenance worker: ${VERSION} does not carry one" + else + install_component "worker-rust" "$os" "$arch" + fi + ;; + *) + install_component "$COMPONENT" "$os" "$arch" + ;; + esac + + echo "" + ok "Installation complete!" + if [ "$COMPONENT" = "weed" ] || [ "$COMPONENT" = "all" ]; then + info " weed: ${INSTALL_DIR}/weed" + fi + if [ "$COMPONENT" = "volume-rust" ] || [ "$COMPONENT" = "all" ]; then + info " weed-volume: ${INSTALL_DIR}/weed-volume" + fi + if [ "$WORKER_INSTALLED" = true ]; then + info " weed-worker: ${INSTALL_DIR}/weed-worker" + fi + echo "" + info "Quick start:" + info " weed master # Start master server" + info " weed volume -mserver=localhost:9333 # Start Go volume server" + info " weed-volume -mserver localhost:9333 # Start Rust volume server" + if [ "$WORKER_INSTALLED" = true ]; then + info " weed-worker --admin localhost:23646 # Start the Rust maintenance worker" + fi +} + +main diff --git a/k8s/charts/artifacthub-repo.yml b/k8s/charts/artifacthub-repo.yml new file mode 100644 index 0000000..2fac2ae --- /dev/null +++ b/k8s/charts/artifacthub-repo.yml @@ -0,0 +1,16 @@ +# Artifact Hub repository metadata file +# +# Some settings like the verified publisher flag or the ignored packages won't +# be applied until the next time the repository is processed. Please keep in +# mind that the repository won't be processed if it has not changed since the +# last time it was processed. Depending on the repository kind, this is checked +# in a different way. For Helm http based repositories, we consider it has +# changed if the `index.yaml` file changes. For git based repositories, it does +# when the hash of the last commit in the branch you set up changes. This does +# NOT apply to ownership claim operations, which are processed immediately. +# + +repositoryID: 5b2f1fe2-20e5-486e-9746-183484642aa2 +# owners: # (optional, used to claim repository ownership) +# - name: username +# email: email diff --git a/k8s/charts/seaweedfs/.helmignore b/k8s/charts/seaweedfs/.helmignore new file mode 100644 index 0000000..50af031 --- /dev/null +++ b/k8s/charts/seaweedfs/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/k8s/charts/seaweedfs/Chart.yaml b/k8s/charts/seaweedfs/Chart.yaml new file mode 100644 index 0000000..871233c --- /dev/null +++ b/k8s/charts/seaweedfs/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +description: SeaweedFS +name: seaweedfs +appVersion: "4.47" +# Dev note: Trigger a helm chart release by `git tag -a helm-` +version: 4.47.1 diff --git a/k8s/charts/seaweedfs/README.md b/k8s/charts/seaweedfs/README.md new file mode 100644 index 0000000..69bcd18 --- /dev/null +++ b/k8s/charts/seaweedfs/README.md @@ -0,0 +1,624 @@ +# SEAWEEDFS - helm chart (2.x+) + + +### Add the helm repo + +```bash +helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm +``` + +### Install the helm chart + +```bash +helm install seaweedfs seaweedfs/seaweedfs +``` + +### (Recommended) Provide `values.yaml` + +```bash +helm install --values=values.yaml seaweedfs seaweedfs/seaweedfs +``` + +## Info: +* master/filer/volume are stateful sets with anti-affinity on the hostname, +so your deployment will be spread/HA. +* leveldb2 is the default filer backend; a mysql-compatible database (memsql, ...) enables HA (multiple filer instances) and the backup/HA it can provide. +* with `filer.extraEnvironmentVars.WEED_MYSQL_ENABLED` set to `"true"`, mysql user/password are created in a k8s secret (default: `-seaweedfs-db-secret`) and injected to the filer with ENV. On any other store neither the secret nor the `WEED_MYSQL_*` env, plain or secret-backed, is rendered. +* cert config exists and can be enabled, but not been tested, requires cert-manager to be installed. + +## Prerequisites +### Database + +leveldb is the default database, this supports multiple filer replicas that will [sync automatically](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication), with some [limitations](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication#limitation). + +When the [limitations](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication#limitation) apply, or for a large number of filer replicas, an external datastore is recommended. + +Such as MySQL-compatible database, as specified in the `values.yaml` at `filer.extraEnvironmentVars`. +This database should be pre-configured and initialized. If using the default `db-init-config`, the configmap name is now dynamic (e.g., `-seaweedfs-db-init-config`). You can override this name via `filer.dbInitConfigName`. + +To initialize manually: +```sql +CREATE TABLE IF NOT EXISTS `filemeta` ( + `dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field', + `name` VARCHAR(766) NOT NULL COMMENT 'directory or file name', + `directory` TEXT NOT NULL COMMENT 'full path to parent directory', + `meta` LONGBLOB, + PRIMARY KEY (`dirhash`, `name`) +) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +``` + +Alternative database can also be configured (e.g. leveldb, postgres) following the instructions at `filer.extraEnvironmentVars`. + +#### RocksDB variant + +The `_large_disk_rocksdb` image tag ships with RocksDB pre-configured as the filer backend. +To use this image with the Helm chart, override the image on all three components and disable +the chart's default `WEED_LEVELDB2_ENABLED`, which would otherwise re-enable LevelDB2 and +override the image's built-in RocksDB configuration: + +```yaml +# Replace with the desired seaweedfs version, e.g. 3.80_large_disk_rocksdb. +master: + imageOverride: chrislusf/seaweedfs:_large_disk_rocksdb + +volume: + imageOverride: chrislusf/seaweedfs:_large_disk_rocksdb + +filer: + enablePVC: true + imageOverride: chrislusf/seaweedfs:_large_disk_rocksdb + extraEnvironmentVars: + WEED_LEVELDB2_ENABLED: "false" +``` + +Notes: + +* `master` and `volume` use the same image tag so that all components share a consistent + SeaweedFS build; RocksDB itself is only used by the filer. +* `filer.enablePVC: true` (or another form of persistent storage for the filer) is required + so that the RocksDB metadata store survives pod restarts — otherwise metadata will be lost. + +### Node Labels +Kubernetes nodes can have labels which help to define which node(Host) will run which pod: + +Here is an example: +* s3/filer/master needs the label **sw-backend=true** +* volume need the label **sw-volume=true** + +to label a node to be able to run all pod types in k8s: +``` +kubectl label node YOUR_NODE_NAME sw-volume=true sw-backend=true +``` + +on production k8s deployment you will want each pod to have a different host, +especially the volume server and the masters, all pods (master/volume/filer) +should have anti-affinity rules to disallow running multiple component pods on the same host. + +If you still want to run multiple pods of the same component (master/volume/filer) on the same host please set/update the corresponding affinity rule in values.yaml to an empty one: + +```affinity: ""``` + +## PVC - storage class ### + +On the volume stateful set added support for k8s PVC, currently example +with the simple local-path-provisioner from Rancher (comes included with k3d / k3s) +https://github.com/rancher/local-path-provisioner + +you can use ANY storage class you like, just update the correct storage-class +for your deployment. + +### Master data: hostPath vs a claim + +The master's `-mdir` holds its Raft log and snapshots, and with them the +cluster's identity (its topology UUID). `master.data.type` defaults to +`hostPath`, which does not follow a pod to another node: a master that is +rescheduled comes back with an empty data directory and a brand new cluster +UUID. With the chart's default of a single master replica there is no peer to +recover the identity from either. + +Putting the master's data on a claim avoids that: + +```yaml +master: + data: + type: "persistentVolumeClaim" + size: "1Gi" + storageClass: "" # empty uses the cluster's default StorageClass +``` + +Raft state is small, so a modest claim is enough — sizing matters far more for +volume and filer. + +The default is left at `hostPath` for backward compatibility: +`volumeClaimTemplates` is immutable on a StatefulSet, so flipping the type on a +release that already exists fails, whether the chart changes the default or you +change it yourself: + +```text +StatefulSet.apps "-seaweedfs-master" is invalid: spec: Forbidden: +updates to statefulset spec for fields other than 'replicas', ... are forbidden +``` + +New installs can set the claim from the start. To move an **existing** release +onto a claim without losing the cluster UUID, use the migration below. The +claim has to be seeded while the master is stopped: a running master rewrites +its Raft state, so copying into a live pod is silently undone by the next +restart. + +The steps below are for the chart's default of a single master +(`master.replicas: 1`). With several master replicas, repeat steps 1, 3 and 4 +for every ordinal, or migrate one at a time and let the remaining quorum +re-replicate. + +Take the names from the cluster rather than assembling them — the release +name, `nameOverride` and `fullnameOverride` all feed the chart's fullname +helper, so `-seaweedfs` is not always right: + +```bash +NS=; REL= +# scope by instance as well as component: several releases can share a namespace +STS=$(kubectl -n $NS get sts \ + -l app.kubernetes.io/instance=$REL,app.kubernetes.io/component=master \ + -o jsonpath='{.items[0].metadata.name}') +POD=$STS-0 +# a StatefulSet names its claims