snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/tls_rotation/tls_rotation_integration_test.go

@@ -0,0 +1,323 @@
+// Package tls_rotation exercises HTTPS certificate rotation end-to-end:
+// start a real `weed master` with an HTTPS listener, capture the leaf
+// served at handshake time, rewrite the cert/key files on disk, and
+// assert that a subsequent handshake sees the new leaf — all without
+// stopping the master process. The test shortens the reloader's refresh
+// window to ~half a second via WEED_TLS_CERT_REFRESH_INTERVAL so it
+// completes in seconds rather than hours.
+package tls_rotation
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"math/big"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"syscall"
+	"testing"
+	"time"
+)
+
+// TestMasterHTTPSCertRotation boots `weed master` with HTTPS, confirms
+// the initial leaf is served, rotates the cert/key pair on disk, and
+// asserts the rotated leaf is served on subsequent TLS handshakes.
+func TestMasterHTTPSCertRotation(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping HTTPS rotation integration test in -short mode")
+	}
+
+	weedBin := findWeedBinary(t)
+
+	dir := t.TempDir()
+	tlsDir := filepath.Join(dir, "tls")
+	if err := os.MkdirAll(tlsDir, 0o755); err != nil {
+		t.Fatalf("mkdir tls: %v", err)
+	}
+	certPath := filepath.Join(tlsDir, "server.crt")
+	keyPath := filepath.Join(tlsDir, "server.key")
+
+	ca, caKey := generateCA(t)
+	leafSerial1 := big.NewInt(10001)
+	leafSerial2 := big.NewInt(10002)
+
+	// Initial leaf on disk.
+	writeLeaf(t, certPath, keyPath, ca, caKey, leafSerial1)
+
+	masterDir := filepath.Join(dir, "master")
+	if err := os.MkdirAll(masterDir, 0o755); err != nil {
+		t.Fatalf("mkdir master: %v", err)
+	}
+	// Empty security.toml so the master doesn't pick up a user's
+	// ~/.seaweedfs/security.toml during the test.
+	if err := os.WriteFile(filepath.Join(masterDir, "security.toml"), []byte("# test\n"), 0o644); err != nil {
+		t.Fatalf("write security.toml: %v", err)
+	}
+
+	// Master auto-derives gRPC port as port+10000 when -port.grpc is
+	// unset, so both must fit in uint16. Pin both explicitly.
+	port, grpcPort := getFreeTCPPort(t), getFreeTCPPort(t)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, weedBin, "master",
+		"-ip", "127.0.0.1",
+		"-port", strconv.Itoa(port),
+		"-port.grpc", strconv.Itoa(grpcPort),
+		"-mdir", masterDir,
+	)
+	cmd.Dir = masterDir
+	cmd.Env = append(os.Environ(),
+		// Isolate HOME so the subprocess cannot pick up a developer's
+		// ~/.seaweedfs/security.toml. Viper's AddConfigPath uses the
+		// literal string "$HOME/.seaweedfs" without env expansion today,
+		// so this is only belt-and-braces — but it insures us against a
+		// future viper upgrade that does expand env vars.
+		"HOME="+dir,
+		"WEED_HTTPS_MASTER_CERT="+certPath,
+		"WEED_HTTPS_MASTER_KEY="+keyPath,
+		// Short refresh window so rotation completes in seconds.
+		"WEED_TLS_CERT_REFRESH_INTERVAL=500ms",
+	)
+	logPath := filepath.Join(masterDir, "master.log")
+	logOut, err := os.Create(logPath)
+	if err != nil {
+		t.Fatalf("create master log: %v", err)
+	}
+	cmd.Stdout = logOut
+	cmd.Stderr = logOut
+
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("start master: %v", err)
+	}
+	t.Cleanup(func() {
+		if cmd.Process != nil {
+			_ = cmd.Process.Signal(syscall.SIGTERM)
+			done := make(chan struct{})
+			go func() { _ = cmd.Wait(); close(done) }()
+			select {
+			case <-done:
+			case <-time.After(10 * time.Second):
+				_ = cmd.Process.Kill()
+				<-done
+			}
+		}
+		_ = logOut.Close()
+		if t.Failed() {
+			if b, readErr := os.ReadFile(logPath); readErr == nil {
+				t.Logf("master.log:\n%s", string(b))
+			}
+		}
+	})
+
+	caPool := x509.NewCertPool()
+	caPool.AddCert(ca)
+
+	addr := fmt.Sprintf("127.0.0.1:%d", port)
+
+	// 1. Wait for the initial leaf to appear. Master takes a few seconds
+	//    to open its HTTPS listener.
+	waitForCert(t, addr, caPool, leafSerial1, 30*time.Second, "initial cert")
+
+	// Sanity: same handshake twice still observes the initial leaf.
+	got, err := peekServerCert(addr, caPool)
+	if err != nil || got == nil || got.SerialNumber.Cmp(leafSerial1) != 0 {
+		t.Fatalf("second probe before rotation did not return initial leaf: cert=%v err=%v", got, err)
+	}
+
+	// 2. Rotate on disk. pemfile watches mtime, so each file's write is
+	//    an atomic rename (tempfile in the same directory).
+	writeLeaf(t, certPath, keyPath, ca, caKey, leafSerial2)
+
+	// 3. Wait for new leaf to take over. With a 500ms refresh and no
+	//    connection pooling (tls.Dial opens a fresh conn each time), this
+	//    should take a couple of seconds.
+	waitForCert(t, addr, caPool, leafSerial2, 15*time.Second, "rotated cert")
+}
+
+// waitForCert polls until a TLS handshake against addr yields a peer
+// cert with the expected serial, or fails the test at the deadline.
+// The last handshake error is surfaced in the fatal message so that a
+// CI flake makes the root cause obvious (master didn't come up, TLS
+// handshake rejected, CA pool mismatch, etc.).
+func waitForCert(t *testing.T, addr string, caPool *x509.CertPool, wantSerial *big.Int, within time.Duration, label string) {
+	t.Helper()
+	deadline := time.Now().Add(within)
+	var lastErr error
+	var lastSerial *big.Int
+	for time.Now().Before(deadline) {
+		cert, err := peekServerCert(addr, caPool)
+		if err != nil {
+			lastErr = err
+		} else if cert != nil {
+			lastSerial = cert.SerialNumber
+			if cert.SerialNumber.Cmp(wantSerial) == 0 {
+				return
+			}
+		}
+		time.Sleep(250 * time.Millisecond)
+	}
+	t.Fatalf("timeout waiting for %s (want serial %s, last seen %v, last err %v)", label, wantSerial, lastSerial, lastErr)
+}
+
+// peekServerCert opens a one-shot TLS connection and returns the leaf.
+// Errors (dial failure, handshake rejection, empty peer chain) are
+// returned rather than swallowed, so the caller can surface them when
+// the test times out.
+func peekServerCert(addr string, caPool *x509.CertPool) (*x509.Certificate, error) {
+	d := &net.Dialer{Timeout: 2 * time.Second}
+	conn, err := tls.DialWithDialer(d, "tcp", addr, &tls.Config{
+		RootCAs:    caPool,
+		ServerName: "localhost",
+		MinVersion: tls.VersionTLS12,
+	})
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+	state := conn.ConnectionState()
+	if len(state.PeerCertificates) == 0 {
+		return nil, fmt.Errorf("handshake returned empty peer chain")
+	}
+	return state.PeerCertificates[0], nil
+}
+
+func getFreeTCPPort(t *testing.T) int {
+	t.Helper()
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen ephemeral: %v", err)
+	}
+	port := ln.Addr().(*net.TCPAddr).Port
+	_ = ln.Close()
+	return port
+}
+
+func findWeedBinary(t *testing.T) string {
+	t.Helper()
+	candidates := []string{
+		"../../weed/weed",
+		"../weed/weed",
+		"./weed",
+	}
+	for _, c := range candidates {
+		if _, err := os.Stat(c); err == nil {
+			abs, absErr := filepath.Abs(c)
+			if absErr == nil {
+				return abs
+			}
+			return c
+		}
+	}
+	if path, err := exec.LookPath("weed"); err == nil {
+		return path
+	}
+	t.Skip("weed binary not found — build with `cd weed && go build` first")
+	return ""
+}
+
+// --- cert fixtures -------------------------------------------------------
+
+// generateCA returns a self-signed CA cert and its private key.
+func generateCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey) {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("gen CA key: %v", err)
+	}
+	tmpl := &x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "tls-rotation-test-CA"},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().Add(24 * time.Hour),
+		IsCA:                  true,
+		BasicConstraintsValid: true,
+		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	if err != nil {
+		t.Fatalf("create CA cert: %v", err)
+	}
+	parsed, err := x509.ParseCertificate(der)
+	if err != nil {
+		t.Fatalf("parse CA cert: %v", err)
+	}
+	return parsed, key
+}
+
+// writeLeaf signs a new leaf cert with the given serial and writes it
+// plus its key to the given paths via atomic rename — the pattern
+// Kubernetes (cert-manager → Secret volume mount) produces in practice.
+func writeLeaf(t *testing.T, certPath, keyPath string, ca *x509.Certificate, caKey *ecdsa.PrivateKey, serial *big.Int) {
+	t.Helper()
+	leafKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("gen leaf key: %v", err)
+	}
+	tmpl := &x509.Certificate{
+		SerialNumber: serial,
+		Subject: pkix.Name{
+			CommonName: "localhost",
+		},
+		NotBefore:   time.Now().Add(-time.Hour),
+		NotAfter:    time.Now().Add(24 * time.Hour),
+		KeyUsage:    x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+		DNSNames:    []string{"localhost"},
+		IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, ca, &leafKey.PublicKey, caKey)
+	if err != nil {
+		t.Fatalf("create leaf cert: %v", err)
+	}
+
+	atomicWritePEM(t, certPath, "CERTIFICATE", der)
+
+	keyDER, err := x509.MarshalECPrivateKey(leafKey)
+	if err != nil {
+		t.Fatalf("marshal leaf key: %v", err)
+	}
+	atomicWritePEM(t, keyPath, "EC PRIVATE KEY", keyDER)
+}
+
+// atomicWritePEM writes a PEM file via tempfile-in-same-directory plus
+// rename, matching what kubelet does when it swaps the ..data symlink
+// for a renewed Secret. Ensures the reader never sees a truncated file.
+func atomicWritePEM(t *testing.T, path, blockType string, der []byte) {
+	t.Helper()
+	dir := filepath.Dir(path)
+	tmp, err := os.CreateTemp(dir, ".tls-*")
+	if err != nil {
+		t.Fatalf("create tempfile: %v", err)
+	}
+	ok := false
+	defer func() {
+		if !ok {
+			_ = os.Remove(tmp.Name())
+		}
+	}()
+	if err := pem.Encode(tmp, &pem.Block{Type: blockType, Bytes: der}); err != nil {
+		tmp.Close()
+		t.Fatalf("pem encode: %v", err)
+	}
+	if err := tmp.Close(); err != nil {
+		t.Fatalf("close tempfile: %v", err)
+	}
+	if err := os.Chmod(tmp.Name(), 0o600); err != nil {
+		t.Fatalf("chmod tempfile: %v", err)
+	}
+	if err := os.Rename(tmp.Name(), path); err != nil {
+		t.Fatalf("rename tempfile onto %s: %v", path, err)
+	}
+	ok = true
+}