snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/pb/Makefile

@@ -0,0 +1,28 @@
+all: gen
+
+.PHONY : gen
+
+gen:
+	protoc master.proto --go_out=./master_pb --go-grpc_out=./master_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
+	protoc volume_server.proto --go_out=./volume_server_pb --go-grpc_out=./volume_server_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
+	protoc filer.proto --go_out=./filer_pb --go-grpc_out=./filer_pb --go-vtproto_out=./filer_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative --go-vtproto_opt=paths=source_relative,features=marshal+unmarshal+size
+	protoc remote.proto --go_out=./remote_pb --go-grpc_out=./remote_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
+	protoc iam.proto --go_out=./iam_pb --go-grpc_out=./iam_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
+	protoc mount.proto --go_out=./mount_pb --go-grpc_out=./mount_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
+	mkdir -p ./mount_peer_pb
+	protoc mount_peer.proto --go_out=./mount_peer_pb --go-grpc_out=./mount_peer_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
+	protoc s3.proto --go_out=./s3_pb --go-grpc_out=./s3_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
+	mkdir -p ./s3_lifecycle_pb
+	protoc s3_lifecycle.proto --go_out=./s3_lifecycle_pb --go-grpc_out=./s3_lifecycle_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
+	protoc mq_broker.proto --go_out=./mq_pb --go-grpc_out=./mq_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
+	protoc mq_schema.proto --go_out=./schema_pb --go-grpc_out=./schema_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
+	protoc mq_agent.proto --go_out=./mq_agent_pb --go-grpc_out=./mq_agent_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
+	protoc worker.proto --go_out=./worker_pb --go-grpc_out=./worker_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
+	mkdir -p ./plugin_pb
+	protoc plugin.proto --go_out=./plugin_pb --go-grpc_out=./plugin_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
+	# protoc filer.proto --java_out=../../other/java/client/src/main/java
+	cp filer.proto ../../other/java/client/src/main/proto
+	cp volume_server.proto master.proto remote.proto ../../seaweed-volume/proto/
+
+fbs:
+	flatc --go -o . --go-namespace message_fbs message.fbs

weed/pb/filer.proto

@@ -0,0 +1,955 @@
+syntax = "proto3";
+
+package filer_pb;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb";
+option java_package = "seaweedfs.client";
+option java_outer_classname = "FilerProto";
+
+//////////////////////////////////////////////////
+
+service SeaweedFiler {
+
+    rpc LookupDirectoryEntry (LookupDirectoryEntryRequest) returns (LookupDirectoryEntryResponse) {
+    }
+
+    rpc LookupDirectoryEntries (LookupDirectoryEntriesRequest) returns (LookupDirectoryEntriesResponse) {
+    }
+
+    rpc ListEntries (ListEntriesRequest) returns (stream ListEntriesResponse) {
+    }
+
+    rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) {
+    }
+
+    rpc UpdateEntry (UpdateEntryRequest) returns (UpdateEntryResponse) {
+    }
+
+    rpc TouchAccessTime (TouchAccessTimeRequest) returns (TouchAccessTimeResponse) {
+    }
+
+    rpc AppendToEntry (AppendToEntryRequest) returns (AppendToEntryResponse) {
+    }
+
+    rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
+    }
+
+    rpc ObjectTransaction (ObjectTransactionRequest) returns (ObjectTransactionResponse) {
+    }
+
+    rpc ObjectTransactionBatch (ObjectTransactionBatchRequest) returns (ObjectTransactionBatchResponse) {
+    }
+
+    rpc PosixLock (PosixLockRequest) returns (PosixLockResponse) {
+    }
+
+    rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
+    }
+    rpc StreamRenameEntry (StreamRenameEntryRequest) returns (stream StreamRenameEntryResponse) {
+    }
+
+    rpc StreamMutateEntry (stream StreamMutateEntryRequest) returns (stream StreamMutateEntryResponse) {
+    }
+
+    rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
+    }
+
+    rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) {
+    }
+
+    rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) {
+    }
+
+    rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
+    }
+
+    rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
+    }
+
+    rpc Ping (PingRequest) returns (PingResponse) {
+    }
+
+    rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) {
+    }
+
+    rpc TraverseBfsMetadata (TraverseBfsMetadataRequest) returns (stream TraverseBfsMetadataResponse) {
+    }
+
+    rpc SubscribeMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) {
+    }
+
+    rpc SubscribeLocalMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) {
+    }
+
+    // List the metadata subscribers currently connected to this filer
+    // (FUSE mounts, S3, filer.sync, peer filers, ...).
+    rpc ListMetadataSubscribers (ListMetadataSubscribersRequest) returns (ListMetadataSubscribersResponse) {
+    }
+
+    rpc KvGet (KvGetRequest) returns (KvGetResponse) {
+    }
+
+    rpc KvPut (KvPutRequest) returns (KvPutResponse) {
+    }
+
+    rpc CacheRemoteObjectToLocalCluster (CacheRemoteObjectToLocalClusterRequest) returns (CacheRemoteObjectToLocalClusterResponse) {
+    }
+
+    rpc DistributedLock(LockRequest) returns (LockResponse) {
+    }
+    rpc DistributedUnlock(UnlockRequest) returns (UnlockResponse) {
+    }
+    rpc FindLockOwner(FindLockOwnerRequest) returns (FindLockOwnerResponse) {
+    }
+    // distributed lock management internal use only
+    rpc TransferLocks(TransferLocksRequest) returns (TransferLocksResponse) {
+    }
+    rpc ReplicateLock(ReplicateLockRequest) returns (ReplicateLockResponse) {
+    }
+
+    // Peer chunk sharing — tier 1: mount-server registry.
+    // See design-weed-mount-peer-chunk-sharing.md for details.
+    rpc MountRegister (MountRegisterRequest) returns (MountRegisterResponse) {
+    }
+    rpc MountList (MountListRequest) returns (MountListResponse) {
+    }
+}
+
+//////////////////////////////////////////////////
+
+message LookupDirectoryEntryRequest {
+    string directory = 1;
+    string name = 2;
+}
+
+message LookupDirectoryEntryResponse {
+    Entry entry = 1;
+    // filer log position stamped before the entry read: every event at or
+    // below it is reflected in the returned entry
+    int64 log_ts_ns = 2;
+    // signature of the filer whose clock stamped log_ts_ns; positions are
+    // only comparable with events that filer logged
+    int32 log_signature = 3;
+}
+
+message ListEntriesRequest {
+    string directory = 1;
+    string prefix = 2;
+    string startFromFileName = 3;
+    bool inclusiveStartFrom = 4;
+    uint32 limit = 5;
+    int64 snapshot_ts_ns = 6;
+    // Leave the chunk list out of every entry in the response. The attributes
+    // still carry the file size, so a listing that only reads attributes can
+    // ask for this and skip the largest part of the payload.
+    bool omit_chunks = 7;
+}
+
+message ListEntriesResponse {
+    Entry entry = 1;
+    int64 snapshot_ts_ns = 2;
+}
+
+message RemoteEntry {
+    string storage_name = 1;
+    int64 last_local_sync_ts_ns = 2;
+    string remote_e_tag = 3;
+    int64 remote_mtime = 4;
+    int64 remote_size = 5;
+    // unset when the remote listing does not report encodings (S3);
+    // empty when the remote object authoritatively has none
+    optional string remote_content_encoding = 6;
+}
+message Entry {
+    string name = 1;
+    bool is_directory = 2;
+    repeated FileChunk chunks = 3;
+    FuseAttributes attributes = 4;
+    map<string, bytes> extended = 5;
+    bytes hard_link_id = 7;
+    int32 hard_link_counter = 8; // only exists in hard link meta data
+    bytes content = 9; // if not empty, the file content
+
+    RemoteEntry remote_entry = 10;
+    int64 quota = 11; // for bucket only. Positive/Negative means enabled/disabled.
+    int64 worm_enforced_at_ts_ns = 12;
+}
+
+message FullEntry {
+    string dir = 1;
+    Entry entry = 2;
+}
+
+message EventNotification {
+    Entry old_entry = 1;
+    Entry new_entry = 2;
+    bool delete_chunks = 3;
+    string new_parent_path = 4;
+    bool is_from_other_cluster = 5;
+    repeated int32 signatures = 6;
+}
+
+enum SSEType {
+    NONE = 0;    // No server-side encryption
+    SSE_C = 1;   // Server-Side Encryption with Customer-Provided Keys  
+    SSE_KMS = 2; // Server-Side Encryption with KMS-Managed Keys
+    SSE_S3 = 3;  // Server-Side Encryption with S3-Managed Keys
+}
+
+message FileChunk {
+    string file_id = 1; // to be deprecated
+    int64 offset = 2;
+    uint64 size = 3;
+    int64 modified_ts_ns = 4;
+    string e_tag = 5;
+    string source_file_id = 6; // to be deprecated
+    FileId fid = 7;
+    FileId source_fid = 8;
+    bytes cipher_key = 9;
+    bool is_compressed = 10;
+    bool is_chunk_manifest = 11; // content is a list of FileChunks
+    SSEType sse_type = 12;           // Server-side encryption type
+    bytes sse_metadata = 13;         // Serialized SSE metadata for this chunk (SSE-C, SSE-KMS, or SSE-S3)
+}
+
+message FileChunkManifest {
+    repeated FileChunk chunks = 1;
+}
+
+message FileId {
+    uint32 volume_id = 1;
+    uint64 file_key = 2;
+    fixed32 cookie = 3;
+}
+
+message FuseAttributes {
+    uint64 file_size = 1;
+    int64 mtime = 2; // unix time in seconds
+    uint32 file_mode = 3;
+    uint32 uid = 4;
+    uint32 gid = 5;
+    int64 crtime = 6; // unix time in seconds
+    string mime = 7;
+    int32 ttl_sec = 10;
+    string user_name = 11; // for hdfs
+    repeated string group_name = 12; // for hdfs
+    string symlink_target = 13;
+    bytes md5 = 14;
+    uint32 rdev = 16;
+    uint64 inode = 17;
+    int64 ctime = 18; // unix time in seconds, inode change time
+    int32 mtime_ns = 19; // nanosecond component of mtime (0-999999999)
+    int32 ctime_ns = 20; // nanosecond component of ctime (0-999999999)
+    int32 crtime_ns = 21; // nanosecond component of crtime (0-999999999)
+    int64 atime = 22; // unix time in seconds, last access time
+    int32 atime_ns = 23; // nanosecond component of atime (0-999999999)
+}
+
+message CreateEntryRequest {
+    string directory = 1;
+    Entry entry = 2;
+    bool o_excl = 3;
+    bool is_from_other_cluster = 4;
+    repeated int32 signatures = 5;
+    bool skip_check_parent_directory = 6;
+    // Optional precondition evaluated against the current entry atomically with
+    // the write, under the filer's per-path lock.
+    WriteCondition condition = 7;
+    // Set on a create a filer forwarded to the entry's ring owner, so the owner
+    // applies it locally instead of forwarding again. Clients leave it unset.
+    bool is_moved = 8;
+}
+
+// WriteCondition is the precondition the filer evaluates against the existing
+// entry before writing, under the per-path lock. A failed condition returns
+// FilerError PRECONDITION_FAILED. The client maps request semantics (e.g. RFC
+// 7232) to clauses; the filer just compares.
+//
+// A condition is a list of clauses that ALL must hold (logical AND). One clause
+// is the common case; several express what a single comparison cannot: an ETag
+// set (If-Match / If-None-Match with multiple values), weak-ETag comparison, and
+// compound conditions (e.g. If-Match + If-Unmodified-Since together).
+message WriteCondition {
+    enum Kind {
+        NONE                  = 0; // unconditional
+        IF_NOT_EXISTS         = 1; // fail if the entry exists (If-None-Match: *)
+        IF_EXISTS             = 2; // fail if the entry is absent (If-Match: *)
+        IF_ETAG_MATCH         = 3; // fail if absent or etag matches none of the set (If-Match)
+        IF_ETAG_NOT_MATCH     = 4; // fail if present and etag matches any of the set (If-None-Match)
+        IF_UNMODIFIED_SINCE   = 5; // fail if present and mtime > unix_time
+        IF_MODIFIED_SINCE     = 6; // fail if present and mtime <= unix_time
+        IF_EXTENDED_NOT_EQUAL = 7; // fail if present and extended[ext_key] == ext_value
+        IF_EXTENDED_TIME_ELAPSED = 8; // fail if present and extended[ext_key] (unix seconds) is in the future
+        IF_CHUNKS_EQUAL       = 9; // fail unless the stored chunk fid multiset equals fids (absent entry = no chunks)
+    }
+    // Clause is one primitive comparison. IF_ETAG_MATCH holds when the current
+    // entry's ETag equals any value in etags; IF_ETAG_NOT_MATCH holds when it
+    // equals none. allow_weak permits weak-comparison (ignoring the W/ prefix).
+    //
+    // The IF_EXTENDED_* kinds are generic guards on an extended attribute, used
+    // to enforce object-lock without teaching the filer S3 semantics:
+    // IF_EXTENDED_NOT_EQUAL expresses a legal hold (block while a key equals a
+    // value), and IF_EXTENDED_TIME_ELAPSED expresses retention (block while a
+    // stored unix-second deadline is in the future, compared to the filer's
+    // clock). The caller composes these and, for governance-bypass, simply omits
+    // the retention clause when the bypass is authorized — the filer makes no
+    // authorization decision.
+    //
+    // IF_CHUNKS_EQUAL guards a chunk-preserving read-modify-write: the stored
+    // chunk fid set must still equal what the caller read, so a stale write
+    // cannot resurrect needles that a concurrent update already diffed away
+    // and queued for deletion. An empty fids list expects no chunks.
+    message Clause {
+        Kind kind = 1;
+        repeated string etags = 2; // ETag set for IF_ETAG_* kinds
+        int64 unix_time = 3;       // bound (unix seconds) for IF_*_SINCE kinds
+        bool allow_weak = 4;       // compare ETags ignoring the weak (W/) marker
+        string ext_key = 5;        // extended attribute name for IF_EXTENDED_* kinds
+        string ext_value = 6;      // blocking value for IF_EXTENDED_NOT_EQUAL
+        string gate_key = 7;       // IF_EXTENDED_TIME_ELAPSED: only enforce when extended[gate_key] == gate_value
+        string gate_value = 8;     // gate value (e.g. retention mode COMPLIANCE for governance bypass)
+        repeated string fids = 9;  // chunk fid strings for IF_CHUNKS_EQUAL
+    }
+    repeated Clause clauses = 1;   // all must hold (logical AND)
+}
+
+// Structured error codes for filer entry operations.
+// Values are stable — do not reorder or reuse numbers.
+enum FilerError {
+    OK                    = 0;
+    ENTRY_NAME_TOO_LONG   = 1; // name exceeds max_file_name_length
+    PARENT_IS_FILE        = 2; // parent path component is a file, not a directory
+    EXISTING_IS_DIRECTORY = 3; // cannot overwrite directory with file
+    EXISTING_IS_FILE      = 4; // cannot overwrite file with directory
+    ENTRY_ALREADY_EXISTS  = 5; // O_EXCL and entry already exists
+    PRECONDITION_FAILED   = 6; // WriteCondition not satisfied
+}
+
+// ObjectMutation is one entry-level change applied by ObjectTransaction. All
+// mutations of a transaction run under a single per-path lock (the request's
+// lock_key) and in order, so the gateway can describe a multi-entry object
+// operation as one request instead of holding a distributed lock across
+// several RPCs. Data-bearing writes (entries with chunks) should be written
+// before the transaction; mutations here are metadata-scoped.
+message ObjectMutation {
+    enum Type {
+        PUT              = 0; // create or replace the entry (entry field)
+        DELETE           = 1; // delete the entry at directory/name (no error if absent)
+        PATCH_EXTENDED   = 2; // merge set_extended / remove delete_extended on the entry
+        RECOMPUTE_LATEST = 3; // scan a directory and re-point a parent entry (recompute)
+    }
+    Type type = 1;
+    string directory = 2;
+    string name = 3;                       // entry name for DELETE / PATCH_EXTENDED / RECOMPUTE_LATEST (the pointer entry)
+    Entry entry = 4;                        // full entry for PUT
+    map<string, bytes> set_extended = 5;   // PATCH_EXTENDED: keys to set
+    repeated string delete_extended = 6;   // PATCH_EXTENDED: keys to remove
+    bool is_delete_data = 7;               // DELETE: also delete chunk data
+    bool is_recursive = 8;                 // DELETE: recurse into a directory
+    Recompute recompute = 9;               // RECOMPUTE_LATEST parameters
+    bool set_content = 10;                 // PATCH_EXTENDED: replace Entry.content with content
+    bytes content = 11;                    // PATCH_EXTENDED: new Entry.content when set_content
+    bool touch_mtime = 12;                 // PATCH_EXTENDED: set the entry's Mtime to now (e.g. a metadata-replace copy)
+    bool remove_empty_parent = 13;         // DELETE: also remove the parent directory when the delete leaves it empty (best-effort)
+}
+
+// Recompute re-derives a pointer entry (directory/name on the mutation) from the
+// current contents of a scanned directory, atomically under the transaction's
+// lock. It is mechanical: the filer picks the child that sorts first or last by
+// name and copies the requested fields into the pointer; it has no knowledge of
+// what the entries mean. The caller (which does know the versioning scheme)
+// supplies the sort direction and the key mappings. This covers re-pointing the
+// latest version after a specific version is deleted, where the scan must run
+// under the lock.
+message Recompute {
+    string scan_dir = 1;                   // directory whose direct children are scanned
+    bool descending = 2;                   // pick the child that sorts last by name (else first)
+    map<string, string> copy_extended = 3; // pointer extended key -> source extended key on the chosen child
+    string name_to_key = 4;                // if set, store the chosen child's name under this pointer key
+    string size_to_key = 5;                // if set, store the chosen child's FileSize (decimal) under this pointer key
+    string mtime_to_key = 6;               // if set, store the chosen child's Mtime (decimal) under this pointer key
+    string demote_key = 7;                 // if set, stamp demote_value on the prior name_to_key target when it changes
+    bytes demote_value = 8;                // value for demote_key
+    string exclude_name = 9;               // if set, skip this child when scanning (e.g. a version about to be deleted)
+}
+
+// ObjectTransactionRequest applies an ordered list of mutations atomically with
+// respect to other writers of the same object, by holding the filer's per-path
+// lock on lock_key for the whole transaction. The optional condition is checked
+// first, against condition_key when set, else lock_key. Callers set route_key to
+// the object's stable owner ring key; a filer that is not the owner forwards the
+// transaction one hop to the owner, so a stale ring view is tolerated.
+message ObjectTransactionRequest {
+    string lock_key = 1;               // object path to lock and to evaluate the condition against
+    WriteCondition condition = 2;      // optional precondition, checked under the lock
+    repeated ObjectMutation mutations = 3;
+    bool is_from_other_cluster = 4;
+    repeated int32 signatures = 5;
+    string condition_key = 6;          // if set, evaluate the condition against this entry instead of lock_key (still locking lock_key)
+    string route_key = 7;              // ring key identifying the owner filer; a non-owner forwards the whole transaction to it
+    bool is_moved = 8;                 // set on a forwarded transaction so the receiver applies it locally instead of forwarding again
+}
+
+message ObjectTransactionResponse {
+    string error = 1;
+    FilerError error_code = 2;
+}
+
+// PosixLockRange is one advisory byte-range lock. Owner identity is (sid, owner):
+// sid is the mount session, owner the FUSE lock owner within it, so owners from
+// different mounts never alias. end is inclusive (max uint64 = to EOF); is_flock
+// separates the flock and fcntl namespaces, which never conflict.
+message PosixLockRange {
+    uint64 start = 1;
+    uint64 end = 2;
+    uint32 type = 3;       // 1=read, 2=write, 3=unlock
+    uint64 sid = 4;
+    uint64 owner = 5;
+    uint32 pid = 6;        // holder pid, for get_lk reporting only
+    bool is_flock = 7;
+}
+
+// PosixLock routes an advisory lock operation to the inode's owner filer, which
+// holds the authoritative in-memory lock table. key is the inode identity ring
+// key (the file path, or hl:<HardLinkId> for a hardlink) used both to resolve the
+// owner and to index the table. A non-owner filer forwards the request one hop;
+// is_moved bounds it so a stale ring view cannot loop.
+message PosixLockRequest {
+    string key = 1;
+    bool is_moved = 2;
+    PosixLockOp op = 3;
+    PosixLockRange lock = 4;
+    // locks carries the full set a mount holds on key for a KEEP_ALIVE
+    // re-assertion, so the current owner filer can rebuild its in-memory state
+    // after an ownership change or restart. lock.sid identifies the session.
+    repeated PosixLockRange locks = 5;
+    // cooling_probe marks a dual-read a new owner sends to the previous owner
+    // during a ring change, so the previous owner answers from local state
+    // without itself cooling-off (no recursion).
+    bool cooling_probe = 6;
+}
+
+enum PosixLockOp {
+    TRY_LOCK = 0;             // grant lock or report conflict (non-blocking)
+    UNLOCK = 1;              // release lock's owner's locks over its range
+    GET_LK = 2;             // report a conflicting lock, if any
+    RELEASE_POSIX_OWNER = 3; // drop the owner's fcntl locks (flush-time)
+    RELEASE_FLOCK_OWNER = 4; // drop the owner's flock locks (release-time)
+    KEEP_ALIVE = 5;         // renew the session's lease on this owner (lock.sid)
+}
+
+message PosixLockResponse {
+    bool granted = 1;            // for TRY_LOCK: whether the lock was granted
+    bool has_conflict = 2;       // whether conflict is populated
+    PosixLockRange conflict = 3; // the blocking lock (TRY_LOCK conflict / GET_LK result)
+}
+
+// ObjectTransactionBatch applies several object transactions in one round trip,
+// each under its own per-path lock and independent of the others (no cross-key
+// atomicity). A caller groups keys that route to the same owner filer and sends
+// one batch per owner, e.g. for a multi-object delete. Each response is parallel
+// to its request.
+message ObjectTransactionBatchRequest {
+    repeated ObjectTransactionRequest transactions = 1;
+}
+
+message ObjectTransactionBatchResponse {
+    repeated ObjectTransactionResponse responses = 1;
+}
+
+message CreateEntryResponse {
+    string error = 1;                          // kept for human readability + backward compat
+    SubscribeMetadataResponse metadata_event = 2;
+    FilerError error_code = 3;                 // machine-readable error code
+    // filer log position stamped under the path lock before the write:
+    // every event at or below it is reflected in the acknowledged state
+    int64 log_ts_ns = 4;
+    int32 log_signature = 5; // filer whose clock stamped log_ts_ns
+}
+
+message UpdateEntryRequest {
+    string directory = 1;
+    Entry entry = 2;
+    bool is_from_other_cluster = 3;
+    repeated int32 signatures = 4;
+    map<string, bytes> expected_extended = 5;
+    // Optional precondition evaluated against the current entry atomically with
+    // the write, under the filer's per-path lock. The caller must route the
+    // key's writes to this entry's owner filer for the check to be authoritative.
+    WriteCondition condition = 6;
+}
+message UpdateEntryResponse {
+    SubscribeMetadataResponse metadata_event = 1;
+    // filer log position stamped under the path lock before the write:
+    // every event at or below it is reflected in the acknowledged state
+    int64 log_ts_ns = 2;
+    int32 log_signature = 3; // filer whose clock stamped log_ts_ns
+}
+
+message TouchAccessTimeRequest {
+    string directory = 1;
+    string name = 2;
+    int64 client_atime_ns = 3; // nanoseconds since epoch; filer may override with relatime
+}
+message TouchAccessTimeResponse {
+    int64 persisted_atime_ns = 1; // nanoseconds since epoch; 0 if no update was performed
+    bool updated = 2;
+}
+
+message AppendToEntryRequest {
+    string directory = 1;
+    string entry_name = 2;
+    repeated FileChunk chunks = 3;
+}
+message AppendToEntryResponse {
+}
+
+message DeleteEntryRequest {
+    string directory = 1;
+    string name = 2;
+    // bool is_directory = 3;
+    bool is_delete_data = 4;
+    bool is_recursive = 5;
+    bool ignore_recursive_error = 6;
+    bool is_from_other_cluster = 7;
+    repeated int32 signatures = 8;
+    int64 if_not_modified_after = 9;
+}
+
+message DeleteEntryResponse {
+    string error = 1;
+    SubscribeMetadataResponse metadata_event = 2;
+}
+
+message AtomicRenameEntryRequest {
+    string old_directory = 1;
+    string old_name = 2;
+    string new_directory = 3;
+    string new_name = 4;
+    repeated int32 signatures = 5;
+}
+
+message AtomicRenameEntryResponse {
+}
+
+message StreamRenameEntryRequest {
+    string old_directory = 1;
+    string old_name = 2;
+    string new_directory = 3;
+    string new_name = 4;
+    repeated int32 signatures = 5;
+}
+message StreamRenameEntryResponse {
+    string directory = 1;
+    EventNotification event_notification = 2;
+    int64 ts_ns = 3;
+}
+message AssignVolumeRequest {
+    int32 count = 1;
+    string collection = 2;
+    string replication = 3;
+    int32 ttl_sec = 4;
+    string data_center = 5;
+    string path = 6;
+    string rack = 7;
+    string data_node = 9;
+    string disk_type = 8;
+    uint64 expected_data_size = 10; // hint for size-aware volume selection
+}
+
+message AssignVolumeResponse {
+    string file_id = 1;
+    int32 count = 4;
+    string auth = 5;
+    string collection = 6;
+    string replication = 7;
+    string error = 8;
+    Location location = 9;
+    repeated Location replicas = 10;
+    // fsync is the storage rule's fsync decision for the assigned path, so the
+    // client can carry it onto the volume server upload request.
+    bool fsync = 11;
+}
+
+message LookupVolumeRequest {
+    repeated string volume_ids = 1;
+}
+
+message Locations {
+    repeated Location locations = 1;
+}
+
+message Location {
+    string url = 1;
+    string public_url = 2;
+    uint32 grpc_port = 3;
+    string data_center = 4;
+    bool data_in_remote = 5;
+}
+message LookupVolumeResponse {
+    map<string, Locations> locations_map = 1;
+}
+
+message Collection {
+    string name = 1;
+}
+message CollectionListRequest {
+    bool include_normal_volumes = 1;
+    bool include_ec_volumes = 2;
+}
+message CollectionListResponse {
+    repeated Collection collections = 1;
+}
+message DeleteCollectionRequest {
+    string collection = 1;
+}
+
+message DeleteCollectionResponse {
+}
+
+message StatisticsRequest {
+    string replication = 1;
+    string collection = 2;
+    string ttl = 3;
+    string disk_type = 4;
+}
+message StatisticsResponse {
+    uint64 total_size = 4;
+    uint64 used_size = 5;
+    uint64 file_count = 6;
+    // sizes counting one copy of the data, as reported by the master
+    uint64 logical_total_size = 7;
+    uint64 logical_used_size = 8;
+}
+
+message PingRequest {
+    string target = 1; // default to ping itself
+    string target_type = 2;
+}
+message PingResponse {
+    int64 start_time_ns = 1;
+    int64 remote_time_ns = 2;
+    int64 stop_time_ns = 3;
+}
+
+message GetFilerConfigurationRequest {
+}
+message GetFilerConfigurationResponse {
+    repeated string masters = 1;
+    string replication = 2;
+    string collection = 3;
+    uint32 max_mb = 4;
+    string dir_buckets = 5;
+    bool cipher = 7;
+    int32 signature = 8;
+    string metrics_address = 9;
+    int32 metrics_interval_sec = 10;
+    string version = 11;
+    string cluster_id = 12;
+    string filer_group = 13;
+    int32 major_version = 14;
+    int32 minor_version = 15;
+}
+
+message SubscribeMetadataRequest {
+    string client_name = 1;
+    string path_prefix = 2;
+    int64 since_ns = 3;
+    int32 signature = 4;
+    repeated string path_prefixes = 6;
+    int32 client_id = 7;
+    int64 until_ns = 8;
+    int32 client_epoch = 9;
+    repeated string directories = 10;  // exact directory to watch
+    bool client_supports_batching = 11; // client can unpack SubscribeMetadataResponse.events
+    bool client_supports_metadata_chunks = 12; // client can read log file chunks from volume servers
+    bool client_supports_idle_heartbeat = 13; // server may send empty responses carrying the current time while the client is caught up
+}
+message SubscribeMetadataResponse {
+    string directory = 1;
+    EventNotification event_notification = 2;
+    int64 ts_ns = 3;
+    repeated SubscribeMetadataResponse events = 4; // batch of additional events (backlog catch-up)
+    repeated LogFileChunkRef log_file_refs = 5;    // log file chunk refs for client direct-read
+    int64 flushed_ts_ns = 6; // local log-buffer flush watermark: everything at or below it is on disk
+}
+message ListMetadataSubscribersRequest {
+    repeated string client_types = 1; // optional filter by client type, e.g. "mount"; empty = all
+}
+message ListMetadataSubscribersResponse {
+    repeated MetadataSubscriber subscribers = 1;
+}
+message MetadataSubscriber {
+    string client_name = 1;     // "<type>@<address>"
+    string client_type = 2;     // e.g. "mount", "sw-vfs", "s3", "filer:<addr>"
+    string address = 3;         // client peer address
+    string path_prefix = 4;     // subscribed path prefix
+    int32 client_id = 5;
+    int32 client_epoch = 6;
+    int64 connected_at_ns = 7;
+    string filer_address = 8;   // the filer this subscriber is connected to
+}
+// A persisted log file that the client can read directly from volume servers.
+// The file format is: [4-byte size | protobuf LogEntry] repeated.
+// Each LogEntry.Data contains a marshaled SubscribeMetadataResponse.
+message LogFileChunkRef {
+    repeated FileChunk chunks = 1; // chunk references (fids) to read from volume servers
+    int64 file_ts_ns = 2;         // minute-level timestamp of the log file
+    string filer_id = 3;          // filer signature suffix from log filename
+}
+
+message TraverseBfsMetadataRequest {
+    string directory = 1;
+    repeated string excluded_prefixes = 2;
+}
+message TraverseBfsMetadataResponse {
+    string directory = 1;
+    Entry entry = 2;
+}
+
+message LogEntry {
+    int64 ts_ns = 1;
+    int32 partition_key_hash = 2;
+    bytes data = 3;
+    bytes key = 4;
+    int64 offset = 5;  // Sequential offset within partition
+}
+
+message KeepConnectedRequest {
+    string name = 1;
+    uint32 grpc_port = 2;
+    repeated string resources = 3;
+}
+message KeepConnectedResponse {
+}
+
+message LocateBrokerRequest {
+    string resource = 1;
+}
+
+message LocateBrokerResponse {
+    bool found = 1;
+    // if found, send the exact address
+    // if not found, send the full list of existing brokers
+    message Resource {
+        string grpc_addresses = 1;
+        int32 resource_count = 2;
+    }
+    repeated Resource resources = 2;
+}
+
+/////////////////////////
+// Key-Value operations
+/////////////////////////
+message KvGetRequest {
+    bytes key = 1;
+}
+message KvGetResponse {
+    bytes value = 1;
+    string error = 2;
+}
+message KvPutRequest {
+    bytes key = 1;
+    bytes value = 2;
+}
+message KvPutResponse {
+    string error = 1;
+}
+
+/////////////////////////
+// path-based configurations
+/////////////////////////
+message FilerConf {
+    int32 version = 1;
+    message PathConf {
+        string location_prefix = 1;
+        string collection = 2;
+        string replication = 3;
+        string ttl = 4;
+        string disk_type = 5;
+        bool fsync = 6;
+        uint32 volume_growth_count = 7;
+        bool read_only = 8;
+        string data_center = 9;
+        string rack = 10;
+        string data_node = 11;
+        uint32 max_file_name_length = 12;
+        bool disable_chunk_deletion = 13;
+        // unset inherits from the enclosing path rule, set overrides it
+        optional bool worm = 14;
+        uint64 worm_grace_period_seconds = 15;
+        uint64 worm_retention_time_seconds = 16;
+    }
+    repeated PathConf locations = 2;
+}
+
+/////////////////////////
+// Remote Storage related
+/////////////////////////
+message CacheRemoteObjectToLocalClusterRequest {
+    string directory = 1;
+    string name = 2;
+    int32 chunk_concurrency = 3; // parallel chunk downloads per file, 0 = default (8)
+    int32 download_concurrency = 4; // multipart download concurrency per chunk (if supported by remote storage), 0 = default (5 for S3)
+}
+message CacheRemoteObjectToLocalClusterResponse {
+    Entry entry = 1;
+    SubscribeMetadataResponse metadata_event = 2;
+    // filer log position stamped before the entry read: every event at or
+    // below it is reflected in the returned entry
+    int64 log_ts_ns = 3;
+    int32 log_signature = 4; // filer whose clock stamped log_ts_ns
+}
+
+/////////////////////////
+// distributed lock management
+/////////////////////////
+message LockRequest {
+    string name = 1;
+    int64 seconds_to_lock = 2;
+    string renew_token = 3;
+    bool is_moved = 4;
+    string owner = 5;
+}
+message LockResponse {
+    string renew_token = 1;
+    string lock_owner = 2;
+    string lock_host_moved_to = 3;
+    string error = 4;
+    int64 generation = 5;
+}
+message UnlockRequest {
+    string name = 1;
+    string renew_token = 2;
+    bool is_moved = 3;
+}
+message UnlockResponse {
+    string error = 1;
+    string moved_to = 2;
+}
+message FindLockOwnerRequest {
+    string name = 1;
+    bool is_moved = 2;
+}
+message FindLockOwnerResponse {
+    string owner = 1;
+}
+message Lock {
+    string name = 1;
+    string renew_token = 2;
+    int64 expired_at_ns = 3;
+    string owner = 4;
+    int64 generation = 5;
+    bool is_backup = 6;
+    int64 seq = 7;
+}
+message TransferLocksRequest {
+    repeated Lock locks = 1;
+}
+message TransferLocksResponse {
+}
+message ReplicateLockRequest {
+    string name = 1;
+    string renew_token = 2;
+    int64 expired_at_ns = 3;
+    string owner = 4;
+    int64 generation = 5;
+    bool is_unlock = 6;
+    int64 seq = 7;
+}
+message ReplicateLockResponse {
+}
+
+//////////////////////////////////////////////////
+// StreamMutateEntry: ordered bidirectional streaming for all filer mutations.
+// All create/update/delete/rename operations from a single mount go through
+// one stream, preserving mutation ordering and eliminating per-request
+// connection overhead.
+
+message StreamMutateEntryRequest {
+    uint64 request_id = 1;
+    oneof request {
+        CreateEntryRequest create_request = 2;
+        UpdateEntryRequest update_request = 3;
+        DeleteEntryRequest delete_request = 4;
+        StreamRenameEntryRequest rename_request = 5;
+    }
+}
+
+message StreamMutateEntryResponse {
+    uint64 request_id = 1;
+    bool is_last = 2; // always true except for rename, which sends multiple events
+    oneof response {
+        CreateEntryResponse create_response = 3;
+        UpdateEntryResponse update_response = 4;
+        DeleteEntryResponse delete_response = 5;
+        StreamRenameEntryResponse rename_response = 6;
+    }
+    string error = 7; // human-readable error message when the operation failed
+    int32 errno = 8;  // POSIX errno (e.g. ENOENT=2, ENOTEMPTY=66) for direct FUSE status mapping
+}
+
+//////////////////////////////////////////////////
+// Peer chunk sharing — mount-server registry
+//////////////////////////////////////////////////
+
+message MountRegisterRequest {
+    string peer_addr = 1;    // host:port where this mount serves peer chunk requests
+    string rack = 2;         // locality label (rack); used for peer ranking
+    int32 ttl_seconds = 3;   // how long the filer should keep this entry without a heartbeat
+    string data_center = 4;  // locality label (data center); coarser than rack
+}
+
+message MountRegisterResponse {
+}
+
+message MountListRequest {
+}
+
+message MountListResponse {
+    repeated MountInfo mounts = 1;
+}
+
+message MountInfo {
+    string peer_addr = 1;
+    string rack = 2;
+    int64 last_seen_ns = 3;
+    string data_center = 4;
+}
+
+// LookupDirectoryEntriesRequest batches independent exact-path lookups into
+// one bounded RPC. Results preserve this order. Empty batches and batches over
+// 4096 requests are rejected. Declared last to keep generated message indices
+// stable.
+message LookupDirectoryEntriesRequest {
+    repeated LookupDirectoryEntryRequest requests = 1;
+    // Cache callers may treat an Entry whose Volume the master no longer
+    // reports as a miss. A lookup the filer could not complete still marks the
+    // Entry with an error. Ordinary filesystem callers keep the fail-closed
+    // error in both cases.
+    bool unavailable_volume_is_miss = 2;
+}
+
+message LookupDirectoryEntryResult {
+    // found reports metadata presence. An error makes this item unusable even
+    // when entry is present, for example when a chunk volume cannot be located.
+    bool found = 1;
+    Entry entry = 2;
+    string error = 3;
+    // Per-entry read fence. Every filer event at or below this position is
+    // reflected in found/entry/error for this item.
+    int64 log_ts_ns = 4;
+    int32 log_signature = 5;
+}
+
+message LookupDirectoryEntriesResponse {
+    repeated LookupDirectoryEntryResult results = 1;
+    // One deduplicated location set for every volume referenced by returned
+    // Entry.chunks. A missing/unavailable volume has an empty Locations value
+    // and marks each affected result with an error.
+    map<string, Locations> locations_map = 2;
+    // Short-lived, exact-FID read capabilities. Direct data-plane clients must
+    // present the matching token to a Volume instead of treating a shared
+    // service credential as authority to read arbitrary needles.
+    map<string, string> read_auth = 3;
+}

weed/pb/filer_pb/filer_client.go

@@ -0,0 +1,357 @@
+package filer_pb
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var (
+	OS_UID = uint32(os.Getuid())
+	OS_GID = uint32(os.Getgid())
+)
+
+type FilerClient interface {
+	WithFilerClient(streamingMode bool, fn func(SeaweedFilerClient) error) error // 15 implementation
+	AdjustedUrl(location *Location) string
+	GetDataCenter() string
+}
+
+// GetEntry returns the entry together with the log position the filer stamped
+// for it and the signature of the filer that stamped it — positions are only
+// comparable within one filer's clock. Both are zero against a filer that does
+// not stamp them.
+func GetEntry(ctx context.Context, filerClient FilerClient, fullFilePath util.FullPath) (entry *Entry, logTsNs int64, logSignature int32, err error) {
+
+	dir, name := fullFilePath.DirAndName()
+
+	err = filerClient.WithFilerClient(false, func(client SeaweedFilerClient) error {
+
+		request := &LookupDirectoryEntryRequest{
+			Directory: dir,
+			Name:      name,
+		}
+
+		// glog.V(3).Infof("read %s request: %v", fullFilePath, request)
+		resp, err := LookupEntry(ctx, client, request)
+		if err != nil {
+			glog.V(3).InfofCtx(ctx, "read %s %v: %v", fullFilePath, resp, err)
+			return err
+		}
+
+		if resp.Entry == nil {
+			// glog.V(3).Infof("read %s entry: %v", fullFilePath, entry)
+			return nil
+		}
+
+		entry = resp.Entry
+		logTsNs = resp.LogTsNs
+		logSignature = resp.LogSignature
+		return nil
+	})
+
+	return
+}
+
+// ListSnapshotTsNsTrailerKey carries the listing snapshot in the ListEntries
+// stream trailer, so empty listings can convey it without a response older
+// consumers would mistake for an entry.
+const ListSnapshotTsNsTrailerKey = "sw-list-snapshot-ts-ns"
+
+type EachEntryFunction func(entry *Entry, isLast bool) error
+
+func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath util.FullPath, prefix string, fn EachEntryFunction) (err error) {
+	_, err = ReadDirAllEntriesWithSnapshot(ctx, filerClient, fullDirPath, prefix, fn)
+	return err
+}
+
+func ReadDirAllEntriesWithSnapshot(ctx context.Context, filerClient FilerClient, fullDirPath util.FullPath, prefix string, fn EachEntryFunction) (snapshotTsNs int64, err error) {
+	var counter uint32
+	var startFrom string
+	var counterFunc = func(entry *Entry, isLast bool) error {
+		counter++
+		startFrom = entry.Name
+		return fn(entry, isLast)
+	}
+
+	var paginationLimit uint32 = 10000
+
+	if snapshotTsNs, err = doListWithSnapshot(ctx, filerClient, fullDirPath, prefix, counterFunc, "", false, paginationLimit, 0); err != nil {
+		return snapshotTsNs, err
+	}
+
+	for counter == paginationLimit {
+		counter = 0
+		lastStartFrom := startFrom
+		if _, err = doListWithSnapshot(ctx, filerClient, fullDirPath, prefix, counterFunc, startFrom, false, paginationLimit, snapshotTsNs); err != nil {
+			return snapshotTsNs, err
+		}
+		// A full page that ends on the name it started from would loop forever;
+		// a store whose ordering does not advance past the cursor causes this.
+		if counter == paginationLimit && startFrom == lastStartFrom {
+			return snapshotTsNs, fmt.Errorf("list %s: pagination stuck at %q", fullDirPath, startFrom)
+		}
+	}
+
+	return snapshotTsNs, nil
+}
+
+func List(ctx context.Context, filerClient FilerClient, parentDirectoryPath, prefix string, fn EachEntryFunction, startFrom string, inclusive bool, limit uint32) (err error) {
+	return filerClient.WithFilerClient(false, func(client SeaweedFilerClient) error {
+		return doSeaweedList(ctx, client, util.FullPath(parentDirectoryPath), prefix, fn, startFrom, inclusive, limit)
+	})
+}
+
+func doListWithSnapshot(ctx context.Context, filerClient FilerClient, fullDirPath util.FullPath, prefix string, fn EachEntryFunction, startFrom string, inclusive bool, limit uint32, snapshotTsNs int64) (actualSnapshotTsNs int64, err error) {
+	err = filerClient.WithFilerClient(false, func(client SeaweedFilerClient) error {
+		actualSnapshotTsNs, err = DoSeaweedListWithSnapshot(ctx, client, fullDirPath, prefix, fn, startFrom, inclusive, limit, snapshotTsNs)
+		return err
+	})
+	return actualSnapshotTsNs, err
+}
+
+func SeaweedList(ctx context.Context, client SeaweedFilerClient, parentDirectoryPath, prefix string, fn EachEntryFunction, startFrom string, inclusive bool, limit uint32) (err error) {
+	return doSeaweedList(ctx, client, util.FullPath(parentDirectoryPath), prefix, fn, startFrom, inclusive, limit)
+}
+
+func doSeaweedList(ctx context.Context, client SeaweedFilerClient, fullDirPath util.FullPath, prefix string, fn EachEntryFunction, startFrom string, inclusive bool, limit uint32) (err error) {
+	_, err = DoSeaweedListWithSnapshot(ctx, client, fullDirPath, prefix, fn, startFrom, inclusive, limit, 0)
+	return err
+}
+
+func DoSeaweedListWithSnapshot(ctx context.Context, client SeaweedFilerClient, fullDirPath util.FullPath, prefix string, fn EachEntryFunction, startFrom string, inclusive bool, limit uint32, snapshotTsNs int64) (actualSnapshotTsNs int64, err error) {
+	// Redundancy limit to make it correctly judge whether it is the last file.
+	redLimit := limit
+
+	if limit < math.MaxInt32 && limit != 0 {
+		redLimit = limit + 1
+	}
+	if redLimit > math.MaxInt32 {
+		redLimit = math.MaxInt32
+	}
+	request := &ListEntriesRequest{
+		Directory:          string(fullDirPath),
+		Prefix:             prefix,
+		StartFromFileName:  startFrom,
+		Limit:              redLimit,
+		InclusiveStartFrom: inclusive,
+		SnapshotTsNs:       snapshotTsNs,
+		OmitChunks:         ChunksOmitted(ctx),
+	}
+
+	// Preserve the caller-requested snapshot so pagination uses the same
+	// boundary across pages. For first requests (snapshotTsNs==0) we do NOT
+	// synthesize a client-side timestamp — if the server returns no entries,
+	// we return 0 so callers like CompleteDirectoryBuild know no server
+	// snapshot was received and can replay all buffered events without
+	// clock-skew-sensitive filtering.
+	actualSnapshotTsNs = snapshotTsNs
+
+	glog.V(4).InfofCtx(ctx, "read directory: %v", request)
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	stream, err := client.ListEntries(ctx, request)
+	if err != nil {
+		// fullDirPath is the caller's bucket and prefix; keep the status the filer
+		// sent so a retry is decided on that rather than on this message
+		return actualSnapshotTsNs, fmt.Errorf("list %s: %w", fullDirPath, err)
+	}
+
+	var prevEntry *Entry
+	count := 0
+	for {
+		resp, recvErr := stream.Recv()
+		if recvErr != nil {
+			if recvErr == io.EOF {
+				if prevEntry != nil {
+					if err := fn(prevEntry, true); err != nil {
+						return actualSnapshotTsNs, err
+					}
+				}
+				// An empty listing carries no in-band snapshot (a snapshot-only
+				// response would be read as an entry by older consumers); newer
+				// filers send it in the stream trailer instead.
+				if actualSnapshotTsNs == 0 {
+					if values := stream.Trailer().Get(ListSnapshotTsNsTrailerKey); len(values) > 0 {
+						if trailerTsNs, parseErr := strconv.ParseInt(values[0], 10, 64); parseErr == nil {
+							actualSnapshotTsNs = trailerTsNs
+						}
+					}
+				}
+				break
+			} else {
+				return actualSnapshotTsNs, recvErr
+			}
+		}
+		if resp.SnapshotTsNs != 0 {
+			actualSnapshotTsNs = resp.SnapshotTsNs
+		}
+		if prevEntry != nil {
+			if err := fn(prevEntry, false); err != nil {
+				return actualSnapshotTsNs, err
+			}
+		}
+		prevEntry = resp.Entry
+		count++
+		if count > int(limit) && limit != 0 {
+			prevEntry = nil
+		}
+	}
+
+	return actualSnapshotTsNs, nil
+}
+
+func Exists(ctx context.Context, filerClient FilerClient, parentDirectoryPath string, entryName string, isDirectory bool) (exists bool, err error) {
+
+	err = filerClient.WithFilerClient(false, func(client SeaweedFilerClient) error {
+
+		request := &LookupDirectoryEntryRequest{
+			Directory: parentDirectoryPath,
+			Name:      entryName,
+		}
+
+		glog.V(4).InfofCtx(ctx, "exists entry %v/%v: %v", parentDirectoryPath, entryName, request)
+		resp, err := LookupEntry(ctx, client, request)
+		if err != nil {
+			if err == ErrNotFound {
+				exists = false
+				return nil
+			}
+			glog.V(0).InfofCtx(ctx, "exists entry %v: %v", request, err)
+			return fmt.Errorf("exists entry %s/%s: %v", parentDirectoryPath, entryName, err)
+		}
+
+		exists = resp.Entry.IsDirectory == isDirectory
+
+		return nil
+	})
+
+	return
+}
+
+func Mkdir(ctx context.Context, filerClient FilerClient, parentDirectoryPath string, dirName string, fn func(entry *Entry)) error {
+	return filerClient.WithFilerClient(false, func(client SeaweedFilerClient) error {
+		return DoMkdir(ctx, client, parentDirectoryPath, dirName, fn)
+	})
+}
+
+func DoMkdir(ctx context.Context, client SeaweedFilerClient, parentDirectoryPath string, dirName string, fn func(entry *Entry)) error {
+	entry := &Entry{
+		Name:        dirName,
+		IsDirectory: true,
+		Attributes: &FuseAttributes{
+			Mtime:    time.Now().Unix(),
+			Crtime:   time.Now().Unix(),
+			FileMode: uint32(0777 | os.ModeDir),
+			Uid:      OS_UID,
+			Gid:      OS_GID,
+		},
+	}
+
+	if fn != nil {
+		fn(entry)
+	}
+
+	request := &CreateEntryRequest{
+		Directory: parentDirectoryPath,
+		Entry:     entry,
+	}
+
+	glog.V(1).InfofCtx(ctx, "mkdir: %v", request)
+	if err := CreateEntry(ctx, client, request); err != nil {
+		glog.V(0).InfofCtx(ctx, "mkdir %v: %v", request, err)
+		return fmt.Errorf("mkdir %s/%s: %w", parentDirectoryPath, dirName, err)
+	}
+
+	return nil
+}
+
+func MkFile(ctx context.Context, filerClient FilerClient, parentDirectoryPath string, fileName string, chunks []*FileChunk, fn func(entry *Entry)) error {
+	return filerClient.WithFilerClient(false, func(client SeaweedFilerClient) error {
+
+		entry := &Entry{
+			Name:        fileName,
+			IsDirectory: false,
+			Attributes: &FuseAttributes{
+				Mtime:    time.Now().Unix(),
+				Crtime:   time.Now().Unix(),
+				FileMode: uint32(0770),
+				Uid:      OS_UID,
+				Gid:      OS_GID,
+			},
+			Chunks: chunks,
+		}
+
+		if fn != nil {
+			fn(entry)
+		}
+
+		request := &CreateEntryRequest{
+			Directory: parentDirectoryPath,
+			Entry:     entry,
+		}
+
+		glog.V(1).InfofCtx(ctx, "create file: %s/%s", parentDirectoryPath, fileName)
+		if err := CreateEntry(ctx, client, request); err != nil {
+			glog.V(0).InfofCtx(ctx, "create file %v:%v", request, err)
+			return fmt.Errorf("create file %s/%s: %w", parentDirectoryPath, fileName, err)
+		}
+
+		return nil
+	})
+}
+
+func Remove(ctx context.Context, filerClient FilerClient, parentDirectoryPath, name string, isDeleteData, isRecursive, ignoreRecursiveErr, isFromOtherCluster bool, signatures []int32) error {
+	_, err := RemoveWithResponse(ctx, filerClient, parentDirectoryPath, name, isDeleteData, isRecursive, ignoreRecursiveErr, isFromOtherCluster, signatures)
+	return err
+}
+
+func DoRemove(ctx context.Context, client SeaweedFilerClient, parentDirectoryPath string, name string, isDeleteData bool, isRecursive bool, ignoreRecursiveErr bool, isFromOtherCluster bool, signatures []int32) error {
+	_, err := DoRemoveWithResponse(ctx, client, parentDirectoryPath, name, isDeleteData, isRecursive, ignoreRecursiveErr, isFromOtherCluster, signatures)
+	return err
+}
+
+func RemoveWithResponse(ctx context.Context, filerClient FilerClient, parentDirectoryPath, name string, isDeleteData, isRecursive, ignoreRecursiveErr, isFromOtherCluster bool, signatures []int32) (*DeleteEntryResponse, error) {
+	var resp *DeleteEntryResponse
+	err := filerClient.WithFilerClient(false, func(client SeaweedFilerClient) error {
+		var innerErr error
+		resp, innerErr = DoRemoveWithResponse(ctx, client, parentDirectoryPath, name, isDeleteData, isRecursive, ignoreRecursiveErr, isFromOtherCluster, signatures)
+		return innerErr
+	})
+	return resp, err
+}
+
+func DoRemoveWithResponse(ctx context.Context, client SeaweedFilerClient, parentDirectoryPath string, name string, isDeleteData bool, isRecursive bool, ignoreRecursiveErr bool, isFromOtherCluster bool, signatures []int32) (*DeleteEntryResponse, error) {
+	deleteEntryRequest := &DeleteEntryRequest{
+		Directory:            parentDirectoryPath,
+		Name:                 name,
+		IsDeleteData:         isDeleteData,
+		IsRecursive:          isRecursive,
+		IgnoreRecursiveError: ignoreRecursiveErr,
+		IsFromOtherCluster:   isFromOtherCluster,
+		Signatures:           signatures,
+	}
+	if resp, err := client.DeleteEntry(ctx, deleteEntryRequest); err != nil {
+		if strings.Contains(err.Error(), ErrNotFound.Error()) {
+			return nil, nil
+		}
+		return nil, err
+	} else {
+		if resp.Error != "" {
+			if strings.Contains(resp.Error, ErrNotFound.Error()) {
+				return nil, nil
+			}
+			return nil, errors.New(resp.Error)
+		}
+		return resp, nil
+	}
+}

weed/pb/filer_pb/filer_client_bfs.go

@@ -0,0 +1,141 @@
+package filer_pb
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TraverseBfs(ctx context.Context, filerClient FilerClient, parentPath util.FullPath, fn func(parentPath util.FullPath, entry *Entry) error) (err error) {
+	K := 5
+
+	// callers hand in user-typed paths, and every entry is reported relative
+	// to this one, so a trailing slash must not reach the callback
+	parentPath = util.NormalizePath(string(parentPath))
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	queue := util.NewQueue[util.FullPath]()
+	var pending sync.WaitGroup
+	pending.Add(1)
+	queue.Enqueue(parentPath)
+
+	var once sync.Once
+	var firstErr error
+
+	// A directory delivered twice (a page-boundary race with concurrent
+	// renames, or a store whose listing order misbehaves) must not be walked
+	// twice: the second walk re-lists the same subtree and can keep the
+	// traversal from ever terminating.
+	var visited sync.Map
+	visited.Store(string(parentPath), struct{}{})
+
+	enqueue := func(p util.FullPath) bool {
+		// Stop expanding traversal once canceled (e.g. first error encountered).
+		if ctx.Err() != nil {
+			return false
+		}
+		if _, seen := visited.LoadOrStore(string(p), struct{}{}); seen {
+			return true
+		}
+		pending.Add(1)
+		queue.Enqueue(p)
+		return true
+	}
+
+	done := make(chan struct{})
+	var workers sync.WaitGroup
+	for i := 0; i < K; i++ {
+		workers.Add(1)
+		go func() {
+			defer workers.Done()
+			for {
+				select {
+				case <-done:
+					return
+				default:
+				}
+
+				dir := queue.Dequeue()
+				if dir == "" {
+					// queue is empty for now
+					select {
+					case <-done:
+						return
+					case <-time.After(50 * time.Millisecond):
+						continue
+					}
+				}
+
+				// Always mark the directory as done so the closer can finish.
+				if ctx.Err() == nil {
+					processErr := processOneDirectory(ctx, filerClient, dir, enqueue, fn)
+					if processErr != nil {
+						once.Do(func() {
+							firstErr = processErr
+							cancel()
+						})
+					}
+				}
+				pending.Done()
+			}
+		}()
+	}
+
+	pending.Wait()
+	close(done)
+
+	workers.Wait()
+
+	return firstErr
+}
+
+func processOneDirectory(ctx context.Context, filerClient FilerClient, parentPath util.FullPath, enqueue func(p util.FullPath) bool, fn func(parentPath util.FullPath, entry *Entry) error) (err error) {
+
+	return ReadDirAllEntries(ctx, filerClient, parentPath, "", func(entry *Entry, isLast bool) error {
+
+		if err := fn(parentPath, entry); err != nil {
+			return err
+		}
+
+		if entry.IsDirectory {
+			if !enqueue(parentPath.Child(entry.Name)) {
+				return ctx.Err()
+			}
+		}
+		return nil
+	})
+
+}
+
+func StreamBfs(client SeaweedFilerClient, dir util.FullPath, olderThanTsNs int64, fn func(parentPath util.FullPath, entry *Entry) error) (err error) {
+	glog.V(0).Infof("TraverseBfsMetadata %v if before %v", dir, time.Unix(0, olderThanTsNs))
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	stream, err := client.TraverseBfsMetadata(ctx, &TraverseBfsMetadataRequest{
+		Directory: string(dir),
+	})
+	if err != nil {
+		return fmt.Errorf("traverse bfs metadata: %w", err)
+	}
+	for {
+		resp, err := stream.Recv()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return fmt.Errorf("traverse bfs metadata: %w", err)
+		}
+		if err := fn(util.FullPath(resp.Directory), resp.Entry); err != nil {
+			return err
+		}
+	}
+	return nil
+}

weed/pb/filer_pb/filer_client_bfs_test.go

@@ -0,0 +1,191 @@
+package filer_pb
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"reflect"
+	"sort"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// stubFiler serves canned listings and counts how often each directory is listed.
+type stubFiler struct {
+	UnimplementedSeaweedFilerServer
+	mu       sync.Mutex
+	listed   map[string]int
+	listings func(req *ListEntriesRequest, send func(*Entry) error) error
+}
+
+func (s *stubFiler) ListEntries(req *ListEntriesRequest, stream SeaweedFiler_ListEntriesServer) error {
+	s.mu.Lock()
+	if s.listed == nil {
+		s.listed = make(map[string]int)
+	}
+	s.listed[req.Directory]++
+	s.mu.Unlock()
+	return s.listings(req, func(entry *Entry) error {
+		return stream.Send(&ListEntriesResponse{Entry: entry})
+	})
+}
+
+func (s *stubFiler) listedCount(dir string) int {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.listed[dir]
+}
+
+type stubFilerClient struct {
+	conn *grpc.ClientConn
+}
+
+func (c *stubFilerClient) WithFilerClient(streamingMode bool, fn func(SeaweedFilerClient) error) error {
+	return fn(NewSeaweedFilerClient(c.conn))
+}
+
+func (c *stubFilerClient) AdjustedUrl(location *Location) string { return location.Url }
+
+func (c *stubFilerClient) GetDataCenter() string { return "" }
+
+func startStubFiler(t *testing.T, s *stubFiler) *stubFilerClient {
+	t.Helper()
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	grpcServer := grpc.NewServer()
+	RegisterSeaweedFilerServer(grpcServer, s)
+	go grpcServer.Serve(listener)
+	t.Cleanup(grpcServer.Stop)
+	conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { conn.Close() })
+	return &stubFilerClient{conn: conn}
+}
+
+// A directory delivered twice by a misbehaving listing must still be walked once.
+func TestTraverseBfsVisitsDuplicateDirectoryOnce(t *testing.T) {
+	server := &stubFiler{}
+	server.listings = func(req *ListEntriesRequest, send func(*Entry) error) error {
+		switch req.Directory {
+		case "/":
+			if req.StartFromFileName != "" {
+				return nil
+			}
+			for _, name := range []string{"dup", "dup"} {
+				if err := send(&Entry{Name: name, IsDirectory: true}); err != nil {
+					return err
+				}
+			}
+		case "/dup":
+			if req.StartFromFileName != "" {
+				return nil
+			}
+			for _, name := range []string{"a.txt", "b.txt"} {
+				if err := send(&Entry{Name: name}); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	}
+	filerClient := startStubFiler(t, server)
+
+	var mu sync.Mutex
+	seen := make(map[string]int)
+	err := TraverseBfs(context.Background(), filerClient, "/", func(parentPath util.FullPath, entry *Entry) error {
+		mu.Lock()
+		seen[string(parentPath.Child(entry.Name))]++
+		mu.Unlock()
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got := server.listedCount("/dup"); got != 1 {
+		t.Fatalf("listed /dup %d times, want 1", got)
+	}
+	for _, path := range []string{"/dup/a.txt", "/dup/b.txt"} {
+		if seen[path] != 1 {
+			t.Fatalf("visited %s %d times, want 1", path, seen[path])
+		}
+	}
+}
+
+// A store whose pagination never advances past the cursor must error out
+// instead of re-listing the same page forever.
+func TestReadDirAllEntriesStuckPagination(t *testing.T) {
+	server := &stubFiler{}
+	server.listings = func(req *ListEntriesRequest, send func(*Entry) error) error {
+		for i := uint32(0); i < req.Limit; i++ {
+			if err := send(&Entry{Name: fmt.Sprintf("f%05d", i)}); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+	filerClient := startStubFiler(t, server)
+
+	done := make(chan error, 1)
+	go func() {
+		done <- ReadDirAllEntries(context.Background(), filerClient, "/", "", func(entry *Entry, isLast bool) error {
+			return nil
+		})
+	}()
+
+	select {
+	case err := <-done:
+		if err == nil || !strings.Contains(err.Error(), "pagination stuck") {
+			t.Fatalf("want pagination stuck error, got %v", err)
+		}
+	case <-time.After(30 * time.Second):
+		t.Fatal("listing loops forever on a non-advancing store")
+	}
+}
+
+// A start path with a trailing slash must still descend into subdirectories.
+func TestTraverseBfsTrailingSlashRoot(t *testing.T) {
+	server := &stubFiler{}
+	server.listings = func(req *ListEntriesRequest, send func(*Entry) error) error {
+		if req.StartFromFileName != "" {
+			return nil
+		}
+		// the filer trims a single trailing slash, nothing more
+		switch strings.TrimSuffix(req.Directory, "/") {
+		case "/buckets/data":
+			return send(&Entry{Name: "sub", IsDirectory: true})
+		case "/buckets/data/sub":
+			return send(&Entry{Name: "a.txt"})
+		}
+		return nil
+	}
+	filerClient := startStubFiler(t, server)
+
+	var mu sync.Mutex
+	var seen []string
+	err := TraverseBfs(context.Background(), filerClient, "/buckets/data/", func(parentPath util.FullPath, entry *Entry) error {
+		mu.Lock()
+		seen = append(seen, fmt.Sprintf("%s -> %s", parentPath, entry.Name))
+		mu.Unlock()
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sort.Strings(seen)
+	want := []string{"/buckets/data -> sub", "/buckets/data/sub -> a.txt"}
+	if !reflect.DeepEqual(seen, want) {
+		t.Fatalf("visited %v, want %v", seen, want)
+	}
+}

weed/pb/filer_pb/filer_client_snapshot_test.go

@@ -0,0 +1,165 @@
+package filer_pb
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/protobuf/proto"
+)
+
+type snapshotListStream struct {
+	responses []*ListEntriesResponse
+	index     int
+}
+
+func (s *snapshotListStream) Recv() (*ListEntriesResponse, error) {
+	if s.index >= len(s.responses) {
+		return nil, io.EOF
+	}
+	resp := s.responses[s.index]
+	s.index++
+	return resp, nil
+}
+
+func (s *snapshotListStream) Header() (metadata.MD, error) { return metadata.MD{}, nil }
+func (s *snapshotListStream) Trailer() metadata.MD         { return metadata.MD{} }
+func (s *snapshotListStream) CloseSend() error             { return nil }
+func (s *snapshotListStream) Context() context.Context     { return context.Background() }
+func (s *snapshotListStream) SendMsg(any) error            { return nil }
+func (s *snapshotListStream) RecvMsg(any) error            { return nil }
+
+type snapshotListClient struct {
+	SeaweedFilerClient
+	entries    []*Entry
+	requests   []*ListEntriesRequest
+	snapshotTs int64
+	listCalled bool
+}
+
+func (c *snapshotListClient) ListEntries(ctx context.Context, in *ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ListEntriesResponse], error) {
+	c.listCalled = true
+	c.requests = append(c.requests, proto.Clone(in).(*ListEntriesRequest))
+
+	start := 0
+	if in.StartFromFileName != "" {
+		start = len(c.entries)
+		for i, entry := range c.entries {
+			if entry.Name == in.StartFromFileName {
+				start = i
+				if !in.InclusiveStartFrom {
+					start++
+				}
+				break
+			}
+		}
+	}
+
+	end := len(c.entries)
+	if in.Limit > 0 && start+int(in.Limit) < end {
+		end = start + int(in.Limit)
+	}
+
+	snapshotTs := in.SnapshotTsNs
+	if snapshotTs == 0 {
+		snapshotTs = c.snapshotTs
+	}
+
+	responses := make([]*ListEntriesResponse, 0, end-start)
+	for i, entry := range c.entries[start:end] {
+		resp := &ListEntriesResponse{
+			Entry: entry,
+		}
+		if i == 0 {
+			resp.SnapshotTsNs = snapshotTs
+		}
+		responses = append(responses, resp)
+	}
+
+	return &snapshotListStream{responses: responses}, nil
+}
+
+type snapshotFilerAccessor struct {
+	client SeaweedFilerClient
+}
+
+func (a *snapshotFilerAccessor) WithFilerClient(_ bool, fn func(SeaweedFilerClient) error) error {
+	return fn(a.client)
+}
+
+func (a *snapshotFilerAccessor) AdjustedUrl(*Location) string { return "" }
+func (a *snapshotFilerAccessor) GetDataCenter() string        { return "" }
+
+func TestReadDirAllEntriesWithSnapshotCarriesSnapshotAcrossPages(t *testing.T) {
+	entries := make([]*Entry, 0, 10001)
+	for i := 0; i < 10001; i++ {
+		entries = append(entries, &Entry{Name: fmt.Sprintf("entry-%05d", i), Attributes: &FuseAttributes{}})
+	}
+
+	client := &snapshotListClient{
+		entries:    entries,
+		snapshotTs: 123456789,
+	}
+	accessor := &snapshotFilerAccessor{client: client}
+
+	var listed []string
+	snapshotTs, err := ReadDirAllEntriesWithSnapshot(context.Background(), accessor, util.FullPath("/dir"), "", func(entry *Entry, isLast bool) error {
+		listed = append(listed, entry.Name)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("ReadDirAllEntriesWithSnapshot: %v", err)
+	}
+
+	if got := len(listed); got != len(entries) {
+		t.Fatalf("listed %d entries, want %d", got, len(entries))
+	}
+	if snapshotTs != client.snapshotTs {
+		t.Fatalf("snapshotTs = %d, want %d", snapshotTs, client.snapshotTs)
+	}
+	if got := len(client.requests); got != 2 {
+		t.Fatalf("request count = %d, want 2", got)
+	}
+	if client.requests[0].SnapshotTsNs != 0 {
+		t.Fatalf("first request snapshot = %d, want 0", client.requests[0].SnapshotTsNs)
+	}
+	if client.requests[1].SnapshotTsNs != client.snapshotTs {
+		t.Fatalf("second request snapshot = %d, want %d", client.requests[1].SnapshotTsNs, client.snapshotTs)
+	}
+	if client.requests[1].StartFromFileName != entries[9999].Name {
+		t.Fatalf("second request marker = %q, want %q", client.requests[1].StartFromFileName, entries[9999].Name)
+	}
+}
+
+func TestReadDirAllEntriesWithSnapshotEmptyDirectory(t *testing.T) {
+	client := &snapshotListClient{
+		entries:    nil, // empty directory
+		snapshotTs: 999888777,
+	}
+	accessor := &snapshotFilerAccessor{client: client}
+
+	var listed []string
+	snapshotTs, err := ReadDirAllEntriesWithSnapshot(context.Background(), accessor, util.FullPath("/empty"), "", func(entry *Entry, isLast bool) error {
+		listed = append(listed, entry.Name)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("ReadDirAllEntriesWithSnapshot: %v", err)
+	}
+	if len(listed) != 0 {
+		t.Fatalf("listed %d entries, want 0", len(listed))
+	}
+	// When the server sends no entries (empty directory), no snapshot is
+	// received. The client returns 0 so callers like CompleteDirectoryBuild
+	// know to replay all buffered events without clock-skew filtering.
+	if snapshotTs != 0 {
+		t.Fatalf("snapshotTs = %d, want 0 for empty directory", snapshotTs)
+	}
+	if !client.listCalled {
+		t.Fatal("ListEntries was not invoked for the empty directory")
+	}
+}

weed/pb/filer_pb/filer_pb_event_scan.go

@@ -0,0 +1,142 @@
+package filer_pb
+
+import (
+	"google.golang.org/protobuf/encoding/protowire"
+)
+
+// ScanMetadataEventSkeleton extracts just the fields the subscription matcher
+// reads, without materializing the entries' chunk lists. ok=false means the
+// caller must fall back to a full unmarshal.
+func ScanMetadataEventSkeleton(data []byte) (skeleton *SubscribeMetadataResponse, ok bool) {
+	skeleton = &SubscribeMetadataResponse{}
+	rest := data
+	for len(rest) > 0 {
+		num, typ, n := protowire.ConsumeTag(rest)
+		if n < 0 {
+			return nil, false
+		}
+		rest = rest[n:]
+		switch {
+		case num == 1 && typ == protowire.BytesType: // directory
+			v, m := protowire.ConsumeBytes(rest)
+			if m < 0 {
+				return nil, false
+			}
+			skeleton.Directory = string(v)
+			rest = rest[m:]
+		case num == 2 && typ == protowire.BytesType: // event_notification
+			if skeleton.EventNotification != nil {
+				// repeated occurrences of a message field merge; punt to a full decode
+				return nil, false
+			}
+			v, m := protowire.ConsumeBytes(rest)
+			if m < 0 {
+				return nil, false
+			}
+			notification, scanOk := scanEventNotificationSkeleton(v)
+			if !scanOk {
+				return nil, false
+			}
+			skeleton.EventNotification = notification
+			rest = rest[m:]
+		case num == 3 && typ == protowire.VarintType: // ts_ns
+			v, m := protowire.ConsumeVarint(rest)
+			if m < 0 {
+				return nil, false
+			}
+			skeleton.TsNs = int64(v)
+			rest = rest[m:]
+		default:
+			m := protowire.ConsumeFieldValue(num, typ, rest)
+			if m < 0 {
+				return nil, false
+			}
+			rest = rest[m:]
+		}
+	}
+	return skeleton, true
+}
+
+func scanEventNotificationSkeleton(data []byte) (*EventNotification, bool) {
+	notification := &EventNotification{}
+	rest := data
+	for len(rest) > 0 {
+		num, typ, n := protowire.ConsumeTag(rest)
+		if n < 0 {
+			return nil, false
+		}
+		rest = rest[n:]
+		switch {
+		case num == 1 && typ == protowire.BytesType: // old_entry
+			if notification.OldEntry != nil {
+				return nil, false
+			}
+			v, m := protowire.ConsumeBytes(rest)
+			if m < 0 {
+				return nil, false
+			}
+			name, scanOk := scanEntryName(v)
+			if !scanOk {
+				return nil, false
+			}
+			notification.OldEntry = &Entry{Name: name}
+			rest = rest[m:]
+		case num == 2 && typ == protowire.BytesType: // new_entry
+			if notification.NewEntry != nil {
+				return nil, false
+			}
+			v, m := protowire.ConsumeBytes(rest)
+			if m < 0 {
+				return nil, false
+			}
+			name, scanOk := scanEntryName(v)
+			if !scanOk {
+				return nil, false
+			}
+			notification.NewEntry = &Entry{Name: name}
+			rest = rest[m:]
+		case num == 4 && typ == protowire.BytesType: // new_parent_path
+			v, m := protowire.ConsumeBytes(rest)
+			if m < 0 {
+				return nil, false
+			}
+			notification.NewParentPath = string(v)
+			rest = rest[m:]
+		default:
+			m := protowire.ConsumeFieldValue(num, typ, rest)
+			if m < 0 {
+				return nil, false
+			}
+			rest = rest[m:]
+		}
+	}
+	return notification, true
+}
+
+// scanEntryName walks the whole entry so field order does not matter.
+func scanEntryName(data []byte) (string, bool) {
+	name := ""
+	rest := data
+	for len(rest) > 0 {
+		num, typ, n := protowire.ConsumeTag(rest)
+		if n < 0 {
+			return "", false
+		}
+		rest = rest[n:]
+		if num == 1 && typ == protowire.BytesType { // name; last occurrence wins like proto merge
+			v, m := protowire.ConsumeBytes(rest)
+			if m < 0 {
+				return "", false
+			}
+			name = string(v)
+			rest = rest[m:]
+			continue
+		}
+		m := protowire.ConsumeFieldValue(num, typ, rest)
+		if m < 0 {
+			return "", false
+		}
+		rest = rest[m:]
+	}
+	return name, true
+}

weed/pb/filer_pb/filer_pb_event_scan_bench_test.go

@@ -0,0 +1,60 @@
+package filer_pb
+
+import (
+	"testing"
+
+	"google.golang.org/protobuf/proto"
+)
+
+func benchmarkEventBytes(b *testing.B, chunkCount int) []byte {
+	b.Helper()
+	chunks := make([]*FileChunk, chunkCount)
+	for i := range chunks {
+		chunks[i] = &FileChunk{FileId: "3,1234567890ab", Offset: int64(i) * 4096, Size: 4096, ModifiedTsNs: int64(i)}
+	}
+	data, err := proto.Marshal(&SubscribeMetadataResponse{
+		Directory: "/data/pvc-42",
+		TsNs:      123456789,
+		EventNotification: &EventNotification{
+			OldEntry: &Entry{Name: "0.log", Chunks: chunks},
+			NewEntry: &Entry{Name: "0.log", Chunks: chunks},
+		},
+	})
+	if err != nil {
+		b.Fatal(err)
+	}
+	return data
+}
+
+// the non-matching subscriber case, which is what a per-path fleet hits for
+// almost every entry
+
+func BenchmarkSubscriptionMatchFullDecode(b *testing.B) {
+	data := benchmarkEventBytes(b, 1000)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		event := &SubscribeMetadataResponse{}
+		if err := proto.Unmarshal(data, event); err != nil {
+			b.Fatal(err)
+		}
+		if MetadataEventMatchesSubscription(event, "/data/pvc-7/", nil, nil) {
+			b.Fatal("unexpected match")
+		}
+	}
+}
+
+func BenchmarkSubscriptionMatchSkeleton(b *testing.B) {
+	data := benchmarkEventBytes(b, 1000)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		skeleton, ok := ScanMetadataEventSkeleton(data)
+		if !ok {
+			b.Fatal("scan failed")
+		}
+		if MetadataEventMatchesSubscription(skeleton, "/data/pvc-7/", nil, nil) {
+			b.Fatal("unexpected match")
+		}
+	}
+}

weed/pb/filer_pb/filer_pb_event_scan_test.go

@@ -0,0 +1,112 @@
+package filer_pb
+
+import (
+	"testing"
+
+	"google.golang.org/protobuf/encoding/protowire"
+	"google.golang.org/protobuf/proto"
+)
+
+func marshalEvent(t *testing.T, event *SubscribeMetadataResponse) []byte {
+	t.Helper()
+	data, err := proto.Marshal(event)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return data
+}
+
+func bigEntry(name string) *Entry {
+	chunks := make([]*FileChunk, 50)
+	for i := range chunks {
+		chunks[i] = &FileChunk{FileId: "3,1234567890ab", Offset: int64(i) * 100, Size: 100}
+	}
+	return &Entry{Name: name, Chunks: chunks, Attributes: &FuseAttributes{FileSize: 5000, Mtime: 1234567890}}
+}
+
+func TestScanMetadataEventSkeletonMatchesFullDecode(t *testing.T) {
+	events := map[string]*SubscribeMetadataResponse{
+		"create": {Directory: "/data/pvc-7", TsNs: 100,
+			EventNotification: &EventNotification{NewEntry: bigEntry("a.log")}},
+		"update": {Directory: "/data/pvc-7", TsNs: 200,
+			EventNotification: &EventNotification{OldEntry: bigEntry("a.log"), NewEntry: bigEntry("a.log")}},
+		"delete": {Directory: "/data/pvc-7", TsNs: 300,
+			EventNotification: &EventNotification{OldEntry: bigEntry("a.log"), DeleteChunks: true}},
+		"rename within dir": {Directory: "/data/pvc-7", TsNs: 400,
+			EventNotification: &EventNotification{OldEntry: bigEntry("a.log"), NewEntry: bigEntry("b.log")}},
+		"rename across dirs": {Directory: "/data/pvc-7", TsNs: 500,
+			EventNotification: &EventNotification{OldEntry: bigEntry("a.log"), NewEntry: bigEntry("a.log"), NewParentPath: "/data/pvc-9"}},
+		"empty notification": {Directory: "/data/pvc-7", TsNs: 600,
+			EventNotification: &EventNotification{}},
+		"no notification": {Directory: "/data/pvc-7", TsNs: 700},
+	}
+	filters := []struct {
+		prefix   string
+		prefixes []string
+		dirs     []string
+	}{
+		{prefix: "/data/pvc-7/"},
+		{prefix: "/data/pvc-9/"},
+		{prefix: "/data/pvc-1/"},
+		{prefix: "/"},
+		{prefixes: []string{"/other/", "/data/pvc-9/"}},
+		{dirs: []string{"/data/pvc-7"}},
+		{dirs: []string{"/data/pvc-9"}},
+	}
+
+	for name, event := range events {
+		skeleton, ok := ScanMetadataEventSkeleton(marshalEvent(t, event))
+		if !ok {
+			t.Fatalf("%s: scan failed", name)
+		}
+		if skeleton.TsNs != event.TsNs {
+			t.Errorf("%s: ts %d != %d", name, skeleton.TsNs, event.TsNs)
+		}
+		for _, f := range filters {
+			want := MetadataEventMatchesSubscription(event, f.prefix, f.prefixes, f.dirs)
+			got := MetadataEventMatchesSubscription(skeleton, f.prefix, f.prefixes, f.dirs)
+			if got != want {
+				t.Errorf("%s with filter %+v: skeleton match %v, full match %v", name, f, got, want)
+			}
+		}
+	}
+}
+
+func TestScanMetadataEventSkeletonFallsBack(t *testing.T) {
+	if _, ok := ScanMetadataEventSkeleton([]byte{0xff, 0xff, 0xff}); ok {
+		t.Error("malformed payload must not scan")
+	}
+
+	// two event_notification occurrences merge under proto semantics; the
+	// scanner must punt rather than guess
+	one := marshalEvent(t, &SubscribeMetadataResponse{
+		EventNotification: &EventNotification{OldEntry: &Entry{Name: "a"}}})
+	two := append(append([]byte{}, one...), one...)
+	if _, ok := ScanMetadataEventSkeleton(two); ok {
+		t.Error("duplicated message field must fall back to full decode")
+	}
+
+	// same for a duplicated entry inside the notification
+	entry, err := proto.Marshal(&Entry{Name: "a"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var notification []byte
+	for i := 0; i < 2; i++ {
+		notification = protowire.AppendTag(notification, 1, protowire.BytesType) // old_entry
+		notification = protowire.AppendBytes(notification, entry)
+	}
+	var response []byte
+	response = protowire.AppendTag(response, 2, protowire.BytesType) // event_notification
+	response = protowire.AppendBytes(response, notification)
+	if _, ok := ScanMetadataEventSkeleton(response); ok {
+		t.Error("duplicated nested entry field must fall back to full decode")
+	}
+}
+
+func TestScanMetadataEventSkeletonEmptyPayload(t *testing.T) {
+	skeleton, ok := ScanMetadataEventSkeleton(nil)
+	if !ok || skeleton.Directory != "" || skeleton.EventNotification != nil {
+		t.Fatalf("empty payload: %+v ok=%v", skeleton, ok)
+	}
+}

weed/pb/filer_pb/filer_pb_helper.go

@@ -0,0 +1,376 @@
+package filer_pb
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/viant/ptrie"
+	"google.golang.org/protobuf/proto"
+)
+
+const cutoffTimeNewEmptyDir = 3
+
+func (entry *Entry) IsInRemoteOnly() bool {
+	return len(entry.GetChunks()) == 0 && entry.RemoteEntry != nil && entry.RemoteEntry.RemoteSize > 0
+}
+
+func (entry *Entry) IsDirectoryKeyObject() bool {
+	// Also true for a file promoted to a directory by a child write, which keeps its
+	// chunks/content, or its remote entry when the file was tiered to remote storage.
+	return entry.IsDirectory &&
+		((entry.Attributes != nil && entry.Attributes.Mime != "") || len(entry.GetChunks()) > 0 || len(entry.GetContent()) > 0 || entry.IsInRemoteOnly() || entry.IsPrefixObject())
+}
+
+// IsPrefixObject reports whether the directory entry also holds the object named by its
+// path without a trailing slash, which an empty object leaves no other trace of.
+func (entry *Entry) IsPrefixObject() bool {
+	if entry == nil || !entry.IsDirectory {
+		return false
+	}
+	_, marked := entry.Extended[s3_constants.SeaweedFSPrefixObject]
+	return marked
+}
+
+// MarkPrefixObject turns a file entry into the directory entry that stores it, so keys
+// nested under its key can be created beside it.
+func (entry *Entry) MarkPrefixObject() {
+	entry.IsDirectory = true
+	if entry.Attributes != nil {
+		entry.Attributes.FileMode |= uint32(os.ModeDir) | 0111
+	}
+	if entry.Extended == nil {
+		entry.Extended = make(map[string][]byte)
+	}
+	entry.Extended[s3_constants.SeaweedFSPrefixObject] = []byte("true")
+}
+
+func (entry *Entry) GetExpiryTime() (expiryTime int64) {
+	// For S3 objects with lifecycle expiration, use Mtime (modification time)
+	// For regular TTL entries, use Crtime (creation time) for backward compatibility
+	if entry.Extended != nil {
+		if _, hasS3Expiry := entry.Extended[s3_constants.SeaweedFSExpiresS3]; hasS3Expiry {
+			// S3 lifecycle expiration: base TTL on modification time
+			expiryTime = entry.Attributes.Mtime
+			if expiryTime == 0 {
+				expiryTime = entry.Attributes.Crtime
+			}
+			expiryTime += int64(entry.Attributes.TtlSec)
+			return expiryTime
+		}
+	}
+
+	// Regular TTL expiration: base on creation time only
+	expiryTime = entry.Attributes.Crtime + int64(entry.Attributes.TtlSec)
+	return expiryTime
+}
+
+func (entry *Entry) IsExpired() bool {
+	return entry != nil && entry.Attributes != nil && entry.Attributes.TtlSec > 0 &&
+		time.Now().Unix() >= entry.GetExpiryTime()
+}
+
+func (entry *Entry) FileMode() (fileMode os.FileMode) {
+	if entry != nil && entry.Attributes != nil {
+		fileMode = os.FileMode(entry.Attributes.FileMode)
+	}
+	return
+}
+
+func (entry *Entry) IsOlderDir() bool {
+	return entry.IsDirectory && entry.Attributes != nil && entry.Attributes.Mime == "" && entry.Attributes.GetCrtime() <= time.Now().Unix()-cutoffTimeNewEmptyDir
+}
+
+func ToFileIdObject(fileIdStr string) (*FileId, error) {
+	t, err := needle.ParseFileIdFromString(fileIdStr)
+	if err != nil {
+		return nil, err
+	}
+	return &FileId{
+		VolumeId: uint32(t.VolumeId),
+		Cookie:   uint32(t.Cookie),
+		FileKey:  uint64(t.Key),
+	}, nil
+
+}
+
+func (fid *FileId) toFileIdString() string {
+	return needle.NewFileId(needle.VolumeId(fid.VolumeId), fid.FileKey, fid.Cookie).String()
+}
+
+func (c *FileChunk) GetFileIdString() string {
+	if c.FileId != "" {
+		return c.FileId
+	}
+	if c.Fid != nil {
+		c.FileId = c.Fid.toFileIdString()
+		return c.FileId
+	}
+	return ""
+}
+
+func BeforeEntrySerialization(chunks []*FileChunk) {
+
+	for _, chunk := range chunks {
+
+		if chunk.FileId != "" {
+			if fid, err := ToFileIdObject(chunk.FileId); err == nil {
+				chunk.Fid = fid
+				chunk.FileId = ""
+			}
+		}
+
+		if chunk.SourceFileId != "" {
+			if fid, err := ToFileIdObject(chunk.SourceFileId); err == nil {
+				chunk.SourceFid = fid
+				chunk.SourceFileId = ""
+			}
+		}
+
+	}
+}
+
+func AfterEntryDeserialization(chunks []*FileChunk) {
+
+	for _, chunk := range chunks {
+
+		if chunk.Fid != nil && chunk.FileId == "" {
+			chunk.FileId = chunk.Fid.toFileIdString()
+		}
+
+		if chunk.SourceFid != nil && chunk.SourceFileId == "" {
+			chunk.SourceFileId = chunk.SourceFid.toFileIdString()
+		}
+
+	}
+}
+
+func CreateEntry(ctx context.Context, client SeaweedFilerClient, request *CreateEntryRequest) error {
+	_, err := CreateEntryWithResponse(ctx, client, request)
+	return err
+}
+
+func CreateEntryWithResponse(ctx context.Context, client SeaweedFilerClient, request *CreateEntryRequest) (*CreateEntryResponse, error) {
+	resp, err := client.CreateEntry(ctx, request)
+	if err != nil {
+		glog.V(1).InfofCtx(ctx, "create entry %s/%s %v: %v", request.Directory, request.Entry.Name, request.OExcl, err)
+		return nil, fmt.Errorf("CreateEntry: %w", err)
+	}
+	if resp.ErrorCode != FilerError_OK {
+		glog.V(1).InfofCtx(ctx, "create entry %s/%s %v: %v (code %v)", request.Directory, request.Entry.Name, request.OExcl, resp.Error, resp.ErrorCode)
+		if sentinel := FilerErrorToSentinel(resp.ErrorCode); sentinel != nil {
+			return nil, fmt.Errorf("CreateEntry %s/%s: %w", request.Directory, request.Entry.Name, sentinel)
+		}
+		return nil, fmt.Errorf("CreateEntry: %w", errors.New(resp.Error))
+	}
+	if resp.Error != "" {
+		glog.V(1).InfofCtx(ctx, "create entry %s/%s %v: %v", request.Directory, request.Entry.Name, request.OExcl, resp.Error)
+		return nil, fmt.Errorf("CreateEntry: %w", errors.New(resp.Error))
+	}
+	return resp, nil
+}
+
+func UpdateEntry(ctx context.Context, client SeaweedFilerClient, request *UpdateEntryRequest) error {
+	_, err := UpdateEntryWithResponse(ctx, client, request)
+	return err
+}
+
+func UpdateEntryWithResponse(ctx context.Context, client SeaweedFilerClient, request *UpdateEntryRequest) (*UpdateEntryResponse, error) {
+	resp, err := client.UpdateEntry(ctx, request)
+	if err != nil {
+		glog.V(1).InfofCtx(ctx, "update entry %s/%s :%v", request.Directory, request.Entry.Name, err)
+		return nil, fmt.Errorf("UpdateEntry: %w", err)
+	}
+	return resp, nil
+}
+
+func LookupEntry(ctx context.Context, client SeaweedFilerClient, request *LookupDirectoryEntryRequest) (*LookupDirectoryEntryResponse, error) {
+	resp, err := client.LookupDirectoryEntry(ctx, request)
+	if err != nil {
+		if err == ErrNotFound || strings.Contains(err.Error(), ErrNotFound.Error()) {
+			return nil, ErrNotFound
+		}
+		glog.V(3).InfofCtx(ctx, "read %s/%v: %v", request.Directory, request.Name, err)
+		return nil, fmt.Errorf("LookupEntry1: %w", err)
+	}
+	if resp.Entry == nil {
+		return nil, ErrNotFound
+	}
+	return resp, nil
+}
+
+var ErrNotFound = errors.New("filer: no entry is found in filer store")
+
+// Sentinel errors for filer entry operations.
+// These are set by the filer and reconstructed from FilerError codes after
+// crossing the gRPC boundary, so consumers can use errors.Is() instead of
+// parsing error strings.
+var (
+	ErrEntryNameTooLong    = errors.New("entry name too long")
+	ErrParentIsFile        = errors.New("parent path is a file")
+	ErrExistingIsDirectory = errors.New("existing entry is a directory")
+	ErrExistingIsFile      = errors.New("existing entry is a file")
+	ErrEntryAlreadyExists  = errors.New("entry already exists")
+)
+
+// FilerErrorToSentinel maps a proto FilerError code to its sentinel error.
+// Returns nil for OK or unknown codes.
+func FilerErrorToSentinel(code FilerError) error {
+	switch code {
+	case FilerError_ENTRY_NAME_TOO_LONG:
+		return ErrEntryNameTooLong
+	case FilerError_PARENT_IS_FILE:
+		return ErrParentIsFile
+	case FilerError_EXISTING_IS_DIRECTORY:
+		return ErrExistingIsDirectory
+	case FilerError_EXISTING_IS_FILE:
+		return ErrExistingIsFile
+	case FilerError_ENTRY_ALREADY_EXISTS:
+		return ErrEntryAlreadyExists
+	default:
+		return nil
+	}
+}
+
+func IsEmpty(event *SubscribeMetadataResponse) bool {
+	return event.EventNotification.NewEntry == nil && event.EventNotification.OldEntry == nil
+}
+
+func IsCreate(event *SubscribeMetadataResponse) bool {
+	return event.EventNotification.NewEntry != nil && event.EventNotification.OldEntry == nil
+}
+
+func IsUpdate(event *SubscribeMetadataResponse) bool {
+	return event.EventNotification.NewEntry != nil &&
+		event.EventNotification.OldEntry != nil &&
+		event.Directory == event.EventNotification.NewParentPath &&
+		event.EventNotification.NewEntry.Name == event.EventNotification.OldEntry.Name
+}
+
+func IsDelete(event *SubscribeMetadataResponse) bool {
+	return event.EventNotification.NewEntry == nil && event.EventNotification.OldEntry != nil
+}
+
+func IsRename(event *SubscribeMetadataResponse) bool {
+	return event.EventNotification.NewEntry != nil &&
+		event.EventNotification.OldEntry != nil &&
+		(event.Directory != event.EventNotification.NewParentPath ||
+			event.EventNotification.NewEntry.Name != event.EventNotification.OldEntry.Name)
+}
+
+func MetadataEventSourceDirectory(event *SubscribeMetadataResponse) string {
+	if event == nil {
+		return ""
+	}
+	return event.Directory
+}
+
+func MetadataEventTargetDirectory(event *SubscribeMetadataResponse) string {
+	if event == nil {
+		return ""
+	}
+	if event.EventNotification != nil && event.EventNotification.NewParentPath != "" {
+		return event.EventNotification.NewParentPath
+	}
+	return event.Directory
+}
+
+func metadataEventSourceEntryName(event *SubscribeMetadataResponse) string {
+	if event == nil || event.EventNotification == nil {
+		return ""
+	}
+	if event.EventNotification.OldEntry != nil {
+		return event.EventNotification.OldEntry.Name
+	}
+	if event.EventNotification.NewEntry != nil {
+		return event.EventNotification.NewEntry.Name
+	}
+	return ""
+}
+
+func metadataEventTargetEntryName(event *SubscribeMetadataResponse) string {
+	if event == nil || event.EventNotification == nil {
+		return ""
+	}
+	if event.EventNotification.NewEntry != nil {
+		return event.EventNotification.NewEntry.Name
+	}
+	if event.EventNotification.OldEntry != nil {
+		return event.EventNotification.OldEntry.Name
+	}
+	return ""
+}
+
+func MetadataEventSourceFullPath(event *SubscribeMetadataResponse) string {
+	return util.Join(MetadataEventSourceDirectory(event), metadataEventSourceEntryName(event))
+}
+
+func MetadataEventTargetFullPath(event *SubscribeMetadataResponse) string {
+	return util.Join(MetadataEventTargetDirectory(event), metadataEventTargetEntryName(event))
+}
+
+func MetadataEventTouchesDirectory(event *SubscribeMetadataResponse, dir string) bool {
+	if MetadataEventSourceDirectory(event) == dir {
+		return true
+	}
+	return event != nil &&
+		event.EventNotification != nil &&
+		event.EventNotification.NewEntry != nil &&
+		MetadataEventTargetDirectory(event) == dir
+}
+
+func MetadataEventMatchesSubscription(event *SubscribeMetadataResponse, pathPrefix string, pathPrefixes []string, directories []string) bool {
+	if event == nil {
+		return false
+	}
+
+	if metadataEventMatchesPath(MetadataEventSourceFullPath(event), MetadataEventSourceDirectory(event), pathPrefix, pathPrefixes, directories) {
+		return true
+	}
+
+	return event.EventNotification != nil &&
+		event.EventNotification.NewEntry != nil &&
+		metadataEventMatchesPath(MetadataEventTargetFullPath(event), MetadataEventTargetDirectory(event), pathPrefix, pathPrefixes, directories)
+}
+
+func metadataEventMatchesPath(fullPath, dirPath, pathPrefix string, pathPrefixes []string, directories []string) bool {
+	if hasPrefixIn(fullPath, pathPrefixes) {
+		return true
+	}
+	if matchByDirectory(dirPath, directories) {
+		return true
+	}
+	return strings.HasPrefix(fullPath, pathPrefix)
+}
+
+func hasPrefixIn(text string, prefixes []string) bool {
+	for _, p := range prefixes {
+		if strings.HasPrefix(text, p) {
+			return true
+		}
+	}
+	return false
+}
+
+func matchByDirectory(dirPath string, directories []string) bool {
+	for _, dir := range directories {
+		if dirPath == dir {
+			return true
+		}
+	}
+	return false
+}
+
+var _ = ptrie.KeyProvider(&FilerConf_PathConf{})
+
+func (fp *FilerConf_PathConf) Key() interface{} {
+	key, _ := proto.Marshal(fp)
+	return string(key)
+}

weed/pb/filer_pb/filer_pb_helper_test.go

@@ -0,0 +1,61 @@
+package filer_pb
+
+import (
+	"os"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+)
+
+func TestIsDirectoryKeyObject(t *testing.T) {
+	chunk := []*FileChunk{{FileId: "1,01", Size: 75}}
+
+	cases := []struct {
+		name string
+		e    *Entry
+		want bool
+	}{
+		{"plain directory", &Entry{IsDirectory: true, Attributes: &FuseAttributes{}}, false},
+		{"directory marker with mime", &Entry{IsDirectory: true, Attributes: &FuseAttributes{Mime: "application/octet-stream"}}, true},
+		{"directory promoted from file keeps chunks", &Entry{IsDirectory: true, Attributes: &FuseAttributes{}, Chunks: chunk}, true},
+		{"directory promoted from small file keeps content", &Entry{IsDirectory: true, Attributes: &FuseAttributes{}, Content: []byte("abc")}, true},
+		{"directory promoted from remote-tiered file", &Entry{IsDirectory: true, Attributes: &FuseAttributes{}, RemoteEntry: &RemoteEntry{RemoteSize: 100}}, true},
+		{"directory with chunks and nil attributes", &Entry{IsDirectory: true, Chunks: chunk}, true},
+		{"regular file with chunks", &Entry{IsDirectory: false, Attributes: &FuseAttributes{}, Chunks: chunk}, false},
+		{"remote mount directory has no remote size", &Entry{IsDirectory: true, Attributes: &FuseAttributes{}, RemoteEntry: &RemoteEntry{StorageName: "s3"}}, false},
+		{"empty prefix object has only the mark", &Entry{IsDirectory: true, Attributes: &FuseAttributes{}, Extended: map[string][]byte{s3_constants.SeaweedFSPrefixObject: []byte("true")}}, true},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			if got := c.e.IsDirectoryKeyObject(); got != c.want {
+				t.Errorf("IsDirectoryKeyObject() = %v, want %v", got, c.want)
+			}
+		})
+	}
+}
+
+func TestMarkPrefixObject(t *testing.T) {
+	entry := &Entry{Name: "foo", Attributes: &FuseAttributes{FileMode: 0644}}
+
+	if entry.IsPrefixObject() {
+		t.Fatal("a file is not a prefix object")
+	}
+
+	entry.MarkPrefixObject()
+
+	if !entry.IsDirectory || !entry.IsPrefixObject() {
+		t.Errorf("MarkPrefixObject() left %+v", entry)
+	}
+	// The filer derives the entry type from the mode, so the directory bit has to be
+	// on it for the store to keep this as a directory.
+	if mode := os.FileMode(entry.Attributes.FileMode); mode&os.ModeDir == 0 || mode.Perm()&0111 != 0111 {
+		t.Errorf("FileMode = %v, want a traversable directory", mode)
+	}
+
+	// A directory the mark was stripped from is a plain directory again.
+	delete(entry.Extended, s3_constants.SeaweedFSPrefixObject)
+	if entry.IsPrefixObject() || entry.IsDirectoryKeyObject() {
+		t.Error("a demoted directory names no key")
+	}
+}

weed/pb/filer_pb/filer_vtproto_test.go

@@ -0,0 +1,306 @@
+package filer_pb
+
+import (
+	"fmt"
+	"testing"
+
+	"google.golang.org/protobuf/proto"
+)
+
+// sampleResponse builds a metadata event exercising the fields touched on the
+// hot subscribe path: nested entries, a repeated chunk list, a map, and byte
+// fields.
+func sampleResponse() *SubscribeMetadataResponse {
+	entry := func(name string) *Entry {
+		return &Entry{
+			Name:        name,
+			IsDirectory: false,
+			Chunks: []*FileChunk{
+				{
+					FileId:       "3,01637037d6",
+					Offset:       0,
+					Size:         1234,
+					ModifiedTsNs: 111,
+					ETag:         "abc",
+					Fid:          &FileId{VolumeId: 3, FileKey: 0x1637037, Cookie: 0xd6},
+					CipherKey:    []byte{1, 2, 3, 4},
+				},
+			},
+			Attributes: &FuseAttributes{
+				FileSize: 1234, Mtime: 222, FileMode: 0644, Uid: 1000, Gid: 1000,
+				GroupName: []string{"staff", "wheel"}, Md5: []byte{9, 8, 7},
+			},
+			Extended: map[string][]byte{
+				"foo": []byte("bar"),
+				"baz": []byte("qux"),
+				"k3":  []byte("v3"),
+			},
+			Content: []byte("hello"),
+		}
+	}
+	return &SubscribeMetadataResponse{
+		Directory: "/buckets/test/dir",
+		EventNotification: &EventNotification{
+			OldEntry:           entry("old.txt"),
+			NewEntry:           entry("new.txt"),
+			DeleteChunks:       true,
+			NewParentPath:      "/buckets/test/dir2",
+			IsFromOtherCluster: true,
+			Signatures:         []int32{101, 202, 303},
+		},
+		TsNs: 1234567890,
+	}
+}
+
+// eventWithChunks models a real create/update event: a file split into n chunks,
+// each a nested FileChunk+FileId message. n scales the number of nested messages
+// the decoder must allocate, which is where vtproto's edge over reflection grows.
+func eventWithChunks(n int) *SubscribeMetadataResponse {
+	chunks := make([]*FileChunk, n)
+	for i := 0; i < n; i++ {
+		chunks[i] = &FileChunk{
+			FileId:       fmt.Sprintf("%d,%08x%04x", i%64, i*7919, i%251),
+			Offset:       int64(i) * 8 << 20,
+			Size:         8 << 20,
+			ModifiedTsNs: 1700000000000000000 + int64(i),
+			ETag:         fmt.Sprintf("etag-%d", i),
+			Fid:          &FileId{VolumeId: uint32(i%64 + 1), FileKey: uint64(i * 7919), Cookie: uint32(i % 251)},
+		}
+	}
+	return &SubscribeMetadataResponse{
+		Directory: "/buckets/data/2026/07/15",
+		EventNotification: &EventNotification{
+			NewEntry: &Entry{
+				Name:       "object.bin",
+				Chunks:     chunks,
+				Attributes: &FuseAttributes{FileSize: uint64(n) * 8 << 20, Mtime: 1700000000, FileMode: 0644, Uid: 1000, Gid: 1000},
+			},
+			Signatures: []int32{12345},
+		},
+		TsNs: 1700000000000000000,
+	}
+}
+
+// TestVTWireCompatibility guards that the vtproto-generated MarshalVT/UnmarshalVT
+// stay wire-compatible with the reflection-based proto path in both directions,
+// so adopting VT on the metadata hot paths cannot silently corrupt events.
+func TestVTWireCompatibility(t *testing.T) {
+	orig := sampleResponse()
+
+	// vt marshal -> proto unmarshal
+	vtBytes, err := orig.MarshalVT()
+	if err != nil {
+		t.Fatalf("MarshalVT: %v", err)
+	}
+	var fromVT SubscribeMetadataResponse
+	if err := proto.Unmarshal(vtBytes, &fromVT); err != nil {
+		t.Fatalf("proto.Unmarshal(vtBytes): %v", err)
+	}
+	if !proto.Equal(orig, &fromVT) {
+		t.Fatalf("proto.Unmarshal(MarshalVT) diverged from original")
+	}
+
+	// proto marshal -> vt unmarshal
+	protoBytes, err := proto.Marshal(orig)
+	if err != nil {
+		t.Fatalf("proto.Marshal: %v", err)
+	}
+	var fromProto SubscribeMetadataResponse
+	if err := fromProto.UnmarshalVT(protoBytes); err != nil {
+		t.Fatalf("UnmarshalVT(protoBytes): %v", err)
+	}
+	if !proto.Equal(orig, &fromProto) {
+		t.Fatalf("UnmarshalVT(proto.Marshal) diverged from original")
+	}
+
+	// MarshalToSizedBufferVT into a pre-sized slice matches MarshalVT, the write
+	// path used by the log buffer.
+	sized := make([]byte, orig.SizeVT())
+	if _, err := orig.MarshalToSizedBufferVT(sized); err != nil {
+		t.Fatalf("MarshalToSizedBufferVT: %v", err)
+	}
+	var fromSized SubscribeMetadataResponse
+	if err := fromSized.UnmarshalVT(sized); err != nil {
+		t.Fatalf("UnmarshalVT(sized): %v", err)
+	}
+	if !proto.Equal(orig, &fromSized) {
+		t.Fatalf("MarshalToSizedBufferVT diverged from original")
+	}
+
+	// LogEntry carrying a marshaled event, mirroring the log buffer write path.
+	le := &LogEntry{TsNs: 7, PartitionKeyHash: 9, Data: vtBytes, Key: []byte("k"), Offset: 42}
+	leBytes, err := le.MarshalVT()
+	if err != nil {
+		t.Fatalf("LogEntry.MarshalVT: %v", err)
+	}
+	var leDecoded LogEntry
+	if err := proto.Unmarshal(leBytes, &leDecoded); err != nil {
+		t.Fatalf("proto.Unmarshal(LogEntry): %v", err)
+	}
+	if !proto.Equal(le, &leDecoded) {
+		t.Fatalf("LogEntry vt/proto roundtrip diverged")
+	}
+}
+
+// TestUTF8Validation documents the deliberate UTF-8 difference between the two
+// decoders. UnmarshalVT skips proto3's UTF-8 validation for string fields, so
+// the metadata delivery paths keep decoding SubscribeMetadataResponse with
+// proto.Unmarshal; only LogEntry (bytes-only) decodes use UnmarshalVT.
+func TestUTF8Validation(t *testing.T) {
+	// SubscribeMetadataResponse{Directory: "\xff"}: field 1 (string), wire type 2,
+	// length 1, byte 0xff.
+	malformed := []byte{0x0a, 0x01, 0xff}
+
+	var viaProto SubscribeMetadataResponse
+	if err := proto.Unmarshal(malformed, &viaProto); err == nil {
+		t.Fatal("proto.Unmarshal accepted invalid UTF-8 in a string field; expected rejection")
+	}
+
+	var viaVT SubscribeMetadataResponse
+	if err := viaVT.UnmarshalVT(malformed); err != nil {
+		t.Fatalf("UnmarshalVT rejected invalid UTF-8: %v", err)
+	}
+	if viaVT.Directory != "\xff" {
+		t.Fatalf("UnmarshalVT Directory = %q, want \\xff", viaVT.Directory)
+	}
+
+	// LogEntry carries only bytes fields, so the decoders agree on arbitrary bytes;
+	// that is why the log entry decode paths can use UnmarshalVT.
+	// LogEntry{Data: "\xff"}: field 3 (bytes), wire type 2, length 1, byte 0xff.
+	leBytes := []byte{0x1a, 0x01, 0xff}
+	var leProto LogEntry
+	if err := proto.Unmarshal(leBytes, &leProto); err != nil {
+		t.Fatalf("proto.Unmarshal LogEntry: %v", err)
+	}
+	var leVT LogEntry
+	if err := leVT.UnmarshalVT(leBytes); err != nil {
+		t.Fatalf("UnmarshalVT LogEntry: %v", err)
+	}
+	if !proto.Equal(&leProto, &leVT) {
+		t.Fatal("LogEntry proto/VT decode of a bytes field diverged")
+	}
+}
+
+var chunkCounts = []int{1, 8, 32, 128}
+
+// BenchmarkUnmarshal is the decode hot path: each subscriber decodes every
+// delivered event, so this cost is paid once per (event, subscriber).
+func BenchmarkUnmarshal(b *testing.B) {
+	for _, n := range chunkCounts {
+		data, _ := proto.Marshal(eventWithChunks(n))
+		b.Run(fmt.Sprintf("proto/chunks=%d", n), func(b *testing.B) {
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				var m SubscribeMetadataResponse
+				if err := proto.Unmarshal(data, &m); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+		b.Run(fmt.Sprintf("vt/chunks=%d", n), func(b *testing.B) {
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				var m SubscribeMetadataResponse
+				if err := m.UnmarshalVT(data); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}
+
+// BenchmarkMarshal is the encode path: each metadata write marshals one LogEntry
+// payload.
+func BenchmarkMarshal(b *testing.B) {
+	for _, n := range chunkCounts {
+		ev := eventWithChunks(n)
+		b.Run(fmt.Sprintf("proto/chunks=%d", n), func(b *testing.B) {
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				if _, err := proto.Marshal(ev); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+		b.Run(fmt.Sprintf("vt/chunks=%d", n), func(b *testing.B) {
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				if _, err := ev.MarshalVT(); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}
+
+// BenchmarkMarshalIntoBuffer mirrors the log buffer write path: marshal into a
+// reused destination buffer. proto.Marshal must allocate a fresh slice and copy;
+// SizeVT + MarshalToSizedBufferVT writes straight into the destination.
+func BenchmarkMarshalIntoBuffer(b *testing.B) {
+	for _, n := range chunkCounts {
+		ev := eventWithChunks(n)
+		dst := make([]byte, 0, ev.SizeVT()*2)
+		b.Run(fmt.Sprintf("proto+copy/chunks=%d", n), func(b *testing.B) {
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				data, err := proto.Marshal(ev)
+				if err != nil {
+					b.Fatal(err)
+				}
+				dst = dst[:len(data)]
+				copy(dst, data)
+			}
+		})
+		b.Run(fmt.Sprintf("vt-sized/chunks=%d", n), func(b *testing.B) {
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				size := ev.SizeVT()
+				dst = dst[:size]
+				if _, err := ev.MarshalToSizedBufferVT(dst); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}
+
+// The log position fields must survive VT round-trips, or a VT-marshaled
+// response silently loses the version the mount fences handles with.
+func TestVtProtoPreservesLogTsNs(t *testing.T) {
+	messages := []interface {
+		MarshalVT() ([]byte, error)
+	}{
+		&LookupDirectoryEntryResponse{LogTsNs: 42},
+		&CacheRemoteObjectToLocalClusterResponse{LogTsNs: 43},
+		&CreateEntryResponse{LogTsNs: 44},
+		&UpdateEntryResponse{LogTsNs: 45},
+	}
+	for i, m := range messages {
+		data, err := m.MarshalVT()
+		if err != nil {
+			t.Fatalf("MarshalVT %d: %v", i, err)
+		}
+		switch v := m.(type) {
+		case *LookupDirectoryEntryResponse:
+			out := &LookupDirectoryEntryResponse{}
+			if err := out.UnmarshalVT(data); err != nil || out.LogTsNs != v.LogTsNs {
+				t.Fatalf("LookupDirectoryEntryResponse round-trip LogTsNs = %d, %v; want %d", out.LogTsNs, err, v.LogTsNs)
+			}
+		case *CacheRemoteObjectToLocalClusterResponse:
+			out := &CacheRemoteObjectToLocalClusterResponse{}
+			if err := out.UnmarshalVT(data); err != nil || out.LogTsNs != v.LogTsNs {
+				t.Fatalf("CacheRemoteObjectToLocalClusterResponse round-trip LogTsNs = %d, %v; want %d", out.LogTsNs, err, v.LogTsNs)
+			}
+		case *CreateEntryResponse:
+			out := &CreateEntryResponse{}
+			if err := out.UnmarshalVT(data); err != nil || out.LogTsNs != v.LogTsNs {
+				t.Fatalf("CreateEntryResponse round-trip LogTsNs = %d, %v; want %d", out.LogTsNs, err, v.LogTsNs)
+			}
+		case *UpdateEntryResponse:
+			out := &UpdateEntryResponse{}
+			if err := out.UnmarshalVT(data); err != nil || out.LogTsNs != v.LogTsNs {
+				t.Fatalf("UpdateEntryResponse round-trip LogTsNs = %d, %v; want %d", out.LogTsNs, err, v.LogTsNs)
+			}
+		}
+	}
+}

weed/pb/filer_pb/listing_options.go

@@ -0,0 +1,22 @@
+package filer_pb
+
+import "context"
+
+type omitChunksKey struct{}
+
+// WithChunksOmitted marks a listing as wanting attributes only. A listing over
+// gRPC then asks the filer to leave the chunk lists out, and a listing served
+// from a local store skips building them. The file size is carried in the
+// attributes either way.
+//
+// Only a caller that reads attributes and nothing else may use this. In
+// particular a listing that populates a cache, or one whose entries can be
+// written back or deleted, needs the chunks.
+func WithChunksOmitted(ctx context.Context) context.Context {
+	return context.WithValue(ctx, omitChunksKey{}, true)
+}
+
+// ChunksOmitted reports whether the listing wants attributes only.
+func ChunksOmitted(ctx context.Context) bool {
+	return ctx.Value(omitChunksKey{}) != nil
+}

weed/pb/filer_pb_direct_read.go

@@ -0,0 +1,354 @@
+package pb
+
+import (
+	"container/heap"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// LogFileReaderFn creates an io.ReadCloser for a set of file chunks.
+type LogFileReaderFn func(chunks []*filer_pb.FileChunk) (io.ReadCloser, error)
+
+// logEntryChannelSize bounds decoded entries in flight per filer stream.
+const logEntryChannelSize = 4096
+
+// maxLogEntrySize guards the per-entry allocation against a corrupt size
+// prefix, mirroring the filer package's unexported constant.
+const maxLogEntrySize = 1 << 30
+
+// ErrLogFileRead marks a failure to read persisted log chunks from volume
+// servers. It surfaces as the metadata stream's error, but says nothing about
+// the filer connection that carried the stream.
+var ErrLogFileRead = errors.New("read log file refs")
+
+// errReaderStopped signals that the entry consumer asked to stop (the merge
+// loop aborted or the caller hit a processing error). It is not a read failure.
+var errReaderStopped = errors.New("log entry reader stopped")
+
+// PathFilter holds subscription path filtering parameters, matching the
+// server-side eachEventNotificationFn filtering logic.
+type PathFilter struct {
+	PathPrefix             string
+	AdditionalPathPrefixes []string
+	DirectoriesToWatch     []string
+}
+
+// ReadLogFileRefs reads log file data directly from volume servers using the
+// chunk references, merges entries from multiple filers in timestamp order
+// (same algorithm as the server's OrderedLogVisitor), applies path filtering,
+// and invokes processEventFn for each matching event.
+//
+// Filers are read in parallel (one goroutine per filer), each streaming
+// entries through a bounded channel.
+func ReadLogFileRefs(
+	refs []*filer_pb.LogFileChunkRef,
+	newReader LogFileReaderFn,
+	startTsNs, stopTsNs int64,
+	filter PathFilter,
+	processEventFn ProcessMetadataFunc,
+) (lastTsNs int64, err error) {
+
+	if len(refs) == 0 {
+		return
+	}
+
+	// Group refs by filer ID, preserving order within each filer.
+	perFiler := make(map[string][]*filer_pb.LogFileChunkRef)
+	var filerOrder []string
+	for _, ref := range refs {
+		if len(ref.Chunks) == 0 {
+			continue
+		}
+		if _, seen := perFiler[ref.FilerId]; !seen {
+			filerOrder = append(filerOrder, ref.FilerId)
+		}
+		perFiler[ref.FilerId] = append(perFiler[ref.FilerId], ref)
+	}
+
+	if len(filerOrder) == 0 {
+		return
+	}
+
+	return readFilersMerged(filerOrder, perFiler, newReader, startTsNs, stopTsNs, filter, processEventFn)
+}
+
+// readFilersMerged reads files from each filer in parallel (one producer
+// goroutine per filer streaming decoded entries through a bounded channel),
+// then merges entries in timestamp order via min-heap. A single filer is the
+// degenerate one-stream merge.
+func readFilersMerged(
+	filerOrder []string,
+	perFiler map[string][]*filer_pb.LogFileChunkRef,
+	newReader LogFileReaderFn,
+	startTsNs, stopTsNs int64,
+	filter PathFilter,
+	processEventFn ProcessMetadataFunc,
+) (lastTsNs int64, err error) {
+
+	type filerStream struct {
+		filerId string
+		entryCh chan *filer_pb.LogEntry
+	}
+
+	streams := make([]filerStream, len(filerOrder))
+
+	// A genuine (non chunk-not-found) read error must fail the whole replay: the
+	// caller advances its cursor only on success, so a swallowed error leaves a
+	// permanent gap. stop aborts the other readers; fatalErr is read on exit.
+	//
+	// Closing stop is the only cleanup: producers observe it at every send and
+	// every file boundary and exit on their own. An abort must not wait for
+	// them — a producer can be wedged in an uncancellable chunk read, and
+	// joining it would stall the caller's retry loop behind a dead connection.
+	stop := make(chan struct{})
+	var stopOnce sync.Once
+	closeStop := func() { stopOnce.Do(func() { close(stop) }) }
+	defer closeStop()
+	var fatalMu sync.Mutex
+	var fatalErr error
+	setFatal := func(e error) {
+		fatalMu.Lock()
+		if fatalErr == nil {
+			fatalErr = e
+		}
+		fatalMu.Unlock()
+		closeStop()
+	}
+	fatal := func() error {
+		fatalMu.Lock()
+		defer fatalMu.Unlock()
+		return fatalErr
+	}
+
+	for i, filerId := range filerOrder {
+		entryCh := make(chan *filer_pb.LogEntry, logEntryChannelSize)
+		streams[i] = filerStream{filerId: filerId, entryCh: entryCh}
+
+		go func(refs []*filer_pb.LogFileChunkRef, ch chan *filer_pb.LogEntry) {
+			defer close(ch)
+			readFilerFilesToChannel(refs, newReader, startTsNs, stopTsNs, ch, stop, setFatal)
+		}(perFiler[filerId], entryCh)
+	}
+
+	// Seed the min-heap with the first entry from each filer
+	pq := &logEntryHeap{}
+	heap.Init(pq)
+	for i := range streams {
+		select {
+		case entry, ok := <-streams[i].entryCh:
+			if ok {
+				heap.Push(pq, &logEntryHeapItem{entry: entry, filerIdx: i})
+			}
+		case <-stop:
+			return lastTsNs, fatal()
+		}
+	}
+
+	// Merge loop
+	for pq.Len() > 0 {
+		// stop is closed only by setFatal here, so a closed stop means a reader
+		// aborted; lock-free bail on the hot path.
+		select {
+		case <-stop:
+			return lastTsNs, fatal()
+		default:
+		}
+
+		item := heap.Pop(pq).(*logEntryHeapItem)
+
+		lastTsNs, err = processOneLogEntry(item.entry, filter, processEventFn)
+		if err != nil {
+			return
+		}
+
+		select {
+		case entry, ok := <-streams[item.filerIdx].entryCh:
+			if ok {
+				heap.Push(pq, &logEntryHeapItem{entry: entry, filerIdx: item.filerIdx})
+			}
+		case <-stop:
+			return lastTsNs, fatal()
+		}
+	}
+
+	// All channels closed: every producer has finished (setFatal, if any,
+	// happened before its close).
+	if fe := fatal(); fe != nil {
+		return lastTsNs, fe
+	}
+	return
+}
+
+func readFilerFilesToChannel(
+	refs []*filer_pb.LogFileChunkRef,
+	newReader LogFileReaderFn,
+	startTsNs, stopTsNs int64,
+	ch chan *filer_pb.LogEntry,
+	stop <-chan struct{},
+	setFatal func(error),
+) {
+	var sent int
+	sendEntry := func(entry *filer_pb.LogEntry) error {
+		// Prefer stop over a send the buffer could still absorb, so an aborted
+		// producer quits at the next entry instead of filling the channel.
+		select {
+		case <-stop:
+			return errReaderStopped
+		default:
+		}
+		select {
+		case ch <- entry:
+			sent++
+			return nil
+		case <-stop:
+			return errReaderStopped
+		}
+	}
+
+	for _, ref := range refs {
+		select {
+		case <-stop:
+			return
+		default:
+		}
+
+		sentBefore := sent
+		streamErr := streamLogFileEntries(newReader, ref.Chunks, startTsNs, stopTsNs, sendEntry)
+		if streamErr == errReaderStopped {
+			return
+		}
+		if streamErr != nil {
+			if isChunkNotFound(streamErr) {
+				// A mid-file not-found still delivered the readable prefix; say so
+				// rather than pretending the whole file was skipped.
+				glog.V(0).Infof("skip log file filer=%s ts=%d after %d entries: %v", ref.FilerId, ref.FileTsNs, sent-sentBefore, streamErr)
+				continue
+			}
+			setFatal(fmt.Errorf("read log file filer=%s ts=%d: %w", ref.FilerId, ref.FileTsNs, streamErr))
+			return
+		}
+	}
+}
+
+func processOneLogEntry(logEntry *filer_pb.LogEntry, filter PathFilter, processEventFn ProcessMetadataFunc) (int64, error) {
+	event := &filer_pb.SubscribeMetadataResponse{}
+	// proto.Unmarshal (not UnmarshalVT) validates UTF-8 in string fields, so
+	// malformed metadata is skipped here instead of reaching path filtering.
+	if err := proto.Unmarshal(logEntry.Data, event); err != nil {
+		glog.Errorf("unmarshal log entry: %v", err)
+		return 0, nil // skip corrupt entries
+	}
+	if !matchesFilter(event, filter) {
+		return event.TsNs, nil
+	}
+	if err := processEventFn(event); err != nil {
+		return event.TsNs, fmt.Errorf("process event: %w", err)
+	}
+	return event.TsNs, nil
+}
+
+// --- path filtering (mirrors server-side eachEventNotificationFn logic) ---
+
+const systemLogDir = "/topics/.system/log"
+
+func matchesFilter(resp *filer_pb.SubscribeMetadataResponse, filter PathFilter) bool {
+	fullpath := filer_pb.MetadataEventSourceFullPath(resp)
+
+	// Skip internal meta log entries
+	if strings.HasPrefix(fullpath, systemLogDir) {
+		return false
+	}
+
+	return filer_pb.MetadataEventMatchesSubscription(resp, filter.PathPrefix, filter.AdditionalPathPrefixes, filter.DirectoriesToWatch)
+}
+
+// isChunkNotFound checks if an error indicates a missing volume chunk.
+// Matches the server-side isChunkNotFoundError logic.
+func isChunkNotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+	s := err.Error()
+	return strings.Contains(s, "not found") || strings.Contains(s, "status 404")
+}
+
+// --- min-heap for merging entries across filers ---
+
+type logEntryHeapItem struct {
+	entry    *filer_pb.LogEntry
+	filerIdx int
+}
+
+type logEntryHeap []*logEntryHeapItem
+
+func (h logEntryHeap) Len() int           { return len(h) }
+func (h logEntryHeap) Less(i, j int) bool { return h[i].entry.TsNs < h[j].entry.TsNs }
+func (h logEntryHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
+func (h *logEntryHeap) Push(x any)        { *h = append(*h, x.(*logEntryHeapItem)) }
+func (h *logEntryHeap) Pop() any {
+	old := *h
+	n := len(old)
+	item := old[n-1]
+	old[n-1] = nil
+	*h = old[:n-1]
+	return item
+}
+
+// --- log file parsing (uses io.ReadFull for correct partial-read handling) ---
+
+// streamLogFileEntries reads a log file's chunks and invokes eachFn for every
+// entry as it is decoded, without buffering the whole file: a multi-GB log
+// file costs one entry plus the chunk reader's buffer at a time, not O(file
+// size). An eachFn error stops the read and is propagated verbatim.
+func streamLogFileEntries(newReader LogFileReaderFn, chunks []*filer_pb.FileChunk, startTsNs, stopTsNs int64, eachFn func(*filer_pb.LogEntry) error) error {
+	reader, err := newReader(chunks)
+	if err != nil {
+		return fmt.Errorf("create reader: %w", err)
+	}
+	defer reader.Close()
+
+	sizeBuf := make([]byte, 4)
+
+	for {
+		if _, err := io.ReadFull(reader, sizeBuf); err != nil {
+			if err == io.EOF || err == io.ErrUnexpectedEOF {
+				return nil
+			}
+			return err
+		}
+
+		size := util.BytesToUint32(sizeBuf)
+		if size > maxLogEntrySize {
+			return fmt.Errorf("entry size %d exceeds %d", size, maxLogEntrySize)
+		}
+		entryData := make([]byte, size)
+		if _, err := io.ReadFull(reader, entryData); err != nil {
+			return err
+		}
+
+		logEntry := &filer_pb.LogEntry{}
+		if err := logEntry.UnmarshalVT(entryData); err != nil {
+			return err
+		}
+
+		if logEntry.TsNs <= startTsNs {
+			continue
+		}
+		if stopTsNs != 0 && logEntry.TsNs > stopTsNs {
+			return nil
+		}
+
+		if err := eachFn(logEntry); err != nil {
+			return err
+		}
+	}
+}

weed/pb/filer_pb_direct_read_test.go

@@ -0,0 +1,477 @@
+package pb
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// buildLogFileData creates the on-disk log file format:
+// [4-byte size | protobuf LogEntry] repeated.
+func buildLogFileData(events []*filer_pb.SubscribeMetadataResponse) []byte {
+	var buf bytes.Buffer
+	for _, event := range events {
+		eventData, _ := proto.Marshal(event)
+		logEntry := &filer_pb.LogEntry{
+			TsNs: event.TsNs,
+			Data: eventData,
+			Key:  []byte(event.Directory),
+		}
+		entryData, _ := proto.Marshal(logEntry)
+		sizeBuf := make([]byte, 4)
+		util.Uint32toBytes(sizeBuf, uint32(len(entryData)))
+		buf.Write(sizeBuf)
+		buf.Write(entryData)
+	}
+	return buf.Bytes()
+}
+
+func makeSubEvent(dir, name string, tsNs int64) *filer_pb.SubscribeMetadataResponse {
+	return &filer_pb.SubscribeMetadataResponse{
+		Directory: dir,
+		TsNs:      tsNs,
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name:        name,
+				IsDirectory: false,
+			},
+		},
+	}
+}
+
+// delayedReader wraps data with a per-open latency to simulate volume server I/O.
+type delayedReader struct {
+	data     []byte
+	delay    time.Duration
+	openedAt time.Time
+}
+
+func (r *delayedReader) Read(p []byte) (int, error) {
+	if r.openedAt.IsZero() {
+		r.openedAt = time.Now()
+		time.Sleep(r.delay)
+	}
+	if len(r.data) == 0 {
+		return 0, io.EOF
+	}
+	n := copy(p, r.data)
+	r.data = r.data[n:]
+	return n, nil
+}
+
+func (r *delayedReader) Close() error { return nil }
+
+type testLogFiles struct {
+	refs      []*filer_pb.LogFileChunkRef
+	fileData  map[string][]byte // key: "filerId:fileTsNs" → raw log file bytes
+	fileDelay time.Duration
+}
+
+func newTestLogFiles(numFilers, filesPerFiler, eventsPerFile int, fileDelay time.Duration) *testLogFiles {
+	t := &testLogFiles{
+		fileData:  make(map[string][]byte),
+		fileDelay: fileDelay,
+	}
+
+	baseTs := time.Now().Add(-time.Hour).UnixNano()
+	tsCounter := int64(0)
+
+	for f := 0; f < numFilers; f++ {
+		filerId := fmt.Sprintf("filer%02d", f)
+		for file := 0; file < filesPerFiler; file++ {
+			fileTsNs := baseTs + int64(file)*int64(time.Minute)
+
+			events := make([]*filer_pb.SubscribeMetadataResponse, eventsPerFile)
+			for i := 0; i < eventsPerFile; i++ {
+				tsCounter++
+				ts := baseTs + tsCounter
+				events[i] = makeSubEvent(
+					fmt.Sprintf("/data/%s/dir%02d", filerId, file),
+					fmt.Sprintf("file%04d.txt", i),
+					ts,
+				)
+			}
+
+			data := buildLogFileData(events)
+			key := fmt.Sprintf("%s:%d", filerId, fileTsNs)
+			t.fileData[key] = data
+
+			t.refs = append(t.refs, &filer_pb.LogFileChunkRef{
+				Chunks: []*filer_pb.FileChunk{{
+					FileId: key,
+				}},
+				FileTsNs: fileTsNs,
+				FilerId:  filerId,
+			})
+		}
+	}
+	return t
+}
+
+func (t *testLogFiles) readerFn() LogFileReaderFn {
+	return func(chunks []*filer_pb.FileChunk) (io.ReadCloser, error) {
+		if len(chunks) == 0 {
+			return nil, fmt.Errorf("no chunks")
+		}
+		key := chunks[0].FileId
+		data, ok := t.fileData[key]
+		if !ok {
+			return nil, fmt.Errorf("file not found: %s", key)
+		}
+		dataCopy := make([]byte, len(data))
+		copy(dataCopy, data)
+		return &delayedReader{data: dataCopy, delay: t.fileDelay}, nil
+	}
+}
+
+func (t *testLogFiles) totalEvents() int {
+	total := 0
+	for _, data := range t.fileData {
+		pos := 0
+		for pos+4 <= len(data) {
+			size := int(util.BytesToUint32(data[pos : pos+4]))
+			pos += 4 + size
+			total++
+		}
+	}
+	return total
+}
+
+// assertOrderedReplay reads all refs and checks every event arrives exactly
+// once, in timestamp order.
+func assertOrderedReplay(t *testing.T, files *testLogFiles) {
+	t.Helper()
+
+	var timestamps []int64
+	_, err := ReadLogFileRefs(files.refs, files.readerFn(), 0, 0,
+		PathFilter{PathPrefix: "/"},
+		func(resp *filer_pb.SubscribeMetadataResponse) error {
+			timestamps = append(timestamps, resp.TsNs)
+			return nil
+		})
+	if err != nil {
+		t.Fatalf("ReadLogFileRefs: %v", err)
+	}
+
+	if got, want := len(timestamps), files.totalEvents(); got != want {
+		t.Fatalf("expected %d events, got %d", want, got)
+	}
+	for i := 1; i < len(timestamps); i++ {
+		if timestamps[i] < timestamps[i-1] {
+			t.Fatalf("out of order at index %d: ts[%d]=%d > ts[%d]=%d",
+				i, i-1, timestamps[i-1], i, timestamps[i])
+		}
+	}
+}
+
+// TestReadLogFileRefsMergeOrder verifies that entries from multiple filers are
+// delivered in correct timestamp order.
+func TestReadLogFileRefsMergeOrder(t *testing.T) {
+	assertOrderedReplay(t, newTestLogFiles(3, 2, 50, 0))
+}
+
+// TestReadLogFileRefsPathFilter verifies path filtering including system log exclusion.
+func TestReadLogFileRefsPathFilter(t *testing.T) {
+	files := newTestLogFiles(2, 2, 50, 0)
+	total := files.totalEvents()
+
+	var allCount, filteredCount int64
+	_, err := ReadLogFileRefs(files.refs, files.readerFn(), 0, 0,
+		PathFilter{PathPrefix: "/"},
+		func(resp *filer_pb.SubscribeMetadataResponse) error {
+			allCount++
+			return nil
+		})
+	if err != nil {
+		t.Fatalf("ReadLogFileRefs (all): %v", err)
+	}
+
+	_, err = ReadLogFileRefs(files.refs, files.readerFn(), 0, 0,
+		PathFilter{PathPrefix: "/data/filer00/"},
+		func(resp *filer_pb.SubscribeMetadataResponse) error {
+			filteredCount++
+			return nil
+		})
+	if err != nil {
+		t.Fatalf("ReadLogFileRefs (filtered): %v", err)
+	}
+
+	t.Logf("Total events: %d, matching /data/filer00/: %d", allCount, filteredCount)
+
+	if allCount != int64(total) {
+		t.Errorf("expected %d total events, got %d", total, allCount)
+	}
+	if filteredCount >= allCount {
+		t.Errorf("filter should reduce events: all=%d filtered=%d", allCount, filteredCount)
+	}
+	if filteredCount == 0 {
+		t.Errorf("filter matched zero events")
+	}
+}
+
+// TestDirectReadVsServerSideThroughput compares:
+//   - Server-side: sequential file read → gRPC send per event
+//   - Client direct-read: parallel filers + streaming + no gRPC
+func TestDirectReadVsServerSideThroughput(t *testing.T) {
+	const (
+		numFilers     = 3
+		filesPerFiler = 7
+		eventsPerFile = 300
+		fileReadDelay = 2 * time.Millisecond
+		sendDelay     = 20 * time.Microsecond
+	)
+
+	files := newTestLogFiles(numFilers, filesPerFiler, eventsPerFile, fileReadDelay)
+
+	var serverRate float64
+	t.Run("server_side_sequential", func(t *testing.T) {
+		var processed int64
+		start := time.Now()
+
+		for _, ref := range files.refs {
+			time.Sleep(fileReadDelay)
+			key := ref.Chunks[0].FileId
+			data := files.fileData[key]
+			pos := 0
+			for pos+4 <= len(data) {
+				size := int(util.BytesToUint32(data[pos : pos+4]))
+				pos += 4 + size
+				time.Sleep(sendDelay)
+				atomic.AddInt64(&processed, 1)
+			}
+		}
+		elapsed := time.Since(start)
+		serverRate = float64(processed) / elapsed.Seconds()
+		t.Logf("server-side: %d events  %v  %6.0f events/sec  (%d files sequential + %v send/event)",
+			processed, elapsed.Round(time.Millisecond), serverRate,
+			numFilers*filesPerFiler, sendDelay)
+	})
+
+	var directRate float64
+	t.Run("client_direct_read_parallel_streaming", func(t *testing.T) {
+		var processed int64
+		start := time.Now()
+
+		_, err := ReadLogFileRefs(files.refs, files.readerFn(), 0, 0,
+			PathFilter{PathPrefix: "/"},
+			func(resp *filer_pb.SubscribeMetadataResponse) error {
+				atomic.AddInt64(&processed, 1)
+				return nil
+			})
+		if err != nil {
+			t.Fatalf("ReadLogFileRefs: %v", err)
+		}
+		elapsed := time.Since(start)
+		directRate = float64(processed) / elapsed.Seconds()
+		t.Logf("direct-read: %d events  %v  %6.0f events/sec  (%d filers parallel + streaming, no gRPC)",
+			processed, elapsed.Round(time.Millisecond), directRate, numFilers)
+	})
+
+	if serverRate > 0 {
+		t.Logf("Speedup: %.1fx (parallel + streaming + no gRPC vs server-side sequential)", directRate/serverRate)
+	}
+}
+
+// failingReaderFn returns err for the given file key, delegating otherwise.
+func failingReaderFn(base LogFileReaderFn, failKey string, err error) LogFileReaderFn {
+	return func(chunks []*filer_pb.FileChunk) (io.ReadCloser, error) {
+		if len(chunks) > 0 && chunks[0].FileId == failKey {
+			return nil, err
+		}
+		return base(chunks)
+	}
+}
+
+// A real (non not-found) read error must fail the whole replay, not silently
+// drop the file and advance the cursor.
+func TestReadLogFileRefsMultiFilerGenuineErrorAborts(t *testing.T) {
+	files := newTestLogFiles(3, 2, 10, 0)
+	failKey := files.refs[2].Chunks[0].FileId // filer01's first file
+	readerFn := failingReaderFn(files.readerFn(), failKey, fmt.Errorf("failed to locate %s", failKey))
+
+	var count int64
+	_, err := ReadLogFileRefs(files.refs, readerFn, 0, 0,
+		PathFilter{PathPrefix: "/"},
+		func(resp *filer_pb.SubscribeMetadataResponse) error {
+			atomic.AddInt64(&count, 1)
+			return nil
+		})
+	if err == nil {
+		t.Fatalf("expected error from genuine read failure, got nil (delivered=%d)", count)
+	}
+}
+
+// A chunk-not-found error skips only that file (volume gone), not the replay.
+func TestReadLogFileRefsMultiFilerNotFoundSkips(t *testing.T) {
+	files := newTestLogFiles(3, 2, 10, 0)
+	skipKey := files.refs[2].Chunks[0].FileId // filer01's first file
+	readerFn := failingReaderFn(files.readerFn(), skipKey, fmt.Errorf("volume not found: %s", skipKey))
+
+	var count int64
+	_, err := ReadLogFileRefs(files.refs, readerFn, 0, 0,
+		PathFilter{PathPrefix: "/"},
+		func(resp *filer_pb.SubscribeMetadataResponse) error {
+			atomic.AddInt64(&count, 1)
+			return nil
+		})
+	if err != nil {
+		t.Fatalf("chunk-not-found should be skipped, got error: %v", err)
+	}
+	expected := int64(files.totalEvents() - 10) // one skipped file's events
+	if count != expected {
+		t.Fatalf("expected %d events after skipping one file, got %d", expected, count)
+	}
+}
+
+// TestReadLogFileRefsSingleFilerOrder covers the single-filer path: every
+// entry across all files, in order.
+func TestReadLogFileRefsSingleFilerOrder(t *testing.T) {
+	assertOrderedReplay(t, newTestLogFiles(1, 4, 50, 0))
+}
+
+// TestReadLogFileRefsSingleFilerProcessErrorStops verifies that the callback's
+// own error propagates and aborts the stream promptly, mid-file.
+func TestReadLogFileRefsSingleFilerProcessErrorStops(t *testing.T) {
+	files := newTestLogFiles(1, 3, 100, 0)
+
+	var count int
+	wantErr := fmt.Errorf("boom")
+	_, err := ReadLogFileRefs(files.refs, files.readerFn(), 0, 0,
+		PathFilter{PathPrefix: "/"},
+		func(resp *filer_pb.SubscribeMetadataResponse) error {
+			count++
+			if count == 5 {
+				return wantErr
+			}
+			return nil
+		})
+	if !errors.Is(err, wantErr) {
+		t.Fatalf("expected processing error to propagate, got: %v", err)
+	}
+	// Should stop near the failing event, not process the whole 300-event set.
+	if count > 20 {
+		t.Fatalf("expected prompt stop after error, processed %d events", count)
+	}
+}
+
+// A corrupt size prefix must fail the replay instead of allocating gigabytes.
+func TestReadLogFileRefsCorruptSizePrefix(t *testing.T) {
+	data := make([]byte, 4)
+	util.Uint32toBytes(data, 0xFFFFFFF0)
+	refs := []*filer_pb.LogFileChunkRef{{
+		Chunks:   []*filer_pb.FileChunk{{FileId: "corrupt"}},
+		FileTsNs: 1,
+		FilerId:  "filer00",
+	}}
+	readerFn := func(chunks []*filer_pb.FileChunk) (io.ReadCloser, error) {
+		return io.NopCloser(bytes.NewReader(data)), nil
+	}
+
+	_, err := ReadLogFileRefs(refs, readerFn, 0, 0, PathFilter{PathPrefix: "/"},
+		func(*filer_pb.SubscribeMetadataResponse) error { return nil })
+	if err == nil || !strings.Contains(err.Error(), "exceeds") {
+		t.Fatalf("expected size-cap error, got: %v", err)
+	}
+}
+
+// blockingReader blocks in Read until released.
+type blockingReader struct{ release chan struct{} }
+
+func (r *blockingReader) Read(p []byte) (int, error) { <-r.release; return 0, io.EOF }
+func (r *blockingReader) Close() error               { return nil }
+
+// An abort (fatal error on one filer) must not wait for another filer's
+// in-flight chunk read: the replay returns promptly and the wedged producer
+// exits on its own once its read completes.
+func TestReadLogFileRefsAbortDoesNotJoinWedgedReader(t *testing.T) {
+	files := newTestLogFiles(2, 1, 10, 0)
+	release := make(chan struct{})
+	t.Cleanup(func() { close(release) })
+
+	wedgedKey := files.refs[0].Chunks[0].FileId // filer00 wedges mid-read
+	failKey := files.refs[1].Chunks[0].FileId   // filer01 fails for real
+	base := files.readerFn()
+	readerFn := func(chunks []*filer_pb.FileChunk) (io.ReadCloser, error) {
+		switch chunks[0].FileId {
+		case wedgedKey:
+			return &blockingReader{release: release}, nil
+		case failKey:
+			return nil, fmt.Errorf("failed to locate %s", failKey)
+		}
+		return base(chunks)
+	}
+
+	done := make(chan error, 1)
+	go func() {
+		_, err := ReadLogFileRefs(files.refs, readerFn, 0, 0, PathFilter{PathPrefix: "/"},
+			func(*filer_pb.SubscribeMetadataResponse) error { return nil })
+		done <- err
+	}()
+
+	select {
+	case err := <-done:
+		if err == nil {
+			t.Fatalf("expected the fatal read error to propagate")
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatalf("ReadLogFileRefs did not return while a peer reader was wedged")
+	}
+}
+
+// TestReadLogFileRefsSingleFilerNotFoundSkips confirms a chunk-not-found on the
+// single-filer path skips just that file, not the whole replay.
+func TestReadLogFileRefsSingleFilerNotFoundSkips(t *testing.T) {
+	files := newTestLogFiles(1, 3, 10, 0)
+	skipKey := files.refs[1].Chunks[0].FileId // second file
+	readerFn := failingReaderFn(files.readerFn(), skipKey, fmt.Errorf("volume not found: %s", skipKey))
+
+	var count int
+	_, err := ReadLogFileRefs(files.refs, readerFn, 0, 0,
+		PathFilter{PathPrefix: "/"},
+		func(resp *filer_pb.SubscribeMetadataResponse) error {
+			count++
+			return nil
+		})
+	if err != nil {
+		t.Fatalf("chunk-not-found should be skipped, got error: %v", err)
+	}
+	if want := files.totalEvents() - 10; count != want {
+		t.Fatalf("expected %d events after skipping one file, got %d", want, count)
+	}
+}
+
+// TestProcessOneLogEntrySkipsInvalidUTF8 guards that an event whose string field
+// is not valid UTF-8 is rejected by proto.Unmarshal and never reaches path
+// filtering or the consumer. UnmarshalVT would accept it, so the decode here
+// must stay on proto.Unmarshal.
+func TestProcessOneLogEntrySkipsInvalidUTF8(t *testing.T) {
+	// SubscribeMetadataResponse{Directory: "\xff"}: field 1 (string), wire type 2,
+	// length 1, byte 0xff — a syntactically valid message with an invalid-UTF-8
+	// string, which proto3 forbids.
+	logEntry := &filer_pb.LogEntry{Data: []byte{0x0a, 0x01, 0xff}}
+
+	called := false
+	tsNs, err := processOneLogEntry(logEntry, PathFilter{PathPrefix: "/"}, func(resp *filer_pb.SubscribeMetadataResponse) error {
+		called = true
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("processOneLogEntry err = %v, want nil (corrupt entries are skipped)", err)
+	}
+	if called {
+		t.Fatal("consumer was invoked for a malformed (invalid UTF-8) event; validation was bypassed")
+	}
+	if tsNs != 0 {
+		t.Fatalf("tsNs = %d, want 0 for a skipped entry", tsNs)
+	}
+}

weed/pb/filer_pb_tail.go

@@ -0,0 +1,216 @@
+package pb
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+)
+
+type EventErrorType int
+
+const (
+	TrivialOnError EventErrorType = iota
+	FatalOnError
+	RetryForeverOnError
+	DontLogError
+)
+
+// MetadataFollowOption is used to control the behavior of the metadata following
+// process. Part of it is used as a cursor to resume the following process.
+type MetadataFollowOption struct {
+	ClientName             string
+	ClientId               int32
+	ClientEpoch            int32
+	SelfSignature          int32
+	PathPrefix             string
+	AdditionalPathPrefixes []string
+	DirectoriesToWatch     []string
+	StartTsNs              int64
+	StopTsNs               int64
+	EventErrorType         EventErrorType
+	// LogFileReaderFn, when non-nil, enables metadata chunks mode:
+	// the server sends log file chunk fids instead of streaming events,
+	// and the client reads directly from volume servers.
+	LogFileReaderFn LogFileReaderFn
+	// OnIdleHeartbeat, when non-nil, opts in to idle heartbeats: while the
+	// subscriber is caught up the server periodically sends an empty response
+	// carrying the current time, and this is called with that timestamp. It is
+	// a freshness signal only and does not advance StartTsNs, so the resume
+	// checkpoint stays on the last real event.
+	OnIdleHeartbeat func(tsNs int64)
+}
+
+type ProcessMetadataFunc func(resp *filer_pb.SubscribeMetadataResponse) error
+
+func FollowMetadata(filerAddress ServerAddress, grpcDialOption grpc.DialOption, option *MetadataFollowOption, processEventFn ProcessMetadataFunc) error {
+
+	err := WithFilerClient(true, option.SelfSignature, filerAddress, grpcDialOption, makeSubscribeMetadataFunc(option, processEventFn))
+	if err != nil {
+		return fmt.Errorf("subscribing filer meta change: %w", err)
+	}
+	return err
+}
+
+func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient, option *MetadataFollowOption, processEventFn ProcessMetadataFunc) error {
+
+	err := filerClient.WithFilerClient(true, makeSubscribeMetadataFunc(option, processEventFn))
+	if err != nil {
+		return fmt.Errorf("subscribing filer meta change: %w", err)
+	}
+
+	return nil
+}
+
+func makeSubscribeMetadataFunc(option *MetadataFollowOption, processEventFn ProcessMetadataFunc) func(client filer_pb.SeaweedFilerClient) error {
+	return func(client filer_pb.SeaweedFilerClient) error {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
+			ClientName:                   option.ClientName,
+			PathPrefix:                   option.PathPrefix,
+			PathPrefixes:                 option.AdditionalPathPrefixes,
+			Directories:                  option.DirectoriesToWatch,
+			SinceNs:                      option.StartTsNs,
+			Signature:                    option.SelfSignature,
+			ClientId:                     option.ClientId,
+			ClientEpoch:                  option.ClientEpoch,
+			UntilNs:                      option.StopTsNs,
+			ClientSupportsBatching:       true,
+			ClientSupportsMetadataChunks: option.LogFileReaderFn != nil,
+			ClientSupportsIdleHeartbeat:  option.OnIdleHeartbeat != nil,
+		})
+		if err != nil {
+			return fmt.Errorf("subscribe: %w", err)
+		}
+
+		handleErr := func(resp *filer_pb.SubscribeMetadataResponse, err error) {
+			switch option.EventErrorType {
+			case TrivialOnError:
+				glog.Errorf("process %v: %v", resp, err)
+			case FatalOnError:
+				glog.Fatalf("process %v: %v", resp, err)
+			case RetryForeverOnError:
+				util.RetryUntil("followMetaUpdates", func() error {
+					return processEventFn(resp)
+				}, func(err error) bool {
+					glog.Errorf("process %v: %v", resp, err)
+					return true
+				})
+			case DontLogError:
+				// pass
+			default:
+				glog.Errorf("process %v: %v", resp, err)
+			}
+		}
+
+		// handleOneEvent processes a single response, whether it arrived as the
+		// batch envelope or inside resp.Events. Freshness signals carry a timestamp
+		// but no entry: the idle heartbeat (nil EventNotification) and the
+		// MaxUnsyncedEvents marker (empty one). Both route to OnIdleHeartbeat,
+		// else the marker pins sync_offset to the stale processed watermark and a
+		// nil heartbeat folded into a batch nil-derefs in the sync job path.
+		handleOneEvent := func(resp *filer_pb.SubscribeMetadataResponse) {
+			if resp.EventNotification == nil || filer_pb.IsEmpty(resp) {
+				if resp.TsNs > 0 && option.OnIdleHeartbeat != nil {
+					option.OnIdleHeartbeat(resp.TsNs)
+				}
+				// The marker advances the resume cursor past the filtered range; the
+				// heartbeat leaves StartTsNs put so a restart cannot outrun a straggler.
+				if resp.EventNotification != nil && resp.TsNs > 0 {
+					option.StartTsNs = resp.TsNs
+				}
+				return
+			}
+			if err := processEventFn(resp); err != nil {
+				handleErr(resp, err)
+			}
+			option.StartTsNs = resp.TsNs
+		}
+
+		var pendingRefs []*filer_pb.LogFileChunkRef
+
+		// drainPendingRefs reads whatever chunk refs have accumulated. The server
+		// sends refs in their own responses, so they can only be read once
+		// something tells us the run of refs has ended — either a normal event, or
+		// the end of the stream. A bounded subscription (StopTsNs set, range
+		// already in the past) may get nothing but refs and then EOF, so draining
+		// only on the former silently returns no events at all.
+		drainPendingRefs := func() error {
+			if len(pendingRefs) == 0 || option.LogFileReaderFn == nil {
+				return nil
+			}
+			lastTs, readErr := ReadLogFileRefs(pendingRefs, option.LogFileReaderFn,
+				option.StartTsNs, option.StopTsNs,
+				PathFilter{
+					PathPrefix:             option.PathPrefix,
+					AdditionalPathPrefixes: option.AdditionalPathPrefixes,
+					DirectoriesToWatch:     option.DirectoriesToWatch,
+				},
+				processEventFn)
+			if readErr != nil {
+				return fmt.Errorf("%w: %w", ErrLogFileRead, readErr)
+			}
+			if lastTs > 0 {
+				option.StartTsNs = lastTs
+			}
+			pendingRefs = nil
+			return nil
+		}
+
+		for {
+			resp, listenErr := stream.Recv()
+			if listenErr == io.EOF {
+				return drainPendingRefs()
+			}
+			if listenErr != nil {
+				return listenErr
+			}
+
+			// Accumulate log file chunk references (metadata chunks mode)
+			if len(resp.LogFileRefs) > 0 {
+				pendingRefs = append(pendingRefs, resp.LogFileRefs...)
+				continue
+			}
+
+			// Process accumulated refs before handling normal events (transition point)
+			if err := drainPendingRefs(); err != nil {
+				return err
+			}
+
+			// Process the envelope event (top-level fields) and any batched tail.
+			// The server folds a backlog into one response: the first event lives in
+			// the top-level fields, the rest in resp.Events. Either slot can hold a
+			// freshness signal, so both go through the same handler.
+			handleOneEvent(resp)
+			for _, batchedEvent := range resp.Events {
+				handleOneEvent(batchedEvent)
+			}
+		}
+	}
+}
+
+func AddOffsetFunc(processEventFn ProcessMetadataFunc, offsetInterval time.Duration, offsetFunc func(counter int64, offset int64) error) ProcessMetadataFunc {
+	var counter int64
+	var lastWriteTime = time.Now()
+	return func(resp *filer_pb.SubscribeMetadataResponse) error {
+		if err := processEventFn(resp); err != nil {
+			return err
+		}
+		counter++
+		if lastWriteTime.Add(offsetInterval).Before(time.Now()) {
+			lastWriteTime = time.Now()
+			if err := offsetFunc(counter, resp.TsNs); err != nil {
+				return err
+			}
+			counter = 0
+		}
+		return nil
+	}
+
+}

weed/pb/filer_pb_tail_test.go

@@ -0,0 +1,194 @@
+package pb
+
+import (
+	"context"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"google.golang.org/grpc"
+)
+
+// fakeSubscribeStream plays back a scripted sequence of responses, one per Recv.
+type fakeSubscribeStream struct {
+	grpc.ClientStream
+	responses []*filer_pb.SubscribeMetadataResponse
+	delay     time.Duration
+	idx       int
+}
+
+func (s *fakeSubscribeStream) Recv() (*filer_pb.SubscribeMetadataResponse, error) {
+	if s.idx >= len(s.responses) {
+		return nil, io.EOF
+	}
+	if s.idx > 0 && s.delay > 0 {
+		// advance the wall clock so AddOffsetFunc's interval gate opens
+		time.Sleep(s.delay)
+	}
+	r := s.responses[s.idx]
+	s.idx++
+	return r, nil
+}
+
+// fakeFilerClient only needs SubscribeMetadata; the embedded nil interface
+// covers the rest, which makeSubscribeMetadataFunc never calls.
+type fakeFilerClient struct {
+	filer_pb.SeaweedFilerClient
+	stream *fakeSubscribeStream
+}
+
+func (c *fakeFilerClient) SubscribeMetadata(ctx context.Context, in *filer_pb.SubscribeMetadataRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.SubscribeMetadataResponse], error) {
+	return c.stream, nil
+}
+
+// TestFilerSyncOffsetStaysFreshOnFilteredMarker: on a read-only watched path with
+// a busy source, the MaxUnsyncedEvents marker (empty EventNotification, fresh
+// timestamp) must be treated as a freshness signal, not fed to the offset path
+// where it would pin the gauge to the stale watermark. Wiring mirrors filer.sync.
+func TestFilerSyncOffsetStaysFreshOnFilteredMarker(t *testing.T) {
+	const oldEventTs = int64(1_000_000_000) // t0: last real synced event (stale)
+	nowTs := time.Now().UnixNano()          // current source time
+	markerTs := nowTs + int64(time.Second)
+
+	var watermark = oldEventTs // MetadataProcessor.processedTsWatermark
+
+	type gaugeWrite struct {
+		src string
+		ts  int64
+	}
+	var timeline []gaugeWrite
+	var heartbeatCalls, markerToProcessFn int
+
+	// AddSyncJob drops empty events and does not advance the watermark; the real
+	// processor is checked in command.TestMetadataProcessorEmptyMarkerKeepsWatermarkStale.
+	realProcessFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
+		if filer_pb.IsEmpty(resp) {
+			markerToProcessFn++
+			return nil
+		}
+		watermark = resp.TsNs
+		return nil
+	}
+	// offsetFunc publishes the watermark to the gauge (filer_sync.go).
+	processEventFn := AddOffsetFunc(realProcessFn, 0, func(counter, lastTsNs int64) error {
+		timeline = append(timeline, gaugeWrite{"offset", watermark})
+		return nil
+	})
+
+	option := &MetadataFollowOption{
+		ClientName:     "syncFrom_A_To_B",
+		StartTsNs:      oldEventTs,
+		EventErrorType: DontLogError,
+		OnIdleHeartbeat: func(tsNs int64) {
+			heartbeatCalls++
+			timeline = append(timeline, gaugeWrite{"heartbeat", tsNs})
+		},
+	}
+
+	stream := &fakeSubscribeStream{
+		delay: 2 * time.Millisecond,
+		responses: []*filer_pb.SubscribeMetadataResponse{
+			// real create on the watched path, long ago -> watermark = t0
+			{Directory: "/watched", TsNs: oldEventTs, EventNotification: &filer_pb.EventNotification{
+				NewEntry: &filer_pb.Entry{Name: "file"},
+			}},
+			// genuine idle heartbeat
+			{TsNs: nowTs},
+			// MaxUnsyncedEvents marker: empty EventNotification, fresh timestamp
+			{TsNs: markerTs, EventNotification: &filer_pb.EventNotification{}},
+		},
+	}
+
+	if err := makeSubscribeMetadataFunc(option, processEventFn)(&fakeFilerClient{stream: stream}); err != nil {
+		t.Fatalf("follow: %v", err)
+	}
+
+	t.Logf("gauge timeline: %+v", timeline)
+
+	// the marker fires OnIdleHeartbeat instead of reaching processEventFn
+	if markerToProcessFn != 0 {
+		t.Errorf("empty marker must not reach processEventFn, got %d calls", markerToProcessFn)
+	}
+	if heartbeatCalls != 2 {
+		t.Errorf("expected OnIdleHeartbeat for both the heartbeat and the marker (2), got %d", heartbeatCalls)
+	}
+	if option.StartTsNs != markerTs {
+		t.Errorf("marker should advance StartTsNs to %d, got %d", markerTs, option.StartTsNs)
+	}
+
+	// gauge stays fresh: last write is the marker's timestamp, not the stale watermark
+	last := timeline[len(timeline)-1]
+	if last.src != "heartbeat" || last.ts != markerTs {
+		t.Fatalf("expected final gauge write fresh at %d, got %+v (spike is back if stale %d)", markerTs, last, oldEventTs)
+	}
+}
+
+// TestFilerSyncBatchedFreshnessSignalDoesNotCrash: while catching up after a
+// peer outage the server folds a backlog into one batched response: the first
+// event in the top-level fields, the rest in resp.Events. The drain can pull an
+// idle heartbeat (nil EventNotification) into that tail. The batched tail must
+// get the same freshness-signal handling as the envelope, else processEventFn
+// (filer.sync's AddSyncJob) nil-derefs in IsEmpty. The processEventFn here
+// mirrors that first call.
+func TestFilerSyncBatchedFreshnessSignalDoesNotCrash(t *testing.T) {
+	const ts1 = int64(1_000_000_000)      // envelope: real create
+	const ts2 = int64(1_000_000_001)      // batched: real create
+	const markerTs = int64(1_000_000_002) // batched: MaxUnsyncedEvents marker (empty entry)
+	hbTs := time.Now().UnixNano()         // batched: idle heartbeat (nil EventNotification), fresh
+
+	var realEvents []int64
+	var heartbeatTs []int64
+
+	// Mirrors AddSyncJob: IsEmpty nil-derefs on a nil EventNotification.
+	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
+		if filer_pb.IsEmpty(resp) {
+			return nil
+		}
+		realEvents = append(realEvents, resp.TsNs)
+		return nil
+	}
+
+	option := &MetadataFollowOption{
+		ClientName:     "syncFrom_A_To_B",
+		StartTsNs:      ts1,
+		EventErrorType: DontLogError,
+		OnIdleHeartbeat: func(tsNs int64) {
+			heartbeatTs = append(heartbeatTs, tsNs)
+		},
+	}
+
+	// One batched response: envelope plus a tail mixing a real event, a marker,
+	// and a fresh heartbeat, as the server emits while draining a backlog. The
+	// heartbeat is last and carries the largest timestamp on purpose.
+	stream := &fakeSubscribeStream{
+		responses: []*filer_pb.SubscribeMetadataResponse{{
+			Directory:         "/watched",
+			TsNs:              ts1,
+			EventNotification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "a"}},
+			Events: []*filer_pb.SubscribeMetadataResponse{
+				{Directory: "/watched", TsNs: ts2, EventNotification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "b"}}},
+				{TsNs: markerTs, EventNotification: &filer_pb.EventNotification{}}, // marker: empty entry
+				{TsNs: hbTs}, // idle heartbeat: nil EventNotification
+			},
+		}},
+	}
+
+	if err := makeSubscribeMetadataFunc(option, processEventFn)(&fakeFilerClient{stream: stream}); err != nil {
+		t.Fatalf("follow: %v", err)
+	}
+
+	// Both real events reached processEventFn; neither freshness signal did.
+	if len(realEvents) != 2 || realEvents[0] != ts1 || realEvents[1] != ts2 {
+		t.Errorf("expected real events [%d %d], got %v", ts1, ts2, realEvents)
+	}
+	// The batched marker and heartbeat both fired OnIdleHeartbeat.
+	if len(heartbeatTs) != 2 || heartbeatTs[0] != markerTs || heartbeatTs[1] != hbTs {
+		t.Errorf("expected heartbeats [%d %d], got %v", markerTs, hbTs, heartbeatTs)
+	}
+	// The marker advances the resume cursor; the heartbeat does not, even though
+	// it is last and carries the largest timestamp. StartTsNs ends at the marker.
+	if option.StartTsNs != markerTs {
+		t.Errorf("expected StartTsNs %d (marker), got %d (heartbeat must not advance the cursor)", markerTs, option.StartTsNs)
+	}
+}

weed/pb/grpc_client_cascade_test.go

@@ -0,0 +1,209 @@
+package pb
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net"
+	"net/http"
+	"sync"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+// cascadeFilerServer answers a "slow" lookup only once its caller is known to be
+// in flight, so a second caller can fail while the first still holds the shared
+// cached ClientConn.
+type cascadeFilerServer struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+	inFlight  chan struct{}
+	release   chan struct{}
+	closeOnce sync.Once
+}
+
+func (s *cascadeFilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
+	if req.Name == "slow" {
+		s.closeOnce.Do(func() { close(s.inFlight) })
+		select {
+		case <-s.release:
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
+	return &filer_pb.LookupDirectoryEntryResponse{}, nil
+}
+
+// startCascadeFiler serves a fake filer and resets the process-wide connection
+// cache so the test owns the shared ClientConn under exercise.
+func startCascadeFiler(t *testing.T) (*cascadeFilerServer, string, grpc.DialOption) {
+	t.Helper()
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	fake := &cascadeFilerServer{inFlight: make(chan struct{}), release: make(chan struct{})}
+	server := grpc.NewServer()
+	filer_pb.RegisterSeaweedFilerServer(server, fake)
+	go server.Serve(listener)
+
+	grpcClientsLock.Lock()
+	previous := grpcClients
+	grpcClients = make(map[string]*versionedGrpcClient)
+	grpcClientsLock.Unlock()
+	t.Cleanup(func() {
+		grpcClientsLock.Lock()
+		cached := grpcClients
+		grpcClients = previous
+		grpcClientsLock.Unlock()
+		for _, connection := range cached {
+			connection.Close()
+		}
+		server.Stop()
+	})
+
+	return fake, listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())
+}
+
+// TestWithGrpcClient_AbandonedRequestKeepsSharedConnection reproduces
+// seaweedfs#10947 end to end: an S3-style caller that hands WithGrpcClient a
+// context.Background() while its RPC rides the (now abandoned) HTTP request
+// context used to close the shared filer ClientConn, killing every concurrent
+// upload on it with "the client connection is closing".
+func TestWithGrpcClient_AbandonedRequestKeepsSharedConnection(t *testing.T) {
+	fake, address, dialOption := startCascadeFiler(t)
+
+	var concurrentErr error
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		concurrentErr = WithGrpcClient(context.Background(), false, 0, func(connection *grpc.ClientConn) error {
+			_, err := filer_pb.NewSeaweedFilerClient(connection).LookupDirectoryEntry(context.Background(),
+				&filer_pb.LookupDirectoryEntryRequest{Directory: "/buckets/b", Name: "slow"})
+			return err
+		}, address, false, dialOption)
+	}()
+	<-fake.inFlight
+
+	abandoned, cancel := context.WithCancel(context.Background())
+	cancel()
+	if err := WithGrpcClient(context.Background(), false, 0, func(connection *grpc.ClientConn) error {
+		_, err := filer_pb.NewSeaweedFilerClient(connection).LookupDirectoryEntry(abandoned,
+			&filer_pb.LookupDirectoryEntryRequest{Directory: "/buckets/b", Name: "fast"})
+		return err
+	}, address, false, dialOption); err == nil {
+		t.Fatal("the abandoned request should have failed")
+	}
+
+	close(fake.release)
+	wg.Wait()
+	if concurrentErr != nil {
+		t.Fatalf("concurrent caller must survive an unrelated abandoned request: %v", concurrentErr)
+	}
+}
+
+func (s *cascadeFilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, stream filer_pb.SeaweedFiler_SubscribeMetadataServer) error {
+	if !req.ClientSupportsMetadataChunks {
+		return nil
+	}
+	return stream.Send(&filer_pb.SubscribeMetadataResponse{
+		LogFileRefs: []*filer_pb.LogFileChunkRef{{
+			FilerId:  "5319c0e8",
+			FileTsNs: time.Now().UnixNano(),
+			Chunks:   []*filer_pb.FileChunk{{FileId: "4471,ce598bfff8416e", Size: 64}},
+		}},
+	})
+}
+
+// TestWithGrpcClient_EndedStreamKeepsSharedConnection covers the other half of
+// seaweedfs#10947: the S3 gateway follows filer metadata on a long-lived
+// SubscribeMetadata stream and reconnects forever. Every ordinary end of that
+// stream used to drop the shared filer ClientConn the request handlers use,
+// firing the same burst of "the client connection is closing" failures.
+func TestWithGrpcClient_EndedStreamKeepsSharedConnection(t *testing.T) {
+	fake, address, dialOption := startCascadeFiler(t)
+
+	var concurrentErr error
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		concurrentErr = WithGrpcClient(context.Background(), false, 0, func(connection *grpc.ClientConn) error {
+			_, err := filer_pb.NewSeaweedFilerClient(connection).LookupDirectoryEntry(context.Background(),
+				&filer_pb.LookupDirectoryEntryRequest{Directory: "/buckets/b", Name: "slow"})
+			return err
+		}, address, false, dialOption)
+	}()
+	<-fake.inFlight
+
+	if err := WithGrpcClient(context.Background(), true, 0, func(connection *grpc.ClientConn) error {
+		stream, err := filer_pb.NewSeaweedFilerClient(connection).SubscribeMetadata(context.Background(), &filer_pb.SubscribeMetadataRequest{})
+		if err != nil {
+			return err
+		}
+		_, err = stream.Recv()
+		return err
+	}, address, false, dialOption); err == nil {
+		t.Fatal("the ended stream should have reported io.EOF")
+	}
+
+	close(fake.release)
+	wg.Wait()
+	if concurrentErr != nil {
+		t.Fatalf("concurrent caller must survive an unrelated stream ending: %v", concurrentErr)
+	}
+}
+
+// TestWithGrpcClient_LogChunkReadFailureKeepsSharedConnection covers a
+// subscriber replaying persisted log chunks over HTTP: an unreachable volume
+// server reports "connection refused" as the subscription's error, which used
+// to drop the shared filer ClientConn and cancel the assign and upload RPCs
+// riding on it.
+func TestWithGrpcClient_LogChunkReadFailureKeepsSharedConnection(t *testing.T) {
+	fake, address, dialOption := startCascadeFiler(t)
+
+	var concurrentErr error
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		concurrentErr = WithGrpcClient(context.Background(), false, 0, func(connection *grpc.ClientConn) error {
+			_, err := filer_pb.NewSeaweedFilerClient(connection).LookupDirectoryEntry(context.Background(),
+				&filer_pb.LookupDirectoryEntryRequest{Directory: "/buckets/b", Name: "slow"})
+			return err
+		}, address, false, dialOption)
+	}()
+	<-fake.inFlight
+
+	option := &MetadataFollowOption{
+		ClientName:     "mount",
+		PathPrefix:     "/",
+		EventErrorType: DontLogError,
+		LogFileReaderFn: func(chunks []*filer_pb.FileChunk) (io.ReadCloser, error) {
+			// An unreachable volume server, as a filerProxy mount sees it.
+			response, err := http.Get("http://127.0.0.1:1/" + chunks[0].FileId)
+			if err != nil {
+				return nil, err
+			}
+			return response.Body, nil
+		},
+	}
+	subscribe := makeSubscribeMetadataFunc(option, func(resp *filer_pb.SubscribeMetadataResponse) error { return nil })
+	err := WithGrpcClient(context.Background(), true, 0, func(connection *grpc.ClientConn) error {
+		return subscribe(filer_pb.NewSeaweedFilerClient(connection))
+	}, address, false, dialOption)
+	if err == nil || !errors.Is(err, ErrLogFileRead) {
+		t.Fatalf("the replay should have failed reading the log chunk: %v", err)
+	}
+
+	close(fake.release)
+	wg.Wait()
+	if concurrentErr != nil {
+		t.Fatalf("concurrent caller must survive a log chunk read failure: %v", concurrentErr)
+	}
+}

weed/pb/grpc_client_server.go

@@ -0,0 +1,645 @@
+package pb
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand/v2"
+	"net"
+	"net/http"
+	"os"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/seaweedfs/seaweedfs/weed/util/request_id"
+	"google.golang.org/grpc/metadata"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/keepalive"
+	"google.golang.org/grpc/status"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+)
+
+const (
+	Max_Message_Size = 1 << 30 // 1 GB
+
+	// gRPC keepalive settings - must be consistent between client and server
+	GrpcKeepAliveTime        = 60 * time.Second // ping interval when no activity
+	GrpcKeepAliveTimeout     = 20 * time.Second // ping timeout
+	GrpcKeepAliveMinimumTime = 20 * time.Second // minimum interval between client pings (enforcement)
+)
+
+var (
+	// cache grpc connections
+	grpcClients     = make(map[string]*versionedGrpcClient)
+	grpcClientsLock sync.Mutex
+
+	// localGrpcSockets maps gRPC port numbers to Unix socket paths for
+	// gRPC services running in this process. Used by both the server side
+	// (to start serving on the socket) and the client side (to dial it
+	// instead of TCP for same-host RPCs).
+	localGrpcSockets = make(map[int]string)
+	// localGrpcHosts maps gRPC port → set of host strings that count as
+	// "this machine" for that port. Only dials whose target host is in this
+	// set are routed through the Unix socket; dials to other hosts that
+	// happen to use the same port number go over TCP as normal.
+	//
+	// Without this host check the hijack was keyed purely on port, so a
+	// remote server reusing one of our local gRPC ports (e.g. a standalone
+	// `weed volume -port=7334` defaulting `port.grpc=17334` against a
+	// `weed server` whose in-process volume socket is also on 17334) had
+	// its inbound RPCs silently rerouted to our local Unix socket — see
+	// issue #9254.
+	localGrpcHosts       = make(map[int]map[string]struct{})
+	localGrpcSocketsLock sync.RWMutex
+)
+
+type versionedGrpcClient struct {
+	*grpc.ClientConn
+	version  int
+	errCount int
+}
+
+func init() {
+	t := http.DefaultTransport.(*http.Transport)
+	t.MaxIdleConnsPerHost = 1024
+	t.MaxIdleConns = 1024
+	// Bind outbound HTTP to the -ip.bind source address. Reads the setting
+	// per dial, so it applies regardless of when SetOutboundLocalIP runs
+	// relative to this init.
+	t.DialContext = util.OutboundDialContext
+}
+
+// RegisterLocalGrpcSocket registers a Unix socket path for a gRPC service
+// running on host:grpcPort in this process. After this is called, any
+// GrpcDial to host:grpcPort (or to a loopback alias of host on the same
+// port) is routed through the Unix socket. Dials to any other host on the
+// same port still go over TCP.
+//
+// No-op on Windows: the /tmp/...sock paths callers pass are POSIX-only and
+// the listen/dial would fail at runtime, taking gRPC down with it (#9430).
+// Skipping registration leaves the maps empty, so ServeGrpcOnLocalSocket
+// and resolveLocalGrpcSocket short-circuit and same-host RPCs go over TCP.
+func RegisterLocalGrpcSocket(host string, grpcPort int, socketPath string) {
+	if runtime.GOOS == "windows" {
+		return
+	}
+	localGrpcSocketsLock.Lock()
+	defer localGrpcSocketsLock.Unlock()
+	localGrpcSockets[grpcPort] = socketPath
+	hosts, ok := localGrpcHosts[grpcPort]
+	if !ok {
+		hosts = make(map[string]struct{})
+		localGrpcHosts[grpcPort] = hosts
+	}
+	// The advertised host plus the well-known loopback aliases all refer
+	// to "this machine"; the empty entry covers SplitHostPort outputs like
+	// ":17334".
+	for _, h := range []string{host, "", "localhost", "127.0.0.1", "::1"} {
+		hosts[h] = struct{}{}
+	}
+}
+
+// GetLocalGrpcSocket returns the Unix socket path for a gRPC port, or empty if not registered.
+func GetLocalGrpcSocket(grpcPort int) string {
+	localGrpcSocketsLock.RLock()
+	defer localGrpcSocketsLock.RUnlock()
+	return localGrpcSockets[grpcPort]
+}
+
+// resolveLocalGrpcSocket returns the Unix socket path to use for an outgoing
+// gRPC dial, or empty if the dial should go over TCP. It matches both the
+// host and the port: a remote peer that happens to reuse a local service's
+// gRPC port must not be rerouted to the local socket (issue #9254).
+func resolveLocalGrpcSocket(address string) string {
+	host, portStr, err := net.SplitHostPort(address)
+	if err != nil {
+		return ""
+	}
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		return ""
+	}
+	localGrpcSocketsLock.RLock()
+	defer localGrpcSocketsLock.RUnlock()
+	if _, ok := localGrpcHosts[port][host]; !ok {
+		return ""
+	}
+	return localGrpcSockets[port]
+}
+
+// ServeGrpcOnLocalSocket starts serving a gRPC server on a Unix socket
+// if one is registered for the given port.
+func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) {
+	socketPath := GetLocalGrpcSocket(grpcPort)
+	if socketPath == "" {
+		return
+	}
+	if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
+		glog.Warningf("Failed to remove old gRPC socket %s: %v", socketPath, err)
+	}
+	lc := net.ListenConfig{Control: setLocalSocketBuffers}
+	listener, err := lc.Listen(context.Background(), "unix", socketPath)
+	if err != nil {
+		glog.Errorf("Failed to listen on gRPC Unix socket %s: %v", socketPath, err)
+		return
+	}
+	listener = &localSocketListener{Listener: listener}
+	glog.V(0).Infof("gRPC also listening on Unix socket %s", socketPath)
+	go func() {
+		if err := grpcServer.Serve(listener); err != nil && err != grpc.ErrServerStopped {
+			glog.Errorf("gRPC Unix socket server error on %s: %v", socketPath, err)
+		}
+		os.Remove(socketPath)
+	}()
+}
+
+// localSocketListener re-applies the buffer sizes to every accepted connection.
+type localSocketListener struct {
+	net.Listener
+}
+
+func (l *localSocketListener) Accept() (net.Conn, error) {
+	c, err := l.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	applyLocalSocketBuffers(c)
+	return c, nil
+}
+
+func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
+	var options []grpc.ServerOption
+	options = append(options,
+		grpc.KeepaliveParams(keepalive.ServerParameters{
+			Time:    GrpcKeepAliveTime,    // server pings client if no activity for this long
+			Timeout: GrpcKeepAliveTimeout, // ping timeout
+		}),
+		grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
+			MinTime:             GrpcKeepAliveMinimumTime, // min time a client should wait before sending a ping
+			PermitWithoutStream: true,
+		}),
+		grpc.MaxRecvMsgSize(Max_Message_Size),
+		grpc.MaxSendMsgSize(Max_Message_Size),
+		grpc.MaxConcurrentStreams(1000),          // Allow more concurrent streams
+		grpc.InitialWindowSize(16*1024*1024),     // 16MB initial window
+		grpc.InitialConnWindowSize(16*1024*1024), // 16MB connection window
+		grpc.MaxHeaderListSize(8*1024*1024),      // 8MB header list limit
+		grpc.UnaryInterceptor(requestIDUnaryInterceptor()),
+	)
+	for _, opt := range opts {
+		if opt != nil {
+			options = append(options, opt)
+		}
+	}
+	return grpc.NewServer(options...)
+}
+
+// GrpcDial establishes a gRPC connection.
+// IMPORTANT: This function intentionally uses the deprecated grpc.DialContext/grpc.Dial behavior
+// to preserve the "passthrough" resolver semantics required for Kubernetes ndots/search-domain DNS behavior.
+// This allows kube DNS suffixes to be correctly appended by the OS resolver.
+//
+// Switching to grpc.NewClient (which defaults to the "dns" resolver) would break this behavior
+// in environments with ndots:5 and many-dot hostnames.
+//
+// Safe alternatives if switching to grpc.NewClient:
+//  1. Prefix the target with "passthrough:///" (e.g., "passthrough:///my-service:8080"). This is the recommended primary migration path.
+//  2. Call resolver.SetDefaultScheme("passthrough") exactly once during init().
+//     WARNING: This is NOT thread-safe, and mutates global resolver state affecting all grpc.NewClient calls in the process.
+func GrpcDial(ctx context.Context, address string, waitForReady bool, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
+	var options []grpc.DialOption
+
+	// Route through Unix socket if one is registered for this address's port
+	if socketPath := resolveLocalGrpcSocket(address); socketPath != "" {
+		options = append(options, grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) {
+			d := net.Dialer{Control: setLocalSocketBuffers}
+			return d.DialContext(ctx, "unix", socketPath)
+		}))
+	} else {
+		// Always install so a conn cached before SetOutboundLocalIP binds once set;
+		// the stock net.Dialer until then preserves gRPC's default dial behavior.
+		options = append(options, grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
+			if util.OutboundLocalAddr() == nil {
+				var d net.Dialer
+				return d.DialContext(ctx, "tcp", addr)
+			}
+			return util.OutboundDialContext(ctx, "tcp", addr)
+		}))
+	}
+
+	options = append(options,
+		grpc.WithDefaultCallOptions(
+			grpc.MaxCallSendMsgSize(Max_Message_Size),
+			grpc.MaxCallRecvMsgSize(Max_Message_Size),
+			grpc.WaitForReady(waitForReady),
+		),
+		grpc.WithKeepaliveParams(keepalive.ClientParameters{
+			Time:    GrpcKeepAliveTime,    // client ping server if no activity for this long
+			Timeout: GrpcKeepAliveTimeout, // ping timeout
+			// Disable pings when there are no active streams to avoid triggering
+			// server enforcement for too-frequent pings from idle clients.
+			PermitWithoutStream: false,
+		}))
+	for _, opt := range opts {
+		if opt != nil {
+			options = append(options, opt)
+		}
+	}
+	return grpc.DialContext(ctx, address, options...)
+}
+
+func getOrCreateConnection(address string, waitForReady bool, opts ...grpc.DialOption) (*versionedGrpcClient, error) {
+
+	grpcClientsLock.Lock()
+	defer grpcClientsLock.Unlock()
+
+	existingConnection, found := grpcClients[address]
+	if found {
+		glog.V(4).Infof("gRPC cache hit for %s (version %d)", address, existingConnection.version)
+		return existingConnection, nil
+	}
+
+	glog.V(2).Infof("Creating new gRPC connection to %s", address)
+	ctx := context.Background()
+	grpcConnection, err := GrpcDial(ctx, address, waitForReady, opts...)
+	if err != nil {
+		return nil, fmt.Errorf("fail to dial %s: %v", address, err)
+	}
+
+	vgc := &versionedGrpcClient{
+		grpcConnection,
+		rand.Int(),
+		0,
+	}
+	grpcClients[address] = vgc
+	glog.V(2).Infof("New gRPC connection established to %s (version %d)", address, vgc.version)
+
+	return vgc, nil
+}
+
+func requestIDUnaryInterceptor() grpc.UnaryServerInterceptor {
+	return func(
+		ctx context.Context,
+		req interface{},
+		info *grpc.UnaryServerInfo,
+		handler grpc.UnaryHandler,
+	) (interface{}, error) {
+		// Get request ID from incoming metadata
+		var reqID string
+		if incomingMd, ok := metadata.FromIncomingContext(ctx); ok {
+			if idList := incomingMd.Get(request_id.AmzRequestIDHeader); len(idList) > 0 {
+				reqID = idList[0]
+			}
+		}
+		if reqID == "" {
+			reqID = uuid.New().String()
+		}
+
+		// Store request ID in context for handlers to access
+		ctx = request_id.Set(ctx, reqID)
+
+		// Also set outgoing context so handlers making downstream gRPC calls
+		// will automatically propagate the request ID
+		ctx = metadata.AppendToOutgoingContext(ctx, request_id.AmzRequestIDHeader, reqID)
+
+		// Set trailer with request ID for response
+		grpc.SetTrailer(ctx, metadata.Pairs(request_id.AmzRequestIDHeader, reqID))
+
+		return handler(ctx, req)
+	}
+}
+
+// InvalidateGrpcConnection drops any cached gRPC ClientConn for the given
+// address. Use when a higher-level signal (for example a streaming master
+// connection detecting its peer has died) indicates the cached channel is
+// stale, even though gRPC itself may still believe the channel is healthy.
+// Silently returns if no cached connection exists.
+func InvalidateGrpcConnection(address string) {
+	grpcClientsLock.Lock()
+	vgc, ok := grpcClients[address]
+	if ok {
+		delete(grpcClients, address)
+	}
+	grpcClientsLock.Unlock()
+	if ok {
+		glog.V(1).Infof("Invalidating cached gRPC connection to %s", address)
+		vgc.Close()
+	}
+}
+
+// grpcMarshalErrorPrefix is the library-owned prefix gRPC prepends to every
+// client-side proto marshal failure; see grpc-go rpc_util.go encode():
+//
+//	status.Errorf(codes.Internal, "grpc: error while marshaling: %v", ...)
+//
+// The "grpc:" token is reserved for gRPC internal diagnostics and will not
+// collide with user-produced Internal statuses.
+const grpcMarshalErrorPrefix = "grpc: error while marshaling"
+
+// grpcStatusError is the minimal interface every gRPC status error satisfies.
+// Using it with errors.As lets us reach through arbitrary fmt.Errorf("...: %w")
+// wrapping to pull out the *original* gRPC Status — important because
+// status.FromError rewrites Status.Message with the outermost err.Error()
+// whenever it has to unwrap, which would defeat a prefix check on the
+// library-owned message.
+type grpcStatusError interface{ GRPCStatus() *status.Status }
+
+// isClientSideMarshalError reports whether err originates from gRPC failing to
+// marshal the outgoing request on the client side. These errors are surfaced
+// with codes.Internal (because gRPC has no better code for them) but do NOT
+// reflect a problem with the TCP/HTTP2 connection: the request never left the
+// process. Tearing down the shared cached ClientConn in response would cancel
+// every other in-flight RPC on that connection — which is exactly the cascade
+// seen in seaweedfs#9139 when a single file with invalid-UTF-8 bytes in its
+// name produces an avalanche of "connection is closing" errors on unrelated
+// LookupEntry / ReadDirAll / UpdateEntry calls.
+//
+// errors.Is against a proto-level sentinel is not viable: gRPC's encode()
+// collapses the inner proto error with "%v" before wrapping it in a Status,
+// so the original error type does not survive. The structural signal that
+// *does* survive is the Status itself — we recover it via errors.As and
+// match the library-owned "grpc:" prefix to disambiguate from a server-side
+// application Internal status that genuinely warrants invalidation.
+func isClientSideMarshalError(err error) bool {
+	var gse grpcStatusError
+	if !errors.As(err, &gse) {
+		return false
+	}
+	s := gse.GRPCStatus()
+	if s == nil {
+		return false
+	}
+	return s.Code() == codes.Internal && strings.HasPrefix(s.Message(), grpcMarshalErrorPrefix)
+}
+
+// shouldInvalidateConnection checks if an error indicates the cached connection
+// should be invalidated. ctx must be the context that bounds this RPC attempt; it
+// disambiguates a genuinely broken channel from the RPC hitting its own deadline
+// or cancellation. A nil or non-cancellable ctx carries no such evidence.
+func shouldInvalidateConnection(ctx context.Context, err error) bool {
+	if err == nil {
+		return false
+	}
+
+	// Client-side marshal errors (e.g. invalid UTF-8 in a proto string field)
+	// are per-request bugs, not connection failures. Do not poison the shared
+	// ClientConn because of them.
+	if isClientSideMarshalError(err) {
+		return false
+	}
+
+	// A metadata subscriber reads log chunks over HTTP from volume servers and
+	// returns what went wrong there through the stream. Those failures read like
+	// transport failures below, and would drop a filer channel that is fine.
+	if errors.Is(err, ErrLogFileRead) {
+		return false
+	}
+
+	// gRPC raises this locally, before the RPC reaches the wire, when this
+	// process already closed the ClientConn. The caller is a bystander of
+	// someone else's teardown, and knows nothing about the peer. Its message
+	// is the only thing separating it from any other Canceled, so the plain
+	// codes.Canceled its deprecation notice recommends cannot stand in: that
+	// is the very code this function has to keep telling apart below.
+	if errors.Is(err, grpc.ErrClientConnClosing) {
+		return false
+	}
+
+	// Check gRPC status codes first (more reliable)
+	if s, ok := status.FromError(err); ok {
+		code := s.Code()
+		switch code {
+		case codes.Unavailable, codes.Aborted, codes.Internal:
+			return true
+		case codes.ResourceExhausted:
+			// Server-side backpressure on a healthy channel (e.g. the master's
+			// assign shed while volume growth is in flight), not a dead connection.
+			// Invalidating would Close() the shared conn and cancel every other
+			// in-flight RPC with "the client connection is closing"; keep it and let
+			// the caller retry.
+			return false
+		case codes.Canceled, codes.DeadlineExceeded:
+			// Ambiguous: a stale cached channel rejects RPCs this way (a peer
+			// restart behind a k8s Service VIP), and so does the RPC's own context
+			// expiring. Tearing the channel down for the latter cancels every other
+			// in-flight RPC on it with "the client connection is closing", turning
+			// one abandoned request into a flood of failures. Only a cancellable
+			// context bounds an RPC attempt and can tell the two apart:
+			// Background/TODO never expires, so its Err() would answer
+			// "stale channel" for every caller with no deadline to honor.
+			return ctx != nil && ctx.Done() != nil && ctx.Err() == nil
+		}
+	}
+
+	// Fall back to string matching for transport-level errors not captured by gRPC codes
+	errStr := err.Error()
+	errLower := strings.ToLower(errStr)
+	return strings.Contains(errLower, "transport") ||
+		strings.Contains(errLower, "connection closed") ||
+		strings.Contains(errLower, "dns") ||
+		strings.Contains(errLower, "connection refused") ||
+		strings.Contains(errLower, "no route to host") ||
+		strings.Contains(errLower, "network is unreachable") ||
+		strings.Contains(errLower, "connection reset")
+}
+
+// WithGrpcClient In streamingMode, always use a fresh connection. Otherwise, try to reuse an existing connection.
+// ctx must be the context that bounds the RPC fn issues, so that a
+// Canceled/DeadlineExceeded fired by ctx itself does not tear down the shared
+// cached ClientConn out from under other concurrent callers. Pass
+// context.Background() when fn picks its own context: a Canceled/DeadlineExceeded
+// then never invalidates, since nothing here can attribute it to the channel.
+func WithGrpcClient(ctx context.Context, streamingMode bool, signature int32, fn func(*grpc.ClientConn) error, address string, waitForReady bool, opts ...grpc.DialOption) error {
+
+	if !streamingMode {
+		vgc, err := getOrCreateConnection(address, waitForReady, opts...)
+		if err != nil {
+			return fmt.Errorf("getOrCreateConnection %s: %v", address, err)
+		}
+		executionErr := fn(vgc.ClientConn)
+		if executionErr != nil && shouldInvalidateConnection(ctx, executionErr) {
+			grpcClientsLock.Lock()
+			t, ok := grpcClients[address]
+			shouldClose := ok && t.version == vgc.version
+			if shouldClose {
+				delete(grpcClients, address)
+			}
+			grpcClientsLock.Unlock()
+			if shouldClose {
+				glog.V(1).Infof("Removing cached gRPC connection to %s due to error: %v", address, executionErr)
+				vgc.Close()
+			}
+		}
+		return executionErr
+	}
+
+	// Streaming mode: dedicate a fresh ClientConn to this call.
+	dialCtx := context.Background()
+	if signature != 0 {
+		// Optimize: Use AppendToOutgoingContext instead of creating new map
+		dialCtx = metadata.AppendToOutgoingContext(dialCtx, "sw-client-id", fmt.Sprintf("%d", signature))
+	}
+	grpcConnection, err := GrpcDial(dialCtx, address, waitForReady, opts...)
+	if err != nil {
+		return fmt.Errorf("fail to dial %s: %v", address, err)
+	}
+	defer grpcConnection.Close()
+	executionErr := fn(grpcConnection)
+	if executionErr != nil && shouldInvalidateConnection(ctx, executionErr) {
+		// The streaming channel is dedicated to this caller, but unrelated
+		// request-path callers share a cached non-streaming ClientConn to the
+		// same peer. When the stream dies of a peer that went away, drop that
+		// cached channel so the next caller dials fresh: this recovers cases
+		// where a stable L4 endpoint (k8s Service VIP, external LB) hides a
+		// peer restart from the transport layer, leaving the cached ClientConn
+		// healthy-looking but silently cancelling RPCs. A stream that merely
+		// ended, or that its own caller gave up on, says nothing about the peer.
+		InvalidateGrpcConnection(address)
+	}
+	return executionErr
+}
+
+func hostAndPort(address string) (host string, port uint64, err error) {
+	colonIndex := strings.LastIndex(address, ":")
+	if colonIndex < 0 {
+		return "", 0, fmt.Errorf("server should have hostname:port format: %v", address)
+	}
+	dotIndex := strings.LastIndex(address, ".")
+	if dotIndex > colonIndex {
+		// port format is "port.grpcPort"
+		port, err = strconv.ParseUint(address[colonIndex+1:dotIndex], 10, 64)
+		if err != nil {
+			return "", 0, fmt.Errorf("server port parse error: %w", err)
+		}
+		return address[:colonIndex], port, err
+	}
+	port, err = strconv.ParseUint(address[colonIndex+1:], 10, 64)
+	if err != nil {
+		return "", 0, fmt.Errorf("server port parse error: %w", err)
+	}
+
+	return address[:colonIndex], port, err
+}
+
+func ServerToGrpcAddress(server string) (serverGrpcAddress string) {
+
+	colonIndex := strings.LastIndex(server, ":")
+	if colonIndex >= 0 {
+		if dotIndex := strings.LastIndex(server, "."); dotIndex > colonIndex {
+			// port format is "port.grpcPort"
+			// return the host:grpcPort
+			host := server[:colonIndex]
+			grpcPort := server[dotIndex+1:]
+			if _, err := strconv.ParseUint(grpcPort, 10, 64); err == nil {
+				return util.JoinHostPort(host, int(0+util.ParseInt(grpcPort, 0)))
+			}
+		}
+	}
+
+	host, port, parseErr := hostAndPort(server)
+	if parseErr != nil {
+		glog.Fatalf("server address %s parse error: %v", server, parseErr)
+	}
+
+	grpcPort := int(port) + 10000
+
+	return util.JoinHostPort(host, grpcPort)
+}
+
+func GrpcAddressToServerAddress(grpcAddress string) (serverAddress string) {
+	host, grpcPort, parseErr := hostAndPort(grpcAddress)
+	if parseErr != nil {
+		glog.Fatalf("server grpc address %s parse error: %v", grpcAddress, parseErr)
+	}
+
+	port := int(grpcPort) - 10000
+
+	return util.JoinHostPort(host, port)
+}
+
+// WithMasterClient threads the context bounding this RPC attempt into the
+// connection-invalidation decision, so a Canceled/DeadlineExceeded from the
+// attempt's own timeout does not invalidate the shared cached master connection.
+// Pass context.Background() when fn picks its own context.
+func WithMasterClient(ctx context.Context, streamingMode bool, master ServerAddress, grpcDialOption grpc.DialOption, waitForReady bool, fn func(client master_pb.SeaweedClient) error) error {
+	return WithGrpcClient(ctx, streamingMode, 0, func(grpcConnection *grpc.ClientConn) error {
+		client := master_pb.NewSeaweedClient(grpcConnection)
+		return fn(client)
+	}, master.ToGrpcAddress(), waitForReady, grpcDialOption)
+
+}
+
+func WithVolumeServerClient(streamingMode bool, volumeServer ServerAddress, grpcDialOption grpc.DialOption, fn func(client volume_server_pb.VolumeServerClient) error) error {
+	return WithGrpcClient(context.Background(), streamingMode, 0, func(grpcConnection *grpc.ClientConn) error {
+		client := volume_server_pb.NewVolumeServerClient(grpcConnection)
+		return fn(client)
+	}, volumeServer.ToGrpcAddress(), false, grpcDialOption)
+
+}
+
+func WithOneOfGrpcMasterClients(streamingMode bool, masterGrpcAddresses map[string]ServerAddress, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) (err error) {
+
+	for _, masterGrpcAddress := range masterGrpcAddresses {
+		err = WithGrpcClient(context.Background(), streamingMode, 0, func(grpcConnection *grpc.ClientConn) error {
+			client := master_pb.NewSeaweedClient(grpcConnection)
+			return fn(client)
+		}, masterGrpcAddress.ToGrpcAddress(), false, grpcDialOption)
+		if err == nil {
+			return nil
+		}
+	}
+
+	return err
+}
+
+func WithBrokerGrpcClient(streamingMode bool, brokerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client mq_pb.SeaweedMessagingClient) error) error {
+
+	return WithGrpcClient(context.Background(), streamingMode, 0, func(grpcConnection *grpc.ClientConn) error {
+		client := mq_pb.NewSeaweedMessagingClient(grpcConnection)
+		return fn(client)
+	}, brokerGrpcAddress, false, grpcDialOption)
+
+}
+
+func WithFilerClient(streamingMode bool, signature int32, filer ServerAddress, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {
+
+	return WithGrpcFilerClient(streamingMode, signature, filer, grpcDialOption, fn)
+
+}
+
+func WithGrpcFilerClient(streamingMode bool, signature int32, filerAddress ServerAddress, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {
+
+	return WithGrpcClient(context.Background(), streamingMode, signature, func(grpcConnection *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+		return fn(client)
+	}, filerAddress.ToGrpcAddress(), false, grpcDialOption)
+
+}
+
+func WithOneOfGrpcFilerClients(streamingMode bool, filerAddresses []ServerAddress, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) (err error) {
+
+	for _, filerAddress := range filerAddresses {
+		err = WithGrpcClient(context.Background(), streamingMode, 0, func(grpcConnection *grpc.ClientConn) error {
+			client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+			return fn(client)
+		}, filerAddress.ToGrpcAddress(), false, grpcDialOption)
+		if err == nil {
+			return nil
+		}
+	}
+
+	return err
+}

weed/pb/grpc_client_server_test.go

@@ -0,0 +1,208 @@
+package pb
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// TestShouldInvalidateConnection_MarshalErrorIsPerRequest ensures that a
+// client-side proto marshal failure does NOT cause the shared cached
+// ClientConn to be torn down. Tearing it down would cancel every other
+// in-flight RPC (seaweedfs#9139: one file with invalid-UTF-8 bytes triggered
+// an avalanche of "connection is closing" errors on unrelated operations).
+func TestShouldInvalidateConnection_MarshalErrorIsPerRequest(t *testing.T) {
+	// Reproduces the exact error gRPC returns when a string field in the
+	// outgoing request contains invalid UTF-8 bytes.
+	marshalErr := status.Error(codes.Internal,
+		"grpc: error while marshaling: string field contains invalid UTF-8")
+	if shouldInvalidateConnection(context.Background(), marshalErr) {
+		t.Fatalf("client-side marshal error must not invalidate the shared connection")
+	}
+
+	// Same error wrapped with fmt.Errorf (common when callers add context).
+	wrapped := fmt.Errorf("upload data: %w", marshalErr)
+	if shouldInvalidateConnection(context.Background(), wrapped) {
+		t.Fatalf("wrapped marshal error must not invalidate the shared connection")
+	}
+}
+
+// TestShouldInvalidateConnection_CallerContextExpiryIsPerRequest ensures that a
+// Canceled/DeadlineExceeded caused by the caller's own context expiring does NOT
+// tear down the shared cached ClientConn. Doing so would cancel every other
+// in-flight RPC on it with "the client connection is closing" — the cascade
+// that turned one slow chunk assign into a flood of failures during a
+// high-concurrency upload (seaweedfs#9765).
+func TestShouldInvalidateConnection_CallerContextExpiryIsPerRequest(t *testing.T) {
+	expired, cancel := context.WithCancel(context.Background())
+	cancel()
+	for _, code := range []codes.Code{codes.Canceled, codes.DeadlineExceeded} {
+		err := status.Error(code, "context expired")
+		if shouldInvalidateConnection(expired, err) {
+			t.Fatalf("%v with an expired caller context must not invalidate the shared connection", code)
+		}
+	}
+}
+
+// TestShouldInvalidateConnection_StaleChannelStillInvalidates ensures the
+// carve-out above is gated on the RPC's context: a Canceled/DeadlineExceeded
+// while that context is still live is the genuine stale-channel signal (e.g. a
+// peer restart behind a k8s Service VIP) and must still invalidate so the next
+// attempt dials fresh.
+func TestShouldInvalidateConnection_StaleChannelStillInvalidates(t *testing.T) {
+	live, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	for _, code := range []codes.Code{codes.Canceled, codes.DeadlineExceeded} {
+		err := status.Error(code, "stale channel")
+		if !shouldInvalidateConnection(live, err) {
+			t.Fatalf("%v with a live attempt context must still invalidate the connection", code)
+		}
+	}
+}
+
+// TestShouldInvalidateConnection_NonCancellableContextIsNoEvidence covers
+// seaweedfs#10947. Background/TODO never expires, so Err() stays nil forever and
+// the guard above answered "stale channel" for every caller that has no deadline
+// to honor — which is almost all of them, the S3 gateway included. One client
+// abandoning a request then closed the shared filer channel and every concurrent
+// multipart part died with "the client connection is closing".
+func TestShouldInvalidateConnection_NonCancellableContextIsNoEvidence(t *testing.T) {
+	for _, ctx := range []context.Context{context.Background(), context.TODO(), nil} {
+		for _, code := range []codes.Code{codes.Canceled, codes.DeadlineExceeded} {
+			err := status.Error(code, "context canceled")
+			if shouldInvalidateConnection(ctx, err) {
+				t.Fatalf("%v must not invalidate the shared connection on a non-cancellable context", code)
+			}
+		}
+	}
+}
+
+// TestShouldInvalidateConnection_ClientConnClosingIsNotStale ensures a caller
+// whose RPC was rejected because this process is already closing the shared
+// ClientConn does not close it again. gRPC raises ErrClientConnClosing locally,
+// so it is only ever the footprint of another goroutine's teardown - reading it
+// as a stale-channel signal lets one teardown re-arm itself across the herd of
+// callers it just cancelled.
+func TestShouldInvalidateConnection_ClientConnClosingIsNotStale(t *testing.T) {
+	live, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	for _, err := range []error{grpc.ErrClientConnClosing, fmt.Errorf("assign volume: %w", grpc.ErrClientConnClosing)} {
+		if shouldInvalidateConnection(live, err) {
+			t.Fatalf("%v must not invalidate the shared connection", err)
+		}
+	}
+}
+
+// TestShouldInvalidateConnection_ResourceExhaustedIsPerRequest ensures the
+// master's growth-in-progress assign shed (codes.ResourceExhausted) does NOT
+// tear down the shared cached ClientConn. The shed fires per-request across a
+// herd of concurrent assigns; invalidating on it would cancel every other
+// in-flight assign with "the client connection is closing" — the cascade in
+// seaweedfs#10118. It is retried by the caller without touching the channel.
+func TestShouldInvalidateConnection_ResourceExhaustedIsPerRequest(t *testing.T) {
+	shed := status.Error(codes.ResourceExhausted, "no writable volumes for x, volume growth in progress")
+	if shouldInvalidateConnection(context.Background(), shed) {
+		t.Fatalf("ResourceExhausted shed must not invalidate the shared connection")
+	}
+	if shouldInvalidateConnection(nil, shed) {
+		t.Fatalf("ResourceExhausted shed must not invalidate the shared connection (nil ctx)")
+	}
+}
+
+// TestShouldInvalidateConnection_GenuineInternalStillInvalidates ensures the
+// marshal-error carve-out does not swallow real server-side Internal errors,
+// which previously caused — and should continue to cause — connection
+// invalidation.
+func TestShouldInvalidateConnection_GenuineInternalStillInvalidates(t *testing.T) {
+	serverInternal := status.Error(codes.Internal, "stream terminated by RST_STREAM with code 2")
+	if !shouldInvalidateConnection(context.Background(), serverInternal) {
+		t.Fatalf("genuine server-side Internal must still invalidate the connection")
+	}
+}
+
+// TestShouldInvalidateConnection_TransportErrorsStillInvalidate is a
+// regression guard for the string-matching fallback path (e.g. a raw
+// "connection refused" from net.Dial that never acquired a gRPC status).
+func TestShouldInvalidateConnection_TransportErrorsStillInvalidate(t *testing.T) {
+	for _, msg := range []string{
+		"rpc error: code = Unavailable desc = transport is closing",
+		"dial tcp: connection refused",
+		"read: connection reset by peer",
+	} {
+		if !shouldInvalidateConnection(context.Background(), fmt.Errorf("%s", msg)) {
+			t.Fatalf("transport error %q must still invalidate", msg)
+		}
+	}
+}
+
+// TestIsClientSideMarshalError_RequiresGrpcStatus ensures the carve-out is
+// type-based (via errors.As on the grpc status interface), not a naive
+// string match against arbitrary errors that happen to mention marshaling.
+// A plain errors.New(...) with the same prefix must NOT be treated as a
+// per-request marshal error — we have no evidence the connection is healthy.
+func TestIsClientSideMarshalError_RequiresGrpcStatus(t *testing.T) {
+	impostor := fmt.Errorf("grpc: error while marshaling: synthetic non-status error")
+	if isClientSideMarshalError(impostor) {
+		t.Fatalf("plain error must not match the marshal-error carve-out")
+	}
+}
+
+// TestResolveLocalGrpcSocket_RemotePortCollision is a regression test for
+// issue #9254. A `weed server` process registers a Unix socket for its
+// in-process volume server on host A. A standalone `weed volume` on host B
+// happens to use the same gRPC port. Dials from the master to host B must
+// continue out over TCP — they must NOT be hijacked into host A's local
+// socket on the basis of port match alone.
+func TestResolveLocalGrpcSocket_RemotePortCollision(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("Unix-socket routing is disabled on Windows (#9430)")
+	}
+	// Snapshot and restore global state so the test does not leak into others.
+	localGrpcSocketsLock.Lock()
+	prevSockets := localGrpcSockets
+	prevHosts := localGrpcHosts
+	localGrpcSockets = make(map[int]string)
+	localGrpcHosts = make(map[int]map[string]struct{})
+	localGrpcSocketsLock.Unlock()
+	t.Cleanup(func() {
+		localGrpcSocketsLock.Lock()
+		localGrpcSockets = prevSockets
+		localGrpcHosts = prevHosts
+		localGrpcSocketsLock.Unlock()
+	})
+
+	const localHost = "10.0.0.2"
+	const remoteHost = "10.0.0.3"
+	const collidingPort = 17334
+	const socketPath = "/tmp/seaweedfs-volume-grpc-17334.sock"
+
+	RegisterLocalGrpcSocket(localHost, collidingPort, socketPath)
+
+	cases := []struct {
+		name    string
+		address string
+		want    string
+	}{
+		{"local advertised host routes to socket", localHost + ":17334", socketPath},
+		{"loopback v4 routes to socket", "127.0.0.1:17334", socketPath},
+		{"localhost routes to socket", "localhost:17334", socketPath},
+		{"loopback v6 routes to socket", "[::1]:17334", socketPath},
+		{"empty host (bare port) routes to socket", ":17334", socketPath},
+		{"remote host with same port stays on TCP", remoteHost + ":17334", ""},
+		{"unrelated host with same port stays on TCP", "192.168.1.5:17334", ""},
+		{"unregistered port stays on TCP", localHost + ":17335", ""},
+		{"malformed address stays on TCP", "not-a-host-port", ""},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := resolveLocalGrpcSocket(tc.address); got != tc.want {
+				t.Fatalf("resolveLocalGrpcSocket(%q) = %q, want %q", tc.address, got, tc.want)
+			}
+		})
+	}
+}

weed/pb/iam.proto

@@ -0,0 +1,340 @@
+syntax = "proto3";
+
+package iam_pb;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb";
+option java_package = "seaweedfs.client";
+option java_outer_classname = "IamProto";
+
+//////////////////////////////////////////////////
+
+service SeaweedIdentityAccessManagement {
+    // Configuration Management
+    rpc GetConfiguration (GetConfigurationRequest) returns (GetConfigurationResponse);
+    rpc PutConfiguration (PutConfigurationRequest) returns (PutConfigurationResponse);
+    
+    // User Management
+    rpc CreateUser (CreateUserRequest) returns (CreateUserResponse);
+    rpc GetUser (GetUserRequest) returns (GetUserResponse);
+    rpc UpdateUser (UpdateUserRequest) returns (UpdateUserResponse);
+    rpc DeleteUser (DeleteUserRequest) returns (DeleteUserResponse);
+    rpc ListUsers (ListUsersRequest) returns (ListUsersResponse);
+    
+    // Access Key Management
+    rpc CreateAccessKey (CreateAccessKeyRequest) returns (CreateAccessKeyResponse);
+    rpc DeleteAccessKey (DeleteAccessKeyRequest) returns (DeleteAccessKeyResponse);
+    rpc GetUserByAccessKey (GetUserByAccessKeyRequest) returns (GetUserByAccessKeyResponse);
+    
+    // Policy Management
+    rpc PutPolicy (PutPolicyRequest) returns (PutPolicyResponse);
+    rpc GetPolicy (GetPolicyRequest) returns (GetPolicyResponse);
+    rpc ListPolicies (ListPoliciesRequest) returns (ListPoliciesResponse);
+    rpc DeletePolicy (DeletePolicyRequest) returns (DeletePolicyResponse);
+
+    // Service Account Management
+    rpc CreateServiceAccount (CreateServiceAccountRequest) returns (CreateServiceAccountResponse);
+    rpc UpdateServiceAccount (UpdateServiceAccountRequest) returns (UpdateServiceAccountResponse);
+    rpc DeleteServiceAccount (DeleteServiceAccountRequest) returns (DeleteServiceAccountResponse);
+    rpc GetServiceAccount (GetServiceAccountRequest) returns (GetServiceAccountResponse);
+    rpc ListServiceAccounts (ListServiceAccountsRequest) returns (ListServiceAccountsResponse);
+    rpc GetServiceAccountByAccessKey (GetServiceAccountByAccessKeyRequest) returns (GetServiceAccountByAccessKeyResponse);
+}
+
+//////////////////////////////////////////////////
+// Configuration Management Messages
+
+message GetConfigurationRequest {
+}
+
+message GetConfigurationResponse {
+    S3ApiConfiguration configuration = 1;
+}
+
+message PutConfigurationRequest {
+    S3ApiConfiguration configuration = 1;
+}
+
+message PutConfigurationResponse {
+}
+
+//////////////////////////////////////////////////
+// User Management Messages
+
+message CreateUserRequest {
+    Identity identity = 1;
+}
+
+message CreateUserResponse {
+}
+
+message GetUserRequest {
+    string username = 1;
+}
+
+message GetUserResponse {
+    Identity identity = 1;
+}
+
+message UpdateUserRequest {
+    string username = 1;
+    Identity identity = 2;
+}
+
+message UpdateUserResponse {
+}
+
+message DeleteUserRequest {
+    string username = 1;
+}
+
+message DeleteUserResponse {
+}
+
+message ListUsersRequest {
+}
+
+message ListUsersResponse {
+    repeated string usernames = 1;
+}
+
+//////////////////////////////////////////////////
+// Access Key Management Messages
+
+message CreateAccessKeyRequest {
+    string username = 1;
+    Credential credential = 2;
+}
+
+message CreateAccessKeyResponse {
+}
+
+message DeleteAccessKeyRequest {
+    string username = 1;
+    string access_key = 2;
+}
+
+message DeleteAccessKeyResponse {
+}
+
+message GetUserByAccessKeyRequest {
+    string access_key = 1;
+}
+
+message GetUserByAccessKeyResponse {
+    Identity identity = 1;
+}
+
+message ListAccessKeysRequest {
+    string username = 1;
+}
+
+message ListAccessKeysResponse {
+    repeated Credential access_keys = 1;
+}
+
+// User Policy Management Messages
+message PutUserPolicyRequest {
+    string username = 1;
+    string policy_name = 2;
+    string policy_document = 3;
+}
+
+message PutUserPolicyResponse {
+}
+
+message GetUserPolicyRequest {
+    string username = 1;
+    string policy_name = 2;
+}
+
+message GetUserPolicyResponse {
+    string username = 1;
+    string policy_name = 2;
+    string policy_document = 3;
+}
+
+message DeleteUserPolicyRequest {
+    string username = 1;
+    string policy_name = 2;
+}
+
+message DeleteUserPolicyResponse {
+}
+
+//////////////////////////////////////////////////
+
+message S3ApiConfiguration {
+    repeated Identity identities = 1;
+    repeated Account accounts = 2;
+    repeated ServiceAccount service_accounts = 3;
+    repeated Policy policies = 4;
+    repeated Group groups = 5;
+}
+
+message Group {
+    string name = 1;
+    repeated string members = 2;       // usernames
+    repeated string policy_names = 3;  // attached managed policy names
+    bool disabled = 4;
+}
+
+message Identity {
+    string name = 1;
+    repeated Credential credentials = 2;
+    repeated string actions = 3;
+    Account account = 4;
+    bool disabled = 5;  // User status: false = enabled (default), true = disabled
+    repeated string service_account_ids = 6;  // IDs of service accounts owned by this user
+    repeated string policy_names = 7;
+    bool is_static = 8;  // Loaded from static config file (read-only, not editable via API)
+    repeated UserTag tags = 9;
+}
+
+message UserTag {
+    string key = 1;
+    string value = 2;
+}
+
+message Credential {
+    string access_key = 1;
+    string secret_key = 2;
+    string status = 3;  // Access key status: "Active" or "Inactive"
+}
+
+message Account {
+    string id = 1;
+    string display_name = 2;
+    string email_address = 3;
+}
+
+// ServiceAccount represents a service account - special credentials for applications.
+// Service accounts are linked to a parent user and can have restricted permissions.
+message ServiceAccount {
+    string id = 1;                     // Unique identifier (e.g., "sa-xxxxx")
+    string parent_user = 2;            // Parent identity name
+    string description = 3;            // Optional description
+    Credential credential = 4;         // Access key/secret for this service account
+    repeated string actions = 5;       // Allowed actions (subset of parent)
+    int64 expiration = 6;              // Unix timestamp, 0 = no expiration
+    bool disabled = 7;                 // Status: false = enabled (default)
+    int64 created_at = 8;              // Creation timestamp
+    string created_by = 9;             // Who created this service account
+}
+
+message PutPolicyRequest {
+    string name = 1;
+    string content = 2;
+}
+
+message PutPolicyResponse {
+}
+
+message GetPolicyRequest {
+    string name = 1;
+}
+
+message GetPolicyResponse {
+    string name = 1;
+    string content = 2;
+}
+
+message ListPoliciesRequest {
+}
+
+message ListPoliciesResponse {
+    repeated Policy policies = 1;
+}
+
+message DeletePolicyRequest {
+    string name = 1;
+}
+
+message DeletePolicyResponse {
+}
+
+message Policy {
+    string name = 1;
+    string content = 2; // JSON content of the policy
+}
+
+//////////////////////////////////////////////////
+// Service Account Messages
+
+message CreateServiceAccountRequest {
+    ServiceAccount service_account = 1;
+}
+
+message CreateServiceAccountResponse {
+}
+
+message UpdateServiceAccountRequest {
+    string id = 1;
+    ServiceAccount service_account = 2;
+}
+
+message UpdateServiceAccountResponse {
+}
+
+message DeleteServiceAccountRequest {
+    string id = 1;
+}
+
+message DeleteServiceAccountResponse {
+}
+
+message GetServiceAccountRequest {
+    string id = 1;
+}
+
+message GetServiceAccountResponse {
+    ServiceAccount service_account = 1;
+}
+
+message ListServiceAccountsRequest {
+}
+
+message ListServiceAccountsResponse {
+    repeated ServiceAccount service_accounts = 1;
+}
+
+message GetServiceAccountByAccessKeyRequest {
+    string access_key = 1;
+}
+
+message GetServiceAccountByAccessKeyResponse {
+    ServiceAccount service_account = 1;
+}
+
+
+//////////////////////////////////////////////////
+// S3 IAM Cache Management
+// Designed for unidirectional propagation from Filer to S3 Servers
+
+
+message PutIdentityRequest {
+    Identity identity = 1;
+}
+
+message PutIdentityResponse {
+}
+
+message RemoveIdentityRequest {
+    string username = 1;
+}
+
+message RemoveIdentityResponse {
+}
+
+message PutGroupRequest {
+    Group group = 1;
+}
+
+message PutGroupResponse {
+}
+
+message RemoveGroupRequest {
+    string group_name = 1;
+}
+
+message RemoveGroupResponse {
+}

weed/pb/iam_pb/iam_grpc.pb.go

@@ -0,0 +1,854 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.5.1
+// - protoc             v6.33.4
+// source: iam.proto
+
+package iam_pb
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.64.0 or later.
+const _ = grpc.SupportPackageIsVersion9
+
+const (
+	SeaweedIdentityAccessManagement_GetConfiguration_FullMethodName             = "/iam_pb.SeaweedIdentityAccessManagement/GetConfiguration"
+	SeaweedIdentityAccessManagement_PutConfiguration_FullMethodName             = "/iam_pb.SeaweedIdentityAccessManagement/PutConfiguration"
+	SeaweedIdentityAccessManagement_CreateUser_FullMethodName                   = "/iam_pb.SeaweedIdentityAccessManagement/CreateUser"
+	SeaweedIdentityAccessManagement_GetUser_FullMethodName                      = "/iam_pb.SeaweedIdentityAccessManagement/GetUser"
+	SeaweedIdentityAccessManagement_UpdateUser_FullMethodName                   = "/iam_pb.SeaweedIdentityAccessManagement/UpdateUser"
+	SeaweedIdentityAccessManagement_DeleteUser_FullMethodName                   = "/iam_pb.SeaweedIdentityAccessManagement/DeleteUser"
+	SeaweedIdentityAccessManagement_ListUsers_FullMethodName                    = "/iam_pb.SeaweedIdentityAccessManagement/ListUsers"
+	SeaweedIdentityAccessManagement_CreateAccessKey_FullMethodName              = "/iam_pb.SeaweedIdentityAccessManagement/CreateAccessKey"
+	SeaweedIdentityAccessManagement_DeleteAccessKey_FullMethodName              = "/iam_pb.SeaweedIdentityAccessManagement/DeleteAccessKey"
+	SeaweedIdentityAccessManagement_GetUserByAccessKey_FullMethodName           = "/iam_pb.SeaweedIdentityAccessManagement/GetUserByAccessKey"
+	SeaweedIdentityAccessManagement_PutPolicy_FullMethodName                    = "/iam_pb.SeaweedIdentityAccessManagement/PutPolicy"
+	SeaweedIdentityAccessManagement_GetPolicy_FullMethodName                    = "/iam_pb.SeaweedIdentityAccessManagement/GetPolicy"
+	SeaweedIdentityAccessManagement_ListPolicies_FullMethodName                 = "/iam_pb.SeaweedIdentityAccessManagement/ListPolicies"
+	SeaweedIdentityAccessManagement_DeletePolicy_FullMethodName                 = "/iam_pb.SeaweedIdentityAccessManagement/DeletePolicy"
+	SeaweedIdentityAccessManagement_CreateServiceAccount_FullMethodName         = "/iam_pb.SeaweedIdentityAccessManagement/CreateServiceAccount"
+	SeaweedIdentityAccessManagement_UpdateServiceAccount_FullMethodName         = "/iam_pb.SeaweedIdentityAccessManagement/UpdateServiceAccount"
+	SeaweedIdentityAccessManagement_DeleteServiceAccount_FullMethodName         = "/iam_pb.SeaweedIdentityAccessManagement/DeleteServiceAccount"
+	SeaweedIdentityAccessManagement_GetServiceAccount_FullMethodName            = "/iam_pb.SeaweedIdentityAccessManagement/GetServiceAccount"
+	SeaweedIdentityAccessManagement_ListServiceAccounts_FullMethodName          = "/iam_pb.SeaweedIdentityAccessManagement/ListServiceAccounts"
+	SeaweedIdentityAccessManagement_GetServiceAccountByAccessKey_FullMethodName = "/iam_pb.SeaweedIdentityAccessManagement/GetServiceAccountByAccessKey"
+)
+
+// SeaweedIdentityAccessManagementClient is the client API for SeaweedIdentityAccessManagement service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type SeaweedIdentityAccessManagementClient interface {
+	// Configuration Management
+	GetConfiguration(ctx context.Context, in *GetConfigurationRequest, opts ...grpc.CallOption) (*GetConfigurationResponse, error)
+	PutConfiguration(ctx context.Context, in *PutConfigurationRequest, opts ...grpc.CallOption) (*PutConfigurationResponse, error)
+	// User Management
+	CreateUser(ctx context.Context, in *CreateUserRequest, opts ...grpc.CallOption) (*CreateUserResponse, error)
+	GetUser(ctx context.Context, in *GetUserRequest, opts ...grpc.CallOption) (*GetUserResponse, error)
+	UpdateUser(ctx context.Context, in *UpdateUserRequest, opts ...grpc.CallOption) (*UpdateUserResponse, error)
+	DeleteUser(ctx context.Context, in *DeleteUserRequest, opts ...grpc.CallOption) (*DeleteUserResponse, error)
+	ListUsers(ctx context.Context, in *ListUsersRequest, opts ...grpc.CallOption) (*ListUsersResponse, error)
+	// Access Key Management
+	CreateAccessKey(ctx context.Context, in *CreateAccessKeyRequest, opts ...grpc.CallOption) (*CreateAccessKeyResponse, error)
+	DeleteAccessKey(ctx context.Context, in *DeleteAccessKeyRequest, opts ...grpc.CallOption) (*DeleteAccessKeyResponse, error)
+	GetUserByAccessKey(ctx context.Context, in *GetUserByAccessKeyRequest, opts ...grpc.CallOption) (*GetUserByAccessKeyResponse, error)
+	// Policy Management
+	PutPolicy(ctx context.Context, in *PutPolicyRequest, opts ...grpc.CallOption) (*PutPolicyResponse, error)
+	GetPolicy(ctx context.Context, in *GetPolicyRequest, opts ...grpc.CallOption) (*GetPolicyResponse, error)
+	ListPolicies(ctx context.Context, in *ListPoliciesRequest, opts ...grpc.CallOption) (*ListPoliciesResponse, error)
+	DeletePolicy(ctx context.Context, in *DeletePolicyRequest, opts ...grpc.CallOption) (*DeletePolicyResponse, error)
+	// Service Account Management
+	CreateServiceAccount(ctx context.Context, in *CreateServiceAccountRequest, opts ...grpc.CallOption) (*CreateServiceAccountResponse, error)
+	UpdateServiceAccount(ctx context.Context, in *UpdateServiceAccountRequest, opts ...grpc.CallOption) (*UpdateServiceAccountResponse, error)
+	DeleteServiceAccount(ctx context.Context, in *DeleteServiceAccountRequest, opts ...grpc.CallOption) (*DeleteServiceAccountResponse, error)
+	GetServiceAccount(ctx context.Context, in *GetServiceAccountRequest, opts ...grpc.CallOption) (*GetServiceAccountResponse, error)
+	ListServiceAccounts(ctx context.Context, in *ListServiceAccountsRequest, opts ...grpc.CallOption) (*ListServiceAccountsResponse, error)
+	GetServiceAccountByAccessKey(ctx context.Context, in *GetServiceAccountByAccessKeyRequest, opts ...grpc.CallOption) (*GetServiceAccountByAccessKeyResponse, error)
+}
+
+type seaweedIdentityAccessManagementClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewSeaweedIdentityAccessManagementClient(cc grpc.ClientConnInterface) SeaweedIdentityAccessManagementClient {
+	return &seaweedIdentityAccessManagementClient{cc}
+}
+
+func (c *seaweedIdentityAccessManagementClient) GetConfiguration(ctx context.Context, in *GetConfigurationRequest, opts ...grpc.CallOption) (*GetConfigurationResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(GetConfigurationResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_GetConfiguration_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) PutConfiguration(ctx context.Context, in *PutConfigurationRequest, opts ...grpc.CallOption) (*PutConfigurationResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(PutConfigurationResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_PutConfiguration_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) CreateUser(ctx context.Context, in *CreateUserRequest, opts ...grpc.CallOption) (*CreateUserResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(CreateUserResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_CreateUser_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) GetUser(ctx context.Context, in *GetUserRequest, opts ...grpc.CallOption) (*GetUserResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(GetUserResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_GetUser_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) UpdateUser(ctx context.Context, in *UpdateUserRequest, opts ...grpc.CallOption) (*UpdateUserResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(UpdateUserResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_UpdateUser_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) DeleteUser(ctx context.Context, in *DeleteUserRequest, opts ...grpc.CallOption) (*DeleteUserResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(DeleteUserResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_DeleteUser_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) ListUsers(ctx context.Context, in *ListUsersRequest, opts ...grpc.CallOption) (*ListUsersResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ListUsersResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_ListUsers_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) CreateAccessKey(ctx context.Context, in *CreateAccessKeyRequest, opts ...grpc.CallOption) (*CreateAccessKeyResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(CreateAccessKeyResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_CreateAccessKey_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) DeleteAccessKey(ctx context.Context, in *DeleteAccessKeyRequest, opts ...grpc.CallOption) (*DeleteAccessKeyResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(DeleteAccessKeyResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_DeleteAccessKey_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) GetUserByAccessKey(ctx context.Context, in *GetUserByAccessKeyRequest, opts ...grpc.CallOption) (*GetUserByAccessKeyResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(GetUserByAccessKeyResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_GetUserByAccessKey_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) PutPolicy(ctx context.Context, in *PutPolicyRequest, opts ...grpc.CallOption) (*PutPolicyResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(PutPolicyResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_PutPolicy_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) GetPolicy(ctx context.Context, in *GetPolicyRequest, opts ...grpc.CallOption) (*GetPolicyResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(GetPolicyResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_GetPolicy_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) ListPolicies(ctx context.Context, in *ListPoliciesRequest, opts ...grpc.CallOption) (*ListPoliciesResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ListPoliciesResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_ListPolicies_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) DeletePolicy(ctx context.Context, in *DeletePolicyRequest, opts ...grpc.CallOption) (*DeletePolicyResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(DeletePolicyResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_DeletePolicy_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) CreateServiceAccount(ctx context.Context, in *CreateServiceAccountRequest, opts ...grpc.CallOption) (*CreateServiceAccountResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(CreateServiceAccountResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_CreateServiceAccount_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) UpdateServiceAccount(ctx context.Context, in *UpdateServiceAccountRequest, opts ...grpc.CallOption) (*UpdateServiceAccountResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(UpdateServiceAccountResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_UpdateServiceAccount_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) DeleteServiceAccount(ctx context.Context, in *DeleteServiceAccountRequest, opts ...grpc.CallOption) (*DeleteServiceAccountResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(DeleteServiceAccountResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_DeleteServiceAccount_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) GetServiceAccount(ctx context.Context, in *GetServiceAccountRequest, opts ...grpc.CallOption) (*GetServiceAccountResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(GetServiceAccountResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_GetServiceAccount_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) ListServiceAccounts(ctx context.Context, in *ListServiceAccountsRequest, opts ...grpc.CallOption) (*ListServiceAccountsResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ListServiceAccountsResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_ListServiceAccounts_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedIdentityAccessManagementClient) GetServiceAccountByAccessKey(ctx context.Context, in *GetServiceAccountByAccessKeyRequest, opts ...grpc.CallOption) (*GetServiceAccountByAccessKeyResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(GetServiceAccountByAccessKeyResponse)
+	err := c.cc.Invoke(ctx, SeaweedIdentityAccessManagement_GetServiceAccountByAccessKey_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// SeaweedIdentityAccessManagementServer is the server API for SeaweedIdentityAccessManagement service.
+// All implementations must embed UnimplementedSeaweedIdentityAccessManagementServer
+// for forward compatibility.
+type SeaweedIdentityAccessManagementServer interface {
+	// Configuration Management
+	GetConfiguration(context.Context, *GetConfigurationRequest) (*GetConfigurationResponse, error)
+	PutConfiguration(context.Context, *PutConfigurationRequest) (*PutConfigurationResponse, error)
+	// User Management
+	CreateUser(context.Context, *CreateUserRequest) (*CreateUserResponse, error)
+	GetUser(context.Context, *GetUserRequest) (*GetUserResponse, error)
+	UpdateUser(context.Context, *UpdateUserRequest) (*UpdateUserResponse, error)
+	DeleteUser(context.Context, *DeleteUserRequest) (*DeleteUserResponse, error)
+	ListUsers(context.Context, *ListUsersRequest) (*ListUsersResponse, error)
+	// Access Key Management
+	CreateAccessKey(context.Context, *CreateAccessKeyRequest) (*CreateAccessKeyResponse, error)
+	DeleteAccessKey(context.Context, *DeleteAccessKeyRequest) (*DeleteAccessKeyResponse, error)
+	GetUserByAccessKey(context.Context, *GetUserByAccessKeyRequest) (*GetUserByAccessKeyResponse, error)
+	// Policy Management
+	PutPolicy(context.Context, *PutPolicyRequest) (*PutPolicyResponse, error)
+	GetPolicy(context.Context, *GetPolicyRequest) (*GetPolicyResponse, error)
+	ListPolicies(context.Context, *ListPoliciesRequest) (*ListPoliciesResponse, error)
+	DeletePolicy(context.Context, *DeletePolicyRequest) (*DeletePolicyResponse, error)
+	// Service Account Management
+	CreateServiceAccount(context.Context, *CreateServiceAccountRequest) (*CreateServiceAccountResponse, error)
+	UpdateServiceAccount(context.Context, *UpdateServiceAccountRequest) (*UpdateServiceAccountResponse, error)
+	DeleteServiceAccount(context.Context, *DeleteServiceAccountRequest) (*DeleteServiceAccountResponse, error)
+	GetServiceAccount(context.Context, *GetServiceAccountRequest) (*GetServiceAccountResponse, error)
+	ListServiceAccounts(context.Context, *ListServiceAccountsRequest) (*ListServiceAccountsResponse, error)
+	GetServiceAccountByAccessKey(context.Context, *GetServiceAccountByAccessKeyRequest) (*GetServiceAccountByAccessKeyResponse, error)
+	mustEmbedUnimplementedSeaweedIdentityAccessManagementServer()
+}
+
+// UnimplementedSeaweedIdentityAccessManagementServer must be embedded to have
+// forward compatible implementations.
+//
+// NOTE: this should be embedded by value instead of pointer to avoid a nil
+// pointer dereference when methods are called.
+type UnimplementedSeaweedIdentityAccessManagementServer struct{}
+
+func (UnimplementedSeaweedIdentityAccessManagementServer) GetConfiguration(context.Context, *GetConfigurationRequest) (*GetConfigurationResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetConfiguration not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) PutConfiguration(context.Context, *PutConfigurationRequest) (*PutConfigurationResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method PutConfiguration not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) CreateUser(context.Context, *CreateUserRequest) (*CreateUserResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method CreateUser not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) GetUser(context.Context, *GetUserRequest) (*GetUserResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetUser not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) UpdateUser(context.Context, *UpdateUserRequest) (*UpdateUserResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method UpdateUser not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) DeleteUser(context.Context, *DeleteUserRequest) (*DeleteUserResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method DeleteUser not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) ListUsers(context.Context, *ListUsersRequest) (*ListUsersResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ListUsers not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) CreateAccessKey(context.Context, *CreateAccessKeyRequest) (*CreateAccessKeyResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method CreateAccessKey not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) DeleteAccessKey(context.Context, *DeleteAccessKeyRequest) (*DeleteAccessKeyResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method DeleteAccessKey not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) GetUserByAccessKey(context.Context, *GetUserByAccessKeyRequest) (*GetUserByAccessKeyResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetUserByAccessKey not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) PutPolicy(context.Context, *PutPolicyRequest) (*PutPolicyResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method PutPolicy not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) GetPolicy(context.Context, *GetPolicyRequest) (*GetPolicyResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetPolicy not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) ListPolicies(context.Context, *ListPoliciesRequest) (*ListPoliciesResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ListPolicies not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) DeletePolicy(context.Context, *DeletePolicyRequest) (*DeletePolicyResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method DeletePolicy not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) CreateServiceAccount(context.Context, *CreateServiceAccountRequest) (*CreateServiceAccountResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method CreateServiceAccount not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) UpdateServiceAccount(context.Context, *UpdateServiceAccountRequest) (*UpdateServiceAccountResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method UpdateServiceAccount not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) DeleteServiceAccount(context.Context, *DeleteServiceAccountRequest) (*DeleteServiceAccountResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method DeleteServiceAccount not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) GetServiceAccount(context.Context, *GetServiceAccountRequest) (*GetServiceAccountResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetServiceAccount not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) ListServiceAccounts(context.Context, *ListServiceAccountsRequest) (*ListServiceAccountsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ListServiceAccounts not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) GetServiceAccountByAccessKey(context.Context, *GetServiceAccountByAccessKeyRequest) (*GetServiceAccountByAccessKeyResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetServiceAccountByAccessKey not implemented")
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) mustEmbedUnimplementedSeaweedIdentityAccessManagementServer() {
+}
+func (UnimplementedSeaweedIdentityAccessManagementServer) testEmbeddedByValue() {}
+
+// UnsafeSeaweedIdentityAccessManagementServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to SeaweedIdentityAccessManagementServer will
+// result in compilation errors.
+type UnsafeSeaweedIdentityAccessManagementServer interface {
+	mustEmbedUnimplementedSeaweedIdentityAccessManagementServer()
+}
+
+func RegisterSeaweedIdentityAccessManagementServer(s grpc.ServiceRegistrar, srv SeaweedIdentityAccessManagementServer) {
+	// If the following call pancis, it indicates UnimplementedSeaweedIdentityAccessManagementServer was
+	// embedded by pointer and is nil.  This will cause panics if an
+	// unimplemented method is ever invoked, so we test this at initialization
+	// time to prevent it from happening at runtime later due to I/O.
+	if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
+		t.testEmbeddedByValue()
+	}
+	s.RegisterService(&SeaweedIdentityAccessManagement_ServiceDesc, srv)
+}
+
+func _SeaweedIdentityAccessManagement_GetConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetConfigurationRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).GetConfiguration(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_GetConfiguration_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).GetConfiguration(ctx, req.(*GetConfigurationRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_PutConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(PutConfigurationRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).PutConfiguration(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_PutConfiguration_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).PutConfiguration(ctx, req.(*PutConfigurationRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_CreateUser_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(CreateUserRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).CreateUser(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_CreateUser_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).CreateUser(ctx, req.(*CreateUserRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_GetUser_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetUserRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).GetUser(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_GetUser_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).GetUser(ctx, req.(*GetUserRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_UpdateUser_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(UpdateUserRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).UpdateUser(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_UpdateUser_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).UpdateUser(ctx, req.(*UpdateUserRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_DeleteUser_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(DeleteUserRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).DeleteUser(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_DeleteUser_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).DeleteUser(ctx, req.(*DeleteUserRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_ListUsers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ListUsersRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).ListUsers(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_ListUsers_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).ListUsers(ctx, req.(*ListUsersRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_CreateAccessKey_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(CreateAccessKeyRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).CreateAccessKey(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_CreateAccessKey_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).CreateAccessKey(ctx, req.(*CreateAccessKeyRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_DeleteAccessKey_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(DeleteAccessKeyRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).DeleteAccessKey(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_DeleteAccessKey_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).DeleteAccessKey(ctx, req.(*DeleteAccessKeyRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_GetUserByAccessKey_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetUserByAccessKeyRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).GetUserByAccessKey(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_GetUserByAccessKey_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).GetUserByAccessKey(ctx, req.(*GetUserByAccessKeyRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_PutPolicy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(PutPolicyRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).PutPolicy(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_PutPolicy_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).PutPolicy(ctx, req.(*PutPolicyRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_GetPolicy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetPolicyRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).GetPolicy(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_GetPolicy_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).GetPolicy(ctx, req.(*GetPolicyRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_ListPolicies_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ListPoliciesRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).ListPolicies(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_ListPolicies_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).ListPolicies(ctx, req.(*ListPoliciesRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_DeletePolicy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(DeletePolicyRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).DeletePolicy(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_DeletePolicy_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).DeletePolicy(ctx, req.(*DeletePolicyRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_CreateServiceAccount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(CreateServiceAccountRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).CreateServiceAccount(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_CreateServiceAccount_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).CreateServiceAccount(ctx, req.(*CreateServiceAccountRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_UpdateServiceAccount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(UpdateServiceAccountRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).UpdateServiceAccount(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_UpdateServiceAccount_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).UpdateServiceAccount(ctx, req.(*UpdateServiceAccountRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_DeleteServiceAccount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(DeleteServiceAccountRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).DeleteServiceAccount(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_DeleteServiceAccount_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).DeleteServiceAccount(ctx, req.(*DeleteServiceAccountRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_GetServiceAccount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetServiceAccountRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).GetServiceAccount(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_GetServiceAccount_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).GetServiceAccount(ctx, req.(*GetServiceAccountRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_ListServiceAccounts_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ListServiceAccountsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).ListServiceAccounts(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_ListServiceAccounts_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).ListServiceAccounts(ctx, req.(*ListServiceAccountsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedIdentityAccessManagement_GetServiceAccountByAccessKey_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetServiceAccountByAccessKeyRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedIdentityAccessManagementServer).GetServiceAccountByAccessKey(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedIdentityAccessManagement_GetServiceAccountByAccessKey_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedIdentityAccessManagementServer).GetServiceAccountByAccessKey(ctx, req.(*GetServiceAccountByAccessKeyRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// SeaweedIdentityAccessManagement_ServiceDesc is the grpc.ServiceDesc for SeaweedIdentityAccessManagement service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var SeaweedIdentityAccessManagement_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "iam_pb.SeaweedIdentityAccessManagement",
+	HandlerType: (*SeaweedIdentityAccessManagementServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "GetConfiguration",
+			Handler:    _SeaweedIdentityAccessManagement_GetConfiguration_Handler,
+		},
+		{
+			MethodName: "PutConfiguration",
+			Handler:    _SeaweedIdentityAccessManagement_PutConfiguration_Handler,
+		},
+		{
+			MethodName: "CreateUser",
+			Handler:    _SeaweedIdentityAccessManagement_CreateUser_Handler,
+		},
+		{
+			MethodName: "GetUser",
+			Handler:    _SeaweedIdentityAccessManagement_GetUser_Handler,
+		},
+		{
+			MethodName: "UpdateUser",
+			Handler:    _SeaweedIdentityAccessManagement_UpdateUser_Handler,
+		},
+		{
+			MethodName: "DeleteUser",
+			Handler:    _SeaweedIdentityAccessManagement_DeleteUser_Handler,
+		},
+		{
+			MethodName: "ListUsers",
+			Handler:    _SeaweedIdentityAccessManagement_ListUsers_Handler,
+		},
+		{
+			MethodName: "CreateAccessKey",
+			Handler:    _SeaweedIdentityAccessManagement_CreateAccessKey_Handler,
+		},
+		{
+			MethodName: "DeleteAccessKey",
+			Handler:    _SeaweedIdentityAccessManagement_DeleteAccessKey_Handler,
+		},
+		{
+			MethodName: "GetUserByAccessKey",
+			Handler:    _SeaweedIdentityAccessManagement_GetUserByAccessKey_Handler,
+		},
+		{
+			MethodName: "PutPolicy",
+			Handler:    _SeaweedIdentityAccessManagement_PutPolicy_Handler,
+		},
+		{
+			MethodName: "GetPolicy",
+			Handler:    _SeaweedIdentityAccessManagement_GetPolicy_Handler,
+		},
+		{
+			MethodName: "ListPolicies",
+			Handler:    _SeaweedIdentityAccessManagement_ListPolicies_Handler,
+		},
+		{
+			MethodName: "DeletePolicy",
+			Handler:    _SeaweedIdentityAccessManagement_DeletePolicy_Handler,
+		},
+		{
+			MethodName: "CreateServiceAccount",
+			Handler:    _SeaweedIdentityAccessManagement_CreateServiceAccount_Handler,
+		},
+		{
+			MethodName: "UpdateServiceAccount",
+			Handler:    _SeaweedIdentityAccessManagement_UpdateServiceAccount_Handler,
+		},
+		{
+			MethodName: "DeleteServiceAccount",
+			Handler:    _SeaweedIdentityAccessManagement_DeleteServiceAccount_Handler,
+		},
+		{
+			MethodName: "GetServiceAccount",
+			Handler:    _SeaweedIdentityAccessManagement_GetServiceAccount_Handler,
+		},
+		{
+			MethodName: "ListServiceAccounts",
+			Handler:    _SeaweedIdentityAccessManagement_ListServiceAccounts_Handler,
+		},
+		{
+			MethodName: "GetServiceAccountByAccessKey",
+			Handler:    _SeaweedIdentityAccessManagement_GetServiceAccountByAccessKey_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "iam.proto",
+}

weed/pb/local_socket_buffers.go

@@ -0,0 +1,39 @@
+//go:build !windows
+
+package pb
+
+import (
+	"net"
+	"syscall"
+)
+
+// Unix socket buffers default small and never autotune (208KB on Linux, 8KB on
+// macOS). Once a buffer cannot absorb what gRPC's loopyWriter emits for the
+// in-flight streams the writer blocks, both peers latch grpc-go's control-buffer
+// throttle, and the connection deadlocks for good. Best effort: a value the OS
+// clamps down only lowers the concurrency this survives.
+const localSocketBufBytes = 8 << 20
+
+func setLocalSocketBufferFD(fd uintptr) {
+	syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, localSocketBufBytes)
+	syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, localSocketBufBytes)
+}
+
+func setLocalSocketBuffers(_, _ string, c syscall.RawConn) error {
+	return c.Control(setLocalSocketBufferFD)
+}
+
+// applyLocalSocketBuffers tunes an already-established connection. Linux does
+// not carry the listener's buffer sizes onto accepted sockets, so the server
+// half needs setting explicitly or only the dialing side gets the headroom.
+func applyLocalSocketBuffers(c net.Conn) {
+	sc, ok := c.(syscall.Conn)
+	if !ok {
+		return
+	}
+	raw, err := sc.SyscallConn()
+	if err != nil {
+		return
+	}
+	raw.Control(setLocalSocketBufferFD)
+}

weed/pb/local_socket_buffers_windows.go

@@ -0,0 +1,13 @@
+//go:build windows
+
+package pb
+
+import (
+	"net"
+	"syscall"
+)
+
+// Windows never registers local Unix sockets, so there is nothing to tune.
+func setLocalSocketBuffers(_, _ string, _ syscall.RawConn) error { return nil }
+
+func applyLocalSocketBuffers(_ net.Conn) {}

weed/pb/master.proto

@@ -0,0 +1,611 @@
+syntax = "proto3";
+
+package master_pb;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/master_pb";
+
+import "volume_server.proto";
+
+//////////////////////////////////////////////////
+
+service Seaweed {
+  rpc SendHeartbeat (stream Heartbeat) returns (stream HeartbeatResponse) {
+  }
+  rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) {
+  }
+  rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) {
+  }
+  rpc Assign (AssignRequest) returns (AssignResponse) {
+  }
+  rpc StreamAssign (stream AssignRequest) returns (stream AssignResponse) {
+  }
+  rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
+  }
+  rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) {
+  }
+  rpc CollectionDelete (CollectionDeleteRequest) returns (CollectionDeleteResponse) {
+  }
+  rpc VolumeList (VolumeListRequest) returns (VolumeListResponse) {
+  }
+  rpc VolumeListStream (VolumeListRequest) returns (stream VolumeListStreamResponse) {
+  }
+  rpc LookupEcVolume (LookupEcVolumeRequest) returns (LookupEcVolumeResponse) {
+  }
+  rpc VacuumVolume (VacuumVolumeRequest) returns (VacuumVolumeResponse) {
+  }
+  rpc DisableVacuum (DisableVacuumRequest) returns (DisableVacuumResponse) {
+  }
+  rpc EnableVacuum (EnableVacuumRequest) returns (EnableVacuumResponse) {
+  }
+  rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
+  }
+  rpc GetMasterConfiguration (GetMasterConfigurationRequest) returns (GetMasterConfigurationResponse) {
+  }
+  rpc ListClusterNodes (ListClusterNodesRequest) returns (ListClusterNodesResponse) {
+  }
+  rpc LeaseAdminToken (LeaseAdminTokenRequest) returns (LeaseAdminTokenResponse) {
+  }
+  rpc ReleaseAdminToken (ReleaseAdminTokenRequest) returns (ReleaseAdminTokenResponse) {
+  }
+  rpc GetAdminLockStatus (GetAdminLockStatusRequest) returns (GetAdminLockStatusResponse) {
+  }
+  rpc Ping (PingRequest) returns (PingResponse) {
+  }
+  rpc RaftListClusterServers (RaftListClusterServersRequest) returns (RaftListClusterServersResponse) {
+  }
+  rpc RaftAddServer (RaftAddServerRequest) returns (RaftAddServerResponse) {
+  }
+  rpc RaftRemoveServer (RaftRemoveServerRequest) returns (RaftRemoveServerResponse) {
+  }
+  rpc RaftLeadershipTransfer (RaftLeadershipTransferRequest) returns (RaftLeadershipTransferResponse) {
+  }
+  rpc VolumeGrow (VolumeGrowRequest) returns (VolumeGrowResponse) {
+  }
+  rpc CollectionStatistics (CollectionStatisticsRequest) returns (CollectionStatisticsResponse) {
+  }
+}
+
+//////////////////////////////////////////////////
+
+message DiskTag {
+  uint32 disk_id = 1;
+  repeated string tags = 2;
+  // Physical disk descriptor, reported for every location including empty ones.
+  string type = 3;
+  int64 max_volume_count = 4;
+}
+
+message Heartbeat {
+  string ip = 1;
+  uint32 port = 2;
+  string public_url = 3;
+  uint64 max_file_key = 5;
+  string data_center = 6;
+  string rack = 7;
+  uint32 admin_port = 8;
+  repeated VolumeInformationMessage volumes = 9;
+  // delta volumes
+  repeated VolumeShortInformationMessage new_volumes = 10;
+  repeated VolumeShortInformationMessage deleted_volumes = 11;
+  bool has_no_volumes = 12;
+
+  // erasure coding
+  repeated VolumeEcShardInformationMessage ec_shards = 16;
+  // delta erasure coding shards
+  repeated VolumeEcShardInformationMessage new_ec_shards = 17;
+  repeated VolumeEcShardInformationMessage deleted_ec_shards = 18;
+  bool has_no_ec_shards = 19;
+
+  map<string, uint32> max_volume_counts = 4;
+  uint32 grpc_port = 20;
+  repeated string location_uuids = 21;
+  string id = 22; // volume server id, independent of ip:port for stable identification
+
+  // state flags
+  volume_server_pb.VolumeServerState state = 23;
+
+  repeated DiskTag disk_tags = 24;
+
+  // physical disk capacity per disk type, in bytes, from the underlying filesystem
+  map<string, uint64> disk_total_bytes = 25;
+  map<string, uint64> disk_free_bytes = 26;
+
+  // Digest of every volume in this heartbeat's view of the server, letting the
+  // master check its copy is current without being sent the whole list. Absent
+  // from servers that do not compute it, and distinct from a digest of 0, which
+  // is what a server holding no volumes reports.
+  optional uint64 volume_digest = 27;
+  // Volumes whose reported state changed since the last heartbeat, sent in
+  // place of `volumes`. A master that does not understand this never sets
+  // volume_digest_supported, so it keeps being sent the whole list.
+  repeated VolumeInformationMessage changed_volumes = 28;
+}
+
+message HeartbeatResponse {
+  uint64 volume_size_limit = 1;
+  string leader = 2;
+  string metrics_address = 3;
+  uint32 metrics_interval_seconds = 4;
+  repeated StorageBackend storage_backends = 5;
+  repeated string duplicated_uuids = 6;
+  bool preallocate = 7;
+  // The master's view of this server's volumes disagrees with the reported
+  // digest, so it needs the full volume list rather than changes alone.
+  bool resend_full_volume_list = 8;
+  // The master compares volume digests, so a server that reports one may send
+  // changed_volumes in place of its whole list.
+  bool volume_digest_supported = 9;
+}
+
+message VolumeInformationMessage {
+  uint32 id = 1;
+  uint64 size = 2;
+  string collection = 3;
+  uint64 file_count = 4;
+  uint64 delete_count = 5;
+  uint64 deleted_byte_count = 6;
+  bool read_only = 7;
+  bool read_only_can_delete = 17;
+  uint32 replica_placement = 8;
+  uint32 version = 9;
+  uint32 ttl = 10;
+  uint32 compact_revision = 11;
+  int64 modified_at_second = 12;
+  string remote_storage_name = 13;
+  string remote_storage_key = 14;
+  string disk_type = 15;
+  uint32 disk_id = 16;
+}
+
+message VolumeShortInformationMessage {
+  uint32 id = 1;
+  string collection = 3;
+  uint32 replica_placement = 8;
+  uint32 version = 9;
+  uint32 ttl = 10;
+  string disk_type = 15;
+  uint32 disk_id = 16;
+  bool read_only = 17;
+  bool read_only_can_delete = 18;
+}
+
+message VolumeEcShardInformationMessage {
+  uint32 id = 1;
+  string collection = 2;
+  uint32 ec_index_bits = 3;
+  string disk_type = 4;
+  uint64 expire_at_sec = 5; // used to record the destruction time of ec volume
+  uint32 disk_id = 6;
+  repeated int64 shard_sizes = 7; // optimized: sizes for shards in order of set bits in ec_index_bits
+  uint64 file_count = 8;   // total needles in the .ecx index (live + tombstoned)
+  uint64 delete_count = 9; // node-local tombstones in the .ecj deletion journal
+  // encode-run identity (unix nanos) from the .vif EcShardConfig; lets the admin
+  // group shards by encode generation. Numbered 14 (not 10) to skip the
+  // enterprise fork's reserved 10-13.
+  int64 encode_ts_ns = 14;
+  // fields 15-19 reserved for future upstream open-source additions.
+  // fields 20+ are owned by the enterprise fork (e.g. data_shards/parity_shards)
+  // and must not be used here without coordination.
+}
+
+message StorageBackend {
+  string type = 1;
+  string id = 2;
+  map<string, string> properties = 3;
+}
+
+message Empty {
+}
+
+message SuperBlockExtra {
+  message ErasureCoding {
+    uint32 data = 1;
+    uint32 parity = 2;
+    repeated uint32 volume_ids = 3;
+  }
+  ErasureCoding erasure_coding = 1;
+}
+
+message KeepConnectedRequest {
+  string client_type = 1;
+  string client_address = 3;
+  string version = 4;
+  string filer_group = 5;
+  string data_center = 6;
+  string rack = 7;
+}
+
+message VolumeLocation {
+  string url = 1;
+  string public_url = 2;
+  repeated uint32 new_vids = 3;
+  repeated uint32 deleted_vids = 4;
+  string leader = 5; // optional when leader is not itself
+  string data_center = 6; // optional when DataCenter is in use
+  uint32 grpc_port = 7;
+  repeated uint32 new_ec_vids = 8;
+  repeated uint32 deleted_ec_vids = 9;
+  repeated uint32 remote_vids = 10;
+  repeated uint32 read_only_vids = 11;
+  repeated uint32 read_only_can_delete_vids = 12;
+}
+
+message ClusterNodeUpdate {
+  string node_type = 1;
+  string address = 2;
+  bool is_add = 4;
+  string filer_group = 5;
+  int64 created_at_ns = 6;
+}
+
+message KeepConnectedResponse {
+  VolumeLocation volume_location = 1;
+  ClusterNodeUpdate cluster_node_update = 2;
+  LockRingUpdate lock_ring_update = 3;
+}
+
+// LockRingUpdate is sent by the master to all filers when the lock ring
+// membership changes. The master batches rapid changes (e.g., node drop + join)
+// and sends the complete member list atomically, avoiding intermediate ring
+// states that would cause unnecessary lock churn.
+message LockRingUpdate {
+  string filer_group = 1;
+  repeated string servers = 2;
+  int64 version = 3;
+}
+
+message LookupVolumeRequest {
+  repeated string volume_or_file_ids = 1;
+  string collection = 2; // optional, a bit faster if provided.
+}
+message LookupVolumeResponse {
+  message VolumeIdLocation {
+    string volume_or_file_id = 1;
+    repeated Location locations = 2;
+    string error = 3;
+    string auth = 4;
+  }
+  repeated VolumeIdLocation volume_id_locations = 1;
+}
+
+message Location {
+  string url = 1;
+  string public_url = 2;
+  uint32 grpc_port = 3;
+  string data_center = 4;
+  bool data_in_remote = 5;
+  bool read_only = 6;
+  bool read_only_can_delete = 7;
+}
+
+message AssignRequest {
+  uint64 count = 1;
+  string replication = 2;
+  string collection = 3;
+  string ttl = 4;
+  string data_center = 5;
+  string rack = 6;
+  string data_node = 7;
+  uint32 memory_map_max_size_mb = 8;
+  uint32 writable_volume_count = 9;
+  string disk_type = 10;
+  uint64 expected_data_size = 11; // hint for size-aware volume selection
+}
+
+message VolumeGrowRequest {
+  uint32 writable_volume_count = 1;
+  string replication = 2;
+  string collection = 3;
+  string ttl = 4;
+  string data_center = 5;
+  string rack = 6;
+  string data_node = 7;
+  uint32 memory_map_max_size_mb = 8;
+  string disk_type = 9;
+}
+
+message AssignResponse {
+  string fid = 1;
+  uint64 count = 4;
+  string error = 5;
+  string auth = 6;
+  repeated Location replicas = 7;
+  Location location = 8;
+}
+
+message StatisticsRequest {
+  string replication = 1;
+  string collection = 2;
+  string ttl = 3;
+  string disk_type = 4;
+}
+message StatisticsResponse {
+  uint64 total_size = 4;
+  uint64 used_size = 5;
+  uint64 file_count = 6;
+  // sizes counting one copy of the data: a single replica of a regular volume,
+  // the data shards of an ec volume. logical_total_size scales the free space
+  // by the copies the requested replication makes.
+  uint64 logical_total_size = 7;
+  uint64 logical_used_size = 8;
+}
+
+//
+// collection related
+//
+message Collection {
+  string name = 1;
+}
+message CollectionListRequest {
+  bool include_normal_volumes = 1;
+  bool include_ec_volumes = 2;
+}
+message CollectionListResponse {
+  repeated Collection collections = 1;
+}
+
+// Summarises what each collection holds, so a caller tracking usage does not
+// have to be sent every volume in the cluster to add it up itself.
+message CollectionStatisticsRequest {
+}
+message CollectionStatisticsResponse {
+  repeated CollectionStatistics collections = 1;
+}
+message CollectionStatistics {
+  string collection = 1;
+  uint64 file_count = 2;
+  uint64 delete_count = 3;
+  uint64 deleted_byte_count = 4;
+  // one copy of the data: a single replica of a regular volume, the data
+  // shards of an ec volume
+  uint64 size = 5;
+  // what is on disk: every replica, and parity shards
+  uint64 physical_size = 6;
+  uint64 volume_count = 7;
+}
+
+message CollectionDeleteRequest {
+  string name = 1;
+}
+message CollectionDeleteResponse {
+}
+
+//
+// volume related
+//
+message DiskInfo {
+  string type = 1;
+  int64 volume_count = 2;
+  int64 max_volume_count = 3;
+  int64 free_volume_count = 4;
+  int64 active_volume_count = 5;
+  repeated VolumeInformationMessage volume_infos = 6;
+  repeated VolumeEcShardInformationMessage ec_shard_infos = 7;
+  int64 remote_volume_count = 8;
+  // On a per-physical-disk DiskInfo (from SplitByPhysicalDisk) this is the disk's
+  // identity; on the type-keyed aggregate it is only a representative fallback
+  // (the first volume's disk id).
+  uint32 disk_id = 9;
+  repeated string tags = 10;
+  // Max volume count for every physical disk of this type, keyed by disk id,
+  // including disks with no volumes or EC shards; recovers empty disks that
+  // carry no per-volume/per-shard records.
+  map<uint32, int64> max_volume_count_by_disk = 11;
+  // physical disk capacity in bytes, from the underlying filesystem (0 if unknown)
+  uint64 disk_total_bytes = 12;
+  uint64 disk_free_bytes = 13;
+}
+message DataNodeInfo {
+  string id = 1;
+  map<string, DiskInfo> diskInfos = 2;
+  uint32 grpc_port = 3;
+  string address = 4; // ip:port for connecting to the volume server
+}
+message RackInfo {
+  string id = 1;
+  repeated DataNodeInfo data_node_infos = 2;
+  map<string, DiskInfo> diskInfos = 3;
+}
+message DataCenterInfo {
+  string id = 1;
+  repeated RackInfo rack_infos = 2;
+  map<string, DiskInfo> diskInfos = 3;
+}
+message TopologyInfo {
+  string id = 1;
+  repeated DataCenterInfo data_center_infos = 2;
+  map<string, DiskInfo> diskInfos = 3;
+}
+message VolumeListRequest {
+  // Empty and zero take everything. Only the volumes and ec shards listed
+  // under a disk are selected; the topology and its disk counters are always
+  // reported in full.
+  string collection = 1;
+  repeated uint32 volume_ids = 2;
+  // The one collection the empty string cannot name. A named collection wins.
+  bool default_collection_only = 3;
+  // Empty and zero take everything. Wildcards are supported.
+  string remote_storage_name = 4;
+  bool local_volume_only = 5;
+  // The topology, its disks and their counters alone, without the volumes
+  // and ec shards. Selecting volumes above contradicts this and is refused.
+  // A master that predates this field ignores it and answers in full.
+  bool topology_only = 6;
+}
+message VolumeListResponse {
+  TopologyInfo topology_info = 1;
+  uint64 volume_size_limit_mb = 2;
+}
+
+// VolumeListStream answers the same request as VolumeList without either end
+// holding every volume in the cluster at once. At 800k volumes the reply is
+// 36MB on the wire but 305MB as messages, which the master built in full
+// before sending any of it.
+message VolumeListStreamResponse {
+  // Sent once, first, listing no volumes: the topology, its disks and their
+  // counters. Every message after carries volumes for one of those disks.
+  VolumeListResponse header = 1;
+  // Which disk this batch is from. A disk arrives over as many batches as it
+  // takes, so append rather than assign.
+  string data_center = 2;
+  string rack = 3;
+  string data_node = 4;
+  string disk_type = 5;
+  repeated VolumeInformationMessage volume_infos = 6;
+  repeated VolumeEcShardInformationMessage ec_shard_infos = 7;
+}
+
+message LookupEcVolumeRequest {
+  uint32 volume_id = 1;
+}
+message LookupEcVolumeResponse {
+  uint32 volume_id = 1;
+  message EcShardIdLocation {
+    uint32 shard_id = 1;
+    repeated Location locations = 2;
+  }
+  repeated EcShardIdLocation shard_id_locations = 2;
+}
+
+message VacuumVolumeRequest {
+  float garbage_threshold = 1;
+  uint32 volume_id = 2;
+  string collection = 3;
+}
+message VacuumVolumeResponse {
+}
+
+message DisableVacuumRequest {
+    bool by_plugin = 1;
+}
+message DisableVacuumResponse {
+}
+
+message EnableVacuumRequest {
+    bool by_plugin = 1;
+}
+message EnableVacuumResponse {
+}
+
+message VolumeMarkReadonlyRequest {
+  string ip = 1;
+  uint32 port = 2;
+  uint32 volume_id = 4;
+  string collection = 5;
+  uint32 replica_placement = 6;
+  uint32 version = 7;
+  uint32 ttl = 8;
+  string disk_type = 9;
+  bool is_readonly = 10;
+}
+message VolumeMarkReadonlyResponse {
+}
+
+message GetMasterConfigurationRequest {
+}
+message GetMasterConfigurationResponse {
+  string metrics_address = 1;
+  uint32 metrics_interval_seconds = 2;
+  repeated StorageBackend storage_backends = 3;
+  string default_replication = 4;
+  string leader = 5;
+  uint32 volume_size_limit_m_b = 6;
+  bool volume_preallocate = 7;
+  // MIGRATION: fields 8-9 help migrate master.toml [master.maintenance] to admin script plugin. Remove after March 2027.
+  string maintenance_scripts = 8;
+  uint32 maintenance_sleep_minutes = 9;
+}
+
+message ListClusterNodesRequest {
+  string client_type = 1;
+  string filer_group = 2;
+  int32 limit = 4;
+}
+message ListClusterNodesResponse {
+  message ClusterNode {
+    string address = 1;
+    string version = 2;
+    int64 created_at_ns = 4;
+    string data_center = 5;
+    string rack = 6;
+  }
+  repeated ClusterNode cluster_nodes = 1;
+}
+
+message LeaseAdminTokenRequest {
+  int64 previous_token = 1;
+  int64 previous_lock_time = 2;
+  string lock_name = 3;
+  string client_name = 4;
+  string message = 5;
+}
+message LeaseAdminTokenResponse {
+  int64 token = 1;
+  int64 lock_ts_ns = 2;
+}
+
+message ReleaseAdminTokenRequest {
+  int64 previous_token = 1;
+  int64 previous_lock_time = 2;
+  string lock_name = 3;
+}
+message ReleaseAdminTokenResponse {
+}
+
+message GetAdminLockStatusRequest {
+  string lock_name = 1;
+}
+message GetAdminLockStatusResponse {
+  bool is_locked = 1;
+  string client_name = 2;
+  string message = 3;
+}
+
+message PingRequest {
+  string target = 1; // default to ping itself
+  string target_type = 2;
+}
+message PingResponse {
+  int64 start_time_ns = 1;
+  int64 remote_time_ns = 2;
+  int64 stop_time_ns = 3;
+}
+
+message RaftAddServerRequest {
+  string id = 1;
+  string address = 2;
+  bool voter = 3;
+}
+message RaftAddServerResponse {
+}
+
+message RaftRemoveServerRequest {
+  string id = 1;
+  bool force = 2;
+}
+message RaftRemoveServerResponse {
+}
+
+message RaftListClusterServersRequest {
+}
+message RaftListClusterServersResponse {
+  message ClusterServers {
+    string id = 1;
+    string address = 2;
+    string suffrage = 3;
+    bool   isLeader = 4;
+  }
+  repeated ClusterServers cluster_servers = 1;
+}
+
+message RaftLeadershipTransferRequest {
+  string target_id = 1;       // Optional: target server ID. If empty, transfers to any eligible follower
+  string target_address = 2;  // Optional: target server address. Required if target_id is specified
+}
+message RaftLeadershipTransferResponse {
+  string previous_leader = 1;
+  string new_leader = 2;
+}
+
+message VolumeGrowResponse {
+}

weed/pb/master_pb/master_helper.go

@@ -0,0 +1,151 @@
+package master_pb
+
+import "sort"
+
+func (v *VolumeLocation) IsEmptyUrl() bool {
+	return v.Url == "" || v.Url == ":0"
+}
+
+// SplitByPhysicalDisk returns one DiskInfo per physical disk. The wire format
+// keys DataNodeInfo.DiskInfos by disk type, collapsing same-type disks into one
+// entry, so this rebuilds the per-disk view. MaxVolumeCountByDisk, when present,
+// is the authoritative set (empty disks included) and gives each disk its exact
+// max; otherwise the set is the DiskIds seen on records with aggregate Max/Free
+// split evenly.
+func (d *DiskInfo) SplitByPhysicalDisk() []*DiskInfo {
+	if d == nil {
+		return nil
+	}
+
+	// DiskId 0 is overloaded: it is both the first physical disk (Locations[0])
+	// and the protobuf default for "unset". Only treat 0 as unset when every
+	// record reports 0 (the legacy case where the volume server didn't populate
+	// DiskId). If any record carries a non-zero DiskId, the reporting is real and
+	// a 0 means physical disk 0 — keep it distinct instead of folding it onto
+	// d.DiskId, which would merge two physical disks and drop disk 0 from view.
+	hasNonZero := false
+	for _, vi := range d.VolumeInfos {
+		if vi.DiskId != 0 {
+			hasNonZero = true
+			break
+		}
+	}
+	if !hasNonZero {
+		for _, eci := range d.EcShardInfos {
+			if eci.DiskId != 0 {
+				hasNonZero = true
+				break
+			}
+		}
+	}
+
+	normalize := func(id uint32) uint32 {
+		if id == 0 && !hasNonZero && d.DiskId != 0 {
+			return d.DiskId
+		}
+		return id
+	}
+
+	perDiskVolumes := make(map[uint32][]*VolumeInformationMessage)
+	for _, vi := range d.VolumeInfos {
+		id := normalize(vi.DiskId)
+		perDiskVolumes[id] = append(perDiskVolumes[id], vi)
+	}
+	perDiskShards := make(map[uint32][]*VolumeEcShardInformationMessage)
+	for _, eci := range d.EcShardInfos {
+		id := normalize(eci.DiskId)
+		perDiskShards[id] = append(perDiskShards[id], eci)
+	}
+
+	diskIDs := make(map[uint32]struct{})
+	for id := range perDiskVolumes {
+		diskIDs[id] = struct{}{}
+	}
+	for id := range perDiskShards {
+		diskIDs[id] = struct{}{}
+	}
+
+	// MaxVolumeCountByDisk (when present) is the authoritative set with each
+	// disk's exact max.
+	exactMax := len(d.MaxVolumeCountByDisk) > 0
+	for diskID := range d.MaxVolumeCountByDisk {
+		diskIDs[diskID] = struct{}{}
+	}
+
+	if len(diskIDs) == 0 {
+		diskIDs[d.DiskId] = struct{}{}
+	}
+
+	// A lone disk equal to the aggregate needs no reconstruction; exactMax disks
+	// always reconstruct so they report their own max.
+	if !exactMax && len(diskIDs) == 1 {
+		for diskID := range diskIDs {
+			if diskID == d.DiskId {
+				return []*DiskInfo{d}
+			}
+		}
+	}
+
+	// Sort disk IDs so the remainder distribution is deterministic and the
+	// reconstructed slice is in DiskId order, which is what downstream
+	// renderers expect.
+	ids := make([]uint32, 0, len(diskIDs))
+	for id := range diskIDs {
+		ids = append(ids, id)
+	}
+	sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] })
+
+	count := int64(len(ids))
+	// share returns total / count, plus one extra for the first
+	// (total % count) entries so the sum of shares equals total. Without
+	// the remainder distribution, splitting 10 across 3 disks would yield
+	// 3+3+3 = 9 and under-report aggregate capacity.
+	share := func(total int64, idx int) int64 {
+		base := total / count
+		if int64(idx) < total%count {
+			return base + 1
+		}
+		return base
+	}
+
+	result := make([]*DiskInfo, 0, len(ids))
+	for i, diskID := range ids {
+		var activeCount, remoteCount int64
+		for _, vi := range perDiskVolumes[diskID] {
+			if !vi.ReadOnly {
+				activeCount++
+			}
+			if vi.RemoteStorageName != "" {
+				remoteCount++
+			}
+		}
+		volumeCount := int64(len(perDiskVolumes[diskID]))
+		maxVolumeCount := share(d.MaxVolumeCount, i)
+		freeVolumeCount := share(d.FreeVolumeCount, i)
+		if exactMax {
+			maxVolumeCount = d.MaxVolumeCountByDisk[diskID]
+			// EC slots aren't subtracted (needs the configurable ratio, outside
+			// this package); planners subtract them from EcShardInfos. Clamp so an
+			// over-allocated disk can't report negative free.
+			freeVolumeCount = maxVolumeCount - (volumeCount - remoteCount)
+			if freeVolumeCount < 0 {
+				freeVolumeCount = 0
+			}
+		}
+		result = append(result, &DiskInfo{
+			Type:              d.Type,
+			MaxVolumeCount:    maxVolumeCount,
+			VolumeCount:       volumeCount,
+			FreeVolumeCount:   freeVolumeCount,
+			ActiveVolumeCount: activeCount,
+			RemoteVolumeCount: remoteCount,
+			VolumeInfos:       perDiskVolumes[diskID],
+			EcShardInfos:      perDiskShards[diskID],
+			DiskId:            diskID,
+			Tags:              append([]string(nil), d.Tags...),
+			DiskTotalBytes:    uint64(share(int64(d.DiskTotalBytes), i)),
+			DiskFreeBytes:     uint64(share(int64(d.DiskFreeBytes), i)),
+		})
+	}
+	return result
+}

weed/pb/master_pb/master_helper_test.go

@@ -0,0 +1,338 @@
+package master_pb
+
+import (
+	"sort"
+	"testing"
+)
+
+// MaxVolumeCountByDisk surfaces empty disks and gives each its exact max.
+func TestDiskInfoSplitByPhysicalDisk_includesEmptyDiskFromPhysicalDisks(t *testing.T) {
+	d := &DiskInfo{
+		Type:           "hdd",
+		MaxVolumeCount: 1000, // per-type aggregate
+		VolumeInfos: []*VolumeInformationMessage{
+			{Id: 10, DiskId: 0},
+			{Id: 11, DiskId: 1},
+		},
+		MaxVolumeCountByDisk: map[uint32]int64{
+			0: 350,
+			1: 350,
+			2: 300, // empty disk, no volumes/shards
+		},
+	}
+
+	got := d.SplitByPhysicalDisk()
+	if len(got) != 3 {
+		t.Fatalf("want 3 physical disks including the empty one, got %d", len(got))
+	}
+	byID := map[uint32]*DiskInfo{}
+	for _, di := range got {
+		byID[di.DiskId] = di
+	}
+
+	empty, ok := byID[2]
+	if !ok {
+		t.Fatalf("empty disk 2 not surfaced; got %v", byID)
+	}
+	if empty.VolumeCount != 0 {
+		t.Errorf("empty disk: want 0 volumes, got %d", empty.VolumeCount)
+	}
+	if empty.FreeVolumeCount != 300 {
+		t.Errorf("empty disk free: want its full max 300, got %d", empty.FreeVolumeCount)
+	}
+
+	// Exact per-disk max, not the even split (which would give ~334/333/333).
+	if byID[0].MaxVolumeCount != 350 || byID[1].MaxVolumeCount != 350 || byID[2].MaxVolumeCount != 300 {
+		t.Errorf("want exact per-disk max 350/350/300, got %d/%d/%d",
+			byID[0].MaxVolumeCount, byID[1].MaxVolumeCount, byID[2].MaxVolumeCount)
+	}
+	if byID[0].FreeVolumeCount != 349 {
+		t.Errorf("disk 0 free: want 349 (350-1 volume), got %d", byID[0].FreeVolumeCount)
+	}
+}
+
+// An over-allocated disk reports zero free, not negative.
+func TestDiskInfoSplitByPhysicalDisk_clampsNegativeFreeOnOverAllocation(t *testing.T) {
+	d := &DiskInfo{
+		Type: "hdd",
+		VolumeInfos: []*VolumeInformationMessage{
+			{Id: 1, DiskId: 0},
+			{Id: 2, DiskId: 0},
+			{Id: 3, DiskId: 0},
+		},
+		MaxVolumeCountByDisk: map[uint32]int64{
+			0: 2, // 3 volumes on a max-2 disk
+		},
+	}
+
+	got := d.SplitByPhysicalDisk()
+	if len(got) != 1 {
+		t.Fatalf("want 1 disk, got %d", len(got))
+	}
+	if got[0].FreeVolumeCount != 0 {
+		t.Errorf("over-allocated disk free: want clamped 0, got %d", got[0].FreeVolumeCount)
+	}
+}
+
+func TestDiskInfoSplitByPhysicalDisk_collapsesOnSingleDisk(t *testing.T) {
+	d := &DiskInfo{
+		Type:           "hdd",
+		MaxVolumeCount: 10,
+		VolumeInfos: []*VolumeInformationMessage{
+			{Id: 1, DiskId: 0},
+			{Id: 2, DiskId: 0},
+		},
+		DiskId: 0,
+	}
+
+	got := d.SplitByPhysicalDisk()
+	if len(got) != 1 {
+		t.Fatalf("want 1 split disk, got %d", len(got))
+	}
+	if got[0] != d {
+		t.Errorf("single-disk input should be returned unchanged; got a copy")
+	}
+}
+
+func TestDiskInfoSplitByPhysicalDisk_splitsByVolumeDiskId(t *testing.T) {
+	d := &DiskInfo{
+		Type:              "hdd",
+		MaxVolumeCount:    60,
+		FreeVolumeCount:   30,
+		ActiveVolumeCount: 12,
+		VolumeInfos: []*VolumeInformationMessage{
+			{Id: 10, DiskId: 0},
+			{Id: 11, DiskId: 0},
+			{Id: 20, DiskId: 1},
+			{Id: 21, DiskId: 2},
+			{Id: 22, DiskId: 2},
+			{Id: 23, DiskId: 2},
+		},
+	}
+
+	got := d.SplitByPhysicalDisk()
+	if len(got) != 3 {
+		t.Fatalf("want 3 split disks, got %d", len(got))
+	}
+
+	byID := map[uint32]*DiskInfo{}
+	for _, di := range got {
+		byID[di.DiskId] = di
+	}
+	for _, want := range []uint32{0, 1, 2} {
+		if _, ok := byID[want]; !ok {
+			t.Errorf("missing DiskId=%d in split result", want)
+		}
+	}
+
+	if byID[0].VolumeCount != 2 {
+		t.Errorf("disk 0: want 2 volumes, got %d", byID[0].VolumeCount)
+	}
+	if byID[1].VolumeCount != 1 {
+		t.Errorf("disk 1: want 1 volume, got %d", byID[1].VolumeCount)
+	}
+	if byID[2].VolumeCount != 3 {
+		t.Errorf("disk 2: want 3 volumes, got %d", byID[2].VolumeCount)
+	}
+
+	// Capacity is split evenly across the reconstructed disks. With 3 disks
+	// and a max of 60, every reconstructed disk gets 20.
+	for id, di := range byID {
+		if di.MaxVolumeCount != 20 {
+			t.Errorf("disk %d: want MaxVolumeCount=20 (60/3), got %d", id, di.MaxVolumeCount)
+		}
+	}
+
+	// Disk type is preserved on every reconstructed entry so writers that
+	// label by type still see "hdd".
+	for id, di := range byID {
+		if di.Type != "hdd" {
+			t.Errorf("disk %d: want Type=hdd, got %q", id, di.Type)
+		}
+	}
+}
+
+// TestDiskInfoSplitByPhysicalDisk_preservesAggregateCapacityWithRemainder
+// pins the invariant that the sum of reconstructed counters equals the
+// original aggregate, even when the count does not divide evenly.
+func TestDiskInfoSplitByPhysicalDisk_preservesAggregateCapacityWithRemainder(t *testing.T) {
+	d := &DiskInfo{
+		Type:            "hdd",
+		MaxVolumeCount:  10,
+		FreeVolumeCount: 7,
+		VolumeInfos: []*VolumeInformationMessage{
+			{Id: 1, DiskId: 0},
+			{Id: 2, DiskId: 1},
+			{Id: 3, DiskId: 2},
+		},
+	}
+
+	got := d.SplitByPhysicalDisk()
+	if len(got) != 3 {
+		t.Fatalf("want 3 disks, got %d", len(got))
+	}
+
+	var sumMax, sumFree int64
+	for _, di := range got {
+		sumMax += di.MaxVolumeCount
+		sumFree += di.FreeVolumeCount
+	}
+	if sumMax != 10 {
+		t.Errorf("sum of MaxVolumeCount = %d, want 10 (lossless split)", sumMax)
+	}
+	if sumFree != 7 {
+		t.Errorf("sum of FreeVolumeCount = %d, want 7 (lossless split)", sumFree)
+	}
+}
+
+// TestDiskInfoSplitByPhysicalDisk_countsActiveAndRemoteExactly verifies
+// the per-disk ActiveVolumeCount and RemoteVolumeCount are derived from
+// the actual VolumeInfos rather than an even split of the node totals.
+func TestDiskInfoSplitByPhysicalDisk_countsActiveAndRemoteExactly(t *testing.T) {
+	d := &DiskInfo{
+		Type: "hdd",
+		VolumeInfos: []*VolumeInformationMessage{
+			{Id: 1, DiskId: 0, ReadOnly: false},
+			{Id: 2, DiskId: 0, ReadOnly: true},
+			{Id: 3, DiskId: 1, ReadOnly: false, RemoteStorageName: "s3"},
+			{Id: 4, DiskId: 2, ReadOnly: false},
+			{Id: 5, DiskId: 2, ReadOnly: false, RemoteStorageName: "s3"},
+		},
+	}
+
+	got := d.SplitByPhysicalDisk()
+	byID := map[uint32]*DiskInfo{}
+	for _, di := range got {
+		byID[di.DiskId] = di
+	}
+
+	cases := []struct {
+		id         uint32
+		wantActive int64
+		wantRemote int64
+	}{
+		{0, 1, 0},
+		{1, 1, 1},
+		{2, 2, 1},
+	}
+	for _, c := range cases {
+		di := byID[c.id]
+		if di == nil {
+			t.Errorf("missing disk %d", c.id)
+			continue
+		}
+		if di.ActiveVolumeCount != c.wantActive {
+			t.Errorf("disk %d: ActiveVolumeCount = %d, want %d", c.id, di.ActiveVolumeCount, c.wantActive)
+		}
+		if di.RemoteVolumeCount != c.wantRemote {
+			t.Errorf("disk %d: RemoteVolumeCount = %d, want %d", c.id, di.RemoteVolumeCount, c.wantRemote)
+		}
+	}
+}
+
+func TestDiskInfoSplitByPhysicalDisk_splitsByEcShardDiskId(t *testing.T) {
+	d := &DiskInfo{
+		Type: "hdd",
+		EcShardInfos: []*VolumeEcShardInformationMessage{
+			{Id: 100, DiskId: 4},
+			{Id: 101, DiskId: 4},
+			{Id: 102, DiskId: 5},
+		},
+	}
+
+	got := d.SplitByPhysicalDisk()
+
+	ids := make([]int, 0, len(got))
+	for _, di := range got {
+		ids = append(ids, int(di.DiskId))
+	}
+	sort.Ints(ids)
+	if len(ids) != 2 || ids[0] != 4 || ids[1] != 5 {
+		t.Fatalf("want split disk ids [4,5], got %v", ids)
+	}
+}
+
+func TestDiskInfoSplitByPhysicalDisk_normalizesZeroToOuterDiskId(t *testing.T) {
+	// Older payloads / fixtures omit the per-record DiskId. The outer DiskId
+	// is the authoritative fallback.
+	d := &DiskInfo{
+		Type:   "hdd",
+		DiskId: 7,
+		VolumeInfos: []*VolumeInformationMessage{
+			{Id: 1, DiskId: 0}, // normalize to 7
+			{Id: 2, DiskId: 0}, // normalize to 7
+		},
+	}
+	got := d.SplitByPhysicalDisk()
+	if len(got) != 1 {
+		t.Fatalf("want 1 disk, got %d", len(got))
+	}
+	if got[0].DiskId != 7 {
+		t.Errorf("want DiskId=7 after normalization, got %d", got[0].DiskId)
+	}
+}
+
+// TestDiskInfoSplitByPhysicalDisk_keepsRealDiskZeroWhenMixed reproduces a
+// multi-disk node where physical disk 0 (Locations[0]) holds volumes and the
+// aggregate DiskId is seeded from a non-zero sibling (volumes[0] landed on
+// disk 6). DiskId 0 must stay its own physical disk, not get folded onto disk
+// 6 — folding drops a disk from the count and doubles the sibling's volumes.
+func TestDiskInfoSplitByPhysicalDisk_keepsRealDiskZeroWhenMixed(t *testing.T) {
+	d := &DiskInfo{
+		Type:           "hdd",
+		MaxVolumeCount: 30,
+		DiskId:         6, // seeded from volumes[0].DiskId in ToDiskInfo
+		VolumeInfos: []*VolumeInformationMessage{
+			{Id: 60, DiskId: 6},
+			{Id: 61, DiskId: 6},
+			{Id: 1, DiskId: 0}, // real disk 0, must not be remapped to 6
+			{Id: 2, DiskId: 0},
+			{Id: 30, DiskId: 3},
+		},
+	}
+
+	got := d.SplitByPhysicalDisk()
+	byID := map[uint32]*DiskInfo{}
+	for _, di := range got {
+		byID[di.DiskId] = di
+	}
+
+	if len(got) != 3 {
+		t.Fatalf("want 3 physical disks (0, 3, 6), got %d: %v", len(got), byID)
+	}
+	if _, ok := byID[0]; !ok {
+		t.Fatalf("physical disk 0 was dropped; got disks %v", byID)
+	}
+	if byID[0].VolumeCount != 2 {
+		t.Errorf("disk 0: want 2 volumes, got %d", byID[0].VolumeCount)
+	}
+	if byID[6].VolumeCount != 2 {
+		t.Errorf("disk 6: want 2 volumes (not merged with disk 0), got %d", byID[6].VolumeCount)
+	}
+	if byID[3].VolumeCount != 1 {
+		t.Errorf("disk 3: want 1 volume, got %d", byID[3].VolumeCount)
+	}
+
+	var sumMax int64
+	for _, di := range got {
+		sumMax += di.MaxVolumeCount
+	}
+	if sumMax != 30 {
+		t.Errorf("sum of MaxVolumeCount = %d, want 30 (lossless split)", sumMax)
+	}
+}
+
+func TestDiskInfoSplitByPhysicalDisk_nilSafe(t *testing.T) {
+	var d *DiskInfo
+	if got := d.SplitByPhysicalDisk(); got != nil {
+		t.Errorf("nil receiver should return nil slice, got %v", got)
+	}
+}
+
+func TestDiskInfoSplitByPhysicalDisk_emptyDiskReturnsSelf(t *testing.T) {
+	d := &DiskInfo{Type: "hdd", MaxVolumeCount: 10, DiskId: 3}
+	got := d.SplitByPhysicalDisk()
+	if len(got) != 1 || got[0] != d {
+		t.Errorf("empty disk should be passed through unchanged; got %v", got)
+	}
+}

weed/pb/message.fbs

@@ -0,0 +1,23 @@
+table NameValue {
+    name:string (key);
+    value:string;
+}
+table Message {
+    seq_delta:int32         (id:0);
+    ts_ms_delta:int32       (id:1);
+    properties:[NameValue]  (id:2);
+    key:string              (id:3); // bytes
+    data:string             (id:4); // bytes
+}
+
+table MessageBatch {
+    producer_id:int32       (id:0);
+    producer_epoch:int32    (id:1);
+    segment_id:int32        (id:2);
+    flags: int32            (id:3);
+    segment_seq_base:int64          (id:4);
+    segment_seq_max_delta:int32     (id:5);
+    ts_ms_base:int64        (id:6);
+    ts_ms_max_delta:int32         (id:7);
+    messages: [Message]     (id:8);
+}

weed/pb/message_fbs/Message.go

@@ -0,0 +1,119 @@
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package message_fbs
+
+import (
+	flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type Message struct {
+	_tab flatbuffers.Table
+}
+
+func GetRootAsMessage(buf []byte, offset flatbuffers.UOffsetT) *Message {
+	n := flatbuffers.GetUOffsetT(buf[offset:])
+	x := &Message{}
+	x.Init(buf, n+offset)
+	return x
+}
+
+func GetSizePrefixedRootAsMessage(buf []byte, offset flatbuffers.UOffsetT) *Message {
+	n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:])
+	x := &Message{}
+	x.Init(buf, n+offset+flatbuffers.SizeUint32)
+	return x
+}
+
+func (rcv *Message) Init(buf []byte, i flatbuffers.UOffsetT) {
+	rcv._tab.Bytes = buf
+	rcv._tab.Pos = i
+}
+
+func (rcv *Message) Table() flatbuffers.Table {
+	return rcv._tab
+}
+
+func (rcv *Message) SeqDelta() int32 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+	if o != 0 {
+		return rcv._tab.GetInt32(o + rcv._tab.Pos)
+	}
+	return 0
+}
+
+func (rcv *Message) MutateSeqDelta(n int32) bool {
+	return rcv._tab.MutateInt32Slot(4, n)
+}
+
+func (rcv *Message) TsMsDelta() int32 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+	if o != 0 {
+		return rcv._tab.GetInt32(o + rcv._tab.Pos)
+	}
+	return 0
+}
+
+func (rcv *Message) MutateTsMsDelta(n int32) bool {
+	return rcv._tab.MutateInt32Slot(6, n)
+}
+
+func (rcv *Message) Properties(obj *NameValue, j int) bool {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+	if o != 0 {
+		x := rcv._tab.Vector(o)
+		x += flatbuffers.UOffsetT(j) * 4
+		x = rcv._tab.Indirect(x)
+		obj.Init(rcv._tab.Bytes, x)
+		return true
+	}
+	return false
+}
+
+func (rcv *Message) PropertiesLength() int {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+	if o != 0 {
+		return rcv._tab.VectorLen(o)
+	}
+	return 0
+}
+
+func (rcv *Message) Key() []byte {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
+	if o != 0 {
+		return rcv._tab.ByteVector(o + rcv._tab.Pos)
+	}
+	return nil
+}
+
+func (rcv *Message) Data() []byte {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(12))
+	if o != 0 {
+		return rcv._tab.ByteVector(o + rcv._tab.Pos)
+	}
+	return nil
+}
+
+func MessageStart(builder *flatbuffers.Builder) {
+	builder.StartObject(5)
+}
+func MessageAddSeqDelta(builder *flatbuffers.Builder, seqDelta int32) {
+	builder.PrependInt32Slot(0, seqDelta, 0)
+}
+func MessageAddTsMsDelta(builder *flatbuffers.Builder, tsMsDelta int32) {
+	builder.PrependInt32Slot(1, tsMsDelta, 0)
+}
+func MessageAddProperties(builder *flatbuffers.Builder, properties flatbuffers.UOffsetT) {
+	builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(properties), 0)
+}
+func MessageStartPropertiesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+	return builder.StartVector(4, numElems, 4)
+}
+func MessageAddKey(builder *flatbuffers.Builder, key flatbuffers.UOffsetT) {
+	builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(key), 0)
+}
+func MessageAddData(builder *flatbuffers.Builder, data flatbuffers.UOffsetT) {
+	builder.PrependUOffsetTSlot(4, flatbuffers.UOffsetT(data), 0)
+}
+func MessageEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+	return builder.EndObject()
+}

weed/pb/message_fbs/MessageBatch.go

@@ -0,0 +1,187 @@
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package message_fbs
+
+import (
+	flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type MessageBatch struct {
+	_tab flatbuffers.Table
+}
+
+func GetRootAsMessageBatch(buf []byte, offset flatbuffers.UOffsetT) *MessageBatch {
+	n := flatbuffers.GetUOffsetT(buf[offset:])
+	x := &MessageBatch{}
+	x.Init(buf, n+offset)
+	return x
+}
+
+func GetSizePrefixedRootAsMessageBatch(buf []byte, offset flatbuffers.UOffsetT) *MessageBatch {
+	n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:])
+	x := &MessageBatch{}
+	x.Init(buf, n+offset+flatbuffers.SizeUint32)
+	return x
+}
+
+func (rcv *MessageBatch) Init(buf []byte, i flatbuffers.UOffsetT) {
+	rcv._tab.Bytes = buf
+	rcv._tab.Pos = i
+}
+
+func (rcv *MessageBatch) Table() flatbuffers.Table {
+	return rcv._tab
+}
+
+func (rcv *MessageBatch) ProducerId() int32 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+	if o != 0 {
+		return rcv._tab.GetInt32(o + rcv._tab.Pos)
+	}
+	return 0
+}
+
+func (rcv *MessageBatch) MutateProducerId(n int32) bool {
+	return rcv._tab.MutateInt32Slot(4, n)
+}
+
+func (rcv *MessageBatch) ProducerEpoch() int32 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+	if o != 0 {
+		return rcv._tab.GetInt32(o + rcv._tab.Pos)
+	}
+	return 0
+}
+
+func (rcv *MessageBatch) MutateProducerEpoch(n int32) bool {
+	return rcv._tab.MutateInt32Slot(6, n)
+}
+
+func (rcv *MessageBatch) SegmentId() int32 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+	if o != 0 {
+		return rcv._tab.GetInt32(o + rcv._tab.Pos)
+	}
+	return 0
+}
+
+func (rcv *MessageBatch) MutateSegmentId(n int32) bool {
+	return rcv._tab.MutateInt32Slot(8, n)
+}
+
+func (rcv *MessageBatch) Flags() int32 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
+	if o != 0 {
+		return rcv._tab.GetInt32(o + rcv._tab.Pos)
+	}
+	return 0
+}
+
+func (rcv *MessageBatch) MutateFlags(n int32) bool {
+	return rcv._tab.MutateInt32Slot(10, n)
+}
+
+func (rcv *MessageBatch) SegmentSeqBase() int64 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(12))
+	if o != 0 {
+		return rcv._tab.GetInt64(o + rcv._tab.Pos)
+	}
+	return 0
+}
+
+func (rcv *MessageBatch) MutateSegmentSeqBase(n int64) bool {
+	return rcv._tab.MutateInt64Slot(12, n)
+}
+
+func (rcv *MessageBatch) SegmentSeqMaxDelta() int32 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(14))
+	if o != 0 {
+		return rcv._tab.GetInt32(o + rcv._tab.Pos)
+	}
+	return 0
+}
+
+func (rcv *MessageBatch) MutateSegmentSeqMaxDelta(n int32) bool {
+	return rcv._tab.MutateInt32Slot(14, n)
+}
+
+func (rcv *MessageBatch) TsMsBase() int64 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(16))
+	if o != 0 {
+		return rcv._tab.GetInt64(o + rcv._tab.Pos)
+	}
+	return 0
+}
+
+func (rcv *MessageBatch) MutateTsMsBase(n int64) bool {
+	return rcv._tab.MutateInt64Slot(16, n)
+}
+
+func (rcv *MessageBatch) TsMsMaxDelta() int32 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(18))
+	if o != 0 {
+		return rcv._tab.GetInt32(o + rcv._tab.Pos)
+	}
+	return 0
+}
+
+func (rcv *MessageBatch) MutateTsMsMaxDelta(n int32) bool {
+	return rcv._tab.MutateInt32Slot(18, n)
+}
+
+func (rcv *MessageBatch) Messages(obj *Message, j int) bool {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(20))
+	if o != 0 {
+		x := rcv._tab.Vector(o)
+		x += flatbuffers.UOffsetT(j) * 4
+		x = rcv._tab.Indirect(x)
+		obj.Init(rcv._tab.Bytes, x)
+		return true
+	}
+	return false
+}
+
+func (rcv *MessageBatch) MessagesLength() int {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(20))
+	if o != 0 {
+		return rcv._tab.VectorLen(o)
+	}
+	return 0
+}
+
+func MessageBatchStart(builder *flatbuffers.Builder) {
+	builder.StartObject(9)
+}
+func MessageBatchAddProducerId(builder *flatbuffers.Builder, producerId int32) {
+	builder.PrependInt32Slot(0, producerId, 0)
+}
+func MessageBatchAddProducerEpoch(builder *flatbuffers.Builder, producerEpoch int32) {
+	builder.PrependInt32Slot(1, producerEpoch, 0)
+}
+func MessageBatchAddSegmentId(builder *flatbuffers.Builder, segmentId int32) {
+	builder.PrependInt32Slot(2, segmentId, 0)
+}
+func MessageBatchAddFlags(builder *flatbuffers.Builder, flags int32) {
+	builder.PrependInt32Slot(3, flags, 0)
+}
+func MessageBatchAddSegmentSeqBase(builder *flatbuffers.Builder, segmentSeqBase int64) {
+	builder.PrependInt64Slot(4, segmentSeqBase, 0)
+}
+func MessageBatchAddSegmentSeqMaxDelta(builder *flatbuffers.Builder, segmentSeqMaxDelta int32) {
+	builder.PrependInt32Slot(5, segmentSeqMaxDelta, 0)
+}
+func MessageBatchAddTsMsBase(builder *flatbuffers.Builder, tsMsBase int64) {
+	builder.PrependInt64Slot(6, tsMsBase, 0)
+}
+func MessageBatchAddTsMsMaxDelta(builder *flatbuffers.Builder, tsMsMaxDelta int32) {
+	builder.PrependInt32Slot(7, tsMsMaxDelta, 0)
+}
+func MessageBatchAddMessages(builder *flatbuffers.Builder, messages flatbuffers.UOffsetT) {
+	builder.PrependUOffsetTSlot(8, flatbuffers.UOffsetT(messages), 0)
+}
+func MessageBatchStartMessagesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+	return builder.StartVector(4, numElems, 4)
+}
+func MessageBatchEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+	return builder.EndObject()
+}

weed/pb/message_fbs/NameValue.go

@@ -0,0 +1,63 @@
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package message_fbs
+
+import (
+	flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type NameValue struct {
+	_tab flatbuffers.Table
+}
+
+func GetRootAsNameValue(buf []byte, offset flatbuffers.UOffsetT) *NameValue {
+	n := flatbuffers.GetUOffsetT(buf[offset:])
+	x := &NameValue{}
+	x.Init(buf, n+offset)
+	return x
+}
+
+func GetSizePrefixedRootAsNameValue(buf []byte, offset flatbuffers.UOffsetT) *NameValue {
+	n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:])
+	x := &NameValue{}
+	x.Init(buf, n+offset+flatbuffers.SizeUint32)
+	return x
+}
+
+func (rcv *NameValue) Init(buf []byte, i flatbuffers.UOffsetT) {
+	rcv._tab.Bytes = buf
+	rcv._tab.Pos = i
+}
+
+func (rcv *NameValue) Table() flatbuffers.Table {
+	return rcv._tab
+}
+
+func (rcv *NameValue) Name() []byte {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+	if o != 0 {
+		return rcv._tab.ByteVector(o + rcv._tab.Pos)
+	}
+	return nil
+}
+
+func (rcv *NameValue) Value() []byte {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+	if o != 0 {
+		return rcv._tab.ByteVector(o + rcv._tab.Pos)
+	}
+	return nil
+}
+
+func NameValueStart(builder *flatbuffers.Builder) {
+	builder.StartObject(2)
+}
+func NameValueAddName(builder *flatbuffers.Builder, name flatbuffers.UOffsetT) {
+	builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(name), 0)
+}
+func NameValueAddValue(builder *flatbuffers.Builder, value flatbuffers.UOffsetT) {
+	builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(value), 0)
+}
+func NameValueEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+	return builder.EndObject()
+}

weed/pb/mount.proto

@@ -0,0 +1,25 @@
+syntax = "proto3";
+
+package messaging_pb;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/mount_pb";
+option java_package = "seaweedfs.client";
+option java_outer_classname = "MountProto";
+
+//////////////////////////////////////////////////
+
+service SeaweedMount {
+
+    rpc Configure (ConfigureRequest) returns (ConfigureResponse) {
+    }
+
+}
+
+//////////////////////////////////////////////////
+
+message ConfigureRequest {
+    int64 collection_capacity = 1;
+}
+
+message ConfigureResponse {
+}

weed/pb/mount_pb/mount.pb.go

@@ -0,0 +1,166 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.6
+// 	protoc        v6.33.4
+// source: mount.proto
+
+package mount_pb
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+	unsafe "unsafe"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type ConfigureRequest struct {
+	state              protoimpl.MessageState `protogen:"open.v1"`
+	CollectionCapacity int64                  `protobuf:"varint,1,opt,name=collection_capacity,json=collectionCapacity,proto3" json:"collection_capacity,omitempty"`
+	unknownFields      protoimpl.UnknownFields
+	sizeCache          protoimpl.SizeCache
+}
+
+func (x *ConfigureRequest) Reset() {
+	*x = ConfigureRequest{}
+	mi := &file_mount_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ConfigureRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ConfigureRequest) ProtoMessage() {}
+
+func (x *ConfigureRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_mount_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ConfigureRequest.ProtoReflect.Descriptor instead.
+func (*ConfigureRequest) Descriptor() ([]byte, []int) {
+	return file_mount_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *ConfigureRequest) GetCollectionCapacity() int64 {
+	if x != nil {
+		return x.CollectionCapacity
+	}
+	return 0
+}
+
+type ConfigureResponse struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ConfigureResponse) Reset() {
+	*x = ConfigureResponse{}
+	mi := &file_mount_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ConfigureResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ConfigureResponse) ProtoMessage() {}
+
+func (x *ConfigureResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_mount_proto_msgTypes[1]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ConfigureResponse.ProtoReflect.Descriptor instead.
+func (*ConfigureResponse) Descriptor() ([]byte, []int) {
+	return file_mount_proto_rawDescGZIP(), []int{1}
+}
+
+var File_mount_proto protoreflect.FileDescriptor
+
+const file_mount_proto_rawDesc = "" +
+	"\n" +
+	"\vmount.proto\x12\fmessaging_pb\"C\n" +
+	"\x10ConfigureRequest\x12/\n" +
+	"\x13collection_capacity\x18\x01 \x01(\x03R\x12collectionCapacity\"\x13\n" +
+	"\x11ConfigureResponse2^\n" +
+	"\fSeaweedMount\x12N\n" +
+	"\tConfigure\x12\x1e.messaging_pb.ConfigureRequest\x1a\x1f.messaging_pb.ConfigureResponse\"\x00BO\n" +
+	"\x10seaweedfs.clientB\n" +
+	"MountProtoZ/github.com/seaweedfs/seaweedfs/weed/pb/mount_pbb\x06proto3"
+
+var (
+	file_mount_proto_rawDescOnce sync.Once
+	file_mount_proto_rawDescData []byte
+)
+
+func file_mount_proto_rawDescGZIP() []byte {
+	file_mount_proto_rawDescOnce.Do(func() {
+		file_mount_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_mount_proto_rawDesc), len(file_mount_proto_rawDesc)))
+	})
+	return file_mount_proto_rawDescData
+}
+
+var file_mount_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_mount_proto_goTypes = []any{
+	(*ConfigureRequest)(nil),  // 0: messaging_pb.ConfigureRequest
+	(*ConfigureResponse)(nil), // 1: messaging_pb.ConfigureResponse
+}
+var file_mount_proto_depIdxs = []int32{
+	0, // 0: messaging_pb.SeaweedMount.Configure:input_type -> messaging_pb.ConfigureRequest
+	1, // 1: messaging_pb.SeaweedMount.Configure:output_type -> messaging_pb.ConfigureResponse
+	1, // [1:2] is the sub-list for method output_type
+	0, // [0:1] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_mount_proto_init() }
+func file_mount_proto_init() {
+	if File_mount_proto != nil {
+		return
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_mount_proto_rawDesc), len(file_mount_proto_rawDesc)),
+			NumEnums:      0,
+			NumMessages:   2,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_mount_proto_goTypes,
+		DependencyIndexes: file_mount_proto_depIdxs,
+		MessageInfos:      file_mount_proto_msgTypes,
+	}.Build()
+	File_mount_proto = out.File
+	file_mount_proto_goTypes = nil
+	file_mount_proto_depIdxs = nil
+}

weed/pb/mount_pb/mount_grpc.pb.go

@@ -0,0 +1,121 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.5.1
+// - protoc             v6.33.4
+// source: mount.proto
+
+package mount_pb
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.64.0 or later.
+const _ = grpc.SupportPackageIsVersion9
+
+const (
+	SeaweedMount_Configure_FullMethodName = "/messaging_pb.SeaweedMount/Configure"
+)
+
+// SeaweedMountClient is the client API for SeaweedMount service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type SeaweedMountClient interface {
+	Configure(ctx context.Context, in *ConfigureRequest, opts ...grpc.CallOption) (*ConfigureResponse, error)
+}
+
+type seaweedMountClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewSeaweedMountClient(cc grpc.ClientConnInterface) SeaweedMountClient {
+	return &seaweedMountClient{cc}
+}
+
+func (c *seaweedMountClient) Configure(ctx context.Context, in *ConfigureRequest, opts ...grpc.CallOption) (*ConfigureResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ConfigureResponse)
+	err := c.cc.Invoke(ctx, SeaweedMount_Configure_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// SeaweedMountServer is the server API for SeaweedMount service.
+// All implementations must embed UnimplementedSeaweedMountServer
+// for forward compatibility.
+type SeaweedMountServer interface {
+	Configure(context.Context, *ConfigureRequest) (*ConfigureResponse, error)
+	mustEmbedUnimplementedSeaweedMountServer()
+}
+
+// UnimplementedSeaweedMountServer must be embedded to have
+// forward compatible implementations.
+//
+// NOTE: this should be embedded by value instead of pointer to avoid a nil
+// pointer dereference when methods are called.
+type UnimplementedSeaweedMountServer struct{}
+
+func (UnimplementedSeaweedMountServer) Configure(context.Context, *ConfigureRequest) (*ConfigureResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Configure not implemented")
+}
+func (UnimplementedSeaweedMountServer) mustEmbedUnimplementedSeaweedMountServer() {}
+func (UnimplementedSeaweedMountServer) testEmbeddedByValue()                      {}
+
+// UnsafeSeaweedMountServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to SeaweedMountServer will
+// result in compilation errors.
+type UnsafeSeaweedMountServer interface {
+	mustEmbedUnimplementedSeaweedMountServer()
+}
+
+func RegisterSeaweedMountServer(s grpc.ServiceRegistrar, srv SeaweedMountServer) {
+	// If the following call pancis, it indicates UnimplementedSeaweedMountServer was
+	// embedded by pointer and is nil.  This will cause panics if an
+	// unimplemented method is ever invoked, so we test this at initialization
+	// time to prevent it from happening at runtime later due to I/O.
+	if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
+		t.testEmbeddedByValue()
+	}
+	s.RegisterService(&SeaweedMount_ServiceDesc, srv)
+}
+
+func _SeaweedMount_Configure_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ConfigureRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMountServer).Configure(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMount_Configure_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMountServer).Configure(ctx, req.(*ConfigureRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// SeaweedMount_ServiceDesc is the grpc.ServiceDesc for SeaweedMount service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var SeaweedMount_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "messaging_pb.SeaweedMount",
+	HandlerType: (*SeaweedMountServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Configure",
+			Handler:    _SeaweedMount_Configure_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "mount.proto",
+}

weed/pb/mount_peer.proto

@@ -0,0 +1,95 @@
+syntax = "proto3";
+
+package mount_peer_pb;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/mount_peer_pb";
+
+//////////////////////////////////////////////////
+// Peer chunk sharing — mount-to-mount chunk directory
+//
+// Each weed mount exposes this service on its -peer.listen address.
+// Ownership of fid -> holders tracking is sharded across the mount fleet
+// via rendezvous (HRW) hashing on the registered mount list. See
+// design-weed-mount-peer-chunk-sharing.md for protocol details.
+//////////////////////////////////////////////////
+
+service MountPeer {
+
+    // ChunkAnnounce: the caller asserts it currently holds the listed fids in
+    // its local chunk cache and is willing to serve them to peers. The
+    // receiver accepts only fids for which it is the HRW-assigned owner on
+    // its current seed view; others are returned in rejected_file_ids.
+    rpc ChunkAnnounce (ChunkAnnounceRequest) returns (ChunkAnnounceResponse) {
+    }
+
+    // ChunkLookup: asks the receiver for known holders of each requested fid.
+    // The receiver responds only for fids it owns; others are listed in
+    // not_owner_file_ids so the caller can retry against the correct owner
+    // after refreshing its own seed view.
+    rpc ChunkLookup (ChunkLookupRequest) returns (ChunkLookupResponse) {
+    }
+
+    // FetchChunk: server-streams the bytes of a cached chunk to a peer.
+    // Streaming avoids the default gRPC 4 MiB message cap for typical
+    // 16 MiB chunks and lets the receiver assemble into a preallocated
+    // buffer. Not-cached / cache-miss returns a gRPC NOT_FOUND status.
+    // The fetcher re-verifies MD5 against expected_etag end-to-end after
+    // the stream completes.
+    rpc FetchChunk (FetchChunkRequest) returns (stream FetchChunkResponse) {
+    }
+}
+
+message ChunkAnnounceRequest {
+    repeated string file_ids = 1;
+    string peer_addr = 2;
+    string rack = 3;
+    int32 ttl_seconds = 4;
+    string data_center = 5;
+}
+
+message ChunkAnnounceResponse {
+    repeated string rejected_file_ids = 1; // receiver is not the owner of these fids
+}
+
+message ChunkLookupRequest {
+    repeated string file_ids = 1;
+}
+
+message ChunkLookupResponse {
+    map<string, PeerSet> peers_by_fid = 1;
+    repeated string not_owner_file_ids = 2;
+}
+
+message PeerSet {
+    repeated PeerInfo peers = 1;
+}
+
+message PeerInfo {
+    string peer_addr = 1;
+    string rack = 2;
+    string data_center = 3;
+}
+
+message FetchChunkRequest {
+    string file_id = 1;
+    string expected_etag = 2;  // caller's expected MD5 over the FULL chunk.
+                                // Only meaningful when offset=0 and length=0
+                                // (a whole-chunk fetch); partial reads can't
+                                // be verified against a whole-chunk MD5.
+                                // Fetcher re-verifies end-to-end.
+    uint64 expected_size = 3;  // filer-reported chunk byte count; server sizes
+                                // its cache-read buffer to exactly this so a
+                                // sub-max-part-size chunk doesn't trigger the
+                                // cache wrapper's all-or-nothing miss path.
+    uint64 offset = 4;         // optional: byte offset within the chunk to
+                                // start reading from. 0 (default) means a
+                                // whole-chunk transfer starting at byte zero.
+    uint64 length = 5;         // optional: number of bytes to return from
+                                // offset. 0 (default) means "until end of
+                                // chunk". Range is capped server-side by
+                                // maxFetchChunkBytes regardless.
+}
+
+message FetchChunkResponse {
+    bytes data = 1;  // next frame of chunk bytes; concatenate across stream
+}

weed/pb/mount_peer_pb/mount_peer.pb.go

@@ -0,0 +1,581 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.6
+// 	protoc        v6.33.4
+// source: mount_peer.proto
+
+package mount_peer_pb
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+	unsafe "unsafe"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type ChunkAnnounceRequest struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	FileIds       []string               `protobuf:"bytes,1,rep,name=file_ids,json=fileIds,proto3" json:"file_ids,omitempty"`
+	PeerAddr      string                 `protobuf:"bytes,2,opt,name=peer_addr,json=peerAddr,proto3" json:"peer_addr,omitempty"`
+	Rack          string                 `protobuf:"bytes,3,opt,name=rack,proto3" json:"rack,omitempty"`
+	TtlSeconds    int32                  `protobuf:"varint,4,opt,name=ttl_seconds,json=ttlSeconds,proto3" json:"ttl_seconds,omitempty"`
+	DataCenter    string                 `protobuf:"bytes,5,opt,name=data_center,json=dataCenter,proto3" json:"data_center,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ChunkAnnounceRequest) Reset() {
+	*x = ChunkAnnounceRequest{}
+	mi := &file_mount_peer_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ChunkAnnounceRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ChunkAnnounceRequest) ProtoMessage() {}
+
+func (x *ChunkAnnounceRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_mount_peer_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ChunkAnnounceRequest.ProtoReflect.Descriptor instead.
+func (*ChunkAnnounceRequest) Descriptor() ([]byte, []int) {
+	return file_mount_peer_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *ChunkAnnounceRequest) GetFileIds() []string {
+	if x != nil {
+		return x.FileIds
+	}
+	return nil
+}
+
+func (x *ChunkAnnounceRequest) GetPeerAddr() string {
+	if x != nil {
+		return x.PeerAddr
+	}
+	return ""
+}
+
+func (x *ChunkAnnounceRequest) GetRack() string {
+	if x != nil {
+		return x.Rack
+	}
+	return ""
+}
+
+func (x *ChunkAnnounceRequest) GetTtlSeconds() int32 {
+	if x != nil {
+		return x.TtlSeconds
+	}
+	return 0
+}
+
+func (x *ChunkAnnounceRequest) GetDataCenter() string {
+	if x != nil {
+		return x.DataCenter
+	}
+	return ""
+}
+
+type ChunkAnnounceResponse struct {
+	state           protoimpl.MessageState `protogen:"open.v1"`
+	RejectedFileIds []string               `protobuf:"bytes,1,rep,name=rejected_file_ids,json=rejectedFileIds,proto3" json:"rejected_file_ids,omitempty"` // receiver is not the owner of these fids
+	unknownFields   protoimpl.UnknownFields
+	sizeCache       protoimpl.SizeCache
+}
+
+func (x *ChunkAnnounceResponse) Reset() {
+	*x = ChunkAnnounceResponse{}
+	mi := &file_mount_peer_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ChunkAnnounceResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ChunkAnnounceResponse) ProtoMessage() {}
+
+func (x *ChunkAnnounceResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_mount_peer_proto_msgTypes[1]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ChunkAnnounceResponse.ProtoReflect.Descriptor instead.
+func (*ChunkAnnounceResponse) Descriptor() ([]byte, []int) {
+	return file_mount_peer_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *ChunkAnnounceResponse) GetRejectedFileIds() []string {
+	if x != nil {
+		return x.RejectedFileIds
+	}
+	return nil
+}
+
+type ChunkLookupRequest struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	FileIds       []string               `protobuf:"bytes,1,rep,name=file_ids,json=fileIds,proto3" json:"file_ids,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ChunkLookupRequest) Reset() {
+	*x = ChunkLookupRequest{}
+	mi := &file_mount_peer_proto_msgTypes[2]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ChunkLookupRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ChunkLookupRequest) ProtoMessage() {}
+
+func (x *ChunkLookupRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_mount_peer_proto_msgTypes[2]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ChunkLookupRequest.ProtoReflect.Descriptor instead.
+func (*ChunkLookupRequest) Descriptor() ([]byte, []int) {
+	return file_mount_peer_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *ChunkLookupRequest) GetFileIds() []string {
+	if x != nil {
+		return x.FileIds
+	}
+	return nil
+}
+
+type ChunkLookupResponse struct {
+	state           protoimpl.MessageState `protogen:"open.v1"`
+	PeersByFid      map[string]*PeerSet    `protobuf:"bytes,1,rep,name=peers_by_fid,json=peersByFid,proto3" json:"peers_by_fid,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	NotOwnerFileIds []string               `protobuf:"bytes,2,rep,name=not_owner_file_ids,json=notOwnerFileIds,proto3" json:"not_owner_file_ids,omitempty"`
+	unknownFields   protoimpl.UnknownFields
+	sizeCache       protoimpl.SizeCache
+}
+
+func (x *ChunkLookupResponse) Reset() {
+	*x = ChunkLookupResponse{}
+	mi := &file_mount_peer_proto_msgTypes[3]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ChunkLookupResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ChunkLookupResponse) ProtoMessage() {}
+
+func (x *ChunkLookupResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_mount_peer_proto_msgTypes[3]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ChunkLookupResponse.ProtoReflect.Descriptor instead.
+func (*ChunkLookupResponse) Descriptor() ([]byte, []int) {
+	return file_mount_peer_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *ChunkLookupResponse) GetPeersByFid() map[string]*PeerSet {
+	if x != nil {
+		return x.PeersByFid
+	}
+	return nil
+}
+
+func (x *ChunkLookupResponse) GetNotOwnerFileIds() []string {
+	if x != nil {
+		return x.NotOwnerFileIds
+	}
+	return nil
+}
+
+type PeerSet struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Peers         []*PeerInfo            `protobuf:"bytes,1,rep,name=peers,proto3" json:"peers,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *PeerSet) Reset() {
+	*x = PeerSet{}
+	mi := &file_mount_peer_proto_msgTypes[4]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *PeerSet) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*PeerSet) ProtoMessage() {}
+
+func (x *PeerSet) ProtoReflect() protoreflect.Message {
+	mi := &file_mount_peer_proto_msgTypes[4]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use PeerSet.ProtoReflect.Descriptor instead.
+func (*PeerSet) Descriptor() ([]byte, []int) {
+	return file_mount_peer_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *PeerSet) GetPeers() []*PeerInfo {
+	if x != nil {
+		return x.Peers
+	}
+	return nil
+}
+
+type PeerInfo struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	PeerAddr      string                 `protobuf:"bytes,1,opt,name=peer_addr,json=peerAddr,proto3" json:"peer_addr,omitempty"`
+	Rack          string                 `protobuf:"bytes,2,opt,name=rack,proto3" json:"rack,omitempty"`
+	DataCenter    string                 `protobuf:"bytes,3,opt,name=data_center,json=dataCenter,proto3" json:"data_center,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *PeerInfo) Reset() {
+	*x = PeerInfo{}
+	mi := &file_mount_peer_proto_msgTypes[5]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *PeerInfo) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*PeerInfo) ProtoMessage() {}
+
+func (x *PeerInfo) ProtoReflect() protoreflect.Message {
+	mi := &file_mount_peer_proto_msgTypes[5]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use PeerInfo.ProtoReflect.Descriptor instead.
+func (*PeerInfo) Descriptor() ([]byte, []int) {
+	return file_mount_peer_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *PeerInfo) GetPeerAddr() string {
+	if x != nil {
+		return x.PeerAddr
+	}
+	return ""
+}
+
+func (x *PeerInfo) GetRack() string {
+	if x != nil {
+		return x.Rack
+	}
+	return ""
+}
+
+func (x *PeerInfo) GetDataCenter() string {
+	if x != nil {
+		return x.DataCenter
+	}
+	return ""
+}
+
+type FetchChunkRequest struct {
+	state        protoimpl.MessageState `protogen:"open.v1"`
+	FileId       string                 `protobuf:"bytes,1,opt,name=file_id,json=fileId,proto3" json:"file_id,omitempty"`
+	ExpectedEtag string                 `protobuf:"bytes,2,opt,name=expected_etag,json=expectedEtag,proto3" json:"expected_etag,omitempty"` // caller's expected MD5 over the FULL chunk.
+	// Only meaningful when offset=0 and length=0
+	// (a whole-chunk fetch); partial reads can't
+	// be verified against a whole-chunk MD5.
+	// Fetcher re-verifies end-to-end.
+	ExpectedSize uint64 `protobuf:"varint,3,opt,name=expected_size,json=expectedSize,proto3" json:"expected_size,omitempty"` // filer-reported chunk byte count; server sizes
+	// its cache-read buffer to exactly this so a
+	// sub-max-part-size chunk doesn't trigger the
+	// cache wrapper's all-or-nothing miss path.
+	Offset uint64 `protobuf:"varint,4,opt,name=offset,proto3" json:"offset,omitempty"` // optional: byte offset within the chunk to
+	// start reading from. 0 (default) means a
+	// whole-chunk transfer starting at byte zero.
+	Length        uint64 `protobuf:"varint,5,opt,name=length,proto3" json:"length,omitempty"` // optional: number of bytes to return from
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *FetchChunkRequest) Reset() {
+	*x = FetchChunkRequest{}
+	mi := &file_mount_peer_proto_msgTypes[6]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *FetchChunkRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*FetchChunkRequest) ProtoMessage() {}
+
+func (x *FetchChunkRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_mount_peer_proto_msgTypes[6]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use FetchChunkRequest.ProtoReflect.Descriptor instead.
+func (*FetchChunkRequest) Descriptor() ([]byte, []int) {
+	return file_mount_peer_proto_rawDescGZIP(), []int{6}
+}
+
+func (x *FetchChunkRequest) GetFileId() string {
+	if x != nil {
+		return x.FileId
+	}
+	return ""
+}
+
+func (x *FetchChunkRequest) GetExpectedEtag() string {
+	if x != nil {
+		return x.ExpectedEtag
+	}
+	return ""
+}
+
+func (x *FetchChunkRequest) GetExpectedSize() uint64 {
+	if x != nil {
+		return x.ExpectedSize
+	}
+	return 0
+}
+
+func (x *FetchChunkRequest) GetOffset() uint64 {
+	if x != nil {
+		return x.Offset
+	}
+	return 0
+}
+
+func (x *FetchChunkRequest) GetLength() uint64 {
+	if x != nil {
+		return x.Length
+	}
+	return 0
+}
+
+type FetchChunkResponse struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Data          []byte                 `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` // next frame of chunk bytes; concatenate across stream
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *FetchChunkResponse) Reset() {
+	*x = FetchChunkResponse{}
+	mi := &file_mount_peer_proto_msgTypes[7]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *FetchChunkResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*FetchChunkResponse) ProtoMessage() {}
+
+func (x *FetchChunkResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_mount_peer_proto_msgTypes[7]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use FetchChunkResponse.ProtoReflect.Descriptor instead.
+func (*FetchChunkResponse) Descriptor() ([]byte, []int) {
+	return file_mount_peer_proto_rawDescGZIP(), []int{7}
+}
+
+func (x *FetchChunkResponse) GetData() []byte {
+	if x != nil {
+		return x.Data
+	}
+	return nil
+}
+
+var File_mount_peer_proto protoreflect.FileDescriptor
+
+const file_mount_peer_proto_rawDesc = "" +
+	"\n" +
+	"\x10mount_peer.proto\x12\rmount_peer_pb\"\xa4\x01\n" +
+	"\x14ChunkAnnounceRequest\x12\x19\n" +
+	"\bfile_ids\x18\x01 \x03(\tR\afileIds\x12\x1b\n" +
+	"\tpeer_addr\x18\x02 \x01(\tR\bpeerAddr\x12\x12\n" +
+	"\x04rack\x18\x03 \x01(\tR\x04rack\x12\x1f\n" +
+	"\vttl_seconds\x18\x04 \x01(\x05R\n" +
+	"ttlSeconds\x12\x1f\n" +
+	"\vdata_center\x18\x05 \x01(\tR\n" +
+	"dataCenter\"C\n" +
+	"\x15ChunkAnnounceResponse\x12*\n" +
+	"\x11rejected_file_ids\x18\x01 \x03(\tR\x0frejectedFileIds\"/\n" +
+	"\x12ChunkLookupRequest\x12\x19\n" +
+	"\bfile_ids\x18\x01 \x03(\tR\afileIds\"\xef\x01\n" +
+	"\x13ChunkLookupResponse\x12T\n" +
+	"\fpeers_by_fid\x18\x01 \x03(\v22.mount_peer_pb.ChunkLookupResponse.PeersByFidEntryR\n" +
+	"peersByFid\x12+\n" +
+	"\x12not_owner_file_ids\x18\x02 \x03(\tR\x0fnotOwnerFileIds\x1aU\n" +
+	"\x0fPeersByFidEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12,\n" +
+	"\x05value\x18\x02 \x01(\v2\x16.mount_peer_pb.PeerSetR\x05value:\x028\x01\"8\n" +
+	"\aPeerSet\x12-\n" +
+	"\x05peers\x18\x01 \x03(\v2\x17.mount_peer_pb.PeerInfoR\x05peers\"\\\n" +
+	"\bPeerInfo\x12\x1b\n" +
+	"\tpeer_addr\x18\x01 \x01(\tR\bpeerAddr\x12\x12\n" +
+	"\x04rack\x18\x02 \x01(\tR\x04rack\x12\x1f\n" +
+	"\vdata_center\x18\x03 \x01(\tR\n" +
+	"dataCenter\"\xa6\x01\n" +
+	"\x11FetchChunkRequest\x12\x17\n" +
+	"\afile_id\x18\x01 \x01(\tR\x06fileId\x12#\n" +
+	"\rexpected_etag\x18\x02 \x01(\tR\fexpectedEtag\x12#\n" +
+	"\rexpected_size\x18\x03 \x01(\x04R\fexpectedSize\x12\x16\n" +
+	"\x06offset\x18\x04 \x01(\x04R\x06offset\x12\x16\n" +
+	"\x06length\x18\x05 \x01(\x04R\x06length\"(\n" +
+	"\x12FetchChunkResponse\x12\x12\n" +
+	"\x04data\x18\x01 \x01(\fR\x04data2\x98\x02\n" +
+	"\tMountPeer\x12\\\n" +
+	"\rChunkAnnounce\x12#.mount_peer_pb.ChunkAnnounceRequest\x1a$.mount_peer_pb.ChunkAnnounceResponse\"\x00\x12V\n" +
+	"\vChunkLookup\x12!.mount_peer_pb.ChunkLookupRequest\x1a\".mount_peer_pb.ChunkLookupResponse\"\x00\x12U\n" +
+	"\n" +
+	"FetchChunk\x12 .mount_peer_pb.FetchChunkRequest\x1a!.mount_peer_pb.FetchChunkResponse\"\x000\x01B6Z4github.com/seaweedfs/seaweedfs/weed/pb/mount_peer_pbb\x06proto3"
+
+var (
+	file_mount_peer_proto_rawDescOnce sync.Once
+	file_mount_peer_proto_rawDescData []byte
+)
+
+func file_mount_peer_proto_rawDescGZIP() []byte {
+	file_mount_peer_proto_rawDescOnce.Do(func() {
+		file_mount_peer_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_mount_peer_proto_rawDesc), len(file_mount_peer_proto_rawDesc)))
+	})
+	return file_mount_peer_proto_rawDescData
+}
+
+var file_mount_peer_proto_msgTypes = make([]protoimpl.MessageInfo, 9)
+var file_mount_peer_proto_goTypes = []any{
+	(*ChunkAnnounceRequest)(nil),  // 0: mount_peer_pb.ChunkAnnounceRequest
+	(*ChunkAnnounceResponse)(nil), // 1: mount_peer_pb.ChunkAnnounceResponse
+	(*ChunkLookupRequest)(nil),    // 2: mount_peer_pb.ChunkLookupRequest
+	(*ChunkLookupResponse)(nil),   // 3: mount_peer_pb.ChunkLookupResponse
+	(*PeerSet)(nil),               // 4: mount_peer_pb.PeerSet
+	(*PeerInfo)(nil),              // 5: mount_peer_pb.PeerInfo
+	(*FetchChunkRequest)(nil),     // 6: mount_peer_pb.FetchChunkRequest
+	(*FetchChunkResponse)(nil),    // 7: mount_peer_pb.FetchChunkResponse
+	nil,                           // 8: mount_peer_pb.ChunkLookupResponse.PeersByFidEntry
+}
+var file_mount_peer_proto_depIdxs = []int32{
+	8, // 0: mount_peer_pb.ChunkLookupResponse.peers_by_fid:type_name -> mount_peer_pb.ChunkLookupResponse.PeersByFidEntry
+	5, // 1: mount_peer_pb.PeerSet.peers:type_name -> mount_peer_pb.PeerInfo
+	4, // 2: mount_peer_pb.ChunkLookupResponse.PeersByFidEntry.value:type_name -> mount_peer_pb.PeerSet
+	0, // 3: mount_peer_pb.MountPeer.ChunkAnnounce:input_type -> mount_peer_pb.ChunkAnnounceRequest
+	2, // 4: mount_peer_pb.MountPeer.ChunkLookup:input_type -> mount_peer_pb.ChunkLookupRequest
+	6, // 5: mount_peer_pb.MountPeer.FetchChunk:input_type -> mount_peer_pb.FetchChunkRequest
+	1, // 6: mount_peer_pb.MountPeer.ChunkAnnounce:output_type -> mount_peer_pb.ChunkAnnounceResponse
+	3, // 7: mount_peer_pb.MountPeer.ChunkLookup:output_type -> mount_peer_pb.ChunkLookupResponse
+	7, // 8: mount_peer_pb.MountPeer.FetchChunk:output_type -> mount_peer_pb.FetchChunkResponse
+	6, // [6:9] is the sub-list for method output_type
+	3, // [3:6] is the sub-list for method input_type
+	3, // [3:3] is the sub-list for extension type_name
+	3, // [3:3] is the sub-list for extension extendee
+	0, // [0:3] is the sub-list for field type_name
+}
+
+func init() { file_mount_peer_proto_init() }
+func file_mount_peer_proto_init() {
+	if File_mount_peer_proto != nil {
+		return
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_mount_peer_proto_rawDesc), len(file_mount_peer_proto_rawDesc)),
+			NumEnums:      0,
+			NumMessages:   9,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_mount_peer_proto_goTypes,
+		DependencyIndexes: file_mount_peer_proto_depIdxs,
+		MessageInfos:      file_mount_peer_proto_msgTypes,
+	}.Build()
+	File_mount_peer_proto = out.File
+	file_mount_peer_proto_goTypes = nil
+	file_mount_peer_proto_depIdxs = nil
+}

weed/pb/mount_peer_pb/mount_peer_grpc.pb.go

@@ -0,0 +1,229 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.5.1
+// - protoc             v6.33.4
+// source: mount_peer.proto
+
+package mount_peer_pb
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.64.0 or later.
+const _ = grpc.SupportPackageIsVersion9
+
+const (
+	MountPeer_ChunkAnnounce_FullMethodName = "/mount_peer_pb.MountPeer/ChunkAnnounce"
+	MountPeer_ChunkLookup_FullMethodName   = "/mount_peer_pb.MountPeer/ChunkLookup"
+	MountPeer_FetchChunk_FullMethodName    = "/mount_peer_pb.MountPeer/FetchChunk"
+)
+
+// MountPeerClient is the client API for MountPeer service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type MountPeerClient interface {
+	// ChunkAnnounce: the caller asserts it currently holds the listed fids in
+	// its local chunk cache and is willing to serve them to peers. The
+	// receiver accepts only fids for which it is the HRW-assigned owner on
+	// its current seed view; others are returned in rejected_file_ids.
+	ChunkAnnounce(ctx context.Context, in *ChunkAnnounceRequest, opts ...grpc.CallOption) (*ChunkAnnounceResponse, error)
+	// ChunkLookup: asks the receiver for known holders of each requested fid.
+	// The receiver responds only for fids it owns; others are listed in
+	// not_owner_file_ids so the caller can retry against the correct owner
+	// after refreshing its own seed view.
+	ChunkLookup(ctx context.Context, in *ChunkLookupRequest, opts ...grpc.CallOption) (*ChunkLookupResponse, error)
+	// FetchChunk: server-streams the bytes of a cached chunk to a peer.
+	// Streaming avoids the default gRPC 4 MiB message cap for typical
+	// 16 MiB chunks and lets the receiver assemble into a preallocated
+	// buffer. Not-cached / cache-miss returns a gRPC NOT_FOUND status.
+	// The fetcher re-verifies MD5 against expected_etag end-to-end after
+	// the stream completes.
+	FetchChunk(ctx context.Context, in *FetchChunkRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[FetchChunkResponse], error)
+}
+
+type mountPeerClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewMountPeerClient(cc grpc.ClientConnInterface) MountPeerClient {
+	return &mountPeerClient{cc}
+}
+
+func (c *mountPeerClient) ChunkAnnounce(ctx context.Context, in *ChunkAnnounceRequest, opts ...grpc.CallOption) (*ChunkAnnounceResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ChunkAnnounceResponse)
+	err := c.cc.Invoke(ctx, MountPeer_ChunkAnnounce_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *mountPeerClient) ChunkLookup(ctx context.Context, in *ChunkLookupRequest, opts ...grpc.CallOption) (*ChunkLookupResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ChunkLookupResponse)
+	err := c.cc.Invoke(ctx, MountPeer_ChunkLookup_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *mountPeerClient) FetchChunk(ctx context.Context, in *FetchChunkRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[FetchChunkResponse], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &MountPeer_ServiceDesc.Streams[0], MountPeer_FetchChunk_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[FetchChunkRequest, FetchChunkResponse]{ClientStream: stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type MountPeer_FetchChunkClient = grpc.ServerStreamingClient[FetchChunkResponse]
+
+// MountPeerServer is the server API for MountPeer service.
+// All implementations must embed UnimplementedMountPeerServer
+// for forward compatibility.
+type MountPeerServer interface {
+	// ChunkAnnounce: the caller asserts it currently holds the listed fids in
+	// its local chunk cache and is willing to serve them to peers. The
+	// receiver accepts only fids for which it is the HRW-assigned owner on
+	// its current seed view; others are returned in rejected_file_ids.
+	ChunkAnnounce(context.Context, *ChunkAnnounceRequest) (*ChunkAnnounceResponse, error)
+	// ChunkLookup: asks the receiver for known holders of each requested fid.
+	// The receiver responds only for fids it owns; others are listed in
+	// not_owner_file_ids so the caller can retry against the correct owner
+	// after refreshing its own seed view.
+	ChunkLookup(context.Context, *ChunkLookupRequest) (*ChunkLookupResponse, error)
+	// FetchChunk: server-streams the bytes of a cached chunk to a peer.
+	// Streaming avoids the default gRPC 4 MiB message cap for typical
+	// 16 MiB chunks and lets the receiver assemble into a preallocated
+	// buffer. Not-cached / cache-miss returns a gRPC NOT_FOUND status.
+	// The fetcher re-verifies MD5 against expected_etag end-to-end after
+	// the stream completes.
+	FetchChunk(*FetchChunkRequest, grpc.ServerStreamingServer[FetchChunkResponse]) error
+	mustEmbedUnimplementedMountPeerServer()
+}
+
+// UnimplementedMountPeerServer must be embedded to have
+// forward compatible implementations.
+//
+// NOTE: this should be embedded by value instead of pointer to avoid a nil
+// pointer dereference when methods are called.
+type UnimplementedMountPeerServer struct{}
+
+func (UnimplementedMountPeerServer) ChunkAnnounce(context.Context, *ChunkAnnounceRequest) (*ChunkAnnounceResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ChunkAnnounce not implemented")
+}
+func (UnimplementedMountPeerServer) ChunkLookup(context.Context, *ChunkLookupRequest) (*ChunkLookupResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ChunkLookup not implemented")
+}
+func (UnimplementedMountPeerServer) FetchChunk(*FetchChunkRequest, grpc.ServerStreamingServer[FetchChunkResponse]) error {
+	return status.Errorf(codes.Unimplemented, "method FetchChunk not implemented")
+}
+func (UnimplementedMountPeerServer) mustEmbedUnimplementedMountPeerServer() {}
+func (UnimplementedMountPeerServer) testEmbeddedByValue()                   {}
+
+// UnsafeMountPeerServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to MountPeerServer will
+// result in compilation errors.
+type UnsafeMountPeerServer interface {
+	mustEmbedUnimplementedMountPeerServer()
+}
+
+func RegisterMountPeerServer(s grpc.ServiceRegistrar, srv MountPeerServer) {
+	// If the following call pancis, it indicates UnimplementedMountPeerServer was
+	// embedded by pointer and is nil.  This will cause panics if an
+	// unimplemented method is ever invoked, so we test this at initialization
+	// time to prevent it from happening at runtime later due to I/O.
+	if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
+		t.testEmbeddedByValue()
+	}
+	s.RegisterService(&MountPeer_ServiceDesc, srv)
+}
+
+func _MountPeer_ChunkAnnounce_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ChunkAnnounceRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(MountPeerServer).ChunkAnnounce(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: MountPeer_ChunkAnnounce_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(MountPeerServer).ChunkAnnounce(ctx, req.(*ChunkAnnounceRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _MountPeer_ChunkLookup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ChunkLookupRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(MountPeerServer).ChunkLookup(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: MountPeer_ChunkLookup_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(MountPeerServer).ChunkLookup(ctx, req.(*ChunkLookupRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _MountPeer_FetchChunk_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(FetchChunkRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(MountPeerServer).FetchChunk(m, &grpc.GenericServerStream[FetchChunkRequest, FetchChunkResponse]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type MountPeer_FetchChunkServer = grpc.ServerStreamingServer[FetchChunkResponse]
+
+// MountPeer_ServiceDesc is the grpc.ServiceDesc for MountPeer service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var MountPeer_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "mount_peer_pb.MountPeer",
+	HandlerType: (*MountPeerServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "ChunkAnnounce",
+			Handler:    _MountPeer_ChunkAnnounce_Handler,
+		},
+		{
+			MethodName: "ChunkLookup",
+			Handler:    _MountPeer_ChunkLookup_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "FetchChunk",
+			Handler:       _MountPeer_FetchChunk_Handler,
+			ServerStreams: true,
+		},
+	},
+	Metadata: "mount_peer.proto",
+}

weed/pb/mq_agent.proto

@@ -0,0 +1,85 @@
+syntax = "proto3";
+
+package messaging_pb;
+
+import "mq_schema.proto";
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/mq_agent_pb";
+option java_package = "seaweedfs.mq_agent";
+option java_outer_classname = "MessageQueueAgentProto";
+
+//////////////////////////////////////////////////
+
+service SeaweedMessagingAgent {
+
+    // Publishing
+    rpc StartPublishSession (StartPublishSessionRequest) returns (StartPublishSessionResponse) {
+    }
+    rpc ClosePublishSession (ClosePublishSessionRequest) returns (ClosePublishSessionResponse) {
+    }
+    rpc PublishRecord (stream PublishRecordRequest) returns (stream PublishRecordResponse) {
+    }
+
+    // Subscribing
+    rpc SubscribeRecord (stream SubscribeRecordRequest) returns (stream SubscribeRecordResponse) {
+    }
+
+}
+
+//////////////////////////////////////////////////
+message StartPublishSessionRequest {
+    schema_pb.Topic topic = 1;
+    int32 partition_count = 2;
+    schema_pb.RecordType record_type = 3;
+    string publisher_name = 4;
+}
+message StartPublishSessionResponse {
+    string error = 1;
+    int64 session_id = 2;
+}
+message ClosePublishSessionRequest {
+    int64 session_id = 1;
+}
+message ClosePublishSessionResponse {
+    string error = 1;
+}
+
+//////////////////////////////////////////////////
+message PublishRecordRequest {
+    int64 session_id = 1; // session_id is required for the first record
+    bytes key = 2;
+    schema_pb.RecordValue value = 3;
+}
+message PublishRecordResponse {
+    int64 ack_sequence = 1;
+    string error = 2;
+    int64 base_offset = 3;  // First offset assigned to this batch
+    int64 last_offset = 4;  // Last offset assigned to this batch
+}
+//////////////////////////////////////////////////
+message SubscribeRecordRequest {
+    message InitSubscribeRecordRequest {
+        string consumer_group = 1;
+        string consumer_group_instance_id = 2;
+        schema_pb.Topic topic = 4;
+        repeated schema_pb.PartitionOffset partition_offsets = 5;
+        schema_pb.OffsetType offset_type = 6;
+        int64 offset_ts_ns = 7;
+        string filter = 10;
+        int32 max_subscribed_partitions = 11;
+        int32 sliding_window_size = 12;
+    }
+    InitSubscribeRecordRequest init = 1;
+    int64 ack_sequence = 2;
+    bytes ack_key = 3;
+}
+message SubscribeRecordResponse {
+    bytes key = 2;
+    schema_pb.RecordValue value = 3;
+    int64 ts_ns = 4;
+    string error = 5;
+    bool is_end_of_stream = 6;
+    bool is_end_of_topic = 7;
+    int64 offset = 8;  // Sequential offset within partition
+}
+//////////////////////////////////////////////////

weed/pb/mq_agent_pb/mq_agent.pb.go

@@ -0,0 +1,763 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.6
+// 	protoc        v6.33.4
+// source: mq_agent.proto
+
+package mq_agent_pb
+
+import (
+	schema_pb "github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+	unsafe "unsafe"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// ////////////////////////////////////////////////
+type StartPublishSessionRequest struct {
+	state          protoimpl.MessageState `protogen:"open.v1"`
+	Topic          *schema_pb.Topic       `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"`
+	PartitionCount int32                  `protobuf:"varint,2,opt,name=partition_count,json=partitionCount,proto3" json:"partition_count,omitempty"`
+	RecordType     *schema_pb.RecordType  `protobuf:"bytes,3,opt,name=record_type,json=recordType,proto3" json:"record_type,omitempty"`
+	PublisherName  string                 `protobuf:"bytes,4,opt,name=publisher_name,json=publisherName,proto3" json:"publisher_name,omitempty"`
+	unknownFields  protoimpl.UnknownFields
+	sizeCache      protoimpl.SizeCache
+}
+
+func (x *StartPublishSessionRequest) Reset() {
+	*x = StartPublishSessionRequest{}
+	mi := &file_mq_agent_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *StartPublishSessionRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*StartPublishSessionRequest) ProtoMessage() {}
+
+func (x *StartPublishSessionRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_mq_agent_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use StartPublishSessionRequest.ProtoReflect.Descriptor instead.
+func (*StartPublishSessionRequest) Descriptor() ([]byte, []int) {
+	return file_mq_agent_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *StartPublishSessionRequest) GetTopic() *schema_pb.Topic {
+	if x != nil {
+		return x.Topic
+	}
+	return nil
+}
+
+func (x *StartPublishSessionRequest) GetPartitionCount() int32 {
+	if x != nil {
+		return x.PartitionCount
+	}
+	return 0
+}
+
+func (x *StartPublishSessionRequest) GetRecordType() *schema_pb.RecordType {
+	if x != nil {
+		return x.RecordType
+	}
+	return nil
+}
+
+func (x *StartPublishSessionRequest) GetPublisherName() string {
+	if x != nil {
+		return x.PublisherName
+	}
+	return ""
+}
+
+type StartPublishSessionResponse struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Error         string                 `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"`
+	SessionId     int64                  `protobuf:"varint,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *StartPublishSessionResponse) Reset() {
+	*x = StartPublishSessionResponse{}
+	mi := &file_mq_agent_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *StartPublishSessionResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*StartPublishSessionResponse) ProtoMessage() {}
+
+func (x *StartPublishSessionResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_mq_agent_proto_msgTypes[1]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use StartPublishSessionResponse.ProtoReflect.Descriptor instead.
+func (*StartPublishSessionResponse) Descriptor() ([]byte, []int) {
+	return file_mq_agent_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *StartPublishSessionResponse) GetError() string {
+	if x != nil {
+		return x.Error
+	}
+	return ""
+}
+
+func (x *StartPublishSessionResponse) GetSessionId() int64 {
+	if x != nil {
+		return x.SessionId
+	}
+	return 0
+}
+
+type ClosePublishSessionRequest struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	SessionId     int64                  `protobuf:"varint,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ClosePublishSessionRequest) Reset() {
+	*x = ClosePublishSessionRequest{}
+	mi := &file_mq_agent_proto_msgTypes[2]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ClosePublishSessionRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ClosePublishSessionRequest) ProtoMessage() {}
+
+func (x *ClosePublishSessionRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_mq_agent_proto_msgTypes[2]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ClosePublishSessionRequest.ProtoReflect.Descriptor instead.
+func (*ClosePublishSessionRequest) Descriptor() ([]byte, []int) {
+	return file_mq_agent_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *ClosePublishSessionRequest) GetSessionId() int64 {
+	if x != nil {
+		return x.SessionId
+	}
+	return 0
+}
+
+type ClosePublishSessionResponse struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Error         string                 `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ClosePublishSessionResponse) Reset() {
+	*x = ClosePublishSessionResponse{}
+	mi := &file_mq_agent_proto_msgTypes[3]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ClosePublishSessionResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ClosePublishSessionResponse) ProtoMessage() {}
+
+func (x *ClosePublishSessionResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_mq_agent_proto_msgTypes[3]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ClosePublishSessionResponse.ProtoReflect.Descriptor instead.
+func (*ClosePublishSessionResponse) Descriptor() ([]byte, []int) {
+	return file_mq_agent_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *ClosePublishSessionResponse) GetError() string {
+	if x != nil {
+		return x.Error
+	}
+	return ""
+}
+
+// ////////////////////////////////////////////////
+type PublishRecordRequest struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	SessionId     int64                  `protobuf:"varint,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // session_id is required for the first record
+	Key           []byte                 `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
+	Value         *schema_pb.RecordValue `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *PublishRecordRequest) Reset() {
+	*x = PublishRecordRequest{}
+	mi := &file_mq_agent_proto_msgTypes[4]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *PublishRecordRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*PublishRecordRequest) ProtoMessage() {}
+
+func (x *PublishRecordRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_mq_agent_proto_msgTypes[4]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use PublishRecordRequest.ProtoReflect.Descriptor instead.
+func (*PublishRecordRequest) Descriptor() ([]byte, []int) {
+	return file_mq_agent_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *PublishRecordRequest) GetSessionId() int64 {
+	if x != nil {
+		return x.SessionId
+	}
+	return 0
+}
+
+func (x *PublishRecordRequest) GetKey() []byte {
+	if x != nil {
+		return x.Key
+	}
+	return nil
+}
+
+func (x *PublishRecordRequest) GetValue() *schema_pb.RecordValue {
+	if x != nil {
+		return x.Value
+	}
+	return nil
+}
+
+type PublishRecordResponse struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	AckSequence   int64                  `protobuf:"varint,1,opt,name=ack_sequence,json=ackSequence,proto3" json:"ack_sequence,omitempty"`
+	Error         string                 `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"`
+	BaseOffset    int64                  `protobuf:"varint,3,opt,name=base_offset,json=baseOffset,proto3" json:"base_offset,omitempty"` // First offset assigned to this batch
+	LastOffset    int64                  `protobuf:"varint,4,opt,name=last_offset,json=lastOffset,proto3" json:"last_offset,omitempty"` // Last offset assigned to this batch
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *PublishRecordResponse) Reset() {
+	*x = PublishRecordResponse{}
+	mi := &file_mq_agent_proto_msgTypes[5]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *PublishRecordResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*PublishRecordResponse) ProtoMessage() {}
+
+func (x *PublishRecordResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_mq_agent_proto_msgTypes[5]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use PublishRecordResponse.ProtoReflect.Descriptor instead.
+func (*PublishRecordResponse) Descriptor() ([]byte, []int) {
+	return file_mq_agent_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *PublishRecordResponse) GetAckSequence() int64 {
+	if x != nil {
+		return x.AckSequence
+	}
+	return 0
+}
+
+func (x *PublishRecordResponse) GetError() string {
+	if x != nil {
+		return x.Error
+	}
+	return ""
+}
+
+func (x *PublishRecordResponse) GetBaseOffset() int64 {
+	if x != nil {
+		return x.BaseOffset
+	}
+	return 0
+}
+
+func (x *PublishRecordResponse) GetLastOffset() int64 {
+	if x != nil {
+		return x.LastOffset
+	}
+	return 0
+}
+
+// ////////////////////////////////////////////////
+type SubscribeRecordRequest struct {
+	state         protoimpl.MessageState                             `protogen:"open.v1"`
+	Init          *SubscribeRecordRequest_InitSubscribeRecordRequest `protobuf:"bytes,1,opt,name=init,proto3" json:"init,omitempty"`
+	AckSequence   int64                                              `protobuf:"varint,2,opt,name=ack_sequence,json=ackSequence,proto3" json:"ack_sequence,omitempty"`
+	AckKey        []byte                                             `protobuf:"bytes,3,opt,name=ack_key,json=ackKey,proto3" json:"ack_key,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *SubscribeRecordRequest) Reset() {
+	*x = SubscribeRecordRequest{}
+	mi := &file_mq_agent_proto_msgTypes[6]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *SubscribeRecordRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SubscribeRecordRequest) ProtoMessage() {}
+
+func (x *SubscribeRecordRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_mq_agent_proto_msgTypes[6]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SubscribeRecordRequest.ProtoReflect.Descriptor instead.
+func (*SubscribeRecordRequest) Descriptor() ([]byte, []int) {
+	return file_mq_agent_proto_rawDescGZIP(), []int{6}
+}
+
+func (x *SubscribeRecordRequest) GetInit() *SubscribeRecordRequest_InitSubscribeRecordRequest {
+	if x != nil {
+		return x.Init
+	}
+	return nil
+}
+
+func (x *SubscribeRecordRequest) GetAckSequence() int64 {
+	if x != nil {
+		return x.AckSequence
+	}
+	return 0
+}
+
+func (x *SubscribeRecordRequest) GetAckKey() []byte {
+	if x != nil {
+		return x.AckKey
+	}
+	return nil
+}
+
+type SubscribeRecordResponse struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Key           []byte                 `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
+	Value         *schema_pb.RecordValue `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"`
+	TsNs          int64                  `protobuf:"varint,4,opt,name=ts_ns,json=tsNs,proto3" json:"ts_ns,omitempty"`
+	Error         string                 `protobuf:"bytes,5,opt,name=error,proto3" json:"error,omitempty"`
+	IsEndOfStream bool                   `protobuf:"varint,6,opt,name=is_end_of_stream,json=isEndOfStream,proto3" json:"is_end_of_stream,omitempty"`
+	IsEndOfTopic  bool                   `protobuf:"varint,7,opt,name=is_end_of_topic,json=isEndOfTopic,proto3" json:"is_end_of_topic,omitempty"`
+	Offset        int64                  `protobuf:"varint,8,opt,name=offset,proto3" json:"offset,omitempty"` // Sequential offset within partition
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *SubscribeRecordResponse) Reset() {
+	*x = SubscribeRecordResponse{}
+	mi := &file_mq_agent_proto_msgTypes[7]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *SubscribeRecordResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SubscribeRecordResponse) ProtoMessage() {}
+
+func (x *SubscribeRecordResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_mq_agent_proto_msgTypes[7]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SubscribeRecordResponse.ProtoReflect.Descriptor instead.
+func (*SubscribeRecordResponse) Descriptor() ([]byte, []int) {
+	return file_mq_agent_proto_rawDescGZIP(), []int{7}
+}
+
+func (x *SubscribeRecordResponse) GetKey() []byte {
+	if x != nil {
+		return x.Key
+	}
+	return nil
+}
+
+func (x *SubscribeRecordResponse) GetValue() *schema_pb.RecordValue {
+	if x != nil {
+		return x.Value
+	}
+	return nil
+}
+
+func (x *SubscribeRecordResponse) GetTsNs() int64 {
+	if x != nil {
+		return x.TsNs
+	}
+	return 0
+}
+
+func (x *SubscribeRecordResponse) GetError() string {
+	if x != nil {
+		return x.Error
+	}
+	return ""
+}
+
+func (x *SubscribeRecordResponse) GetIsEndOfStream() bool {
+	if x != nil {
+		return x.IsEndOfStream
+	}
+	return false
+}
+
+func (x *SubscribeRecordResponse) GetIsEndOfTopic() bool {
+	if x != nil {
+		return x.IsEndOfTopic
+	}
+	return false
+}
+
+func (x *SubscribeRecordResponse) GetOffset() int64 {
+	if x != nil {
+		return x.Offset
+	}
+	return 0
+}
+
+type SubscribeRecordRequest_InitSubscribeRecordRequest struct {
+	state                   protoimpl.MessageState       `protogen:"open.v1"`
+	ConsumerGroup           string                       `protobuf:"bytes,1,opt,name=consumer_group,json=consumerGroup,proto3" json:"consumer_group,omitempty"`
+	ConsumerGroupInstanceId string                       `protobuf:"bytes,2,opt,name=consumer_group_instance_id,json=consumerGroupInstanceId,proto3" json:"consumer_group_instance_id,omitempty"`
+	Topic                   *schema_pb.Topic             `protobuf:"bytes,4,opt,name=topic,proto3" json:"topic,omitempty"`
+	PartitionOffsets        []*schema_pb.PartitionOffset `protobuf:"bytes,5,rep,name=partition_offsets,json=partitionOffsets,proto3" json:"partition_offsets,omitempty"`
+	OffsetType              schema_pb.OffsetType         `protobuf:"varint,6,opt,name=offset_type,json=offsetType,proto3,enum=schema_pb.OffsetType" json:"offset_type,omitempty"`
+	OffsetTsNs              int64                        `protobuf:"varint,7,opt,name=offset_ts_ns,json=offsetTsNs,proto3" json:"offset_ts_ns,omitempty"`
+	Filter                  string                       `protobuf:"bytes,10,opt,name=filter,proto3" json:"filter,omitempty"`
+	MaxSubscribedPartitions int32                        `protobuf:"varint,11,opt,name=max_subscribed_partitions,json=maxSubscribedPartitions,proto3" json:"max_subscribed_partitions,omitempty"`
+	SlidingWindowSize       int32                        `protobuf:"varint,12,opt,name=sliding_window_size,json=slidingWindowSize,proto3" json:"sliding_window_size,omitempty"`
+	unknownFields           protoimpl.UnknownFields
+	sizeCache               protoimpl.SizeCache
+}
+
+func (x *SubscribeRecordRequest_InitSubscribeRecordRequest) Reset() {
+	*x = SubscribeRecordRequest_InitSubscribeRecordRequest{}
+	mi := &file_mq_agent_proto_msgTypes[8]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *SubscribeRecordRequest_InitSubscribeRecordRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SubscribeRecordRequest_InitSubscribeRecordRequest) ProtoMessage() {}
+
+func (x *SubscribeRecordRequest_InitSubscribeRecordRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_mq_agent_proto_msgTypes[8]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SubscribeRecordRequest_InitSubscribeRecordRequest.ProtoReflect.Descriptor instead.
+func (*SubscribeRecordRequest_InitSubscribeRecordRequest) Descriptor() ([]byte, []int) {
+	return file_mq_agent_proto_rawDescGZIP(), []int{6, 0}
+}
+
+func (x *SubscribeRecordRequest_InitSubscribeRecordRequest) GetConsumerGroup() string {
+	if x != nil {
+		return x.ConsumerGroup
+	}
+	return ""
+}
+
+func (x *SubscribeRecordRequest_InitSubscribeRecordRequest) GetConsumerGroupInstanceId() string {
+	if x != nil {
+		return x.ConsumerGroupInstanceId
+	}
+	return ""
+}
+
+func (x *SubscribeRecordRequest_InitSubscribeRecordRequest) GetTopic() *schema_pb.Topic {
+	if x != nil {
+		return x.Topic
+	}
+	return nil
+}
+
+func (x *SubscribeRecordRequest_InitSubscribeRecordRequest) GetPartitionOffsets() []*schema_pb.PartitionOffset {
+	if x != nil {
+		return x.PartitionOffsets
+	}
+	return nil
+}
+
+func (x *SubscribeRecordRequest_InitSubscribeRecordRequest) GetOffsetType() schema_pb.OffsetType {
+	if x != nil {
+		return x.OffsetType
+	}
+	return schema_pb.OffsetType(0)
+}
+
+func (x *SubscribeRecordRequest_InitSubscribeRecordRequest) GetOffsetTsNs() int64 {
+	if x != nil {
+		return x.OffsetTsNs
+	}
+	return 0
+}
+
+func (x *SubscribeRecordRequest_InitSubscribeRecordRequest) GetFilter() string {
+	if x != nil {
+		return x.Filter
+	}
+	return ""
+}
+
+func (x *SubscribeRecordRequest_InitSubscribeRecordRequest) GetMaxSubscribedPartitions() int32 {
+	if x != nil {
+		return x.MaxSubscribedPartitions
+	}
+	return 0
+}
+
+func (x *SubscribeRecordRequest_InitSubscribeRecordRequest) GetSlidingWindowSize() int32 {
+	if x != nil {
+		return x.SlidingWindowSize
+	}
+	return 0
+}
+
+var File_mq_agent_proto protoreflect.FileDescriptor
+
+const file_mq_agent_proto_rawDesc = "" +
+	"\n" +
+	"\x0emq_agent.proto\x12\fmessaging_pb\x1a\x0fmq_schema.proto\"\xcc\x01\n" +
+	"\x1aStartPublishSessionRequest\x12&\n" +
+	"\x05topic\x18\x01 \x01(\v2\x10.schema_pb.TopicR\x05topic\x12'\n" +
+	"\x0fpartition_count\x18\x02 \x01(\x05R\x0epartitionCount\x126\n" +
+	"\vrecord_type\x18\x03 \x01(\v2\x15.schema_pb.RecordTypeR\n" +
+	"recordType\x12%\n" +
+	"\x0epublisher_name\x18\x04 \x01(\tR\rpublisherName\"R\n" +
+	"\x1bStartPublishSessionResponse\x12\x14\n" +
+	"\x05error\x18\x01 \x01(\tR\x05error\x12\x1d\n" +
+	"\n" +
+	"session_id\x18\x02 \x01(\x03R\tsessionId\";\n" +
+	"\x1aClosePublishSessionRequest\x12\x1d\n" +
+	"\n" +
+	"session_id\x18\x01 \x01(\x03R\tsessionId\"3\n" +
+	"\x1bClosePublishSessionResponse\x12\x14\n" +
+	"\x05error\x18\x01 \x01(\tR\x05error\"u\n" +
+	"\x14PublishRecordRequest\x12\x1d\n" +
+	"\n" +
+	"session_id\x18\x01 \x01(\x03R\tsessionId\x12\x10\n" +
+	"\x03key\x18\x02 \x01(\fR\x03key\x12,\n" +
+	"\x05value\x18\x03 \x01(\v2\x16.schema_pb.RecordValueR\x05value\"\x92\x01\n" +
+	"\x15PublishRecordResponse\x12!\n" +
+	"\fack_sequence\x18\x01 \x01(\x03R\vackSequence\x12\x14\n" +
+	"\x05error\x18\x02 \x01(\tR\x05error\x12\x1f\n" +
+	"\vbase_offset\x18\x03 \x01(\x03R\n" +
+	"baseOffset\x12\x1f\n" +
+	"\vlast_offset\x18\x04 \x01(\x03R\n" +
+	"lastOffset\"\xfb\x04\n" +
+	"\x16SubscribeRecordRequest\x12S\n" +
+	"\x04init\x18\x01 \x01(\v2?.messaging_pb.SubscribeRecordRequest.InitSubscribeRecordRequestR\x04init\x12!\n" +
+	"\fack_sequence\x18\x02 \x01(\x03R\vackSequence\x12\x17\n" +
+	"\aack_key\x18\x03 \x01(\fR\x06ackKey\x1a\xcf\x03\n" +
+	"\x1aInitSubscribeRecordRequest\x12%\n" +
+	"\x0econsumer_group\x18\x01 \x01(\tR\rconsumerGroup\x12;\n" +
+	"\x1aconsumer_group_instance_id\x18\x02 \x01(\tR\x17consumerGroupInstanceId\x12&\n" +
+	"\x05topic\x18\x04 \x01(\v2\x10.schema_pb.TopicR\x05topic\x12G\n" +
+	"\x11partition_offsets\x18\x05 \x03(\v2\x1a.schema_pb.PartitionOffsetR\x10partitionOffsets\x126\n" +
+	"\voffset_type\x18\x06 \x01(\x0e2\x15.schema_pb.OffsetTypeR\n" +
+	"offsetType\x12 \n" +
+	"\foffset_ts_ns\x18\a \x01(\x03R\n" +
+	"offsetTsNs\x12\x16\n" +
+	"\x06filter\x18\n" +
+	" \x01(\tR\x06filter\x12:\n" +
+	"\x19max_subscribed_partitions\x18\v \x01(\x05R\x17maxSubscribedPartitions\x12.\n" +
+	"\x13sliding_window_size\x18\f \x01(\x05R\x11slidingWindowSize\"\xec\x01\n" +
+	"\x17SubscribeRecordResponse\x12\x10\n" +
+	"\x03key\x18\x02 \x01(\fR\x03key\x12,\n" +
+	"\x05value\x18\x03 \x01(\v2\x16.schema_pb.RecordValueR\x05value\x12\x13\n" +
+	"\x05ts_ns\x18\x04 \x01(\x03R\x04tsNs\x12\x14\n" +
+	"\x05error\x18\x05 \x01(\tR\x05error\x12'\n" +
+	"\x10is_end_of_stream\x18\x06 \x01(\bR\risEndOfStream\x12%\n" +
+	"\x0fis_end_of_topic\x18\a \x01(\bR\fisEndOfTopic\x12\x16\n" +
+	"\x06offset\x18\b \x01(\x03R\x06offset2\xb9\x03\n" +
+	"\x15SeaweedMessagingAgent\x12l\n" +
+	"\x13StartPublishSession\x12(.messaging_pb.StartPublishSessionRequest\x1a).messaging_pb.StartPublishSessionResponse\"\x00\x12l\n" +
+	"\x13ClosePublishSession\x12(.messaging_pb.ClosePublishSessionRequest\x1a).messaging_pb.ClosePublishSessionResponse\"\x00\x12^\n" +
+	"\rPublishRecord\x12\".messaging_pb.PublishRecordRequest\x1a#.messaging_pb.PublishRecordResponse\"\x00(\x010\x01\x12d\n" +
+	"\x0fSubscribeRecord\x12$.messaging_pb.SubscribeRecordRequest\x1a%.messaging_pb.SubscribeRecordResponse\"\x00(\x010\x01B`\n" +
+	"\x12seaweedfs.mq_agentB\x16MessageQueueAgentProtoZ2github.com/seaweedfs/seaweedfs/weed/pb/mq_agent_pbb\x06proto3"
+
+var (
+	file_mq_agent_proto_rawDescOnce sync.Once
+	file_mq_agent_proto_rawDescData []byte
+)
+
+func file_mq_agent_proto_rawDescGZIP() []byte {
+	file_mq_agent_proto_rawDescOnce.Do(func() {
+		file_mq_agent_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_mq_agent_proto_rawDesc), len(file_mq_agent_proto_rawDesc)))
+	})
+	return file_mq_agent_proto_rawDescData
+}
+
+var file_mq_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 9)
+var file_mq_agent_proto_goTypes = []any{
+	(*StartPublishSessionRequest)(nil),                        // 0: messaging_pb.StartPublishSessionRequest
+	(*StartPublishSessionResponse)(nil),                       // 1: messaging_pb.StartPublishSessionResponse
+	(*ClosePublishSessionRequest)(nil),                        // 2: messaging_pb.ClosePublishSessionRequest
+	(*ClosePublishSessionResponse)(nil),                       // 3: messaging_pb.ClosePublishSessionResponse
+	(*PublishRecordRequest)(nil),                              // 4: messaging_pb.PublishRecordRequest
+	(*PublishRecordResponse)(nil),                             // 5: messaging_pb.PublishRecordResponse
+	(*SubscribeRecordRequest)(nil),                            // 6: messaging_pb.SubscribeRecordRequest
+	(*SubscribeRecordResponse)(nil),                           // 7: messaging_pb.SubscribeRecordResponse
+	(*SubscribeRecordRequest_InitSubscribeRecordRequest)(nil), // 8: messaging_pb.SubscribeRecordRequest.InitSubscribeRecordRequest
+	(*schema_pb.Topic)(nil),                                   // 9: schema_pb.Topic
+	(*schema_pb.RecordType)(nil),                              // 10: schema_pb.RecordType
+	(*schema_pb.RecordValue)(nil),                             // 11: schema_pb.RecordValue
+	(*schema_pb.PartitionOffset)(nil),                         // 12: schema_pb.PartitionOffset
+	(schema_pb.OffsetType)(0),                                 // 13: schema_pb.OffsetType
+}
+var file_mq_agent_proto_depIdxs = []int32{
+	9,  // 0: messaging_pb.StartPublishSessionRequest.topic:type_name -> schema_pb.Topic
+	10, // 1: messaging_pb.StartPublishSessionRequest.record_type:type_name -> schema_pb.RecordType
+	11, // 2: messaging_pb.PublishRecordRequest.value:type_name -> schema_pb.RecordValue
+	8,  // 3: messaging_pb.SubscribeRecordRequest.init:type_name -> messaging_pb.SubscribeRecordRequest.InitSubscribeRecordRequest
+	11, // 4: messaging_pb.SubscribeRecordResponse.value:type_name -> schema_pb.RecordValue
+	9,  // 5: messaging_pb.SubscribeRecordRequest.InitSubscribeRecordRequest.topic:type_name -> schema_pb.Topic
+	12, // 6: messaging_pb.SubscribeRecordRequest.InitSubscribeRecordRequest.partition_offsets:type_name -> schema_pb.PartitionOffset
+	13, // 7: messaging_pb.SubscribeRecordRequest.InitSubscribeRecordRequest.offset_type:type_name -> schema_pb.OffsetType
+	0,  // 8: messaging_pb.SeaweedMessagingAgent.StartPublishSession:input_type -> messaging_pb.StartPublishSessionRequest
+	2,  // 9: messaging_pb.SeaweedMessagingAgent.ClosePublishSession:input_type -> messaging_pb.ClosePublishSessionRequest
+	4,  // 10: messaging_pb.SeaweedMessagingAgent.PublishRecord:input_type -> messaging_pb.PublishRecordRequest
+	6,  // 11: messaging_pb.SeaweedMessagingAgent.SubscribeRecord:input_type -> messaging_pb.SubscribeRecordRequest
+	1,  // 12: messaging_pb.SeaweedMessagingAgent.StartPublishSession:output_type -> messaging_pb.StartPublishSessionResponse
+	3,  // 13: messaging_pb.SeaweedMessagingAgent.ClosePublishSession:output_type -> messaging_pb.ClosePublishSessionResponse
+	5,  // 14: messaging_pb.SeaweedMessagingAgent.PublishRecord:output_type -> messaging_pb.PublishRecordResponse
+	7,  // 15: messaging_pb.SeaweedMessagingAgent.SubscribeRecord:output_type -> messaging_pb.SubscribeRecordResponse
+	12, // [12:16] is the sub-list for method output_type
+	8,  // [8:12] is the sub-list for method input_type
+	8,  // [8:8] is the sub-list for extension type_name
+	8,  // [8:8] is the sub-list for extension extendee
+	0,  // [0:8] is the sub-list for field type_name
+}
+
+func init() { file_mq_agent_proto_init() }
+func file_mq_agent_proto_init() {
+	if File_mq_agent_proto != nil {
+		return
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_mq_agent_proto_rawDesc), len(file_mq_agent_proto_rawDesc)),
+			NumEnums:      0,
+			NumMessages:   9,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_mq_agent_proto_goTypes,
+		DependencyIndexes: file_mq_agent_proto_depIdxs,
+		MessageInfos:      file_mq_agent_proto_msgTypes,
+	}.Build()
+	File_mq_agent_proto = out.File
+	file_mq_agent_proto_goTypes = nil
+	file_mq_agent_proto_depIdxs = nil
+}

weed/pb/mq_agent_pb/mq_agent_grpc.pb.go

@@ -0,0 +1,228 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.5.1
+// - protoc             v6.33.4
+// source: mq_agent.proto
+
+package mq_agent_pb
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.64.0 or later.
+const _ = grpc.SupportPackageIsVersion9
+
+const (
+	SeaweedMessagingAgent_StartPublishSession_FullMethodName = "/messaging_pb.SeaweedMessagingAgent/StartPublishSession"
+	SeaweedMessagingAgent_ClosePublishSession_FullMethodName = "/messaging_pb.SeaweedMessagingAgent/ClosePublishSession"
+	SeaweedMessagingAgent_PublishRecord_FullMethodName       = "/messaging_pb.SeaweedMessagingAgent/PublishRecord"
+	SeaweedMessagingAgent_SubscribeRecord_FullMethodName     = "/messaging_pb.SeaweedMessagingAgent/SubscribeRecord"
+)
+
+// SeaweedMessagingAgentClient is the client API for SeaweedMessagingAgent service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type SeaweedMessagingAgentClient interface {
+	// Publishing
+	StartPublishSession(ctx context.Context, in *StartPublishSessionRequest, opts ...grpc.CallOption) (*StartPublishSessionResponse, error)
+	ClosePublishSession(ctx context.Context, in *ClosePublishSessionRequest, opts ...grpc.CallOption) (*ClosePublishSessionResponse, error)
+	PublishRecord(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[PublishRecordRequest, PublishRecordResponse], error)
+	// Subscribing
+	SubscribeRecord(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[SubscribeRecordRequest, SubscribeRecordResponse], error)
+}
+
+type seaweedMessagingAgentClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewSeaweedMessagingAgentClient(cc grpc.ClientConnInterface) SeaweedMessagingAgentClient {
+	return &seaweedMessagingAgentClient{cc}
+}
+
+func (c *seaweedMessagingAgentClient) StartPublishSession(ctx context.Context, in *StartPublishSessionRequest, opts ...grpc.CallOption) (*StartPublishSessionResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(StartPublishSessionResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessagingAgent_StartPublishSession_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingAgentClient) ClosePublishSession(ctx context.Context, in *ClosePublishSessionRequest, opts ...grpc.CallOption) (*ClosePublishSessionResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ClosePublishSessionResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessagingAgent_ClosePublishSession_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingAgentClient) PublishRecord(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[PublishRecordRequest, PublishRecordResponse], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &SeaweedMessagingAgent_ServiceDesc.Streams[0], SeaweedMessagingAgent_PublishRecord_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[PublishRecordRequest, PublishRecordResponse]{ClientStream: stream}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessagingAgent_PublishRecordClient = grpc.BidiStreamingClient[PublishRecordRequest, PublishRecordResponse]
+
+func (c *seaweedMessagingAgentClient) SubscribeRecord(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[SubscribeRecordRequest, SubscribeRecordResponse], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &SeaweedMessagingAgent_ServiceDesc.Streams[1], SeaweedMessagingAgent_SubscribeRecord_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[SubscribeRecordRequest, SubscribeRecordResponse]{ClientStream: stream}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessagingAgent_SubscribeRecordClient = grpc.BidiStreamingClient[SubscribeRecordRequest, SubscribeRecordResponse]
+
+// SeaweedMessagingAgentServer is the server API for SeaweedMessagingAgent service.
+// All implementations must embed UnimplementedSeaweedMessagingAgentServer
+// for forward compatibility.
+type SeaweedMessagingAgentServer interface {
+	// Publishing
+	StartPublishSession(context.Context, *StartPublishSessionRequest) (*StartPublishSessionResponse, error)
+	ClosePublishSession(context.Context, *ClosePublishSessionRequest) (*ClosePublishSessionResponse, error)
+	PublishRecord(grpc.BidiStreamingServer[PublishRecordRequest, PublishRecordResponse]) error
+	// Subscribing
+	SubscribeRecord(grpc.BidiStreamingServer[SubscribeRecordRequest, SubscribeRecordResponse]) error
+	mustEmbedUnimplementedSeaweedMessagingAgentServer()
+}
+
+// UnimplementedSeaweedMessagingAgentServer must be embedded to have
+// forward compatible implementations.
+//
+// NOTE: this should be embedded by value instead of pointer to avoid a nil
+// pointer dereference when methods are called.
+type UnimplementedSeaweedMessagingAgentServer struct{}
+
+func (UnimplementedSeaweedMessagingAgentServer) StartPublishSession(context.Context, *StartPublishSessionRequest) (*StartPublishSessionResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method StartPublishSession not implemented")
+}
+func (UnimplementedSeaweedMessagingAgentServer) ClosePublishSession(context.Context, *ClosePublishSessionRequest) (*ClosePublishSessionResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ClosePublishSession not implemented")
+}
+func (UnimplementedSeaweedMessagingAgentServer) PublishRecord(grpc.BidiStreamingServer[PublishRecordRequest, PublishRecordResponse]) error {
+	return status.Errorf(codes.Unimplemented, "method PublishRecord not implemented")
+}
+func (UnimplementedSeaweedMessagingAgentServer) SubscribeRecord(grpc.BidiStreamingServer[SubscribeRecordRequest, SubscribeRecordResponse]) error {
+	return status.Errorf(codes.Unimplemented, "method SubscribeRecord not implemented")
+}
+func (UnimplementedSeaweedMessagingAgentServer) mustEmbedUnimplementedSeaweedMessagingAgentServer() {}
+func (UnimplementedSeaweedMessagingAgentServer) testEmbeddedByValue()                               {}
+
+// UnsafeSeaweedMessagingAgentServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to SeaweedMessagingAgentServer will
+// result in compilation errors.
+type UnsafeSeaweedMessagingAgentServer interface {
+	mustEmbedUnimplementedSeaweedMessagingAgentServer()
+}
+
+func RegisterSeaweedMessagingAgentServer(s grpc.ServiceRegistrar, srv SeaweedMessagingAgentServer) {
+	// If the following call pancis, it indicates UnimplementedSeaweedMessagingAgentServer was
+	// embedded by pointer and is nil.  This will cause panics if an
+	// unimplemented method is ever invoked, so we test this at initialization
+	// time to prevent it from happening at runtime later due to I/O.
+	if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
+		t.testEmbeddedByValue()
+	}
+	s.RegisterService(&SeaweedMessagingAgent_ServiceDesc, srv)
+}
+
+func _SeaweedMessagingAgent_StartPublishSession_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(StartPublishSessionRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingAgentServer).StartPublishSession(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessagingAgent_StartPublishSession_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingAgentServer).StartPublishSession(ctx, req.(*StartPublishSessionRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessagingAgent_ClosePublishSession_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ClosePublishSessionRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingAgentServer).ClosePublishSession(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessagingAgent_ClosePublishSession_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingAgentServer).ClosePublishSession(ctx, req.(*ClosePublishSessionRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessagingAgent_PublishRecord_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(SeaweedMessagingAgentServer).PublishRecord(&grpc.GenericServerStream[PublishRecordRequest, PublishRecordResponse]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessagingAgent_PublishRecordServer = grpc.BidiStreamingServer[PublishRecordRequest, PublishRecordResponse]
+
+func _SeaweedMessagingAgent_SubscribeRecord_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(SeaweedMessagingAgentServer).SubscribeRecord(&grpc.GenericServerStream[SubscribeRecordRequest, SubscribeRecordResponse]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessagingAgent_SubscribeRecordServer = grpc.BidiStreamingServer[SubscribeRecordRequest, SubscribeRecordResponse]
+
+// SeaweedMessagingAgent_ServiceDesc is the grpc.ServiceDesc for SeaweedMessagingAgent service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var SeaweedMessagingAgent_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "messaging_pb.SeaweedMessagingAgent",
+	HandlerType: (*SeaweedMessagingAgentServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "StartPublishSession",
+			Handler:    _SeaweedMessagingAgent_StartPublishSession_Handler,
+		},
+		{
+			MethodName: "ClosePublishSession",
+			Handler:    _SeaweedMessagingAgent_ClosePublishSession_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "PublishRecord",
+			Handler:       _SeaweedMessagingAgent_PublishRecord_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "SubscribeRecord",
+			Handler:       _SeaweedMessagingAgent_SubscribeRecord_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "mq_agent.proto",
+}

weed/pb/mq_agent_pb/publish_response_test.go

@@ -0,0 +1,103 @@
+package mq_agent_pb
+
+import (
+	"testing"
+
+	"google.golang.org/protobuf/proto"
+)
+
+func TestPublishRecordResponseSerialization(t *testing.T) {
+	// Test that PublishRecordResponse can serialize/deserialize with new offset fields
+	original := &PublishRecordResponse{
+		AckSequence: 123,
+		Error:       "",
+		BaseOffset:  1000, // New field
+		LastOffset:  1005, // New field
+	}
+
+	// Test proto marshaling/unmarshaling
+	data, err := proto.Marshal(original)
+	if err != nil {
+		t.Fatalf("Failed to marshal PublishRecordResponse: %v", err)
+	}
+
+	restored := &PublishRecordResponse{}
+	err = proto.Unmarshal(data, restored)
+	if err != nil {
+		t.Fatalf("Failed to unmarshal PublishRecordResponse: %v", err)
+	}
+
+	// Verify all fields are preserved
+	if restored.AckSequence != original.AckSequence {
+		t.Errorf("AckSequence = %d, want %d", restored.AckSequence, original.AckSequence)
+	}
+	if restored.BaseOffset != original.BaseOffset {
+		t.Errorf("BaseOffset = %d, want %d", restored.BaseOffset, original.BaseOffset)
+	}
+	if restored.LastOffset != original.LastOffset {
+		t.Errorf("LastOffset = %d, want %d", restored.LastOffset, original.LastOffset)
+	}
+}
+
+func TestSubscribeRecordResponseSerialization(t *testing.T) {
+	// Test that SubscribeRecordResponse can serialize/deserialize with new offset field
+	original := &SubscribeRecordResponse{
+		Key:           []byte("test-key"),
+		TsNs:          1234567890,
+		Error:         "",
+		IsEndOfStream: false,
+		IsEndOfTopic:  false,
+		Offset:        42, // New field
+	}
+
+	// Test proto marshaling/unmarshaling
+	data, err := proto.Marshal(original)
+	if err != nil {
+		t.Fatalf("Failed to marshal SubscribeRecordResponse: %v", err)
+	}
+
+	restored := &SubscribeRecordResponse{}
+	err = proto.Unmarshal(data, restored)
+	if err != nil {
+		t.Fatalf("Failed to unmarshal SubscribeRecordResponse: %v", err)
+	}
+
+	// Verify all fields are preserved
+	if restored.TsNs != original.TsNs {
+		t.Errorf("TsNs = %d, want %d", restored.TsNs, original.TsNs)
+	}
+	if restored.Offset != original.Offset {
+		t.Errorf("Offset = %d, want %d", restored.Offset, original.Offset)
+	}
+	if string(restored.Key) != string(original.Key) {
+		t.Errorf("Key = %s, want %s", string(restored.Key), string(original.Key))
+	}
+}
+
+func TestPublishRecordResponseBackwardCompatibility(t *testing.T) {
+	// Test that PublishRecordResponse without offset fields still works
+	original := &PublishRecordResponse{
+		AckSequence: 123,
+		Error:       "",
+		// BaseOffset and LastOffset not set (defaults to 0)
+	}
+
+	data, err := proto.Marshal(original)
+	if err != nil {
+		t.Fatalf("Failed to marshal PublishRecordResponse: %v", err)
+	}
+
+	restored := &PublishRecordResponse{}
+	err = proto.Unmarshal(data, restored)
+	if err != nil {
+		t.Fatalf("Failed to unmarshal PublishRecordResponse: %v", err)
+	}
+
+	// Offset fields should default to 0
+	if restored.BaseOffset != 0 {
+		t.Errorf("BaseOffset = %d, want 0", restored.BaseOffset)
+	}
+	if restored.LastOffset != 0 {
+		t.Errorf("LastOffset = %d, want 0", restored.LastOffset)
+	}
+}

weed/pb/mq_broker.proto

@@ -0,0 +1,509 @@
+syntax = "proto3";
+
+package messaging_pb;
+
+import "mq_schema.proto";
+import "filer.proto";
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb";
+option java_package = "seaweedfs.mq";
+option java_outer_classname = "MessageQueueProto";
+
+//////////////////////////////////////////////////
+
+service SeaweedMessaging {
+
+    // control plane
+    rpc FindBrokerLeader (FindBrokerLeaderRequest) returns (FindBrokerLeaderResponse) {
+    }
+
+    // control plane for balancer
+    rpc PublisherToPubBalancer (stream PublisherToPubBalancerRequest) returns (stream PublisherToPubBalancerResponse) {
+    }
+    rpc BalanceTopics (BalanceTopicsRequest) returns (BalanceTopicsResponse) {
+    }
+
+    // control plane for topic partitions
+    rpc ListTopics (ListTopicsRequest) returns (ListTopicsResponse) {
+    }
+    rpc TopicExists (TopicExistsRequest) returns (TopicExistsResponse) {
+    }
+    rpc ConfigureTopic (ConfigureTopicRequest) returns (ConfigureTopicResponse) {
+    }
+    rpc LookupTopicBrokers (LookupTopicBrokersRequest) returns (LookupTopicBrokersResponse) {
+    }
+    rpc GetTopicConfiguration (GetTopicConfigurationRequest) returns (GetTopicConfigurationResponse) {
+    }
+    rpc GetTopicPublishers (GetTopicPublishersRequest) returns (GetTopicPublishersResponse) {
+    }
+    rpc GetTopicSubscribers (GetTopicSubscribersRequest) returns (GetTopicSubscribersResponse) {
+    }
+
+    // invoked by the balancer, running on each broker
+    rpc AssignTopicPartitions (AssignTopicPartitionsRequest) returns (AssignTopicPartitionsResponse) {
+    }
+    rpc ClosePublishers(ClosePublishersRequest) returns (ClosePublishersResponse) {
+    }
+    rpc CloseSubscribers(CloseSubscribersRequest) returns (CloseSubscribersResponse) {
+    }
+
+    // subscriber connects to broker balancer, which coordinates with the subscribers
+    rpc SubscriberToSubCoordinator (stream SubscriberToSubCoordinatorRequest) returns (stream SubscriberToSubCoordinatorResponse) {
+    }
+
+    // data plane for each topic partition
+    rpc PublishMessage (stream PublishMessageRequest) returns (stream PublishMessageResponse) {
+    }
+    rpc SubscribeMessage (stream SubscribeMessageRequest) returns (stream SubscribeMessageResponse) {
+    }
+    // The lead broker asks a follower broker to follow itself
+    rpc PublishFollowMe (stream PublishFollowMeRequest) returns (stream PublishFollowMeResponse) {
+    }
+    rpc SubscribeFollowMe (stream SubscribeFollowMeRequest) returns (SubscribeFollowMeResponse) {
+    }
+    
+    // Stateless fetch API (Kafka-style) - request/response pattern
+    // This is the recommended API for Kafka gateway and other stateless clients
+    // No streaming, no session state - each request is completely independent
+    rpc FetchMessage (FetchMessageRequest) returns (FetchMessageResponse) {
+    }
+
+    // SQL query support - get unflushed messages from broker's in-memory buffer (streaming)
+    rpc GetUnflushedMessages (GetUnflushedMessagesRequest) returns (stream GetUnflushedMessagesResponse) {
+    }
+
+    // Get comprehensive partition range information (offsets, timestamps, and other fields)
+    rpc GetPartitionRangeInfo (GetPartitionRangeInfoRequest) returns (GetPartitionRangeInfoResponse) {
+    }
+
+    // Removed Kafka Gateway Registration - no longer needed
+}
+
+//////////////////////////////////////////////////
+
+message FindBrokerLeaderRequest {
+    string filer_group = 1;
+}
+
+message FindBrokerLeaderResponse {
+    string broker = 1;
+}
+
+//////////////////////////////////////////////////
+message BrokerStats {
+    int32 cpu_usage_percent = 1;
+    map<string, TopicPartitionStats> stats = 2;
+}
+message TopicPartitionStats {
+    schema_pb.Topic topic = 1;
+    schema_pb.Partition partition = 2;
+    int32 publisher_count = 3;
+    int32 subscriber_count = 4;
+    string follower = 5;
+}
+
+
+message PublisherToPubBalancerRequest {
+    message InitMessage {
+        string broker = 1;
+    }
+    oneof message {
+        InitMessage init = 1;
+        BrokerStats stats = 2;
+    }
+}
+message PublisherToPubBalancerResponse {
+}
+
+message BalanceTopicsRequest {
+}
+message BalanceTopicsResponse {
+}
+
+//////////////////////////////////////////////////
+message TopicRetention {
+    int64 retention_seconds = 1; // retention duration in seconds
+    bool enabled = 2; // whether retention is enabled
+}
+
+message ConfigureTopicRequest {
+    schema_pb.Topic topic = 1;
+    int32 partition_count = 2;
+    TopicRetention retention = 3;
+    schema_pb.RecordType message_record_type = 4;  // Complete flat schema for the message
+    repeated string key_columns = 5;              // Names of columns that form the key
+    string schema_format = 6;                      // Serialization format: "AVRO", "PROTOBUF", "JSON_SCHEMA", or empty for schemaless
+}
+message ConfigureTopicResponse {
+    repeated BrokerPartitionAssignment broker_partition_assignments = 2;
+    TopicRetention retention = 3;
+    schema_pb.RecordType message_record_type = 4;  // Complete flat schema for the message
+    repeated string key_columns = 5;              // Names of columns that form the key
+    string schema_format = 6;                      // Serialization format: "AVRO", "PROTOBUF", "JSON_SCHEMA", or empty for schemaless
+}
+message ListTopicsRequest {
+}
+message ListTopicsResponse {
+    repeated schema_pb.Topic topics = 1;
+}
+message TopicExistsRequest {
+    schema_pb.Topic topic = 1;
+}
+message TopicExistsResponse {
+    bool exists = 1;
+}
+message LookupTopicBrokersRequest {
+    schema_pb.Topic topic = 1;
+}
+message LookupTopicBrokersResponse {
+    schema_pb.Topic topic = 1;
+    repeated BrokerPartitionAssignment broker_partition_assignments = 2;
+}
+message BrokerPartitionAssignment {
+    schema_pb.Partition partition = 1;
+    string leader_broker = 2;
+    string follower_broker = 3;
+}
+message GetTopicConfigurationRequest {
+    schema_pb.Topic topic = 1;
+}
+message GetTopicConfigurationResponse {
+    schema_pb.Topic topic = 1;
+    int32 partition_count = 2;
+    repeated BrokerPartitionAssignment broker_partition_assignments = 3;
+    int64 created_at_ns = 4;
+    int64 last_updated_ns = 5;
+    TopicRetention retention = 6;
+    schema_pb.RecordType message_record_type = 7;  // Complete flat schema for the message
+    repeated string key_columns = 8;              // Names of columns that form the key
+    string schema_format = 9;                      // Serialization format: "AVRO", "PROTOBUF", "JSON_SCHEMA", or empty for schemaless
+}
+
+message GetTopicPublishersRequest {
+    schema_pb.Topic topic = 1;
+}
+message GetTopicPublishersResponse {
+    repeated TopicPublisher publishers = 1;
+}
+
+message GetTopicSubscribersRequest {
+    schema_pb.Topic topic = 1;
+}
+message GetTopicSubscribersResponse {
+    repeated TopicSubscriber subscribers = 1;
+}
+
+message TopicPublisher {
+    string publisher_name = 1;
+    string client_id = 2;
+    schema_pb.Partition partition = 3;
+    int64 connect_time_ns = 4;
+    int64 last_seen_time_ns = 5;
+    string broker = 6;
+    bool is_active = 7;
+    int64 last_published_offset = 8;
+    int64 last_acked_offset = 9;
+}
+
+message TopicSubscriber {
+    string consumer_group = 1;
+    string consumer_id = 2;
+    string client_id = 3;
+    schema_pb.Partition partition = 4;
+    int64 connect_time_ns = 5;
+    int64 last_seen_time_ns = 6;
+    string broker = 7;
+    bool is_active = 8;
+    int64 current_offset = 9; // last acknowledged offset
+    int64 last_received_offset = 10;
+}
+
+message AssignTopicPartitionsRequest {
+    schema_pb.Topic topic = 1;
+    repeated BrokerPartitionAssignment broker_partition_assignments = 2;
+    bool is_leader = 3;
+    bool is_draining = 4;
+}
+message AssignTopicPartitionsResponse {
+}
+
+message SubscriberToSubCoordinatorRequest {
+    message InitMessage {
+        string consumer_group = 1;
+        string consumer_group_instance_id = 2;
+        schema_pb.Topic topic = 3;
+        // The consumer group instance will be assigned at most max_partition_count partitions.
+        // If the number of partitions is less than the sum of max_partition_count,
+        // the consumer group instance may be assigned partitions less than max_partition_count.
+        // Default is 1.
+        int32 max_partition_count = 4;
+        // If consumer group instance changes, wait for rebalance_seconds before reassigning partitions
+        // Exception: if adding a new consumer group instance and sum of max_partition_count equals the number of partitions,
+        // the rebalance will happen immediately.
+        // Default is 10 seconds.
+        int32 rebalance_seconds = 5;
+    }
+    message AckUnAssignmentMessage {
+        schema_pb.Partition partition = 1;
+    }
+    message AckAssignmentMessage {
+        schema_pb.Partition partition = 1;
+    }
+    oneof message {
+        InitMessage init = 1;
+        AckAssignmentMessage ack_assignment = 2;
+        AckUnAssignmentMessage ack_un_assignment = 3;
+    }
+}
+message SubscriberToSubCoordinatorResponse {
+    message Assignment {
+        BrokerPartitionAssignment partition_assignment = 1;
+    }
+    message UnAssignment {
+        schema_pb.Partition partition = 1;
+    }
+    oneof message {
+        Assignment assignment = 1;
+        UnAssignment un_assignment = 2;
+    }
+}
+
+//////////////////////////////////////////////////
+message ControlMessage {
+    bool is_close = 1;
+    string publisher_name = 2;
+}
+message DataMessage {
+    bytes key = 1;
+    bytes value = 2;
+    int64 ts_ns = 3;
+    ControlMessage ctrl = 4;
+}
+message PublishMessageRequest {
+    message InitMessage {
+        schema_pb.Topic topic = 1;
+        schema_pb.Partition partition = 2;
+        int32 ack_interval = 3;
+        string follower_broker = 4;
+        string publisher_name = 5; // for debugging
+    }
+    oneof message {
+        InitMessage init = 1;
+        DataMessage data = 2;
+    }
+}
+message PublishMessageResponse {
+    int64 ack_ts_ns = 1;  // Acknowledgment timestamp in nanoseconds
+    string error = 2;
+    bool should_close = 3;
+    int32 error_code = 4; // Structured error code for reliable error mapping
+    int64 assigned_offset = 5; // The actual offset assigned by SeaweedMQ for this message
+}
+message PublishFollowMeRequest {
+    message InitMessage {
+        schema_pb.Topic topic = 1;
+        schema_pb.Partition partition = 2;
+    }
+    message FlushMessage {
+        int64 ts_ns = 1;
+    }
+    message CloseMessage {
+    }
+    oneof message {
+        InitMessage init = 1;
+        DataMessage data = 2;
+        FlushMessage flush = 3;
+        CloseMessage close = 4;
+    }
+}
+message PublishFollowMeResponse {
+    int64 ack_ts_ns = 1;
+}
+message SubscribeMessageRequest {
+    message InitMessage {
+        string consumer_group = 1;
+        string consumer_id = 2;
+        string client_id = 3;
+        schema_pb.Topic topic = 4;
+        schema_pb.PartitionOffset partition_offset = 5;
+        schema_pb.OffsetType offset_type = 6;
+        string filter = 10;
+        string follower_broker = 11;
+        int32 sliding_window_size = 12;
+    }
+    message AckMessage {
+        int64 ts_ns = 1;  // Timestamp in nanoseconds for acknowledgment tracking
+        bytes key = 2;
+    }
+    message SeekMessage {
+        int64 offset = 1;  // New offset to seek to
+        schema_pb.OffsetType offset_type = 2;  // EXACT_OFFSET, RESET_TO_LATEST, etc.
+    }
+    oneof message {
+        InitMessage init = 1;
+        AckMessage ack = 2;
+        SeekMessage seek = 3;
+    }
+}
+message SubscribeMessageResponse {
+    message SubscribeCtrlMessage {
+        string error = 1;
+        bool is_end_of_stream = 2;
+        bool is_end_of_topic = 3;
+    }
+    oneof message {
+        SubscribeCtrlMessage ctrl = 1;
+        DataMessage data = 2;
+    }
+}
+message SubscribeFollowMeRequest {
+    message InitMessage {
+        schema_pb.Topic topic = 1;
+        schema_pb.Partition partition = 2;
+        string consumer_group = 3;
+    }
+    message AckMessage {
+        int64 ts_ns = 1;
+    }
+    message CloseMessage {
+    }
+    oneof message {
+        InitMessage init = 1;
+        AckMessage ack = 2;
+        CloseMessage close = 3;
+    }
+}
+message SubscribeFollowMeResponse {
+    int64 ack_ts_ns = 1;
+}
+
+//////////////////////////////////////////////////
+// Stateless Fetch API (Kafka-style)
+// Unlike SubscribeMessage which maintains long-lived Subscribe loops,
+// FetchMessage is completely stateless - each request is independent.
+// This eliminates concurrent access issues and stream corruption.
+//
+// Key differences from SubscribeMessage:
+// 1. Request/Response pattern (not streaming)
+// 2. No session state maintained
+// 3. Each fetch is independent
+// 4. Natural support for concurrent reads at different offsets
+// 5. Client manages offset tracking (like Kafka)
+//////////////////////////////////////////////////
+
+message FetchMessageRequest {
+    // Topic and partition to fetch from
+    schema_pb.Topic topic = 1;
+    schema_pb.Partition partition = 2;
+    
+    // Starting offset for this fetch
+    int64 start_offset = 3;
+    
+    // Maximum number of bytes to return (limit response size)
+    int32 max_bytes = 4;
+    
+    // Maximum number of messages to return
+    int32 max_messages = 5;
+    
+    // Maximum time to wait for data if partition is empty (milliseconds)
+    // 0 = return immediately, >0 = wait up to this long
+    int32 max_wait_ms = 6;
+    
+    // Minimum bytes before responding (0 = respond immediately)
+    // This allows batching for efficiency
+    int32 min_bytes = 7;
+    
+    // Consumer identity (for monitoring/debugging)
+    string consumer_group = 8;
+    string consumer_id = 9;
+}
+
+message FetchMessageResponse {
+    // Messages fetched (may be empty if no data available)
+    repeated DataMessage messages = 1;
+    
+    // Metadata about partition state
+    int64 high_water_mark = 2;  // Highest offset available
+    int64 log_start_offset = 3;  // Earliest offset available
+    bool end_of_partition = 4;   // True if no more data available
+    
+    // Error handling
+    string error = 5;
+    int32 error_code = 6;
+    
+    // Next offset to fetch (for client convenience)
+    // Client should fetch from this offset next
+    int64 next_offset = 7;
+}
+
+message ClosePublishersRequest {
+    schema_pb.Topic topic = 1;
+    int64 unix_time_ns = 2;
+}
+message ClosePublishersResponse {
+}
+message CloseSubscribersRequest {
+    schema_pb.Topic topic = 1;
+    int64 unix_time_ns = 2;
+}
+message CloseSubscribersResponse {
+}
+
+//////////////////////////////////////////////////
+// SQL query support messages
+
+message GetUnflushedMessagesRequest {
+    schema_pb.Topic topic = 1;
+    schema_pb.Partition partition = 2;
+    int64 start_buffer_offset = 3; // Filter by buffer offset (messages from buffers >= this offset)
+}
+
+message GetUnflushedMessagesResponse {
+    filer_pb.LogEntry message = 1;       // Single message per response (streaming)
+    string error = 2;           // Error message if any
+    bool end_of_stream = 3;     // Indicates this is the final response
+}
+
+//////////////////////////////////////////////////
+// Partition range information messages
+
+message GetPartitionRangeInfoRequest {
+    schema_pb.Topic topic = 1;
+    schema_pb.Partition partition = 2;
+}
+
+message GetPartitionRangeInfoResponse {
+    // Offset range information
+    OffsetRangeInfo offset_range = 1;
+    
+    // Timestamp range information
+    TimestampRangeInfo timestamp_range = 2;
+    
+    // Future: ID range information (for ordered IDs, UUIDs, etc.)
+    // IdRangeInfo id_range = 3;
+    
+    // Partition metadata
+    int64 record_count = 10;
+    int64 active_subscriptions = 11;
+    string error = 12;
+}
+
+message OffsetRangeInfo {
+    int64 earliest_offset = 1;
+    int64 latest_offset = 2;
+    int64 high_water_mark = 3;
+}
+
+message TimestampRangeInfo {
+    int64 earliest_timestamp_ns = 1;  // Earliest message timestamp in nanoseconds
+    int64 latest_timestamp_ns = 2;    // Latest message timestamp in nanoseconds
+}
+
+// Future extension for ID ranges
+// message IdRangeInfo {
+//     string earliest_id = 1;
+//     string latest_id = 2;
+//     string id_type = 3;  // "uuid", "sequential", "custom", etc.
+// }
+
+// Removed Kafka Gateway Registration messages - no longer needed

weed/pb/mq_pb/mq_broker_grpc.pb.go

@@ -0,0 +1,872 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.5.1
+// - protoc             v6.33.4
+// source: mq_broker.proto
+
+package mq_pb
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.64.0 or later.
+const _ = grpc.SupportPackageIsVersion9
+
+const (
+	SeaweedMessaging_FindBrokerLeader_FullMethodName           = "/messaging_pb.SeaweedMessaging/FindBrokerLeader"
+	SeaweedMessaging_PublisherToPubBalancer_FullMethodName     = "/messaging_pb.SeaweedMessaging/PublisherToPubBalancer"
+	SeaweedMessaging_BalanceTopics_FullMethodName              = "/messaging_pb.SeaweedMessaging/BalanceTopics"
+	SeaweedMessaging_ListTopics_FullMethodName                 = "/messaging_pb.SeaweedMessaging/ListTopics"
+	SeaweedMessaging_TopicExists_FullMethodName                = "/messaging_pb.SeaweedMessaging/TopicExists"
+	SeaweedMessaging_ConfigureTopic_FullMethodName             = "/messaging_pb.SeaweedMessaging/ConfigureTopic"
+	SeaweedMessaging_LookupTopicBrokers_FullMethodName         = "/messaging_pb.SeaweedMessaging/LookupTopicBrokers"
+	SeaweedMessaging_GetTopicConfiguration_FullMethodName      = "/messaging_pb.SeaweedMessaging/GetTopicConfiguration"
+	SeaweedMessaging_GetTopicPublishers_FullMethodName         = "/messaging_pb.SeaweedMessaging/GetTopicPublishers"
+	SeaweedMessaging_GetTopicSubscribers_FullMethodName        = "/messaging_pb.SeaweedMessaging/GetTopicSubscribers"
+	SeaweedMessaging_AssignTopicPartitions_FullMethodName      = "/messaging_pb.SeaweedMessaging/AssignTopicPartitions"
+	SeaweedMessaging_ClosePublishers_FullMethodName            = "/messaging_pb.SeaweedMessaging/ClosePublishers"
+	SeaweedMessaging_CloseSubscribers_FullMethodName           = "/messaging_pb.SeaweedMessaging/CloseSubscribers"
+	SeaweedMessaging_SubscriberToSubCoordinator_FullMethodName = "/messaging_pb.SeaweedMessaging/SubscriberToSubCoordinator"
+	SeaweedMessaging_PublishMessage_FullMethodName             = "/messaging_pb.SeaweedMessaging/PublishMessage"
+	SeaweedMessaging_SubscribeMessage_FullMethodName           = "/messaging_pb.SeaweedMessaging/SubscribeMessage"
+	SeaweedMessaging_PublishFollowMe_FullMethodName            = "/messaging_pb.SeaweedMessaging/PublishFollowMe"
+	SeaweedMessaging_SubscribeFollowMe_FullMethodName          = "/messaging_pb.SeaweedMessaging/SubscribeFollowMe"
+	SeaweedMessaging_FetchMessage_FullMethodName               = "/messaging_pb.SeaweedMessaging/FetchMessage"
+	SeaweedMessaging_GetUnflushedMessages_FullMethodName       = "/messaging_pb.SeaweedMessaging/GetUnflushedMessages"
+	SeaweedMessaging_GetPartitionRangeInfo_FullMethodName      = "/messaging_pb.SeaweedMessaging/GetPartitionRangeInfo"
+)
+
+// SeaweedMessagingClient is the client API for SeaweedMessaging service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type SeaweedMessagingClient interface {
+	// control plane
+	FindBrokerLeader(ctx context.Context, in *FindBrokerLeaderRequest, opts ...grpc.CallOption) (*FindBrokerLeaderResponse, error)
+	// control plane for balancer
+	PublisherToPubBalancer(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[PublisherToPubBalancerRequest, PublisherToPubBalancerResponse], error)
+	BalanceTopics(ctx context.Context, in *BalanceTopicsRequest, opts ...grpc.CallOption) (*BalanceTopicsResponse, error)
+	// control plane for topic partitions
+	ListTopics(ctx context.Context, in *ListTopicsRequest, opts ...grpc.CallOption) (*ListTopicsResponse, error)
+	TopicExists(ctx context.Context, in *TopicExistsRequest, opts ...grpc.CallOption) (*TopicExistsResponse, error)
+	ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error)
+	LookupTopicBrokers(ctx context.Context, in *LookupTopicBrokersRequest, opts ...grpc.CallOption) (*LookupTopicBrokersResponse, error)
+	GetTopicConfiguration(ctx context.Context, in *GetTopicConfigurationRequest, opts ...grpc.CallOption) (*GetTopicConfigurationResponse, error)
+	GetTopicPublishers(ctx context.Context, in *GetTopicPublishersRequest, opts ...grpc.CallOption) (*GetTopicPublishersResponse, error)
+	GetTopicSubscribers(ctx context.Context, in *GetTopicSubscribersRequest, opts ...grpc.CallOption) (*GetTopicSubscribersResponse, error)
+	// invoked by the balancer, running on each broker
+	AssignTopicPartitions(ctx context.Context, in *AssignTopicPartitionsRequest, opts ...grpc.CallOption) (*AssignTopicPartitionsResponse, error)
+	ClosePublishers(ctx context.Context, in *ClosePublishersRequest, opts ...grpc.CallOption) (*ClosePublishersResponse, error)
+	CloseSubscribers(ctx context.Context, in *CloseSubscribersRequest, opts ...grpc.CallOption) (*CloseSubscribersResponse, error)
+	// subscriber connects to broker balancer, which coordinates with the subscribers
+	SubscriberToSubCoordinator(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[SubscriberToSubCoordinatorRequest, SubscriberToSubCoordinatorResponse], error)
+	// data plane for each topic partition
+	PublishMessage(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[PublishMessageRequest, PublishMessageResponse], error)
+	SubscribeMessage(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[SubscribeMessageRequest, SubscribeMessageResponse], error)
+	// The lead broker asks a follower broker to follow itself
+	PublishFollowMe(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[PublishFollowMeRequest, PublishFollowMeResponse], error)
+	SubscribeFollowMe(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[SubscribeFollowMeRequest, SubscribeFollowMeResponse], error)
+	// Stateless fetch API (Kafka-style) - request/response pattern
+	// This is the recommended API for Kafka gateway and other stateless clients
+	// No streaming, no session state - each request is completely independent
+	FetchMessage(ctx context.Context, in *FetchMessageRequest, opts ...grpc.CallOption) (*FetchMessageResponse, error)
+	// SQL query support - get unflushed messages from broker's in-memory buffer (streaming)
+	GetUnflushedMessages(ctx context.Context, in *GetUnflushedMessagesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[GetUnflushedMessagesResponse], error)
+	// Get comprehensive partition range information (offsets, timestamps, and other fields)
+	GetPartitionRangeInfo(ctx context.Context, in *GetPartitionRangeInfoRequest, opts ...grpc.CallOption) (*GetPartitionRangeInfoResponse, error)
+}
+
+type seaweedMessagingClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewSeaweedMessagingClient(cc grpc.ClientConnInterface) SeaweedMessagingClient {
+	return &seaweedMessagingClient{cc}
+}
+
+func (c *seaweedMessagingClient) FindBrokerLeader(ctx context.Context, in *FindBrokerLeaderRequest, opts ...grpc.CallOption) (*FindBrokerLeaderResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(FindBrokerLeaderResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessaging_FindBrokerLeader_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) PublisherToPubBalancer(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[PublisherToPubBalancerRequest, PublisherToPubBalancerResponse], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &SeaweedMessaging_ServiceDesc.Streams[0], SeaweedMessaging_PublisherToPubBalancer_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[PublisherToPubBalancerRequest, PublisherToPubBalancerResponse]{ClientStream: stream}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessaging_PublisherToPubBalancerClient = grpc.BidiStreamingClient[PublisherToPubBalancerRequest, PublisherToPubBalancerResponse]
+
+func (c *seaweedMessagingClient) BalanceTopics(ctx context.Context, in *BalanceTopicsRequest, opts ...grpc.CallOption) (*BalanceTopicsResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(BalanceTopicsResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessaging_BalanceTopics_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) ListTopics(ctx context.Context, in *ListTopicsRequest, opts ...grpc.CallOption) (*ListTopicsResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ListTopicsResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessaging_ListTopics_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) TopicExists(ctx context.Context, in *TopicExistsRequest, opts ...grpc.CallOption) (*TopicExistsResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(TopicExistsResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessaging_TopicExists_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ConfigureTopicResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessaging_ConfigureTopic_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) LookupTopicBrokers(ctx context.Context, in *LookupTopicBrokersRequest, opts ...grpc.CallOption) (*LookupTopicBrokersResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(LookupTopicBrokersResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessaging_LookupTopicBrokers_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) GetTopicConfiguration(ctx context.Context, in *GetTopicConfigurationRequest, opts ...grpc.CallOption) (*GetTopicConfigurationResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(GetTopicConfigurationResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessaging_GetTopicConfiguration_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) GetTopicPublishers(ctx context.Context, in *GetTopicPublishersRequest, opts ...grpc.CallOption) (*GetTopicPublishersResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(GetTopicPublishersResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessaging_GetTopicPublishers_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) GetTopicSubscribers(ctx context.Context, in *GetTopicSubscribersRequest, opts ...grpc.CallOption) (*GetTopicSubscribersResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(GetTopicSubscribersResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessaging_GetTopicSubscribers_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) AssignTopicPartitions(ctx context.Context, in *AssignTopicPartitionsRequest, opts ...grpc.CallOption) (*AssignTopicPartitionsResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(AssignTopicPartitionsResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessaging_AssignTopicPartitions_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) ClosePublishers(ctx context.Context, in *ClosePublishersRequest, opts ...grpc.CallOption) (*ClosePublishersResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ClosePublishersResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessaging_ClosePublishers_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) CloseSubscribers(ctx context.Context, in *CloseSubscribersRequest, opts ...grpc.CallOption) (*CloseSubscribersResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(CloseSubscribersResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessaging_CloseSubscribers_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) SubscriberToSubCoordinator(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[SubscriberToSubCoordinatorRequest, SubscriberToSubCoordinatorResponse], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &SeaweedMessaging_ServiceDesc.Streams[1], SeaweedMessaging_SubscriberToSubCoordinator_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[SubscriberToSubCoordinatorRequest, SubscriberToSubCoordinatorResponse]{ClientStream: stream}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessaging_SubscriberToSubCoordinatorClient = grpc.BidiStreamingClient[SubscriberToSubCoordinatorRequest, SubscriberToSubCoordinatorResponse]
+
+func (c *seaweedMessagingClient) PublishMessage(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[PublishMessageRequest, PublishMessageResponse], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &SeaweedMessaging_ServiceDesc.Streams[2], SeaweedMessaging_PublishMessage_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[PublishMessageRequest, PublishMessageResponse]{ClientStream: stream}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessaging_PublishMessageClient = grpc.BidiStreamingClient[PublishMessageRequest, PublishMessageResponse]
+
+func (c *seaweedMessagingClient) SubscribeMessage(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[SubscribeMessageRequest, SubscribeMessageResponse], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &SeaweedMessaging_ServiceDesc.Streams[3], SeaweedMessaging_SubscribeMessage_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[SubscribeMessageRequest, SubscribeMessageResponse]{ClientStream: stream}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessaging_SubscribeMessageClient = grpc.BidiStreamingClient[SubscribeMessageRequest, SubscribeMessageResponse]
+
+func (c *seaweedMessagingClient) PublishFollowMe(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[PublishFollowMeRequest, PublishFollowMeResponse], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &SeaweedMessaging_ServiceDesc.Streams[4], SeaweedMessaging_PublishFollowMe_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[PublishFollowMeRequest, PublishFollowMeResponse]{ClientStream: stream}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessaging_PublishFollowMeClient = grpc.BidiStreamingClient[PublishFollowMeRequest, PublishFollowMeResponse]
+
+func (c *seaweedMessagingClient) SubscribeFollowMe(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[SubscribeFollowMeRequest, SubscribeFollowMeResponse], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &SeaweedMessaging_ServiceDesc.Streams[5], SeaweedMessaging_SubscribeFollowMe_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[SubscribeFollowMeRequest, SubscribeFollowMeResponse]{ClientStream: stream}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessaging_SubscribeFollowMeClient = grpc.ClientStreamingClient[SubscribeFollowMeRequest, SubscribeFollowMeResponse]
+
+func (c *seaweedMessagingClient) FetchMessage(ctx context.Context, in *FetchMessageRequest, opts ...grpc.CallOption) (*FetchMessageResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(FetchMessageResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessaging_FetchMessage_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) GetUnflushedMessages(ctx context.Context, in *GetUnflushedMessagesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[GetUnflushedMessagesResponse], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &SeaweedMessaging_ServiceDesc.Streams[6], SeaweedMessaging_GetUnflushedMessages_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[GetUnflushedMessagesRequest, GetUnflushedMessagesResponse]{ClientStream: stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessaging_GetUnflushedMessagesClient = grpc.ServerStreamingClient[GetUnflushedMessagesResponse]
+
+func (c *seaweedMessagingClient) GetPartitionRangeInfo(ctx context.Context, in *GetPartitionRangeInfoRequest, opts ...grpc.CallOption) (*GetPartitionRangeInfoResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(GetPartitionRangeInfoResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessaging_GetPartitionRangeInfo_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// SeaweedMessagingServer is the server API for SeaweedMessaging service.
+// All implementations must embed UnimplementedSeaweedMessagingServer
+// for forward compatibility.
+type SeaweedMessagingServer interface {
+	// control plane
+	FindBrokerLeader(context.Context, *FindBrokerLeaderRequest) (*FindBrokerLeaderResponse, error)
+	// control plane for balancer
+	PublisherToPubBalancer(grpc.BidiStreamingServer[PublisherToPubBalancerRequest, PublisherToPubBalancerResponse]) error
+	BalanceTopics(context.Context, *BalanceTopicsRequest) (*BalanceTopicsResponse, error)
+	// control plane for topic partitions
+	ListTopics(context.Context, *ListTopicsRequest) (*ListTopicsResponse, error)
+	TopicExists(context.Context, *TopicExistsRequest) (*TopicExistsResponse, error)
+	ConfigureTopic(context.Context, *ConfigureTopicRequest) (*ConfigureTopicResponse, error)
+	LookupTopicBrokers(context.Context, *LookupTopicBrokersRequest) (*LookupTopicBrokersResponse, error)
+	GetTopicConfiguration(context.Context, *GetTopicConfigurationRequest) (*GetTopicConfigurationResponse, error)
+	GetTopicPublishers(context.Context, *GetTopicPublishersRequest) (*GetTopicPublishersResponse, error)
+	GetTopicSubscribers(context.Context, *GetTopicSubscribersRequest) (*GetTopicSubscribersResponse, error)
+	// invoked by the balancer, running on each broker
+	AssignTopicPartitions(context.Context, *AssignTopicPartitionsRequest) (*AssignTopicPartitionsResponse, error)
+	ClosePublishers(context.Context, *ClosePublishersRequest) (*ClosePublishersResponse, error)
+	CloseSubscribers(context.Context, *CloseSubscribersRequest) (*CloseSubscribersResponse, error)
+	// subscriber connects to broker balancer, which coordinates with the subscribers
+	SubscriberToSubCoordinator(grpc.BidiStreamingServer[SubscriberToSubCoordinatorRequest, SubscriberToSubCoordinatorResponse]) error
+	// data plane for each topic partition
+	PublishMessage(grpc.BidiStreamingServer[PublishMessageRequest, PublishMessageResponse]) error
+	SubscribeMessage(grpc.BidiStreamingServer[SubscribeMessageRequest, SubscribeMessageResponse]) error
+	// The lead broker asks a follower broker to follow itself
+	PublishFollowMe(grpc.BidiStreamingServer[PublishFollowMeRequest, PublishFollowMeResponse]) error
+	SubscribeFollowMe(grpc.ClientStreamingServer[SubscribeFollowMeRequest, SubscribeFollowMeResponse]) error
+	// Stateless fetch API (Kafka-style) - request/response pattern
+	// This is the recommended API for Kafka gateway and other stateless clients
+	// No streaming, no session state - each request is completely independent
+	FetchMessage(context.Context, *FetchMessageRequest) (*FetchMessageResponse, error)
+	// SQL query support - get unflushed messages from broker's in-memory buffer (streaming)
+	GetUnflushedMessages(*GetUnflushedMessagesRequest, grpc.ServerStreamingServer[GetUnflushedMessagesResponse]) error
+	// Get comprehensive partition range information (offsets, timestamps, and other fields)
+	GetPartitionRangeInfo(context.Context, *GetPartitionRangeInfoRequest) (*GetPartitionRangeInfoResponse, error)
+	mustEmbedUnimplementedSeaweedMessagingServer()
+}
+
+// UnimplementedSeaweedMessagingServer must be embedded to have
+// forward compatible implementations.
+//
+// NOTE: this should be embedded by value instead of pointer to avoid a nil
+// pointer dereference when methods are called.
+type UnimplementedSeaweedMessagingServer struct{}
+
+func (UnimplementedSeaweedMessagingServer) FindBrokerLeader(context.Context, *FindBrokerLeaderRequest) (*FindBrokerLeaderResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method FindBrokerLeader not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) PublisherToPubBalancer(grpc.BidiStreamingServer[PublisherToPubBalancerRequest, PublisherToPubBalancerResponse]) error {
+	return status.Errorf(codes.Unimplemented, "method PublisherToPubBalancer not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) BalanceTopics(context.Context, *BalanceTopicsRequest) (*BalanceTopicsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method BalanceTopics not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) ListTopics(context.Context, *ListTopicsRequest) (*ListTopicsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ListTopics not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) TopicExists(context.Context, *TopicExistsRequest) (*TopicExistsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method TopicExists not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) ConfigureTopic(context.Context, *ConfigureTopicRequest) (*ConfigureTopicResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ConfigureTopic not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) LookupTopicBrokers(context.Context, *LookupTopicBrokersRequest) (*LookupTopicBrokersResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method LookupTopicBrokers not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) GetTopicConfiguration(context.Context, *GetTopicConfigurationRequest) (*GetTopicConfigurationResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetTopicConfiguration not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) GetTopicPublishers(context.Context, *GetTopicPublishersRequest) (*GetTopicPublishersResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetTopicPublishers not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) GetTopicSubscribers(context.Context, *GetTopicSubscribersRequest) (*GetTopicSubscribersResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetTopicSubscribers not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) AssignTopicPartitions(context.Context, *AssignTopicPartitionsRequest) (*AssignTopicPartitionsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method AssignTopicPartitions not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) ClosePublishers(context.Context, *ClosePublishersRequest) (*ClosePublishersResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ClosePublishers not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) CloseSubscribers(context.Context, *CloseSubscribersRequest) (*CloseSubscribersResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method CloseSubscribers not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) SubscriberToSubCoordinator(grpc.BidiStreamingServer[SubscriberToSubCoordinatorRequest, SubscriberToSubCoordinatorResponse]) error {
+	return status.Errorf(codes.Unimplemented, "method SubscriberToSubCoordinator not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) PublishMessage(grpc.BidiStreamingServer[PublishMessageRequest, PublishMessageResponse]) error {
+	return status.Errorf(codes.Unimplemented, "method PublishMessage not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) SubscribeMessage(grpc.BidiStreamingServer[SubscribeMessageRequest, SubscribeMessageResponse]) error {
+	return status.Errorf(codes.Unimplemented, "method SubscribeMessage not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) PublishFollowMe(grpc.BidiStreamingServer[PublishFollowMeRequest, PublishFollowMeResponse]) error {
+	return status.Errorf(codes.Unimplemented, "method PublishFollowMe not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) SubscribeFollowMe(grpc.ClientStreamingServer[SubscribeFollowMeRequest, SubscribeFollowMeResponse]) error {
+	return status.Errorf(codes.Unimplemented, "method SubscribeFollowMe not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) FetchMessage(context.Context, *FetchMessageRequest) (*FetchMessageResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method FetchMessage not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) GetUnflushedMessages(*GetUnflushedMessagesRequest, grpc.ServerStreamingServer[GetUnflushedMessagesResponse]) error {
+	return status.Errorf(codes.Unimplemented, "method GetUnflushedMessages not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) GetPartitionRangeInfo(context.Context, *GetPartitionRangeInfoRequest) (*GetPartitionRangeInfoResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetPartitionRangeInfo not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) mustEmbedUnimplementedSeaweedMessagingServer() {}
+func (UnimplementedSeaweedMessagingServer) testEmbeddedByValue()                          {}
+
+// UnsafeSeaweedMessagingServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to SeaweedMessagingServer will
+// result in compilation errors.
+type UnsafeSeaweedMessagingServer interface {
+	mustEmbedUnimplementedSeaweedMessagingServer()
+}
+
+func RegisterSeaweedMessagingServer(s grpc.ServiceRegistrar, srv SeaweedMessagingServer) {
+	// If the following call pancis, it indicates UnimplementedSeaweedMessagingServer was
+	// embedded by pointer and is nil.  This will cause panics if an
+	// unimplemented method is ever invoked, so we test this at initialization
+	// time to prevent it from happening at runtime later due to I/O.
+	if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
+		t.testEmbeddedByValue()
+	}
+	s.RegisterService(&SeaweedMessaging_ServiceDesc, srv)
+}
+
+func _SeaweedMessaging_FindBrokerLeader_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(FindBrokerLeaderRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).FindBrokerLeader(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessaging_FindBrokerLeader_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).FindBrokerLeader(ctx, req.(*FindBrokerLeaderRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_PublisherToPubBalancer_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(SeaweedMessagingServer).PublisherToPubBalancer(&grpc.GenericServerStream[PublisherToPubBalancerRequest, PublisherToPubBalancerResponse]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessaging_PublisherToPubBalancerServer = grpc.BidiStreamingServer[PublisherToPubBalancerRequest, PublisherToPubBalancerResponse]
+
+func _SeaweedMessaging_BalanceTopics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(BalanceTopicsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).BalanceTopics(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessaging_BalanceTopics_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).BalanceTopics(ctx, req.(*BalanceTopicsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_ListTopics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ListTopicsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).ListTopics(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessaging_ListTopics_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).ListTopics(ctx, req.(*ListTopicsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_TopicExists_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(TopicExistsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).TopicExists(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessaging_TopicExists_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).TopicExists(ctx, req.(*TopicExistsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_ConfigureTopic_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ConfigureTopicRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).ConfigureTopic(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessaging_ConfigureTopic_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).ConfigureTopic(ctx, req.(*ConfigureTopicRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_LookupTopicBrokers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(LookupTopicBrokersRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).LookupTopicBrokers(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessaging_LookupTopicBrokers_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).LookupTopicBrokers(ctx, req.(*LookupTopicBrokersRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_GetTopicConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetTopicConfigurationRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).GetTopicConfiguration(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessaging_GetTopicConfiguration_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).GetTopicConfiguration(ctx, req.(*GetTopicConfigurationRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_GetTopicPublishers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetTopicPublishersRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).GetTopicPublishers(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessaging_GetTopicPublishers_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).GetTopicPublishers(ctx, req.(*GetTopicPublishersRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_GetTopicSubscribers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetTopicSubscribersRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).GetTopicSubscribers(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessaging_GetTopicSubscribers_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).GetTopicSubscribers(ctx, req.(*GetTopicSubscribersRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_AssignTopicPartitions_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(AssignTopicPartitionsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).AssignTopicPartitions(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessaging_AssignTopicPartitions_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).AssignTopicPartitions(ctx, req.(*AssignTopicPartitionsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_ClosePublishers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ClosePublishersRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).ClosePublishers(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessaging_ClosePublishers_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).ClosePublishers(ctx, req.(*ClosePublishersRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_CloseSubscribers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(CloseSubscribersRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).CloseSubscribers(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessaging_CloseSubscribers_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).CloseSubscribers(ctx, req.(*CloseSubscribersRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_SubscriberToSubCoordinator_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(SeaweedMessagingServer).SubscriberToSubCoordinator(&grpc.GenericServerStream[SubscriberToSubCoordinatorRequest, SubscriberToSubCoordinatorResponse]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessaging_SubscriberToSubCoordinatorServer = grpc.BidiStreamingServer[SubscriberToSubCoordinatorRequest, SubscriberToSubCoordinatorResponse]
+
+func _SeaweedMessaging_PublishMessage_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(SeaweedMessagingServer).PublishMessage(&grpc.GenericServerStream[PublishMessageRequest, PublishMessageResponse]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessaging_PublishMessageServer = grpc.BidiStreamingServer[PublishMessageRequest, PublishMessageResponse]
+
+func _SeaweedMessaging_SubscribeMessage_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(SeaweedMessagingServer).SubscribeMessage(&grpc.GenericServerStream[SubscribeMessageRequest, SubscribeMessageResponse]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessaging_SubscribeMessageServer = grpc.BidiStreamingServer[SubscribeMessageRequest, SubscribeMessageResponse]
+
+func _SeaweedMessaging_PublishFollowMe_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(SeaweedMessagingServer).PublishFollowMe(&grpc.GenericServerStream[PublishFollowMeRequest, PublishFollowMeResponse]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessaging_PublishFollowMeServer = grpc.BidiStreamingServer[PublishFollowMeRequest, PublishFollowMeResponse]
+
+func _SeaweedMessaging_SubscribeFollowMe_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(SeaweedMessagingServer).SubscribeFollowMe(&grpc.GenericServerStream[SubscribeFollowMeRequest, SubscribeFollowMeResponse]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessaging_SubscribeFollowMeServer = grpc.ClientStreamingServer[SubscribeFollowMeRequest, SubscribeFollowMeResponse]
+
+func _SeaweedMessaging_FetchMessage_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(FetchMessageRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).FetchMessage(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessaging_FetchMessage_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).FetchMessage(ctx, req.(*FetchMessageRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_GetUnflushedMessages_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(GetUnflushedMessagesRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(SeaweedMessagingServer).GetUnflushedMessages(m, &grpc.GenericServerStream[GetUnflushedMessagesRequest, GetUnflushedMessagesResponse]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type SeaweedMessaging_GetUnflushedMessagesServer = grpc.ServerStreamingServer[GetUnflushedMessagesResponse]
+
+func _SeaweedMessaging_GetPartitionRangeInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetPartitionRangeInfoRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).GetPartitionRangeInfo(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessaging_GetPartitionRangeInfo_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).GetPartitionRangeInfo(ctx, req.(*GetPartitionRangeInfoRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// SeaweedMessaging_ServiceDesc is the grpc.ServiceDesc for SeaweedMessaging service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var SeaweedMessaging_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "messaging_pb.SeaweedMessaging",
+	HandlerType: (*SeaweedMessagingServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "FindBrokerLeader",
+			Handler:    _SeaweedMessaging_FindBrokerLeader_Handler,
+		},
+		{
+			MethodName: "BalanceTopics",
+			Handler:    _SeaweedMessaging_BalanceTopics_Handler,
+		},
+		{
+			MethodName: "ListTopics",
+			Handler:    _SeaweedMessaging_ListTopics_Handler,
+		},
+		{
+			MethodName: "TopicExists",
+			Handler:    _SeaweedMessaging_TopicExists_Handler,
+		},
+		{
+			MethodName: "ConfigureTopic",
+			Handler:    _SeaweedMessaging_ConfigureTopic_Handler,
+		},
+		{
+			MethodName: "LookupTopicBrokers",
+			Handler:    _SeaweedMessaging_LookupTopicBrokers_Handler,
+		},
+		{
+			MethodName: "GetTopicConfiguration",
+			Handler:    _SeaweedMessaging_GetTopicConfiguration_Handler,
+		},
+		{
+			MethodName: "GetTopicPublishers",
+			Handler:    _SeaweedMessaging_GetTopicPublishers_Handler,
+		},
+		{
+			MethodName: "GetTopicSubscribers",
+			Handler:    _SeaweedMessaging_GetTopicSubscribers_Handler,
+		},
+		{
+			MethodName: "AssignTopicPartitions",
+			Handler:    _SeaweedMessaging_AssignTopicPartitions_Handler,
+		},
+		{
+			MethodName: "ClosePublishers",
+			Handler:    _SeaweedMessaging_ClosePublishers_Handler,
+		},
+		{
+			MethodName: "CloseSubscribers",
+			Handler:    _SeaweedMessaging_CloseSubscribers_Handler,
+		},
+		{
+			MethodName: "FetchMessage",
+			Handler:    _SeaweedMessaging_FetchMessage_Handler,
+		},
+		{
+			MethodName: "GetPartitionRangeInfo",
+			Handler:    _SeaweedMessaging_GetPartitionRangeInfo_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "PublisherToPubBalancer",
+			Handler:       _SeaweedMessaging_PublisherToPubBalancer_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "SubscriberToSubCoordinator",
+			Handler:       _SeaweedMessaging_SubscriberToSubCoordinator_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "PublishMessage",
+			Handler:       _SeaweedMessaging_PublishMessage_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "SubscribeMessage",
+			Handler:       _SeaweedMessaging_SubscribeMessage_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "PublishFollowMe",
+			Handler:       _SeaweedMessaging_PublishFollowMe_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "SubscribeFollowMe",
+			Handler:       _SeaweedMessaging_SubscribeFollowMe_Handler,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "GetUnflushedMessages",
+			Handler:       _SeaweedMessaging_GetUnflushedMessages_Handler,
+			ServerStreams: true,
+		},
+	},
+	Metadata: "mq_broker.proto",
+}

weed/pb/mq_schema.proto

@@ -0,0 +1,134 @@
+syntax = "proto3";
+
+package schema_pb;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/schema_pb";
+
+///////////////////////////
+// Topic definition
+///////////////////////////
+
+message Topic {
+    string namespace = 1;
+    string name = 2;
+}
+message Partition {
+    int32 ring_size = 1;
+    int32 range_start = 2;
+    int32 range_stop = 3;
+    int64 unix_time_ns = 4;
+}
+
+message Offset {
+    Topic topic = 1;
+    repeated PartitionOffset partition_offsets = 2;
+}
+
+enum OffsetType {
+    RESUME_OR_EARLIEST = 0;
+    RESET_TO_EARLIEST = 5;
+    EXACT_TS_NS = 10;
+    RESET_TO_LATEST = 15;
+    RESUME_OR_LATEST = 20;
+    // Offset-based positioning
+    EXACT_OFFSET = 25;
+    RESET_TO_OFFSET = 30;
+}
+
+message PartitionOffset {
+    Partition partition = 1;
+    int64 start_ts_ns = 2;
+    int64 start_offset = 3;  // For offset-based positioning
+}
+
+///////////////////////////
+// schema definition
+///////////////////////////
+
+message RecordType {
+    repeated Field fields = 1;
+}
+
+message Field {
+    string name = 1;
+    int32 field_index = 2;
+    Type type = 3;
+    bool is_repeated = 4;
+    bool is_required = 5;
+}
+
+message Type {
+    oneof kind {
+        ScalarType scalar_type = 1;
+        RecordType record_type = 2;
+        ListType list_type = 3;
+    }
+}
+
+enum ScalarType {
+    BOOL = 0;
+    INT32 = 1;
+    INT64 = 3;
+    FLOAT = 4;
+    DOUBLE = 5;
+    BYTES = 6;
+    STRING = 7;
+    // Parquet logical types for analytics
+    TIMESTAMP = 8;  // UTC timestamp (microseconds since epoch)
+    DATE = 9;       // Date (days since epoch)
+    DECIMAL = 10;   // Arbitrary precision decimal
+    TIME = 11;      // Time of day (microseconds)
+}
+
+message ListType {
+    Type element_type = 1;
+}
+
+///////////////////////////
+// value definition
+///////////////////////////
+message RecordValue {
+    map<string, Value> fields = 1;
+}
+message Value {
+    oneof kind {
+        bool bool_value = 1;
+        int32 int32_value = 2;
+        int64 int64_value = 3;
+        float float_value = 4;
+        double double_value = 5;
+        bytes bytes_value = 6;
+        string string_value = 7;
+        // Parquet logical type values
+        TimestampValue timestamp_value = 8;
+        DateValue date_value = 9;
+        DecimalValue decimal_value = 10;
+        TimeValue time_value = 11;
+        // Complex types
+        ListValue list_value = 14;
+        RecordValue record_value = 15;
+    }
+}
+// Parquet logical type value messages
+message TimestampValue {
+    int64 timestamp_micros = 1;  // Microseconds since Unix epoch (UTC)
+    bool is_utc = 2;             // True if UTC, false if local time
+}
+
+message DateValue {
+    int32 days_since_epoch = 1;  // Days since Unix epoch (1970-01-01)
+}
+
+message DecimalValue {
+    bytes value = 1;             // Arbitrary precision decimal as bytes
+    int32 precision = 2;         // Total number of digits
+    int32 scale = 3;             // Number of digits after decimal point
+}
+
+message TimeValue {
+    int64 time_micros = 1;       // Microseconds since midnight
+}
+
+message ListValue {
+    repeated Value values = 1;
+}

weed/pb/plugin.proto

@@ -0,0 +1,515 @@
+syntax = "proto3";
+
+package plugin;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb";
+
+import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
+
+// PluginControlService is the admin-facing stream API for external workers.
+// Workers initiate and keep this stream alive; all control plane traffic flows through it.
+service PluginControlService {
+  rpc WorkerStream(stream WorkerToAdminMessage) returns (stream AdminToWorkerMessage);
+}
+
+// WorkerToAdminMessage carries worker-originated events and responses.
+message WorkerToAdminMessage {
+  string worker_id = 1;
+  google.protobuf.Timestamp sent_at = 2;
+
+  oneof body {
+    WorkerHello hello = 10;
+    WorkerHeartbeat heartbeat = 11;
+    WorkerAcknowledge acknowledge = 12;
+    ConfigSchemaResponse config_schema_response = 13;
+    DetectionProposals detection_proposals = 14;
+    DetectionComplete detection_complete = 15;
+    JobProgressUpdate job_progress_update = 16;
+    JobCompleted job_completed = 17;
+    WorkerObservations observations = 18;
+    ObjectPreviewResponse object_preview_response = 19;
+  }
+}
+
+// AdminToWorkerMessage carries commands and lifecycle notifications from admin.
+message AdminToWorkerMessage {
+  string request_id = 1;
+  google.protobuf.Timestamp sent_at = 2;
+
+  oneof body {
+    AdminHello hello = 10;
+    RequestConfigSchema request_config_schema = 11;
+    RunDetectionRequest run_detection_request = 12;
+    ExecuteJobRequest execute_job_request = 13;
+    CancelRequest cancel_request = 14;
+    AdminShutdown shutdown = 15;
+    RequestObjectPreview request_object_preview = 16;
+  }
+}
+
+message WorkerHello {
+  string worker_id = 1;
+  string worker_instance_id = 2;
+  string address = 3;
+  string worker_version = 4;
+  string protocol_version = 5;
+  repeated JobTypeCapability capabilities = 6;
+  map<string, string> metadata = 7;
+}
+
+message AdminHello {
+  bool accepted = 1;
+  string message = 2;
+  int32 heartbeat_interval_seconds = 3;
+  int32 reconnect_delay_seconds = 4;
+}
+
+message WorkerHeartbeat {
+  string worker_id = 1;
+  repeated RunningWork running_work = 2;
+  int32 detection_slots_used = 3;
+  int32 detection_slots_total = 4;
+  int32 execution_slots_used = 5;
+  int32 execution_slots_total = 6;
+  map<string, int32> queued_jobs_by_type = 7;
+  map<string, string> metadata = 8;
+}
+
+message WorkerAcknowledge {
+  string request_id = 1;
+  bool accepted = 2;
+  string message = 3;
+}
+
+message RunningWork {
+  string work_id = 1;
+  WorkKind kind = 2;
+  string job_type = 3;
+  JobState state = 4;
+  double progress_percent = 5;
+  string stage = 6;
+}
+
+message JobTypeCapability {
+  string job_type = 1;
+  bool can_detect = 2;
+  bool can_execute = 3;
+  int32 max_detection_concurrency = 4;
+  int32 max_execution_concurrency = 5;
+  string display_name = 6;
+  string description = 7;
+  int32 weight = 8;
+}
+
+message RequestConfigSchema {
+  string job_type = 1;
+  bool force_refresh = 2;
+}
+
+message ConfigSchemaResponse {
+  string request_id = 1;
+  string job_type = 2;
+  bool success = 3;
+  string error_message = 4;
+  JobTypeDescriptor job_type_descriptor = 5;
+}
+
+// JobTypeDescriptor defines one job type contract, including UI schema and defaults.
+message JobTypeDescriptor {
+  string job_type = 1;
+  string display_name = 2;
+  string description = 3;
+  string icon = 4;
+  uint32 descriptor_version = 5;
+
+  // Admin-owned options such as detection frequency and dispatch concurrency.
+  ConfigForm admin_config_form = 6;
+  // Worker-owned options used during detection and execution.
+  ConfigForm worker_config_form = 7;
+
+  AdminRuntimeDefaults admin_runtime_defaults = 8;
+  map<string, ConfigValue> worker_default_values = 9;
+}
+
+message ConfigForm {
+  string form_id = 1;
+  string title = 2;
+  string description = 3;
+  repeated ConfigSection sections = 4;
+  map<string, ConfigValue> default_values = 5;
+}
+
+message ConfigSection {
+  string section_id = 1;
+  string title = 2;
+  string description = 3;
+  repeated ConfigField fields = 4;
+}
+
+message ConfigField {
+  string name = 1;
+  string label = 2;
+  string description = 3;
+  string help_text = 4;
+  string placeholder = 5;
+
+  ConfigFieldType field_type = 6;
+  ConfigWidget widget = 7;
+
+  bool required = 8;
+  bool read_only = 9;
+  bool sensitive = 10;
+
+  ConfigValue min_value = 11;
+  ConfigValue max_value = 12;
+
+  repeated ConfigOption options = 13;
+  repeated ValidationRule validation_rules = 14;
+
+  // Simple visibility dependency: show this field when the referenced field equals value.
+  string visible_when_field = 15;
+  ConfigValue visible_when_equals = 16;
+}
+
+message ConfigOption {
+  string value = 1;
+  string label = 2;
+  string description = 3;
+  bool disabled = 4;
+}
+
+message ValidationRule {
+  ValidationRuleType type = 1;
+  string expression = 2;
+  string error_message = 3;
+}
+
+message ConfigValue {
+  oneof kind {
+    bool bool_value = 1;
+    int64 int64_value = 2;
+    double double_value = 3;
+    string string_value = 4;
+    bytes bytes_value = 5;
+    google.protobuf.Duration duration_value = 6;
+    StringList string_list = 7;
+    Int64List int64_list = 8;
+    DoubleList double_list = 9;
+    BoolList bool_list = 10;
+    ValueList list_value = 11;
+    ValueMap map_value = 12;
+  }
+}
+
+message StringList {
+  repeated string values = 1;
+}
+
+message Int64List {
+  repeated int64 values = 1;
+}
+
+message DoubleList {
+  repeated double values = 1;
+}
+
+message BoolList {
+  repeated bool values = 1;
+}
+
+message ValueList {
+  repeated ConfigValue values = 1;
+}
+
+message ValueMap {
+  map<string, ConfigValue> fields = 1;
+}
+
+message AdminRuntimeDefaults {
+  bool enabled = 1;
+  int32 detection_interval_minutes = 2;
+  int32 detection_timeout_seconds = 3;
+  int32 max_jobs_per_detection = 4;
+  int32 global_execution_concurrency = 5;
+  int32 per_worker_execution_concurrency = 6;
+  int32 retry_limit = 7;
+  int32 retry_backoff_seconds = 8;
+  int32 job_type_max_runtime_seconds = 9;
+  // Per-attempt execution deadline for one job. Distinct from
+  // detection_timeout (which bounds the detection scan) and
+  // job_type_max_runtime (a lane-wide budget across all attempts).
+  int32 execution_timeout_seconds = 10;
+}
+
+message AdminRuntimeConfig {
+  bool enabled = 1;
+  int32 detection_interval_minutes = 2;
+  int32 detection_timeout_seconds = 3;
+  int32 max_jobs_per_detection = 4;
+  int32 global_execution_concurrency = 5;
+  int32 per_worker_execution_concurrency = 6;
+  int32 retry_limit = 7;
+  int32 retry_backoff_seconds = 8;
+  int32 job_type_max_runtime_seconds = 9;
+  int32 execution_timeout_seconds = 10;
+}
+
+message RunDetectionRequest {
+  string request_id = 1;
+  string job_type = 2;
+  int64 detection_sequence = 3;
+
+  AdminRuntimeConfig admin_runtime = 4;
+  map<string, ConfigValue> admin_config_values = 5;
+  map<string, ConfigValue> worker_config_values = 6;
+
+  ClusterContext cluster_context = 7;
+  google.protobuf.Timestamp last_successful_run = 8;
+  int32 max_results = 9;
+}
+
+message DetectionProposals {
+  string request_id = 1;
+  string job_type = 2;
+  repeated JobProposal proposals = 3;
+  bool has_more = 4;
+}
+
+message DetectionComplete {
+  string request_id = 1;
+  string job_type = 2;
+  bool success = 3;
+  string error_message = 4;
+  int32 total_proposals = 5;
+}
+
+message JobProposal {
+  string proposal_id = 1;
+  string dedupe_key = 2;
+  string job_type = 3;
+  JobPriority priority = 4;
+  string summary = 5;
+  string detail = 6;
+  map<string, ConfigValue> parameters = 7;
+  map<string, string> labels = 8;
+  google.protobuf.Timestamp not_before = 9;
+  google.protobuf.Timestamp expires_at = 10;
+}
+
+message ExecuteJobRequest {
+  string request_id = 1;
+  JobSpec job = 2;
+  AdminRuntimeConfig admin_runtime = 3;
+  map<string, ConfigValue> admin_config_values = 4;
+  map<string, ConfigValue> worker_config_values = 5;
+  ClusterContext cluster_context = 6;
+  int32 attempt = 7;
+}
+
+message JobSpec {
+  string job_id = 1;
+  string job_type = 2;
+  string dedupe_key = 3;
+  JobPriority priority = 4;
+  string summary = 5;
+  string detail = 6;
+  map<string, ConfigValue> parameters = 7;
+  map<string, string> labels = 8;
+  google.protobuf.Timestamp created_at = 9;
+  google.protobuf.Timestamp scheduled_at = 10;
+}
+
+message JobProgressUpdate {
+  string request_id = 1;
+  string job_id = 2;
+  string job_type = 3;
+  JobState state = 4;
+  double progress_percent = 5;
+  string stage = 6;
+  string message = 7;
+  map<string, ConfigValue> metrics = 8;
+  repeated ActivityEvent activities = 9;
+  google.protobuf.Timestamp updated_at = 10;
+}
+
+message JobCompleted {
+  string request_id = 1;
+  string job_id = 2;
+  string job_type = 3;
+  bool success = 4;
+  string error_message = 5;
+  JobResult result = 6;
+  repeated ActivityEvent activities = 7;
+  google.protobuf.Timestamp completed_at = 8;
+}
+
+message JobResult {
+  map<string, ConfigValue> output_values = 1;
+  string summary = 2;
+}
+
+message ClusterContext {
+  repeated string master_grpc_addresses = 1;
+  // Filers in pb.ServerAddress form (host:httpPort.grpcPort). Consumers
+  // convert to a gRPC or HTTP address as needed; see weed/pb/server_address.go.
+  repeated string filer_addresses = 2;
+  repeated string volume_grpc_addresses = 3;
+  map<string, string> metadata = 4;
+  repeated string s3_grpc_addresses = 5;
+}
+
+message ActivityEvent {
+  ActivitySource source = 1;
+  string message = 2;
+  string stage = 3;
+  map<string, ConfigValue> details = 4;
+  google.protobuf.Timestamp created_at = 5;
+}
+
+message CancelRequest {
+  string target_id = 1;
+  WorkKind target_kind = 2;
+  string reason = 3;
+  bool force = 4;
+}
+
+message AdminShutdown {
+  string reason = 1;
+  int32 grace_period_seconds = 2;
+}
+
+// PersistedJobTypeConfig is the admin-side on-disk model per job type.
+message PersistedJobTypeConfig {
+  string job_type = 1;
+  uint32 descriptor_version = 2;
+  map<string, ConfigValue> admin_config_values = 3;
+  map<string, ConfigValue> worker_config_values = 4;
+  AdminRuntimeConfig admin_runtime = 5;
+  google.protobuf.Timestamp updated_at = 6;
+  string updated_by = 7;
+}
+
+enum WorkKind {
+  WORK_KIND_UNSPECIFIED = 0;
+  WORK_KIND_DETECTION = 1;
+  WORK_KIND_EXECUTION = 2;
+}
+
+enum JobPriority {
+  JOB_PRIORITY_UNSPECIFIED = 0;
+  JOB_PRIORITY_LOW = 1;
+  JOB_PRIORITY_NORMAL = 2;
+  JOB_PRIORITY_HIGH = 3;
+  JOB_PRIORITY_CRITICAL = 4;
+}
+
+enum JobState {
+  JOB_STATE_UNSPECIFIED = 0;
+  JOB_STATE_PENDING = 1;
+  JOB_STATE_ASSIGNED = 2;
+  JOB_STATE_RUNNING = 3;
+  JOB_STATE_SUCCEEDED = 4;
+  JOB_STATE_FAILED = 5;
+  JOB_STATE_CANCELED = 6;
+}
+
+enum ConfigFieldType {
+  CONFIG_FIELD_TYPE_UNSPECIFIED = 0;
+  CONFIG_FIELD_TYPE_BOOL = 1;
+  CONFIG_FIELD_TYPE_INT64 = 2;
+  CONFIG_FIELD_TYPE_DOUBLE = 3;
+  CONFIG_FIELD_TYPE_STRING = 4;
+  CONFIG_FIELD_TYPE_BYTES = 5;
+  CONFIG_FIELD_TYPE_DURATION = 6;
+  CONFIG_FIELD_TYPE_ENUM = 7;
+  CONFIG_FIELD_TYPE_LIST = 8;
+  CONFIG_FIELD_TYPE_OBJECT = 9;
+}
+
+enum ConfigWidget {
+  CONFIG_WIDGET_UNSPECIFIED = 0;
+  CONFIG_WIDGET_TOGGLE = 1;
+  CONFIG_WIDGET_TEXT = 2;
+  CONFIG_WIDGET_TEXTAREA = 3;
+  CONFIG_WIDGET_NUMBER = 4;
+  CONFIG_WIDGET_SELECT = 5;
+  CONFIG_WIDGET_MULTI_SELECT = 6;
+  CONFIG_WIDGET_DURATION = 7;
+  CONFIG_WIDGET_PASSWORD = 8;
+}
+
+enum ValidationRuleType {
+  VALIDATION_RULE_TYPE_UNSPECIFIED = 0;
+  VALIDATION_RULE_TYPE_REGEX = 1;
+  VALIDATION_RULE_TYPE_MIN_LENGTH = 2;
+  VALIDATION_RULE_TYPE_MAX_LENGTH = 3;
+  VALIDATION_RULE_TYPE_MIN_ITEMS = 4;
+  VALIDATION_RULE_TYPE_MAX_ITEMS = 5;
+  VALIDATION_RULE_TYPE_CUSTOM = 6;
+}
+
+enum ActivitySource {
+  ACTIVITY_SOURCE_UNSPECIFIED = 0;
+  ACTIVITY_SOURCE_ADMIN = 1;
+  ACTIVITY_SOURCE_DETECTOR = 2;
+  ACTIVITY_SOURCE_EXECUTOR = 3;
+}
+
+// WorkerObservations reports what a worker learned about the objects it
+// inspected. A worker opens a table to decide whether it needs a job; what it
+// saw is worth keeping either way, because for a format the cluster cannot
+// read, the worker is the only thing that can describe it.
+//
+// Observations are not work. Admin caches them and serves them back, so a
+// worker that goes away leaves its last ones standing rather than blanking a
+// page.
+message WorkerObservations {
+  string job_type = 1;
+  repeated ObjectObservation observations = 2;
+}
+
+// ObjectObservation is one object as a worker last saw it.
+message ObjectObservation {
+  // Identifier of the object, e.g. ["bucket", "namespace", "table"].
+  repeated string object_id = 1;
+  // What kind of object this is, e.g. "table".
+  string object_kind = 2;
+  // The object's format, e.g. "LANCE", so a reader can tell whose observation
+  // this is without parsing the attributes.
+  string format = 3;
+  // Whatever the worker can cheaply say: schema, row count, fragment count.
+  // The keys are the worker's to choose.
+  map<string, ConfigValue> attributes = 4;
+  google.protobuf.Timestamp observed_at = 5;
+}
+
+// RequestObjectPreview asks a worker for sample rows of an object admin cannot
+// read itself. Unlike an observation this is not cached: it is fetched when
+// someone opens the page, because rows are the object's data rather than a
+// description of it, and holding a copy in admin is neither fresh nor its
+// business.
+message RequestObjectPreview {
+  repeated string object_id = 1;
+  // The format admin believes this object is, so a worker that does not own it
+  // can decline instead of guessing.
+  string format = 2;
+  int32 row_limit = 3;
+}
+
+// ObjectPreviewResponse carries the sample back, already rendered as text. The
+// worker is the only thing that knows the object's types, so it formats them;
+// admin displays what it is given.
+message ObjectPreviewResponse {
+  string request_id = 6;
+  bool success = 1;
+  string error_message = 2;
+  repeated string columns = 3;
+  repeated PreviewRow rows = 4;
+  // Rows in the object, which is not the number sampled.
+  int64 total_rows = 5;
+}
+
+message PreviewRow {
+  repeated string values = 1;
+}

weed/pb/plugin_pb/plugin_grpc.pb.go

@@ -0,0 +1,121 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.5.1
+// - protoc             v6.33.4
+// source: plugin.proto
+
+package plugin_pb
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.64.0 or later.
+const _ = grpc.SupportPackageIsVersion9
+
+const (
+	PluginControlService_WorkerStream_FullMethodName = "/plugin.PluginControlService/WorkerStream"
+)
+
+// PluginControlServiceClient is the client API for PluginControlService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+//
+// PluginControlService is the admin-facing stream API for external workers.
+// Workers initiate and keep this stream alive; all control plane traffic flows through it.
+type PluginControlServiceClient interface {
+	WorkerStream(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[WorkerToAdminMessage, AdminToWorkerMessage], error)
+}
+
+type pluginControlServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewPluginControlServiceClient(cc grpc.ClientConnInterface) PluginControlServiceClient {
+	return &pluginControlServiceClient{cc}
+}
+
+func (c *pluginControlServiceClient) WorkerStream(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[WorkerToAdminMessage, AdminToWorkerMessage], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &PluginControlService_ServiceDesc.Streams[0], PluginControlService_WorkerStream_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[WorkerToAdminMessage, AdminToWorkerMessage]{ClientStream: stream}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type PluginControlService_WorkerStreamClient = grpc.BidiStreamingClient[WorkerToAdminMessage, AdminToWorkerMessage]
+
+// PluginControlServiceServer is the server API for PluginControlService service.
+// All implementations must embed UnimplementedPluginControlServiceServer
+// for forward compatibility.
+//
+// PluginControlService is the admin-facing stream API for external workers.
+// Workers initiate and keep this stream alive; all control plane traffic flows through it.
+type PluginControlServiceServer interface {
+	WorkerStream(grpc.BidiStreamingServer[WorkerToAdminMessage, AdminToWorkerMessage]) error
+	mustEmbedUnimplementedPluginControlServiceServer()
+}
+
+// UnimplementedPluginControlServiceServer must be embedded to have
+// forward compatible implementations.
+//
+// NOTE: this should be embedded by value instead of pointer to avoid a nil
+// pointer dereference when methods are called.
+type UnimplementedPluginControlServiceServer struct{}
+
+func (UnimplementedPluginControlServiceServer) WorkerStream(grpc.BidiStreamingServer[WorkerToAdminMessage, AdminToWorkerMessage]) error {
+	return status.Errorf(codes.Unimplemented, "method WorkerStream not implemented")
+}
+func (UnimplementedPluginControlServiceServer) mustEmbedUnimplementedPluginControlServiceServer() {}
+func (UnimplementedPluginControlServiceServer) testEmbeddedByValue()                              {}
+
+// UnsafePluginControlServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to PluginControlServiceServer will
+// result in compilation errors.
+type UnsafePluginControlServiceServer interface {
+	mustEmbedUnimplementedPluginControlServiceServer()
+}
+
+func RegisterPluginControlServiceServer(s grpc.ServiceRegistrar, srv PluginControlServiceServer) {
+	// If the following call pancis, it indicates UnimplementedPluginControlServiceServer was
+	// embedded by pointer and is nil.  This will cause panics if an
+	// unimplemented method is ever invoked, so we test this at initialization
+	// time to prevent it from happening at runtime later due to I/O.
+	if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
+		t.testEmbeddedByValue()
+	}
+	s.RegisterService(&PluginControlService_ServiceDesc, srv)
+}
+
+func _PluginControlService_WorkerStream_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(PluginControlServiceServer).WorkerStream(&grpc.GenericServerStream[WorkerToAdminMessage, AdminToWorkerMessage]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type PluginControlService_WorkerStreamServer = grpc.BidiStreamingServer[WorkerToAdminMessage, AdminToWorkerMessage]
+
+// PluginControlService_ServiceDesc is the grpc.ServiceDesc for PluginControlService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var PluginControlService_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "plugin.PluginControlService",
+	HandlerType: (*PluginControlServiceServer)(nil),
+	Methods:     []grpc.MethodDesc{},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "WorkerStream",
+			Handler:       _PluginControlService_WorkerStream_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "plugin.proto",
+}

weed/pb/proto_read_write_test.go

@@ -0,0 +1,43 @@
+package pb
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	jsonpb "google.golang.org/protobuf/encoding/protojson"
+)
+
+func TestJsonpMarshalUnmarshal(t *testing.T) {
+
+	tv := &volume_server_pb.RemoteFile{
+		BackendType: "aws",
+		BackendId:   "",
+		FileSize:    12,
+	}
+
+	m := jsonpb.MarshalOptions{
+		EmitUnpopulated: true,
+		Indent:          "  ",
+	}
+
+	if text, err := m.Marshal(tv); err != nil {
+		fmt.Printf("marshal error: %v\n", err)
+	} else {
+		fmt.Printf("marshalled: %s\n", string(text))
+	}
+
+	rawJson := `{
+		"backendType":"aws",
+		"backendId":"temp",
+		"fileSize":12
+	}`
+
+	tv1 := &volume_server_pb.RemoteFile{}
+	if err := jsonpb.Unmarshal([]byte(rawJson), tv1); err != nil {
+		fmt.Printf("unmarshal error: %v\n", err)
+	}
+
+	fmt.Printf("unmarshalled: %+v\n", tv1)
+
+}

weed/pb/remote.proto

@@ -0,0 +1,83 @@
+syntax = "proto3";
+
+package remote_pb;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb";
+option java_package = "seaweedfs.client";
+option java_outer_classname = "FilerProto";
+
+/////////////////////////
+// Remote Storage related
+/////////////////////////
+message RemoteConf {
+  string type = 1;
+  string name = 2;
+  string s3_access_key = 4;
+  string s3_secret_key = 5;
+  string s3_region = 6;
+  string s3_endpoint = 7;
+  string s3_storage_class = 8;
+  bool s3_force_path_style = 9;
+  bool s3_support_tagging = 13;
+  bool s3_v4_signature = 11;
+
+  string gcs_google_application_credentials = 10;
+  string gcs_project_id = 12;
+
+  string azure_account_name = 15;
+  string azure_account_key = 16;
+  string azure_client_id = 17;
+  string azure_endpoint = 18;
+
+  string backblaze_key_id = 20;
+  string backblaze_application_key = 21;
+  string backblaze_endpoint = 22;
+  string backblaze_region = 23;
+
+  string aliyun_access_key = 25;
+  string aliyun_secret_key = 26;
+  string aliyun_endpoint = 27;
+  string aliyun_region = 28;
+
+  string tencent_secret_id = 30;
+  string tencent_secret_key = 31;
+  string tencent_endpoint = 32;
+
+  string baidu_access_key = 35;
+  string baidu_secret_key = 36;
+  string baidu_endpoint = 37;
+  string baidu_region = 38;
+
+  string wasabi_access_key = 40;
+  string wasabi_secret_key = 41;
+  string wasabi_endpoint = 42;
+  string wasabi_region = 43;
+
+  string filebase_access_key = 60;
+  string filebase_secret_key = 61;
+  string filebase_endpoint = 62;
+
+  string storj_access_key = 65;
+  string storj_secret_key = 66;
+  string storj_endpoint = 67;
+
+  string contabo_access_key = 68;
+  string contabo_secret_key = 69;
+  string contabo_endpoint = 70;
+  string contabo_region = 71;
+
+  uint32 upload_concurrency = 72; // multipart upload concurrency per file, 0 = client default (1 for S3, 16 for Azure)
+  uint32 download_concurrency = 73; // multipart download concurrency per read, 0 = client default (5 for S3, 16 for Azure)
+}
+
+message RemoteStorageMapping {
+  map<string,RemoteStorageLocation> mappings = 1;
+  string primary_bucket_storage_name = 2;
+}
+message RemoteStorageLocation {
+  string name = 1;
+  string bucket = 2;
+  string path = 3;
+  int32 listing_cache_ttl_seconds = 4; // 0 = disabled; >0 enables on-demand directory listing with this TTL in seconds
+  optional int32 cache_wait_ms = 5; // unset = size based default; 0 = read straight from the remote without caching
+}

weed/pb/remote_pb/remote.pb.go

@@ -0,0 +1,691 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.6
+// 	protoc        v6.33.4
+// source: remote.proto
+
+package remote_pb
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+	unsafe "unsafe"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// ///////////////////////
+// Remote Storage related
+// ///////////////////////
+type RemoteConf struct {
+	state                           protoimpl.MessageState `protogen:"open.v1"`
+	Type                            string                 `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"`
+	Name                            string                 `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
+	S3AccessKey                     string                 `protobuf:"bytes,4,opt,name=s3_access_key,json=s3AccessKey,proto3" json:"s3_access_key,omitempty"`
+	S3SecretKey                     string                 `protobuf:"bytes,5,opt,name=s3_secret_key,json=s3SecretKey,proto3" json:"s3_secret_key,omitempty"`
+	S3Region                        string                 `protobuf:"bytes,6,opt,name=s3_region,json=s3Region,proto3" json:"s3_region,omitempty"`
+	S3Endpoint                      string                 `protobuf:"bytes,7,opt,name=s3_endpoint,json=s3Endpoint,proto3" json:"s3_endpoint,omitempty"`
+	S3StorageClass                  string                 `protobuf:"bytes,8,opt,name=s3_storage_class,json=s3StorageClass,proto3" json:"s3_storage_class,omitempty"`
+	S3ForcePathStyle                bool                   `protobuf:"varint,9,opt,name=s3_force_path_style,json=s3ForcePathStyle,proto3" json:"s3_force_path_style,omitempty"`
+	S3SupportTagging                bool                   `protobuf:"varint,13,opt,name=s3_support_tagging,json=s3SupportTagging,proto3" json:"s3_support_tagging,omitempty"`
+	S3V4Signature                   bool                   `protobuf:"varint,11,opt,name=s3_v4_signature,json=s3V4Signature,proto3" json:"s3_v4_signature,omitempty"`
+	GcsGoogleApplicationCredentials string                 `protobuf:"bytes,10,opt,name=gcs_google_application_credentials,json=gcsGoogleApplicationCredentials,proto3" json:"gcs_google_application_credentials,omitempty"`
+	GcsProjectId                    string                 `protobuf:"bytes,12,opt,name=gcs_project_id,json=gcsProjectId,proto3" json:"gcs_project_id,omitempty"`
+	AzureAccountName                string                 `protobuf:"bytes,15,opt,name=azure_account_name,json=azureAccountName,proto3" json:"azure_account_name,omitempty"`
+	AzureAccountKey                 string                 `protobuf:"bytes,16,opt,name=azure_account_key,json=azureAccountKey,proto3" json:"azure_account_key,omitempty"`
+	AzureClientId                   string                 `protobuf:"bytes,17,opt,name=azure_client_id,json=azureClientId,proto3" json:"azure_client_id,omitempty"`
+	AzureEndpoint                   string                 `protobuf:"bytes,18,opt,name=azure_endpoint,json=azureEndpoint,proto3" json:"azure_endpoint,omitempty"`
+	BackblazeKeyId                  string                 `protobuf:"bytes,20,opt,name=backblaze_key_id,json=backblazeKeyId,proto3" json:"backblaze_key_id,omitempty"`
+	BackblazeApplicationKey         string                 `protobuf:"bytes,21,opt,name=backblaze_application_key,json=backblazeApplicationKey,proto3" json:"backblaze_application_key,omitempty"`
+	BackblazeEndpoint               string                 `protobuf:"bytes,22,opt,name=backblaze_endpoint,json=backblazeEndpoint,proto3" json:"backblaze_endpoint,omitempty"`
+	BackblazeRegion                 string                 `protobuf:"bytes,23,opt,name=backblaze_region,json=backblazeRegion,proto3" json:"backblaze_region,omitempty"`
+	AliyunAccessKey                 string                 `protobuf:"bytes,25,opt,name=aliyun_access_key,json=aliyunAccessKey,proto3" json:"aliyun_access_key,omitempty"`
+	AliyunSecretKey                 string                 `protobuf:"bytes,26,opt,name=aliyun_secret_key,json=aliyunSecretKey,proto3" json:"aliyun_secret_key,omitempty"`
+	AliyunEndpoint                  string                 `protobuf:"bytes,27,opt,name=aliyun_endpoint,json=aliyunEndpoint,proto3" json:"aliyun_endpoint,omitempty"`
+	AliyunRegion                    string                 `protobuf:"bytes,28,opt,name=aliyun_region,json=aliyunRegion,proto3" json:"aliyun_region,omitempty"`
+	TencentSecretId                 string                 `protobuf:"bytes,30,opt,name=tencent_secret_id,json=tencentSecretId,proto3" json:"tencent_secret_id,omitempty"`
+	TencentSecretKey                string                 `protobuf:"bytes,31,opt,name=tencent_secret_key,json=tencentSecretKey,proto3" json:"tencent_secret_key,omitempty"`
+	TencentEndpoint                 string                 `protobuf:"bytes,32,opt,name=tencent_endpoint,json=tencentEndpoint,proto3" json:"tencent_endpoint,omitempty"`
+	BaiduAccessKey                  string                 `protobuf:"bytes,35,opt,name=baidu_access_key,json=baiduAccessKey,proto3" json:"baidu_access_key,omitempty"`
+	BaiduSecretKey                  string                 `protobuf:"bytes,36,opt,name=baidu_secret_key,json=baiduSecretKey,proto3" json:"baidu_secret_key,omitempty"`
+	BaiduEndpoint                   string                 `protobuf:"bytes,37,opt,name=baidu_endpoint,json=baiduEndpoint,proto3" json:"baidu_endpoint,omitempty"`
+	BaiduRegion                     string                 `protobuf:"bytes,38,opt,name=baidu_region,json=baiduRegion,proto3" json:"baidu_region,omitempty"`
+	WasabiAccessKey                 string                 `protobuf:"bytes,40,opt,name=wasabi_access_key,json=wasabiAccessKey,proto3" json:"wasabi_access_key,omitempty"`
+	WasabiSecretKey                 string                 `protobuf:"bytes,41,opt,name=wasabi_secret_key,json=wasabiSecretKey,proto3" json:"wasabi_secret_key,omitempty"`
+	WasabiEndpoint                  string                 `protobuf:"bytes,42,opt,name=wasabi_endpoint,json=wasabiEndpoint,proto3" json:"wasabi_endpoint,omitempty"`
+	WasabiRegion                    string                 `protobuf:"bytes,43,opt,name=wasabi_region,json=wasabiRegion,proto3" json:"wasabi_region,omitempty"`
+	FilebaseAccessKey               string                 `protobuf:"bytes,60,opt,name=filebase_access_key,json=filebaseAccessKey,proto3" json:"filebase_access_key,omitempty"`
+	FilebaseSecretKey               string                 `protobuf:"bytes,61,opt,name=filebase_secret_key,json=filebaseSecretKey,proto3" json:"filebase_secret_key,omitempty"`
+	FilebaseEndpoint                string                 `protobuf:"bytes,62,opt,name=filebase_endpoint,json=filebaseEndpoint,proto3" json:"filebase_endpoint,omitempty"`
+	StorjAccessKey                  string                 `protobuf:"bytes,65,opt,name=storj_access_key,json=storjAccessKey,proto3" json:"storj_access_key,omitempty"`
+	StorjSecretKey                  string                 `protobuf:"bytes,66,opt,name=storj_secret_key,json=storjSecretKey,proto3" json:"storj_secret_key,omitempty"`
+	StorjEndpoint                   string                 `protobuf:"bytes,67,opt,name=storj_endpoint,json=storjEndpoint,proto3" json:"storj_endpoint,omitempty"`
+	ContaboAccessKey                string                 `protobuf:"bytes,68,opt,name=contabo_access_key,json=contaboAccessKey,proto3" json:"contabo_access_key,omitempty"`
+	ContaboSecretKey                string                 `protobuf:"bytes,69,opt,name=contabo_secret_key,json=contaboSecretKey,proto3" json:"contabo_secret_key,omitempty"`
+	ContaboEndpoint                 string                 `protobuf:"bytes,70,opt,name=contabo_endpoint,json=contaboEndpoint,proto3" json:"contabo_endpoint,omitempty"`
+	ContaboRegion                   string                 `protobuf:"bytes,71,opt,name=contabo_region,json=contaboRegion,proto3" json:"contabo_region,omitempty"`
+	UploadConcurrency               uint32                 `protobuf:"varint,72,opt,name=upload_concurrency,json=uploadConcurrency,proto3" json:"upload_concurrency,omitempty"`       // multipart upload concurrency per file, 0 = client default (1 for S3, 16 for Azure)
+	DownloadConcurrency             uint32                 `protobuf:"varint,73,opt,name=download_concurrency,json=downloadConcurrency,proto3" json:"download_concurrency,omitempty"` // multipart download concurrency per read, 0 = client default (5 for S3, 16 for Azure)
+	unknownFields                   protoimpl.UnknownFields
+	sizeCache                       protoimpl.SizeCache
+}
+
+func (x *RemoteConf) Reset() {
+	*x = RemoteConf{}
+	mi := &file_remote_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *RemoteConf) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RemoteConf) ProtoMessage() {}
+
+func (x *RemoteConf) ProtoReflect() protoreflect.Message {
+	mi := &file_remote_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RemoteConf.ProtoReflect.Descriptor instead.
+func (*RemoteConf) Descriptor() ([]byte, []int) {
+	return file_remote_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *RemoteConf) GetType() string {
+	if x != nil {
+		return x.Type
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetS3AccessKey() string {
+	if x != nil {
+		return x.S3AccessKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetS3SecretKey() string {
+	if x != nil {
+		return x.S3SecretKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetS3Region() string {
+	if x != nil {
+		return x.S3Region
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetS3Endpoint() string {
+	if x != nil {
+		return x.S3Endpoint
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetS3StorageClass() string {
+	if x != nil {
+		return x.S3StorageClass
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetS3ForcePathStyle() bool {
+	if x != nil {
+		return x.S3ForcePathStyle
+	}
+	return false
+}
+
+func (x *RemoteConf) GetS3SupportTagging() bool {
+	if x != nil {
+		return x.S3SupportTagging
+	}
+	return false
+}
+
+func (x *RemoteConf) GetS3V4Signature() bool {
+	if x != nil {
+		return x.S3V4Signature
+	}
+	return false
+}
+
+func (x *RemoteConf) GetGcsGoogleApplicationCredentials() string {
+	if x != nil {
+		return x.GcsGoogleApplicationCredentials
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetGcsProjectId() string {
+	if x != nil {
+		return x.GcsProjectId
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetAzureAccountName() string {
+	if x != nil {
+		return x.AzureAccountName
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetAzureAccountKey() string {
+	if x != nil {
+		return x.AzureAccountKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetAzureClientId() string {
+	if x != nil {
+		return x.AzureClientId
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetAzureEndpoint() string {
+	if x != nil {
+		return x.AzureEndpoint
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetBackblazeKeyId() string {
+	if x != nil {
+		return x.BackblazeKeyId
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetBackblazeApplicationKey() string {
+	if x != nil {
+		return x.BackblazeApplicationKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetBackblazeEndpoint() string {
+	if x != nil {
+		return x.BackblazeEndpoint
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetBackblazeRegion() string {
+	if x != nil {
+		return x.BackblazeRegion
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetAliyunAccessKey() string {
+	if x != nil {
+		return x.AliyunAccessKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetAliyunSecretKey() string {
+	if x != nil {
+		return x.AliyunSecretKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetAliyunEndpoint() string {
+	if x != nil {
+		return x.AliyunEndpoint
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetAliyunRegion() string {
+	if x != nil {
+		return x.AliyunRegion
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetTencentSecretId() string {
+	if x != nil {
+		return x.TencentSecretId
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetTencentSecretKey() string {
+	if x != nil {
+		return x.TencentSecretKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetTencentEndpoint() string {
+	if x != nil {
+		return x.TencentEndpoint
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetBaiduAccessKey() string {
+	if x != nil {
+		return x.BaiduAccessKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetBaiduSecretKey() string {
+	if x != nil {
+		return x.BaiduSecretKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetBaiduEndpoint() string {
+	if x != nil {
+		return x.BaiduEndpoint
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetBaiduRegion() string {
+	if x != nil {
+		return x.BaiduRegion
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetWasabiAccessKey() string {
+	if x != nil {
+		return x.WasabiAccessKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetWasabiSecretKey() string {
+	if x != nil {
+		return x.WasabiSecretKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetWasabiEndpoint() string {
+	if x != nil {
+		return x.WasabiEndpoint
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetWasabiRegion() string {
+	if x != nil {
+		return x.WasabiRegion
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetFilebaseAccessKey() string {
+	if x != nil {
+		return x.FilebaseAccessKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetFilebaseSecretKey() string {
+	if x != nil {
+		return x.FilebaseSecretKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetFilebaseEndpoint() string {
+	if x != nil {
+		return x.FilebaseEndpoint
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetStorjAccessKey() string {
+	if x != nil {
+		return x.StorjAccessKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetStorjSecretKey() string {
+	if x != nil {
+		return x.StorjSecretKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetStorjEndpoint() string {
+	if x != nil {
+		return x.StorjEndpoint
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetContaboAccessKey() string {
+	if x != nil {
+		return x.ContaboAccessKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetContaboSecretKey() string {
+	if x != nil {
+		return x.ContaboSecretKey
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetContaboEndpoint() string {
+	if x != nil {
+		return x.ContaboEndpoint
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetContaboRegion() string {
+	if x != nil {
+		return x.ContaboRegion
+	}
+	return ""
+}
+
+func (x *RemoteConf) GetUploadConcurrency() uint32 {
+	if x != nil {
+		return x.UploadConcurrency
+	}
+	return 0
+}
+
+func (x *RemoteConf) GetDownloadConcurrency() uint32 {
+	if x != nil {
+		return x.DownloadConcurrency
+	}
+	return 0
+}
+
+type RemoteStorageMapping struct {
+	state                    protoimpl.MessageState            `protogen:"open.v1"`
+	Mappings                 map[string]*RemoteStorageLocation `protobuf:"bytes,1,rep,name=mappings,proto3" json:"mappings,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	PrimaryBucketStorageName string                            `protobuf:"bytes,2,opt,name=primary_bucket_storage_name,json=primaryBucketStorageName,proto3" json:"primary_bucket_storage_name,omitempty"`
+	unknownFields            protoimpl.UnknownFields
+	sizeCache                protoimpl.SizeCache
+}
+
+func (x *RemoteStorageMapping) Reset() {
+	*x = RemoteStorageMapping{}
+	mi := &file_remote_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *RemoteStorageMapping) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RemoteStorageMapping) ProtoMessage() {}
+
+func (x *RemoteStorageMapping) ProtoReflect() protoreflect.Message {
+	mi := &file_remote_proto_msgTypes[1]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RemoteStorageMapping.ProtoReflect.Descriptor instead.
+func (*RemoteStorageMapping) Descriptor() ([]byte, []int) {
+	return file_remote_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *RemoteStorageMapping) GetMappings() map[string]*RemoteStorageLocation {
+	if x != nil {
+		return x.Mappings
+	}
+	return nil
+}
+
+func (x *RemoteStorageMapping) GetPrimaryBucketStorageName() string {
+	if x != nil {
+		return x.PrimaryBucketStorageName
+	}
+	return ""
+}
+
+type RemoteStorageLocation struct {
+	state                  protoimpl.MessageState `protogen:"open.v1"`
+	Name                   string                 `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Bucket                 string                 `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"`
+	Path                   string                 `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"`
+	ListingCacheTtlSeconds int32                  `protobuf:"varint,4,opt,name=listing_cache_ttl_seconds,json=listingCacheTtlSeconds,proto3" json:"listing_cache_ttl_seconds,omitempty"` // 0 = disabled; >0 enables on-demand directory listing with this TTL in seconds
+	CacheWaitMs            *int32                 `protobuf:"varint,5,opt,name=cache_wait_ms,json=cacheWaitMs,proto3,oneof" json:"cache_wait_ms,omitempty"`                              // unset = size based default; 0 = read straight from the remote without caching
+	unknownFields          protoimpl.UnknownFields
+	sizeCache              protoimpl.SizeCache
+}
+
+func (x *RemoteStorageLocation) Reset() {
+	*x = RemoteStorageLocation{}
+	mi := &file_remote_proto_msgTypes[2]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *RemoteStorageLocation) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RemoteStorageLocation) ProtoMessage() {}
+
+func (x *RemoteStorageLocation) ProtoReflect() protoreflect.Message {
+	mi := &file_remote_proto_msgTypes[2]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RemoteStorageLocation.ProtoReflect.Descriptor instead.
+func (*RemoteStorageLocation) Descriptor() ([]byte, []int) {
+	return file_remote_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *RemoteStorageLocation) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
+
+func (x *RemoteStorageLocation) GetBucket() string {
+	if x != nil {
+		return x.Bucket
+	}
+	return ""
+}
+
+func (x *RemoteStorageLocation) GetPath() string {
+	if x != nil {
+		return x.Path
+	}
+	return ""
+}
+
+func (x *RemoteStorageLocation) GetListingCacheTtlSeconds() int32 {
+	if x != nil {
+		return x.ListingCacheTtlSeconds
+	}
+	return 0
+}
+
+func (x *RemoteStorageLocation) GetCacheWaitMs() int32 {
+	if x != nil && x.CacheWaitMs != nil {
+		return *x.CacheWaitMs
+	}
+	return 0
+}
+
+var File_remote_proto protoreflect.FileDescriptor
+
+const file_remote_proto_rawDesc = "" +
+	"\n" +
+	"\fremote.proto\x12\tremote_pb\"\xcc\x0f\n" +
+	"\n" +
+	"RemoteConf\x12\x12\n" +
+	"\x04type\x18\x01 \x01(\tR\x04type\x12\x12\n" +
+	"\x04name\x18\x02 \x01(\tR\x04name\x12\"\n" +
+	"\rs3_access_key\x18\x04 \x01(\tR\vs3AccessKey\x12\"\n" +
+	"\rs3_secret_key\x18\x05 \x01(\tR\vs3SecretKey\x12\x1b\n" +
+	"\ts3_region\x18\x06 \x01(\tR\bs3Region\x12\x1f\n" +
+	"\vs3_endpoint\x18\a \x01(\tR\n" +
+	"s3Endpoint\x12(\n" +
+	"\x10s3_storage_class\x18\b \x01(\tR\x0es3StorageClass\x12-\n" +
+	"\x13s3_force_path_style\x18\t \x01(\bR\x10s3ForcePathStyle\x12,\n" +
+	"\x12s3_support_tagging\x18\r \x01(\bR\x10s3SupportTagging\x12&\n" +
+	"\x0fs3_v4_signature\x18\v \x01(\bR\rs3V4Signature\x12K\n" +
+	"\"gcs_google_application_credentials\x18\n" +
+	" \x01(\tR\x1fgcsGoogleApplicationCredentials\x12$\n" +
+	"\x0egcs_project_id\x18\f \x01(\tR\fgcsProjectId\x12,\n" +
+	"\x12azure_account_name\x18\x0f \x01(\tR\x10azureAccountName\x12*\n" +
+	"\x11azure_account_key\x18\x10 \x01(\tR\x0fazureAccountKey\x12&\n" +
+	"\x0fazure_client_id\x18\x11 \x01(\tR\razureClientId\x12%\n" +
+	"\x0eazure_endpoint\x18\x12 \x01(\tR\razureEndpoint\x12(\n" +
+	"\x10backblaze_key_id\x18\x14 \x01(\tR\x0ebackblazeKeyId\x12:\n" +
+	"\x19backblaze_application_key\x18\x15 \x01(\tR\x17backblazeApplicationKey\x12-\n" +
+	"\x12backblaze_endpoint\x18\x16 \x01(\tR\x11backblazeEndpoint\x12)\n" +
+	"\x10backblaze_region\x18\x17 \x01(\tR\x0fbackblazeRegion\x12*\n" +
+	"\x11aliyun_access_key\x18\x19 \x01(\tR\x0faliyunAccessKey\x12*\n" +
+	"\x11aliyun_secret_key\x18\x1a \x01(\tR\x0faliyunSecretKey\x12'\n" +
+	"\x0faliyun_endpoint\x18\x1b \x01(\tR\x0ealiyunEndpoint\x12#\n" +
+	"\raliyun_region\x18\x1c \x01(\tR\faliyunRegion\x12*\n" +
+	"\x11tencent_secret_id\x18\x1e \x01(\tR\x0ftencentSecretId\x12,\n" +
+	"\x12tencent_secret_key\x18\x1f \x01(\tR\x10tencentSecretKey\x12)\n" +
+	"\x10tencent_endpoint\x18  \x01(\tR\x0ftencentEndpoint\x12(\n" +
+	"\x10baidu_access_key\x18# \x01(\tR\x0ebaiduAccessKey\x12(\n" +
+	"\x10baidu_secret_key\x18$ \x01(\tR\x0ebaiduSecretKey\x12%\n" +
+	"\x0ebaidu_endpoint\x18% \x01(\tR\rbaiduEndpoint\x12!\n" +
+	"\fbaidu_region\x18& \x01(\tR\vbaiduRegion\x12*\n" +
+	"\x11wasabi_access_key\x18( \x01(\tR\x0fwasabiAccessKey\x12*\n" +
+	"\x11wasabi_secret_key\x18) \x01(\tR\x0fwasabiSecretKey\x12'\n" +
+	"\x0fwasabi_endpoint\x18* \x01(\tR\x0ewasabiEndpoint\x12#\n" +
+	"\rwasabi_region\x18+ \x01(\tR\fwasabiRegion\x12.\n" +
+	"\x13filebase_access_key\x18< \x01(\tR\x11filebaseAccessKey\x12.\n" +
+	"\x13filebase_secret_key\x18= \x01(\tR\x11filebaseSecretKey\x12+\n" +
+	"\x11filebase_endpoint\x18> \x01(\tR\x10filebaseEndpoint\x12(\n" +
+	"\x10storj_access_key\x18A \x01(\tR\x0estorjAccessKey\x12(\n" +
+	"\x10storj_secret_key\x18B \x01(\tR\x0estorjSecretKey\x12%\n" +
+	"\x0estorj_endpoint\x18C \x01(\tR\rstorjEndpoint\x12,\n" +
+	"\x12contabo_access_key\x18D \x01(\tR\x10contaboAccessKey\x12,\n" +
+	"\x12contabo_secret_key\x18E \x01(\tR\x10contaboSecretKey\x12)\n" +
+	"\x10contabo_endpoint\x18F \x01(\tR\x0fcontaboEndpoint\x12%\n" +
+	"\x0econtabo_region\x18G \x01(\tR\rcontaboRegion\x12-\n" +
+	"\x12upload_concurrency\x18H \x01(\rR\x11uploadConcurrency\x121\n" +
+	"\x14download_concurrency\x18I \x01(\rR\x13downloadConcurrency\"\xff\x01\n" +
+	"\x14RemoteStorageMapping\x12I\n" +
+	"\bmappings\x18\x01 \x03(\v2-.remote_pb.RemoteStorageMapping.MappingsEntryR\bmappings\x12=\n" +
+	"\x1bprimary_bucket_storage_name\x18\x02 \x01(\tR\x18primaryBucketStorageName\x1a]\n" +
+	"\rMappingsEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x126\n" +
+	"\x05value\x18\x02 \x01(\v2 .remote_pb.RemoteStorageLocationR\x05value:\x028\x01\"\xcd\x01\n" +
+	"\x15RemoteStorageLocation\x12\x12\n" +
+	"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" +
+	"\x06bucket\x18\x02 \x01(\tR\x06bucket\x12\x12\n" +
+	"\x04path\x18\x03 \x01(\tR\x04path\x129\n" +
+	"\x19listing_cache_ttl_seconds\x18\x04 \x01(\x05R\x16listingCacheTtlSeconds\x12'\n" +
+	"\rcache_wait_ms\x18\x05 \x01(\x05H\x00R\vcacheWaitMs\x88\x01\x01B\x10\n" +
+	"\x0e_cache_wait_msBP\n" +
+	"\x10seaweedfs.clientB\n" +
+	"FilerProtoZ0github.com/seaweedfs/seaweedfs/weed/pb/remote_pbb\x06proto3"
+
+var (
+	file_remote_proto_rawDescOnce sync.Once
+	file_remote_proto_rawDescData []byte
+)
+
+func file_remote_proto_rawDescGZIP() []byte {
+	file_remote_proto_rawDescOnce.Do(func() {
+		file_remote_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_remote_proto_rawDesc), len(file_remote_proto_rawDesc)))
+	})
+	return file_remote_proto_rawDescData
+}
+
+var file_remote_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
+var file_remote_proto_goTypes = []any{
+	(*RemoteConf)(nil),            // 0: remote_pb.RemoteConf
+	(*RemoteStorageMapping)(nil),  // 1: remote_pb.RemoteStorageMapping
+	(*RemoteStorageLocation)(nil), // 2: remote_pb.RemoteStorageLocation
+	nil,                           // 3: remote_pb.RemoteStorageMapping.MappingsEntry
+}
+var file_remote_proto_depIdxs = []int32{
+	3, // 0: remote_pb.RemoteStorageMapping.mappings:type_name -> remote_pb.RemoteStorageMapping.MappingsEntry
+	2, // 1: remote_pb.RemoteStorageMapping.MappingsEntry.value:type_name -> remote_pb.RemoteStorageLocation
+	2, // [2:2] is the sub-list for method output_type
+	2, // [2:2] is the sub-list for method input_type
+	2, // [2:2] is the sub-list for extension type_name
+	2, // [2:2] is the sub-list for extension extendee
+	0, // [0:2] is the sub-list for field type_name
+}
+
+func init() { file_remote_proto_init() }
+func file_remote_proto_init() {
+	if File_remote_proto != nil {
+		return
+	}
+	file_remote_proto_msgTypes[2].OneofWrappers = []any{}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_remote_proto_rawDesc), len(file_remote_proto_rawDesc)),
+			NumEnums:      0,
+			NumMessages:   4,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_remote_proto_goTypes,
+		DependencyIndexes: file_remote_proto_depIdxs,
+		MessageInfos:      file_remote_proto_msgTypes,
+	}.Build()
+	File_remote_proto = out.File
+	file_remote_proto_goTypes = nil
+	file_remote_proto_depIdxs = nil
+}

weed/pb/remote_pb/remote_pb_helper.go

@@ -0,0 +1,8 @@
+package remote_pb
+
+import "google.golang.org/protobuf/proto"
+
+func (fp *RemoteStorageLocation) Key() interface{} {
+	key, _ := proto.Marshal(fp)
+	return string(key)
+}

weed/pb/s3.proto

@@ -0,0 +1,63 @@
+syntax = "proto3";
+
+package messaging_pb;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb";
+option java_package = "seaweedfs.client";
+option java_outer_classname = "S3Proto";
+
+//////////////////////////////////////////////////
+
+import "iam.proto";
+
+// Designed for unidirectional propagation from Filer to S3 Servers
+service SeaweedS3IamCache {
+    rpc PutIdentity (iam_pb.PutIdentityRequest) returns (iam_pb.PutIdentityResponse);
+    rpc RemoveIdentity (iam_pb.RemoveIdentityRequest) returns (iam_pb.RemoveIdentityResponse);
+    rpc PutPolicy (iam_pb.PutPolicyRequest) returns (iam_pb.PutPolicyResponse);
+    rpc GetPolicy (iam_pb.GetPolicyRequest) returns (iam_pb.GetPolicyResponse);
+    rpc ListPolicies (iam_pb.ListPoliciesRequest) returns (iam_pb.ListPoliciesResponse);
+    rpc DeletePolicy (iam_pb.DeletePolicyRequest) returns (iam_pb.DeletePolicyResponse);
+    rpc PutGroup (iam_pb.PutGroupRequest) returns (iam_pb.PutGroupResponse);
+    rpc RemoveGroup (iam_pb.RemoveGroupRequest) returns (iam_pb.RemoveGroupResponse);
+}
+
+//////////////////////////////////////////////////
+
+message S3CircuitBreakerConfig {
+    S3CircuitBreakerOptions global=1;
+    map<string, S3CircuitBreakerOptions> buckets= 2;
+}
+
+message S3CircuitBreakerOptions {
+    bool enabled=1;
+    map<string, int64> actions = 2;
+}
+
+//////////////////////////////////////////////////
+// Bucket Metadata
+
+message CORSRule {
+    repeated string allowed_headers = 1;
+    repeated string allowed_methods = 2;
+    repeated string allowed_origins = 3;
+    repeated string expose_headers = 4;
+    int32 max_age_seconds = 5;
+    string id = 6;
+}
+
+message CORSConfiguration {
+    repeated CORSRule cors_rules = 1;
+}
+
+message BucketMetadata {
+    map<string, string> tags = 1;
+    CORSConfiguration cors = 2;
+    EncryptionConfiguration encryption = 3;
+}
+
+message EncryptionConfiguration {
+    string sse_algorithm = 1; // "AES256" or "aws:kms"
+    string kms_key_id = 2; // KMS key ID (optional for aws:kms)
+    bool bucket_key_enabled = 3; // S3 Bucket Keys optimization
+}

weed/pb/s3_lifecycle.proto

@@ -0,0 +1,340 @@
+syntax = "proto3";
+
+package s3_lifecycle_pb;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb";
+option java_package = "seaweedfs.client";
+option java_outer_classname = "S3LifecycleProto";
+
+// Storage layout reference (see S3_LIFECYCLE_REDESIGN.md):
+//
+//   /etc/s3/lifecycle/<bucket>/<rule_hash_hex>/<action_kind>/state    -> LifecycleState
+//   /etc/s3/lifecycle/<bucket>/<rule_hash_hex>/<action_kind>/pending  -> repeated PendingItem
+//   /etc/s3/lifecycle/<bucket>/_bootstrap                             -> BootstrapState
+//   /etc/s3/lifecycle/_reader/reader_state                            -> ReaderState
+//   /etc/s3/lifecycle/_reader/blockers                                -> repeated BlockerRecord
+//   /etc/s3/lifecycle/_reader/retry_budget                            -> repeated RetryBudgetEntry
+//
+// State is per-(rule_hash, action_kind), not per-rule: a single XML <Rule>
+// with N action sub-elements expands into N compiled actions, each with
+// its own state file and its own pending file. Sibling actions of the
+// same rule can degrade or activate independently.
+
+// ActionKind classifies the lifecycle action a single compiled entry
+// represents. A single XML rule may produce multiple compiled entries —
+// one per populated action.
+enum ActionKind {
+  ACTION_KIND_UNSPECIFIED = 0;
+  EXPIRATION_DAYS         = 1; // Expiration.Days
+  EXPIRATION_DATE         = 2; // Expiration.Date
+  NONCURRENT_DAYS         = 3; // NoncurrentVersionExpiration.NoncurrentDays (with optional NewerNoncurrent retention)
+  NEWER_NONCURRENT        = 4; // NoncurrentVersionExpiration.NewerNoncurrentVersions, count-only (no NoncurrentDays)
+  ABORT_MPU               = 5; // AbortIncompleteMultipartUpload.DaysAfterInitiation
+  EXPIRED_DELETE_MARKER   = 6; // Expiration.ExpiredObjectDeleteMarker
+}
+
+// SeaweedS3LifecycleInternal is the worker-to-S3 service that performs the
+// actual lifecycle deletion. The lifecycle worker computes the (rule, action)
+// verdict locally; the S3 server is the only component allowed to mutate the
+// filer state, so it gets the final word: it re-fetches the live entry,
+// verifies the EntryIdentity CAS, runs object-lock protections, and dispatches
+// to the appropriate internal helper.
+service SeaweedS3LifecycleInternal {
+  rpc LifecycleDelete(LifecycleDeleteRequest) returns (LifecycleDeleteResponse);
+}
+
+// LifecycleDeleteOutcome captures every per-event verdict the worker needs.
+// Cursor advance / pending mutation rules:
+//
+//   DONE                 -> advance cursor / drop pending
+//   NOOP_RESOLVED        -> advance cursor / drop pending (object already gone or stale identity)
+//   SKIPPED_OBJECT_LOCK  -> log + counter; advance cursor (per design: object lock is operator concern)
+//   RETRY_LATER          -> hold cursor; feed retry-budget; next batch retries from same position
+//   BLOCKED              -> hold cursor; durable BlockerRecord written; operator must intervene
+enum LifecycleDeleteOutcome {
+  LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED = 0;
+  DONE                                 = 1;
+  NOOP_RESOLVED                        = 2;
+  SKIPPED_OBJECT_LOCK                  = 3;
+  RETRY_LATER                          = 4;
+  BLOCKED                              = 5;
+}
+
+message LifecycleDeleteRequest {
+  // Routing.
+  string     bucket       = 1;
+  string     object_path  = 2; // bucket-relative; no leading slash
+  string     version_id   = 3; // empty for non-versioned
+  bytes      rule_hash    = 4; // 8 bytes; matches the rule's per-rule dir
+  ActionKind action_kind  = 5; // chooses dispatch within the rule
+
+  // Stream context (echoed in BlockerRecord on FATAL outcomes so operators
+  // can resolve the right (shard, delay) tuple).
+  string  stream_shard         = 10;
+  int64   stream_delay_seconds = 11;
+  int64   stream_position_ts_ns = 12;
+  int64   stream_position_offset = 13;
+
+  // CAS witness. The server re-fetches the live entry and aborts as
+  // NOOP_RESOLVED (STALE_IDENTITY) if any field doesn't match.
+  EntryIdentity expected_identity = 20;
+
+  // Snapshot id at the time the worker computed this verdict; the server
+  // verifies the (rule_hash, action_kind) is still in the current policy
+  // snapshot and aborts as NOOP_RESOLVED (STALE_POLICY) if not.
+  uint64 engine_snapshot_id = 21;
+}
+
+message LifecycleDeleteResponse {
+  LifecycleDeleteOutcome outcome = 1;
+  // Human-readable cause, e.g. "STALE_IDENTITY: mtime drift",
+  // "FATAL_EVENT_ERROR: malformed entry", or "TRANSPORT_ERROR: filer
+  // unavailable". Echoed into BlockerRecord.last_error on FATAL.
+  string reason = 2;
+}
+
+// MessagePosition mirrors weed/util/log_buffer.MessagePosition for durable
+// storage. ts_ns + offset uniquely identifies a position within a per-filer
+// log buffer; per-shard cursors (see ReaderState) use this shape to skip past
+// already-processed events when replaying.
+message MessagePosition {
+  int64 ts_ns  = 1;
+  int64 offset = 2;
+}
+
+// Cursor map keyed by filer_id. Each delay group's cursor (ReaderState) and
+// the predicate cursor (ReaderState) are per-filer-shard maps because
+// LogEntry.Offset is per-filer and not globally unique.
+message FilerShardCursor {
+  string          filer_id = 1;
+  MessagePosition position = 2;
+}
+
+// EntryIdentity is the CAS witness used by LifecycleDelete to detect that
+// the live entry hasn't changed between worker decision and server execution.
+// Mismatch -> STALE_IDENTITY -> NOOP_RESOLVED.
+message EntryIdentity {
+  int64  mtime_ns         = 1;
+  int64  size             = 2;
+  string head_fid         = 3;
+  bytes  extended_hash    = 4; // sha256 over sorted Extended map; cheap and stable
+}
+
+// LifecycleState captures the durable per-action scheduling state. Keyed by
+// (rule_hash, action_kind); persisted at
+//   /etc/s3/lifecycle/<bucket>/<rule_hash_hex>/<action_kind>/state
+//
+// A single XML <Rule> with N populated action sub-elements expands into N
+// LifecycleState records, one per sibling action.
+message LifecycleState {
+  // Proto3 best practice: zero value is an UNSPECIFIED sentinel so that
+  // legacy / partially-populated payloads don't silently default to a
+  // semantically active value. Persisted state files always set explicit
+  // non-zero values; reading UNSPECIFIED indicates corruption or a missing
+  // field.
+  enum RuleMode {
+    RULE_MODE_UNSPECIFIED = 0;
+    EVENT_DRIVEN          = 1;
+    SCAN_AT_DATE          = 2;
+    SCAN_ONLY             = 3;
+    DISABLED              = 4;
+    PENDING_BOOTSTRAP     = 5;
+  }
+  // DEGRADED_REASON_UNSPECIFIED replaces the old NONE sentinel: an unset
+  // / zero degraded_reason means "no active degradation" — operators
+  // treat UNSPECIFIED as healthy.
+  enum DegradedReason {
+    DEGRADED_REASON_UNSPECIFIED = 0;
+    LAG_HIGH                    = 1;
+    PENDING_FULL                = 2;
+    DELETE_FAILURES             = 3;
+    OPERATOR_PAUSED             = 4;
+    RETENTION_BELOW_HORIZON     = 5;
+    LOST_LOG                    = 6;
+  }
+
+  bytes      rule_hash                = 1;  // 8 bytes; matches the parent rule dir
+  ActionKind action_kind              = 2;  // matches the leaf dir name
+  string     rule_id                  = 3;  // display only; identical across sibling actions
+  RuleMode   mode                     = 4;
+  DegradedReason degraded_reason      = 5;
+  int64      degraded_since_ns        = 6;
+  bool       bootstrap_complete       = 7;
+  int64      bootstrap_started_at_ns  = 8;
+  int64      bootstrap_completed_at_ns = 9;
+
+  // Safety-scan scheduling. Set by the safety-scan job when a SCAN_ONLY
+  // (or periodic safety-scan) bootstrap pass completes; consulted by the
+  // detector to decide whether to emit the next bootstrap task.
+  int64 last_safety_scan_ts_ns = 10;
+  int64 next_safety_scan_ts_ns = 11;
+
+  // Counters for observability; no behavior depends on these.
+  // - evaluated_total: live entries the worker considered for this action
+  //   (filter-matched + due-checked); includes objects that ended up not
+  //   yet eligible.
+  // - expired_total: successful DONE outcomes (object/version/marker/MPU
+  //   removed) under this action.
+  // - metadata_only_total: subset of expired_total where the worker
+  //   short-circuited via volume-TTL routing (chunks left to volume GC).
+  // - error_total: outcomes that did NOT advance the cursor under this
+  //   action (RETRY_LATER, BLOCKED, FATAL_EVENT_ERROR).
+  int64 evaluated_total      = 12;
+  int64 expired_total        = 13;
+  int64 metadata_only_total  = 14;
+  int64 error_total          = 15;
+}
+
+// PendingItem records a not-yet-due eligibility for late-predicate exceptions
+// (e.g. tag added at age 30d on a 60d rule). Drained by s3.lifecycle.drain.
+// One use only — not the steady-state path.
+message PendingItem {
+  string         object_path       = 1;
+  string         version_id        = 2;
+  int64          due_at_ns         = 3;
+  EntryIdentity  expected_identity = 4;
+}
+
+// BootstrapState tracks the bucket walker's resume point. Operator-resolution
+// of a BOOTSTRAP-stream blocker does NOT mutate this field — the next
+// scheduled bootstrap task picks up from last_scanned_path and re-walks.
+message BootstrapState {
+  string last_scanned_path = 1;
+  int64  started_at_ns     = 2;
+  int64  completed_at_ns   = 3; // zero while in progress
+}
+
+// ReaderState is the cluster-singleton reader's durable cursors. One file at
+// /etc/s3/lifecycle/_reader/reader_state, written by the singleton task.
+message ReaderState {
+  string primary_filer_endpoint = 1;
+
+  // Per-delay-group, per-filer-shard cursors. Outer key is delay_seconds.
+  // Inner FilerShardCursor list maps filer_id -> MessagePosition.
+  map<int64, FilerShardCursorList> last_processed_original = 2;
+
+  // Predicate-change pass cursor, keyed by filer_id.
+  repeated FilerShardCursor last_processed_predicate = 3;
+
+  // Filer-shard keys whose cursor reached range.latest and were safely GC'd.
+  // Used by the lost-log gate at task entry to distinguish "F was tail-drained
+  // safely then pruned" from "F's logs vanished before catch-up". Encoded as
+  // stream-specific keys (same shape as RetryBudgetEntry.StreamKey).
+  repeated TailDrainedKey tail_drained_streams = 4;
+
+  int64 last_checkpoint_ns = 5;
+}
+
+message FilerShardCursorList {
+  repeated FilerShardCursor cursors = 1;
+}
+
+// TailDrainedKey identifies a (stream_kind, shard, [delay]) tuple whose
+// cursor previously caught up to range.latest. Persisted in ReaderState so
+// the lost-log gate can distinguish safe drain from data loss.
+message TailDrainedKey {
+  StreamKind stream_kind   = 1;
+  string     filer_id      = 2;
+  int64      delay_seconds = 3; // populated when stream_kind == ORIGINAL
+}
+
+// StreamKind classifies the four blockable streams. ORIGINAL/PREDICATE pause
+// reader cursors; BOOTSTRAP pauses a bucket walker; PENDING pauses a rule's
+// drain task. Zero is an UNSPECIFIED sentinel — a BlockerRecord whose
+// stream_kind reads as UNSPECIFIED is a corruption signal, not a default.
+enum StreamKind {
+  STREAM_KIND_UNSPECIFIED = 0;
+  ORIGINAL                = 1;
+  PREDICATE               = 2;
+  BOOTSTRAP               = 3;
+  PENDING                 = 4;
+}
+
+// BlockerRecord is the durable record of a paused stream. The cursor stays
+// at the failing position until the operator clears it via
+// `s3.lifecycle.blockers retry|resume|quarantine`. There is no automatic
+// dead-letter — silently routing a failed delete decision aside is unsafe
+// for lifecycle.
+message BlockerRecord {
+  StreamKind stream_kind = 1;
+
+  // Stream-specific scoping; populated per stream_kind.
+  string          shard         = 2; // filer_id; ORIGINAL/PREDICATE only
+  int64           delay_seconds = 3; // ORIGINAL only
+  MessagePosition position      = 4; // ORIGINAL/PREDICATE only
+
+  // Common context. bucket/object_path/version_id always populated.
+  // rule_hash and action_kind are OPTIONAL: empty / UNSPECIFIED for
+  // pre-evaluation failures (e.g. handleEvent fetchLive FATAL where no
+  // action has been evaluated). When populated, both are set together —
+  // the failure is bound to a specific (rule, action) pair.
+  bytes      rule_hash    = 5;
+  string     bucket       = 6;
+  string     object_path  = 7;
+  string     version_id   = 8;
+  ActionKind action_kind  = 14; // optional; UNSPECIFIED for pre-evaluation failures
+
+  string reason          = 9;  // "FATAL_EVENT_ERROR: malformed entry"
+  string last_error      = 10; // raw error string from the last attempt
+  int64  first_seen_at_ns = 11;
+  int64  last_retry_at_ns = 12;
+  int32  retry_count      = 13;
+}
+
+// StreamKey is the shape used by retry-budget tracking. One of four shapes
+// per stream_kind, since the identifying tuple differs.
+message StreamKey {
+  oneof key {
+    OriginalKey  original  = 1;
+    PredicateKey predicate = 2;
+    BootstrapKey bootstrap = 3;
+    PendingKey   pending   = 4;
+  }
+}
+
+message OriginalKey {
+  string          shard         = 1;
+  int64           delay_seconds = 2;
+  MessagePosition position      = 3;
+}
+message PredicateKey {
+  string          shard    = 1;
+  MessagePosition position = 2;
+}
+message BootstrapKey {
+  string     bucket      = 1;
+  string     object_path = 2;
+  string     version_id  = 3;
+  bytes      rule_hash   = 4;
+  ActionKind action_kind = 5; // distinguishes per-action streams under one rule
+}
+message PendingKey {
+  string     bucket      = 1;
+  bytes      rule_hash   = 2;
+  string     object_path = 3;
+  string     version_id  = 4;
+  ActionKind action_kind = 5;
+}
+
+// RetryBudgetEntry tracks consecutive RETRY_LATER outcomes for a stream key.
+// Promotes to BLOCKED when consecutive_retries >= retryBudgetMax (30) or age
+// >= retryBudgetMaxAge (4h). Compacts on success (clearRetryBudget).
+message RetryBudgetEntry {
+  StreamKey key                 = 1;
+  int32     consecutive_retries = 2;
+  int64     first_seen_at_ns    = 3;
+  int64     last_retry_at_ns    = 4;
+}
+
+// RetryTarget carries routing info needed to (a) write a BlockerRecord on
+// promotion and (b) re-invoke the right primitive on operator retry. We
+// deliberately do NOT carry expected_identity or action: the handler
+// re-fetches live state and re-evaluates from scratch.
+message RetryTarget {
+  StreamKey  key         = 1;
+  string     bucket      = 2;
+  string     object_path = 3;
+  string     version_id  = 4;
+  bytes      rule_hash   = 5;
+  ActionKind action_kind = 6;
+}

weed/pb/s3_lifecycle_pb/s3_lifecycle_grpc.pb.go

@@ -0,0 +1,136 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.5.1
+// - protoc             v6.33.4
+// source: s3_lifecycle.proto
+
+package s3_lifecycle_pb
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.64.0 or later.
+const _ = grpc.SupportPackageIsVersion9
+
+const (
+	SeaweedS3LifecycleInternal_LifecycleDelete_FullMethodName = "/s3_lifecycle_pb.SeaweedS3LifecycleInternal/LifecycleDelete"
+)
+
+// SeaweedS3LifecycleInternalClient is the client API for SeaweedS3LifecycleInternal service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+//
+// SeaweedS3LifecycleInternal is the worker-to-S3 service that performs the
+// actual lifecycle deletion. The lifecycle worker computes the (rule, action)
+// verdict locally; the S3 server is the only component allowed to mutate the
+// filer state, so it gets the final word: it re-fetches the live entry,
+// verifies the EntryIdentity CAS, runs object-lock protections, and dispatches
+// to the appropriate internal helper.
+type SeaweedS3LifecycleInternalClient interface {
+	LifecycleDelete(ctx context.Context, in *LifecycleDeleteRequest, opts ...grpc.CallOption) (*LifecycleDeleteResponse, error)
+}
+
+type seaweedS3LifecycleInternalClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewSeaweedS3LifecycleInternalClient(cc grpc.ClientConnInterface) SeaweedS3LifecycleInternalClient {
+	return &seaweedS3LifecycleInternalClient{cc}
+}
+
+func (c *seaweedS3LifecycleInternalClient) LifecycleDelete(ctx context.Context, in *LifecycleDeleteRequest, opts ...grpc.CallOption) (*LifecycleDeleteResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(LifecycleDeleteResponse)
+	err := c.cc.Invoke(ctx, SeaweedS3LifecycleInternal_LifecycleDelete_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// SeaweedS3LifecycleInternalServer is the server API for SeaweedS3LifecycleInternal service.
+// All implementations must embed UnimplementedSeaweedS3LifecycleInternalServer
+// for forward compatibility.
+//
+// SeaweedS3LifecycleInternal is the worker-to-S3 service that performs the
+// actual lifecycle deletion. The lifecycle worker computes the (rule, action)
+// verdict locally; the S3 server is the only component allowed to mutate the
+// filer state, so it gets the final word: it re-fetches the live entry,
+// verifies the EntryIdentity CAS, runs object-lock protections, and dispatches
+// to the appropriate internal helper.
+type SeaweedS3LifecycleInternalServer interface {
+	LifecycleDelete(context.Context, *LifecycleDeleteRequest) (*LifecycleDeleteResponse, error)
+	mustEmbedUnimplementedSeaweedS3LifecycleInternalServer()
+}
+
+// UnimplementedSeaweedS3LifecycleInternalServer must be embedded to have
+// forward compatible implementations.
+//
+// NOTE: this should be embedded by value instead of pointer to avoid a nil
+// pointer dereference when methods are called.
+type UnimplementedSeaweedS3LifecycleInternalServer struct{}
+
+func (UnimplementedSeaweedS3LifecycleInternalServer) LifecycleDelete(context.Context, *LifecycleDeleteRequest) (*LifecycleDeleteResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method LifecycleDelete not implemented")
+}
+func (UnimplementedSeaweedS3LifecycleInternalServer) mustEmbedUnimplementedSeaweedS3LifecycleInternalServer() {
+}
+func (UnimplementedSeaweedS3LifecycleInternalServer) testEmbeddedByValue() {}
+
+// UnsafeSeaweedS3LifecycleInternalServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to SeaweedS3LifecycleInternalServer will
+// result in compilation errors.
+type UnsafeSeaweedS3LifecycleInternalServer interface {
+	mustEmbedUnimplementedSeaweedS3LifecycleInternalServer()
+}
+
+func RegisterSeaweedS3LifecycleInternalServer(s grpc.ServiceRegistrar, srv SeaweedS3LifecycleInternalServer) {
+	// If the following call pancis, it indicates UnimplementedSeaweedS3LifecycleInternalServer was
+	// embedded by pointer and is nil.  This will cause panics if an
+	// unimplemented method is ever invoked, so we test this at initialization
+	// time to prevent it from happening at runtime later due to I/O.
+	if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
+		t.testEmbeddedByValue()
+	}
+	s.RegisterService(&SeaweedS3LifecycleInternal_ServiceDesc, srv)
+}
+
+func _SeaweedS3LifecycleInternal_LifecycleDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(LifecycleDeleteRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedS3LifecycleInternalServer).LifecycleDelete(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedS3LifecycleInternal_LifecycleDelete_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedS3LifecycleInternalServer).LifecycleDelete(ctx, req.(*LifecycleDeleteRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// SeaweedS3LifecycleInternal_ServiceDesc is the grpc.ServiceDesc for SeaweedS3LifecycleInternal service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var SeaweedS3LifecycleInternal_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "s3_lifecycle_pb.SeaweedS3LifecycleInternal",
+	HandlerType: (*SeaweedS3LifecycleInternalServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "LifecycleDelete",
+			Handler:    _SeaweedS3LifecycleInternal_LifecycleDelete_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "s3_lifecycle.proto",
+}

weed/pb/s3_pb/s3.pb.go

@@ -0,0 +1,523 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.6
+// 	protoc        v6.33.4
+// source: s3.proto
+
+package s3_pb
+
+import (
+	iam_pb "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+	unsafe "unsafe"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type S3CircuitBreakerConfig struct {
+	state         protoimpl.MessageState              `protogen:"open.v1"`
+	Global        *S3CircuitBreakerOptions            `protobuf:"bytes,1,opt,name=global,proto3" json:"global,omitempty"`
+	Buckets       map[string]*S3CircuitBreakerOptions `protobuf:"bytes,2,rep,name=buckets,proto3" json:"buckets,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *S3CircuitBreakerConfig) Reset() {
+	*x = S3CircuitBreakerConfig{}
+	mi := &file_s3_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *S3CircuitBreakerConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*S3CircuitBreakerConfig) ProtoMessage() {}
+
+func (x *S3CircuitBreakerConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_s3_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use S3CircuitBreakerConfig.ProtoReflect.Descriptor instead.
+func (*S3CircuitBreakerConfig) Descriptor() ([]byte, []int) {
+	return file_s3_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *S3CircuitBreakerConfig) GetGlobal() *S3CircuitBreakerOptions {
+	if x != nil {
+		return x.Global
+	}
+	return nil
+}
+
+func (x *S3CircuitBreakerConfig) GetBuckets() map[string]*S3CircuitBreakerOptions {
+	if x != nil {
+		return x.Buckets
+	}
+	return nil
+}
+
+type S3CircuitBreakerOptions struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Enabled       bool                   `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"`
+	Actions       map[string]int64       `protobuf:"bytes,2,rep,name=actions,proto3" json:"actions,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *S3CircuitBreakerOptions) Reset() {
+	*x = S3CircuitBreakerOptions{}
+	mi := &file_s3_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *S3CircuitBreakerOptions) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*S3CircuitBreakerOptions) ProtoMessage() {}
+
+func (x *S3CircuitBreakerOptions) ProtoReflect() protoreflect.Message {
+	mi := &file_s3_proto_msgTypes[1]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use S3CircuitBreakerOptions.ProtoReflect.Descriptor instead.
+func (*S3CircuitBreakerOptions) Descriptor() ([]byte, []int) {
+	return file_s3_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *S3CircuitBreakerOptions) GetEnabled() bool {
+	if x != nil {
+		return x.Enabled
+	}
+	return false
+}
+
+func (x *S3CircuitBreakerOptions) GetActions() map[string]int64 {
+	if x != nil {
+		return x.Actions
+	}
+	return nil
+}
+
+type CORSRule struct {
+	state          protoimpl.MessageState `protogen:"open.v1"`
+	AllowedHeaders []string               `protobuf:"bytes,1,rep,name=allowed_headers,json=allowedHeaders,proto3" json:"allowed_headers,omitempty"`
+	AllowedMethods []string               `protobuf:"bytes,2,rep,name=allowed_methods,json=allowedMethods,proto3" json:"allowed_methods,omitempty"`
+	AllowedOrigins []string               `protobuf:"bytes,3,rep,name=allowed_origins,json=allowedOrigins,proto3" json:"allowed_origins,omitempty"`
+	ExposeHeaders  []string               `protobuf:"bytes,4,rep,name=expose_headers,json=exposeHeaders,proto3" json:"expose_headers,omitempty"`
+	MaxAgeSeconds  int32                  `protobuf:"varint,5,opt,name=max_age_seconds,json=maxAgeSeconds,proto3" json:"max_age_seconds,omitempty"`
+	Id             string                 `protobuf:"bytes,6,opt,name=id,proto3" json:"id,omitempty"`
+	unknownFields  protoimpl.UnknownFields
+	sizeCache      protoimpl.SizeCache
+}
+
+func (x *CORSRule) Reset() {
+	*x = CORSRule{}
+	mi := &file_s3_proto_msgTypes[2]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *CORSRule) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CORSRule) ProtoMessage() {}
+
+func (x *CORSRule) ProtoReflect() protoreflect.Message {
+	mi := &file_s3_proto_msgTypes[2]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use CORSRule.ProtoReflect.Descriptor instead.
+func (*CORSRule) Descriptor() ([]byte, []int) {
+	return file_s3_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *CORSRule) GetAllowedHeaders() []string {
+	if x != nil {
+		return x.AllowedHeaders
+	}
+	return nil
+}
+
+func (x *CORSRule) GetAllowedMethods() []string {
+	if x != nil {
+		return x.AllowedMethods
+	}
+	return nil
+}
+
+func (x *CORSRule) GetAllowedOrigins() []string {
+	if x != nil {
+		return x.AllowedOrigins
+	}
+	return nil
+}
+
+func (x *CORSRule) GetExposeHeaders() []string {
+	if x != nil {
+		return x.ExposeHeaders
+	}
+	return nil
+}
+
+func (x *CORSRule) GetMaxAgeSeconds() int32 {
+	if x != nil {
+		return x.MaxAgeSeconds
+	}
+	return 0
+}
+
+func (x *CORSRule) GetId() string {
+	if x != nil {
+		return x.Id
+	}
+	return ""
+}
+
+type CORSConfiguration struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	CorsRules     []*CORSRule            `protobuf:"bytes,1,rep,name=cors_rules,json=corsRules,proto3" json:"cors_rules,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *CORSConfiguration) Reset() {
+	*x = CORSConfiguration{}
+	mi := &file_s3_proto_msgTypes[3]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *CORSConfiguration) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CORSConfiguration) ProtoMessage() {}
+
+func (x *CORSConfiguration) ProtoReflect() protoreflect.Message {
+	mi := &file_s3_proto_msgTypes[3]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use CORSConfiguration.ProtoReflect.Descriptor instead.
+func (*CORSConfiguration) Descriptor() ([]byte, []int) {
+	return file_s3_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *CORSConfiguration) GetCorsRules() []*CORSRule {
+	if x != nil {
+		return x.CorsRules
+	}
+	return nil
+}
+
+type BucketMetadata struct {
+	state         protoimpl.MessageState   `protogen:"open.v1"`
+	Tags          map[string]string        `protobuf:"bytes,1,rep,name=tags,proto3" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	Cors          *CORSConfiguration       `protobuf:"bytes,2,opt,name=cors,proto3" json:"cors,omitempty"`
+	Encryption    *EncryptionConfiguration `protobuf:"bytes,3,opt,name=encryption,proto3" json:"encryption,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *BucketMetadata) Reset() {
+	*x = BucketMetadata{}
+	mi := &file_s3_proto_msgTypes[4]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *BucketMetadata) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*BucketMetadata) ProtoMessage() {}
+
+func (x *BucketMetadata) ProtoReflect() protoreflect.Message {
+	mi := &file_s3_proto_msgTypes[4]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use BucketMetadata.ProtoReflect.Descriptor instead.
+func (*BucketMetadata) Descriptor() ([]byte, []int) {
+	return file_s3_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *BucketMetadata) GetTags() map[string]string {
+	if x != nil {
+		return x.Tags
+	}
+	return nil
+}
+
+func (x *BucketMetadata) GetCors() *CORSConfiguration {
+	if x != nil {
+		return x.Cors
+	}
+	return nil
+}
+
+func (x *BucketMetadata) GetEncryption() *EncryptionConfiguration {
+	if x != nil {
+		return x.Encryption
+	}
+	return nil
+}
+
+type EncryptionConfiguration struct {
+	state            protoimpl.MessageState `protogen:"open.v1"`
+	SseAlgorithm     string                 `protobuf:"bytes,1,opt,name=sse_algorithm,json=sseAlgorithm,proto3" json:"sse_algorithm,omitempty"`                // "AES256" or "aws:kms"
+	KmsKeyId         string                 `protobuf:"bytes,2,opt,name=kms_key_id,json=kmsKeyId,proto3" json:"kms_key_id,omitempty"`                          // KMS key ID (optional for aws:kms)
+	BucketKeyEnabled bool                   `protobuf:"varint,3,opt,name=bucket_key_enabled,json=bucketKeyEnabled,proto3" json:"bucket_key_enabled,omitempty"` // S3 Bucket Keys optimization
+	unknownFields    protoimpl.UnknownFields
+	sizeCache        protoimpl.SizeCache
+}
+
+func (x *EncryptionConfiguration) Reset() {
+	*x = EncryptionConfiguration{}
+	mi := &file_s3_proto_msgTypes[5]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *EncryptionConfiguration) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*EncryptionConfiguration) ProtoMessage() {}
+
+func (x *EncryptionConfiguration) ProtoReflect() protoreflect.Message {
+	mi := &file_s3_proto_msgTypes[5]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use EncryptionConfiguration.ProtoReflect.Descriptor instead.
+func (*EncryptionConfiguration) Descriptor() ([]byte, []int) {
+	return file_s3_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *EncryptionConfiguration) GetSseAlgorithm() string {
+	if x != nil {
+		return x.SseAlgorithm
+	}
+	return ""
+}
+
+func (x *EncryptionConfiguration) GetKmsKeyId() string {
+	if x != nil {
+		return x.KmsKeyId
+	}
+	return ""
+}
+
+func (x *EncryptionConfiguration) GetBucketKeyEnabled() bool {
+	if x != nil {
+		return x.BucketKeyEnabled
+	}
+	return false
+}
+
+var File_s3_proto protoreflect.FileDescriptor
+
+const file_s3_proto_rawDesc = "" +
+	"\n" +
+	"\bs3.proto\x12\fmessaging_pb\x1a\tiam.proto\"\x87\x02\n" +
+	"\x16S3CircuitBreakerConfig\x12=\n" +
+	"\x06global\x18\x01 \x01(\v2%.messaging_pb.S3CircuitBreakerOptionsR\x06global\x12K\n" +
+	"\abuckets\x18\x02 \x03(\v21.messaging_pb.S3CircuitBreakerConfig.BucketsEntryR\abuckets\x1aa\n" +
+	"\fBucketsEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12;\n" +
+	"\x05value\x18\x02 \x01(\v2%.messaging_pb.S3CircuitBreakerOptionsR\x05value:\x028\x01\"\xbd\x01\n" +
+	"\x17S3CircuitBreakerOptions\x12\x18\n" +
+	"\aenabled\x18\x01 \x01(\bR\aenabled\x12L\n" +
+	"\aactions\x18\x02 \x03(\v22.messaging_pb.S3CircuitBreakerOptions.ActionsEntryR\aactions\x1a:\n" +
+	"\fActionsEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+	"\x05value\x18\x02 \x01(\x03R\x05value:\x028\x01\"\xe4\x01\n" +
+	"\bCORSRule\x12'\n" +
+	"\x0fallowed_headers\x18\x01 \x03(\tR\x0eallowedHeaders\x12'\n" +
+	"\x0fallowed_methods\x18\x02 \x03(\tR\x0eallowedMethods\x12'\n" +
+	"\x0fallowed_origins\x18\x03 \x03(\tR\x0eallowedOrigins\x12%\n" +
+	"\x0eexpose_headers\x18\x04 \x03(\tR\rexposeHeaders\x12&\n" +
+	"\x0fmax_age_seconds\x18\x05 \x01(\x05R\rmaxAgeSeconds\x12\x0e\n" +
+	"\x02id\x18\x06 \x01(\tR\x02id\"J\n" +
+	"\x11CORSConfiguration\x125\n" +
+	"\n" +
+	"cors_rules\x18\x01 \x03(\v2\x16.messaging_pb.CORSRuleR\tcorsRules\"\x81\x02\n" +
+	"\x0eBucketMetadata\x12:\n" +
+	"\x04tags\x18\x01 \x03(\v2&.messaging_pb.BucketMetadata.TagsEntryR\x04tags\x123\n" +
+	"\x04cors\x18\x02 \x01(\v2\x1f.messaging_pb.CORSConfigurationR\x04cors\x12E\n" +
+	"\n" +
+	"encryption\x18\x03 \x01(\v2%.messaging_pb.EncryptionConfigurationR\n" +
+	"encryption\x1a7\n" +
+	"\tTagsEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x8a\x01\n" +
+	"\x17EncryptionConfiguration\x12#\n" +
+	"\rsse_algorithm\x18\x01 \x01(\tR\fsseAlgorithm\x12\x1c\n" +
+	"\n" +
+	"kms_key_id\x18\x02 \x01(\tR\bkmsKeyId\x12,\n" +
+	"\x12bucket_key_enabled\x18\x03 \x01(\bR\x10bucketKeyEnabled2\xcd\x04\n" +
+	"\x11SeaweedS3IamCache\x12F\n" +
+	"\vPutIdentity\x12\x1a.iam_pb.PutIdentityRequest\x1a\x1b.iam_pb.PutIdentityResponse\x12O\n" +
+	"\x0eRemoveIdentity\x12\x1d.iam_pb.RemoveIdentityRequest\x1a\x1e.iam_pb.RemoveIdentityResponse\x12@\n" +
+	"\tPutPolicy\x12\x18.iam_pb.PutPolicyRequest\x1a\x19.iam_pb.PutPolicyResponse\x12@\n" +
+	"\tGetPolicy\x12\x18.iam_pb.GetPolicyRequest\x1a\x19.iam_pb.GetPolicyResponse\x12I\n" +
+	"\fListPolicies\x12\x1b.iam_pb.ListPoliciesRequest\x1a\x1c.iam_pb.ListPoliciesResponse\x12I\n" +
+	"\fDeletePolicy\x12\x1b.iam_pb.DeletePolicyRequest\x1a\x1c.iam_pb.DeletePolicyResponse\x12=\n" +
+	"\bPutGroup\x12\x17.iam_pb.PutGroupRequest\x1a\x18.iam_pb.PutGroupResponse\x12F\n" +
+	"\vRemoveGroup\x12\x1a.iam_pb.RemoveGroupRequest\x1a\x1b.iam_pb.RemoveGroupResponseBI\n" +
+	"\x10seaweedfs.clientB\aS3ProtoZ,github.com/seaweedfs/seaweedfs/weed/pb/s3_pbb\x06proto3"
+
+var (
+	file_s3_proto_rawDescOnce sync.Once
+	file_s3_proto_rawDescData []byte
+)
+
+func file_s3_proto_rawDescGZIP() []byte {
+	file_s3_proto_rawDescOnce.Do(func() {
+		file_s3_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_s3_proto_rawDesc), len(file_s3_proto_rawDesc)))
+	})
+	return file_s3_proto_rawDescData
+}
+
+var file_s3_proto_msgTypes = make([]protoimpl.MessageInfo, 9)
+var file_s3_proto_goTypes = []any{
+	(*S3CircuitBreakerConfig)(nil),        // 0: messaging_pb.S3CircuitBreakerConfig
+	(*S3CircuitBreakerOptions)(nil),       // 1: messaging_pb.S3CircuitBreakerOptions
+	(*CORSRule)(nil),                      // 2: messaging_pb.CORSRule
+	(*CORSConfiguration)(nil),             // 3: messaging_pb.CORSConfiguration
+	(*BucketMetadata)(nil),                // 4: messaging_pb.BucketMetadata
+	(*EncryptionConfiguration)(nil),       // 5: messaging_pb.EncryptionConfiguration
+	nil,                                   // 6: messaging_pb.S3CircuitBreakerConfig.BucketsEntry
+	nil,                                   // 7: messaging_pb.S3CircuitBreakerOptions.ActionsEntry
+	nil,                                   // 8: messaging_pb.BucketMetadata.TagsEntry
+	(*iam_pb.PutIdentityRequest)(nil),     // 9: iam_pb.PutIdentityRequest
+	(*iam_pb.RemoveIdentityRequest)(nil),  // 10: iam_pb.RemoveIdentityRequest
+	(*iam_pb.PutPolicyRequest)(nil),       // 11: iam_pb.PutPolicyRequest
+	(*iam_pb.GetPolicyRequest)(nil),       // 12: iam_pb.GetPolicyRequest
+	(*iam_pb.ListPoliciesRequest)(nil),    // 13: iam_pb.ListPoliciesRequest
+	(*iam_pb.DeletePolicyRequest)(nil),    // 14: iam_pb.DeletePolicyRequest
+	(*iam_pb.PutGroupRequest)(nil),        // 15: iam_pb.PutGroupRequest
+	(*iam_pb.RemoveGroupRequest)(nil),     // 16: iam_pb.RemoveGroupRequest
+	(*iam_pb.PutIdentityResponse)(nil),    // 17: iam_pb.PutIdentityResponse
+	(*iam_pb.RemoveIdentityResponse)(nil), // 18: iam_pb.RemoveIdentityResponse
+	(*iam_pb.PutPolicyResponse)(nil),      // 19: iam_pb.PutPolicyResponse
+	(*iam_pb.GetPolicyResponse)(nil),      // 20: iam_pb.GetPolicyResponse
+	(*iam_pb.ListPoliciesResponse)(nil),   // 21: iam_pb.ListPoliciesResponse
+	(*iam_pb.DeletePolicyResponse)(nil),   // 22: iam_pb.DeletePolicyResponse
+	(*iam_pb.PutGroupResponse)(nil),       // 23: iam_pb.PutGroupResponse
+	(*iam_pb.RemoveGroupResponse)(nil),    // 24: iam_pb.RemoveGroupResponse
+}
+var file_s3_proto_depIdxs = []int32{
+	1,  // 0: messaging_pb.S3CircuitBreakerConfig.global:type_name -> messaging_pb.S3CircuitBreakerOptions
+	6,  // 1: messaging_pb.S3CircuitBreakerConfig.buckets:type_name -> messaging_pb.S3CircuitBreakerConfig.BucketsEntry
+	7,  // 2: messaging_pb.S3CircuitBreakerOptions.actions:type_name -> messaging_pb.S3CircuitBreakerOptions.ActionsEntry
+	2,  // 3: messaging_pb.CORSConfiguration.cors_rules:type_name -> messaging_pb.CORSRule
+	8,  // 4: messaging_pb.BucketMetadata.tags:type_name -> messaging_pb.BucketMetadata.TagsEntry
+	3,  // 5: messaging_pb.BucketMetadata.cors:type_name -> messaging_pb.CORSConfiguration
+	5,  // 6: messaging_pb.BucketMetadata.encryption:type_name -> messaging_pb.EncryptionConfiguration
+	1,  // 7: messaging_pb.S3CircuitBreakerConfig.BucketsEntry.value:type_name -> messaging_pb.S3CircuitBreakerOptions
+	9,  // 8: messaging_pb.SeaweedS3IamCache.PutIdentity:input_type -> iam_pb.PutIdentityRequest
+	10, // 9: messaging_pb.SeaweedS3IamCache.RemoveIdentity:input_type -> iam_pb.RemoveIdentityRequest
+	11, // 10: messaging_pb.SeaweedS3IamCache.PutPolicy:input_type -> iam_pb.PutPolicyRequest
+	12, // 11: messaging_pb.SeaweedS3IamCache.GetPolicy:input_type -> iam_pb.GetPolicyRequest
+	13, // 12: messaging_pb.SeaweedS3IamCache.ListPolicies:input_type -> iam_pb.ListPoliciesRequest
+	14, // 13: messaging_pb.SeaweedS3IamCache.DeletePolicy:input_type -> iam_pb.DeletePolicyRequest
+	15, // 14: messaging_pb.SeaweedS3IamCache.PutGroup:input_type -> iam_pb.PutGroupRequest
+	16, // 15: messaging_pb.SeaweedS3IamCache.RemoveGroup:input_type -> iam_pb.RemoveGroupRequest
+	17, // 16: messaging_pb.SeaweedS3IamCache.PutIdentity:output_type -> iam_pb.PutIdentityResponse
+	18, // 17: messaging_pb.SeaweedS3IamCache.RemoveIdentity:output_type -> iam_pb.RemoveIdentityResponse
+	19, // 18: messaging_pb.SeaweedS3IamCache.PutPolicy:output_type -> iam_pb.PutPolicyResponse
+	20, // 19: messaging_pb.SeaweedS3IamCache.GetPolicy:output_type -> iam_pb.GetPolicyResponse
+	21, // 20: messaging_pb.SeaweedS3IamCache.ListPolicies:output_type -> iam_pb.ListPoliciesResponse
+	22, // 21: messaging_pb.SeaweedS3IamCache.DeletePolicy:output_type -> iam_pb.DeletePolicyResponse
+	23, // 22: messaging_pb.SeaweedS3IamCache.PutGroup:output_type -> iam_pb.PutGroupResponse
+	24, // 23: messaging_pb.SeaweedS3IamCache.RemoveGroup:output_type -> iam_pb.RemoveGroupResponse
+	16, // [16:24] is the sub-list for method output_type
+	8,  // [8:16] is the sub-list for method input_type
+	8,  // [8:8] is the sub-list for extension type_name
+	8,  // [8:8] is the sub-list for extension extendee
+	0,  // [0:8] is the sub-list for field type_name
+}
+
+func init() { file_s3_proto_init() }
+func file_s3_proto_init() {
+	if File_s3_proto != nil {
+		return
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_s3_proto_rawDesc), len(file_s3_proto_rawDesc)),
+			NumEnums:      0,
+			NumMessages:   9,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_s3_proto_goTypes,
+		DependencyIndexes: file_s3_proto_depIdxs,
+		MessageInfos:      file_s3_proto_msgTypes,
+	}.Build()
+	File_s3_proto = out.File
+	file_s3_proto_goTypes = nil
+	file_s3_proto_depIdxs = nil
+}

weed/pb/s3_pb/s3_grpc.pb.go

@@ -0,0 +1,392 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.5.1
+// - protoc             v6.33.4
+// source: s3.proto
+
+package s3_pb
+
+import (
+	context "context"
+	iam_pb "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.64.0 or later.
+const _ = grpc.SupportPackageIsVersion9
+
+const (
+	SeaweedS3IamCache_PutIdentity_FullMethodName    = "/messaging_pb.SeaweedS3IamCache/PutIdentity"
+	SeaweedS3IamCache_RemoveIdentity_FullMethodName = "/messaging_pb.SeaweedS3IamCache/RemoveIdentity"
+	SeaweedS3IamCache_PutPolicy_FullMethodName      = "/messaging_pb.SeaweedS3IamCache/PutPolicy"
+	SeaweedS3IamCache_GetPolicy_FullMethodName      = "/messaging_pb.SeaweedS3IamCache/GetPolicy"
+	SeaweedS3IamCache_ListPolicies_FullMethodName   = "/messaging_pb.SeaweedS3IamCache/ListPolicies"
+	SeaweedS3IamCache_DeletePolicy_FullMethodName   = "/messaging_pb.SeaweedS3IamCache/DeletePolicy"
+	SeaweedS3IamCache_PutGroup_FullMethodName       = "/messaging_pb.SeaweedS3IamCache/PutGroup"
+	SeaweedS3IamCache_RemoveGroup_FullMethodName    = "/messaging_pb.SeaweedS3IamCache/RemoveGroup"
+)
+
+// SeaweedS3IamCacheClient is the client API for SeaweedS3IamCache service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+//
+// Designed for unidirectional propagation from Filer to S3 Servers
+type SeaweedS3IamCacheClient interface {
+	PutIdentity(ctx context.Context, in *iam_pb.PutIdentityRequest, opts ...grpc.CallOption) (*iam_pb.PutIdentityResponse, error)
+	RemoveIdentity(ctx context.Context, in *iam_pb.RemoveIdentityRequest, opts ...grpc.CallOption) (*iam_pb.RemoveIdentityResponse, error)
+	PutPolicy(ctx context.Context, in *iam_pb.PutPolicyRequest, opts ...grpc.CallOption) (*iam_pb.PutPolicyResponse, error)
+	GetPolicy(ctx context.Context, in *iam_pb.GetPolicyRequest, opts ...grpc.CallOption) (*iam_pb.GetPolicyResponse, error)
+	ListPolicies(ctx context.Context, in *iam_pb.ListPoliciesRequest, opts ...grpc.CallOption) (*iam_pb.ListPoliciesResponse, error)
+	DeletePolicy(ctx context.Context, in *iam_pb.DeletePolicyRequest, opts ...grpc.CallOption) (*iam_pb.DeletePolicyResponse, error)
+	PutGroup(ctx context.Context, in *iam_pb.PutGroupRequest, opts ...grpc.CallOption) (*iam_pb.PutGroupResponse, error)
+	RemoveGroup(ctx context.Context, in *iam_pb.RemoveGroupRequest, opts ...grpc.CallOption) (*iam_pb.RemoveGroupResponse, error)
+}
+
+type seaweedS3IamCacheClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewSeaweedS3IamCacheClient(cc grpc.ClientConnInterface) SeaweedS3IamCacheClient {
+	return &seaweedS3IamCacheClient{cc}
+}
+
+func (c *seaweedS3IamCacheClient) PutIdentity(ctx context.Context, in *iam_pb.PutIdentityRequest, opts ...grpc.CallOption) (*iam_pb.PutIdentityResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(iam_pb.PutIdentityResponse)
+	err := c.cc.Invoke(ctx, SeaweedS3IamCache_PutIdentity_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedS3IamCacheClient) RemoveIdentity(ctx context.Context, in *iam_pb.RemoveIdentityRequest, opts ...grpc.CallOption) (*iam_pb.RemoveIdentityResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(iam_pb.RemoveIdentityResponse)
+	err := c.cc.Invoke(ctx, SeaweedS3IamCache_RemoveIdentity_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedS3IamCacheClient) PutPolicy(ctx context.Context, in *iam_pb.PutPolicyRequest, opts ...grpc.CallOption) (*iam_pb.PutPolicyResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(iam_pb.PutPolicyResponse)
+	err := c.cc.Invoke(ctx, SeaweedS3IamCache_PutPolicy_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedS3IamCacheClient) GetPolicy(ctx context.Context, in *iam_pb.GetPolicyRequest, opts ...grpc.CallOption) (*iam_pb.GetPolicyResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(iam_pb.GetPolicyResponse)
+	err := c.cc.Invoke(ctx, SeaweedS3IamCache_GetPolicy_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedS3IamCacheClient) ListPolicies(ctx context.Context, in *iam_pb.ListPoliciesRequest, opts ...grpc.CallOption) (*iam_pb.ListPoliciesResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(iam_pb.ListPoliciesResponse)
+	err := c.cc.Invoke(ctx, SeaweedS3IamCache_ListPolicies_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedS3IamCacheClient) DeletePolicy(ctx context.Context, in *iam_pb.DeletePolicyRequest, opts ...grpc.CallOption) (*iam_pb.DeletePolicyResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(iam_pb.DeletePolicyResponse)
+	err := c.cc.Invoke(ctx, SeaweedS3IamCache_DeletePolicy_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedS3IamCacheClient) PutGroup(ctx context.Context, in *iam_pb.PutGroupRequest, opts ...grpc.CallOption) (*iam_pb.PutGroupResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(iam_pb.PutGroupResponse)
+	err := c.cc.Invoke(ctx, SeaweedS3IamCache_PutGroup_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedS3IamCacheClient) RemoveGroup(ctx context.Context, in *iam_pb.RemoveGroupRequest, opts ...grpc.CallOption) (*iam_pb.RemoveGroupResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(iam_pb.RemoveGroupResponse)
+	err := c.cc.Invoke(ctx, SeaweedS3IamCache_RemoveGroup_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// SeaweedS3IamCacheServer is the server API for SeaweedS3IamCache service.
+// All implementations must embed UnimplementedSeaweedS3IamCacheServer
+// for forward compatibility.
+//
+// Designed for unidirectional propagation from Filer to S3 Servers
+type SeaweedS3IamCacheServer interface {
+	PutIdentity(context.Context, *iam_pb.PutIdentityRequest) (*iam_pb.PutIdentityResponse, error)
+	RemoveIdentity(context.Context, *iam_pb.RemoveIdentityRequest) (*iam_pb.RemoveIdentityResponse, error)
+	PutPolicy(context.Context, *iam_pb.PutPolicyRequest) (*iam_pb.PutPolicyResponse, error)
+	GetPolicy(context.Context, *iam_pb.GetPolicyRequest) (*iam_pb.GetPolicyResponse, error)
+	ListPolicies(context.Context, *iam_pb.ListPoliciesRequest) (*iam_pb.ListPoliciesResponse, error)
+	DeletePolicy(context.Context, *iam_pb.DeletePolicyRequest) (*iam_pb.DeletePolicyResponse, error)
+	PutGroup(context.Context, *iam_pb.PutGroupRequest) (*iam_pb.PutGroupResponse, error)
+	RemoveGroup(context.Context, *iam_pb.RemoveGroupRequest) (*iam_pb.RemoveGroupResponse, error)
+	mustEmbedUnimplementedSeaweedS3IamCacheServer()
+}
+
+// UnimplementedSeaweedS3IamCacheServer must be embedded to have
+// forward compatible implementations.
+//
+// NOTE: this should be embedded by value instead of pointer to avoid a nil
+// pointer dereference when methods are called.
+type UnimplementedSeaweedS3IamCacheServer struct{}
+
+func (UnimplementedSeaweedS3IamCacheServer) PutIdentity(context.Context, *iam_pb.PutIdentityRequest) (*iam_pb.PutIdentityResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method PutIdentity not implemented")
+}
+func (UnimplementedSeaweedS3IamCacheServer) RemoveIdentity(context.Context, *iam_pb.RemoveIdentityRequest) (*iam_pb.RemoveIdentityResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method RemoveIdentity not implemented")
+}
+func (UnimplementedSeaweedS3IamCacheServer) PutPolicy(context.Context, *iam_pb.PutPolicyRequest) (*iam_pb.PutPolicyResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method PutPolicy not implemented")
+}
+func (UnimplementedSeaweedS3IamCacheServer) GetPolicy(context.Context, *iam_pb.GetPolicyRequest) (*iam_pb.GetPolicyResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetPolicy not implemented")
+}
+func (UnimplementedSeaweedS3IamCacheServer) ListPolicies(context.Context, *iam_pb.ListPoliciesRequest) (*iam_pb.ListPoliciesResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ListPolicies not implemented")
+}
+func (UnimplementedSeaweedS3IamCacheServer) DeletePolicy(context.Context, *iam_pb.DeletePolicyRequest) (*iam_pb.DeletePolicyResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method DeletePolicy not implemented")
+}
+func (UnimplementedSeaweedS3IamCacheServer) PutGroup(context.Context, *iam_pb.PutGroupRequest) (*iam_pb.PutGroupResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method PutGroup not implemented")
+}
+func (UnimplementedSeaweedS3IamCacheServer) RemoveGroup(context.Context, *iam_pb.RemoveGroupRequest) (*iam_pb.RemoveGroupResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method RemoveGroup not implemented")
+}
+func (UnimplementedSeaweedS3IamCacheServer) mustEmbedUnimplementedSeaweedS3IamCacheServer() {}
+func (UnimplementedSeaweedS3IamCacheServer) testEmbeddedByValue()                           {}
+
+// UnsafeSeaweedS3IamCacheServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to SeaweedS3IamCacheServer will
+// result in compilation errors.
+type UnsafeSeaweedS3IamCacheServer interface {
+	mustEmbedUnimplementedSeaweedS3IamCacheServer()
+}
+
+func RegisterSeaweedS3IamCacheServer(s grpc.ServiceRegistrar, srv SeaweedS3IamCacheServer) {
+	// If the following call pancis, it indicates UnimplementedSeaweedS3IamCacheServer was
+	// embedded by pointer and is nil.  This will cause panics if an
+	// unimplemented method is ever invoked, so we test this at initialization
+	// time to prevent it from happening at runtime later due to I/O.
+	if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
+		t.testEmbeddedByValue()
+	}
+	s.RegisterService(&SeaweedS3IamCache_ServiceDesc, srv)
+}
+
+func _SeaweedS3IamCache_PutIdentity_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(iam_pb.PutIdentityRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedS3IamCacheServer).PutIdentity(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedS3IamCache_PutIdentity_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedS3IamCacheServer).PutIdentity(ctx, req.(*iam_pb.PutIdentityRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedS3IamCache_RemoveIdentity_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(iam_pb.RemoveIdentityRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedS3IamCacheServer).RemoveIdentity(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedS3IamCache_RemoveIdentity_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedS3IamCacheServer).RemoveIdentity(ctx, req.(*iam_pb.RemoveIdentityRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedS3IamCache_PutPolicy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(iam_pb.PutPolicyRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedS3IamCacheServer).PutPolicy(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedS3IamCache_PutPolicy_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedS3IamCacheServer).PutPolicy(ctx, req.(*iam_pb.PutPolicyRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedS3IamCache_GetPolicy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(iam_pb.GetPolicyRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedS3IamCacheServer).GetPolicy(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedS3IamCache_GetPolicy_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedS3IamCacheServer).GetPolicy(ctx, req.(*iam_pb.GetPolicyRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedS3IamCache_ListPolicies_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(iam_pb.ListPoliciesRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedS3IamCacheServer).ListPolicies(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedS3IamCache_ListPolicies_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedS3IamCacheServer).ListPolicies(ctx, req.(*iam_pb.ListPoliciesRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedS3IamCache_DeletePolicy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(iam_pb.DeletePolicyRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedS3IamCacheServer).DeletePolicy(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedS3IamCache_DeletePolicy_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedS3IamCacheServer).DeletePolicy(ctx, req.(*iam_pb.DeletePolicyRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedS3IamCache_PutGroup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(iam_pb.PutGroupRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedS3IamCacheServer).PutGroup(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedS3IamCache_PutGroup_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedS3IamCacheServer).PutGroup(ctx, req.(*iam_pb.PutGroupRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedS3IamCache_RemoveGroup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(iam_pb.RemoveGroupRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedS3IamCacheServer).RemoveGroup(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedS3IamCache_RemoveGroup_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedS3IamCacheServer).RemoveGroup(ctx, req.(*iam_pb.RemoveGroupRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// SeaweedS3IamCache_ServiceDesc is the grpc.ServiceDesc for SeaweedS3IamCache service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var SeaweedS3IamCache_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "messaging_pb.SeaweedS3IamCache",
+	HandlerType: (*SeaweedS3IamCacheServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "PutIdentity",
+			Handler:    _SeaweedS3IamCache_PutIdentity_Handler,
+		},
+		{
+			MethodName: "RemoveIdentity",
+			Handler:    _SeaweedS3IamCache_RemoveIdentity_Handler,
+		},
+		{
+			MethodName: "PutPolicy",
+			Handler:    _SeaweedS3IamCache_PutPolicy_Handler,
+		},
+		{
+			MethodName: "GetPolicy",
+			Handler:    _SeaweedS3IamCache_GetPolicy_Handler,
+		},
+		{
+			MethodName: "ListPolicies",
+			Handler:    _SeaweedS3IamCache_ListPolicies_Handler,
+		},
+		{
+			MethodName: "DeletePolicy",
+			Handler:    _SeaweedS3IamCache_DeletePolicy_Handler,
+		},
+		{
+			MethodName: "PutGroup",
+			Handler:    _SeaweedS3IamCache_PutGroup_Handler,
+		},
+		{
+			MethodName: "RemoveGroup",
+			Handler:    _SeaweedS3IamCache_RemoveGroup_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "s3.proto",
+}

weed/pb/schema_pb/offset_test.go

@@ -0,0 +1,94 @@
+package schema_pb
+
+import (
+	"testing"
+
+	"google.golang.org/protobuf/proto"
+)
+
+func TestOffsetTypeEnums(t *testing.T) {
+	// Test that new offset-based enum values are defined
+	tests := []struct {
+		name     string
+		value    OffsetType
+		expected int32
+	}{
+		{"EXACT_OFFSET", OffsetType_EXACT_OFFSET, 25},
+		{"RESET_TO_OFFSET", OffsetType_RESET_TO_OFFSET, 30},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if int32(tt.value) != tt.expected {
+				t.Errorf("OffsetType_%s = %d, want %d", tt.name, int32(tt.value), tt.expected)
+			}
+		})
+	}
+}
+
+func TestPartitionOffsetSerialization(t *testing.T) {
+	// Test that PartitionOffset can serialize/deserialize with new offset field
+	original := &PartitionOffset{
+		Partition: &Partition{
+			RingSize:   1024,
+			RangeStart: 0,
+			RangeStop:  31,
+			UnixTimeNs: 1234567890,
+		},
+		StartTsNs:   1234567890,
+		StartOffset: 42, // New field
+	}
+
+	// Test proto marshaling/unmarshaling
+	data, err := proto.Marshal(original)
+	if err != nil {
+		t.Fatalf("Failed to marshal PartitionOffset: %v", err)
+	}
+
+	restored := &PartitionOffset{}
+	err = proto.Unmarshal(data, restored)
+	if err != nil {
+		t.Fatalf("Failed to unmarshal PartitionOffset: %v", err)
+	}
+
+	// Verify all fields are preserved
+	if restored.StartTsNs != original.StartTsNs {
+		t.Errorf("StartTsNs = %d, want %d", restored.StartTsNs, original.StartTsNs)
+	}
+	if restored.StartOffset != original.StartOffset {
+		t.Errorf("StartOffset = %d, want %d", restored.StartOffset, original.StartOffset)
+	}
+	if restored.Partition.RingSize != original.Partition.RingSize {
+		t.Errorf("Partition.RingSize = %d, want %d", restored.Partition.RingSize, original.Partition.RingSize)
+	}
+}
+
+func TestPartitionOffsetBackwardCompatibility(t *testing.T) {
+	// Test that PartitionOffset without StartOffset still works
+	original := &PartitionOffset{
+		Partition: &Partition{
+			RingSize:   1024,
+			RangeStart: 0,
+			RangeStop:  31,
+			UnixTimeNs: 1234567890,
+		},
+		StartTsNs: 1234567890,
+		// StartOffset not set (defaults to 0)
+	}
+
+	data, err := proto.Marshal(original)
+	if err != nil {
+		t.Fatalf("Failed to marshal PartitionOffset: %v", err)
+	}
+
+	restored := &PartitionOffset{}
+	err = proto.Unmarshal(data, restored)
+	if err != nil {
+		t.Fatalf("Failed to unmarshal PartitionOffset: %v", err)
+	}
+
+	// StartOffset should default to 0
+	if restored.StartOffset != 0 {
+		t.Errorf("StartOffset = %d, want 0", restored.StartOffset)
+	}
+}

weed/pb/server_address.go

@@ -0,0 +1,186 @@
+package pb
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type ServerAddress string
+type ServerAddresses string
+type ServerSrvAddress string
+
+func NewServerAddress(host string, port int, grpcPort int) ServerAddress {
+	if grpcPort == 0 {
+		return ServerAddress(util.JoinHostPort(host, port))
+	}
+	return ServerAddress(util.JoinHostPort(host, port) + "." + strconv.Itoa(grpcPort))
+}
+
+func NewServerAddressWithGrpcPort(address string, grpcPort int) ServerAddress {
+	if grpcPort == 0 {
+		return ServerAddress(address)
+	}
+	return ServerAddress(address + "." + strconv.Itoa(grpcPort))
+}
+
+func NewServerAddressFromDataNode(dn *master_pb.DataNodeInfo) ServerAddress {
+	// Use Address field if available (new behavior), fall back to Id for backward compatibility
+	addr := dn.Address
+	if addr == "" {
+		addr = dn.Id // backward compatibility: old nodes use ip:port as id
+	}
+	return NewServerAddressWithGrpcPort(addr, int(dn.GrpcPort))
+}
+
+func NewServerAddressFromLocation(dn *master_pb.Location) ServerAddress {
+	return NewServerAddressWithGrpcPort(dn.Url, int(dn.GrpcPort))
+}
+
+func (sa ServerAddress) String() string {
+	return sa.ToHttpAddress()
+}
+
+func (sa ServerAddress) ToHttpAddress() string {
+	portsSepIndex := strings.LastIndex(string(sa), ":")
+	if portsSepIndex < 0 {
+		return string(sa)
+	}
+	if portsSepIndex+1 >= len(sa) {
+		return string(sa)
+	}
+	ports := string(sa[portsSepIndex+1:])
+	sepIndex := strings.LastIndex(string(ports), ".")
+	if sepIndex >= 0 {
+		host := string(sa[0:portsSepIndex])
+		return util.JoinHostPortStr(host, ports[0:sepIndex])
+	}
+	return string(sa)
+}
+
+func (sa ServerAddress) Equals(other ServerAddress) bool {
+	if sa == other {
+		return true
+	}
+	return sa.ToHttpAddress() == other.ToHttpAddress()
+}
+
+func (sa ServerAddress) ToGrpcAddress() string {
+	portsSepIndex := strings.LastIndex(string(sa), ":")
+	if portsSepIndex < 0 {
+		return string(sa)
+	}
+	if portsSepIndex+1 >= len(sa) {
+		return string(sa)
+	}
+	ports := string(sa[portsSepIndex+1:])
+	sepIndex := strings.LastIndex(ports, ".")
+	if sepIndex >= 0 {
+		host := string(sa[0:portsSepIndex])
+		return util.JoinHostPortStr(host, ports[sepIndex+1:])
+	}
+	return ServerToGrpcAddress(string(sa))
+}
+
+// ToHost returns the host part only, without any port information.
+func (sa ServerAddress) ToHost() string {
+	httpAddr := sa.ToHttpAddress()
+
+	host, _, err := net.SplitHostPort(httpAddr)
+	if err == nil {
+		return host
+	}
+
+	// Fallback: if parsing fails, it's likely a host without a port.
+	// Handle bracketed IPv6 (e.g., "[::1]" without port) by trimming brackets.
+	if strings.HasPrefix(httpAddr, "[") && strings.HasSuffix(httpAddr, "]") {
+		return httpAddr[1 : len(httpAddr)-1]
+	}
+	return httpAddr
+}
+
+// LookUp may return an error for some records along with successful lookups - make sure you do not
+// discard `addresses` even if `err == nil`
+func (r ServerSrvAddress) LookUp() (addresses []ServerAddress, err error) {
+	_, records, lookupErr := net.LookupSRV("", "", string(r))
+	if lookupErr != nil {
+		err = fmt.Errorf("lookup SRV address %s: %v", r, lookupErr)
+	}
+	for _, srv := range records {
+		address := fmt.Sprintf("%s:%d", srv.Target, srv.Port)
+		addresses = append(addresses, ServerAddress(address))
+	}
+	return
+}
+
+// ToServiceDiscovery expects one of: a comma-separated list of ip:port, like
+//
+//	10.0.0.1:9999,10.0.0.2:24:9999
+//
+// OR an SRV Record prepended with 'dnssrv+', like:
+//
+//	dnssrv+_grpc._tcp.master.consul
+//	dnssrv+_grpc._tcp.headless.default.svc.cluster.local
+//	dnssrv+seaweed-master.master.consul
+func (sa ServerAddresses) ToServiceDiscovery() (sd *ServerDiscovery) {
+	sd = &ServerDiscovery{}
+	prefix := "dnssrv+"
+	if strings.HasPrefix(string(sa), prefix) {
+		trimmed := strings.TrimPrefix(string(sa), prefix)
+		srv := ServerSrvAddress(trimmed)
+		sd.srvRecord = &srv
+	} else {
+		sd.list = sa.ToAddresses()
+	}
+	return
+}
+
+func (sa ServerAddresses) ToAddresses() (addresses []ServerAddress) {
+	parts := strings.Split(string(sa), ",")
+	for _, address := range parts {
+		if address != "" {
+			addresses = append(addresses, ServerAddress(address))
+		}
+	}
+	return
+}
+
+func (sa ServerAddresses) ToAddressMap() (addresses map[string]ServerAddress) {
+	addresses = make(map[string]ServerAddress)
+	for _, address := range sa.ToAddresses() {
+		addresses[string(address)] = address
+	}
+	return
+}
+
+func ToAddressStrings(addresses []ServerAddress) []string {
+	var strings []string
+	for _, addr := range addresses {
+		strings = append(strings, string(addr))
+	}
+	return strings
+}
+
+func ParseUrl(input string) (address ServerAddress, path string, err error) {
+	if !strings.HasPrefix(input, "http://") {
+		return "", "", fmt.Errorf("url %s needs prefix 'http://'", input)
+	}
+	input = input[7:]
+	pathSeparatorIndex := strings.Index(input, "/")
+	hostAndPorts := input
+	if pathSeparatorIndex > 0 {
+		path = input[pathSeparatorIndex:]
+		hostAndPorts = input[0:pathSeparatorIndex]
+	}
+	commaSeparatorIndex := strings.Index(input, ":")
+	if commaSeparatorIndex < 0 {
+		err = fmt.Errorf("port should be specified in %s", input)
+		return
+	}
+	address = ServerAddress(hostAndPorts)
+	return
+}

weed/pb/server_address_test.go

@@ -0,0 +1,127 @@
+package pb
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestServerAddresses_ToAddressMapOrSrv_shouldRemovePrefix(t *testing.T) {
+	str := ServerAddresses("dnssrv+hello.srv.consul")
+
+	d := str.ToServiceDiscovery()
+
+	expected := ServerSrvAddress("hello.srv.consul")
+	if *d.srvRecord != expected {
+		t.Fatalf(`ServerAddresses("dnssrv+hello.srv.consul") = %s, expected %s`, *d.srvRecord, expected)
+	}
+}
+
+func TestServerAddresses_ToAddressMapOrSrv_shouldHandleIPPortList(t *testing.T) {
+	str := ServerAddresses("10.0.0.1:23,10.0.0.2:24")
+
+	d := str.ToServiceDiscovery()
+
+	if d.srvRecord != nil {
+		t.Fatalf(`ServerAddresses("dnssrv+hello.srv.consul") = %s, expected nil`, *d.srvRecord)
+	}
+
+	expected := []ServerAddress{
+		ServerAddress("10.0.0.1:23"),
+		ServerAddress("10.0.0.2:24"),
+	}
+
+	if !reflect.DeepEqual(d.list, expected) {
+		t.Fatalf(`Expected %q, got %q`, expected, d.list)
+	}
+}
+
+func TestServerAddress_ToHost(t *testing.T) {
+	testCases := []struct {
+		name     string
+		address  ServerAddress
+		expected string
+	}{
+		{
+			name:     "hostname with port",
+			address:  ServerAddress("master.example.com:9333"),
+			expected: "master.example.com",
+		},
+		{
+			name:     "IPv4 with port",
+			address:  ServerAddress("192.168.1.1:9333"),
+			expected: "192.168.1.1",
+		},
+		{
+			name:     "IPv6 with port",
+			address:  ServerAddress("[2001:db8::1]:9333"),
+			expected: "2001:db8::1",
+		},
+		{
+			name:     "hostname without port",
+			address:  ServerAddress("master.example.com"),
+			expected: "master.example.com",
+		},
+		{
+			name:     "hostname with port.grpcPort",
+			address:  ServerAddress("master.example.com:443.10443"),
+			expected: "master.example.com",
+		},
+		{
+			name:     "IPv4 with port.grpcPort",
+			address:  ServerAddress("192.168.1.1:8080.18080"),
+			expected: "192.168.1.1",
+		},
+		{
+			name:     "IPv6 with port.grpcPort",
+			address:  ServerAddress("[2001:db8::1]:8080.18080"),
+			expected: "2001:db8::1",
+		},
+		{
+			name:     "bracketed IPv6 without port",
+			address:  ServerAddress("[2001:db8::1]"),
+			expected: "2001:db8::1",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := tc.address.ToHost()
+			if got != tc.expected {
+				t.Errorf("ServerAddress(%q).ToHost() = %q, want %q", tc.address, got, tc.expected)
+			}
+		})
+	}
+}
+
+func TestIPv6ServerAddressFormatting(t *testing.T) {
+	testCases := []struct {
+		name         string
+		sa           ServerAddress
+		expectedHttp string
+		expectedGrpc string
+	}{
+		{
+			name:         "unbracketed IPv6",
+			sa:           NewServerAddress("2001:db8::1", 8080, 18080),
+			expectedHttp: "[2001:db8::1]:8080",
+			expectedGrpc: "[2001:db8::1]:18080",
+		},
+		{
+			name:         "bracketed IPv6",
+			sa:           NewServerAddressWithGrpcPort("[2001:db8::1]:8080", 18080),
+			expectedHttp: "[2001:db8::1]:8080",
+			expectedGrpc: "[2001:db8::1]:18080",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			if httpAddr := tc.sa.ToHttpAddress(); httpAddr != tc.expectedHttp {
+				t.Errorf("%s: ToHttpAddress() = %s, want %s", tc.name, httpAddr, tc.expectedHttp)
+			}
+			if grpcAddr := tc.sa.ToGrpcAddress(); grpcAddr != tc.expectedGrpc {
+				t.Errorf("%s: ToGrpcAddress() = %s, want %s", tc.name, grpcAddr, tc.expectedGrpc)
+			}
+		})
+	}
+}

weed/pb/server_discovery.go

@@ -0,0 +1,63 @@
+package pb
+
+import (
+	"reflect"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+// ServerDiscovery encodes a way to find at least 1 instance of a service,
+// and provides utility functions to refresh the instance list
+type ServerDiscovery struct {
+	list      []ServerAddress
+	srvRecord *ServerSrvAddress
+}
+
+func NewServiceDiscoveryFromMap(m map[string]ServerAddress) (sd *ServerDiscovery) {
+	sd = &ServerDiscovery{}
+	for _, s := range m {
+		sd.list = append(sd.list, s)
+	}
+	return sd
+}
+
+// RefreshBySrvIfAvailable performs a DNS SRV lookup and updates list with the results
+// of the lookup
+func (sd *ServerDiscovery) RefreshBySrvIfAvailable() {
+	if sd.srvRecord == nil {
+		return
+	}
+	newList, err := sd.srvRecord.LookUp()
+	if err != nil {
+		glog.V(0).Infof("failed to lookup SRV for %s: %v", *sd.srvRecord, err)
+	}
+	if newList == nil || len(newList) == 0 {
+		glog.V(0).Infof("looked up SRV for %s, but found no well-formed names", *sd.srvRecord)
+		return
+	}
+	if !reflect.DeepEqual(sd.list, newList) {
+		sd.list = newList
+	}
+}
+
+// GetInstances returns a copy of the latest known list of addresses
+// call RefreshBySrvIfAvailable prior to this in order to get a more up-to-date view
+func (sd *ServerDiscovery) GetInstances() (addresses []ServerAddress) {
+	for _, a := range sd.list {
+		addresses = append(addresses, a)
+	}
+	return addresses
+}
+func (sd *ServerDiscovery) GetInstancesAsStrings() (addresses []string) {
+	for _, i := range sd.list {
+		addresses = append(addresses, string(i))
+	}
+	return addresses
+}
+func (sd *ServerDiscovery) GetInstancesAsMap() (addresses map[string]ServerAddress) {
+	addresses = make(map[string]ServerAddress)
+	for _, i := range sd.list {
+		addresses[string(i)] = i
+	}
+	return addresses
+}

weed/pb/shared_values.go

@@ -0,0 +1,5 @@
+package pb
+
+const (
+	AdminShellClient = "adminShell"
+)

weed/pb/volume_list_stream.go

@@ -0,0 +1,212 @@
+package pb
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// ReceiveVolumeList reads a streamed volume listing: the topology first, then
+// its volumes in batches as they arrive. A caller that works a volume at a
+// time never holds the cluster; one that needs it whole can use
+// CollectVolumeList.
+//
+// onTopology is given a listing whose disks name themselves but list nothing,
+// because the volumes come through onVolumes instead. That holds however the
+// master answered: one too old for the stream is asked the old way and its
+// reply cut into the same batches, so a caller cannot tell the difference and
+// must not read volumes off the topology either way.
+func ReceiveVolumeList(ctx context.Context, client master_pb.SeaweedClient, request *master_pb.VolumeListRequest,
+	onTopology func(*master_pb.VolumeListResponse) error,
+	onVolumes func(*master_pb.VolumeListStreamResponse) error) error {
+
+	stream, err := client.VolumeListStream(ctx, request)
+	if err == nil {
+		started, streamErr := receiveVolumeListStream(stream, onTopology, onVolumes)
+		// Only a stream that said nothing can be asked again the old way. Past
+		// its first message the master plainly does have the method, and the
+		// error may even be the caller's own, so starting over would hand back
+		// what has already been handed over.
+		if started || status.Code(streamErr) != codes.Unimplemented {
+			return streamErr
+		}
+	} else if status.Code(err) != codes.Unimplemented {
+		return err
+	}
+
+	response, err := client.VolumeList(ctx, request)
+	if err != nil {
+		return err
+	}
+	return replayVolumeList(response, onTopology, onVolumes)
+}
+
+// receiveVolumeListStream reports whether the stream said anything at all,
+// which decides whether it can be started over as an unstreamed listing.
+func receiveVolumeListStream(stream master_pb.Seaweed_VolumeListStreamClient,
+	onTopology func(*master_pb.VolumeListResponse) error,
+	onVolumes func(*master_pb.VolumeListStreamResponse) error) (started bool, err error) {
+
+	told := false
+	for {
+		batch, err := stream.Recv()
+		if err == io.EOF {
+			if !told {
+				return started, fmt.Errorf("volume list stream ended before its topology")
+			}
+			return started, nil
+		}
+		if err != nil {
+			return started, err
+		}
+		started = true
+		if batch.Header != nil {
+			if told {
+				return started, fmt.Errorf("volume list stream sent its topology twice")
+			}
+			told = true
+			if onTopology != nil {
+				if err := onTopology(batch.Header); err != nil {
+					return started, err
+				}
+			}
+			continue
+		}
+		if !told {
+			return started, fmt.Errorf("volume list stream sent volumes before its topology")
+		}
+		if onVolumes != nil {
+			if err := onVolumes(batch); err != nil {
+				return started, err
+			}
+		}
+	}
+}
+
+// replayVolumeList cuts an unstreamed reply into the batches a caller expects.
+// Each disk's volumes are moved out of the topology rather than shared with it,
+// so the topology handed over lists nothing, exactly as a streamed one does.
+func replayVolumeList(response *master_pb.VolumeListResponse,
+	onTopology func(*master_pb.VolumeListResponse) error,
+	onVolumes func(*master_pb.VolumeListStreamResponse) error) error {
+
+	type batch struct {
+		key    [4]string
+		volume []*master_pb.VolumeInformationMessage
+		ec     []*master_pb.VolumeEcShardInformationMessage
+	}
+	var batches []batch
+
+	if response.TopologyInfo != nil {
+		for _, dc := range response.TopologyInfo.DataCenterInfos {
+			for _, rack := range dc.RackInfos {
+				for _, node := range rack.DataNodeInfos {
+					for diskType, disk := range node.DiskInfos {
+						if len(disk.VolumeInfos) == 0 && len(disk.EcShardInfos) == 0 {
+							continue
+						}
+						batches = append(batches, batch{
+							key:    [4]string{dc.Id, rack.Id, node.Id, diskType},
+							volume: disk.VolumeInfos,
+							ec:     disk.EcShardInfos,
+						})
+						disk.VolumeInfos, disk.EcShardInfos = nil, nil
+					}
+				}
+			}
+		}
+	}
+
+	if onTopology != nil {
+		if err := onTopology(response); err != nil {
+			return err
+		}
+	}
+	if onVolumes == nil {
+		return nil
+	}
+	for _, b := range batches {
+		err := onVolumes(&master_pb.VolumeListStreamResponse{
+			DataCenter:   b.key[0],
+			Rack:         b.key[1],
+			DataNode:     b.key[2],
+			DiskType:     b.key[3],
+			VolumeInfos:  b.volume,
+			EcShardInfos: b.ec,
+		})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// CollectVolumeList streams a listing and puts it back together, for callers
+// that need the whole topology. The master still never holds it all, which is
+// the point; this only moves that cost to the caller.
+func CollectVolumeList(ctx context.Context, client master_pb.SeaweedClient, request *master_pb.VolumeListRequest) (*master_pb.VolumeListResponse, error) {
+	var response *master_pb.VolumeListResponse
+	var skipped int
+	disks := make(map[[4]string]*master_pb.DiskInfo)
+
+	err := ReceiveVolumeList(ctx, client, request,
+		func(topology *master_pb.VolumeListResponse) error {
+			response = topology
+			return nil
+		},
+		func(batch *master_pb.VolumeListStreamResponse) error {
+			key := [4]string{batch.DataCenter, batch.Rack, batch.DataNode, batch.DiskType}
+			disk, known := disks[key]
+			if !known {
+				disk = findDisk(response, key)
+				disks[key] = disk
+			}
+			if disk == nil {
+				// A disk registered after the topology went out. It is not in
+				// the listing being rebuilt and has nowhere to go, so leave it
+				// to the next one rather than failing this one -- an unstreamed
+				// listing would not have shown it either, having read each
+				// node's disks once.
+				skipped++
+				return nil
+			}
+			disk.VolumeInfos = append(disk.VolumeInfos, batch.VolumeInfos...)
+			disk.EcShardInfos = append(disk.EcShardInfos, batch.EcShardInfos...)
+			return nil
+		})
+	if err != nil {
+		return nil, err
+	}
+	if skipped > 0 {
+		glog.V(1).Infof("volume list: %d batches were for disks added after the topology was sent", skipped)
+	}
+	return response, nil
+}
+
+func findDisk(response *master_pb.VolumeListResponse, key [4]string) *master_pb.DiskInfo {
+	if response == nil || response.TopologyInfo == nil {
+		return nil
+	}
+	for _, dc := range response.TopologyInfo.DataCenterInfos {
+		if dc.Id != key[0] {
+			continue
+		}
+		for _, rack := range dc.RackInfos {
+			if rack.Id != key[1] {
+				continue
+			}
+			for _, node := range rack.DataNodeInfos {
+				if node.Id != key[2] {
+					continue
+				}
+				return node.DiskInfos[key[3]]
+			}
+		}
+	}
+	return nil
+}

weed/pb/volume_list_stream_test.go

@@ -0,0 +1,306 @@
+package pb
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/test/bufconn"
+	"google.golang.org/protobuf/proto"
+)
+
+// fakeMaster answers a listing either way, so the same assertions can be made
+// of a master that streams and one too old to.
+type fakeMaster struct {
+	master_pb.UnimplementedSeaweedServer
+	response *master_pb.VolumeListResponse
+	streams  bool
+	batch    int
+	// extraBatch is sent after the rest, standing for a disk that registered
+	// once the topology had gone out.
+	extraBatch *master_pb.VolumeListStreamResponse
+	// failAfterHeader stands for a master that plainly has the method but
+	// gives up mid-stream, reporting the one code that means "ask the old way".
+	failAfterHeader bool
+}
+
+func (m *fakeMaster) VolumeList(ctx context.Context, req *master_pb.VolumeListRequest) (*master_pb.VolumeListResponse, error) {
+	return cloneListing(m.response), nil
+}
+
+func (m *fakeMaster) VolumeListStream(req *master_pb.VolumeListRequest, stream master_pb.Seaweed_VolumeListStreamServer) error {
+	if !m.streams {
+		return status_Unimplemented()
+	}
+	full := cloneListing(m.response)
+	header := &master_pb.VolumeListResponse{
+		TopologyInfo:      &master_pb.TopologyInfo{Id: full.TopologyInfo.Id},
+		VolumeSizeLimitMb: full.VolumeSizeLimitMb,
+	}
+	// The header names every disk but lists nothing on it.
+	for _, dc := range full.TopologyInfo.DataCenterInfos {
+		headerDc := &master_pb.DataCenterInfo{Id: dc.Id}
+		for _, rack := range dc.RackInfos {
+			headerRack := &master_pb.RackInfo{Id: rack.Id}
+			for _, node := range rack.DataNodeInfos {
+				headerNode := &master_pb.DataNodeInfo{Id: node.Id, DiskInfos: map[string]*master_pb.DiskInfo{}}
+				for diskType, disk := range node.DiskInfos {
+					headerNode.DiskInfos[diskType] = &master_pb.DiskInfo{Type: diskType, DiskId: disk.DiskId}
+				}
+				headerRack.DataNodeInfos = append(headerRack.DataNodeInfos, headerNode)
+			}
+			headerDc.RackInfos = append(headerDc.RackInfos, headerRack)
+		}
+		header.TopologyInfo.DataCenterInfos = append(header.TopologyInfo.DataCenterInfos, headerDc)
+	}
+	if err := stream.Send(&master_pb.VolumeListStreamResponse{Header: header}); err != nil {
+		return err
+	}
+	if m.failAfterHeader {
+		return status_Unimplemented()
+	}
+
+	for _, dc := range full.TopologyInfo.DataCenterInfos {
+		for _, rack := range dc.RackInfos {
+			for _, node := range rack.DataNodeInfos {
+				for diskType, disk := range node.DiskInfos {
+					for start := 0; start < len(disk.VolumeInfos); start += m.batch {
+						end := min(start+m.batch, len(disk.VolumeInfos))
+						err := stream.Send(&master_pb.VolumeListStreamResponse{
+							DataCenter: dc.Id, Rack: rack.Id, DataNode: node.Id, DiskType: diskType,
+							VolumeInfos: disk.VolumeInfos[start:end],
+						})
+						if err != nil {
+							return err
+						}
+					}
+					if len(disk.EcShardInfos) > 0 {
+						err := stream.Send(&master_pb.VolumeListStreamResponse{
+							DataCenter: dc.Id, Rack: rack.Id, DataNode: node.Id, DiskType: diskType,
+							EcShardInfos: disk.EcShardInfos,
+						})
+						if err != nil {
+							return err
+						}
+					}
+				}
+			}
+		}
+	}
+	if m.extraBatch != nil {
+		if err := stream.Send(m.extraBatch); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func testListing(volumes int) *master_pb.VolumeListResponse {
+	disk := &master_pb.DiskInfo{Type: "", DiskId: 2}
+	for i := 1; i <= volumes; i++ {
+		disk.VolumeInfos = append(disk.VolumeInfos, &master_pb.VolumeInformationMessage{
+			Id: uint32(i), Size: uint64(i) * 100, Collection: "c",
+		})
+	}
+	disk.EcShardInfos = append(disk.EcShardInfos, &master_pb.VolumeEcShardInformationMessage{
+		Id: 900, Collection: "c", EcIndexBits: 0x3fff,
+	})
+	return &master_pb.VolumeListResponse{
+		VolumeSizeLimitMb: 30000,
+		TopologyInfo: &master_pb.TopologyInfo{
+			Id: "topo",
+			DataCenterInfos: []*master_pb.DataCenterInfo{{
+				Id: "dc1",
+				RackInfos: []*master_pb.RackInfo{{
+					Id: "rack1",
+					DataNodeInfos: []*master_pb.DataNodeInfo{{
+						Id:        "10.0.0.1:8080",
+						DiskInfos: map[string]*master_pb.DiskInfo{"": disk},
+					}},
+				}},
+			}},
+		},
+	}
+}
+
+func dial(t *testing.T, master *fakeMaster) master_pb.SeaweedClient {
+	t.Helper()
+	listener := bufconn.Listen(1 << 20)
+	server := grpc.NewServer()
+	master_pb.RegisterSeaweedServer(server, master)
+	go server.Serve(listener)
+	t.Cleanup(server.Stop)
+
+	conn, err := grpc.NewClient("passthrough://bufnet",
+		grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return listener.DialContext(ctx) }),
+		grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { conn.Close() })
+	return master_pb.NewSeaweedClient(conn)
+}
+
+// Whichever way the master answers, a caller sees the same volumes, and sees
+// none of them on the topology it is handed.
+func TestReceiveVolumeListIsTheSameEitherWay(t *testing.T) {
+	const volumes = 250
+	for _, streams := range []bool{true, false} {
+		t.Run(fmt.Sprintf("streaming=%v", streams), func(t *testing.T) {
+			client := dial(t, &fakeMaster{response: testListing(volumes), streams: streams, batch: 32})
+
+			var got []uint32
+			var ec []uint32
+			var topology *master_pb.VolumeListResponse
+			err := ReceiveVolumeList(context.Background(), client, &master_pb.VolumeListRequest{},
+				func(header *master_pb.VolumeListResponse) error {
+					topology = header
+					return nil
+				},
+				func(batch *master_pb.VolumeListStreamResponse) error {
+					for _, v := range batch.VolumeInfos {
+						got = append(got, v.Id)
+					}
+					for _, s := range batch.EcShardInfos {
+						ec = append(ec, s.Id)
+					}
+					return nil
+				})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(got) != volumes {
+				t.Errorf("received %d volumes, want %d", len(got), volumes)
+			}
+			if len(ec) != 1 {
+				t.Errorf("received %d ec shards, want 1", len(ec))
+			}
+			if topology == nil {
+				t.Fatal("never told the topology")
+			}
+			if topology.VolumeSizeLimitMb != 30000 {
+				t.Errorf("volume size limit %d, want 30000", topology.VolumeSizeLimitMb)
+			}
+			disk := topology.TopologyInfo.DataCenterInfos[0].RackInfos[0].DataNodeInfos[0].DiskInfos[""]
+			if len(disk.VolumeInfos) != 0 || len(disk.EcShardInfos) != 0 {
+				t.Errorf("the topology handed over listed %d volumes and %d ec shards, want none",
+					len(disk.VolumeInfos), len(disk.EcShardInfos))
+			}
+		})
+	}
+}
+
+// Reassembly must put back exactly what an unstreamed listing holds -- in
+// particular it must not double the volumes when the master did not stream.
+func TestCollectVolumeListRebuildsTheListing(t *testing.T) {
+	const volumes = 250
+	for _, streams := range []bool{true, false} {
+		t.Run(fmt.Sprintf("streaming=%v", streams), func(t *testing.T) {
+			client := dial(t, &fakeMaster{response: testListing(volumes), streams: streams, batch: 32})
+
+			response, err := CollectVolumeList(context.Background(), client, &master_pb.VolumeListRequest{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			disk := response.TopologyInfo.DataCenterInfos[0].RackInfos[0].DataNodeInfos[0].DiskInfos[""]
+			if len(disk.VolumeInfos) != volumes {
+				t.Fatalf("rebuilt %d volumes, want %d", len(disk.VolumeInfos), volumes)
+			}
+			if len(disk.EcShardInfos) != 1 {
+				t.Fatalf("rebuilt %d ec shards, want 1", len(disk.EcShardInfos))
+			}
+			if disk.DiskId != 2 {
+				t.Errorf("rebuilt disk id %d, want 2", disk.DiskId)
+			}
+			seen := make(map[uint32]int, volumes)
+			for _, v := range disk.VolumeInfos {
+				seen[v.Id]++
+			}
+			for id, n := range seen {
+				if n != 1 {
+					t.Fatalf("volume %d rebuilt %d times", id, n)
+				}
+			}
+		})
+	}
+}
+
+func status_Unimplemented() error {
+	return status.Error(codes.Unimplemented, "this master does not stream volume listings")
+}
+
+func cloneListing(r *master_pb.VolumeListResponse) *master_pb.VolumeListResponse {
+	return proto.Clone(r).(*master_pb.VolumeListResponse)
+}
+
+// A heartbeat can register a disk between the topology going out and the
+// batches following it. Those volumes have nowhere to go in the listing being
+// rebuilt, but they must not fail it: the scan that reads it runs every 30
+// minutes and would lose the whole cluster over one new disk.
+func TestCollectVolumeListSurvivesADiskAddedMidStream(t *testing.T) {
+	const volumes = 100
+	master := &fakeMaster{response: testListing(volumes), streams: true, batch: 32}
+	master.extraBatch = &master_pb.VolumeListStreamResponse{
+		DataCenter: "dc1", Rack: "rack1", DataNode: "10.0.0.1:8080", DiskType: "ssd",
+		VolumeInfos: []*master_pb.VolumeInformationMessage{{Id: 5000, Collection: "c"}},
+	}
+	client := dial(t, master)
+
+	response, err := CollectVolumeList(context.Background(), client, &master_pb.VolumeListRequest{})
+	if err != nil {
+		t.Fatalf("a disk arriving mid-stream failed the listing: %v", err)
+	}
+	node := response.TopologyInfo.DataCenterInfos[0].RackInfos[0].DataNodeInfos[0]
+	if len(node.DiskInfos[""].VolumeInfos) != volumes {
+		t.Errorf("rebuilt %d volumes, want %d", len(node.DiskInfos[""].VolumeInfos), volumes)
+	}
+	if _, appeared := node.DiskInfos["ssd"]; appeared {
+		t.Error("the listing grew a disk its topology never named")
+	}
+}
+
+// A stream that has already spoken cannot be started over as an unstreamed
+// listing: the caller would be handed the same volumes twice.
+func TestReceiveVolumeListDoesNotRestartAStreamThatBegan(t *testing.T) {
+	client := dial(t, &fakeMaster{response: testListing(100), streams: true, batch: 32, failAfterHeader: true})
+
+	topologies, batches := 0, 0
+	err := ReceiveVolumeList(context.Background(), client, &master_pb.VolumeListRequest{},
+		func(*master_pb.VolumeListResponse) error { topologies++; return nil },
+		func(*master_pb.VolumeListStreamResponse) error { batches++; return nil })
+	if err == nil {
+		t.Fatal("a stream that failed after its topology was quietly restarted")
+	}
+	if topologies != 1 {
+		t.Errorf("handed the topology %d times, want 1", topologies)
+	}
+	if batches != 0 {
+		t.Errorf("handed %d volume batches, want none", batches)
+	}
+}
+
+// A caller's own error must reach it, even when it happens to carry the code
+// that means an older master.
+func TestReceiveVolumeListDoesNotRestartOnACallersError(t *testing.T) {
+	client := dial(t, &fakeMaster{response: testListing(100), streams: true, batch: 32})
+
+	batches := 0
+	err := ReceiveVolumeList(context.Background(), client, &master_pb.VolumeListRequest{},
+		nil,
+		func(*master_pb.VolumeListStreamResponse) error {
+			batches++
+			return status.Error(codes.Unimplemented, "the caller cannot handle this")
+		})
+	if err == nil {
+		t.Fatal("the caller's error was swallowed and the listing restarted")
+	}
+	if batches != 1 {
+		t.Errorf("called back %d times, want 1 before giving up", batches)
+	}
+}

weed/pb/volume_server.proto

@@ -0,0 +1,835 @@
+syntax = "proto3";
+
+package volume_server_pb;
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb";
+
+import "remote.proto";
+
+//////////////////////////////////////////////////
+
+// Persistent state for volume servers.
+message VolumeServerState {
+    // whether the server is in maintenance (i.e. read-only) mode.
+    bool maintenance = 1;
+    // incremental version counter
+    uint32 version = 2;
+}
+
+//////////////////////////////////////////////////
+
+service VolumeServer {
+    //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
+    rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) {
+    }
+
+    rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
+    }
+    rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (stream VacuumVolumeCompactResponse) {
+    }
+    rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
+    }
+    rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) {
+    }
+
+    rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
+    }
+    rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) {
+    }
+
+    rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
+    }
+    rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) {
+    }
+
+    rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
+    }
+    rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
+    }
+    rpc VolumeConsolidateIndex (VolumeConsolidateIndexRequest) returns (VolumeConsolidateIndexResponse) {
+    }
+    rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
+    }
+    rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
+    }
+    rpc VolumeMarkWritable (VolumeMarkWritableRequest) returns (VolumeMarkWritableResponse) {
+    }
+    rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) {
+    }
+    rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) {
+    }
+
+    rpc GetState (GetStateRequest) returns (GetStateResponse) {
+    }
+    rpc SetState (SetStateRequest) returns (SetStateResponse) {
+    }
+
+    // copy the .idx .dat files, and mount this volume
+    rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) {
+    }
+    rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
+    }
+    rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
+    }
+    rpc ReceiveFile (stream ReceiveFileRequest) returns (ReceiveFileResponse) {
+    }
+
+    rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
+    }
+    rpc ReadNeedleMeta (ReadNeedleMetaRequest) returns (ReadNeedleMetaResponse) {
+    }
+    rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
+    }
+    rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) {
+    }
+
+    rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
+    }
+    rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
+    }
+
+    // erasure coding
+    rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
+    }
+    rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) {
+    }
+    rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
+    }
+    rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
+    }
+    rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) {
+    }
+    rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
+    }
+    rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
+    }
+    rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) {
+    }
+    rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) {
+    }
+    rpc VolumeEcShardsInfo (VolumeEcShardsInfoRequest) returns (VolumeEcShardsInfoResponse) {
+    }
+
+    // tiered storage
+    rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) {
+    }
+    rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) {
+    }
+
+    rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) {
+    }
+    rpc VolumeServerLeave (VolumeServerLeaveRequest) returns (VolumeServerLeaveResponse) {
+    }
+
+    // remote storage
+    rpc FetchAndWriteNeedle (FetchAndWriteNeedleRequest) returns (FetchAndWriteNeedleResponse) {
+    }
+
+    // scrubbing
+    rpc ScrubVolume (ScrubVolumeRequest) returns (ScrubVolumeResponse) {
+    }
+    rpc ScrubEcVolume (ScrubEcVolumeRequest) returns (ScrubEcVolumeResponse) {
+    }
+
+    // <experimental> query
+    rpc Query (QueryRequest) returns (stream QueriedStripe) {
+    }
+
+    rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) {
+    }
+
+    rpc Ping (PingRequest) returns (PingResponse) {
+    }
+
+}
+
+//////////////////////////////////////////////////
+
+message BatchDeleteRequest {
+    repeated string file_ids = 1;
+    bool skip_cookie_check = 2;
+}
+
+message BatchDeleteResponse {
+    repeated DeleteResult results = 1;
+}
+message DeleteResult {
+    string file_id = 1;
+    int32 status = 2;
+    string error = 3;
+    uint32 size = 4;
+    uint32 version = 5;
+}
+
+message Empty {
+}
+
+message VacuumVolumeCheckRequest {
+    uint32 volume_id = 1;
+}
+message VacuumVolumeCheckResponse {
+    double garbage_ratio = 1;
+}
+
+message VacuumVolumeCompactRequest {
+    uint32 volume_id = 1;
+    int64 preallocate = 2;
+}
+message VacuumVolumeCompactResponse {
+    int64 processed_bytes = 1;
+    float load_avg_1m = 2;
+}
+
+message VacuumVolumeCommitRequest {
+    uint32 volume_id = 1;
+}
+message VacuumVolumeCommitResponse {
+    bool is_read_only = 1;
+    uint64 volume_size = 2;
+}
+
+message VacuumVolumeCleanupRequest {
+    uint32 volume_id = 1;
+}
+message VacuumVolumeCleanupResponse {
+}
+
+message DeleteCollectionRequest {
+    string collection = 1;
+}
+message DeleteCollectionResponse {
+}
+
+message AllocateVolumeRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    int64 preallocate = 3;
+    string replication = 4;
+    string ttl = 5;
+    uint32 memory_map_max_size_mb = 6;
+    string disk_type = 7;
+    uint32 version = 8;
+}
+message AllocateVolumeResponse {
+}
+
+message VolumeSyncStatusRequest {
+    uint32 volume_id = 1;
+}
+message VolumeSyncStatusResponse {
+    uint32 volume_id = 1;
+    string collection = 2;
+    string replication = 4;
+    string ttl = 5;
+    uint64 tail_offset = 6;
+    uint32 compact_revision = 7;
+    uint64 idx_file_size = 8;
+    uint32 version = 9;
+}
+
+message VolumeIncrementalCopyRequest {
+    uint32 volume_id = 1;
+    uint64 since_ns = 2;
+}
+message VolumeIncrementalCopyResponse {
+    bytes file_content = 1;
+}
+
+message VolumeMountRequest {
+    uint32 volume_id = 1;
+    optional string collection = 2;
+}
+message VolumeMountResponse {
+}
+
+message VolumeUnmountRequest {
+    uint32 volume_id = 1;
+}
+message VolumeUnmountResponse {
+}
+
+message VolumeConsolidateIndexRequest {
+    uint32 volume_id = 1;
+}
+message VolumeConsolidateIndexResponse {
+}
+
+message VolumeDeleteRequest {
+    uint32 volume_id = 1;
+    bool only_empty = 2;
+    // when true, do not remove the cloud-tier object backing the volume.
+    // used for moves where another server is taking over the same .vif.
+    bool keep_remote_data = 3;
+}
+message VolumeDeleteResponse {
+}
+
+message VolumeMarkReadonlyRequest {
+    uint32 volume_id = 1;
+    bool persist = 2;
+    // reject writes but keep accepting deletes, so expiring data can drain the volume
+    bool can_delete = 3;
+}
+message VolumeMarkReadonlyResponse {
+}
+
+message VolumeMarkWritableRequest {
+    uint32 volume_id = 1;
+}
+message VolumeMarkWritableResponse {
+}
+
+message VolumeConfigureRequest {
+    uint32 volume_id = 1;
+    string replication = 2;
+}
+message VolumeConfigureResponse {
+    string error = 1;
+}
+
+message VolumeStatusRequest {
+    uint32 volume_id = 1;
+}
+message VolumeStatusResponse {
+    bool is_read_only = 1;
+    uint64 volume_size = 2;
+    uint64 file_count = 3;
+    uint64 file_deleted_count = 4;
+}
+
+message GetStateRequest {
+}
+message GetStateResponse {
+    VolumeServerState state = 1;
+}
+
+message SetStateRequest {
+    // SetState updates *all* volume server flags at once. Retrieve state with GetState(),
+    // modify individual flags as required, then call this RPC to update.
+    VolumeServerState state = 1;
+}
+message SetStateResponse {
+    VolumeServerState state = 1;
+}
+
+message VolumeCopyRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    string replication = 3;
+    string ttl = 4;
+    string source_data_node = 5;
+    string disk_type = 6;
+    int64 io_byte_per_second = 7;
+}
+message VolumeCopyResponse {
+    uint64 last_append_at_ns = 1;
+    int64 processed_bytes = 2;
+}
+
+message CopyFileRequest {
+    uint32 volume_id = 1;
+    string ext = 2;
+    uint32 compaction_revision = 3;
+    uint64 stop_offset = 4;
+    string collection = 5;
+    bool is_ec_volume = 6;
+    bool ignore_source_file_not_found = 7;
+}
+message CopyFileResponse {
+    bytes file_content = 1;
+    int64 modified_ts_ns = 2;
+}
+
+message ReceiveFileRequest {
+    oneof data {
+        ReceiveFileInfo info = 1;
+        bytes file_content = 2;
+    }
+}
+
+message ReceiveFileInfo {
+    uint32 volume_id = 1;
+    string ext = 2;
+    string collection = 3;
+    bool is_ec_volume = 4;
+    uint32 shard_id = 5;
+    uint64 file_size = 6;
+    uint32 disk_id = 7;  // EC shard disk; 0 = auto-select (see VolumeEcShardsCopyRequest.disk_id)
+
+    // Field numbers 8-11 are reserved for versioned-EC; disk_type stays at 12.
+    // Staged-new-volume mode (EC decode onto a clean peer): set on a non-EC push
+    // whose volume does not yet exist on this server. The server picks a disk
+    // location of this medium with a free slot and writes <base><ext>.copying,
+    // finalized by VolumeEcShardsToVolume(from_staged).
+    string disk_type = 12;
+}
+
+message ReceiveFileResponse {
+    uint64 bytes_written = 1;
+    string error = 2;
+}
+
+message ReadNeedleBlobRequest {
+    uint32 volume_id = 1;
+    int64 offset = 3; // actual offset
+    int32 size = 4;
+}
+message ReadNeedleBlobResponse {
+    bytes needle_blob = 1;
+}
+
+message ReadNeedleMetaRequest {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+    int64 offset = 3; // actual offset
+    int32 size = 4;
+}
+message ReadNeedleMetaResponse {
+    uint32 cookie = 1;
+    uint64 last_modified = 2;
+    uint32 crc = 3;
+    string ttl = 4;
+    uint64 append_at_ns = 5;
+}
+
+message WriteNeedleBlobRequest {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+    int32 size = 3;
+    bytes needle_blob = 4;
+}
+message WriteNeedleBlobResponse {
+}
+
+message ReadAllNeedlesRequest {
+    repeated uint32 volume_ids = 1;
+}
+message ReadAllNeedlesResponse {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+    uint32 cookie = 3;
+    bytes needle_blob = 5;
+    bool needle_blob_compressed = 6;
+    uint64 last_modified = 7;
+    uint32 crc = 8;
+    bytes name = 9;
+    bytes mime = 10;
+}
+
+message VolumeTailSenderRequest {
+    uint32 volume_id = 1;
+    uint64 since_ns = 2;
+    uint32 idle_timeout_seconds = 3;
+}
+message VolumeTailSenderResponse {
+    bytes needle_header = 1;
+    bytes needle_body = 2;
+    bool is_last_chunk = 3;
+    uint32 version = 4;
+}
+
+message VolumeTailReceiverRequest {
+    uint32 volume_id = 1;
+    uint64 since_ns = 2;
+    uint32 idle_timeout_seconds = 3;
+    string source_volume_server = 4;
+}
+message VolumeTailReceiverResponse {
+}
+
+message VolumeEcShardsGenerateRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+}
+message VolumeEcShardsGenerateResponse {
+}
+
+message VolumeEcShardsRebuildRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    bool unsafe_ignore_sidecar = 3; // bypass the bitrot-sidecar fail-closed guard (operator override; distinct from ec.rebuild -force)
+}
+message VolumeEcShardsRebuildResponse {
+    repeated uint32 rebuilt_shard_ids = 1;
+}
+
+message VolumeEcShardsCopyRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    repeated uint32 shard_ids = 3;
+    bool copy_ecx_file = 4;
+    string source_data_node = 5;
+    bool copy_ecj_file = 6;
+    bool copy_vif_file = 7;
+    uint32 disk_id = 8;  // Target disk ID for storing EC shards
+    bool copy_ecsum_file = 9; // copy the bitrot checksum sidecar (.ecsum) when present; tolerant of a missing source (no-op), since this non-2PC path has no Prepare backstop
+    int64 io_byte_per_second = 10; // limit the copy rate; 0 falls back to the server's maintenance rate
+}
+message VolumeEcShardsCopyResponse {
+}
+
+message VolumeEcShardsDeleteRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    repeated uint32 shard_ids = 3;
+    bool full_teardown = 4; // pre-encode cleanup: wipe every EC artifact + generation for this volume, not just shard_ids
+    int64 encode_ts_ns = 5; // full_teardown generation fence: delete only a disk whose .vif generation is strictly OLDER than this; preserve same-or-newer, generation 0, and an unreadable .vif. 0 => wipe-all (shell pre-encode / pre-upgrade)
+}
+message VolumeEcShardsDeleteResponse {
+    bool full_teardown_done = 1; // set by a new server that performed full_teardown; absent from an old server lets the caller detect the silent no-op
+}
+
+message VolumeEcShardsMountRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    repeated uint32 shard_ids = 3;
+    string source_disk_type = 4; // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423)
+    bool recover_missing_index = 5; // first fetch a missing .ecx index from a peer so on-disk shards without a local index become mountable (#10104)
+}
+message VolumeEcShardsMountResponse {
+}
+
+message VolumeEcShardsUnmountRequest {
+    uint32 volume_id = 1;
+    repeated uint32 shard_ids = 3;
+    int64 encode_ts_ns = 4; // generation fence: skip a disk whose mounted EC volume is this generation or newer (0 = unfenced, unmount all)
+}
+message VolumeEcShardsUnmountResponse {
+}
+
+message VolumeEcShardReadRequest {
+    uint32 volume_id = 1;
+    uint32 shard_id = 2;
+    int64 offset = 3;
+    int64 size = 4;
+    uint64 file_key = 5;
+    reserved 6;
+    int64 encode_ts_ns = 7; // caller's expected encode time; the server rejects a shard from a different encode run
+}
+message VolumeEcShardReadResponse {
+    bytes data = 1;
+    bool is_deleted = 2;
+    int64 encode_ts_ns = 3; // identity of the shard actually served; client rejects a mismatch (0 = pre-upgrade server)
+}
+
+message VolumeEcBlobDeleteRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    uint64 file_key = 3;
+    uint32 version = 4;
+}
+message VolumeEcBlobDeleteResponse {
+}
+
+message VolumeEcShardsToVolumeRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    // Staged mode: the caller already decoded the EC shards off-box and pushed
+    // .dat/.idx/.vif as <base><ext>.copying to this server (ReceiveFile
+    // staged-new-volume mode). Adopt them as a normal volume instead of decoding
+    // local EC shards in place, so <vid> is never registered as both EC and
+    // normal on one disk.
+    bool from_staged = 3;
+    string disk_type = 4; // target medium's disk location for the normal volume (staged mode)
+}
+message VolumeEcShardsToVolumeResponse {
+}
+
+message VolumeEcShardsInfoRequest {
+    uint32 volume_id = 1;
+}
+message VolumeEcShardsInfoResponse {
+    repeated EcShardInfo ec_shard_infos = 1;
+    uint64 volume_size = 2;
+    uint64 file_count = 3;
+    uint64 file_deleted_count = 4;
+    EcShardConfig ec_shard_config = 10;  // the layout this holder serves reads through; a binary predating a field reports it as unset
+}
+
+message EcShardInfo {
+    uint32 shard_id = 1;
+    int64 size = 2;
+    string collection = 3;
+    uint32 volume_id = 4;
+}
+
+message ReadVolumeFileStatusRequest {
+    uint32 volume_id = 1;
+}
+message ReadVolumeFileStatusResponse {
+    uint32 volume_id = 1;
+    uint64 idx_file_timestamp_seconds = 2;
+    uint64 idx_file_size = 3;
+    uint64 dat_file_timestamp_seconds = 4;
+    uint64 dat_file_size = 5;
+    uint64 file_count = 6;
+    uint32 compaction_revision = 7;
+    string collection = 8;
+    string disk_type = 9;
+    VolumeInfo volume_info = 10;
+    uint32 version = 11;
+}
+
+message DiskStatus {
+    string dir = 1;
+    uint64 all = 2;
+    uint64 used = 3;
+    uint64 free = 4;
+    float percent_free = 5;
+    float percent_used = 6;
+    string disk_type = 7;
+    string error = 8;
+}
+
+message MemStatus {
+    int32 goroutines = 1;
+    uint64 all = 2;
+    uint64 used = 3;
+    uint64 free = 4;
+    uint64 self = 5;
+    uint64 heap = 6;
+    uint64 stack = 7;
+}
+
+// tired storage on volume servers
+message RemoteFile {
+    string backend_type = 1;
+    string backend_id = 2;
+    string key = 3;
+    uint64 offset = 4;
+    uint64 file_size = 5;
+    uint64 modified_time = 6;
+    string extension = 7;
+}
+message VolumeInfo {
+    repeated RemoteFile files = 1;
+    uint32 version = 2;
+    string replication = 3;
+    uint32 bytes_offset = 4;
+    int64 dat_file_size = 5; // store the original dat file size
+    uint64 expire_at_sec = 6; // expiration time of ec volume
+    bool read_only = 7;
+    EcShardConfig ec_shard_config = 8; // EC shard configuration (optional, null = use default 10+4)
+    bool read_only_can_delete = 9; // with read_only: writes are rejected but deletes still land
+}
+
+// EcShardConfig specifies erasure coding shard configuration
+message EcShardConfig {
+    uint32 data_shards = 1;   // Number of data shards (e.g., 10)
+    uint32 parity_shards = 2; // Number of parity shards (e.g., 4)
+    int64 encode_ts_ns = 3;   // encode time (unix nanos); a read served from a shard of a different encode run is rejected
+    int64 block_size = 4;     // uniform block layout: each shard is a single contiguous block of this many bytes; 0 = legacy 1GiB/1MiB two-tier layout
+}
+
+// EcBitrotProtection is the entire content of a bitrot checksum sidecar
+// (<base>.ecsum for the legacy generation, <base>.ecsum.v<N> for vacuum
+// generation N). On disk it is wrapped in a fixed header carrying a CRC32C
+// over this serialized payload (see weed/storage/erasure_coding/ec_bitrot.go).
+message EcBitrotProtection {
+    ChecksumAlgorithm algorithm = 1;       // CRC32C (Castagnoli)
+    uint32 block_size = 2;                  // bytes per checksum block; default 16777216 (16 MiB), a power-of-two multiple of 1 MiB
+    uint32 generation = 3;                  // EC vacuum generation these checksums describe (0 = legacy/fresh); must match the sidecar filename version
+    EcShardConfig ec_shard_config = 4;      // data/parity shard counts at encode time
+    repeated EcShardChecksums shards = 5;   // one entry per shard id in the active layout
+    bytes encode_uuid = 6;                  // random per-encode identity, for stale-sidecar detection across in-place re-encodes
+}
+
+message EcShardChecksums {
+    uint32 shard_id = 1;     // 0..MaxShardCount-1 (custom EC ratios go up to 32)
+    int64 covered_size = 2;  // shard byte length these checksums cover (must equal the on-disk shard length)
+    bytes block_crc32c = 3;  // packed little-endian uint32[] = ceil(covered_size/block_size) entries
+}
+
+enum ChecksumAlgorithm {
+    CHECKSUM_NONE = 0;
+    CHECKSUM_CRC32C = 1;
+}
+message OldVersionVolumeInfo {
+    repeated RemoteFile files = 1;
+    uint32 version = 2;
+    string replication = 3;
+    uint32 BytesOffset = 4;
+    int64 dat_file_size = 5; // store the original dat file size
+    uint64 DestroyTime = 6; // expiration time of ec volume
+    bool read_only = 7;
+}
+
+// tiered storage
+message VolumeTierMoveDatToRemoteRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    string destination_backend_name = 3;
+    bool keep_local_dat_file = 4;
+    int32 concurrency = 5; // multipart upload concurrency, 0 = backend default (5 for S3)
+}
+message VolumeTierMoveDatToRemoteResponse {
+    int64 processed = 1;
+    float processedPercentage = 2;
+}
+
+message VolumeTierMoveDatFromRemoteRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    bool keep_remote_dat_file = 3;
+    int32 concurrency = 4; // multipart download concurrency, 0 = backend default (5 for S3)
+}
+message VolumeTierMoveDatFromRemoteResponse {
+    int64 processed = 1;
+    float processedPercentage = 2;
+}
+
+message VolumeServerStatusRequest {
+
+}
+message VolumeServerStatusResponse {
+    repeated DiskStatus disk_statuses = 1;
+    MemStatus memory_status = 2;
+    string version = 3;
+    string data_center = 4;
+    string rack = 5;
+    VolumeServerState state = 6;
+}
+
+message VolumeServerLeaveRequest {
+}
+message VolumeServerLeaveResponse {
+}
+
+// remote storage
+message FetchAndWriteNeedleRequest {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+    uint32 cookie = 3;
+    int64 offset = 4;
+    int64 size = 5;
+    message Replica {
+        string url = 1;
+        string public_url = 2;
+        int32 grpc_port = 3;
+    }
+    repeated Replica replicas = 6;
+    string auth = 7;
+    int32 download_concurrency = 8; // multipart download concurrency if supported by the remote storage client; for S3, 0 = default (5)
+    // remote conf
+    remote_pb.RemoteConf remote_conf = 15;
+    remote_pb.RemoteStorageLocation remote_location = 16;
+}
+message FetchAndWriteNeedleResponse {
+    string e_tag = 1;
+}
+
+enum VolumeScrubMode {
+    UNKNOWN = 0;
+    INDEX = 1;
+    FULL = 2;
+    LOCAL = 3;
+    CHECKSUM = 4; // EC only: verify each local shard's raw bytes against the bitrot checksum sidecar
+    READS = 5;    // like FULL, but EC intervals no shard can serve are reconstructed from parity
+}
+
+message ScrubVolumeRequest {
+    VolumeScrubMode mode = 1;
+    // optional list of volume IDs to scrub. if empty, all volumes for the server are scrubbed.
+    repeated uint32 volume_ids = 2;
+    bool mark_broken_volumes_readonly = 3;
+}
+message ScrubVolumeResponse {
+    uint64 total_volumes = 1;
+    uint64 total_files = 2;
+    repeated uint32 broken_volume_ids = 3;
+    repeated string details = 4;
+}
+
+message ScrubEcVolumeRequest {
+    VolumeScrubMode mode = 1;
+    // optional list of volume IDs to scrub. if empty, all EC volumes for the server are scrubbed.
+    repeated uint32 volume_ids = 2;
+    bool force_deleted_needles_check = 3; // FULL mode only; may report false positives when EC indexes disagree
+}
+message ScrubEcVolumeResponse {
+    uint64 total_volumes = 1;
+    uint64 total_files = 2;
+    repeated uint32 broken_volume_ids = 3;
+    repeated EcShardInfo broken_shard_infos = 4;
+    repeated string details = 5;
+}
+
+// select on volume servers
+message QueryRequest {
+    repeated string selections = 1;
+    repeated string from_file_ids = 2;
+    message Filter {
+        string field = 1;
+        string operand = 2;
+        string value = 3;
+    }
+    Filter filter = 3;
+
+    message InputSerialization {
+        // NONE | GZIP | BZIP2
+        string compression_type = 1;
+        message CSVInput {
+            string file_header_info = 1; // Valid values: NONE | USE | IGNORE
+            string record_delimiter = 2; // Default: \n
+            string field_delimiter = 3; // Default: ,
+            string quote_character = 4; // Default: "
+            string quote_escape_character = 5; // Default: "
+            string comments = 6; // Default: #
+            // If true, records might contain record delimiters within quote characters
+            bool allow_quoted_record_delimiter = 7; // default False.
+        }
+        message JSONInput {
+            string type = 1; // Valid values: DOCUMENT | LINES
+        }
+        message ParquetInput {
+        }
+
+        CSVInput csv_input = 2;
+        JSONInput json_input = 3;
+        ParquetInput parquet_input = 4;
+    }
+    InputSerialization input_serialization = 4;
+
+    message OutputSerialization {
+        message CSVOutput {
+            string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED
+            string record_delimiter = 2; // Default: \n
+            string field_delimiter = 3; // Default: ,
+            string quote_character = 4; // Default: "
+            string quote_escape_character = 5; // Default: "
+        }
+        message JSONOutput {
+            string record_delimiter = 1;
+        }
+
+        CSVOutput csv_output = 2;
+        JSONOutput json_output = 3;
+    }
+
+    OutputSerialization output_serialization = 5;
+}
+message QueriedStripe {
+    bytes records = 1;
+}
+
+message VolumeNeedleStatusRequest {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+}
+message VolumeNeedleStatusResponse {
+    uint64 needle_id = 1;
+    uint32 cookie = 2;
+    uint32 size = 3;
+    uint64 last_modified = 4;
+    uint32 crc = 5;
+    string ttl = 6;
+}
+
+message PingRequest {
+    string target = 1; // default to ping itself
+    string target_type = 2;
+}
+message PingResponse {
+    int64 start_time_ns = 1;
+    int64 remote_time_ns = 2;
+    int64 stop_time_ns = 3;
+}

weed/pb/volume_server_pb/volume_server_helper.go

@@ -0,0 +1,5 @@
+package volume_server_pb
+
+func (m *RemoteFile) BackendName() string {
+	return m.BackendType + "." + m.BackendId
+}

weed/pb/worker.proto

@@ -0,0 +1,501 @@
+syntax = "proto3";
+
+package worker_pb;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/worker_pb";
+
+// WorkerService provides bidirectional communication between admin and worker
+service WorkerService {
+  // WorkerStream maintains a bidirectional stream for worker communication
+  rpc WorkerStream(stream WorkerMessage) returns (stream AdminMessage);
+}
+
+// WorkerMessage represents messages from worker to admin
+message WorkerMessage {
+  string worker_id = 1;
+  int64 timestamp = 2;
+  
+  oneof message {
+    WorkerRegistration registration = 3;
+    WorkerHeartbeat heartbeat = 4;
+    TaskRequest task_request = 5;
+    TaskUpdate task_update = 6;
+    TaskComplete task_complete = 7;
+    WorkerShutdown shutdown = 8;
+    TaskLogResponse task_log_response = 9;
+  }
+}
+
+// AdminMessage represents messages from admin to worker
+message AdminMessage {
+  string admin_id = 1;
+  int64 timestamp = 2;
+  
+  oneof message {
+    RegistrationResponse registration_response = 3;
+    HeartbeatResponse heartbeat_response = 4;
+    TaskAssignment task_assignment = 5;
+    TaskCancellation task_cancellation = 6;
+    AdminShutdown admin_shutdown = 7;
+    TaskLogRequest task_log_request = 8;
+  }
+}
+
+// WorkerRegistration message when worker connects
+message WorkerRegistration {
+  string worker_id = 1;
+  string address = 2;
+  repeated string capabilities = 3;
+  int32 max_concurrent = 4;
+  map<string, string> metadata = 5;
+}
+
+// RegistrationResponse confirms worker registration
+message RegistrationResponse {
+  bool success = 1;
+  string message = 2;
+  string assigned_worker_id = 3;
+}
+
+// WorkerHeartbeat sent periodically by worker
+message WorkerHeartbeat {
+  string worker_id = 1;
+  string status = 2;
+  int32 current_load = 3;
+  int32 max_concurrent = 4;
+  repeated string current_task_ids = 5;
+  int32 tasks_completed = 6;
+  int32 tasks_failed = 7;
+  int64 uptime_seconds = 8;
+}
+
+// HeartbeatResponse acknowledges heartbeat
+message HeartbeatResponse {
+  bool success = 1;
+  string message = 2;
+}
+
+// TaskRequest from worker asking for new tasks
+message TaskRequest {
+  string worker_id = 1;
+  repeated string capabilities = 2;
+  int32 available_slots = 3;
+}
+
+// TaskAssignment from admin to worker
+message TaskAssignment {
+  string task_id = 1;
+  string task_type = 2;
+  TaskParams params = 3;
+  int32 priority = 4;
+  int64 created_time = 5;
+  map<string, string> metadata = 6;
+}
+
+// TaskParams contains task-specific parameters with typed variants
+message TaskParams {
+  string task_id = 1;                   // ActiveTopology task ID for lifecycle management
+  uint32 volume_id = 2;                 // Primary volume ID for the task
+  string collection = 3;                // Collection name
+  string data_center = 4;               // Primary data center
+  string rack = 5;                      // Primary rack
+  uint64 volume_size = 6;               // Original volume size in bytes for tracking size changes
+  
+  // Unified source and target arrays for all task types
+  repeated TaskSource sources = 7;      // Source locations (volume replicas, EC shards, etc.)
+  repeated TaskTarget targets = 8;      // Target locations (destinations, new replicas, etc.)
+  
+  // Typed task parameters
+  oneof task_params {
+    VacuumTaskParams vacuum_params = 9;
+    ErasureCodingTaskParams erasure_coding_params = 10;
+    BalanceTaskParams balance_params = 11;
+    ReplicationTaskParams replication_params = 12;
+    EcBalanceTaskParams ec_balance_params = 13;
+    S3LifecycleParams s3_lifecycle_params = 14;
+  }
+}
+
+// S3LifecycleParams routes a worker task to one of the three lifecycle
+// subroutines. READ is the per-shard meta-log reader (one task per shard_id
+// at a time); BOOTSTRAP walks a single bucket; DRAIN drains pending
+// exceptions for a single rule.
+message S3LifecycleParams {
+  // Zero is an UNSPECIFIED sentinel: a TaskParams payload whose subtype is
+  // unset must not silently route into a READ task. Callers always populate
+  // one of READ / BOOTSTRAP / DRAIN.
+  enum Subtype {
+    SUBTYPE_UNSPECIFIED = 0;
+    READ                = 1;
+    BOOTSTRAP           = 2;
+    DRAIN               = 3;
+  }
+  Subtype subtype = 1;
+
+  // Required for BOOTSTRAP and DRAIN; ignored for READ. rule_hash is
+  // optional for BOOTSTRAP (omitting walks all rules for the bucket).
+  string bucket    = 2;
+  bytes  rule_hash = 3; // 8 bytes when present
+
+  bool   force                 = 4; // operator override; bypasses scheduling guards
+  int64  batch_time_budget_ns  = 5; // 0 = use default
+  int32  batch_event_budget    = 6; // 0 = use default
+
+  ContinuationHint continuation = 7; // resume hint for kill-resume tasks
+
+  // READ only: which (bucket, key-prefix-hash) shard this task processes.
+  // Range 0..S3LifecycleShardCount-1 (16 shards). Required for READ; ignored
+  // for BOOTSTRAP and DRAIN. Workers receive one READ task per owned shard;
+  // shards distribute across workers via the existing scheduler.
+  int32 shard_id = 8;
+}
+
+// ContinuationHint lets a long-running BOOTSTRAP or READ task hand its
+// resume point to the next scheduled invocation without going through
+// durable state. The durable state files are still authoritative; this
+// is just a scheduling hint to skip an unnecessary fresh start.
+message ContinuationHint {
+  string last_scanned_path = 1; // BOOTSTRAP only
+  int64  last_position_ns  = 2; // READ only (advisory)
+}
+
+// VacuumTaskParams for vacuum operations
+message VacuumTaskParams {
+  double garbage_threshold = 1;           // Minimum garbage ratio to trigger vacuum
+  bool force_vacuum = 2;                  // Force vacuum even if below threshold
+  int32 batch_size = 3;                   // Number of files to process per batch
+  string working_dir = 4;                 // Working directory for temporary files
+  bool verify_checksum = 5;               // Verify file checksums during vacuum
+}
+
+// ErasureCodingTaskParams for EC encoding operations
+message ErasureCodingTaskParams {
+  reserved 7;
+  uint64 estimated_shard_size = 1;        // Estimated size per shard
+  int32 data_shards = 2;                  // Number of data shards (default: 10)
+  int32 parity_shards = 3;                // Number of parity shards (default: 4)
+  string working_dir = 4;                 // Working directory for EC processing
+  string master_client = 5;               // Master server address
+  bool cleanup_source = 6;                // Whether to cleanup source volume after EC
+  string source_disk_type = 8;            // Source volume's disk type, passed to VolumeEcShardsMount so shards report under it (#9423)
+  int64 encode_ts_ns = 9;                 // admin-issued encode generation (unix nanos); fences a stale worker's shard cleanup against a newer run. 0 => unfenced (legacy/shell), falls back to a blanket teardown.
+}
+
+// TaskSource represents a unified source location for any task type
+message TaskSource {
+  string node = 1;                        // Source server address
+  uint32 disk_id = 2;                     // Source disk ID
+  string rack = 3;                        // Source rack for tracking
+  string data_center = 4;                 // Source data center for tracking
+  uint32 volume_id = 5;                   // Volume ID (for volume operations)
+  repeated uint32 shard_ids = 6;          // Shard IDs (for EC shard operations)
+  uint64 estimated_size = 7;              // Estimated size to be processed
+}
+
+// TaskTarget represents a unified target location for any task type
+message TaskTarget {
+  string node = 1;                        // Target server address
+  uint32 disk_id = 2;                     // Target disk ID  
+  string rack = 3;                        // Target rack for tracking
+  string data_center = 4;                 // Target data center for tracking
+  uint32 volume_id = 5;                   // Volume ID (for volume operations)
+  repeated uint32 shard_ids = 6;          // Shard IDs (for EC shard operations)
+  uint64 estimated_size = 7;              // Estimated size to be created
+}
+
+
+
+// BalanceMoveSpec describes a single volume move within a batch balance job
+message BalanceMoveSpec {
+  uint32 volume_id = 1;                   // Volume to move
+  string source_node = 2;                 // Source server address (host:port)
+  string target_node = 3;                 // Destination server address (host:port)
+  string collection = 4;                  // Collection name
+  uint64 volume_size = 5;                 // Volume size in bytes (informational)
+}
+
+// BalanceTaskParams for volume balancing operations
+message BalanceTaskParams {
+  bool force_move = 1;                    // Force move even with conflicts
+  int32 timeout_seconds = 2;              // Operation timeout
+  int32 max_concurrent_moves = 3;         // Max concurrent moves in a batch job (0 = default 5)
+  repeated BalanceMoveSpec moves = 4;     // Batch: multiple volume moves in one job
+  int64 io_byte_per_second = 5;           // limit each move's copy rate; 0 falls back to the volume server's maintenance rate
+}
+
+// ReplicationTaskParams for adding replicas
+message ReplicationTaskParams {
+  int32 replica_count = 1;                // Target replica count
+  bool verify_consistency = 2;            // Verify replica consistency after creation
+}
+
+// TaskUpdate reports task progress
+message TaskUpdate {
+  string task_id = 1;
+  string worker_id = 2;
+  string status = 3;
+  float progress = 4;
+  string message = 5;
+  map<string, string> metadata = 6;
+}
+
+// TaskComplete reports task completion
+message TaskComplete {
+  string task_id = 1;
+  string worker_id = 2;
+  bool success = 3;
+  string error_message = 4;
+  int64 completion_time = 5;
+  map<string, string> result_metadata = 6;
+}
+
+// TaskCancellation from admin to cancel a task
+message TaskCancellation {
+  string task_id = 1;
+  string reason = 2;
+  bool force = 3;
+}
+
+// WorkerShutdown notifies admin that worker is shutting down
+message WorkerShutdown {
+  string worker_id = 1;
+  string reason = 2;
+  repeated string pending_task_ids = 3;
+}
+
+// AdminShutdown notifies worker that admin is shutting down
+message AdminShutdown {
+  string reason = 1;
+  int32 graceful_shutdown_seconds = 2;
+}
+
+// ========== Task Log Messages ==========
+
+// TaskLogRequest requests logs for a specific task
+message TaskLogRequest {
+  string task_id = 1;
+  string worker_id = 2;
+  bool include_metadata = 3;  // Include task metadata
+  int32 max_entries = 4;      // Maximum number of log entries (0 = all)
+  string log_level = 5;       // Filter by log level (INFO, WARNING, ERROR, DEBUG)
+  int64 start_time = 6;       // Unix timestamp for start time filter
+  int64 end_time = 7;         // Unix timestamp for end time filter
+}
+
+// TaskLogResponse returns task logs and metadata
+message TaskLogResponse {
+  string task_id = 1;
+  string worker_id = 2;
+  bool success = 3;
+  string error_message = 4;
+  TaskLogMetadata metadata = 5;
+  repeated TaskLogEntry log_entries = 6;
+}
+
+// TaskLogMetadata contains metadata about task execution
+message TaskLogMetadata {
+  string task_id = 1;
+  string task_type = 2;
+  string worker_id = 3;
+  int64 start_time = 4;
+  int64 end_time = 5;
+  int64 duration_ms = 6;
+  string status = 7;
+  float progress = 8;
+  uint32 volume_id = 9;
+  string server = 10;
+  string collection = 11;
+  string log_file_path = 12;
+  int64 created_at = 13;
+  map<string, string> custom_data = 14;
+}
+
+// TaskLogEntry represents a single log entry
+message TaskLogEntry {
+  int64 timestamp = 1;
+  string level = 2;
+  string message = 3;
+  map<string, string> fields = 4;
+  float progress = 5;
+  string status = 6;
+}
+
+// ========== Maintenance Configuration Messages ==========
+
+// MaintenanceConfig holds configuration for the maintenance system
+message MaintenanceConfig {
+  // optional so a file that predates presence tracking (where an operator's
+  // explicit false and the field's absence are indistinguishable on the wire)
+  // can be told apart from one that explicitly persists the toggle
+  optional bool enabled = 1;
+  int32 scan_interval_seconds = 2;    // How often to scan for maintenance needs
+  int32 worker_timeout_seconds = 3;   // Worker heartbeat timeout
+  int32 task_timeout_seconds = 4;     // Individual task timeout
+  int32 retry_delay_seconds = 5;      // Delay between retries
+  int32 max_retries = 6;              // Default max retries for tasks
+  int32 cleanup_interval_seconds = 7; // How often to clean up old tasks
+  int32 task_retention_seconds = 8;   // How long to keep completed/failed tasks
+  MaintenancePolicy policy = 9;
+}
+
+// MaintenancePolicy defines policies for maintenance operations
+message MaintenancePolicy {
+  map<string, TaskPolicy> task_policies = 1;  // Task type -> policy mapping
+  int32 global_max_concurrent = 2;             // Overall limit across all task types
+  int32 default_repeat_interval_seconds = 3;   // Default seconds if task doesn't specify
+  int32 default_check_interval_seconds = 4;    // Default seconds for periodic checks
+}
+
+// TaskPolicy represents configuration for a specific task type
+message TaskPolicy {
+  bool enabled = 1;
+  int32 max_concurrent = 2;
+  int32 repeat_interval_seconds = 3;  // Seconds to wait before repeating
+  int32 check_interval_seconds = 4;   // Seconds between checks
+  
+  // Typed task-specific configuration (replaces generic map)
+  oneof task_config {
+    VacuumTaskConfig vacuum_config = 5;
+    ErasureCodingTaskConfig erasure_coding_config = 6;
+    BalanceTaskConfig balance_config = 7;
+    ReplicationTaskConfig replication_config = 8;
+    EcBalanceTaskConfig ec_balance_config = 9;
+  }
+}
+
+// Task-specific configuration messages
+
+// VacuumTaskConfig contains vacuum-specific configuration
+message VacuumTaskConfig {
+  double garbage_threshold = 1;     // Minimum garbage ratio to trigger vacuum (0.0-1.0)
+  int32 min_volume_age_hours = 2;   // Minimum age before vacuum is considered
+  int32 min_interval_seconds = 3;   // Minimum time between vacuum operations on the same volume
+}
+
+// ErasureCodingTaskConfig contains EC-specific configuration
+message ErasureCodingTaskConfig {
+  double fullness_ratio = 1;        // Minimum fullness ratio to trigger EC (0.0-1.0)
+  int32 quiet_for_seconds = 2;      // Minimum quiet time before EC
+  int32 min_volume_size_mb = 3;     // Minimum volume size for EC
+  string collection_filter = 4;     // Only process volumes from specific collections
+  repeated string preferred_tags = 5; // Disk tags to prioritize for EC shard placement
+  string replica_placement = 6;     // EC shard replica placement (e.g. "020"); empty falls back to master default replication
+}
+
+// BalanceTaskConfig contains balance-specific configuration
+message BalanceTaskConfig {
+  double imbalance_threshold = 1;   // Threshold for triggering rebalancing (0.0-1.0)
+  int32 min_server_count = 2;       // Minimum number of servers required for balancing
+  int64 io_byte_per_second = 3;     // limit each move's copy rate; 0 falls back to the volume server's maintenance rate
+}
+
+// ReplicationTaskConfig contains replication-specific configuration
+message ReplicationTaskConfig {
+  int32 target_replica_count = 1;   // Target number of replicas
+}
+
+// EcBalanceTaskParams for EC shard balancing operations
+message EcBalanceTaskParams {
+  string disk_type = 1;                    // Disk type filter (hdd, ssd, "")
+  int32 max_parallelization = 2;           // Max parallel shard moves within a batch
+  int32 timeout_seconds = 3;               // Operation timeout per move
+  repeated EcShardMoveSpec moves = 4;      // Batch: multiple shard moves in one job
+  // For a dedup move, the node that keeps the shard. The worker confirms this
+  // node really holds it before deleting the copy, so a topology entry naming a
+  // location that holds nothing cannot cause the last copy to be removed.
+  string dedup_keep_node = 5;
+  int64 io_byte_per_second = 6;            // limit each shard copy's rate; 0 falls back to the volume server's maintenance rate
+}
+
+// EcShardMoveSpec describes a single EC shard move within a batch
+message EcShardMoveSpec {
+  uint32 volume_id = 1;                    // EC volume ID
+  uint32 shard_id = 2;                     // Shard ID (0-13)
+  string collection = 3;                   // Collection name
+  string source_node = 4;                  // Source server address
+  uint32 source_disk_id = 5;              // Source disk ID
+  string target_node = 6;                  // Target server address
+  uint32 target_disk_id = 7;              // Target disk ID
+}
+
+// EcBalanceTaskConfig contains EC balance-specific configuration
+message EcBalanceTaskConfig {
+  double imbalance_threshold = 1;         // Threshold for triggering EC shard rebalancing
+  int32 min_server_count = 2;             // Minimum number of servers required
+  string collection_filter = 3;           // Collection filter
+  string disk_type = 4;                   // Disk type filter
+  repeated string preferred_tags = 5;     // Preferred disk tags for placement
+  string replica_placement = 6;           // EC shard replica placement (e.g. "020"); empty falls back to master default replication
+  int64 io_byte_per_second = 7;           // limit each shard copy's rate; 0 falls back to the volume server's maintenance rate
+}
+
+// ========== Task Persistence Messages ==========
+
+// MaintenanceTaskData represents complete task state for persistence
+message MaintenanceTaskData {
+  string id = 1;
+  string type = 2;
+  string priority = 3;
+  string status = 4;
+  uint32 volume_id = 5;
+  string server = 6;
+  string collection = 7;
+  TaskParams typed_params = 8;
+  string reason = 9;
+  int64 created_at = 10;
+  int64 scheduled_at = 11;
+  int64 started_at = 12;
+  int64 completed_at = 13;
+  string worker_id = 14;
+  string error = 15;
+  double progress = 16;
+  int32 retry_count = 17;
+  int32 max_retries = 18;
+
+  // Enhanced fields for detailed task tracking
+  string created_by = 19;
+  string creation_context = 20;
+  repeated TaskAssignmentRecord assignment_history = 21;
+  string detailed_reason = 22;
+  map<string, string> tags = 23;
+  TaskCreationMetrics creation_metrics = 24;
+}
+
+// TaskAssignmentRecord tracks worker assignments for a task
+message TaskAssignmentRecord {
+  string worker_id = 1;
+  string worker_address = 2;
+  int64 assigned_at = 3;
+  int64 unassigned_at = 4;  // Optional: when worker was unassigned
+  string reason = 5;        // Reason for assignment/unassignment
+}
+
+// TaskCreationMetrics tracks why and how a task was created
+message TaskCreationMetrics {
+  string trigger_metric = 1;           // Name of metric that triggered creation
+  double metric_value = 2;             // Value that triggered creation
+  double threshold = 3;                // Threshold that was exceeded
+  VolumeHealthMetrics volume_metrics = 4; // Volume health at creation time
+  map<string, string> additional_data = 5; // Additional context data
+}
+
+// VolumeHealthMetrics captures volume state at task creation
+message VolumeHealthMetrics {
+  uint64 total_size = 1;
+  uint64 used_size = 2;
+  uint64 garbage_size = 3;
+  double garbage_ratio = 4;
+  int32 file_count = 5;
+  int32 deleted_file_count = 6;
+  int64 last_modified = 7;
+  int32 replica_count = 8;
+  bool is_ec_volume = 9;
+  string collection = 10;
+}
+
+// TaskStateFile wraps task data with metadata for persistence
+message TaskStateFile {
+  MaintenanceTaskData task = 1;
+  int64 last_updated = 2;
+  string admin_version = 3;
+} 

weed/pb/worker_pb/worker_grpc.pb.go

@@ -0,0 +1,121 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.6.2
+// - protoc             v7.35.0
+// source: worker.proto
+
+package worker_pb
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.64.0 or later.
+const _ = grpc.SupportPackageIsVersion9
+
+const (
+	WorkerService_WorkerStream_FullMethodName = "/worker_pb.WorkerService/WorkerStream"
+)
+
+// WorkerServiceClient is the client API for WorkerService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+//
+// WorkerService provides bidirectional communication between admin and worker
+type WorkerServiceClient interface {
+	// WorkerStream maintains a bidirectional stream for worker communication
+	WorkerStream(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[WorkerMessage, AdminMessage], error)
+}
+
+type workerServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewWorkerServiceClient(cc grpc.ClientConnInterface) WorkerServiceClient {
+	return &workerServiceClient{cc}
+}
+
+func (c *workerServiceClient) WorkerStream(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[WorkerMessage, AdminMessage], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &WorkerService_ServiceDesc.Streams[0], WorkerService_WorkerStream_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[WorkerMessage, AdminMessage]{ClientStream: stream}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type WorkerService_WorkerStreamClient = grpc.BidiStreamingClient[WorkerMessage, AdminMessage]
+
+// WorkerServiceServer is the server API for WorkerService service.
+// All implementations must embed UnimplementedWorkerServiceServer
+// for forward compatibility.
+//
+// WorkerService provides bidirectional communication between admin and worker
+type WorkerServiceServer interface {
+	// WorkerStream maintains a bidirectional stream for worker communication
+	WorkerStream(grpc.BidiStreamingServer[WorkerMessage, AdminMessage]) error
+	mustEmbedUnimplementedWorkerServiceServer()
+}
+
+// UnimplementedWorkerServiceServer must be embedded to have
+// forward compatible implementations.
+//
+// NOTE: this should be embedded by value instead of pointer to avoid a nil
+// pointer dereference when methods are called.
+type UnimplementedWorkerServiceServer struct{}
+
+func (UnimplementedWorkerServiceServer) WorkerStream(grpc.BidiStreamingServer[WorkerMessage, AdminMessage]) error {
+	return status.Error(codes.Unimplemented, "method WorkerStream not implemented")
+}
+func (UnimplementedWorkerServiceServer) mustEmbedUnimplementedWorkerServiceServer() {}
+func (UnimplementedWorkerServiceServer) testEmbeddedByValue()                       {}
+
+// UnsafeWorkerServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to WorkerServiceServer will
+// result in compilation errors.
+type UnsafeWorkerServiceServer interface {
+	mustEmbedUnimplementedWorkerServiceServer()
+}
+
+func RegisterWorkerServiceServer(s grpc.ServiceRegistrar, srv WorkerServiceServer) {
+	// If the following call panics, it indicates UnimplementedWorkerServiceServer was
+	// embedded by pointer and is nil.  This will cause panics if an
+	// unimplemented method is ever invoked, so we test this at initialization
+	// time to prevent it from happening at runtime later due to I/O.
+	if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
+		t.testEmbeddedByValue()
+	}
+	s.RegisterService(&WorkerService_ServiceDesc, srv)
+}
+
+func _WorkerService_WorkerStream_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(WorkerServiceServer).WorkerStream(&grpc.GenericServerStream[WorkerMessage, AdminMessage]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type WorkerService_WorkerStreamServer = grpc.BidiStreamingServer[WorkerMessage, AdminMessage]
+
+// WorkerService_ServiceDesc is the grpc.ServiceDesc for WorkerService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var WorkerService_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "worker_pb.WorkerService",
+	HandlerType: (*WorkerServiceServer)(nil),
+	Methods:     []grpc.MethodDesc{},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "WorkerStream",
+			Handler:       _WorkerService_WorkerStream_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "worker.proto",
+}