snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/mount/peer_directory.go

@@ -0,0 +1,283 @@
+package mount
+
+import (
+	"sort"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// maxPeerDirectoryEntries caps the total (fid, holder) edges a single mount
+// will hold in its shard of the chunk directory. Each edge is a
+// (uint32, time.Time) map entry (~40 B with Go's map overhead), plus the
+// per-peer metadata is interned — see peerMetadata below — so 500k edges
+// come out to roughly 20 MB regardless of fleet size. A full directory
+// rejects new announces until TTL frees room.
+const maxPeerDirectoryEntries = 500000
+
+// peerID is the interned identifier for a peer. Peer addresses and locality
+// labels live once in the peers slice; the (fid -> holder) index references
+// peers by this fixed-size id so we avoid paying for the ~48 B worth of
+// string headers (addr, dc, rack) on every edge.
+type peerID uint32
+
+// peerMetadata is the per-peer data that used to be inlined on every
+// holderEntry. Fields are updated in place on each announce so a peer
+// reconfiguring its dc/rack is picked up without churn.
+type peerMetadata struct {
+	addr       string
+	dataCenter string
+	rack       string
+}
+
+// PeerDirectory is the tier-2 chunk directory: an in-memory
+// `fid -> set<peer>` map, holding entries only for fids for which this
+// mount is the HRW-assigned owner. Populated by inbound ChunkAnnounce RPCs,
+// queried by inbound ChunkLookup RPCs. Entries expire on TTL.
+//
+// Peer addresses are interned into peerID. That way the hot index stores
+// `fid -> (peerID -> expiry)`, a map of two fixed-size values, instead of
+// a full (addr, dc, rack, expiry) struct per edge. The peer slice itself
+// is bounded by fleet size (typically a few hundred), so the interning
+// table's memory cost is negligible next to the per-edge savings.
+//
+// Lookup takes only a read lock; TTL eviction is the sole responsibility of
+// the Sweep goroutine, which runs on a periodic ticker. This keeps the hot
+// read path concurrent even under heavy announce traffic.
+//
+// See design-weed-mount-peer-chunk-sharing.md §4.2.2.
+type PeerDirectory struct {
+	mu sync.RWMutex
+
+	// Interning table. peers[id] gives the peer's metadata; peerIDByAddr
+	// maps an address back to its id when a new announce comes in.
+	peers        []peerMetadata
+	peerIDByAddr map[string]peerID
+
+	// fid -> (peerID -> expiry). The inner map's value is a bare timestamp
+	// — metadata is looked up via peers[id] on demand.
+	index map[string]map[peerID]time.Time
+
+	// entryCount is maintained in lock-step with map mutations (under
+	// mu), so Stats() can report O(1) instead of scanning the index.
+	entryCount int
+
+	announces atomic.Int64
+	lookups   atomic.Int64
+	rejected  atomic.Int64
+	evictions atomic.Int64
+
+	clock func() time.Time
+}
+
+// NewPeerDirectory constructs a directory using the real wall clock.
+func NewPeerDirectory() *PeerDirectory {
+	return newPeerDirectoryWithClock(time.Now)
+}
+
+func newPeerDirectoryWithClock(clock func() time.Time) *PeerDirectory {
+	return &PeerDirectory{
+		index:        map[string]map[peerID]time.Time{},
+		peerIDByAddr: map[string]peerID{},
+		clock:        clock,
+	}
+}
+
+// AnnounceResult reports how an inbound announce was handled per-fid: fids
+// this receiver owns were recorded; others were rejected so the caller can
+// retry the correct owner after refreshing its seed view.
+type AnnounceResult struct {
+	Rejected []string
+}
+
+// OwnerCheck is a predicate provided by the caller (backed by HRW on the
+// current seed list) that says whether self is the owner for a given fid.
+type OwnerCheck func(fid string) bool
+
+// internPeer returns the interned id for peerAddr, allocating a new slot
+// if this is the first announce from that peer. Must be called under mu
+// (write). dc/rack are refreshed on each call so reconfig is picked up.
+func (d *PeerDirectory) internPeer(addr, dc, rack string) peerID {
+	if id, ok := d.peerIDByAddr[addr]; ok {
+		p := &d.peers[id]
+		p.dataCenter = dc
+		p.rack = rack
+		return id
+	}
+	id := peerID(len(d.peers))
+	d.peers = append(d.peers, peerMetadata{addr: addr, dataCenter: dc, rack: rack})
+	d.peerIDByAddr[addr] = id
+	return id
+}
+
+// Announce records the caller as a holder for every accepted fid. Any fid
+// for which ownerCheck returns false is rejected and surfaced back. A
+// directory at maxPeerDirectoryEntries silently drops *new* edges
+// (renewals of existing edges continue to succeed).
+func (d *PeerDirectory) Announce(peerAddr, dataCenter, rack string, fids []string, ttl time.Duration, ownerCheck OwnerCheck) AnnounceResult {
+	if ttl <= 0 {
+		ttl = 60 * time.Second
+	}
+	expiry := d.clock().Add(ttl)
+
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	pid := d.internPeer(peerAddr, dataCenter, rack)
+
+	result := AnnounceResult{}
+	for _, fid := range fids {
+		if ownerCheck != nil && !ownerCheck(fid) {
+			result.Rejected = append(result.Rejected, fid)
+			d.rejected.Add(1)
+			continue
+		}
+		holders, ok := d.index[fid]
+		if !ok {
+			if d.entryCount >= maxPeerDirectoryEntries {
+				continue
+			}
+			holders = map[peerID]time.Time{}
+			d.index[fid] = holders
+		}
+		if _, exists := holders[pid]; !exists {
+			if d.entryCount >= maxPeerDirectoryEntries {
+				continue
+			}
+			d.entryCount++
+		}
+		holders[pid] = expiry
+		d.announces.Add(1)
+	}
+	return result
+}
+
+// LookupHolder is the public form of a holder entry. DataCenter and Rack
+// are carried through so the fetcher can re-sort by its own locality
+// before picking a peer to dial.
+type LookupHolder struct {
+	PeerAddr   string
+	DataCenter string
+	Rack       string
+}
+
+// maxLookupHolders caps how many holders a single ChunkLookup response
+// returns per fid. The directory stores a holder per mount caching the
+// fid, which on a large hot file can reach fleet size (hundreds). The
+// fetcher typically only tries the first 1-3 before succeeding, so
+// returning every entry is pure wire-overhead. LRU ordering ensures the
+// freshest holders are the ones kept.
+const maxLookupHolders = 16
+
+// LookupResult holds the merged lookup response. Fids for which this
+// receiver is not the owner are split out so the caller can retry.
+type LookupResult struct {
+	PeersByFid   map[string][]LookupHolder
+	NotOwnerFids []string
+}
+
+// Lookup returns known holders for the requested fids. Expired edges are
+// filtered out of the response but NOT deleted here — Sweep handles that
+// under a write lock on its own schedule, so Lookup can stay read-only and
+// concurrent. A fid owned by this receiver with no known holders returns
+// an empty peer list.
+//
+// Holders are returned in LRU order (most recently announced first). Since
+// each Announce stamps a fresh `now + ttl` expiry, sorting by expiry
+// descending is equivalent to ordering by last-seen descending. Fetchers
+// should try these in the returned order — the most recent announcer is
+// the holder most likely to still have the chunk cached, given that each
+// mount's chunk_cache is itself LRU.
+func (d *PeerDirectory) Lookup(fids []string, ownerCheck OwnerCheck) LookupResult {
+	now := d.clock()
+	result := LookupResult{
+		PeersByFid: map[string][]LookupHolder{},
+	}
+
+	// Collect candidate holders under the read lock, including the peer
+	// metadata copies — Announce mutates peers[id] in place when an
+	// existing peer re-announces, so reading peers[pid] outside the lock
+	// would race with that write. Sorting happens after the unlock.
+	type candidate struct {
+		peerMetadata
+		expiry time.Time
+	}
+	perFid := make(map[string][]candidate, len(fids))
+
+	d.mu.RLock()
+	for _, fid := range fids {
+		d.lookups.Add(1)
+		if ownerCheck != nil && !ownerCheck(fid) {
+			result.NotOwnerFids = append(result.NotOwnerFids, fid)
+			continue
+		}
+		holders, ok := d.index[fid]
+		if !ok {
+			result.PeersByFid[fid] = nil
+			continue
+		}
+		cands := make([]candidate, 0, len(holders))
+		for pid, expiry := range holders {
+			if !expiry.After(now) {
+				continue // Sweep will clean this up
+			}
+			cands = append(cands, candidate{peerMetadata: d.peers[pid], expiry: expiry})
+		}
+		perFid[fid] = cands
+	}
+	d.mu.RUnlock()
+
+	for fid, cands := range perFid {
+		sort.Slice(cands, func(i, j int) bool {
+			if !cands[i].expiry.Equal(cands[j].expiry) {
+				return cands[i].expiry.After(cands[j].expiry) // newest first
+			}
+			return cands[i].addr < cands[j].addr // deterministic tiebreak
+		})
+		// Cap response at maxLookupHolders. The fetcher typically tries 1-3
+		// holders before succeeding; shipping every holder for hot fids is
+		// wire-overhead without hit-rate gain.
+		if len(cands) > maxLookupHolders {
+			cands = cands[:maxLookupHolders]
+		}
+		out := make([]LookupHolder, len(cands))
+		for i, c := range cands {
+			out[i] = LookupHolder{PeerAddr: c.addr, DataCenter: c.dataCenter, Rack: c.rack}
+		}
+		result.PeersByFid[fid] = out
+	}
+	return result
+}
+
+// Sweep purges expired edges. Safe to call periodically. Peer slots are
+// left in place — their count is bounded by fleet size (a few hundred at
+// most) and never grows faster than the fleet itself, so reclaiming them
+// isn't worth the bookkeeping.
+func (d *PeerDirectory) Sweep() int {
+	now := d.clock()
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	n := 0
+	for fid, holders := range d.index {
+		for pid, expiry := range holders {
+			if !expiry.After(now) {
+				delete(holders, pid)
+				d.entryCount--
+				n++
+			}
+		}
+		if len(holders) == 0 {
+			delete(d.index, fid)
+		}
+	}
+	d.evictions.Add(int64(n))
+	return n
+}
+
+// Stats exposes counters for observability. O(1) — entryCount is kept in
+// lock-step with map mutations.
+func (d *PeerDirectory) Stats() (announces, lookups, rejected, evictions int64, entries int) {
+	d.mu.RLock()
+	defer d.mu.RUnlock()
+	return d.announces.Load(), d.lookups.Load(), d.rejected.Load(), d.evictions.Load(), d.entryCount
+}