snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/command/filer_sync_verify.go

@@ -0,0 +1,802 @@
+package command
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"sort"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+)
+
+type SyncVerifyOptions struct {
+	filerA          *string
+	filerB          *string
+	aPath           *string
+	bPath           *string
+	aSecurity       *string
+	bSecurity       *string
+	isActivePassive *bool
+	modifiedTimeAgo *time.Duration
+	jsonOutput      *bool
+}
+
+var syncVerifyOptions SyncVerifyOptions
+
+func init() {
+	cmdFilerSyncVerify.Run = runFilerSyncVerify // break init cycle
+	syncVerifyOptions.filerA = cmdFilerSyncVerify.Flag.String("a", "", "filer A in one SeaweedFS cluster")
+	syncVerifyOptions.filerB = cmdFilerSyncVerify.Flag.String("b", "", "filer B in the other SeaweedFS cluster")
+	syncVerifyOptions.aPath = cmdFilerSyncVerify.Flag.String("a.path", "/", "directory to verify on filer A")
+	syncVerifyOptions.bPath = cmdFilerSyncVerify.Flag.String("b.path", "/", "directory to verify on filer B")
+	syncVerifyOptions.aSecurity = cmdFilerSyncVerify.Flag.String("a.security", "", "security.toml file for filer A when clusters use different certificates")
+	syncVerifyOptions.bSecurity = cmdFilerSyncVerify.Flag.String("b.security", "", "security.toml file for filer B when clusters use different certificates")
+	syncVerifyOptions.isActivePassive = cmdFilerSyncVerify.Flag.Bool("isActivePassive", false, "one directional comparison from A to B; entries only in B are not reported")
+	syncVerifyOptions.modifiedTimeAgo = cmdFilerSyncVerify.Flag.Duration("modifiedTimeAgo", 0, "only verify files modified before this duration ago (e.g. 1h) for sync-lag tolerance")
+	syncVerifyOptions.jsonOutput = cmdFilerSyncVerify.Flag.Bool("jsonOutput", false, "emit NDJSON output (one JSON object per line) for external tooling")
+}
+
+var cmdFilerSyncVerify = &Command{
+	UsageLine: "filer.sync.verify -a=<oneFilerHost>:<oneFilerPort> -b=<otherFilerHost>:<otherFilerPort>",
+	Short:     "compare entries between two filers and report differences",
+	Long: `compare entries between two filers and report differences, then exit.
+
+	Useful for validating active/passive sync targets agree with the source.
+	Reports MISSING (in A but not in B), ONLY_IN_B (in B but not in A; suppressed
+	in active-passive mode), SIZE_MISMATCH, and ETAG_MISMATCH. Honors
+	-modifiedTimeAgo to skip recently-modified files (sync-lag tolerance) and
+	-isActivePassive for unidirectional comparison.
+
+	A chunk-derived ETag (no stored attr.Md5) can differ between clusters only
+	because the chunk slice was assembled in a different order, since ETagChunks
+	does not sort by offset. Such files are byte-identical and reported as
+	CHUNK_REORDER: not counted as errors, always counted in the summary, listed
+	only at -v=1.
+
+	Exits with code 0 on agreement, 2 on differences or operational errors.
+
+`,
+}
+
+func runFilerSyncVerify(cmd *Command, args []string) bool {
+	*syncVerifyOptions.aSecurity = util.ResolvePath(*syncVerifyOptions.aSecurity)
+	*syncVerifyOptions.bSecurity = util.ResolvePath(*syncVerifyOptions.bSecurity)
+	util.LoadSecurityConfiguration()
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+
+	grpcDialOptionA := grpcDialOption
+	grpcDialOptionB := grpcDialOption
+	if *syncVerifyOptions.aSecurity != "" {
+		var err error
+		if grpcDialOptionA, err = security.LoadClientTLSFromFile(*syncVerifyOptions.aSecurity, "grpc.client"); err != nil {
+			glog.Fatalf("load security config for filer A: %v", err)
+		}
+	}
+	if *syncVerifyOptions.bSecurity != "" {
+		var err error
+		if grpcDialOptionB, err = security.LoadClientTLSFromFile(*syncVerifyOptions.bSecurity, "grpc.client"); err != nil {
+			glog.Fatalf("load security config for filer B: %v", err)
+		}
+	}
+
+	filerA := pb.ServerAddress(*syncVerifyOptions.filerA)
+	filerB := pb.ServerAddress(*syncVerifyOptions.filerB)
+
+	if err := runVerifySync(filerA, filerB, *syncVerifyOptions.aPath, *syncVerifyOptions.bPath,
+		*syncVerifyOptions.isActivePassive, *syncVerifyOptions.modifiedTimeAgo,
+		*syncVerifyOptions.jsonOutput,
+		grpcDialOptionA, grpcDialOptionB); err != nil {
+		glog.Errorf("verify sync: %v", err)
+		os.Exit(2)
+	}
+	return true
+}
+
+// verifySyncConcurrency caps concurrent directory I/O across the entire
+// recursive walk. A single shared semaphore is created in runVerifySync and
+// passed down so the limit applies globally — a per-call semaphore would only
+// cap concurrency per directory level, allowing fanout to grow as
+// verifySyncConcurrency^depth on deep trees.
+//
+// Trade-off: higher values reduce wall time on wide trees by parallelizing
+// listEntries RPCs, at the cost of more concurrent load on both filers and
+// more memory from queued goroutines (each waiting goroutine ~2KB stack).
+// Each compareDirectory holds a slot only for its own listing+compare phase
+// and releases before recursing, so a parent never blocks waiting for
+// children to acquire slots.
+const verifySyncConcurrency = 5
+
+// isTooRecent reports whether entry's mtime is past cutoff (sync-lag tolerance).
+// Returns false when cutoff is zero or attributes are missing.
+func isTooRecent(entry *filer_pb.Entry, cutoff time.Time) bool {
+	return !cutoff.IsZero() && entry != nil && entry.Attributes != nil && entry.Attributes.Mtime > cutoff.Unix()
+}
+
+type VerifyResult struct {
+	dirCount      atomic.Int64
+	fileCount     atomic.Int64
+	missingCount  atomic.Int64
+	sizeMismatch  atomic.Int64
+	etagMismatch  atomic.Int64
+	chunkReorder  atomic.Int64 // content-equal; chunks differ only in slice order
+	onlyInB       atomic.Int64
+	skippedRecent atomic.Int64
+
+	// outputMu serializes writes to stdout. Multiple goroutines call
+	// reportDiff concurrently from compareDirectory worker pool.
+	outputMu   sync.Mutex
+	jsonOutput bool
+}
+
+type verifyDiffType int
+
+const (
+	diffMissing      verifyDiffType = iota // in A but not in B
+	diffOnlyInB                            // in B but not in A
+	diffSizeMismatch                       // size differs
+	diffETagMismatch                       // etag differs
+	diffChunkReorder                       // etag differs but content is equal (chunk slice order only)
+)
+
+// diffRecord is the JSON Lines schema for a single diff entry.
+type diffRecord struct {
+	Type          string       `json:"type"`
+	Path          string       `json:"path"`
+	IsDirectory   bool         `json:"isDirectory,omitempty"`
+	A             *entryRecord `json:"a,omitempty"`
+	B             *entryRecord `json:"b,omitempty"`
+	MtimeRelation string       `json:"mtimeRelation,omitempty"` // EQUAL | A_NEWER | B_NEWER
+	MtimeDelta    string       `json:"mtimeDelta,omitempty"`    // human-readable, e.g. "5d", "12h"
+	Hint          string       `json:"hint,omitempty"`          // late_updates_skip_likely | sync_lag_or_event_miss
+}
+
+type entryRecord struct {
+	Size  uint64 `json:"size"`
+	Mtime int64  `json:"mtime"`
+	ETag  string `json:"etag,omitempty"`
+}
+
+type summaryRecord struct {
+	Type          string `json:"type"`
+	Directories   int64  `json:"directories"`
+	Files         int64  `json:"files"`
+	SkippedRecent int64  `json:"skippedRecent"`
+	Missing       int64  `json:"missing"`
+	SizeMismatch  int64  `json:"sizeMismatch"`
+	ETagMismatch  int64  `json:"etagMismatch"`
+	ChunkReorder  int64  `json:"chunkReorder"`
+	OnlyInB       int64  `json:"onlyInB"`
+	TotalErrors   int64  `json:"totalErrors"`
+}
+
+// simpleFilerClient implements filer_pb.FilerClient for gRPC connections
+type simpleFilerClient struct {
+	grpcAddress    pb.ServerAddress
+	grpcDialOption grpc.DialOption
+}
+
+func (c *simpleFilerClient) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
+	return pb.WithGrpcClient(context.Background(), streamingMode, 0, func(grpcConnection *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+		return fn(client)
+	}, c.grpcAddress.ToGrpcAddress(), false, c.grpcDialOption)
+}
+
+func (c *simpleFilerClient) AdjustedUrl(location *filer_pb.Location) string {
+	return location.Url
+}
+
+func (c *simpleFilerClient) GetDataCenter() string {
+	return ""
+}
+
+func runVerifySync(filerA, filerB pb.ServerAddress, aPath, bPath string,
+	isActivePassive bool, modifiedTimeAgo time.Duration,
+	jsonOutput bool,
+	grpcDialOptionA, grpcDialOptionB grpc.DialOption) error {
+
+	clientA := &simpleFilerClient{grpcAddress: filerA, grpcDialOption: grpcDialOptionA}
+	clientB := &simpleFilerClient{grpcAddress: filerB, grpcDialOption: grpcDialOptionB}
+
+	var cutoffTime time.Time
+	if modifiedTimeAgo > 0 {
+		cutoffTime = time.Now().Add(-modifiedTimeAgo)
+	}
+
+	if !jsonOutput {
+		if !cutoffTime.IsZero() {
+			fmt.Fprintf(os.Stdout, "Verifying files modified before %v (modifiedTimeAgo=%v)\n",
+				cutoffTime.Format(time.RFC3339), modifiedTimeAgo)
+		}
+		fmt.Fprintf(os.Stdout, "Comparing %s%s => %s%s (isActivePassive=%v)\n\n",
+			filerA, aPath, filerB, bPath, isActivePassive)
+	}
+
+	result := &VerifyResult{jsonOutput: jsonOutput}
+	ctx := context.Background()
+	sem := make(chan struct{}, verifySyncConcurrency)
+
+	if err := compareDirectory(ctx, clientA, clientB, aPath, bPath, isActivePassive, cutoffTime, sem, result); err != nil {
+		return err
+	}
+
+	totalErrors := result.missingCount.Load() + result.sizeMismatch.Load() + result.etagMismatch.Load()
+	if !isActivePassive {
+		totalErrors += result.onlyInB.Load()
+	}
+
+	if jsonOutput {
+		summary := summaryRecord{
+			Type:          "SUMMARY",
+			Directories:   result.dirCount.Load(),
+			Files:         result.fileCount.Load(),
+			SkippedRecent: result.skippedRecent.Load(),
+			Missing:       result.missingCount.Load(),
+			SizeMismatch:  result.sizeMismatch.Load(),
+			ETagMismatch:  result.etagMismatch.Load(),
+			ChunkReorder:  result.chunkReorder.Load(),
+			OnlyInB:       result.onlyInB.Load(),
+			TotalErrors:   totalErrors,
+		}
+		writeJSONLine(result, summary)
+	} else {
+		fmt.Fprintf(os.Stdout, "\nSummary:\n")
+		fmt.Fprintf(os.Stdout, "  Directories compared: %d\n", result.dirCount.Load())
+		fmt.Fprintf(os.Stdout, "  Files verified:       %d\n", result.fileCount.Load())
+		if result.skippedRecent.Load() > 0 {
+			fmt.Fprintf(os.Stdout, "  Skipped (too recent): %d\n", result.skippedRecent.Load())
+		}
+		fmt.Fprintf(os.Stdout, "  Missing in B:         %d\n", result.missingCount.Load())
+		fmt.Fprintf(os.Stdout, "  Size mismatch:        %d\n", result.sizeMismatch.Load())
+		fmt.Fprintf(os.Stdout, "  ETag mismatch:        %d\n", result.etagMismatch.Load())
+		if n := result.chunkReorder.Load(); n > 0 {
+			fmt.Fprintf(os.Stdout, "  Chunk reorder (equal): %d  (content-equal, not an error; use -v=1 to list)\n", n)
+		}
+		if !isActivePassive {
+			fmt.Fprintf(os.Stdout, "  Only in B:            %d\n", result.onlyInB.Load())
+		}
+		fmt.Fprintf(os.Stdout, "  Total errors:         %d\n", totalErrors)
+	}
+
+	if totalErrors > 0 {
+		return fmt.Errorf("found %d differences", totalErrors)
+	}
+	return nil
+}
+
+// dirEntries holds one directory's entries buffered and sorted in Go byte
+// order, which the merge in compareDirectory requires (it pairs equal names and
+// uses eA.Name < eB.Name as the tie-break). The filer's own list order is not
+// trusted: since PR #9824 (4.32) a filer forces byte-lexicographic listing
+// regardless of SQL collation, while an older filer whose name column uses a
+// locale collation lists case-insensitively. Comparing two such clusters yields
+// the same name set in a different order, desyncing the merge into spurious
+// MISSING / ONLY_IN_B. Sorting both sides client-side makes the order identical
+// regardless of filer version or store backend.
+//
+// The load runs in a background goroutine so both sides of a compareDirectory
+// list concurrently; peek/advance block on `ready` until it completes. Memory
+// is O(directory size) per side — acceptable for a one-shot verify CLI.
+type dirEntries struct {
+	ready   chan struct{} // closed once entries and err are set
+	entries []*filer_pb.Entry
+	idx     int
+	err     error
+}
+
+// newDirEntries starts loading the whole directory and sorting it by Name, then
+// returns immediately. The goroutine exits when listing completes, errors, or
+// ctx is cancelled; it always closes `ready` so consumers never block forever.
+func newDirEntries(ctx context.Context, client filer_pb.FilerClient, dir string) *dirEntries {
+	s := &dirEntries{ready: make(chan struct{})}
+	go func() {
+		defer close(s.ready)
+		s.err = filer_pb.ReadDirAllEntries(ctx, client, util.FullPath(dir), "",
+			func(entry *filer_pb.Entry, isLast bool) error {
+				select {
+				case <-ctx.Done():
+					return ctx.Err()
+				default:
+				}
+				s.entries = append(s.entries, entry)
+				return nil
+			})
+		if s.err == nil {
+			sort.Slice(s.entries, func(i, j int) bool {
+				return s.entries[i].Name < s.entries[j].Name
+			})
+		}
+	}()
+	return s
+}
+
+// wait blocks until the directory has finished loading and sorting. Its close
+// of `ready` happens-before the return, so entries and err are then safe to read
+// without additional synchronisation.
+func (s *dirEntries) wait() { <-s.ready }
+
+// peek returns the next entry without consuming it, or nil at end-of-stream.
+// It blocks until the directory has finished loading and sorting.
+func (s *dirEntries) peek() *filer_pb.Entry {
+	s.wait()
+	if s.idx >= len(s.entries) {
+		return nil
+	}
+	return s.entries[s.idx]
+}
+
+// advance consumes and returns the next entry.
+func (s *dirEntries) advance() *filer_pb.Entry {
+	e := s.peek()
+	if e != nil {
+		s.idx++
+	}
+	return e
+}
+
+func compareDirectory(ctx context.Context,
+	clientA, clientB filer_pb.FilerClient,
+	dirA, dirB string,
+	isActivePassive bool,
+	cutoffTime time.Time,
+	sem chan struct{},
+	result *VerifyResult) error {
+
+	// Hold a slot only for this directory's I/O phase (listings + merge).
+	// Released before recursing so parents never block waiting for children
+	// to acquire slots — see verifySyncConcurrency for the rationale.
+	sem <- struct{}{}
+	released := false
+	releaseSlot := func() {
+		if !released {
+			released = true
+			<-sem
+		}
+	}
+	defer releaseSlot()
+
+	result.dirCount.Add(1)
+
+	// A child context cancels the load goroutines (and their listing RPCs) if
+	// compareDirectory returns early, e.g. on error. Both sides load concurrently.
+	mergeCtx, cancelMerge := context.WithCancel(ctx)
+	defer cancelMerge()
+
+	entriesA := newDirEntries(mergeCtx, clientA, dirA)
+	entriesB := newDirEntries(mergeCtx, clientB, dirB)
+
+	// Both sides are fully buffered before merging; surface any listing error
+	// here so a failed or cancelled listing can't emit bogus diffs from partial
+	// data. wait() makes entries/err visible without extra synchronisation.
+	entriesA.wait()
+	entriesB.wait()
+	if err := entriesA.err; err != nil && err != context.Canceled {
+		return fmt.Errorf("list %s on filer A: %v", dirA, err)
+	}
+	if err := entriesB.err; err != nil && err != context.Canceled {
+		return fmt.Errorf("list %s on filer B: %v", dirB, err)
+	}
+
+	// collect subdirectories for recursive comparison
+	type dirPair struct{ a, b string }
+	var subDirs []dirPair
+
+	for entriesA.peek() != nil || entriesB.peek() != nil {
+		eA := entriesA.peek()
+		eB := entriesB.peek()
+
+		switch {
+		case eA != nil && (eB == nil || eA.Name < eB.Name):
+			// entry only in A
+			entryA := entriesA.advance()
+			if entryA.IsDirectory {
+				// Always recurse for missing-in-B directories: a recent
+				// child write can bump the parent's mtime even though
+				// older missing files exist underneath. The cutoff is
+				// applied per-file inside countMissingRecursive.
+				reportDiff(diffMissing, dirA, entryA, nil, result)
+				countMissingRecursive(ctx, clientA, path.Join(dirA, entryA.Name), cutoffTime, result)
+			} else if isTooRecent(entryA, cutoffTime) {
+				result.skippedRecent.Add(1)
+			} else {
+				reportDiff(diffMissing, dirA, entryA, nil, result)
+			}
+
+		case eB != nil && (eA == nil || eB.Name < eA.Name):
+			// entry only in B
+			entryB := entriesB.advance()
+			if !isActivePassive {
+				if isTooRecent(entryB, cutoffTime) {
+					result.skippedRecent.Add(1)
+				} else {
+					reportDiff(diffOnlyInB, dirB, entryB, nil, result)
+				}
+			}
+
+		default:
+			// same name in both
+			entryA := entriesA.advance()
+			entryB := entriesB.advance()
+
+			if entryA.IsDirectory && entryB.IsDirectory {
+				subDirs = append(subDirs, dirPair{
+					a: path.Join(dirA, entryA.Name),
+					b: path.Join(dirB, entryB.Name),
+				})
+			} else if !entryA.IsDirectory && !entryB.IsDirectory {
+				// Skip if either side was modified recently (sync-lag tolerance).
+				if isTooRecent(entryA, cutoffTime) || isTooRecent(entryB, cutoffTime) {
+					result.skippedRecent.Add(1)
+				} else {
+					compareEntries(dirA, entryA, entryB, result)
+				}
+			} else {
+				// type mismatch: one is dir, other is file
+				reportDiff(diffMissing, dirA, entryA, nil, result)
+				if !isActivePassive {
+					reportDiff(diffOnlyInB, dirB, entryB, nil, result)
+				}
+			}
+		}
+	}
+
+	// Release our slot before recursing so children can acquire it. Holding
+	// it across wg.Wait would deadlock once depth exceeds verifySyncConcurrency.
+	releaseSlot()
+
+	if len(subDirs) > 0 {
+		// Bounded worker pool: cap goroutines per directory level instead
+		// of spawning one per child. A directory with thousands of subdirs
+		// would otherwise park ~2KB per waiting goroutine even though
+		// only `verifySyncConcurrency` can do I/O at once.
+		workers := verifySyncConcurrency
+		if len(subDirs) < workers {
+			workers = len(subDirs)
+		}
+		jobs := make(chan dirPair, len(subDirs))
+		errCh := make(chan error, 1) // first error wins; others dropped
+		var wg sync.WaitGroup
+
+		for i := 0; i < workers; i++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for pair := range jobs {
+					if err := compareDirectory(ctx, clientA, clientB, pair.a, pair.b, isActivePassive, cutoffTime, sem, result); err != nil {
+						select {
+						case errCh <- err:
+						default:
+						}
+					}
+				}
+			}()
+		}
+		for _, pair := range subDirs {
+			jobs <- pair
+		}
+		close(jobs)
+		wg.Wait()
+		close(errCh)
+		if err := <-errCh; err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func compareEntries(dir string, entryA, entryB *filer_pb.Entry, result *VerifyResult) {
+	result.fileCount.Add(1)
+
+	sizeA := filer.FileSize(entryA)
+	sizeB := filer.FileSize(entryB)
+	if sizeA != sizeB {
+		reportDiff(diffSizeMismatch, dir, entryA, entryB, result)
+		return
+	}
+
+	etagA := filer.ETag(entryA)
+	etagB := filer.ETag(entryB)
+	if etagA != etagB {
+		if isChunkReorder(entryA, entryB) {
+			// Same bytes, just a different chunk slice order — not a real diff.
+			reportDiff(diffChunkReorder, dir, entryA, entryB, result)
+		} else {
+			reportDiff(diffETagMismatch, dir, entryA, entryB, result)
+		}
+		return
+	}
+}
+
+// isChunkReorder reports whether two entries with differing file ETags are
+// byte-identical, differing only in chunk slice order. filer.ETagChunks hashes
+// per-chunk MD5s in stored order without sorting by offset, so the same bytes
+// written by different paths (S3 multipart vs filer.backup) can hash
+// differently. If both chunk lists sorted by offset match element-wise on
+// (offset, size, ETag), the files cover the same bytes with the same content
+// and only the slice order differed.
+//
+// Applies only when both ETags come from chunks (a stored attr.Md5 is
+// order-independent, so a mismatch there is a real content difference) and the
+// chunks form a manifest-free, non-overlapping set whose bytes are fully
+// determined by the list — overlaps would need timestamp-based visible-interval
+// resolution, so those stay ETAG_MISMATCH.
+func isChunkReorder(entryA, entryB *filer_pb.Entry) bool {
+	if !usesChunkETag(entryA) || !usesChunkETag(entryB) {
+		return false
+	}
+	a := sortedSimpleChunks(entryA.GetChunks())
+	b := sortedSimpleChunks(entryB.GetChunks())
+	// nil means overlapping or manifest chunks we do not fast-path; a single
+	// chunk hashes directly and a differing count is a real divergence.
+	if a == nil || b == nil || len(a) != len(b) || len(a) < 2 {
+		return false
+	}
+	for i := range a {
+		if a[i].Offset != b[i].Offset || a[i].Size != b[i].Size || a[i].ETag != b[i].ETag {
+			return false
+		}
+	}
+	return true
+}
+
+// usesChunkETag reports whether filer.ETag falls back to ETagChunks for entry,
+// i.e. no stored attr.Md5 whole-content hash is available. Mirrors filer.ETag's
+// own condition so it agrees with how the compared ETag was computed.
+func usesChunkETag(entry *filer_pb.Entry) bool {
+	return entry.Attributes == nil || entry.Attributes.Md5 == nil
+}
+
+// sortedSimpleChunks returns chunks sorted by (offset, ETag) if they are a
+// manifest-free, non-overlapping set whose visible bytes are fully determined
+// by the list and every chunk carries a usable per-chunk ETag; otherwise nil.
+// Overlapping ranges would require resolving the visible interval by chunk
+// timestamp, which this fast path does not attempt.
+func sortedSimpleChunks(chunks []*filer_pb.FileChunk) []*filer_pb.FileChunk {
+	sorted := make([]*filer_pb.FileChunk, len(chunks))
+	copy(sorted, chunks)
+	sort.Slice(sorted, func(i, j int) bool {
+		if sorted[i].Offset != sorted[j].Offset {
+			return sorted[i].Offset < sorted[j].Offset
+		}
+		return sorted[i].ETag < sorted[j].ETag
+	})
+	prevEnd := int64(0)
+	for i, c := range sorted {
+		if c.IsChunkManifest {
+			return nil
+		}
+		// An empty or undecodable ETag is not a content fingerprint, so
+		// element-wise (offset, size, ETag) equality cannot prove the bytes
+		// match — decline rather than assert a false content-equality.
+		if len(util.Base64Md5ToBytes(c.ETag)) == 0 {
+			return nil
+		}
+		if i > 0 && c.Offset < prevEnd {
+			return nil
+		}
+		prevEnd = c.Offset + int64(c.Size)
+	}
+	return sorted
+}
+
+// mtimeRelation classifies B.mtime vs A.mtime. Both entries must be non-nil.
+// Returns relation, absolute delta in seconds, and human-readable string.
+func mtimeRelation(entryA, entryB *filer_pb.Entry) (relation, deltaStr string, deltaSec int64) {
+	if entryA == nil || entryB == nil || entryA.Attributes == nil || entryB.Attributes == nil {
+		return "", "", 0
+	}
+	delta := entryB.Attributes.Mtime - entryA.Attributes.Mtime
+	abs := delta
+	if abs < 0 {
+		abs = -abs
+	}
+	switch {
+	case delta == 0:
+		return "EQUAL", "0s", 0
+	case delta > 0:
+		return "B_NEWER", formatSeconds(abs), abs
+	default:
+		return "A_NEWER", formatSeconds(abs), abs
+	}
+}
+
+func formatSeconds(s int64) string {
+	switch {
+	case s < 60:
+		return fmt.Sprintf("%ds", s)
+	case s < 3600:
+		return fmt.Sprintf("%dm", s/60)
+	case s < 86400:
+		return fmt.Sprintf("%dh", s/3600)
+	default:
+		return fmt.Sprintf("%dd", s/86400)
+	}
+}
+
+// hintFor returns an automatic interpretation hint based on mtime relation.
+// Only emitted for SIZE_MISMATCH and ETAG_MISMATCH where both entries exist.
+func hintFor(relation string) string {
+	switch relation {
+	case "B_NEWER":
+		return "late_updates_skip_likely"
+	case "A_NEWER":
+		return "sync_lag_or_event_miss"
+	}
+	return ""
+}
+
+func reportDiff(diffType verifyDiffType, dir string, entryA, entryB *filer_pb.Entry, result *VerifyResult) {
+	switch diffType {
+	case diffMissing:
+		result.missingCount.Add(1)
+	case diffOnlyInB:
+		result.onlyInB.Add(1)
+	case diffSizeMismatch:
+		result.sizeMismatch.Add(1)
+	case diffETagMismatch:
+		result.etagMismatch.Add(1)
+	case diffChunkReorder:
+		result.chunkReorder.Add(1)
+	}
+
+	if result.jsonOutput {
+		writeJSONDiff(result, diffType, dir, entryA, entryB)
+	} else {
+		// A chunk reorder is content-equal, not a real diff: keep it out of the
+		// default text report and list it only at -v=1. JSON always emits it so
+		// the per-record stream matches the summary count.
+		if diffType == diffChunkReorder && !glog.V(1) {
+			return
+		}
+		writeTextDiff(result, diffType, dir, entryA, entryB)
+	}
+}
+
+func writeTextDiff(result *VerifyResult, diffType verifyDiffType, dir string, entryA, entryB *filer_pb.Entry) {
+	entryPath := path.Join(dir, entryA.Name)
+
+	result.outputMu.Lock()
+	defer result.outputMu.Unlock()
+
+	switch diffType {
+	case diffMissing:
+		if entryA.IsDirectory {
+			fmt.Fprintf(os.Stdout, "[MISSING]       %s/ (directory)\n", entryPath)
+		} else {
+			fmt.Fprintf(os.Stdout, "[MISSING]       %s (size=%d, etag=%s)\n",
+				entryPath, filer.FileSize(entryA), filer.ETag(entryA))
+		}
+	case diffOnlyInB:
+		fmt.Fprintf(os.Stdout, "[ONLY_IN_B]     %s\n", entryPath)
+	case diffSizeMismatch:
+		ann := annotation(entryA, entryB)
+		fmt.Fprintf(os.Stdout, "[SIZE_MISMATCH] %s (a=%d, b=%d%s)\n",
+			entryPath, filer.FileSize(entryA), filer.FileSize(entryB), ann)
+	case diffETagMismatch:
+		ann := annotation(entryA, entryB)
+		fmt.Fprintf(os.Stdout, "[ETAG_MISMATCH] %s (a=%s, b=%s%s)\n",
+			entryPath, filer.ETag(entryA), filer.ETag(entryB), ann)
+	case diffChunkReorder:
+		fmt.Fprintf(os.Stdout, "[CHUNK_REORDER] %s (a=%s, b=%s, content-equal: chunks differ only in slice order)\n",
+			entryPath, filer.ETag(entryA), filer.ETag(entryB))
+	}
+}
+
+// annotation builds the trailing ", mtime: ... [hint]" segment for text output.
+// Returns empty string if entries are unavailable.
+func annotation(entryA, entryB *filer_pb.Entry) string {
+	relation, delta, _ := mtimeRelation(entryA, entryB)
+	if relation == "" {
+		return ""
+	}
+	switch relation {
+	case "EQUAL":
+		return ", mtime equal [chunk-level issue]"
+	case "B_NEWER":
+		return fmt.Sprintf(", B newer +%s [late-updates skip likely]", delta)
+	case "A_NEWER":
+		return fmt.Sprintf(", A newer +%s [sync lag or event miss]", delta)
+	}
+	return ""
+}
+
+func writeJSONDiff(result *VerifyResult, diffType verifyDiffType, dir string, entryA, entryB *filer_pb.Entry) {
+	rec := diffRecord{Path: path.Join(dir, entryA.Name)}
+
+	switch diffType {
+	case diffMissing:
+		rec.Type = "MISSING"
+		rec.IsDirectory = entryA.IsDirectory
+		rec.A = toEntryRecord(entryA)
+	case diffOnlyInB:
+		rec.Type = "ONLY_IN_B"
+		// for diffOnlyInB the existing convention passes the entry as entryA
+		rec.IsDirectory = entryA.IsDirectory
+		rec.B = toEntryRecord(entryA)
+	case diffSizeMismatch:
+		rec.Type = "SIZE_MISMATCH"
+		rec.A = toEntryRecord(entryA)
+		rec.B = toEntryRecord(entryB)
+		relation, delta, _ := mtimeRelation(entryA, entryB)
+		rec.MtimeRelation = relation
+		rec.MtimeDelta = delta
+		rec.Hint = hintFor(relation)
+	case diffETagMismatch:
+		rec.Type = "ETAG_MISMATCH"
+		rec.A = toEntryRecord(entryA)
+		rec.B = toEntryRecord(entryB)
+		relation, delta, _ := mtimeRelation(entryA, entryB)
+		rec.MtimeRelation = relation
+		rec.MtimeDelta = delta
+		rec.Hint = hintFor(relation)
+	case diffChunkReorder:
+		rec.Type = "CHUNK_REORDER"
+		rec.A = toEntryRecord(entryA)
+		rec.B = toEntryRecord(entryB)
+		relation, delta, _ := mtimeRelation(entryA, entryB)
+		rec.MtimeRelation = relation
+		rec.MtimeDelta = delta
+		// no hint: content-equal, no action needed
+	}
+
+	writeJSONLine(result, rec)
+}
+
+func toEntryRecord(entry *filer_pb.Entry) *entryRecord {
+	if entry == nil {
+		return nil
+	}
+	r := &entryRecord{
+		Size: filer.FileSize(entry),
+		ETag: filer.ETag(entry),
+	}
+	if entry.Attributes != nil {
+		r.Mtime = entry.Attributes.Mtime
+	}
+	return r
+}
+
+// writeJSONLine emits a single JSON object followed by newline. Holds outputMu
+// across marshal+write so concurrent goroutines never interleave.
+func writeJSONLine(result *VerifyResult, v any) {
+	data, err := json.Marshal(v)
+	if err != nil {
+		glog.Warningf("marshal verify record: %v", err)
+		return
+	}
+	result.outputMu.Lock()
+	defer result.outputMu.Unlock()
+	os.Stdout.Write(data)
+	os.Stdout.Write([]byte{'\n'})
+}
+
+func countMissingRecursive(ctx context.Context, client filer_pb.FilerClient, dir string, cutoffTime time.Time, result *VerifyResult) {
+	err := filer_pb.ReadDirAllEntries(ctx, client, util.FullPath(dir), "",
+		func(entry *filer_pb.Entry, isLast bool) error {
+			if entry.IsDirectory {
+				countMissingRecursive(ctx, client, path.Join(dir, entry.Name), cutoffTime, result)
+				return nil
+			}
+			if !cutoffTime.IsZero() && entry.Attributes != nil && entry.Attributes.Mtime > cutoffTime.Unix() {
+				result.skippedRecent.Add(1)
+				return nil
+			}
+			reportDiff(diffMissing, dir, entry, nil, result)
+			return nil
+		})
+	if err != nil {
+		glog.Warningf("list missing directory %s: %v", dir, err)
+	}
+}