upstream-seaweedfs/weed/server/volume_grpc_tier_download_test.go

package weed_server

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
	"github.com/seaweedfs/seaweedfs/weed/stats"
	"github.com/seaweedfs/seaweedfs/weed/storage"
	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
	"github.com/seaweedfs/seaweedfs/weed/storage/types"
	"github.com/seaweedfs/seaweedfs/weed/util"

	"google.golang.org/grpc"
)

const tierTestBackendName = "tier_test_local_dir.default"

// tierTestBackend is a fake BackendStorage backed by files in a temp dir. It
// records DeleteFile calls so a test can assert that a shared remote object is
// (or is not) removed.
type tierTestBackend struct {
	root string

	mu      sync.Mutex
	deletes []string

	lastUploadConcurrency   int
	lastDownloadConcurrency int
}

func (b *tierTestBackend) ToProperties() map[string]string { return map[string]string{"root": b.root} }

func (b *tierTestBackend) NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) backend.BackendStorageFile {
	return &tierTestBackendFile{backend: b, key: key, tierInfo: tierInfo}
}

func (b *tierTestBackend) CopyFile(f *os.File, fn func(progressed int64, percentage float32) error, concurrency int) (key string, size int64, err error) {
	b.mu.Lock()
	b.lastUploadConcurrency = concurrency
	b.mu.Unlock()
	key = fmt.Sprintf("obj-%d", time.Now().UnixNano())
	dst := filepath.Join(b.root, key)
	out, err := os.Create(dst)
	if err != nil {
		return "", 0, err
	}
	defer out.Close()
	if _, err = f.Seek(0, io.SeekStart); err != nil {
		return "", 0, err
	}
	written, err := io.Copy(out, f)
	if err != nil {
		return "", 0, err
	}
	return key, written, nil
}

func (b *tierTestBackend) DownloadFile(fileName string, key string, fn func(progressed int64, percentage float32) error, concurrency int) (size int64, err error) {
	b.mu.Lock()
	b.lastDownloadConcurrency = concurrency
	b.mu.Unlock()
	in, err := os.Open(filepath.Join(b.root, key))
	if err != nil {
		return 0, err
	}
	defer in.Close()
	out, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return 0, err
	}
	written, err := io.Copy(out, in)
	if err != nil {
		out.Close()
		return 0, err
	}
	// mirror the real backends: fsync the .dat before close so the caller can
	// trim the .vif and delete the remote object without risking a torn write.
	if syncErr := out.Sync(); syncErr != nil {
		out.Close()
		return 0, syncErr
	}
	if closeErr := out.Close(); closeErr != nil {
		return 0, closeErr
	}
	return written, nil
}

func (b *tierTestBackend) DeleteFile(key string) error {
	b.mu.Lock()
	b.deletes = append(b.deletes, key)
	b.mu.Unlock()
	return os.Remove(filepath.Join(b.root, key))
}

func (b *tierTestBackend) deleteHistory() []string {
	b.mu.Lock()
	defer b.mu.Unlock()
	out := make([]string, len(b.deletes))
	copy(out, b.deletes)
	return out
}

func (b *tierTestBackend) objectExists(key string) bool {
	_, err := os.Stat(filepath.Join(b.root, key))
	return err == nil
}

func (b *tierTestBackend) concurrencySeen() (upload, download int) {
	b.mu.Lock()
	defer b.mu.Unlock()
	return b.lastUploadConcurrency, b.lastDownloadConcurrency
}

type tierTestBackendFile struct {
	backend  *tierTestBackend
	key      string
	tierInfo *volume_server_pb.VolumeInfo
}

func (f *tierTestBackendFile) ReadAt(p []byte, off int64) (int, error) {
	in, err := os.Open(filepath.Join(f.backend.root, f.key))
	if err != nil {
		return 0, err
	}
	defer in.Close()
	return in.ReadAt(p, off)
}
func (f *tierTestBackendFile) WriteAt(p []byte, off int64) (int, error) { panic("not implemented") }
func (f *tierTestBackendFile) Truncate(off int64) error                 { panic("not implemented") }
func (f *tierTestBackendFile) Close() error                             { return nil }
func (f *tierTestBackendFile) Name() string                             { return f.key }
func (f *tierTestBackendFile) Sync() error                              { return nil }
func (f *tierTestBackendFile) GetStat() (int64, time.Time, error) {
	files := f.tierInfo.GetFiles()
	if len(files) == 0 {
		return 0, time.Time{}, fmt.Errorf("remote file info not found")
	}
	return int64(files[0].FileSize), time.Unix(int64(files[0].ModifiedTime), 0), nil
}

// fakeTierStream is a no-op server stream for the tier-download RPC. The
// embedded grpc.ServerStream is nil, so Context is implemented here rather than
// promoted -- the RPC reads it to authorize the caller.
type fakeTierStream struct {
	grpc.ServerStream
}

func (s *fakeTierStream) Send(*volume_server_pb.VolumeTierMoveDatFromRemoteResponse) error {
	return nil
}

func (s *fakeTierStream) Context() context.Context {
	return context.Background()
}

func newTierTestStore(t *testing.T, dir string) *storage.Store {
	t.Helper()
	diskIOProbeConfig := stats.DefaultDiskIOProbeConfig()
	store := storage.NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "",
		[]string{dir}, []int32{100}, []util.MinFreeSpace{{}}, "",
		storage.NeedleMapInMemory, []types.DiskType{types.HardDriveType}, nil, 3, diskIOProbeConfig)

	done := make(chan bool)
	go func() {
		for {
			select {
			case <-store.NewVolumesChan:
			case <-store.DeletedVolumesChan:
			case <-done:
				return
			}
		}
	}()
	t.Cleanup(func() {
		store.Close()
		close(done)
	})
	return store
}

// tierUpVolumeOnDisk creates a real local volume in dir, writes a few needles,
// uploads the .dat to the fake backend, rewrites the .vif to remote mode, and
// removes the local .dat — mirroring volume_grpc_tier_upload.go. It returns the
// remote object key and the original local .dat bytes for later comparison.
func tierUpVolumeOnDisk(t *testing.T, dir string, vid needle.VolumeId, b *tierTestBackend) (key string, localDat []byte) {
	t.Helper()
	store := newTierTestStore(t, dir)
	if err := store.AddVolume(vid, "", storage.NeedleMapInMemory, "000", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0); err != nil {
		t.Fatalf("add volume: %v", err)
	}
	for i := 1; i <= 5; i++ {
		n := new(needle.Needle)
		n.Id = types.Uint64ToNeedleId(uint64(i))
		n.Data = []byte(fmt.Sprintf("payload-%d-localdisk", i))
		n.Checksum = needle.NewCRC(n.Data)
		if _, err := store.WriteVolumeNeedle(vid, n, true, false); err != nil {
			t.Fatalf("write needle %d: %v", i, err)
		}
	}

	v := store.GetVolume(vid)
	if v == nil {
		t.Fatal("volume not found after add")
	}
	diskFile, ok := v.DataBackend.(*backend.DiskFile)
	if !ok {
		t.Fatalf("expected on-disk backend before tier-up, got %T", v.DataBackend)
	}
	datPath := v.FileName(".dat")
	key, size, err := b.CopyFile(diskFile.File, nil, 0)
	if err != nil {
		t.Fatalf("upload to fake backend: %v", err)
	}

	bType, bId := backend.BackendNameToTypeId(tierTestBackendName)
	v.GetVolumeInfo().Files = append(v.GetVolumeInfo().GetFiles(), &volume_server_pb.RemoteFile{
		BackendType:  bType,
		BackendId:    bId,
		Key:          key,
		FileSize:     uint64(size),
		ModifiedTime: uint64(time.Now().Unix()),
		Extension:    ".dat",
	})
	if err := v.SaveVolumeInfo(); err != nil {
		t.Fatalf("save volume info: %v", err)
	}

	localDat, err = os.ReadFile(datPath)
	if err != nil {
		t.Fatalf("read local dat: %v", err)
	}
	if err := store.UnmountVolume(vid); err != nil {
		t.Fatalf("unmount: %v", err)
	}
	if err := os.Remove(datPath); err != nil {
		t.Fatalf("remove local dat: %v", err)
	}
	return key, localDat
}

// TestTierMoveDatFromRemote_KeepRemote_LeavesReplicaLocal reproduces the
// multi-replica tier-download data loss: when KeepRemoteDatFile=true (every
// replica except the last), the replica must end up served from its local .dat
// with the shared remote object intact. Before the fix the keep-remote path
// returned right after the download without trimming the .vif or swapping the
// data backend, so the replica stayed remote-backed and a subsequent delete of
// the shared object on the final replica bricked it.
func TestTierMoveDatFromRemote_KeepRemote_LeavesReplicaLocal(t *testing.T) {
	b := &tierTestBackend{root: t.TempDir()}
	backend.BackendStorages[tierTestBackendName] = b
	t.Cleanup(func() { delete(backend.BackendStorages, tierTestBackendName) })

	dir := t.TempDir()
	const vid = needle.VolumeId(71)
	key, localDat := tierUpVolumeOnDisk(t, dir, vid, b)

	if !b.objectExists(key) {
		t.Fatal("remote object missing after tier-up")
	}

	// Mount the tiered volume the way a volume server does at startup.
	store := newTierTestStore(t, dir)
	v := store.GetVolume(vid)
	if v == nil {
		t.Fatal("tiered volume not loaded by store")
	}
	if !v.HasRemoteFile() {
		t.Fatal("volume should load in remote mode before download")
	}

	vs := &VolumeServer{store: store}
	req := &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{
		VolumeId:          uint32(vid),
		Collection:        "",
		KeepRemoteDatFile: true,
	}
	if err := vs.VolumeTierMoveDatFromRemote(req, &fakeTierStream{}); err != nil {
		t.Fatalf("VolumeTierMoveDatFromRemote: %v", err)
	}

	// keepRemote: the shared object must survive untouched.
	if len(b.deleteHistory()) != 0 {
		t.Fatalf("KeepRemoteDatFile=true must not delete the shared remote object, deletes: %v", b.deleteHistory())
	}
	if !b.objectExists(key) {
		t.Fatal("remote object must still exist after a keep-remote download")
	}

	// The in-memory volume must now be served from a local DiskFile, not the
	// remote backend — this is the assertion that fails on the pre-fix code.
	if _, ok := v.DataBackend.(*backend.DiskFile); !ok {
		t.Fatalf("after keep-remote download the data backend must be local DiskFile, got %T", v.DataBackend)
	}
	// The live volume must also report non-remote immediately, without waiting for
	// a remount: a delete arriving before the remount would otherwise skip the .dat
	// tombstone, and the phantom-.dat guard would stay disabled.
	if v.HasRemoteFile() {
		t.Fatal("after download the live volume must no longer be remote-backed")
	}

	// Remount and confirm the volume is no longer remote-backed and reads come
	// from the local .dat (matching the bytes uploaded before tiering).
	if err := store.UnmountVolume(vid); err != nil {
		t.Fatalf("unmount after download: %v", err)
	}
	if err := store.MountVolume(vid, &req.Collection); err != nil {
		t.Fatalf("remount after download: %v", err)
	}
	v2 := store.GetVolume(vid)
	if v2 == nil {
		t.Fatal("volume missing after remount")
	}
	if v2.HasRemoteFile() {
		t.Fatal("after a keep-remote download the remounted replica must not be remote-backed")
	}
	if _, ok := v2.DataBackend.(*backend.DiskFile); !ok {
		t.Fatalf("remounted replica must read from local DiskFile, got %T", v2.DataBackend)
	}

	// The .vif must no longer reference the remote object.
	storageName, storageKey := v2.RemoteStorageNameKey()
	if storageName != "" || storageKey != "" {
		t.Fatalf("trimmed .vif must not reference remote object, got %q/%q", storageName, storageKey)
	}

	// The local .dat must hold the downloaded content (the original bytes).
	gotDat, err := os.ReadFile(v2.FileName(".dat"))
	if err != nil {
		t.Fatalf("read local dat after download: %v", err)
	}
	if !bytes.Equal(gotDat, localDat) {
		t.Fatalf("local .dat content mismatch: got %d bytes, want %d", len(gotDat), len(localDat))
	}
}

// TestTierMoveConcurrencyPlumbing verifies the -concurrency value travels the
// whole tier-move path to CopyFile and DownloadFile.
func TestTierMoveConcurrencyPlumbing(t *testing.T) {
	b := &tierTestBackend{root: t.TempDir()}
	backend.BackendStorages[tierTestBackendName] = b
	t.Cleanup(func() { delete(backend.BackendStorages, tierTestBackendName) })

	dir := t.TempDir()
	const vid = needle.VolumeId(72)
	tierUpVolumeOnDisk(t, dir, vid, b)

	// download path: request concurrency must reach DownloadFile
	store := newTierTestStore(t, dir)
	v := store.GetVolume(vid)
	if v == nil {
		t.Fatal("tiered volume not loaded by store")
	}
	vs := &VolumeServer{store: store}
	if err := vs.VolumeTierMoveDatFromRemote(&volume_server_pb.VolumeTierMoveDatFromRemoteRequest{
		VolumeId:    uint32(vid),
		Concurrency: 4,
	}, &fakeTierStream{}); err != nil {
		t.Fatalf("VolumeTierMoveDatFromRemote: %v", err)
	}
	if up, down := b.concurrencySeen(); down != 4 {
		t.Fatalf("DownloadFile concurrency = %d (upload=%d), want 4", down, up)
	}

	// upload path: the volume is local again after the download
	if err := vs.VolumeTierMoveDatToRemote(&volume_server_pb.VolumeTierMoveDatToRemoteRequest{
		VolumeId:               uint32(vid),
		DestinationBackendName: tierTestBackendName,
		Concurrency:            3,
	}, &discardServerStream[volume_server_pb.VolumeTierMoveDatToRemoteResponse]{}); err != nil {
		t.Fatalf("VolumeTierMoveDatToRemote: %v", err)
	}
	if up, down := b.concurrencySeen(); up != 3 || down != 4 {
		t.Fatalf("CopyFile concurrency = %d (download=%d), want 3/4", up, down)
	}
}