snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/admin/dash/worker_grpc_server.go

@@ -0,0 +1,860 @@
+package dash
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/status"
+)
+
+const (
+	maxLogFetchLimit   = 1000
+	maxLogMessageSize  = 2000
+	maxLogFieldsCount  = 20
+	logRequestTimeout  = 10 * time.Second
+	logResponseTimeout = 30 * time.Second
+	logSendTimeout     = 10 * time.Second
+)
+
+// WorkerGrpcServer implements the WorkerService gRPC interface
+type WorkerGrpcServer struct {
+	worker_pb.UnimplementedWorkerServiceServer
+	adminServer *AdminServer
+
+	// Worker connection management
+	connections map[string]*WorkerConnection
+	connMutex   sync.RWMutex
+
+	// Log request correlation
+	pendingLogRequests map[string]*LogRequestContext
+	logRequestsMutex   sync.RWMutex
+
+	// gRPC server
+	grpcServer  *grpc.Server
+	listener    net.Listener
+	running     bool
+	stopChan    chan struct{}
+	mtlsEnabled bool
+}
+
+// LogRequestContext tracks pending log requests
+type LogRequestContext struct {
+	TaskID     string
+	WorkerID   string
+	ResponseCh chan *worker_pb.TaskLogResponse
+}
+
+// WorkerConnection represents an active worker connection
+type WorkerConnection struct {
+	workerID      string
+	stream        worker_pb.WorkerService_WorkerStreamServer
+	lastSeen      time.Time
+	capabilities  []MaintenanceTaskType
+	address       string
+	maxConcurrent int32
+	outgoing      chan *worker_pb.AdminMessage
+	ctx           context.Context
+	cancel        context.CancelFunc
+}
+
+// NewWorkerGrpcServer creates a new gRPC server for worker connections
+func NewWorkerGrpcServer(adminServer *AdminServer) *WorkerGrpcServer {
+	return &WorkerGrpcServer{
+		adminServer:        adminServer,
+		connections:        make(map[string]*WorkerConnection),
+		pendingLogRequests: make(map[string]*LogRequestContext),
+		stopChan:           make(chan struct{}),
+	}
+}
+
+// StartWithTLS starts the gRPC server on the specified port with optional TLS.
+// A caller that already holds the port passes its listener instead. When no
+// listener is supplied the server binds to bindIp to honor the operator's -ip.
+func (s *WorkerGrpcServer) StartWithTLS(bindIp string, port int, listener net.Listener) error {
+	if s.running {
+		return fmt.Errorf("worker gRPC server is already running")
+	}
+
+	if listener == nil {
+		var err error
+		listener, err = net.Listen("tcp", util.JoinHostPort(bindIp, port))
+		if err != nil {
+			return fmt.Errorf("failed to listen on port %d: %v", port, err)
+		}
+	}
+
+	// Create gRPC server with optional TLS
+	tlsOption, _ := security.LoadServerTLS(util.GetViper(), "grpc.admin")
+	s.mtlsEnabled = tlsOption != nil
+	grpcServer := pb.NewGrpcServer(tlsOption)
+
+	worker_pb.RegisterWorkerServiceServer(grpcServer, s)
+	if plugin := s.adminServer.GetPlugin(); plugin != nil {
+		plugin_pb.RegisterPluginControlServiceServer(grpcServer, plugin)
+		glog.V(0).Infof("Plugin gRPC service registered on worker gRPC server")
+	}
+
+	s.grpcServer = grpcServer
+	s.listener = listener
+	s.running = true
+
+	// Start background routines
+	go s.cleanupRoutine()
+	go s.activeLogFetchLoop()
+
+	pb.ServeGrpcOnLocalSocket(grpcServer, port)
+
+	// Start serving in a goroutine
+	go func() {
+		if err := s.grpcServer.Serve(listener); err != nil {
+			if s.running {
+				glog.Errorf("Worker gRPC server error: %v", err)
+			}
+		}
+	}()
+
+	return nil
+}
+
+// ListenPort returns the currently bound worker gRPC listen port.
+func (s *WorkerGrpcServer) ListenPort() int {
+	if s == nil || s.listener == nil {
+		return 0
+	}
+	if tcpAddr, ok := s.listener.Addr().(*net.TCPAddr); ok {
+		return tcpAddr.Port
+	}
+	_, portStr, err := net.SplitHostPort(s.listener.Addr().String())
+	if err != nil {
+		return 0
+	}
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		return 0
+	}
+	return port
+}
+
+// Stop stops the gRPC server
+func (s *WorkerGrpcServer) Stop() error {
+	if !s.running {
+		return nil
+	}
+
+	s.running = false
+	close(s.stopChan)
+
+	// Close all worker connections. Cancelling the context stops each
+	// handleOutgoingMessages goroutine; the outgoing channel is never closed.
+	s.connMutex.Lock()
+	for _, conn := range s.connections {
+		conn.cancel()
+	}
+	s.connections = make(map[string]*WorkerConnection)
+	s.connMutex.Unlock()
+
+	// Stop gRPC server
+	if s.grpcServer != nil {
+		s.grpcServer.GracefulStop()
+	}
+
+	// Close listener
+	if s.listener != nil {
+		s.listener.Close()
+	}
+
+	glog.Infof("Worker gRPC server stopped")
+	return nil
+}
+
+// WorkerStream handles bidirectional communication with workers
+func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStreamServer) error {
+	ctx := stream.Context()
+
+	// get client address
+	address := findClientAddress(ctx)
+
+	// Wait for initial registration message
+	msg, err := stream.Recv()
+	if err != nil {
+		return fmt.Errorf("failed to receive registration message: %w", err)
+	}
+
+	registration := msg.GetRegistration()
+	if registration == nil {
+		return fmt.Errorf("first message must be registration")
+	}
+	registration.Address = address
+
+	workerID := registration.WorkerId
+	if workerID == "" {
+		return fmt.Errorf("worker ID cannot be empty")
+	}
+
+	glog.Infof("Worker %s connecting from %s", workerID, registration.Address)
+
+	// Create worker connection
+	connCtx, connCancel := context.WithCancel(ctx)
+	conn := &WorkerConnection{
+		workerID:      workerID,
+		stream:        stream,
+		lastSeen:      time.Now(),
+		address:       registration.Address,
+		maxConcurrent: registration.MaxConcurrent,
+		outgoing:      make(chan *worker_pb.AdminMessage, 100),
+		ctx:           connCtx,
+		cancel:        connCancel,
+	}
+
+	// Convert capabilities
+	capabilities := make([]MaintenanceTaskType, len(registration.Capabilities))
+	for i, cap := range registration.Capabilities {
+		capabilities[i] = MaintenanceTaskType(cap)
+	}
+	conn.capabilities = capabilities
+
+	// Register connection - clean up old connection if worker is reconnecting
+	s.connMutex.Lock()
+	if oldConn, exists := s.connections[workerID]; exists {
+		glog.Infof("Worker %s reconnected, cleaning up old connection", workerID)
+		// Cancel old connection to stop its goroutines. Its handleOutgoingMessages
+		// exits on the cancelled context; the outgoing channel is never closed.
+		oldConn.cancel()
+	}
+	s.connections[workerID] = conn
+	s.connMutex.Unlock()
+	stats_collect.AdminWorkerEventsTotal.WithLabelValues("registered").Inc()
+
+	// Register worker with maintenance manager
+	s.registerWorkerWithManager(conn)
+
+	// IMPORTANT: Start outgoing message handler BEFORE sending registration response
+	// This ensures the handler is ready to process messages and prevents race conditions
+	// where the worker might send requests before we're ready to respond
+	go s.handleOutgoingMessages(conn)
+
+	// Send registration response (after handler is started)
+	regResponse := &worker_pb.AdminMessage{
+		Timestamp: time.Now().Unix(),
+		Message: &worker_pb.AdminMessage_RegistrationResponse{
+			RegistrationResponse: &worker_pb.RegistrationResponse{
+				Success: true,
+				Message: "Worker registered successfully",
+			},
+		},
+	}
+
+	if s.sendToWorker(conn, regResponse, 5*time.Second, "registration response") {
+		glog.V(1).Infof("Registration response sent to worker %s", workerID)
+	}
+
+	// Handle incoming messages
+	for {
+		select {
+		case <-ctx.Done():
+			glog.Infof("Worker %s connection closed: %v", workerID, ctx.Err())
+			s.unregisterWorker(conn, "unregistered")
+			return nil
+		case <-connCtx.Done():
+			glog.Infof("Worker %s connection cancelled", workerID)
+			s.unregisterWorker(conn, "unregistered")
+			return nil
+		default:
+		}
+
+		msg, err := stream.Recv()
+		if err != nil {
+			switch {
+			case err == io.EOF:
+				glog.Infof("Worker %s disconnected", workerID)
+			case errors.Is(err, context.Canceled), status.Code(err) == codes.Canceled:
+				// Graceful shutdown on either side cancels the stream.
+				glog.V(1).Infof("Worker %s stream canceled: %v", workerID, err)
+			default:
+				glog.Errorf("Error receiving from worker %s: %v", workerID, err)
+			}
+			s.unregisterWorker(conn, "unregistered")
+			return err
+		}
+
+		s.connMutex.Lock()
+		conn.lastSeen = time.Now()
+		s.connMutex.Unlock()
+		s.handleWorkerMessage(conn, msg)
+	}
+}
+
+// handleOutgoingMessages sends messages to worker
+func (s *WorkerGrpcServer) handleOutgoingMessages(conn *WorkerConnection) {
+	for {
+		select {
+		case <-conn.ctx.Done():
+			return
+		case msg, ok := <-conn.outgoing:
+			if !ok {
+				return
+			}
+
+			if err := conn.stream.Send(msg); err != nil {
+				glog.Errorf("Failed to send message to worker %s: %v", conn.workerID, err)
+				conn.cancel()
+				return
+			}
+		}
+	}
+}
+
+// handleWorkerMessage processes incoming messages from workers
+func (s *WorkerGrpcServer) handleWorkerMessage(conn *WorkerConnection, msg *worker_pb.WorkerMessage) {
+	workerID := conn.workerID
+
+	switch m := msg.Message.(type) {
+	case *worker_pb.WorkerMessage_Heartbeat:
+		s.handleHeartbeat(conn, m.Heartbeat)
+
+	case *worker_pb.WorkerMessage_TaskRequest:
+		s.handleTaskRequest(conn, m.TaskRequest)
+
+	case *worker_pb.WorkerMessage_TaskUpdate:
+		s.handleTaskUpdate(conn, m.TaskUpdate)
+
+	case *worker_pb.WorkerMessage_TaskComplete:
+		s.handleTaskCompletion(conn, m.TaskComplete)
+
+	case *worker_pb.WorkerMessage_TaskLogResponse:
+		s.handleTaskLogResponse(conn, m.TaskLogResponse)
+
+	case *worker_pb.WorkerMessage_Shutdown:
+		glog.Infof("Worker %s shutting down: %s", workerID, m.Shutdown.Reason)
+		s.unregisterWorker(conn, "unregistered")
+
+	default:
+		glog.Warningf("Unknown message type from worker %s", workerID)
+	}
+}
+
+// registerWorkerWithManager registers the worker with the maintenance manager
+func (s *WorkerGrpcServer) registerWorkerWithManager(conn *WorkerConnection) {
+	if s.adminServer.maintenanceManager == nil {
+		return
+	}
+
+	worker := &MaintenanceWorker{
+		ID:            conn.workerID,
+		Address:       conn.address,
+		LastHeartbeat: time.Now(),
+		Status:        "active",
+		Capabilities:  conn.capabilities,
+		MaxConcurrent: int(conn.maxConcurrent),
+		CurrentLoad:   0,
+	}
+
+	s.adminServer.maintenanceManager.RegisterWorker(worker)
+	glog.V(1).Infof("Registered worker %s with maintenance manager", conn.workerID)
+}
+
+// handleHeartbeat processes heartbeat messages
+func (s *WorkerGrpcServer) handleHeartbeat(conn *WorkerConnection, heartbeat *worker_pb.WorkerHeartbeat) {
+	if s.adminServer.maintenanceManager != nil {
+		s.adminServer.maintenanceManager.UpdateWorkerHeartbeat(conn.workerID)
+	}
+
+	// Send heartbeat response
+	response := &worker_pb.AdminMessage{
+		Timestamp: time.Now().Unix(),
+		Message: &worker_pb.AdminMessage_HeartbeatResponse{
+			HeartbeatResponse: &worker_pb.HeartbeatResponse{
+				Success: true,
+				Message: "Heartbeat acknowledged",
+			},
+		},
+	}
+
+	s.sendToWorker(conn, response, time.Second, "heartbeat response")
+}
+
+// handleTaskRequest processes task requests from workers
+func (s *WorkerGrpcServer) handleTaskRequest(conn *WorkerConnection, request *worker_pb.TaskRequest) {
+
+	if s.adminServer.maintenanceManager == nil {
+		return
+	}
+
+	// Get next task from maintenance manager
+	task := s.adminServer.maintenanceManager.GetNextTask(conn.workerID, conn.capabilities)
+
+	if task != nil {
+
+		// Use typed params directly - master client should already be configured in the params
+		var taskParams *worker_pb.TaskParams
+		if task.TypedParams != nil {
+			taskParams = task.TypedParams
+		} else {
+			// Create basic params if none exist
+			taskParams = &worker_pb.TaskParams{
+				VolumeId:   task.VolumeID,
+				Collection: task.Collection,
+				Sources: []*worker_pb.TaskSource{
+					{
+						Node:     task.Server,
+						VolumeId: task.VolumeID,
+					},
+				},
+			}
+		}
+
+		// Send task assignment
+		assignment := &worker_pb.AdminMessage{
+			Timestamp: time.Now().Unix(),
+			Message: &worker_pb.AdminMessage_TaskAssignment{
+				TaskAssignment: &worker_pb.TaskAssignment{
+					TaskId:      task.ID,
+					TaskType:    string(task.Type),
+					Params:      taskParams,
+					Priority:    int32(task.Priority),
+					CreatedTime: time.Now().Unix(),
+				},
+			},
+		}
+
+		s.sendToWorker(conn, assignment, time.Second, "task assignment")
+	} else {
+		// Send explicit "No Task" response to prevent worker timeout
+		// Workers expect a TaskAssignment message but will sleep if TaskId is empty
+		noTaskAssignment := &worker_pb.AdminMessage{
+			Timestamp: time.Now().Unix(),
+			Message: &worker_pb.AdminMessage_TaskAssignment{
+				TaskAssignment: &worker_pb.TaskAssignment{
+					TaskId: "", // Empty TaskId indicates no task available
+				},
+			},
+		}
+
+		if s.sendToWorker(conn, noTaskAssignment, time.Second, "no-task response") {
+			glog.V(4).Infof("Sent 'No Task' response to worker %s", conn.workerID)
+		}
+	}
+}
+
+// handleTaskUpdate processes task progress updates
+func (s *WorkerGrpcServer) handleTaskUpdate(conn *WorkerConnection, update *worker_pb.TaskUpdate) {
+	if s.adminServer.maintenanceManager != nil {
+		s.adminServer.maintenanceManager.UpdateTaskProgress(update.TaskId, float64(update.Progress))
+		glog.V(3).Infof("Updated task %s progress: %.1f%%", update.TaskId, update.Progress)
+	}
+}
+
+// handleTaskCompletion processes task completion notifications
+func (s *WorkerGrpcServer) handleTaskCompletion(conn *WorkerConnection, completion *worker_pb.TaskComplete) {
+	if s.adminServer.maintenanceManager != nil {
+		errorMsg := ""
+		if !completion.Success {
+			errorMsg = completion.ErrorMessage
+		}
+		s.adminServer.maintenanceManager.CompleteTask(completion.TaskId, errorMsg)
+
+		if completion.Success {
+			glog.V(1).Infof("Worker %s completed task %s successfully", conn.workerID, completion.TaskId)
+		} else {
+			glog.Errorf("Worker %s failed task %s: %s", conn.workerID, completion.TaskId, completion.ErrorMessage)
+		}
+
+		// Fetch and persist logs
+		go s.FetchAndSaveLogs(conn.workerID, completion.TaskId)
+	}
+}
+
+// FetchAndSaveLogs retrieves logs from a worker and saves them to disk
+func (s *WorkerGrpcServer) FetchAndSaveLogs(workerID, taskID string) error {
+	// Add a small initial delay to allow worker to finalize and sync logs
+	// especially when this is called immediately after TaskComplete
+	time.Sleep(300 * time.Millisecond)
+
+	var workerLogs []*worker_pb.TaskLogEntry
+	var err error
+
+	// Retry a few times if fetch fails, as logs might be in the middle of a terminal sync
+	for attempt := 1; attempt <= 3; attempt++ {
+		workerLogs, err = s.RequestTaskLogs(workerID, taskID, maxLogFetchLimit, "")
+		if err == nil {
+			break
+		}
+		if attempt < 3 {
+			glog.V(1).Infof("Fetch logs attempt %d failed for task %s: %v. Retrying in 1s...", attempt, taskID, err)
+			time.Sleep(1 * time.Second)
+		}
+	}
+
+	if err != nil {
+		glog.Warningf("Failed to fetch logs for task %s after 3 attempts: %v", taskID, err)
+		return err
+	}
+
+	// Convert logs
+	var maintenanceLogs []*maintenance.TaskExecutionLog
+	for _, workerLog := range workerLogs {
+		maintenanceLog := &maintenance.TaskExecutionLog{
+			Timestamp: time.Unix(workerLog.Timestamp, 0),
+			Level:     workerLog.Level,
+			Message:   workerLog.Message,
+			Source:    "worker",
+			TaskID:    taskID,
+			WorkerID:  workerID,
+		}
+
+		// Truncate very long messages to prevent rendering issues and disk bloat
+		if len(maintenanceLog.Message) > maxLogMessageSize {
+			maintenanceLog.Message = maintenanceLog.Message[:maxLogMessageSize] + "... (truncated)"
+		}
+
+		// carry structured fields if present
+		if len(workerLog.Fields) > 0 {
+			maintenanceLog.Fields = make(map[string]string)
+			fieldCount := 0
+			for k, v := range workerLog.Fields {
+				if fieldCount >= maxLogFieldsCount {
+					maintenanceLog.Fields["..."] = fmt.Sprintf("(%d more fields truncated)", len(workerLog.Fields)-maxLogFieldsCount)
+					break
+				}
+				maintenanceLog.Fields[k] = v
+				fieldCount++
+			}
+		}
+
+		// carry optional progress/status
+		if workerLog.Progress != 0 {
+			p := float64(workerLog.Progress)
+			maintenanceLog.Progress = &p
+		}
+		if workerLog.Status != "" {
+			maintenanceLog.Status = workerLog.Status
+		}
+		maintenanceLogs = append(maintenanceLogs, maintenanceLog)
+	}
+
+	// Persist logs
+	if s.adminServer.configPersistence != nil {
+		if err := s.adminServer.configPersistence.SaveTaskExecutionLogs(taskID, maintenanceLogs); err != nil {
+			glog.Errorf("Failed to persist logs for task %s: %v", taskID, err)
+			return err
+		}
+	}
+	return nil
+}
+
+// handleTaskLogResponse processes task log responses from workers
+func (s *WorkerGrpcServer) handleTaskLogResponse(conn *WorkerConnection, response *worker_pb.TaskLogResponse) {
+	requestKey := fmt.Sprintf("%s:%s", response.WorkerId, response.TaskId)
+
+	s.logRequestsMutex.RLock()
+	requestContext, exists := s.pendingLogRequests[requestKey]
+	s.logRequestsMutex.RUnlock()
+
+	if !exists {
+		glog.Warningf("Received unexpected log response for task %s from worker %s", response.TaskId, response.WorkerId)
+		return
+	}
+
+	glog.V(1).Infof("Received log response for task %s from worker %s: %d entries", response.TaskId, response.WorkerId, len(response.LogEntries))
+
+	// Send response to waiting channel
+	select {
+	case requestContext.ResponseCh <- response:
+		// Response delivered successfully
+	case <-time.After(time.Second):
+		glog.Warningf("Failed to deliver log response for task %s from worker %s: timeout", response.TaskId, response.WorkerId)
+	}
+
+	// Clean up the pending request
+	s.logRequestsMutex.Lock()
+	delete(s.pendingLogRequests, requestKey)
+	s.logRequestsMutex.Unlock()
+}
+
+// sendToWorker queues a message on conn.outgoing, which is deliberately never
+// closed: it has multiple concurrent senders, so closing it could panic one of
+// them with "send on closed channel". Teardown is signaled via conn.ctx instead;
+// handleOutgoingMessages drains the channel until that context is cancelled.
+// Returns false if the connection closed or the send timed out.
+func (s *WorkerGrpcServer) sendToWorker(conn *WorkerConnection, msg *worker_pb.AdminMessage, timeout time.Duration, description string) bool {
+	select {
+	case conn.outgoing <- msg:
+		return true
+	case <-conn.ctx.Done():
+		glog.V(2).Infof("Dropped %s for worker %s: connection closed", description, conn.workerID)
+		return false
+	case <-time.After(timeout):
+		glog.Warningf("Failed to send %s to worker %s: timeout", description, conn.workerID)
+		return false
+	}
+}
+
+// unregisterWorker removes a worker connection
+func (s *WorkerGrpcServer) unregisterWorker(conn *WorkerConnection, event string) {
+	s.connMutex.Lock()
+	existingConn, exists := s.connections[conn.workerID]
+	if !exists {
+		s.connMutex.Unlock()
+		glog.V(2).Infof("unregisterWorker: worker %s not found in connections map (already unregistered)", conn.workerID)
+		return
+	}
+
+	// Only remove if it matches the specific connection instance
+	if existingConn != conn {
+		s.connMutex.Unlock()
+		glog.V(1).Infof("unregisterWorker: worker %s connection replaced, skipping unregister for old connection", conn.workerID)
+		return
+	}
+
+	// Remove from map first to prevent duplicate cleanup attempts
+	delete(s.connections, conn.workerID)
+	s.connMutex.Unlock()
+	stats_collect.AdminWorkerEventsTotal.WithLabelValues(event).Inc()
+
+	// Cancel context to signal goroutines to stop. The outgoing channel is
+	// never closed (it has multiple senders); handleOutgoingMessages exits on
+	// the cancelled context.
+	conn.cancel()
+
+	glog.V(1).Infof("Unregistered worker %s", conn.workerID)
+}
+
+// cleanupRoutine periodically cleans up stale connections
+func (s *WorkerGrpcServer) cleanupRoutine() {
+	ticker := time.NewTicker(30 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-s.stopChan:
+			return
+		case <-ticker.C:
+			s.cleanupStaleConnections()
+		}
+	}
+}
+
+// cleanupStaleConnections removes connections that haven't been seen recently
+func (s *WorkerGrpcServer) cleanupStaleConnections() {
+	cutoff := time.Now().Add(-2 * time.Minute)
+
+	s.connMutex.Lock()
+	// collect connections to remove first to avoid deadlock if unregisterWorker locks
+	var toRemove []*WorkerConnection
+	for _, conn := range s.connections {
+		if conn.lastSeen.Before(cutoff) {
+			toRemove = append(toRemove, conn)
+		}
+	}
+	s.connMutex.Unlock()
+
+	for _, conn := range toRemove {
+		glog.Warningf("Cleaning up stale worker connection: %s", conn.workerID)
+		s.unregisterWorker(conn, "stale_removed")
+	}
+}
+
+// GetConnectedWorkers returns a list of currently connected workers
+func (s *WorkerGrpcServer) GetConnectedWorkers() []string {
+	s.connMutex.RLock()
+	defer s.connMutex.RUnlock()
+
+	workers := make([]string, 0, len(s.connections))
+	for workerID := range s.connections {
+		workers = append(workers, workerID)
+	}
+	return workers
+}
+
+// RequestTaskLogs requests execution logs from a worker for a specific task
+func (s *WorkerGrpcServer) RequestTaskLogs(workerID, taskID string, maxEntries int32, logLevel string) ([]*worker_pb.TaskLogEntry, error) {
+	s.connMutex.RLock()
+	conn, exists := s.connections[workerID]
+	s.connMutex.RUnlock()
+
+	if !exists {
+		return nil, fmt.Errorf("worker %s is not connected", workerID)
+	}
+
+	// Create response channel for this request
+	responseCh := make(chan *worker_pb.TaskLogResponse, 1)
+	requestKey := fmt.Sprintf("%s:%s", workerID, taskID)
+
+	// Register pending request
+	requestContext := &LogRequestContext{
+		TaskID:     taskID,
+		WorkerID:   workerID,
+		ResponseCh: responseCh,
+	}
+
+	s.logRequestsMutex.Lock()
+	if _, exists := s.pendingLogRequests[requestKey]; exists {
+		s.logRequestsMutex.Unlock()
+		return nil, fmt.Errorf("a log request for task %s is already in progress", taskID)
+	}
+	s.pendingLogRequests[requestKey] = requestContext
+	s.logRequestsMutex.Unlock()
+
+	// Create log request message
+	logRequest := &worker_pb.AdminMessage{
+		AdminId:   "admin-server",
+		Timestamp: time.Now().Unix(),
+		Message: &worker_pb.AdminMessage_TaskLogRequest{
+			TaskLogRequest: &worker_pb.TaskLogRequest{
+				TaskId:          taskID,
+				WorkerId:        workerID,
+				IncludeMetadata: true,
+				MaxEntries:      maxEntries,
+				LogLevel:        logLevel,
+			},
+		},
+	}
+
+	// Send the request through the worker's outgoing channel
+	select {
+	case conn.outgoing <- logRequest:
+		glog.V(1).Infof("Log request sent to worker %s for task %s", workerID, taskID)
+	case <-conn.ctx.Done():
+		s.logRequestsMutex.Lock()
+		if s.pendingLogRequests[requestKey] == requestContext {
+			delete(s.pendingLogRequests, requestKey)
+		}
+		s.logRequestsMutex.Unlock()
+		return nil, fmt.Errorf("worker %s connection closed", workerID)
+	case <-time.After(logSendTimeout):
+		// Clean up pending request on timeout
+		s.logRequestsMutex.Lock()
+		if s.pendingLogRequests[requestKey] == requestContext {
+			delete(s.pendingLogRequests, requestKey)
+		}
+		s.logRequestsMutex.Unlock()
+		return nil, fmt.Errorf("timeout sending log request to worker %s", workerID)
+	}
+
+	// Wait for response
+	select {
+	case response := <-responseCh:
+		if !response.Success {
+			return nil, fmt.Errorf("worker log request failed: %s", response.ErrorMessage)
+		}
+		glog.V(1).Infof("Received %d log entries for task %s from worker %s", len(response.LogEntries), taskID, workerID)
+		return response.LogEntries, nil
+	case <-conn.ctx.Done():
+		s.logRequestsMutex.Lock()
+		if s.pendingLogRequests[requestKey] == requestContext {
+			delete(s.pendingLogRequests, requestKey)
+		}
+		s.logRequestsMutex.Unlock()
+		return nil, fmt.Errorf("worker %s connection closed", workerID)
+	case <-time.After(logResponseTimeout):
+		// Clean up pending request on timeout
+		s.logRequestsMutex.Lock()
+		if s.pendingLogRequests[requestKey] == requestContext {
+			delete(s.pendingLogRequests, requestKey)
+		}
+		s.logRequestsMutex.Unlock()
+		return nil, fmt.Errorf("timeout waiting for log response from worker %s", workerID)
+	}
+}
+
+// RequestTaskLogsFromAllWorkers requests logs for a task from all connected workers
+func (s *WorkerGrpcServer) RequestTaskLogsFromAllWorkers(taskID string, maxEntries int32, logLevel string) (map[string][]*worker_pb.TaskLogEntry, error) {
+	s.connMutex.RLock()
+	workerIDs := make([]string, 0, len(s.connections))
+	for workerID := range s.connections {
+		workerIDs = append(workerIDs, workerID)
+	}
+	s.connMutex.RUnlock()
+
+	results := make(map[string][]*worker_pb.TaskLogEntry)
+
+	for _, workerID := range workerIDs {
+		logs, err := s.RequestTaskLogs(workerID, taskID, maxEntries, logLevel)
+		if err != nil {
+			glog.V(1).Infof("Failed to get logs from worker %s for task %s: %v", workerID, taskID, err)
+			// Store empty result with error information for debugging
+			results[workerID+"_error"] = []*worker_pb.TaskLogEntry{
+				{
+					Timestamp: time.Now().Unix(),
+					Level:     "ERROR",
+					Message:   fmt.Sprintf("Failed to retrieve logs from worker %s: %v", workerID, err),
+					Fields:    map[string]string{"source": "admin"},
+				},
+			}
+			continue
+		}
+		if len(logs) > 0 {
+			results[workerID] = logs
+		} else {
+			glog.V(2).Infof("No logs found for task %s on worker %s", taskID, workerID)
+		}
+	}
+
+	return results, nil
+}
+
+func findClientAddress(ctx context.Context) string {
+	// fmt.Printf("FromContext %+v\n", ctx)
+	pr, ok := peer.FromContext(ctx)
+	if !ok {
+		glog.Error("failed to get peer from ctx")
+		return ""
+	}
+	if pr.Addr == net.Addr(nil) {
+		glog.Error("failed to get peer address")
+		return ""
+	}
+	return pr.Addr.String()
+}
+
+// activeLogFetchLoop periodically fetches logs for all in-progress tasks
+func (s *WorkerGrpcServer) activeLogFetchLoop() {
+	ticker := time.NewTicker(30 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-s.stopChan:
+			return
+		case <-ticker.C:
+			if !s.running || s.adminServer == nil || s.adminServer.maintenanceManager == nil {
+				continue
+			}
+
+			// Get all in-progress tasks
+			tasks := s.adminServer.maintenanceManager.GetTasks(maintenance.TaskStatusInProgress, "", 0)
+			if len(tasks) == 0 {
+				continue
+			}
+
+			glog.V(2).Infof("Background log fetcher: found %d in-progress tasks", len(tasks))
+			for _, task := range tasks {
+				if task.WorkerID != "" {
+					// Use a goroutine to avoid blocking the loop
+					go func(wID, tID string) {
+						if err := s.FetchAndSaveLogs(wID, tID); err != nil {
+							glog.V(2).Infof("Background log fetch failed for task %s on worker %s: %v", tID, wID, err)
+						}
+					}(task.WorkerID, task.ID)
+				}
+			}
+		}
+	}
+}