snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/pb/plugin.proto

@@ -0,0 +1,515 @@
+syntax = "proto3";
+
+package plugin;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb";
+
+import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
+
+// PluginControlService is the admin-facing stream API for external workers.
+// Workers initiate and keep this stream alive; all control plane traffic flows through it.
+service PluginControlService {
+  rpc WorkerStream(stream WorkerToAdminMessage) returns (stream AdminToWorkerMessage);
+}
+
+// WorkerToAdminMessage carries worker-originated events and responses.
+message WorkerToAdminMessage {
+  string worker_id = 1;
+  google.protobuf.Timestamp sent_at = 2;
+
+  oneof body {
+    WorkerHello hello = 10;
+    WorkerHeartbeat heartbeat = 11;
+    WorkerAcknowledge acknowledge = 12;
+    ConfigSchemaResponse config_schema_response = 13;
+    DetectionProposals detection_proposals = 14;
+    DetectionComplete detection_complete = 15;
+    JobProgressUpdate job_progress_update = 16;
+    JobCompleted job_completed = 17;
+    WorkerObservations observations = 18;
+    ObjectPreviewResponse object_preview_response = 19;
+  }
+}
+
+// AdminToWorkerMessage carries commands and lifecycle notifications from admin.
+message AdminToWorkerMessage {
+  string request_id = 1;
+  google.protobuf.Timestamp sent_at = 2;
+
+  oneof body {
+    AdminHello hello = 10;
+    RequestConfigSchema request_config_schema = 11;
+    RunDetectionRequest run_detection_request = 12;
+    ExecuteJobRequest execute_job_request = 13;
+    CancelRequest cancel_request = 14;
+    AdminShutdown shutdown = 15;
+    RequestObjectPreview request_object_preview = 16;
+  }
+}
+
+message WorkerHello {
+  string worker_id = 1;
+  string worker_instance_id = 2;
+  string address = 3;
+  string worker_version = 4;
+  string protocol_version = 5;
+  repeated JobTypeCapability capabilities = 6;
+  map<string, string> metadata = 7;
+}
+
+message AdminHello {
+  bool accepted = 1;
+  string message = 2;
+  int32 heartbeat_interval_seconds = 3;
+  int32 reconnect_delay_seconds = 4;
+}
+
+message WorkerHeartbeat {
+  string worker_id = 1;
+  repeated RunningWork running_work = 2;
+  int32 detection_slots_used = 3;
+  int32 detection_slots_total = 4;
+  int32 execution_slots_used = 5;
+  int32 execution_slots_total = 6;
+  map<string, int32> queued_jobs_by_type = 7;
+  map<string, string> metadata = 8;
+}
+
+message WorkerAcknowledge {
+  string request_id = 1;
+  bool accepted = 2;
+  string message = 3;
+}
+
+message RunningWork {
+  string work_id = 1;
+  WorkKind kind = 2;
+  string job_type = 3;
+  JobState state = 4;
+  double progress_percent = 5;
+  string stage = 6;
+}
+
+message JobTypeCapability {
+  string job_type = 1;
+  bool can_detect = 2;
+  bool can_execute = 3;
+  int32 max_detection_concurrency = 4;
+  int32 max_execution_concurrency = 5;
+  string display_name = 6;
+  string description = 7;
+  int32 weight = 8;
+}
+
+message RequestConfigSchema {
+  string job_type = 1;
+  bool force_refresh = 2;
+}
+
+message ConfigSchemaResponse {
+  string request_id = 1;
+  string job_type = 2;
+  bool success = 3;
+  string error_message = 4;
+  JobTypeDescriptor job_type_descriptor = 5;
+}
+
+// JobTypeDescriptor defines one job type contract, including UI schema and defaults.
+message JobTypeDescriptor {
+  string job_type = 1;
+  string display_name = 2;
+  string description = 3;
+  string icon = 4;
+  uint32 descriptor_version = 5;
+
+  // Admin-owned options such as detection frequency and dispatch concurrency.
+  ConfigForm admin_config_form = 6;
+  // Worker-owned options used during detection and execution.
+  ConfigForm worker_config_form = 7;
+
+  AdminRuntimeDefaults admin_runtime_defaults = 8;
+  map<string, ConfigValue> worker_default_values = 9;
+}
+
+message ConfigForm {
+  string form_id = 1;
+  string title = 2;
+  string description = 3;
+  repeated ConfigSection sections = 4;
+  map<string, ConfigValue> default_values = 5;
+}
+
+message ConfigSection {
+  string section_id = 1;
+  string title = 2;
+  string description = 3;
+  repeated ConfigField fields = 4;
+}
+
+message ConfigField {
+  string name = 1;
+  string label = 2;
+  string description = 3;
+  string help_text = 4;
+  string placeholder = 5;
+
+  ConfigFieldType field_type = 6;
+  ConfigWidget widget = 7;
+
+  bool required = 8;
+  bool read_only = 9;
+  bool sensitive = 10;
+
+  ConfigValue min_value = 11;
+  ConfigValue max_value = 12;
+
+  repeated ConfigOption options = 13;
+  repeated ValidationRule validation_rules = 14;
+
+  // Simple visibility dependency: show this field when the referenced field equals value.
+  string visible_when_field = 15;
+  ConfigValue visible_when_equals = 16;
+}
+
+message ConfigOption {
+  string value = 1;
+  string label = 2;
+  string description = 3;
+  bool disabled = 4;
+}
+
+message ValidationRule {
+  ValidationRuleType type = 1;
+  string expression = 2;
+  string error_message = 3;
+}
+
+message ConfigValue {
+  oneof kind {
+    bool bool_value = 1;
+    int64 int64_value = 2;
+    double double_value = 3;
+    string string_value = 4;
+    bytes bytes_value = 5;
+    google.protobuf.Duration duration_value = 6;
+    StringList string_list = 7;
+    Int64List int64_list = 8;
+    DoubleList double_list = 9;
+    BoolList bool_list = 10;
+    ValueList list_value = 11;
+    ValueMap map_value = 12;
+  }
+}
+
+message StringList {
+  repeated string values = 1;
+}
+
+message Int64List {
+  repeated int64 values = 1;
+}
+
+message DoubleList {
+  repeated double values = 1;
+}
+
+message BoolList {
+  repeated bool values = 1;
+}
+
+message ValueList {
+  repeated ConfigValue values = 1;
+}
+
+message ValueMap {
+  map<string, ConfigValue> fields = 1;
+}
+
+message AdminRuntimeDefaults {
+  bool enabled = 1;
+  int32 detection_interval_minutes = 2;
+  int32 detection_timeout_seconds = 3;
+  int32 max_jobs_per_detection = 4;
+  int32 global_execution_concurrency = 5;
+  int32 per_worker_execution_concurrency = 6;
+  int32 retry_limit = 7;
+  int32 retry_backoff_seconds = 8;
+  int32 job_type_max_runtime_seconds = 9;
+  // Per-attempt execution deadline for one job. Distinct from
+  // detection_timeout (which bounds the detection scan) and
+  // job_type_max_runtime (a lane-wide budget across all attempts).
+  int32 execution_timeout_seconds = 10;
+}
+
+message AdminRuntimeConfig {
+  bool enabled = 1;
+  int32 detection_interval_minutes = 2;
+  int32 detection_timeout_seconds = 3;
+  int32 max_jobs_per_detection = 4;
+  int32 global_execution_concurrency = 5;
+  int32 per_worker_execution_concurrency = 6;
+  int32 retry_limit = 7;
+  int32 retry_backoff_seconds = 8;
+  int32 job_type_max_runtime_seconds = 9;
+  int32 execution_timeout_seconds = 10;
+}
+
+message RunDetectionRequest {
+  string request_id = 1;
+  string job_type = 2;
+  int64 detection_sequence = 3;
+
+  AdminRuntimeConfig admin_runtime = 4;
+  map<string, ConfigValue> admin_config_values = 5;
+  map<string, ConfigValue> worker_config_values = 6;
+
+  ClusterContext cluster_context = 7;
+  google.protobuf.Timestamp last_successful_run = 8;
+  int32 max_results = 9;
+}
+
+message DetectionProposals {
+  string request_id = 1;
+  string job_type = 2;
+  repeated JobProposal proposals = 3;
+  bool has_more = 4;
+}
+
+message DetectionComplete {
+  string request_id = 1;
+  string job_type = 2;
+  bool success = 3;
+  string error_message = 4;
+  int32 total_proposals = 5;
+}
+
+message JobProposal {
+  string proposal_id = 1;
+  string dedupe_key = 2;
+  string job_type = 3;
+  JobPriority priority = 4;
+  string summary = 5;
+  string detail = 6;
+  map<string, ConfigValue> parameters = 7;
+  map<string, string> labels = 8;
+  google.protobuf.Timestamp not_before = 9;
+  google.protobuf.Timestamp expires_at = 10;
+}
+
+message ExecuteJobRequest {
+  string request_id = 1;
+  JobSpec job = 2;
+  AdminRuntimeConfig admin_runtime = 3;
+  map<string, ConfigValue> admin_config_values = 4;
+  map<string, ConfigValue> worker_config_values = 5;
+  ClusterContext cluster_context = 6;
+  int32 attempt = 7;
+}
+
+message JobSpec {
+  string job_id = 1;
+  string job_type = 2;
+  string dedupe_key = 3;
+  JobPriority priority = 4;
+  string summary = 5;
+  string detail = 6;
+  map<string, ConfigValue> parameters = 7;
+  map<string, string> labels = 8;
+  google.protobuf.Timestamp created_at = 9;
+  google.protobuf.Timestamp scheduled_at = 10;
+}
+
+message JobProgressUpdate {
+  string request_id = 1;
+  string job_id = 2;
+  string job_type = 3;
+  JobState state = 4;
+  double progress_percent = 5;
+  string stage = 6;
+  string message = 7;
+  map<string, ConfigValue> metrics = 8;
+  repeated ActivityEvent activities = 9;
+  google.protobuf.Timestamp updated_at = 10;
+}
+
+message JobCompleted {
+  string request_id = 1;
+  string job_id = 2;
+  string job_type = 3;
+  bool success = 4;
+  string error_message = 5;
+  JobResult result = 6;
+  repeated ActivityEvent activities = 7;
+  google.protobuf.Timestamp completed_at = 8;
+}
+
+message JobResult {
+  map<string, ConfigValue> output_values = 1;
+  string summary = 2;
+}
+
+message ClusterContext {
+  repeated string master_grpc_addresses = 1;
+  // Filers in pb.ServerAddress form (host:httpPort.grpcPort). Consumers
+  // convert to a gRPC or HTTP address as needed; see weed/pb/server_address.go.
+  repeated string filer_addresses = 2;
+  repeated string volume_grpc_addresses = 3;
+  map<string, string> metadata = 4;
+  repeated string s3_grpc_addresses = 5;
+}
+
+message ActivityEvent {
+  ActivitySource source = 1;
+  string message = 2;
+  string stage = 3;
+  map<string, ConfigValue> details = 4;
+  google.protobuf.Timestamp created_at = 5;
+}
+
+message CancelRequest {
+  string target_id = 1;
+  WorkKind target_kind = 2;
+  string reason = 3;
+  bool force = 4;
+}
+
+message AdminShutdown {
+  string reason = 1;
+  int32 grace_period_seconds = 2;
+}
+
+// PersistedJobTypeConfig is the admin-side on-disk model per job type.
+message PersistedJobTypeConfig {
+  string job_type = 1;
+  uint32 descriptor_version = 2;
+  map<string, ConfigValue> admin_config_values = 3;
+  map<string, ConfigValue> worker_config_values = 4;
+  AdminRuntimeConfig admin_runtime = 5;
+  google.protobuf.Timestamp updated_at = 6;
+  string updated_by = 7;
+}
+
+enum WorkKind {
+  WORK_KIND_UNSPECIFIED = 0;
+  WORK_KIND_DETECTION = 1;
+  WORK_KIND_EXECUTION = 2;
+}
+
+enum JobPriority {
+  JOB_PRIORITY_UNSPECIFIED = 0;
+  JOB_PRIORITY_LOW = 1;
+  JOB_PRIORITY_NORMAL = 2;
+  JOB_PRIORITY_HIGH = 3;
+  JOB_PRIORITY_CRITICAL = 4;
+}
+
+enum JobState {
+  JOB_STATE_UNSPECIFIED = 0;
+  JOB_STATE_PENDING = 1;
+  JOB_STATE_ASSIGNED = 2;
+  JOB_STATE_RUNNING = 3;
+  JOB_STATE_SUCCEEDED = 4;
+  JOB_STATE_FAILED = 5;
+  JOB_STATE_CANCELED = 6;
+}
+
+enum ConfigFieldType {
+  CONFIG_FIELD_TYPE_UNSPECIFIED = 0;
+  CONFIG_FIELD_TYPE_BOOL = 1;
+  CONFIG_FIELD_TYPE_INT64 = 2;
+  CONFIG_FIELD_TYPE_DOUBLE = 3;
+  CONFIG_FIELD_TYPE_STRING = 4;
+  CONFIG_FIELD_TYPE_BYTES = 5;
+  CONFIG_FIELD_TYPE_DURATION = 6;
+  CONFIG_FIELD_TYPE_ENUM = 7;
+  CONFIG_FIELD_TYPE_LIST = 8;
+  CONFIG_FIELD_TYPE_OBJECT = 9;
+}
+
+enum ConfigWidget {
+  CONFIG_WIDGET_UNSPECIFIED = 0;
+  CONFIG_WIDGET_TOGGLE = 1;
+  CONFIG_WIDGET_TEXT = 2;
+  CONFIG_WIDGET_TEXTAREA = 3;
+  CONFIG_WIDGET_NUMBER = 4;
+  CONFIG_WIDGET_SELECT = 5;
+  CONFIG_WIDGET_MULTI_SELECT = 6;
+  CONFIG_WIDGET_DURATION = 7;
+  CONFIG_WIDGET_PASSWORD = 8;
+}
+
+enum ValidationRuleType {
+  VALIDATION_RULE_TYPE_UNSPECIFIED = 0;
+  VALIDATION_RULE_TYPE_REGEX = 1;
+  VALIDATION_RULE_TYPE_MIN_LENGTH = 2;
+  VALIDATION_RULE_TYPE_MAX_LENGTH = 3;
+  VALIDATION_RULE_TYPE_MIN_ITEMS = 4;
+  VALIDATION_RULE_TYPE_MAX_ITEMS = 5;
+  VALIDATION_RULE_TYPE_CUSTOM = 6;
+}
+
+enum ActivitySource {
+  ACTIVITY_SOURCE_UNSPECIFIED = 0;
+  ACTIVITY_SOURCE_ADMIN = 1;
+  ACTIVITY_SOURCE_DETECTOR = 2;
+  ACTIVITY_SOURCE_EXECUTOR = 3;
+}
+
+// WorkerObservations reports what a worker learned about the objects it
+// inspected. A worker opens a table to decide whether it needs a job; what it
+// saw is worth keeping either way, because for a format the cluster cannot
+// read, the worker is the only thing that can describe it.
+//
+// Observations are not work. Admin caches them and serves them back, so a
+// worker that goes away leaves its last ones standing rather than blanking a
+// page.
+message WorkerObservations {
+  string job_type = 1;
+  repeated ObjectObservation observations = 2;
+}
+
+// ObjectObservation is one object as a worker last saw it.
+message ObjectObservation {
+  // Identifier of the object, e.g. ["bucket", "namespace", "table"].
+  repeated string object_id = 1;
+  // What kind of object this is, e.g. "table".
+  string object_kind = 2;
+  // The object's format, e.g. "LANCE", so a reader can tell whose observation
+  // this is without parsing the attributes.
+  string format = 3;
+  // Whatever the worker can cheaply say: schema, row count, fragment count.
+  // The keys are the worker's to choose.
+  map<string, ConfigValue> attributes = 4;
+  google.protobuf.Timestamp observed_at = 5;
+}
+
+// RequestObjectPreview asks a worker for sample rows of an object admin cannot
+// read itself. Unlike an observation this is not cached: it is fetched when
+// someone opens the page, because rows are the object's data rather than a
+// description of it, and holding a copy in admin is neither fresh nor its
+// business.
+message RequestObjectPreview {
+  repeated string object_id = 1;
+  // The format admin believes this object is, so a worker that does not own it
+  // can decline instead of guessing.
+  string format = 2;
+  int32 row_limit = 3;
+}
+
+// ObjectPreviewResponse carries the sample back, already rendered as text. The
+// worker is the only thing that knows the object's types, so it formats them;
+// admin displays what it is given.
+message ObjectPreviewResponse {
+  string request_id = 6;
+  bool success = 1;
+  string error_message = 2;
+  repeated string columns = 3;
+  repeated PreviewRow rows = 4;
+  // Rows in the object, which is not the number sampled.
+  int64 total_rows = 5;
+}
+
+message PreviewRow {
+  repeated string values = 1;
+}