snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/replication/sub/notification_aws_sqs.go

@@ -0,0 +1,124 @@
+package sub
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/sqs"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/protobuf/proto"
+)
+
+func init() {
+	NotificationInputs = append(NotificationInputs, &AwsSqsInput{})
+}
+
+type AwsSqsInput struct {
+	svc      *sqs.SQS
+	queueUrl string
+}
+
+func (k *AwsSqsInput) GetName() string {
+	return "aws_sqs"
+}
+
+func (k *AwsSqsInput) Initialize(configuration util.Configuration, prefix string) error {
+	glog.V(0).Infof("replication.notification.aws_sqs.region: %v", configuration.GetString(prefix+"region"))
+	glog.V(0).Infof("replication.notification.aws_sqs.sqs_queue_name: %v", configuration.GetString(prefix+"sqs_queue_name"))
+	return k.initialize(
+		configuration.GetString(prefix+"aws_access_key_id"),
+		configuration.GetString(prefix+"aws_secret_access_key"),
+		configuration.GetString(prefix+"region"),
+		configuration.GetString(prefix+"sqs_queue_name"),
+	)
+}
+
+func (k *AwsSqsInput) initialize(awsAccessKeyId, awsSecretAccessKey, region, queueName string) (err error) {
+
+	config := &aws.Config{
+		Region:                        aws.String(region),
+		S3DisableContentMD5Validation: aws.Bool(true),
+	}
+	if awsAccessKeyId != "" && awsSecretAccessKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, awsSecretAccessKey, "")
+	}
+
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return fmt.Errorf("create aws session: %w", err)
+	}
+	k.svc = sqs.New(sess)
+
+	result, err := k.svc.GetQueueUrl(&sqs.GetQueueUrlInput{
+		QueueName: aws.String(queueName),
+	})
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == sqs.ErrCodeQueueDoesNotExist {
+			return fmt.Errorf("unable to find queue %s", queueName)
+		}
+		return fmt.Errorf("get queue %s url: %v", queueName, err)
+	}
+
+	k.queueUrl = *result.QueueUrl
+
+	return nil
+}
+
+func (k *AwsSqsInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error) {
+
+	// receive message
+	result, err := k.svc.ReceiveMessage(&sqs.ReceiveMessageInput{
+		AttributeNames: []*string{
+			aws.String(sqs.MessageSystemAttributeNameSentTimestamp),
+		},
+		MessageAttributeNames: []*string{
+			aws.String(sqs.QueueAttributeNameAll),
+		},
+		QueueUrl:            &k.queueUrl,
+		MaxNumberOfMessages: aws.Int64(1),
+		VisibilityTimeout:   aws.Int64(20), // 20 seconds
+		WaitTimeSeconds:     aws.Int64(20),
+	})
+	if err != nil {
+		err = fmt.Errorf("receive message from sqs %s: %v", k.queueUrl, err)
+		return
+	}
+	if len(result.Messages) == 0 {
+		return
+	}
+
+	// process the message
+	// fmt.Printf("messages: %+v\n", result.Messages[0])
+	keyValue := result.Messages[0].MessageAttributes["key"]
+	key = *keyValue.StringValue
+	text := *result.Messages[0].Body
+	message = &filer_pb.EventNotification{}
+	err = proto.Unmarshal([]byte(text), message)
+	if err != nil {
+		err = fmt.Errorf("unmarshal message from sqs %s: %w", k.queueUrl, err)
+		return
+	}
+
+	// Delete only once the message has been replicated. Deleting on receipt
+	// drops the message for good when the sink write fails; leaving it in the
+	// queue lets the visibility timeout redeliver it.
+	receiptHandle := result.Messages[0].ReceiptHandle
+	onSuccessFn = func() {
+		if _, deleteErr := k.svc.DeleteMessage(&sqs.DeleteMessageInput{
+			QueueUrl:      &k.queueUrl,
+			ReceiptHandle: receiptHandle,
+		}); deleteErr != nil {
+			glog.V(1).Infof("delete message from sqs %s: %v", k.queueUrl, deleteErr)
+		}
+	}
+	onFailureFn = func() {
+		glog.V(1).Infof("keeping message %s in sqs %s for redelivery", key, k.queueUrl)
+	}
+
+	return
+}

weed/replication/sub/notification_gocdk_pub_sub.go

@@ -0,0 +1,167 @@
+//go:build gocdk
+
+package sub
+
+import (
+	"context"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	amqp "github.com/rabbitmq/amqp091-go"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"gocloud.dev/pubsub"
+	_ "gocloud.dev/pubsub/awssnssqs"
+	"gocloud.dev/pubsub/rabbitpubsub"
+	"google.golang.org/protobuf/proto"
+
+	// _ "gocloud.dev/pubsub/azuresb"
+	_ "gocloud.dev/pubsub/gcppubsub"
+	_ "gocloud.dev/pubsub/natspubsub"
+	_ "gocloud.dev/pubsub/rabbitpubsub"
+)
+
+func init() {
+	NotificationInputs = append(NotificationInputs, &GoCDKPubSubInput{})
+}
+
+func getPath(rawUrl string) string {
+	parsedUrl, _ := url.Parse(rawUrl)
+	return path.Join(parsedUrl.Host, parsedUrl.Path)
+}
+
+func QueueDeclareAndBind(conn *amqp.Connection, exchangeUrl string, queueUrl string) error {
+	exchangeName := getPath(exchangeUrl)
+	queueName := getPath(queueUrl)
+	exchangeNameDLX := "DLX." + exchangeName
+	queueNameDLX := "DLX." + queueName
+	ch, err := conn.Channel()
+	if err != nil {
+		glog.Error(err)
+		return err
+	}
+	defer ch.Close()
+	if err := ch.ExchangeDeclare(
+		exchangeNameDLX, "fanout", true, false, false, false, nil); err != nil {
+		glog.Error(err)
+		return err
+	}
+	if err := ch.ExchangeDeclare(
+		exchangeName, "fanout", true, false, false, false, nil); err != nil {
+		glog.Error(err)
+		return err
+	}
+	if _, err := ch.QueueDeclare(
+		queueName, true, false, false, false,
+		amqp.Table{"x-dead-letter-exchange": exchangeNameDLX}); err != nil {
+		glog.Error(err)
+		return err
+	}
+	if err := ch.QueueBind(queueName, "", exchangeName, false, nil); err != nil {
+		glog.Error(err)
+		return err
+	}
+	if _, err := ch.QueueDeclare(
+		queueNameDLX, true, false, false, false,
+		amqp.Table{"x-dead-letter-exchange": exchangeName, "x-message-ttl": 600000}); err != nil {
+		glog.Error(err)
+		return err
+	}
+	if err := ch.QueueBind(queueNameDLX, "", exchangeNameDLX, false, nil); err != nil {
+		glog.Error(err)
+		return err
+	}
+	return nil
+}
+
+type GoCDKPubSubInput struct {
+	sub    *pubsub.Subscription
+	subURL string
+}
+
+func (k *GoCDKPubSubInput) GetName() string {
+	return "gocdk_pub_sub"
+}
+
+func (k *GoCDKPubSubInput) Initialize(configuration util.Configuration, prefix string) error {
+	topicUrl := configuration.GetString(prefix + "topic_url")
+	k.subURL = configuration.GetString(prefix + "sub_url")
+	glog.V(0).Infof("notification.gocdk_pub_sub.sub_url: %v", k.subURL)
+	sub, err := pubsub.OpenSubscription(context.Background(), k.subURL)
+	if err != nil {
+		return err
+	}
+	var conn *amqp.Connection
+	if sub.As(&conn) {
+		ch, err := conn.Channel()
+		if err != nil {
+			return err
+		}
+		defer ch.Close()
+		_, err = ch.QueueInspect(getPath(k.subURL))
+		if err != nil {
+			if strings.HasPrefix(err.Error(), "Exception (404) Reason") {
+				if err := QueueDeclareAndBind(conn, topicUrl, k.subURL); err != nil {
+					return err
+				}
+			} else {
+				return err
+			}
+		}
+	}
+	k.sub = sub
+	return nil
+}
+
+func (k *GoCDKPubSubInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error) {
+	ctx := context.Background()
+	msg, err := k.sub.Receive(ctx)
+	if err != nil {
+		var conn *amqp.Connection
+		if k.sub.As(&conn) && conn.IsClosed() {
+			conn.Close()
+			k.sub.Shutdown(ctx)
+			conn, err = amqp.Dial(os.Getenv("RABBIT_SERVER_URL"))
+			if err != nil {
+				glog.Error(err)
+				time.Sleep(time.Second)
+				return
+			}
+			k.sub = rabbitpubsub.OpenSubscription(conn, getPath(k.subURL), nil)
+			return
+		}
+		// This is permanent cached sub err
+		glog.Fatal(err)
+	}
+	onFailureFn = func() {
+		if msg.Nackable() {
+			isRedelivered := false
+			var delivery amqp.Delivery
+			if msg.As(&delivery) {
+				isRedelivered = delivery.Redelivered
+				glog.Warningf("onFailureFn() metadata: %+v, redelivered: %v", msg.Metadata, delivery.Redelivered)
+			}
+			if isRedelivered {
+				if err := delivery.Nack(false, false); err != nil {
+					glog.Error(err)
+				}
+			} else {
+				msg.Nack()
+			}
+		}
+	}
+	onSuccessFn = func() {
+		msg.Ack()
+	}
+	key = msg.Metadata["key"]
+	message = &filer_pb.EventNotification{}
+	err = proto.Unmarshal(msg.Body, message)
+	if err != nil {
+		return "", nil, onSuccessFn, onFailureFn, err
+	}
+	return key, message, onSuccessFn, onFailureFn, nil
+}

weed/replication/sub/notification_google_pub_sub.go

@@ -0,0 +1,115 @@
+package sub
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"cloud.google.com/go/pubsub"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/api/option"
+	"google.golang.org/protobuf/proto"
+)
+
+func init() {
+	NotificationInputs = append(NotificationInputs, &GooglePubSubInput{})
+}
+
+type GooglePubSubInput struct {
+	sub         *pubsub.Subscription
+	topicName   string
+	messageChan chan *pubsub.Message
+}
+
+func (k *GooglePubSubInput) GetName() string {
+	return "google_pub_sub"
+}
+
+func (k *GooglePubSubInput) Initialize(configuration util.Configuration, prefix string) error {
+	glog.V(0).Infof("notification.google_pub_sub.project_id: %v", configuration.GetString(prefix+"project_id"))
+	glog.V(0).Infof("notification.google_pub_sub.topic: %v", configuration.GetString(prefix+"topic"))
+	return k.initialize(
+		configuration.GetString(prefix+"google_application_credentials"),
+		configuration.GetString(prefix+"project_id"),
+		configuration.GetString(prefix+"topic"),
+	)
+}
+
+func (k *GooglePubSubInput) initialize(google_application_credentials, projectId, topicName string) (err error) {
+
+	ctx := context.Background()
+	// Creates a client.
+	if google_application_credentials == "" {
+		var found bool
+		google_application_credentials, found = os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS")
+		if !found {
+			glog.Fatalf("need to specific GOOGLE_APPLICATION_CREDENTIALS env variable or google_application_credentials in filer.toml")
+		}
+	}
+
+	client, err := pubsub.NewClient(ctx, projectId, option.WithCredentialsFile(google_application_credentials))
+	if err != nil {
+		glog.Fatalf("Failed to create client: %v", err)
+	}
+
+	k.topicName = topicName
+	topic := client.Topic(topicName)
+	if exists, err := topic.Exists(ctx); err == nil {
+		if !exists {
+			topic, err = client.CreateTopic(ctx, topicName)
+			if err != nil {
+				glog.Fatalf("Failed to create topic %s: %v", topicName, err)
+			}
+		}
+	} else {
+		glog.Fatalf("Failed to check topic %s: %v", topicName, err)
+	}
+
+	subscriptionName := "seaweedfs_sub"
+
+	k.sub = client.Subscription(subscriptionName)
+	if exists, err := k.sub.Exists(ctx); err == nil {
+		if !exists {
+			k.sub, err = client.CreateSubscription(ctx, subscriptionName, pubsub.SubscriptionConfig{Topic: topic})
+			if err != nil {
+				glog.Fatalf("Failed to create subscription %s: %v", subscriptionName, err)
+			}
+		}
+	} else {
+		glog.Fatalf("Failed to check subscription %s: %v", topicName, err)
+	}
+
+	k.messageChan = make(chan *pubsub.Message, 1)
+
+	go k.sub.Receive(ctx, func(ctx context.Context, m *pubsub.Message) {
+		k.messageChan <- m
+	})
+
+	return err
+}
+
+func (k *GooglePubSubInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error) {
+
+	m := <-k.messageChan
+
+	onSuccessFn = func() {
+		m.Ack()
+	}
+	onFailureFn = func() {
+		m.Nack()
+	}
+
+	// process the message
+	key = m.Attributes["key"]
+	message = &filer_pb.EventNotification{}
+	err = proto.Unmarshal(m.Data, message)
+
+	if err != nil {
+		err = fmt.Errorf("unmarshal message from google pubsub %s: %v", k.topicName, err)
+		return
+	}
+
+	return
+}

weed/replication/sub/notification_kafka.go

@@ -0,0 +1,203 @@
+package sub
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/Shopify/sarama"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	kafkanotif "github.com/seaweedfs/seaweedfs/weed/notification/kafka"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/protobuf/proto"
+)
+
+func init() {
+	NotificationInputs = append(NotificationInputs, &KafkaInput{})
+}
+
+type KafkaInput struct {
+	topic       string
+	consumer    sarama.Consumer
+	messageChan chan *sarama.ConsumerMessage
+	progress    *KafkaProgress
+}
+
+func (k *KafkaInput) GetName() string {
+	return "kafka"
+}
+
+func (k *KafkaInput) Initialize(configuration util.Configuration, prefix string) error {
+	glog.V(0).Infof("replication.notification.kafka.hosts: %v\n", configuration.GetStringSlice(prefix+"hosts"))
+	glog.V(0).Infof("replication.notification.kafka.topic: %v\n", configuration.GetString(prefix+"topic"))
+	return k.initialize(
+		configuration.GetStringSlice(prefix+"hosts"),
+		configuration.GetString(prefix+"topic"),
+		configuration.GetString(prefix+"offsetFile"),
+		configuration.GetInt(prefix+"offsetSaveIntervalSeconds"),
+		kafkanotif.SASLTLSConfig{
+			SASLEnabled:           configuration.GetBool(prefix + "sasl_enabled"),
+			SASLMechanism:         configuration.GetString(prefix + "sasl_mechanism"),
+			SASLUsername:          configuration.GetString(prefix + "sasl_username"),
+			SASLPassword:          configuration.GetString(prefix + "sasl_password"),
+			TLSEnabled:            configuration.GetBool(prefix + "tls_enabled"),
+			TLSCACert:             configuration.GetString(prefix + "tls_ca_cert"),
+			TLSClientCert:         configuration.GetString(prefix + "tls_client_cert"),
+			TLSClientKey:          configuration.GetString(prefix + "tls_client_key"),
+			TLSInsecureSkipVerify: configuration.GetBool(prefix + "tls_insecure_skip_verify"),
+		},
+	)
+}
+
+func (k *KafkaInput) initialize(hosts []string, topic string, offsetFile string, offsetSaveIntervalSeconds int, saslTLS kafkanotif.SASLTLSConfig) (err error) {
+	config := sarama.NewConfig()
+	config.Consumer.Return.Errors = true
+	if err = kafkanotif.ConfigureSASLTLS(config, saslTLS); err != nil {
+		return fmt.Errorf("kafka consumer security configuration: %w", err)
+	}
+	k.consumer, err = sarama.NewConsumer(hosts, config)
+	if err != nil {
+		return fmt.Errorf("create kafka consumer: %w", err)
+	}
+	glog.V(0).Infof("connected to %v", hosts)
+
+	k.topic = topic
+	k.messageChan = make(chan *sarama.ConsumerMessage, 1)
+
+	partitions, err := k.consumer.Partitions(topic)
+	if err != nil {
+		return fmt.Errorf("get kafka partitions for topic %q: %w", topic, err)
+	}
+
+	progress := loadProgress(offsetFile)
+	if progress == nil || progress.Topic != topic {
+		progress = &KafkaProgress{
+			Topic:            topic,
+			PartitionOffsets: make(map[int32]int64),
+		}
+	}
+	progress.lastSaveTime = time.Now()
+	progress.offsetFile = offsetFile
+	progress.offsetSaveIntervalSeconds = offsetSaveIntervalSeconds
+	progress.failedOffsets = make(map[int32]int64)
+	k.progress = progress
+
+	for _, partition := range partitions {
+		offset, found := progress.PartitionOffsets[partition]
+		if !found {
+			offset = sarama.OffsetOldest
+		} else {
+			offset += 1
+		}
+		partitionConsumer, err := k.consumer.ConsumePartition(topic, partition, offset)
+		if err != nil {
+			return fmt.Errorf("consume kafka topic %q partition %d: %w", topic, partition, err)
+		}
+		go func() {
+			for {
+				select {
+				case err := <-partitionConsumer.Errors():
+					fmt.Println(err)
+				case msg := <-partitionConsumer.Messages():
+					k.messageChan <- msg
+				}
+			}
+		}()
+	}
+
+	return nil
+}
+
+func (k *KafkaInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error) {
+
+	msg := <-k.messageChan
+
+	// Commit only once the message has been replicated. Committing on receipt
+	// leaves nothing to redeliver when the sink write fails.
+	onSuccessFn = func() {
+		if err := k.progress.setOffset(msg.Partition, msg.Offset); err != nil {
+			glog.Warningf("set kafka offset: %v", err)
+		}
+	}
+	onFailureFn = func() {
+		k.progress.markFailed(msg.Partition, msg.Offset)
+	}
+
+	key = string(msg.Key)
+	message = &filer_pb.EventNotification{}
+	err = proto.Unmarshal(msg.Value, message)
+
+	return
+}
+
+type KafkaProgress struct {
+	Topic                     string          `json:"topic"`
+	PartitionOffsets          map[int32]int64 `json:"partitionOffsets"`
+	offsetFile                string
+	lastSaveTime              time.Time
+	offsetSaveIntervalSeconds int
+	// failedOffsets is the oldest offset that failed to replicate in each
+	// partition. Nothing at or past it is ever committed, so a restart
+	// redelivers from the failure instead of resuming after it.
+	failedOffsets map[int32]int64
+	sync.Mutex
+}
+
+func loadProgress(offsetFile string) *KafkaProgress {
+	progress := &KafkaProgress{}
+	data, err := os.ReadFile(offsetFile)
+	if err != nil {
+		glog.Warningf("failed to read kafka progress file: %s", offsetFile)
+		return nil
+	}
+	err = json.Unmarshal(data, progress)
+	if err != nil {
+		glog.Warningf("failed to read kafka progress message: %s", string(data))
+		return nil
+	}
+	return progress
+}
+
+func (progress *KafkaProgress) saveProgress() error {
+	data, err := json.Marshal(progress)
+	if err != nil {
+		return fmt.Errorf("failed to marshal progress: %w", err)
+	}
+	err = util.WriteFile(progress.offsetFile, data, 0640)
+	if err != nil {
+		return fmt.Errorf("failed to save progress to %s: %v", progress.offsetFile, err)
+	}
+
+	progress.lastSaveTime = time.Now()
+	return nil
+}
+
+func (progress *KafkaProgress) setOffset(partition int32, offset int64) error {
+	progress.Lock()
+	defer progress.Unlock()
+
+	if failedOffset, found := progress.failedOffsets[partition]; found && offset >= failedOffset {
+		return nil
+	}
+
+	progress.PartitionOffsets[partition] = offset
+	if int(time.Now().Sub(progress.lastSaveTime).Seconds()) > progress.offsetSaveIntervalSeconds {
+		return progress.saveProgress()
+	}
+	return nil
+}
+
+// markFailed records an offset that could not be replicated, holding the
+// committed offset for that partition behind it.
+func (progress *KafkaProgress) markFailed(partition int32, offset int64) {
+	progress.Lock()
+	defer progress.Unlock()
+
+	if failedOffset, found := progress.failedOffsets[partition]; !found || offset < failedOffset {
+		progress.failedOffsets[partition] = offset
+		glog.Errorf("replicate kafka %s partition %d offset %d failed; holding the committed offset before it", progress.Topic, partition, offset)
+	}
+}

weed/replication/sub/notification_kafka_test.go

@@ -0,0 +1,69 @@
+package sub
+
+import (
+	"testing"
+	"time"
+)
+
+func newTestProgress() *KafkaProgress {
+	return &KafkaProgress{
+		Topic:            "test",
+		PartitionOffsets: make(map[int32]int64),
+		failedOffsets:    make(map[int32]int64),
+		lastSaveTime:     time.Now(),
+		// large enough that no test call reaches saveProgress and touches disk
+		offsetSaveIntervalSeconds: 3600,
+	}
+}
+
+// TestKafkaProgressHoldsOffsetAtFailure verifies that a message that failed to
+// replicate keeps the committed offset behind it, so a restart redelivers it
+// rather than resuming after it.
+func TestKafkaProgressHoldsOffsetAtFailure(t *testing.T) {
+	progress := newTestProgress()
+
+	if err := progress.setOffset(0, 10); err != nil {
+		t.Fatalf("setOffset: %v", err)
+	}
+	if got := progress.PartitionOffsets[0]; got != 10 {
+		t.Fatalf("offset = %d after a replicated message, want 10", got)
+	}
+
+	progress.markFailed(0, 11)
+
+	if err := progress.setOffset(0, 12); err != nil {
+		t.Fatalf("setOffset: %v", err)
+	}
+	if got := progress.PartitionOffsets[0]; got != 10 {
+		t.Fatalf("offset = %d after a later success, want it held at 10", got)
+	}
+
+	// a failure in one partition must not stall the others
+	if err := progress.setOffset(1, 5); err != nil {
+		t.Fatalf("setOffset: %v", err)
+	}
+	if got := progress.PartitionOffsets[1]; got != 5 {
+		t.Fatalf("offset = %d in an unaffected partition, want 5", got)
+	}
+}
+
+// TestKafkaProgressKeepsOldestFailure verifies that the hold point is the
+// oldest failed offset, not the most recent one.
+func TestKafkaProgressKeepsOldestFailure(t *testing.T) {
+	progress := newTestProgress()
+
+	progress.markFailed(0, 20)
+	progress.markFailed(0, 11)
+	progress.markFailed(0, 30)
+
+	if got := progress.failedOffsets[0]; got != 11 {
+		t.Fatalf("held at offset %d, want the oldest failure 11", got)
+	}
+
+	if err := progress.setOffset(0, 10); err != nil {
+		t.Fatalf("setOffset: %v", err)
+	}
+	if got := progress.PartitionOffsets[0]; got != 10 {
+		t.Fatalf("offset = %d, want an offset before the failure to still commit", got)
+	}
+}

weed/replication/sub/notifications.go

@@ -0,0 +1,18 @@
+package sub
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type NotificationInput interface {
+	// GetName gets the name to locate the configuration in sync.toml file
+	GetName() string
+	// Initialize initializes the file store
+	Initialize(configuration util.Configuration, prefix string) error
+	ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error)
+}
+
+var (
+	NotificationInputs []NotificationInput
+)