snapshot of https://github.com/ThreeDotsLabs/watermill.git (master, 2026-09-15; blobs > 131072B omitted)
Some checks failed
master / ci (push) Failing after 0s

This commit is contained in:
upstream-mirror
2026-09-15 21:27:46 +05:00
commit ce933d1277
538 changed files with 46504 additions and 0 deletions

View File

@@ -0,0 +1,4 @@
validation_cmd: "docker compose up"
teardown_cmd: "docker compose down"
timeout: 180
expected_output: "Starting handler"

View File

@@ -0,0 +1,44 @@
# Exactly-once delivery counter
Is exactly-once delivery impossible? Well, it depends a lot on the definition of exactly-once delivery.
When we assume we want to avoid the situation when a message is delivered more than once when our broker or worker died -- it's possible.
I'll say more, it's even possible with Watermill!
![](./at-least-once-delivery.jpg)
*At-least once delivery - this is not what we want!*
There are just two constraints:
1. you need to use a Pub/Sub implementation that does support exactly-once delivery (only [MySQL/PostgreSQL](https://github.com/ThreeDotsLabs/watermill-sql) for now),
2. writes need to go to the same DB.
In practice, our model is pretty similar to how does it work with Kafka exactly-once delivery. If you want to know more details, you can check [their article](https://www.confluent.io/blog/exactly-once-semantics-are-possible-heres-how-apache-kafka-does-it/).
In our example, we use a MySQL database to implement a **simple counter**. It can be triggered by calling the `http://localhost:8080/count/{counterUUID}` endpoint.
Calling this endpoint will publish a message to MySQL via our [Pub/Sub implementation](https://github.com/ThreeDotsLabs/watermill-sql).
The endpoint is provided by [server/main.go](server/main.go).
Later, the message is consumed by [worker/main.go](worker/main.go). The only responsibility of the worker is to update the counter in the MySQL database.
**Counter update is done in the same transaction as message consumption.**
Normally, we would need to de-duplicate messages.
But thanks to that fact and [A.C.I.D](https://en.wikipedia.org/wiki/ACID) even if server, worker or network failure happens during processing our data will stay consistent.
![](./architecture.jpg)
*Watermill's exactly-once delivery*
To check if the created code works, I've created a small `run.go` program, that sends 10k requests to the server and verifies if the count at the end is equal to 10k.
But to not make it too easy, I'm restarting the worker and MySQL a couple of times. I also forgot about graceful shutdown in my worker. ;-)
The biggest downside of this approach is performance. Due to [our benchmark](https://github.com/ThreeDotsLabs/watermill-benchmark#sql-mysql), MySQL subscriber can consume up to 154 messages per second.
Fortunately, it's still 13,305,600 messages per day. It's more than enough for a lot of systems.
## Running
docker-compose up
go run run.go
*Please note that `run.go` needs to be executed by a user having privileges to manage Docker.
It's due to the fact that `run.go` is restarting containers.*

Binary file not shown.

After

Width:  |  Height:  |  Size: 90 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 64 KiB

View File

@@ -0,0 +1,31 @@
services:
server:
image: golang:1.25
restart: unless-stopped
ports:
- 8080:8080
volumes:
- ./server:/app
- $GOPATH/pkg/mod:/go/pkg/mod
working_dir: /app
command: 'go run .'
worker:
image: golang:1.25
restart: unless-stopped
volumes:
- ./worker:/app
- $GOPATH/pkg/mod:/go/pkg/mod
working_dir: /app
command: 'go run .'
mysql:
image: mysql:8.0
restart: unless-stopped
ports:
- 3306:3306
environment:
MYSQL_DATABASE: example
MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
volumes:
- ./schema.sql:/docker-entrypoint-initdb.d/schema.sql

View File

@@ -0,0 +1,173 @@
package main
import (
stdSQL "database/sql"
"fmt"
"net/http"
"os/exec"
"sync"
"time"
"github.com/cheggaaa/pb/v3"
"github.com/go-sql-driver/mysql"
"github.com/google/uuid"
)
const messagesCount = 5000
// at these messages we will restart MySQL
var restartMySQLAt = map[int]struct{}{
50: {},
1000: {},
1500: {},
3000: {},
}
// at these messages we will restart counter worker
var restartWorkerAt = map[int]struct{}{
100: {},
1500: {},
1600: {},
3000: {},
}
const senderGoroutines = 5
func main() {
db := createDB()
counterUUID := uuid.New().String()
wg := &sync.WaitGroup{}
wg.Add(messagesCount)
bar := pb.StartNew(messagesCount)
// sending value to sendCounter counter HTTP call
sendCounter := make(chan struct{}, 0)
go func() {
for i := 0; i < messagesCount; i++ {
sendCounter <- struct{}{}
// let's challenge exactly-once delivery a bit
// normally it should trigger re-delivery of the message
if _, ok := restartMySQLAt[i]; ok {
restartMySQL()
}
if _, ok := restartWorkerAt[i]; ok {
restartWorker()
}
}
close(sendCounter)
}()
for i := 0; i < senderGoroutines; i++ {
go func() {
for range sendCounter {
sendCountRequest(counterUUID)
wg.Done()
bar.Increment()
}
}()
}
wg.Wait()
bar.Finish()
timeout := time.Now().Add(time.Second * 30)
fmt.Println("checking counter with DB, expected count:", messagesCount)
matchedOnce := true
for {
if time.Now().After(timeout) {
fmt.Println("timeout")
break
}
dbCounterValue, err := getDbCounterValue(db, counterUUID)
if err != nil {
fmt.Println("err:", err)
continue
}
fmt.Println("db counter value", dbCounterValue)
if dbCounterValue == messagesCount {
if !matchedOnce {
// let's ensure that nothing new will arrive
matchedOnce = true
time.Sleep(time.Second * 2)
continue
} else {
fmt.Println("expected counter value is matching DB value")
break
}
}
time.Sleep(time.Second)
}
}
func getDbCounterValue(db *stdSQL.DB, counterUUID string) (int, error) {
var dbCounterValue int
row := db.QueryRow("SELECT value from counter WHERE id = ?", counterUUID)
if err := row.Scan(&dbCounterValue); err != nil {
return 0, err
}
return dbCounterValue, nil
}
func restartWorker() {
fmt.Println("restarting worker")
err := exec.Command("docker-compose", "restart", "worker").Run()
if err != nil {
fmt.Println("restarting worker failed", err)
}
}
func restartMySQL() {
fmt.Println("restarting mysql")
err := exec.Command("docker-compose", "restart", "mysql").Run()
if err != nil {
fmt.Println("restarting mysql failed", err)
}
}
func sendCountRequest(counterUUID string) {
for {
resp, err := http.Post("http://localhost:8080/count/"+counterUUID, "", nil)
if err != nil {
time.Sleep(100 * time.Millisecond)
continue
}
resp.Body.Close()
if resp.StatusCode == http.StatusNoContent {
break
}
time.Sleep(100 * time.Millisecond)
}
}
func createDB() *stdSQL.DB {
conf := mysql.NewConfig()
conf.Net = "tcp"
conf.User = "root"
conf.Addr = "localhost"
conf.DBName = "example"
db, err := stdSQL.Open("mysql", conf.FormatDSN())
if err != nil {
panic(err)
}
err = db.Ping()
if err != nil {
panic(err)
}
return db
}

View File

@@ -0,0 +1,4 @@
CREATE TABLE counter (
id VARCHAR(36) NOT NULL UNIQUE,
value int NOT NULL
);

View File

@@ -0,0 +1,25 @@
module exactly-once-delivery
go 1.25.0
require (
github.com/ThreeDotsLabs/watermill v1.5.1
github.com/ThreeDotsLabs/watermill-sql/v4 v4.1.3
github.com/go-chi/chi/v5 v5.2.5
github.com/go-sql-driver/mysql v1.10.0
)
require (
filippo.io/edwards25519 v1.2.0 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.9.2 // indirect
github.com/lib/pq v1.12.3 // indirect
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/sony/gobreaker v1.0.0 // indirect
golang.org/x/text v0.37.0 // indirect
)

View File

@@ -0,0 +1,51 @@
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
github.com/ThreeDotsLabs/watermill v1.5.1 h1:t5xMivyf9tpmU3iozPqyrCZXHvoV1XQDfihas4sV0fY=
github.com/ThreeDotsLabs/watermill v1.5.1/go.mod h1:Uop10dA3VeJWsSvis9qO3vbVY892LARrKAdki6WtXS4=
github.com/ThreeDotsLabs/watermill-sql/v4 v4.1.3 h1:d9niNUM3G9nFH2YRdJ7F+qz3IF0bS2IHq6UB2TKWHGE=
github.com/ThreeDotsLabs/watermill-sql/v4 v4.1.3/go.mod h1:Ce2GVZVnyajAh0AkwxSJXwx8ajBBveu1DI/yatan5jc=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8=
github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts=
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sony/gobreaker v1.0.0 h1:feX5fGGXSl3dYd4aHZItw+FpHLvvoaqkawKjVNiFMNQ=
github.com/sony/gobreaker v1.0.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -0,0 +1,86 @@
package main
import (
stdSQL "database/sql"
"encoding/json"
"log"
"net/http"
"github.com/ThreeDotsLabs/watermill"
"github.com/ThreeDotsLabs/watermill-sql/v4/pkg/sql"
"github.com/ThreeDotsLabs/watermill/message"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
driver "github.com/go-sql-driver/mysql"
)
const topic = "counter"
func main() {
db := createDB()
logger := watermill.NewStdLogger(false, false)
r := chi.NewRouter()
r.Use(middleware.Recoverer)
r.Use(middleware.Logger)
publisher, err := sql.NewPublisher(
sql.BeginnerFromStdSQL(db),
sql.PublisherConfig{
SchemaAdapter: sql.DefaultMySQLSchema{},
},
logger,
)
if err != nil {
panic(err)
}
r.Post("/count/{counterUUID}", func(w http.ResponseWriter, r *http.Request) {
payload, err := json.Marshal(messagePayload{
CounterUUID: chi.URLParam(r, "counterUUID"),
})
if err != nil {
log.Print(err)
w.WriteHeader(http.StatusInternalServerError)
return
}
msg := message.NewMessage(watermill.NewUUID(), payload)
if err := publisher.Publish(topic, msg); err != nil {
log.Print(err)
w.WriteHeader(http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusNoContent)
})
err = http.ListenAndServe(":8080", r)
if err != nil {
panic(err)
}
}
type messagePayload struct {
CounterUUID string `json:"counter_uuid"`
}
func createDB() *stdSQL.DB {
conf := driver.NewConfig()
conf.Net = "tcp"
conf.User = "root"
conf.Addr = "mysql"
conf.DBName = "example"
db, err := stdSQL.Open("mysql", conf.FormatDSN())
if err != nil {
panic(err)
}
err = db.Ping()
if err != nil {
panic(err)
}
return db
}

View File

@@ -0,0 +1,24 @@
module exactly-once-delivery
go 1.25.0
require (
github.com/ThreeDotsLabs/watermill v1.5.1
github.com/ThreeDotsLabs/watermill-sql/v4 v4.1.3
github.com/go-sql-driver/mysql v1.10.0
github.com/pkg/errors v0.9.1
)
require (
filippo.io/edwards25519 v1.2.0 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.9.2 // indirect
github.com/lib/pq v1.12.3 // indirect
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/sony/gobreaker v1.0.0 // indirect
golang.org/x/text v0.37.0 // indirect
)

View File

@@ -0,0 +1,49 @@
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
github.com/ThreeDotsLabs/watermill v1.5.1 h1:t5xMivyf9tpmU3iozPqyrCZXHvoV1XQDfihas4sV0fY=
github.com/ThreeDotsLabs/watermill v1.5.1/go.mod h1:Uop10dA3VeJWsSvis9qO3vbVY892LARrKAdki6WtXS4=
github.com/ThreeDotsLabs/watermill-sql/v4 v4.1.3 h1:d9niNUM3G9nFH2YRdJ7F+qz3IF0bS2IHq6UB2TKWHGE=
github.com/ThreeDotsLabs/watermill-sql/v4 v4.1.3/go.mod h1:Ce2GVZVnyajAh0AkwxSJXwx8ajBBveu1DI/yatan5jc=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8=
github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts=
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sony/gobreaker v1.0.0 h1:feX5fGGXSl3dYd4aHZItw+FpHLvvoaqkawKjVNiFMNQ=
github.com/sony/gobreaker v1.0.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -0,0 +1,148 @@
package main
import (
"context"
stdSQL "database/sql"
"encoding/json"
"os"
"os/signal"
"syscall"
"github.com/ThreeDotsLabs/watermill/message"
driver "github.com/go-sql-driver/mysql"
"github.com/pkg/errors"
"github.com/ThreeDotsLabs/watermill"
"github.com/ThreeDotsLabs/watermill-sql/v4/pkg/sql"
)
const topic = "counter"
func main() {
db := createDB()
logger := watermill.NewStdLogger(false, false)
go runWatermillRouter(db, logger)
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
// no graceful shutdown, to increase chance of problems :-)
<-sigs
}
type messagePayload struct {
CounterUUID string `json:"counter_uuid"`
}
func runWatermillRouter(db *stdSQL.DB, logger watermill.LoggerAdapter) {
subscriber, err := sql.NewSubscriber(
sql.BeginnerFromStdSQL(db),
sql.SubscriberConfig{
SchemaAdapter: sql.DefaultMySQLSchema{},
OffsetsAdapter: sql.DefaultMySQLOffsetsAdapter{},
InitializeSchema: true,
},
logger,
)
if err != nil {
panic(err)
}
router, err := message.NewRouter(message.RouterConfig{}, logger)
if err != nil {
panic(err)
}
router.AddConsumerHandler(
"counter",
topic,
subscriber,
processMessage,
)
if err := router.Run(context.Background()); err != nil {
panic(err)
}
}
func processMessage(msg *message.Message) error {
tx, ok := sql.TxFromContext(msg.Context())
if !ok {
return errors.New("tx not found in message context")
}
payload := messagePayload{}
err := json.Unmarshal(msg.Payload, &payload)
if err != nil {
return errors.Wrap(err, "unable to unmarshal payload")
}
// let's do it more fragile, let's get the value from DB instead of simple increment
counterValue, err := dbCounterValue(msg.Context(), tx, payload.CounterUUID)
if err != nil {
return err
}
counterValue += 1
if err := updateDbCounter(msg.Context(), tx, payload.CounterUUID, counterValue); err != nil {
return err
}
return nil
}
func updateDbCounter(ctx context.Context, tx sql.Tx, counterUUD string, counterValue int) error {
_, err := tx.ExecContext(
ctx,
"INSERT INTO counter (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value = ?",
counterUUD,
counterValue,
counterValue,
)
if err != nil {
return errors.Wrap(err, "can't update counter value")
}
return nil
}
func dbCounterValue(ctx context.Context, tx sql.Tx, counterUUID string) (int, error) {
var counterValue int
rows, err := tx.QueryContext(ctx, "SELECT value from counter WHERE id = ?", counterUUID)
if err != nil {
return 0, errors.Wrap(err, "can't get counter value")
}
if !rows.Next() {
return 0, nil
}
err = rows.Scan(&counterValue)
if err != nil {
return 0, errors.Wrap(err, "can't get counter value")
}
return counterValue, nil
}
func createDB() *stdSQL.DB {
conf := driver.NewConfig()
conf.Net = "tcp"
conf.User = "root"
conf.Addr = "mysql"
conf.DBName = "example"
db, err := stdSQL.Open("mysql", conf.FormatDSN())
if err != nil {
panic(err)
}
err = db.Ping()
if err != nil {
panic(err)
}
return db
}