snapshot of https://github.com/ThreeDotsLabs/watermill.git (master, 2026-09-15; blobs > 131072B omitted)
Some checks failed
master / ci (push) Failing after 0s
Some checks failed
master / ci (push) Failing after 0s
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
validation_cmd: "docker compose up"
|
||||
teardown_cmd: "docker compose down"
|
||||
timeout: 180
|
||||
expected_output: "Starting handler"
|
||||
@@ -0,0 +1,44 @@
|
||||
# Exactly-once delivery counter
|
||||
|
||||
Is exactly-once delivery impossible? Well, it depends a lot on the definition of exactly-once delivery.
|
||||
When we assume we want to avoid the situation when a message is delivered more than once when our broker or worker died -- it's possible.
|
||||
I'll say more, it's even possible with Watermill!
|
||||
|
||||

|
||||
|
||||
*At-least once delivery - this is not what we want!*
|
||||
|
||||
There are just two constraints:
|
||||
1. you need to use a Pub/Sub implementation that does support exactly-once delivery (only [MySQL/PostgreSQL](https://github.com/ThreeDotsLabs/watermill-sql) for now),
|
||||
2. writes need to go to the same DB.
|
||||
|
||||
In practice, our model is pretty similar to how does it work with Kafka exactly-once delivery. If you want to know more details, you can check [their article](https://www.confluent.io/blog/exactly-once-semantics-are-possible-heres-how-apache-kafka-does-it/).
|
||||
|
||||
In our example, we use a MySQL database to implement a **simple counter**. It can be triggered by calling the `http://localhost:8080/count/{counterUUID}` endpoint.
|
||||
Calling this endpoint will publish a message to MySQL via our [Pub/Sub implementation](https://github.com/ThreeDotsLabs/watermill-sql).
|
||||
The endpoint is provided by [server/main.go](server/main.go).
|
||||
|
||||
Later, the message is consumed by [worker/main.go](worker/main.go). The only responsibility of the worker is to update the counter in the MySQL database.
|
||||
**Counter update is done in the same transaction as message consumption.**
|
||||
|
||||
Normally, we would need to de-duplicate messages.
|
||||
But thanks to that fact and [A.C.I.D](https://en.wikipedia.org/wiki/ACID) even if server, worker or network failure happens during processing our data will stay consistent.
|
||||
|
||||

|
||||
|
||||
*Watermill's exactly-once delivery*
|
||||
|
||||
To check if the created code works, I've created a small `run.go` program, that sends 10k requests to the server and verifies if the count at the end is equal to 10k.
|
||||
But to not make it too easy, I'm restarting the worker and MySQL a couple of times. I also forgot about graceful shutdown in my worker. ;-)
|
||||
|
||||
The biggest downside of this approach is performance. Due to [our benchmark](https://github.com/ThreeDotsLabs/watermill-benchmark#sql-mysql), MySQL subscriber can consume up to 154 messages per second.
|
||||
Fortunately, it's still 13,305,600 messages per day. It's more than enough for a lot of systems.
|
||||
|
||||
## Running
|
||||
|
||||
docker-compose up
|
||||
|
||||
go run run.go
|
||||
|
||||
*Please note that `run.go` needs to be executed by a user having privileges to manage Docker.
|
||||
It's due to the fact that `run.go` is restarting containers.*
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 90 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 64 KiB |
@@ -0,0 +1,31 @@
|
||||
services:
|
||||
server:
|
||||
image: golang:1.25
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- 8080:8080
|
||||
volumes:
|
||||
- ./server:/app
|
||||
- $GOPATH/pkg/mod:/go/pkg/mod
|
||||
working_dir: /app
|
||||
command: 'go run .'
|
||||
|
||||
worker:
|
||||
image: golang:1.25
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ./worker:/app
|
||||
- $GOPATH/pkg/mod:/go/pkg/mod
|
||||
working_dir: /app
|
||||
command: 'go run .'
|
||||
|
||||
mysql:
|
||||
image: mysql:8.0
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- 3306:3306
|
||||
environment:
|
||||
MYSQL_DATABASE: example
|
||||
MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
|
||||
volumes:
|
||||
- ./schema.sql:/docker-entrypoint-initdb.d/schema.sql
|
||||
@@ -0,0 +1,173 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
stdSQL "database/sql"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/cheggaaa/pb/v3"
|
||||
"github.com/go-sql-driver/mysql"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const messagesCount = 5000
|
||||
|
||||
// at these messages we will restart MySQL
|
||||
var restartMySQLAt = map[int]struct{}{
|
||||
50: {},
|
||||
1000: {},
|
||||
1500: {},
|
||||
3000: {},
|
||||
}
|
||||
|
||||
// at these messages we will restart counter worker
|
||||
var restartWorkerAt = map[int]struct{}{
|
||||
100: {},
|
||||
1500: {},
|
||||
1600: {},
|
||||
3000: {},
|
||||
}
|
||||
|
||||
const senderGoroutines = 5
|
||||
|
||||
func main() {
|
||||
db := createDB()
|
||||
counterUUID := uuid.New().String()
|
||||
|
||||
wg := &sync.WaitGroup{}
|
||||
wg.Add(messagesCount)
|
||||
|
||||
bar := pb.StartNew(messagesCount)
|
||||
|
||||
// sending value to sendCounter counter HTTP call
|
||||
sendCounter := make(chan struct{}, 0)
|
||||
go func() {
|
||||
for i := 0; i < messagesCount; i++ {
|
||||
sendCounter <- struct{}{}
|
||||
|
||||
// let's challenge exactly-once delivery a bit
|
||||
// normally it should trigger re-delivery of the message
|
||||
if _, ok := restartMySQLAt[i]; ok {
|
||||
restartMySQL()
|
||||
}
|
||||
if _, ok := restartWorkerAt[i]; ok {
|
||||
restartWorker()
|
||||
}
|
||||
}
|
||||
close(sendCounter)
|
||||
}()
|
||||
|
||||
for i := 0; i < senderGoroutines; i++ {
|
||||
go func() {
|
||||
for range sendCounter {
|
||||
sendCountRequest(counterUUID)
|
||||
wg.Done()
|
||||
bar.Increment()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
bar.Finish()
|
||||
|
||||
timeout := time.Now().Add(time.Second * 30)
|
||||
|
||||
fmt.Println("checking counter with DB, expected count:", messagesCount)
|
||||
|
||||
matchedOnce := true
|
||||
|
||||
for {
|
||||
if time.Now().After(timeout) {
|
||||
fmt.Println("timeout")
|
||||
break
|
||||
}
|
||||
|
||||
dbCounterValue, err := getDbCounterValue(db, counterUUID)
|
||||
if err != nil {
|
||||
fmt.Println("err:", err)
|
||||
continue
|
||||
}
|
||||
|
||||
fmt.Println("db counter value", dbCounterValue)
|
||||
if dbCounterValue == messagesCount {
|
||||
if !matchedOnce {
|
||||
// let's ensure that nothing new will arrive
|
||||
matchedOnce = true
|
||||
time.Sleep(time.Second * 2)
|
||||
continue
|
||||
} else {
|
||||
fmt.Println("expected counter value is matching DB value")
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func getDbCounterValue(db *stdSQL.DB, counterUUID string) (int, error) {
|
||||
var dbCounterValue int
|
||||
row := db.QueryRow("SELECT value from counter WHERE id = ?", counterUUID)
|
||||
|
||||
if err := row.Scan(&dbCounterValue); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return dbCounterValue, nil
|
||||
}
|
||||
|
||||
func restartWorker() {
|
||||
fmt.Println("restarting worker")
|
||||
err := exec.Command("docker-compose", "restart", "worker").Run()
|
||||
if err != nil {
|
||||
fmt.Println("restarting worker failed", err)
|
||||
}
|
||||
}
|
||||
|
||||
func restartMySQL() {
|
||||
fmt.Println("restarting mysql")
|
||||
err := exec.Command("docker-compose", "restart", "mysql").Run()
|
||||
if err != nil {
|
||||
fmt.Println("restarting mysql failed", err)
|
||||
}
|
||||
}
|
||||
|
||||
func sendCountRequest(counterUUID string) {
|
||||
for {
|
||||
resp, err := http.Post("http://localhost:8080/count/"+counterUUID, "", nil)
|
||||
if err != nil {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == http.StatusNoContent {
|
||||
break
|
||||
}
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func createDB() *stdSQL.DB {
|
||||
conf := mysql.NewConfig()
|
||||
conf.Net = "tcp"
|
||||
conf.User = "root"
|
||||
conf.Addr = "localhost"
|
||||
conf.DBName = "example"
|
||||
|
||||
db, err := stdSQL.Open("mysql", conf.FormatDSN())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = db.Ping()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return db
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
CREATE TABLE counter (
|
||||
id VARCHAR(36) NOT NULL UNIQUE,
|
||||
value int NOT NULL
|
||||
);
|
||||
@@ -0,0 +1,25 @@
|
||||
module exactly-once-delivery
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1
|
||||
github.com/ThreeDotsLabs/watermill-sql/v4 v4.1.3
|
||||
github.com/go-chi/chi/v5 v5.2.5
|
||||
github.com/go-sql-driver/mysql v1.10.0
|
||||
)
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/pgx/v5 v5.9.2 // indirect
|
||||
github.com/lib/pq v1.12.3 // indirect
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
|
||||
github.com/oklog/ulid v1.3.1 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/sony/gobreaker v1.0.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,51 @@
|
||||
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1 h1:t5xMivyf9tpmU3iozPqyrCZXHvoV1XQDfihas4sV0fY=
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1/go.mod h1:Uop10dA3VeJWsSvis9qO3vbVY892LARrKAdki6WtXS4=
|
||||
github.com/ThreeDotsLabs/watermill-sql/v4 v4.1.3 h1:d9niNUM3G9nFH2YRdJ7F+qz3IF0bS2IHq6UB2TKWHGE=
|
||||
github.com/ThreeDotsLabs/watermill-sql/v4 v4.1.3/go.mod h1:Ce2GVZVnyajAh0AkwxSJXwx8ajBBveu1DI/yatan5jc=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
|
||||
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
|
||||
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
|
||||
github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
|
||||
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
|
||||
github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8=
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts=
|
||||
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/sony/gobreaker v1.0.0 h1:feX5fGGXSl3dYd4aHZItw+FpHLvvoaqkawKjVNiFMNQ=
|
||||
github.com/sony/gobreaker v1.0.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,86 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
stdSQL "database/sql"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
"github.com/ThreeDotsLabs/watermill"
|
||||
"github.com/ThreeDotsLabs/watermill-sql/v4/pkg/sql"
|
||||
"github.com/ThreeDotsLabs/watermill/message"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
driver "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
const topic = "counter"
|
||||
|
||||
func main() {
|
||||
db := createDB()
|
||||
logger := watermill.NewStdLogger(false, false)
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(middleware.Recoverer)
|
||||
r.Use(middleware.Logger)
|
||||
|
||||
publisher, err := sql.NewPublisher(
|
||||
sql.BeginnerFromStdSQL(db),
|
||||
sql.PublisherConfig{
|
||||
SchemaAdapter: sql.DefaultMySQLSchema{},
|
||||
},
|
||||
logger,
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
r.Post("/count/{counterUUID}", func(w http.ResponseWriter, r *http.Request) {
|
||||
payload, err := json.Marshal(messagePayload{
|
||||
CounterUUID: chi.URLParam(r, "counterUUID"),
|
||||
})
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
msg := message.NewMessage(watermill.NewUUID(), payload)
|
||||
|
||||
if err := publisher.Publish(topic, msg); err != nil {
|
||||
log.Print(err)
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
})
|
||||
|
||||
err = http.ListenAndServe(":8080", r)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
type messagePayload struct {
|
||||
CounterUUID string `json:"counter_uuid"`
|
||||
}
|
||||
|
||||
func createDB() *stdSQL.DB {
|
||||
conf := driver.NewConfig()
|
||||
conf.Net = "tcp"
|
||||
conf.User = "root"
|
||||
conf.Addr = "mysql"
|
||||
conf.DBName = "example"
|
||||
|
||||
db, err := stdSQL.Open("mysql", conf.FormatDSN())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = db.Ping()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return db
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
module exactly-once-delivery
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1
|
||||
github.com/ThreeDotsLabs/watermill-sql/v4 v4.1.3
|
||||
github.com/go-sql-driver/mysql v1.10.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
)
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/pgx/v5 v5.9.2 // indirect
|
||||
github.com/lib/pq v1.12.3 // indirect
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
|
||||
github.com/oklog/ulid v1.3.1 // indirect
|
||||
github.com/sony/gobreaker v1.0.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,49 @@
|
||||
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1 h1:t5xMivyf9tpmU3iozPqyrCZXHvoV1XQDfihas4sV0fY=
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1/go.mod h1:Uop10dA3VeJWsSvis9qO3vbVY892LARrKAdki6WtXS4=
|
||||
github.com/ThreeDotsLabs/watermill-sql/v4 v4.1.3 h1:d9niNUM3G9nFH2YRdJ7F+qz3IF0bS2IHq6UB2TKWHGE=
|
||||
github.com/ThreeDotsLabs/watermill-sql/v4 v4.1.3/go.mod h1:Ce2GVZVnyajAh0AkwxSJXwx8ajBBveu1DI/yatan5jc=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
|
||||
github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
|
||||
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
|
||||
github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8=
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts=
|
||||
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/sony/gobreaker v1.0.0 h1:feX5fGGXSl3dYd4aHZItw+FpHLvvoaqkawKjVNiFMNQ=
|
||||
github.com/sony/gobreaker v1.0.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,148 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
stdSQL "database/sql"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/ThreeDotsLabs/watermill/message"
|
||||
driver "github.com/go-sql-driver/mysql"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/ThreeDotsLabs/watermill"
|
||||
"github.com/ThreeDotsLabs/watermill-sql/v4/pkg/sql"
|
||||
)
|
||||
|
||||
const topic = "counter"
|
||||
|
||||
func main() {
|
||||
db := createDB()
|
||||
logger := watermill.NewStdLogger(false, false)
|
||||
|
||||
go runWatermillRouter(db, logger)
|
||||
|
||||
sigs := make(chan os.Signal, 1)
|
||||
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
// no graceful shutdown, to increase chance of problems :-)
|
||||
<-sigs
|
||||
}
|
||||
|
||||
type messagePayload struct {
|
||||
CounterUUID string `json:"counter_uuid"`
|
||||
}
|
||||
|
||||
func runWatermillRouter(db *stdSQL.DB, logger watermill.LoggerAdapter) {
|
||||
subscriber, err := sql.NewSubscriber(
|
||||
sql.BeginnerFromStdSQL(db),
|
||||
sql.SubscriberConfig{
|
||||
SchemaAdapter: sql.DefaultMySQLSchema{},
|
||||
OffsetsAdapter: sql.DefaultMySQLOffsetsAdapter{},
|
||||
InitializeSchema: true,
|
||||
},
|
||||
logger,
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
router, err := message.NewRouter(message.RouterConfig{}, logger)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
router.AddConsumerHandler(
|
||||
"counter",
|
||||
topic,
|
||||
subscriber,
|
||||
processMessage,
|
||||
)
|
||||
|
||||
if err := router.Run(context.Background()); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func processMessage(msg *message.Message) error {
|
||||
tx, ok := sql.TxFromContext(msg.Context())
|
||||
if !ok {
|
||||
return errors.New("tx not found in message context")
|
||||
}
|
||||
|
||||
payload := messagePayload{}
|
||||
err := json.Unmarshal(msg.Payload, &payload)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "unable to unmarshal payload")
|
||||
}
|
||||
|
||||
// let's do it more fragile, let's get the value from DB instead of simple increment
|
||||
counterValue, err := dbCounterValue(msg.Context(), tx, payload.CounterUUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
counterValue += 1
|
||||
|
||||
if err := updateDbCounter(msg.Context(), tx, payload.CounterUUID, counterValue); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func updateDbCounter(ctx context.Context, tx sql.Tx, counterUUD string, counterValue int) error {
|
||||
_, err := tx.ExecContext(
|
||||
ctx,
|
||||
"INSERT INTO counter (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value = ?",
|
||||
counterUUD,
|
||||
counterValue,
|
||||
counterValue,
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "can't update counter value")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func dbCounterValue(ctx context.Context, tx sql.Tx, counterUUID string) (int, error) {
|
||||
var counterValue int
|
||||
rows, err := tx.QueryContext(ctx, "SELECT value from counter WHERE id = ?", counterUUID)
|
||||
if err != nil {
|
||||
return 0, errors.Wrap(err, "can't get counter value")
|
||||
}
|
||||
|
||||
if !rows.Next() {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
err = rows.Scan(&counterValue)
|
||||
if err != nil {
|
||||
return 0, errors.Wrap(err, "can't get counter value")
|
||||
}
|
||||
|
||||
return counterValue, nil
|
||||
}
|
||||
|
||||
func createDB() *stdSQL.DB {
|
||||
conf := driver.NewConfig()
|
||||
conf.Net = "tcp"
|
||||
conf.User = "root"
|
||||
conf.Addr = "mysql"
|
||||
conf.DBName = "example"
|
||||
|
||||
db, err := stdSQL.Open("mysql", conf.FormatDSN())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = db.Ping()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return db
|
||||
}
|
||||
Reference in New Issue
Block a user