snapshot of https://github.com/ThreeDotsLabs/watermill.git (master, 2026-09-15; blobs > 131072B omitted)
Some checks failed
master / ci (push) Failing after 0s

This commit is contained in:
upstream-mirror
2026-09-15 21:27:46 +05:00
commit ce933d1277
538 changed files with 46504 additions and 0 deletions

View File

@@ -0,0 +1,4 @@
validation_cmd: "docker compose up"
teardown_cmd: "docker compose down"
timeout: 180
expected_output: "received user:"

View File

@@ -0,0 +1,66 @@
# Synchronizing Databases (MySQL to PostgreSQL)
This example shows how to use [SQL Pub/Sub](https://github.com/ThreeDotsLabs/watermill-sql) across two different databases.
See also [SQL Pub/Sub documentation](https://watermill.io/pubsubs/sql).
## Background
Synchronizing two databases can be a tough task, especially with different data formats.
This example shows how to migrate a MySQL table to PostgreSQL table using watermill.
The application will first transfer all existing rows and then keep listening for any new inserts,
copying them to the new table as soon, as they appear. Only new rows will be detected, there's no
support for updates or deletes.
The `main.go` file contains watermill-related setup, database connections and the handler translating events
from one format to another. In `mysql.go` and `postgres.go` you will find definitions of `SchemaAdapters` for
each database.
For more detailed description, see [When an SQL database makes a great Pub/Sub](https://threedots.tech/post/when-sql-database-makes-great-pub-sub/).
## Requirements
To run this example you will need Docker and docker-compose installed. See installation guide at https://docs.docker.com/compose/install/
## Running
Run the command and observe standard output. It should print out incoming users.
```bash
docker-compose up
```
Check what's inside MySQL by running:
```
docker-compose exec mysql mysql -e 'select * from watermill.users;'
```
```
+----+------------------+------------+-----------+---------------------+
| id | user | first_name | last_name | created_at |
+----+------------------+------------+-----------+---------------------+
| 1 | Carroll8506 | Marc | Murphy | 2019-09-28 13:51:53 |
| 2 | Metz8415 | Briana | Bauch | 2019-09-28 13:51:54 |
| 3 | Lebsack6887 | Tomasa | Steuber | 2019-09-28 13:51:55 |
| 4 | Hauck4518 | Alexandra | Halvorson | 2019-09-28 13:51:56 |
| 5 | Reynolds7156 | Ariane | Lebsack | 2019-09-28 13:51:57 |
+----+------------------+------------+-----------+---------------------+
```
And the same for PostgreSQL:
```
docker-compose exec postgres psql -U watermill -d watermill -c 'select * from users;'
```
```
id | username | full_name | created_at
-----+------------------+----------------------+---------------------
1 | Carroll8506 | Marc Murphy | 2019-09-28 13:51:53
2 | Metz8415 | Briana Bauch | 2019-09-28 13:51:54
3 | Lebsack6887 | Tomasa Steuber | 2019-09-28 13:51:55
4 | Hauck4518 | Alexandra Halvorson | 2019-09-28 13:51:56
5 | Reynolds7156 | Ariane Lebsack | 2019-09-28 13:51:57
```

View File

@@ -0,0 +1,31 @@
services:
server:
image: golang:1.25
restart: unless-stopped
depends_on:
- mysql
- postgres
volumes:
- .:/app
- $GOPATH/pkg/mod:/go/pkg/mod
working_dir: /app
command: go run .
mysql:
image: mysql:8.0
restart: unless-stopped
ports:
- 3306:3306
environment:
MYSQL_DATABASE: watermill
MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
postgres:
image: postgres:11
restart: unless-stopped
ports:
- 5432:5432
environment:
POSTGRES_USER: watermill
POSTGRES_DB: watermill
POSTGRES_PASSWORD: "password"

View File

@@ -0,0 +1,25 @@
module main.go
go 1.25.0
require (
github.com/ThreeDotsLabs/watermill v1.5.1
github.com/ThreeDotsLabs/watermill-sql/v4 v4.1.3
github.com/brianvoe/gofakeit/v6 v6.28.0
github.com/go-sql-driver/mysql v1.10.0
github.com/lib/pq v1.12.3
)
require (
filippo.io/edwards25519 v1.2.0 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.9.2 // indirect
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/sony/gobreaker v1.0.0 // indirect
golang.org/x/text v0.37.0 // indirect
)

View File

@@ -0,0 +1,51 @@
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
github.com/ThreeDotsLabs/watermill v1.5.1 h1:t5xMivyf9tpmU3iozPqyrCZXHvoV1XQDfihas4sV0fY=
github.com/ThreeDotsLabs/watermill v1.5.1/go.mod h1:Uop10dA3VeJWsSvis9qO3vbVY892LARrKAdki6WtXS4=
github.com/ThreeDotsLabs/watermill-sql/v4 v4.1.3 h1:d9niNUM3G9nFH2YRdJ7F+qz3IF0bS2IHq6UB2TKWHGE=
github.com/ThreeDotsLabs/watermill-sql/v4 v4.1.3/go.mod h1:Ce2GVZVnyajAh0AkwxSJXwx8ajBBveu1DI/yatan5jc=
github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4=
github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8=
github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts=
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sony/gobreaker v1.0.0 h1:feX5fGGXSl3dYd4aHZItw+FpHLvvoaqkawKjVNiFMNQ=
github.com/sony/gobreaker v1.0.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -0,0 +1,195 @@
package main
import (
"bytes"
"context"
stdSQL "database/sql"
"encoding/gob"
"fmt"
"log"
"time"
"github.com/brianvoe/gofakeit/v6"
driver "github.com/go-sql-driver/mysql"
_ "github.com/lib/pq"
"github.com/ThreeDotsLabs/watermill"
"github.com/ThreeDotsLabs/watermill-sql/v4/pkg/sql"
"github.com/ThreeDotsLabs/watermill/message"
"github.com/ThreeDotsLabs/watermill/message/router/middleware"
"github.com/ThreeDotsLabs/watermill/message/router/plugin"
)
var (
logger = watermill.NewStdLogger(false, false)
postgresTable = "users"
mysqlTable = "users"
)
func main() {
router, err := message.NewRouter(message.RouterConfig{}, logger)
if err != nil {
panic(err)
}
router.AddPlugin(plugin.SignalsHandler)
router.AddMiddleware(middleware.Recoverer)
mysqlDB := createMySQLConnection()
postgresDB := createPostgresConnection()
subscriber := createSubscriber(mysqlDB)
publisher := createPublisher(postgresDB)
go simulateEvents(mysqlDB)
router.AddHandler(
"mysql-to-postgres",
mysqlTable,
subscriber,
postgresTable,
publisher,
func(msg *message.Message) ([]*message.Message, error) {
originUser := mysqlUser{}
decoder := gob.NewDecoder(bytes.NewBuffer(msg.Payload))
err := decoder.Decode(&originUser)
if err != nil {
return nil, err
}
log.Printf("received user: %+v", originUser)
newUser := postgresUser{
ID: originUser.ID,
Username: originUser.User,
FullName: fmt.Sprintf("%s %s", originUser.FirstName, originUser.LastName),
CreatedAt: originUser.CreatedAt,
}
var payload bytes.Buffer
encoder := gob.NewEncoder(&payload)
err = encoder.Encode(newUser)
if err != nil {
return nil, err
}
newMessage := message.NewMessage(watermill.NewULID(), payload.Bytes())
return []*message.Message{newMessage}, nil
},
)
if err := router.Run(context.Background()); err != nil {
panic(err)
}
}
func createMySQLConnection() *stdSQL.DB {
conf := driver.NewConfig()
conf.Net = "tcp"
conf.User = "root"
conf.Addr = "mysql"
conf.DBName = "watermill"
conf.ParseTime = true
db, err := stdSQL.Open("mysql", conf.FormatDSN())
if err != nil {
panic(err)
}
err = db.Ping()
if err != nil {
panic(err)
}
return db
}
func createPostgresConnection() *stdSQL.DB {
dsn := "postgres://watermill:password@postgres/watermill?sslmode=disable"
db, err := stdSQL.Open("postgres", dsn)
if err != nil {
panic(err)
}
err = db.Ping()
if err != nil {
panic(err)
}
return db
}
func createSubscriber(db *stdSQL.DB) message.Subscriber {
sub, err := sql.NewSubscriber(
sql.BeginnerFromStdSQL(db),
sql.SubscriberConfig{
SchemaAdapter: mysqlSchemaAdapter{},
OffsetsAdapter: sql.DefaultMySQLOffsetsAdapter{},
InitializeSchema: true,
},
logger,
)
if err != nil {
panic(err)
}
return sub
}
func createPublisher(db *stdSQL.DB) message.Publisher {
pub, err := sql.NewPublisher(
sql.BeginnerFromStdSQL(db),
sql.PublisherConfig{
SchemaAdapter: postgresSchemaAdapter{},
AutoInitializeSchema: true,
},
logger,
)
if err != nil {
panic(err)
}
return pub
}
func simulateEvents(db *stdSQL.DB) {
pub, err := sql.NewPublisher(
sql.BeginnerFromStdSQL(db),
sql.PublisherConfig{
SchemaAdapter: mysqlSchemaAdapter{},
AutoInitializeSchema: true,
},
logger,
)
if err != nil {
panic(err)
}
for {
user := mysqlUser{
User: gofakeit.Username(),
FirstName: gofakeit.FirstName(),
LastName: gofakeit.LastName(),
CreatedAt: time.Now().UTC(),
}
var payload bytes.Buffer
encoder := gob.NewEncoder(&payload)
err := encoder.Encode(user)
if err != nil {
panic(err)
}
err = pub.Publish(mysqlTable, message.NewMessage(
watermill.NewUUID(),
payload.Bytes(),
))
if err != nil {
panic(err)
}
time.Sleep(time.Second)
}
}

View File

@@ -0,0 +1,105 @@
package main
import (
"bytes"
"encoding/gob"
"fmt"
"strings"
"time"
"github.com/ThreeDotsLabs/watermill"
"github.com/ThreeDotsLabs/watermill-sql/v4/pkg/sql"
"github.com/ThreeDotsLabs/watermill/message"
)
type mysqlUser struct {
ID int64
User string
FirstName string
LastName string
CreatedAt time.Time
}
type mysqlSchemaAdapter struct {
sql.DefaultMySQLSchema
}
func (m mysqlSchemaAdapter) SchemaInitializingQueries(params sql.SchemaInitializingQueriesParams) ([]sql.Query, error) {
createQuery := `
CREATE TABLE IF NOT EXISTS ` + params.Topic + ` (
id INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
user VARCHAR(36) NOT NULL,
first_name VARCHAR(36) NOT NULL,
last_name VARCHAR(36) NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
`
return []sql.Query{{Query: createQuery}}, nil
}
func (m mysqlSchemaAdapter) InsertQuery(params sql.InsertQueryParams) (sql.Query, error) {
insertQuery := fmt.Sprintf(
`INSERT INTO %s (user, first_name, last_name, created_at) VALUES %s`,
params.Topic,
strings.TrimRight(strings.Repeat(`(?,?,?,?),`, len(params.Msgs)), ","),
)
var args []interface{}
for _, msg := range params.Msgs {
user := mysqlUser{}
decoder := gob.NewDecoder(bytes.NewBuffer(msg.Payload))
err := decoder.Decode(&user)
if err != nil {
return sql.Query{}, err
}
args = append(args, user.User, user.FirstName, user.LastName, user.CreatedAt)
}
return sql.Query{Query: insertQuery, Args: args}, nil
}
func (m mysqlSchemaAdapter) SelectQuery(params sql.SelectQueryParams) (sql.Query, error) {
nextOffsetQuery, err := params.OffsetsAdapter.NextOffsetQuery(sql.NextOffsetQueryParams{
Topic: params.Topic,
ConsumerGroup: params.ConsumerGroup,
})
if err != nil {
return sql.Query{}, err
}
selectQuery := `
SELECT id, user, first_name, last_name, created_at FROM ` + params.Topic + `
WHERE
id > (` + nextOffsetQuery.Query + `)
ORDER BY
id ASC
LIMIT 1`
return sql.Query{Query: selectQuery, Args: nextOffsetQuery.Args}, nil
}
func (m mysqlSchemaAdapter) UnmarshalMessage(params sql.UnmarshalMessageParams) (_ sql.Row, err error) {
user := mysqlUser{}
err = params.Row.Scan(&user.ID, &user.User, &user.FirstName, &user.LastName, &user.CreatedAt)
if err != nil {
return sql.Row{}, err
}
var payload bytes.Buffer
encoder := gob.NewEncoder(&payload)
err = encoder.Encode(user)
if err != nil {
return sql.Row{}, err
}
msg := message.NewMessage(watermill.NewULID(), payload.Bytes())
return sql.Row{
Offset: user.ID,
Msg: msg,
}, nil
}

View File

@@ -0,0 +1,68 @@
package main
import (
"bytes"
"encoding/gob"
"errors"
"fmt"
"strings"
"time"
"github.com/ThreeDotsLabs/watermill-sql/v4/pkg/sql"
)
type postgresUser struct {
ID int64
Username string
FullName string
CreatedAt time.Time
}
type postgresSchemaAdapter struct {
sql.DefaultPostgreSQLSchema
}
func (p postgresSchemaAdapter) SchemaInitializingQueries(params sql.SchemaInitializingQueriesParams) ([]sql.Query, error) {
createQuery := `
CREATE TABLE IF NOT EXISTS ` + params.Topic + ` (
id INT NOT NULL PRIMARY KEY,
username VARCHAR(36) NOT NULL,
full_name VARCHAR(36) NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
`
return []sql.Query{{Query: createQuery}}, nil
}
func (p postgresSchemaAdapter) InsertQuery(params sql.InsertQueryParams) (sql.Query, error) {
insertQuery := fmt.Sprintf(
`INSERT INTO %s (id, username, full_name, created_at) VALUES %s`,
params.Topic,
strings.TrimRight(strings.Repeat(`($1,$2,$3,$4),`, len(params.Msgs)), ","),
)
var args []interface{}
for _, msg := range params.Msgs {
user := postgresUser{}
decoder := gob.NewDecoder(bytes.NewBuffer(msg.Payload))
err := decoder.Decode(&user)
if err != nil {
return sql.Query{}, err
}
args = append(args, user.ID, user.Username, user.FullName, user.CreatedAt)
}
return sql.Query{Query: insertQuery, Args: args}, nil
}
func (p postgresSchemaAdapter) SelectQuery(params sql.SelectQueryParams) (sql.Query, error) {
// No need to implement this method, as PostgreSQL subscriber is not used in this example.
return sql.Query{}, nil
}
func (p postgresSchemaAdapter) UnmarshalMessage(params sql.UnmarshalMessageParams) (sql.Row, error) {
return sql.Row{}, errors.New("not implemented")
}