snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 03:23:42 +05:00
commit 89f5be005d
3997 changed files with 961970 additions and 0 deletions

View File

@@ -0,0 +1,389 @@
# PostgreSQL Wire Protocol Support for SeaweedFS
## Overview
This design adds native PostgreSQL wire protocol support to SeaweedFS, enabling compatibility with all PostgreSQL clients, tools, and drivers without requiring custom implementations.
## Benefits
### Universal Compatibility
- **Standard PostgreSQL Clients**: psql, pgAdmin, Adminer, etc.
- **JDBC/ODBC Drivers**: Use standard PostgreSQL drivers
- **BI Tools**: Tableau, Power BI, Grafana, Superset with native PostgreSQL connectors
- **ORMs**: Hibernate, ActiveRecord, Django ORM, etc.
- **Programming Languages**: Native PostgreSQL libraries in Python (psycopg2), Node.js (pg), Go (lib/pq), etc.
### Enterprise Integration
- **Existing Infrastructure**: Drop-in replacement for PostgreSQL in read-only scenarios
- **Migration Path**: Easy transition from PostgreSQL-based analytics
- **Tool Ecosystem**: Leverage entire PostgreSQL ecosystem
## Architecture
```
┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐
│ PostgreSQL │ │ PostgreSQL │ │ SeaweedFS │
│ Clients │◄──►│ Protocol │◄──►│ SQL Engine │
│ (psql, etc.) │ │ Server │ │ │
└─────────────────┘ └──────────────────┘ └─────────────────┘
│
▼
┌──────────────────┐
│ Authentication │
│ & Session Mgmt │
└──────────────────┘
```
## Core Components
### 1. PostgreSQL Wire Protocol Handler
```go
// PostgreSQL message types
const (
PG_MSG_STARTUP = 0x00 // Startup message
PG_MSG_QUERY = 'Q' // Simple query
PG_MSG_PARSE = 'P' // Parse (prepared statement)
PG_MSG_BIND = 'B' // Bind parameters
PG_MSG_EXECUTE = 'E' // Execute prepared statement
PG_MSG_DESCRIBE = 'D' // Describe statement/portal
PG_MSG_CLOSE = 'C' // Close statement/portal
PG_MSG_FLUSH = 'H' // Flush
PG_MSG_SYNC = 'S' // Sync
PG_MSG_TERMINATE = 'X' // Terminate connection
PG_MSG_PASSWORD = 'p' // Password message
)
// PostgreSQL response types
const (
PG_RESP_AUTH_OK = 'R' // Authentication OK
PG_RESP_AUTH_REQ = 'R' // Authentication request
PG_RESP_BACKEND_KEY = 'K' // Backend key data
PG_RESP_PARAMETER = 'S' // Parameter status
PG_RESP_READY = 'Z' // Ready for query
PG_RESP_COMMAND = 'C' // Command complete
PG_RESP_DATA_ROW = 'D' // Data row
PG_RESP_ROW_DESC = 'T' // Row description
PG_RESP_PARSE_COMPLETE = '1' // Parse complete
PG_RESP_BIND_COMPLETE = '2' // Bind complete
PG_RESP_CLOSE_COMPLETE = '3' // Close complete
PG_RESP_ERROR = 'E' // Error response
PG_RESP_NOTICE = 'N' // Notice response
)
```
### 2. Session Management
```go
type PostgreSQLSession struct {
conn net.Conn
reader *bufio.Reader
writer *bufio.Writer
authenticated bool
username string
database string
parameters map[string]string
preparedStmts map[string]*PreparedStatement
portals map[string]*Portal
transactionState TransactionState
processID uint32
secretKey uint32
}
type PreparedStatement struct {
name string
query string
paramTypes []uint32
fields []FieldDescription
}
type Portal struct {
name string
statement string
parameters [][]byte
suspended bool
}
```
### 3. SQL Translation Layer
```go
type PostgreSQLTranslator struct {
dialectMap map[string]string
}
// Translates PostgreSQL-specific SQL to SeaweedFS SQL
func (t *PostgreSQLTranslator) TranslateQuery(pgSQL string) (string, error) {
// Handle PostgreSQL-specific syntax:
// - SELECT version() -> SELECT 'SeaweedFS 1.0'
// - SELECT current_database() -> SELECT 'default'
// - SELECT current_user -> SELECT 'seaweedfs'
// - \d commands -> SHOW TABLES/DESCRIBE equivalents
// - PostgreSQL system catalogs -> SeaweedFS equivalents
}
```
### 4. Data Type Mapping
```go
var PostgreSQLTypeMap = map[string]uint32{
"TEXT": 25, // PostgreSQL TEXT type
"VARCHAR": 1043, // PostgreSQL VARCHAR type
"INTEGER": 23, // PostgreSQL INTEGER type
"BIGINT": 20, // PostgreSQL BIGINT type
"FLOAT": 701, // PostgreSQL FLOAT8 type
"BOOLEAN": 16, // PostgreSQL BOOLEAN type
"TIMESTAMP": 1114, // PostgreSQL TIMESTAMP type
"JSON": 114, // PostgreSQL JSON type
}
func SeaweedToPostgreSQLType(seaweedType string) uint32 {
if pgType, exists := PostgreSQLTypeMap[strings.ToUpper(seaweedType)]; exists {
return pgType
}
return 25 // Default to TEXT
}
```
## Protocol Implementation
### 1. Connection Flow
```
Client Server
│ │
├─ StartupMessage ────────────►│
│ ├─ AuthenticationOk
│ ├─ ParameterStatus (multiple)
│ ├─ BackendKeyData
│ └─ ReadyForQuery
│ │
├─ Query('SELECT 1') ─────────►│
│ ├─ RowDescription
│ ├─ DataRow
│ ├─ CommandComplete
│ └─ ReadyForQuery
│ │
├─ Parse('stmt1', 'SELECT $1')►│
│ └─ ParseComplete
├─ Bind('portal1', 'stmt1')───►│
│ └─ BindComplete
├─ Execute('portal1')─────────►│
│ ├─ DataRow (multiple)
│ └─ CommandComplete
├─ Sync ──────────────────────►│
│ └─ ReadyForQuery
│ │
├─ Terminate ─────────────────►│
│ └─ [Connection closed]
```
### 2. Authentication
```go
type AuthMethod int
const (
AuthTrust AuthMethod = iota
AuthPassword
AuthMD5
AuthSASL
)
func (s *PostgreSQLServer) handleAuthentication(session *PostgreSQLSession) error {
switch s.authMethod {
case AuthTrust:
return s.sendAuthenticationOk(session)
case AuthPassword:
return s.handlePasswordAuth(session)
case AuthMD5:
return s.handleMD5Auth(session)
default:
return fmt.Errorf("unsupported auth method")
}
}
```
### 3. Query Processing
```go
func (s *PostgreSQLServer) handleSimpleQuery(session *PostgreSQLSession, query string) error {
// 1. Translate PostgreSQL SQL to SeaweedFS SQL
translatedQuery, err := s.translator.TranslateQuery(query)
if err != nil {
return s.sendError(session, err)
}
// 2. Execute using existing SQL engine
result, err := s.sqlEngine.ExecuteSQL(context.Background(), translatedQuery)
if err != nil {
return s.sendError(session, err)
}
// 3. Send results in PostgreSQL format
err = s.sendRowDescription(session, result.Columns)
if err != nil {
return err
}
for _, row := range result.Rows {
err = s.sendDataRow(session, row)
if err != nil {
return err
}
}
return s.sendCommandComplete(session, fmt.Sprintf("SELECT %d", len(result.Rows)))
}
```
## System Catalogs Support
PostgreSQL clients expect certain system catalogs. We'll implement views for key ones:
```sql
-- pg_tables equivalent
SELECT
'default' as schemaname,
table_name as tablename,
'seaweedfs' as tableowner,
NULL as tablespace,
false as hasindexes,
false as hasrules,
false as hastriggers
FROM information_schema.tables;
-- pg_database equivalent
SELECT
database_name as datname,
'seaweedfs' as datdba,
'UTF8' as encoding,
'C' as datcollate,
'C' as datctype
FROM information_schema.schemata;
-- pg_version equivalent
SELECT 'SeaweedFS 1.0 (PostgreSQL 14.0 compatible)' as version;
```
## Configuration
### Server Configuration
```go
type PostgreSQLServerConfig struct {
Host string
Port int
Database string
AuthMethod AuthMethod
Users map[string]string // username -> password
TLSConfig *tls.Config
MaxConns int
IdleTimeout time.Duration
}
```
### Client Connection String
```bash
# Standard PostgreSQL connection strings work
psql "host=localhost port=5432 dbname=default user=seaweedfs"
PGPASSWORD=secret psql -h localhost -p 5432 -U seaweedfs -d default
# JDBC URL
jdbc:postgresql://localhost:5432/default?user=seaweedfs&password=secret
```
## Command Line Interface
```bash
# Start PostgreSQL protocol server
weed-db -port=5432 -auth=trust
weed-db -port=5432 -auth=password -users="admin:secret;readonly:pass"
weed-db -port=5432 -tls-cert=server.crt -tls-key=server.key
# Configuration options
-host=localhost # Listen host
-port=5432 # PostgreSQL standard port
-auth=trust|password|md5 # Authentication method
-users=user:pass;user2:pass2 # User credentials (password/md5 auth) - use semicolons to separate users
-database=default # Default database name
-max-connections=100 # Maximum concurrent connections
-idle-timeout=1h # Connection idle timeout
-tls-cert="" # TLS certificate file
-tls-key="" # TLS private key file
```
## Client Compatibility Testing
### Essential Clients
- **psql**: PostgreSQL command line client
- **pgAdmin**: Web-based administration tool
- **DBeaver**: Universal database tool
- **DataGrip**: JetBrains database IDE
### Programming Language Drivers
- **Python**: psycopg2, asyncpg
- **Java**: PostgreSQL JDBC driver
- **Node.js**: pg, node-postgres
- **Go**: lib/pq, pgx
- **.NET**: Npgsql
### BI Tools
- **Grafana**: PostgreSQL data source
- **Superset**: PostgreSQL connector
- **Tableau**: PostgreSQL native connector
- **Power BI**: PostgreSQL connector
## Implementation Plan
1. **Phase 1**: Basic wire protocol and simple queries
2. **Phase 2**: Extended query protocol (prepared statements)
3. **Phase 3**: System catalog views
4. **Phase 4**: Advanced features (transactions, notifications)
5. **Phase 5**: Performance optimization and caching
## Limitations
### Read-Only Access
- INSERT/UPDATE/DELETE operations not supported
- Returns appropriate error messages for write operations
### Partial SQL Compatibility
- Subset of PostgreSQL SQL features
- SeaweedFS-specific limitations apply
### System Features
- No stored procedures/functions
- No triggers or constraints
- No user-defined types
- Limited transaction support (mostly no-op)
## Security Considerations
### Authentication
- Support for trust, password, and MD5 authentication
- TLS encryption support
- User access control
### SQL Injection Prevention
- Prepared statements with parameter binding
- Input validation and sanitization
- Query complexity limits
## Performance Optimizations
### Connection Pooling
- Configurable maximum connections
- Connection reuse and idle timeout
- Memory efficient session management
### Query Caching
- Prepared statement caching
- Result set caching for repeated queries
- Metadata caching
### Protocol Efficiency
- Binary result format support
- Batch query processing
- Streaming large result sets
This design provides a comprehensive PostgreSQL wire protocol implementation that makes SeaweedFS accessible to the entire PostgreSQL ecosystem while maintaining compatibility and performance.

View File

@@ -0,0 +1,284 @@
# PostgreSQL Wire Protocol Package
This package implements PostgreSQL wire protocol support for SeaweedFS, enabling universal compatibility with PostgreSQL clients, tools, and applications.
## Package Structure
```
weed/server/postgres/
├── README.md # This documentation
├── server.go # Main PostgreSQL server implementation
├── protocol.go # Wire protocol message handlers with MQ integration
├── DESIGN.md # Architecture and design documentation
└── IMPLEMENTATION.md # Complete implementation guide
```
## Core Components
### `server.go`
- **PostgreSQLServer**: Main server structure with connection management
- **PostgreSQLSession**: Individual client session handling
- **PostgreSQLServerConfig**: Server configuration options
- **Authentication System**: Trust, password, and MD5 authentication
- **TLS Support**: Encrypted connections with custom certificates
- **Connection Pooling**: Resource management and cleanup
### `protocol.go`
- **Wire Protocol Implementation**: Full PostgreSQL 3.0 protocol support
- **Message Handlers**: Startup, query, parse/bind/execute sequences
- **Response Generation**: Row descriptions, data rows, command completion
- **Data Type Mapping**: SeaweedFS to PostgreSQL type conversion
- **SQL Parser**: Uses PostgreSQL-native parser for full dialect compatibility
- **Error Handling**: PostgreSQL-compliant error responses
- **MQ Integration**: Direct integration with SeaweedFS SQL engine for real topic data
- **System Query Support**: Essential PostgreSQL system queries (version, current_user, etc.)
- **Database Context**: Session-based database switching with USE commands
## Key Features
### Real MQ Topic Integration
The PostgreSQL server now directly integrates with SeaweedFS Message Queue topics, providing:
- **Live Topic Discovery**: Automatically discovers MQ namespaces and topics from the filer
- **Real Schema Information**: Reads actual topic schemas from broker configuration
- **Actual Data Access**: Queries real MQ data stored in Parquet and log files
- **Dynamic Updates**: Reflects topic additions and schema changes automatically
- **Consistent SQL Engine**: Uses the same SQL engine as `weed-sql` command
### Database Context Management
- **Session Isolation**: Each PostgreSQL connection has its own database context
- **USE Command Support**: Switch between namespaces using standard `USE database` syntax
- **Auto-Discovery**: Topics are discovered and registered on first access
- **Schema Caching**: Efficient caching of topic schemas and metadata
## Usage
### Import the Package
```go
import "github.com/seaweedfs/seaweedfs/weed/server/postgres"
```
### Create and Start Server
```go
config := &postgres.PostgreSQLServerConfig{
Host: "localhost",
Port: 5432,
AuthMethod: postgres.AuthMD5,
Users: map[string]string{"admin": "secret"},
Database: "default",
MaxConns: 100,
IdleTimeout: time.Hour,
}
server, err := postgres.NewPostgreSQLServer(config, "localhost:9333")
if err != nil {
return err
}
err = server.Start()
if err != nil {
return err
}
// Server is now accepting PostgreSQL connections
```
## Authentication Methods
The package supports three authentication methods:
### Trust Authentication
```go
AuthMethod: postgres.AuthTrust
```
- No password required
- Suitable for development/testing
- Not recommended for production
### Password Authentication
```go
AuthMethod: postgres.AuthPassword,
Users: map[string]string{"user": "password"}
```
- Clear text password transmission
- Simple but less secure
- Requires TLS for production use
### MD5 Authentication
```go
AuthMethod: postgres.AuthMD5,
Users: map[string]string{"user": "password"}
```
- Secure hashed authentication with salt
- **Recommended for production**
- Compatible with all PostgreSQL clients
## TLS Configuration
Enable TLS encryption for secure connections:
```go
cert, err := tls.LoadX509KeyPair("server.crt", "server.key")
if err != nil {
return err
}
config.TLSConfig = &tls.Config{
Certificates: []tls.Certificate{cert},
}
```
## Client Compatibility
This implementation is compatible with:
### Command Line Tools
- `psql` - PostgreSQL command line client
- `pgcli` - Enhanced command line with auto-completion
- Database IDEs (DataGrip, DBeaver)
### Programming Languages
- **Python**: psycopg2, asyncpg
- **Java**: PostgreSQL JDBC driver
- **JavaScript**: pg (node-postgres)
- **Go**: lib/pq, pgx
- **.NET**: Npgsql
- **PHP**: pdo_pgsql
- **Ruby**: pg gem
### BI Tools
- Tableau (native PostgreSQL connector)
- Power BI (PostgreSQL data source)
- Grafana (PostgreSQL plugin)
- Apache Superset
## Supported SQL Operations
### Data Queries
```sql
SELECT * FROM topic_name;
SELECT id, message FROM topic_name WHERE condition;
SELECT COUNT(*) FROM topic_name;
SELECT MIN(id), MAX(id), AVG(amount) FROM topic_name;
```
### Schema Information
```sql
SHOW DATABASES;
SHOW TABLES;
DESCRIBE topic_name;
DESC topic_name;
```
### System Information
```sql
SELECT version();
SELECT current_database();
SELECT current_user;
```
### System Columns
```sql
SELECT id, message, _timestamp_ns, _key, _source FROM topic_name;
```
## Configuration Options
### Server Configuration
- **Host/Port**: Server binding address and port
- **Authentication**: Method and user credentials
- **Database**: Default database/namespace name
- **Connections**: Maximum concurrent connections
- **Timeouts**: Idle connection timeout
- **TLS**: Certificate and encryption settings
### Performance Tuning
- **Connection Limits**: Prevent resource exhaustion
- **Idle Timeout**: Automatic cleanup of unused connections
- **Memory Management**: Efficient session handling
- **Query Streaming**: Large result set support
## Error Handling
The package provides PostgreSQL-compliant error responses:
- **Connection Errors**: Authentication failures, network issues
- **SQL Errors**: Invalid syntax, missing tables
- **Resource Errors**: Connection limits, timeouts
- **Security Errors**: Permission denied, invalid credentials
## Development and Testing
### Unit Tests
Run PostgreSQL package tests:
```bash
go test ./weed/server/postgres
```
### Integration Testing
Use the provided Python test client:
```bash
python postgres-examples/test_client.py --host localhost --port 5432
```
### Manual Testing
Connect with psql:
```bash
psql -h localhost -p 5432 -U seaweedfs -d default
```
## Documentation
- **DESIGN.md**: Complete architecture and design overview
- **IMPLEMENTATION.md**: Detailed implementation guide
- **postgres-examples/**: Client examples and test scripts
- **Command Documentation**: `weed-db -help`
## Security Considerations
### Production Deployment
- Use MD5 or stronger authentication
- Enable TLS encryption
- Configure appropriate connection limits
- Monitor for suspicious activity
- Use strong passwords
- Implement proper firewall rules
### Access Control
- Create dedicated read-only users
- Use principle of least privilege
- Monitor connection patterns
- Log authentication attempts
## Architecture Notes
### SQL Parser Dialect Considerations
**✅ POSTGRESQL ONLY**: SeaweedFS SQL engine exclusively supports PostgreSQL syntax:
- **✅ Core Engine**: `engine.go` uses custom PostgreSQL parser for proper dialect support
- **PostgreSQL Server**: Uses PostgreSQL parser for optimal wire protocol compatibility
- **Parser**: Custom lightweight PostgreSQL parser for full PostgreSQL compatibility
- **Support Status**: Only PostgreSQL syntax is supported - MySQL parsing has been removed
**Key Benefits of PostgreSQL Parser**:
- **Native Dialect Support**: Correctly handles PostgreSQL-specific syntax and semantics
- **System Catalog Compatibility**: Supports `pg_catalog`, `information_schema` queries
- **Operator Compatibility**: Handles `||` string concatenation, PostgreSQL-specific operators
- **Type System Alignment**: Better PostgreSQL type inference and coercion
- **Reduced Translation Overhead**: Eliminates need for dialect translation layer
**PostgreSQL Syntax Support**:
- **Identifier Quoting**: Uses PostgreSQL double quotes (`"`) for identifiers
- **String Concatenation**: Supports PostgreSQL `||` operator
- **System Functions**: Full support for PostgreSQL system catalogs (`pg_catalog`) and functions
- **Standard Compliance**: Follows PostgreSQL SQL standard and dialect
**Implementation Features**:
- Native PostgreSQL query processing in `protocol.go`
- System query support (`SELECT version()`, `BEGIN`, etc.)
- Type mapping between PostgreSQL and SeaweedFS schema types
- Error code mapping to PostgreSQL standards
- Comprehensive PostgreSQL wire protocol support
This package provides enterprise-grade PostgreSQL compatibility, enabling seamless integration of SeaweedFS with the entire PostgreSQL ecosystem.

View File

@@ -0,0 +1,917 @@
package postgres
import (
"context"
"encoding/binary"
"fmt"
"io"
"strconv"
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
"github.com/seaweedfs/seaweedfs/weed/query/engine"
"github.com/seaweedfs/seaweedfs/weed/query/sqltypes"
"github.com/seaweedfs/seaweedfs/weed/util/sqlutil"
"github.com/seaweedfs/seaweedfs/weed/util/version"
)
const (
// maxStartupMessageSize limits the startup handshake body to 10 KiB.
// Startup parameters (username, database, etc.) are always small.
maxStartupMessageSize = 10000
// maxMessageSize limits runtime message bodies to 100 MiB to prevent OOM.
// PostgreSQL protocol allows up to 1 GiB per message in theory, but real
// queries are rarely larger than a few megabytes.
maxMessageSize = 100 * 1024 * 1024 // 100 MiB
// maxAuthMessageSize caps password/MD5 payloads, which arrive before the
// client is authenticated. Credentials are tiny, so the pre-auth allocation
// stays far below maxMessageSize.
maxAuthMessageSize = 10 * 1024 // 10 KiB
)
// mapErrorToPostgreSQLCode maps SeaweedFS SQL engine errors to appropriate PostgreSQL error codes
func mapErrorToPostgreSQLCode(err error) string {
if err == nil {
return "00000" // Success
}
// Use typed errors for robust error mapping
switch err.(type) {
case engine.ParseError:
return "42601" // Syntax error
case engine.TableNotFoundError:
return "42P01" // Undefined table
case engine.ColumnNotFoundError:
return "42703" // Undefined column
case engine.UnsupportedFeatureError:
return "0A000" // Feature not supported
case engine.AggregationError:
// Aggregation errors are usually function-related issues
return "42883" // Undefined function (aggregation function issues)
case engine.DataSourceError:
// Data source errors are usually access or connection issues
return "08000" // Connection exception
case engine.OptimizationError:
// Optimization failures are usually feature limitations
return "0A000" // Feature not supported
case engine.NoSchemaError:
// Topic exists but no schema available
return "42P01" // Undefined table (treat as table not found)
}
// Fallback: analyze error message for backward compatibility with non-typed errors
errLower := strings.ToLower(err.Error())
// Parsing and syntax errors
if strings.Contains(errLower, "parse error") || strings.Contains(errLower, "syntax") {
return "42601" // Syntax error
}
// Unsupported features
if strings.Contains(errLower, "unsupported") || strings.Contains(errLower, "not supported") {
return "0A000" // Feature not supported
}
// Table/topic not found
if strings.Contains(errLower, "not found") ||
(strings.Contains(errLower, "topic") && strings.Contains(errLower, "available")) {
return "42P01" // Undefined table
}
// Column-related errors
if strings.Contains(errLower, "column") || strings.Contains(errLower, "field") {
return "42703" // Undefined column
}
// Multi-table or complex query limitations
if strings.Contains(errLower, "single table") || strings.Contains(errLower, "join") {
return "0A000" // Feature not supported
}
// Default to generic syntax/access error
return "42000" // Syntax error or access rule violation
}
// handleMessage processes a single PostgreSQL protocol message
func (s *PostgreSQLServer) handleMessage(session *PostgreSQLSession) error {
// Read message type
msgType := make([]byte, 1)
_, err := io.ReadFull(session.reader, msgType)
if err != nil {
return err
}
// Read message length
length := make([]byte, 4)
_, err = io.ReadFull(session.reader, length)
if err != nil {
return err
}
rawLength := binary.BigEndian.Uint32(length)
if rawLength < 4 {
return fmt.Errorf("invalid message length: %d (must be >= 4)", rawLength)
}
msgLength := rawLength - 4
if msgLength > maxMessageSize {
return fmt.Errorf("message too large: %d bytes", msgLength)
}
msgBody := make([]byte, msgLength)
if msgLength > 0 {
_, err = io.ReadFull(session.reader, msgBody)
if err != nil {
return err
}
}
// Process message based on type
switch msgType[0] {
case PG_MSG_QUERY:
query := strings.TrimSuffix(string(msgBody), "\x00") // Remove null terminator if present
return s.handleSimpleQuery(session, query)
case PG_MSG_PARSE:
return s.handleParse(session, msgBody)
case PG_MSG_BIND:
return s.handleBind(session, msgBody)
case PG_MSG_EXECUTE:
return s.handleExecute(session, msgBody)
case PG_MSG_DESCRIBE:
return s.handleDescribe(session, msgBody)
case PG_MSG_CLOSE:
return s.handleClose(session, msgBody)
case PG_MSG_FLUSH:
return s.handleFlush(session)
case PG_MSG_SYNC:
return s.handleSync(session)
case PG_MSG_TERMINATE:
return io.EOF // Signal connection termination
default:
return s.sendError(session, "08P01", fmt.Sprintf("unknown message type: %c", msgType[0]))
}
}
// handleSimpleQuery processes a simple query message
func (s *PostgreSQLServer) handleSimpleQuery(session *PostgreSQLSession, query string) error {
glog.V(2).Infof("PostgreSQL Query (ID: %d): %s", session.processID, query)
// Add comprehensive error recovery to prevent crashes
defer func() {
if r := recover(); r != nil {
glog.Errorf("Panic in handleSimpleQuery (ID: %d): %v", session.processID, r)
// Try to send error message
s.sendError(session, "XX000", fmt.Sprintf("Internal error: %v", r))
// Try to send ReadyForQuery to keep connection alive
s.sendReadyForQuery(session)
}
}()
// Handle USE database commands for session context
parts := strings.Fields(strings.TrimSpace(query))
if len(parts) >= 2 && strings.ToUpper(parts[0]) == "USE" {
// Re-join the parts after "USE" to handle names with spaces, then trim.
dbName := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(query), parts[0]))
// Unquote if necessary (handle quoted identifiers like "my-database")
if len(dbName) > 1 && dbName[0] == '"' && dbName[len(dbName)-1] == '"' {
dbName = dbName[1 : len(dbName)-1]
} else if len(dbName) > 1 && dbName[0] == '`' && dbName[len(dbName)-1] == '`' {
// Also handle backtick quotes for MySQL/other client compatibility
dbName = dbName[1 : len(dbName)-1]
}
session.database = dbName
s.sqlEngine.GetCatalog().SetCurrentDatabase(dbName)
// Send command complete for USE
err := s.sendCommandComplete(session, "USE")
if err != nil {
return err
}
// Send ReadyForQuery and exit (don't continue processing)
return s.sendReadyForQuery(session)
}
// Set database context in SQL engine if session database is different from current
if session.database != "" && session.database != s.sqlEngine.GetCatalog().GetCurrentDatabase() {
s.sqlEngine.GetCatalog().SetCurrentDatabase(session.database)
}
// Split query string into individual statements to handle multi-statement queries
queries := sqlutil.SplitStatements(query)
// Execute each statement sequentially
for _, singleQuery := range queries {
cleanQuery := strings.TrimSpace(singleQuery)
if cleanQuery == "" {
continue // Skip empty statements
}
// Handle PostgreSQL-specific system queries directly
if systemResult := s.handleSystemQuery(session, cleanQuery); systemResult != nil {
err := s.sendSystemQueryResult(session, systemResult, cleanQuery)
if err != nil {
return err
}
continue // Continue with next statement
}
// Execute using PostgreSQL-compatible SQL engine for proper dialect support
ctx := context.Background()
var result *engine.QueryResult
var err error
// Execute SQL query with panic recovery to prevent crashes
func() {
defer func() {
if r := recover(); r != nil {
glog.Errorf("Panic in SQL execution (ID: %d, Query: %s): %v", session.processID, cleanQuery, r)
err = fmt.Errorf("internal error during SQL execution: %v", r)
}
}()
// Use the main sqlEngine (now uses CockroachDB parser for PostgreSQL compatibility)
result, err = s.sqlEngine.ExecuteSQL(ctx, cleanQuery)
}()
if err != nil {
// Send error message but keep connection alive
errorCode := mapErrorToPostgreSQLCode(err)
sendErr := s.sendError(session, errorCode, err.Error())
if sendErr != nil {
return sendErr
}
// Send ReadyForQuery to keep connection alive
return s.sendReadyForQuery(session)
}
if result.Error != nil {
// Send error message but keep connection alive
errorCode := mapErrorToPostgreSQLCode(result.Error)
sendErr := s.sendError(session, errorCode, result.Error.Error())
if sendErr != nil {
return sendErr
}
// Send ReadyForQuery to keep connection alive
return s.sendReadyForQuery(session)
}
// Send results for this statement
if len(result.Columns) > 0 {
// Send row description
err = s.sendRowDescription(session, result)
if err != nil {
return err
}
// Send data rows
for _, row := range result.Rows {
err = s.sendDataRow(session, row)
if err != nil {
return err
}
}
}
// Send command complete for this statement
tag := s.getCommandTag(cleanQuery, len(result.Rows))
err = s.sendCommandComplete(session, tag)
if err != nil {
return err
}
}
// Send ready for query after all statements are processed
return s.sendReadyForQuery(session)
}
// SystemQueryResult represents the result of a system query
type SystemQueryResult struct {
Columns []string
Rows [][]string
}
// handleSystemQuery handles PostgreSQL system queries directly
func (s *PostgreSQLServer) handleSystemQuery(session *PostgreSQLSession, query string) *SystemQueryResult {
// Trim and normalize query
query = strings.TrimSpace(query)
query = strings.TrimSuffix(query, ";")
queryLower := strings.ToLower(query)
// Handle essential PostgreSQL system queries
switch queryLower {
case "select version()":
return &SystemQueryResult{
Columns: []string{"version"},
Rows: [][]string{{fmt.Sprintf("SeaweedFS %s (PostgreSQL 14.0 compatible)", version.VERSION_NUMBER)}},
}
case "select current_database()":
return &SystemQueryResult{
Columns: []string{"current_database"},
Rows: [][]string{{s.config.Database}},
}
case "select current_user":
return &SystemQueryResult{
Columns: []string{"current_user"},
Rows: [][]string{{"seaweedfs"}},
}
case "select current_setting('server_version')":
return &SystemQueryResult{
Columns: []string{"server_version"},
Rows: [][]string{{fmt.Sprintf("%s (SeaweedFS)", version.VERSION_NUMBER)}},
}
case "select current_setting('server_encoding')":
return &SystemQueryResult{
Columns: []string{"server_encoding"},
Rows: [][]string{{"UTF8"}},
}
case "select current_setting('client_encoding')":
return &SystemQueryResult{
Columns: []string{"client_encoding"},
Rows: [][]string{{"UTF8"}},
}
}
// Handle transaction commands (no-op for read-only)
switch queryLower {
case "begin", "start transaction":
return &SystemQueryResult{
Columns: []string{"status"},
Rows: [][]string{{"BEGIN"}},
}
case "commit":
return &SystemQueryResult{
Columns: []string{"status"},
Rows: [][]string{{"COMMIT"}},
}
case "rollback":
return &SystemQueryResult{
Columns: []string{"status"},
Rows: [][]string{{"ROLLBACK"}},
}
}
// If starts with SET, return a no-op
if strings.HasPrefix(queryLower, "set ") {
return &SystemQueryResult{
Columns: []string{"status"},
Rows: [][]string{{"SET"}},
}
}
// Return nil to use SQL engine
return nil
}
// sendSystemQueryResult sends the result of a system query
func (s *PostgreSQLServer) sendSystemQueryResult(session *PostgreSQLSession, result *SystemQueryResult, query string) error {
// Add panic recovery to prevent crashes in system query results
defer func() {
if r := recover(); r != nil {
glog.Errorf("Panic in sendSystemQueryResult (ID: %d, Query: %s): %v", session.processID, query, r)
// Try to send error and continue
s.sendError(session, "XX000", fmt.Sprintf("Internal error in system query: %v", r))
}
}()
// Create column descriptions for system query results
columns := make([]string, len(result.Columns))
for i, col := range result.Columns {
columns[i] = col
}
// Convert to sqltypes.Value format
var sqlRows [][]sqltypes.Value
for _, row := range result.Rows {
sqlRow := make([]sqltypes.Value, len(row))
for i, cell := range row {
sqlRow[i] = sqltypes.NewVarChar(cell)
}
sqlRows = append(sqlRows, sqlRow)
}
// Send row description (create a temporary QueryResult for consistency)
tempResult := &engine.QueryResult{
Columns: columns,
Rows: sqlRows,
}
err := s.sendRowDescription(session, tempResult)
if err != nil {
return err
}
// Send data rows
for _, row := range sqlRows {
err = s.sendDataRow(session, row)
if err != nil {
return err
}
}
// Send command complete
tag := s.getCommandTag(query, len(result.Rows))
err = s.sendCommandComplete(session, tag)
if err != nil {
return err
}
// Send ready for query
return s.sendReadyForQuery(session)
}
// handleParse processes a Parse message (prepared statement)
func (s *PostgreSQLServer) handleParse(session *PostgreSQLSession, msgBody []byte) error {
// Parse message format: statement_name\0query\0param_count(int16)[param_type(int32)...]
parts := strings.Split(string(msgBody), "\x00")
if len(parts) < 2 {
return s.sendError(session, "08P01", "invalid Parse message format")
}
stmtName := parts[0]
query := parts[1]
// Create prepared statement
stmt := &PreparedStatement{
Name: stmtName,
Query: query,
ParamTypes: []uint32{},
Fields: []FieldDescription{},
}
session.preparedStmts[stmtName] = stmt
// Send parse complete
return s.sendParseComplete(session)
}
// handleBind processes a Bind message
func (s *PostgreSQLServer) handleBind(session *PostgreSQLSession, msgBody []byte) error {
// For now, simple implementation
// In full implementation, would parse parameters and create portal
// Send bind complete
return s.sendBindComplete(session)
}
// handleExecute processes an Execute message
func (s *PostgreSQLServer) handleExecute(session *PostgreSQLSession, msgBody []byte) error {
// Parse portal name
parts := strings.Split(string(msgBody), "\x00")
if len(parts) == 0 {
return s.sendError(session, "08P01", "invalid Execute message format")
}
portalName := parts[0]
// For now, execute as simple query
// In full implementation, would use portal with parameters
glog.V(2).Infof("PostgreSQL Execute portal (ID: %d): %s", session.processID, portalName)
// Send command complete
err := s.sendCommandComplete(session, "SELECT 0")
if err != nil {
return err
}
return nil
}
// handleDescribe processes a Describe message
func (s *PostgreSQLServer) handleDescribe(session *PostgreSQLSession, msgBody []byte) error {
if len(msgBody) < 2 {
return s.sendError(session, "08P01", "invalid Describe message format")
}
objectType := msgBody[0] // 'S' for statement, 'P' for portal
objectName := string(msgBody[1:])
glog.V(2).Infof("PostgreSQL Describe %c (ID: %d): %s", objectType, session.processID, objectName)
// For now, send empty row description
tempResult := &engine.QueryResult{
Columns: []string{},
Rows: [][]sqltypes.Value{},
}
return s.sendRowDescription(session, tempResult)
}
// handleClose processes a Close message
func (s *PostgreSQLServer) handleClose(session *PostgreSQLSession, msgBody []byte) error {
if len(msgBody) < 2 {
return s.sendError(session, "08P01", "invalid Close message format")
}
objectType := msgBody[0] // 'S' for statement, 'P' for portal
objectName := string(msgBody[1:])
switch objectType {
case 'S':
delete(session.preparedStmts, objectName)
case 'P':
delete(session.portals, objectName)
}
// Send close complete
return s.sendCloseComplete(session)
}
// handleFlush processes a Flush message
func (s *PostgreSQLServer) handleFlush(session *PostgreSQLSession) error {
return session.writer.Flush()
}
// handleSync processes a Sync message
func (s *PostgreSQLServer) handleSync(session *PostgreSQLSession) error {
// Reset transaction state if needed
session.transactionState = PG_TRANS_IDLE
// Send ready for query
return s.sendReadyForQuery(session)
}
// sendParameterStatus sends a parameter status message
func (s *PostgreSQLServer) sendParameterStatus(session *PostgreSQLSession, name, value string) error {
msg := make([]byte, 0)
msg = append(msg, PG_RESP_PARAMETER)
// Calculate length
length := 4 + len(name) + 1 + len(value) + 1
lengthBytes := make([]byte, 4)
binary.BigEndian.PutUint32(lengthBytes, uint32(length))
msg = append(msg, lengthBytes...)
// Add name and value
msg = append(msg, []byte(name)...)
msg = append(msg, 0) // null terminator
msg = append(msg, []byte(value)...)
msg = append(msg, 0) // null terminator
_, err := session.writer.Write(msg)
if err == nil {
err = session.writer.Flush()
}
return err
}
// sendBackendKeyData sends backend key data
func (s *PostgreSQLServer) sendBackendKeyData(session *PostgreSQLSession) error {
msg := make([]byte, 13)
msg[0] = PG_RESP_BACKEND_KEY
binary.BigEndian.PutUint32(msg[1:5], 12)
binary.BigEndian.PutUint32(msg[5:9], session.processID)
binary.BigEndian.PutUint32(msg[9:13], session.secretKey)
_, err := session.writer.Write(msg)
if err == nil {
err = session.writer.Flush()
}
return err
}
// sendReadyForQuery sends ready for query message
func (s *PostgreSQLServer) sendReadyForQuery(session *PostgreSQLSession) error {
msg := make([]byte, 6)
msg[0] = PG_RESP_READY
binary.BigEndian.PutUint32(msg[1:5], 5)
msg[5] = session.transactionState
_, err := session.writer.Write(msg)
if err == nil {
err = session.writer.Flush()
}
return err
}
// sendRowDescription sends row description message
func (s *PostgreSQLServer) sendRowDescription(session *PostgreSQLSession, result *engine.QueryResult) error {
msg := make([]byte, 0)
msg = append(msg, PG_RESP_ROW_DESC)
// Calculate message length
length := 4 + 2 // length + field count
for _, col := range result.Columns {
length += len(col) + 1 + 4 + 2 + 4 + 2 + 4 + 2 // name + null + tableOID + attrNum + typeOID + typeSize + typeMod + format
}
lengthBytes := make([]byte, 4)
binary.BigEndian.PutUint32(lengthBytes, uint32(length))
msg = append(msg, lengthBytes...)
// Field count
fieldCountBytes := make([]byte, 2)
binary.BigEndian.PutUint16(fieldCountBytes, uint16(len(result.Columns)))
msg = append(msg, fieldCountBytes...)
// Field descriptions
for i, col := range result.Columns {
// Field name
msg = append(msg, []byte(col)...)
msg = append(msg, 0) // null terminator
// Table OID (0 for no table)
tableOID := make([]byte, 4)
binary.BigEndian.PutUint32(tableOID, 0)
msg = append(msg, tableOID...)
// Attribute number
attrNum := make([]byte, 2)
binary.BigEndian.PutUint16(attrNum, uint16(i+1))
msg = append(msg, attrNum...)
// Type OID (determine from schema if available, fallback to data inference)
typeOID := s.getPostgreSQLTypeFromSchema(result, col, i)
typeOIDBytes := make([]byte, 4)
binary.BigEndian.PutUint32(typeOIDBytes, typeOID)
msg = append(msg, typeOIDBytes...)
// Type size (-1 for variable length)
typeSize := make([]byte, 2)
binary.BigEndian.PutUint16(typeSize, 0xFFFF) // -1 as uint16
msg = append(msg, typeSize...)
// Type modifier (-1 for default)
typeMod := make([]byte, 4)
binary.BigEndian.PutUint32(typeMod, 0xFFFFFFFF) // -1 as uint32
msg = append(msg, typeMod...)
// Format (0 for text)
format := make([]byte, 2)
binary.BigEndian.PutUint16(format, 0)
msg = append(msg, format...)
}
_, err := session.writer.Write(msg)
if err == nil {
err = session.writer.Flush()
}
return err
}
// sendDataRow sends a data row message
func (s *PostgreSQLServer) sendDataRow(session *PostgreSQLSession, row []sqltypes.Value) error {
msg := make([]byte, 0)
msg = append(msg, PG_RESP_DATA_ROW)
// Calculate message length
length := 4 + 2 // length + field count
for _, value := range row {
if value.IsNull() {
length += 4 // null value length (-1)
} else {
valueStr := value.ToString()
length += 4 + len(valueStr) // field length + data
}
}
lengthBytes := make([]byte, 4)
binary.BigEndian.PutUint32(lengthBytes, uint32(length))
msg = append(msg, lengthBytes...)
// Field count
fieldCountBytes := make([]byte, 2)
binary.BigEndian.PutUint16(fieldCountBytes, uint16(len(row)))
msg = append(msg, fieldCountBytes...)
// Field values
for _, value := range row {
if value.IsNull() {
// Null value
nullLength := make([]byte, 4)
binary.BigEndian.PutUint32(nullLength, 0xFFFFFFFF) // -1 as uint32
msg = append(msg, nullLength...)
} else {
valueStr := value.ToString()
valueLength := make([]byte, 4)
binary.BigEndian.PutUint32(valueLength, uint32(len(valueStr)))
msg = append(msg, valueLength...)
msg = append(msg, []byte(valueStr)...)
}
}
_, err := session.writer.Write(msg)
if err == nil {
err = session.writer.Flush()
}
return err
}
// sendCommandComplete sends command complete message
func (s *PostgreSQLServer) sendCommandComplete(session *PostgreSQLSession, tag string) error {
msg := make([]byte, 0)
msg = append(msg, PG_RESP_COMMAND)
length := 4 + len(tag) + 1
lengthBytes := make([]byte, 4)
binary.BigEndian.PutUint32(lengthBytes, uint32(length))
msg = append(msg, lengthBytes...)
msg = append(msg, []byte(tag)...)
msg = append(msg, 0) // null terminator
_, err := session.writer.Write(msg)
if err == nil {
err = session.writer.Flush()
}
return err
}
// sendParseComplete sends parse complete message
func (s *PostgreSQLServer) sendParseComplete(session *PostgreSQLSession) error {
msg := make([]byte, 5)
msg[0] = PG_RESP_PARSE_COMPLETE
binary.BigEndian.PutUint32(msg[1:5], 4)
_, err := session.writer.Write(msg)
if err == nil {
err = session.writer.Flush()
}
return err
}
// sendBindComplete sends bind complete message
func (s *PostgreSQLServer) sendBindComplete(session *PostgreSQLSession) error {
msg := make([]byte, 5)
msg[0] = PG_RESP_BIND_COMPLETE
binary.BigEndian.PutUint32(msg[1:5], 4)
_, err := session.writer.Write(msg)
if err == nil {
err = session.writer.Flush()
}
return err
}
// sendCloseComplete sends close complete message
func (s *PostgreSQLServer) sendCloseComplete(session *PostgreSQLSession) error {
msg := make([]byte, 5)
msg[0] = PG_RESP_CLOSE_COMPLETE
binary.BigEndian.PutUint32(msg[1:5], 4)
_, err := session.writer.Write(msg)
if err == nil {
err = session.writer.Flush()
}
return err
}
// sendError sends an error message
func (s *PostgreSQLServer) sendError(session *PostgreSQLSession, code, message string) error {
msg := make([]byte, 0)
msg = append(msg, PG_RESP_ERROR)
// Build error fields
fields := fmt.Sprintf("S%s\x00C%s\x00M%s\x00\x00", "ERROR", code, message)
length := 4 + len(fields)
lengthBytes := make([]byte, 4)
binary.BigEndian.PutUint32(lengthBytes, uint32(length))
msg = append(msg, lengthBytes...)
msg = append(msg, []byte(fields)...)
_, err := session.writer.Write(msg)
if err == nil {
err = session.writer.Flush()
}
return err
}
// getCommandTag generates appropriate command tag for query
func (s *PostgreSQLServer) getCommandTag(query string, rowCount int) string {
queryUpper := strings.ToUpper(strings.TrimSpace(query))
if strings.HasPrefix(queryUpper, "SELECT") {
return fmt.Sprintf("SELECT %d", rowCount)
} else if strings.HasPrefix(queryUpper, "INSERT") {
return fmt.Sprintf("INSERT 0 %d", rowCount)
} else if strings.HasPrefix(queryUpper, "UPDATE") {
return fmt.Sprintf("UPDATE %d", rowCount)
} else if strings.HasPrefix(queryUpper, "DELETE") {
return fmt.Sprintf("DELETE %d", rowCount)
} else if strings.HasPrefix(queryUpper, "SHOW") {
return fmt.Sprintf("SELECT %d", rowCount)
} else if strings.HasPrefix(queryUpper, "DESCRIBE") || strings.HasPrefix(queryUpper, "DESC") {
return fmt.Sprintf("SELECT %d", rowCount)
}
return "SELECT 0"
}
// getPostgreSQLTypeFromSchema determines PostgreSQL type OID from schema information first, fallback to data
func (s *PostgreSQLServer) getPostgreSQLTypeFromSchema(result *engine.QueryResult, columnName string, colIndex int) uint32 {
// Try to get type from schema if database and table are available
if result.Database != "" && result.Table != "" {
if tableInfo, err := s.sqlEngine.GetCatalog().GetTableInfo(result.Database, result.Table); err == nil {
if tableInfo.Schema != nil && tableInfo.Schema.RecordType != nil {
// Look for the field in the schema
for _, field := range tableInfo.Schema.RecordType.Fields {
if field.Name == columnName {
return s.mapSchemaTypeToPostgreSQL(field.Type)
}
}
}
}
}
// Handle system columns
switch columnName {
case "_timestamp_ns":
return PG_TYPE_INT8 // PostgreSQL BIGINT for nanosecond timestamps
case "_key":
return PG_TYPE_BYTEA // PostgreSQL BYTEA for binary keys
case "_source":
return PG_TYPE_TEXT // PostgreSQL TEXT for source information
}
// Fallback to data-based inference if schema is not available
return s.getPostgreSQLTypeFromData(result.Columns, result.Rows, colIndex)
}
// mapSchemaTypeToPostgreSQL maps SeaweedFS schema types to PostgreSQL type OIDs
func (s *PostgreSQLServer) mapSchemaTypeToPostgreSQL(fieldType *schema_pb.Type) uint32 {
if fieldType == nil {
return PG_TYPE_TEXT
}
switch kind := fieldType.Kind.(type) {
case *schema_pb.Type_ScalarType:
switch kind.ScalarType {
case schema_pb.ScalarType_BOOL:
return PG_TYPE_BOOL
case schema_pb.ScalarType_INT32:
return PG_TYPE_INT4
case schema_pb.ScalarType_INT64:
return PG_TYPE_INT8
case schema_pb.ScalarType_FLOAT:
return PG_TYPE_FLOAT4
case schema_pb.ScalarType_DOUBLE:
return PG_TYPE_FLOAT8
case schema_pb.ScalarType_BYTES:
return PG_TYPE_BYTEA
case schema_pb.ScalarType_STRING:
return PG_TYPE_TEXT
default:
return PG_TYPE_TEXT
}
case *schema_pb.Type_ListType:
// For list types, we'll represent them as JSON text
return PG_TYPE_JSONB
case *schema_pb.Type_RecordType:
// For nested record types, we'll represent them as JSON text
return PG_TYPE_JSONB
default:
return PG_TYPE_TEXT
}
}
// getPostgreSQLTypeFromData determines PostgreSQL type OID from data (legacy fallback method)
func (s *PostgreSQLServer) getPostgreSQLTypeFromData(columns []string, rows [][]sqltypes.Value, colIndex int) uint32 {
if len(rows) == 0 || colIndex >= len(rows[0]) {
return PG_TYPE_TEXT // Default to text
}
// Sample first non-null value to determine type
for _, row := range rows {
if colIndex < len(row) && !row[colIndex].IsNull() {
value := row[colIndex]
switch value.Type() {
case sqltypes.Int8, sqltypes.Int16, sqltypes.Int32:
return PG_TYPE_INT4
case sqltypes.Int64:
return PG_TYPE_INT8
case sqltypes.Float32, sqltypes.Float64:
return PG_TYPE_FLOAT8
case sqltypes.Bit:
return PG_TYPE_BOOL
case sqltypes.Timestamp, sqltypes.Datetime:
return PG_TYPE_TIMESTAMP
default:
// Try to infer from string content
valueStr := value.ToString()
if _, err := strconv.ParseInt(valueStr, 10, 32); err == nil {
return PG_TYPE_INT4
}
if _, err := strconv.ParseInt(valueStr, 10, 64); err == nil {
return PG_TYPE_INT8
}
if _, err := strconv.ParseFloat(valueStr, 64); err == nil {
return PG_TYPE_FLOAT8
}
if valueStr == "true" || valueStr == "false" {
return PG_TYPE_BOOL
}
return PG_TYPE_TEXT
}
}
}
return PG_TYPE_TEXT // Default to text
}

View File

@@ -0,0 +1,726 @@
package postgres
import (
"bufio"
"crypto/md5"
"crypto/rand"
"crypto/tls"
"encoding/binary"
"fmt"
"io"
"net"
"strings"
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/query/engine"
"github.com/seaweedfs/seaweedfs/weed/util/version"
)
// PostgreSQL protocol constants
const (
// Protocol versions
PG_PROTOCOL_VERSION_3 = 196608 // PostgreSQL 3.0 protocol (0x00030000)
PG_SSL_REQUEST = 80877103 // SSL request (0x04d2162f)
PG_GSSAPI_REQUEST = 80877104 // GSSAPI request (0x04d21630)
// Message types from client
PG_MSG_STARTUP = 0x00
PG_MSG_QUERY = 'Q'
PG_MSG_PARSE = 'P'
PG_MSG_BIND = 'B'
PG_MSG_EXECUTE = 'E'
PG_MSG_DESCRIBE = 'D'
PG_MSG_CLOSE = 'C'
PG_MSG_FLUSH = 'H'
PG_MSG_SYNC = 'S'
PG_MSG_TERMINATE = 'X'
PG_MSG_PASSWORD = 'p'
// Response types to client
PG_RESP_AUTH_OK = 'R'
PG_RESP_BACKEND_KEY = 'K'
PG_RESP_PARAMETER = 'S'
PG_RESP_READY = 'Z'
PG_RESP_COMMAND = 'C'
PG_RESP_DATA_ROW = 'D'
PG_RESP_ROW_DESC = 'T'
PG_RESP_PARSE_COMPLETE = '1'
PG_RESP_BIND_COMPLETE = '2'
PG_RESP_CLOSE_COMPLETE = '3'
PG_RESP_ERROR = 'E'
PG_RESP_NOTICE = 'N'
// Transaction states
PG_TRANS_IDLE = 'I'
PG_TRANS_INTRANS = 'T'
PG_TRANS_ERROR = 'E'
// Authentication methods
AUTH_OK = 0
AUTH_CLEAR = 3
AUTH_MD5 = 5
AUTH_TRUST = 10
// PostgreSQL data types
PG_TYPE_BOOL = 16
PG_TYPE_BYTEA = 17
PG_TYPE_INT8 = 20
PG_TYPE_INT4 = 23
PG_TYPE_TEXT = 25
PG_TYPE_FLOAT4 = 700
PG_TYPE_FLOAT8 = 701
PG_TYPE_VARCHAR = 1043
PG_TYPE_TIMESTAMP = 1114
PG_TYPE_JSON = 114
PG_TYPE_JSONB = 3802
// Default values
DEFAULT_POSTGRES_PORT = 5432
)
// Authentication method type
type AuthMethod int
const (
AuthTrust AuthMethod = iota
AuthPassword
AuthMD5
)
// PostgreSQL server configuration
type PostgreSQLServerConfig struct {
Host string
Port int
AuthMethod AuthMethod
Users map[string]string
TLSConfig *tls.Config
MaxConns int
IdleTimeout time.Duration
StartupTimeout time.Duration // Timeout for client startup handshake
Database string
}
// PostgreSQL server
type PostgreSQLServer struct {
config *PostgreSQLServerConfig
listener net.Listener
sqlEngine *engine.SQLEngine
sessions map[uint32]*PostgreSQLSession
sessionMux sync.RWMutex
shutdown chan struct{}
wg sync.WaitGroup
nextConnID uint32
}
// PostgreSQL session
type PostgreSQLSession struct {
conn net.Conn
reader *bufio.Reader
writer *bufio.Writer
authenticated bool
username string
database string
parameters map[string]string
preparedStmts map[string]*PreparedStatement
portals map[string]*Portal
transactionState byte
processID uint32
secretKey uint32
created time.Time
lastActivity time.Time
mutex sync.Mutex
}
// Prepared statement
type PreparedStatement struct {
Name string
Query string
ParamTypes []uint32
Fields []FieldDescription
}
// Portal (cursor)
type Portal struct {
Name string
Statement string
Parameters [][]byte
Suspended bool
}
// Field description
type FieldDescription struct {
Name string
TableOID uint32
AttrNum int16
TypeOID uint32
TypeSize int16
TypeMod int32
Format int16
}
// NewPostgreSQLServer creates a new PostgreSQL protocol server
func NewPostgreSQLServer(config *PostgreSQLServerConfig, masterAddr string) (*PostgreSQLServer, error) {
if config.Port <= 0 {
config.Port = DEFAULT_POSTGRES_PORT
}
if config.Host == "" {
config.Host = "localhost"
}
if config.Database == "" {
config.Database = "default"
}
if config.MaxConns <= 0 {
config.MaxConns = 100
}
if config.IdleTimeout <= 0 {
config.IdleTimeout = time.Hour
}
if config.StartupTimeout <= 0 {
config.StartupTimeout = 30 * time.Second
}
// Create SQL engine (now uses CockroachDB parser for PostgreSQL compatibility)
sqlEngine := engine.NewSQLEngine(masterAddr)
server := &PostgreSQLServer{
config: config,
sqlEngine: sqlEngine,
sessions: make(map[uint32]*PostgreSQLSession),
shutdown: make(chan struct{}),
nextConnID: 1,
}
return server, nil
}
// Start begins listening for PostgreSQL connections
func (s *PostgreSQLServer) Start() error {
addr := fmt.Sprintf("%s:%d", s.config.Host, s.config.Port)
var listener net.Listener
var err error
if s.config.TLSConfig != nil {
listener, err = tls.Listen("tcp", addr, s.config.TLSConfig)
glog.Infof("PostgreSQL Server with TLS listening on %s", addr)
} else {
listener, err = net.Listen("tcp", addr)
glog.Infof("PostgreSQL Server listening on %s", addr)
}
if err != nil {
return fmt.Errorf("failed to start PostgreSQL server on %s: %v", addr, err)
}
s.listener = listener
// Start accepting connections
s.wg.Add(1)
go s.acceptConnections()
// Start cleanup routine
s.wg.Add(1)
go s.cleanupSessions()
return nil
}
// Stop gracefully shuts down the PostgreSQL server
func (s *PostgreSQLServer) Stop() error {
close(s.shutdown)
if s.listener != nil {
s.listener.Close()
}
// Close all sessions
s.sessionMux.Lock()
for _, session := range s.sessions {
session.close()
}
s.sessions = make(map[uint32]*PostgreSQLSession)
s.sessionMux.Unlock()
s.wg.Wait()
glog.Infof("PostgreSQL Server stopped")
return nil
}
// acceptConnections handles incoming PostgreSQL connections
func (s *PostgreSQLServer) acceptConnections() {
defer s.wg.Done()
for {
select {
case <-s.shutdown:
return
default:
}
conn, err := s.listener.Accept()
if err != nil {
select {
case <-s.shutdown:
return
default:
glog.Errorf("Failed to accept PostgreSQL connection: %v", err)
continue
}
}
// Check connection limit
s.sessionMux.RLock()
sessionCount := len(s.sessions)
s.sessionMux.RUnlock()
if sessionCount >= s.config.MaxConns {
glog.Warningf("Maximum connections reached (%d), rejecting connection from %s",
s.config.MaxConns, conn.RemoteAddr())
conn.Close()
continue
}
s.wg.Add(1)
go s.handleConnection(conn)
}
}
// handleConnection processes a single PostgreSQL connection
func (s *PostgreSQLServer) handleConnection(conn net.Conn) {
defer s.wg.Done()
defer conn.Close()
// Generate unique connection ID
connID := s.generateConnectionID()
secretKey, err := s.generateSecretKey()
if err != nil {
glog.Errorf("handleConnection: %v", err)
return
}
// Create session
session := &PostgreSQLSession{
conn: conn,
reader: bufio.NewReader(conn),
writer: bufio.NewWriter(conn),
authenticated: false,
database: s.config.Database,
parameters: make(map[string]string),
preparedStmts: make(map[string]*PreparedStatement),
portals: make(map[string]*Portal),
transactionState: PG_TRANS_IDLE,
processID: connID,
secretKey: secretKey,
created: time.Now(),
lastActivity: time.Now(),
}
// Register session
s.sessionMux.Lock()
s.sessions[connID] = session
s.sessionMux.Unlock()
// Clean up on exit
defer func() {
s.sessionMux.Lock()
delete(s.sessions, connID)
s.sessionMux.Unlock()
}()
glog.V(2).Infof("New PostgreSQL connection from %s (ID: %d)", conn.RemoteAddr(), connID)
// Handle startup
err = s.handleStartup(session)
if err != nil {
// Handle common disconnection scenarios more gracefully
if strings.Contains(err.Error(), "client disconnected") {
glog.V(1).Infof("Client startup disconnected from %s (ID: %d): %v", conn.RemoteAddr(), connID, err)
} else if strings.Contains(err.Error(), "timeout") {
glog.Warningf("Startup timeout for connection %d from %s: %v", connID, conn.RemoteAddr(), err)
} else {
glog.Errorf("Startup failed for connection %d from %s: %v", connID, conn.RemoteAddr(), err)
}
return
}
// Handle messages
for {
select {
case <-s.shutdown:
return
default:
}
// Set read timeout
conn.SetReadDeadline(time.Now().Add(30 * time.Second))
err := s.handleMessage(session)
if err != nil {
if err == io.EOF {
glog.Infof("PostgreSQL client disconnected (ID: %d)", connID)
} else {
glog.Errorf("Error handling PostgreSQL message (ID: %d): %v", connID, err)
}
return
}
session.lastActivity = time.Now()
}
}
// handleStartup processes the PostgreSQL startup sequence
func (s *PostgreSQLServer) handleStartup(session *PostgreSQLSession) error {
// Set a startup timeout to prevent hanging connections
startupTimeout := s.config.StartupTimeout
session.conn.SetReadDeadline(time.Now().Add(startupTimeout))
defer session.conn.SetReadDeadline(time.Time{}) // Clear timeout
for {
// Read startup message length
length := make([]byte, 4)
_, err := io.ReadFull(session.reader, length)
if err != nil {
if err == io.EOF {
// Client disconnected during startup - this is common for health checks
return fmt.Errorf("client disconnected during startup handshake")
}
if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
return fmt.Errorf("startup handshake timeout after %v", startupTimeout)
}
return fmt.Errorf("failed to read message length during startup: %v", err)
}
msgTotalLen := binary.BigEndian.Uint32(length)
// Prevent unsigned underflow and OOM by checking total message length first
if msgTotalLen < 8 {
return fmt.Errorf("startup message too short: %d bytes", msgTotalLen)
}
msgLength := msgTotalLen - 4
if msgLength > maxStartupMessageSize {
return fmt.Errorf("startup message too large: %d bytes", msgLength)
}
// Read startup message content
msg := make([]byte, msgLength)
_, err = io.ReadFull(session.reader, msg)
if err != nil {
if err == io.EOF {
return fmt.Errorf("client disconnected while reading startup message")
}
if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
return fmt.Errorf("startup message read timeout")
}
return fmt.Errorf("failed to read startup message: %v", err)
}
// Parse protocol version (msgTotalLen >= 8 above guarantees msgLength >= 4)
protocolVersion := binary.BigEndian.Uint32(msg[0:4])
switch protocolVersion {
case PG_SSL_REQUEST:
// Reject SSL request - send 'N' to indicate SSL not supported
_, err = session.conn.Write([]byte{'N'})
if err != nil {
return fmt.Errorf("failed to reject SSL request: %v", err)
}
// Continue loop to read the actual startup message
continue
case PG_GSSAPI_REQUEST:
// Reject GSSAPI request - send 'N' to indicate GSSAPI not supported
_, err = session.conn.Write([]byte{'N'})
if err != nil {
return fmt.Errorf("failed to reject GSSAPI request: %v", err)
}
// Continue loop to read the actual startup message
continue
case PG_PROTOCOL_VERSION_3:
// This is the actual startup message, break out of loop
break
default:
return fmt.Errorf("unsupported protocol version: %d", protocolVersion)
}
// Parse parameters
params := strings.Split(string(msg[4:]), "\x00")
for i := 0; i < len(params)-1; i += 2 {
if params[i] == "user" {
session.username = params[i+1]
} else if params[i] == "database" {
session.database = params[i+1]
}
session.parameters[params[i]] = params[i+1]
}
// Break out of the main loop - we have the startup message
break
}
// Handle authentication
err := s.handleAuthentication(session)
if err != nil {
return err
}
// Send parameter status messages
err = s.sendParameterStatus(session, "server_version", fmt.Sprintf("%s (SeaweedFS)", version.VERSION_NUMBER))
if err != nil {
return err
}
err = s.sendParameterStatus(session, "server_encoding", "UTF8")
if err != nil {
return err
}
err = s.sendParameterStatus(session, "client_encoding", "UTF8")
if err != nil {
return err
}
err = s.sendParameterStatus(session, "DateStyle", "ISO, MDY")
if err != nil {
return err
}
err = s.sendParameterStatus(session, "integer_datetimes", "on")
if err != nil {
return err
}
// Send backend key data
err = s.sendBackendKeyData(session)
if err != nil {
return err
}
// Send ready for query
err = s.sendReadyForQuery(session)
if err != nil {
return err
}
session.authenticated = true
return nil
}
// handleAuthentication processes authentication
func (s *PostgreSQLServer) handleAuthentication(session *PostgreSQLSession) error {
switch s.config.AuthMethod {
case AuthTrust:
return s.sendAuthenticationOk(session)
case AuthPassword:
return s.handlePasswordAuth(session)
case AuthMD5:
return s.handleMD5Auth(session)
default:
return fmt.Errorf("unsupported authentication method")
}
}
// sendAuthenticationOk sends authentication OK message
func (s *PostgreSQLServer) sendAuthenticationOk(session *PostgreSQLSession) error {
msg := make([]byte, 9)
msg[0] = PG_RESP_AUTH_OK
binary.BigEndian.PutUint32(msg[1:5], 8)
binary.BigEndian.PutUint32(msg[5:9], AUTH_OK)
_, err := session.writer.Write(msg)
if err == nil {
err = session.writer.Flush()
}
return err
}
// handlePasswordAuth handles clear password authentication
func (s *PostgreSQLServer) handlePasswordAuth(session *PostgreSQLSession) error {
// Send password request
msg := make([]byte, 9)
msg[0] = PG_RESP_AUTH_OK
binary.BigEndian.PutUint32(msg[1:5], 8)
binary.BigEndian.PutUint32(msg[5:9], AUTH_CLEAR)
_, err := session.writer.Write(msg)
if err != nil {
return err
}
err = session.writer.Flush()
if err != nil {
return err
}
// Read password response
msgType := make([]byte, 1)
_, err = io.ReadFull(session.reader, msgType)
if err != nil {
return err
}
if msgType[0] != PG_MSG_PASSWORD {
return fmt.Errorf("expected password message, got %c", msgType[0])
}
length := make([]byte, 4)
_, err = io.ReadFull(session.reader, length)
if err != nil {
return err
}
rawLength := binary.BigEndian.Uint32(length)
if rawLength < 4 {
return fmt.Errorf("password message too short: %d bytes", rawLength)
}
msgLength := rawLength - 4
if msgLength > maxAuthMessageSize {
return fmt.Errorf("password message too large: %d bytes", msgLength)
}
password := make([]byte, msgLength)
_, err = io.ReadFull(session.reader, password)
if err != nil {
return err
}
// Verify password (safe null-terminator removal)
expectedPassword, exists := s.config.Users[session.username]
passwordStr := strings.TrimSuffix(string(password), "\x00")
if !exists || passwordStr != expectedPassword {
return s.sendError(session, "28P01", "authentication failed for user \""+session.username+"\"")
}
return s.sendAuthenticationOk(session)
}
// handleMD5Auth handles MD5 password authentication
func (s *PostgreSQLServer) handleMD5Auth(session *PostgreSQLSession) error {
// Generate salt
salt := make([]byte, 4)
_, err := rand.Read(salt)
if err != nil {
return err
}
// Send MD5 request
msg := make([]byte, 13)
msg[0] = PG_RESP_AUTH_OK
binary.BigEndian.PutUint32(msg[1:5], 12)
binary.BigEndian.PutUint32(msg[5:9], AUTH_MD5)
copy(msg[9:13], salt)
_, err = session.writer.Write(msg)
if err != nil {
return err
}
err = session.writer.Flush()
if err != nil {
return err
}
// Read password response
msgType := make([]byte, 1)
_, err = io.ReadFull(session.reader, msgType)
if err != nil {
return err
}
if msgType[0] != PG_MSG_PASSWORD {
return fmt.Errorf("expected password message, got %c", msgType[0])
}
length := make([]byte, 4)
_, err = io.ReadFull(session.reader, length)
if err != nil {
return err
}
rawLength := binary.BigEndian.Uint32(length)
if rawLength < 4 {
return fmt.Errorf("MD5 response too short: %d bytes", rawLength)
}
msgLength := rawLength - 4
if msgLength > maxAuthMessageSize {
return fmt.Errorf("MD5 response too large: %d bytes", msgLength)
}
response := make([]byte, msgLength)
_, err = io.ReadFull(session.reader, response)
if err != nil {
return err
}
// Verify MD5 hash
expectedPassword, exists := s.config.Users[session.username]
if !exists {
return s.sendError(session, "28P01", "authentication failed for user \""+session.username+"\"")
}
// Calculate expected hash: md5(md5(password + username) + salt)
inner := md5.Sum([]byte(expectedPassword + session.username))
expected := fmt.Sprintf("md5%x", md5.Sum(append([]byte(fmt.Sprintf("%x", inner)), salt...)))
if strings.TrimSuffix(string(response), "\x00") != expected {
return s.sendError(session, "28P01", "authentication failed for user \""+session.username+"\"")
}
return s.sendAuthenticationOk(session)
}
// generateConnectionID generates a unique connection ID
func (s *PostgreSQLServer) generateConnectionID() uint32 {
s.sessionMux.Lock()
defer s.sessionMux.Unlock()
id := s.nextConnID
s.nextConnID++
return id
}
// generateSecretKey generates a secret key for the connection
func (s *PostgreSQLServer) generateSecretKey() (uint32, error) {
key := make([]byte, 4)
if _, err := rand.Read(key); err != nil {
return 0, fmt.Errorf("generate secret key: %w", err)
}
return binary.BigEndian.Uint32(key), nil
}
// close marks the session as closed
func (s *PostgreSQLSession) close() {
s.mutex.Lock()
defer s.mutex.Unlock()
if s.conn != nil {
s.conn.Close()
s.conn = nil
}
}
// cleanupSessions periodically cleans up idle sessions
func (s *PostgreSQLServer) cleanupSessions() {
defer s.wg.Done()
ticker := time.NewTicker(time.Minute)
defer ticker.Stop()
for {
select {
case <-s.shutdown:
return
case <-ticker.C:
s.cleanupIdleSessions()
}
}
}
// cleanupIdleSessions removes sessions that have been idle too long
func (s *PostgreSQLServer) cleanupIdleSessions() {
now := time.Now()
s.sessionMux.Lock()
defer s.sessionMux.Unlock()
for id, session := range s.sessions {
if now.Sub(session.lastActivity) > s.config.IdleTimeout {
glog.Infof("Closing idle PostgreSQL session %d", id)
session.close()
delete(s.sessions, id)
}
}
}