snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
389
weed/server/postgres/DESIGN.md
Normal file
389
weed/server/postgres/DESIGN.md
Normal file
@@ -0,0 +1,389 @@
|
||||
# PostgreSQL Wire Protocol Support for SeaweedFS
|
||||
|
||||
## Overview
|
||||
|
||||
This design adds native PostgreSQL wire protocol support to SeaweedFS, enabling compatibility with all PostgreSQL clients, tools, and drivers without requiring custom implementations.
|
||||
|
||||
## Benefits
|
||||
|
||||
### Universal Compatibility
|
||||
- **Standard PostgreSQL Clients**: psql, pgAdmin, Adminer, etc.
|
||||
- **JDBC/ODBC Drivers**: Use standard PostgreSQL drivers
|
||||
- **BI Tools**: Tableau, Power BI, Grafana, Superset with native PostgreSQL connectors
|
||||
- **ORMs**: Hibernate, ActiveRecord, Django ORM, etc.
|
||||
- **Programming Languages**: Native PostgreSQL libraries in Python (psycopg2), Node.js (pg), Go (lib/pq), etc.
|
||||
|
||||
### Enterprise Integration
|
||||
- **Existing Infrastructure**: Drop-in replacement for PostgreSQL in read-only scenarios
|
||||
- **Migration Path**: Easy transition from PostgreSQL-based analytics
|
||||
- **Tool Ecosystem**: Leverage entire PostgreSQL ecosystem
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐
|
||||
│ PostgreSQL │ │ PostgreSQL │ │ SeaweedFS │
|
||||
│ Clients │◄──►│ Protocol │◄──►│ SQL Engine │
|
||||
│ (psql, etc.) │ │ Server │ │ │
|
||||
└─────────────────┘ └──────────────────┘ └─────────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────┐
|
||||
│ Authentication │
|
||||
│ & Session Mgmt │
|
||||
└──────────────────┘
|
||||
```
|
||||
|
||||
## Core Components
|
||||
|
||||
### 1. PostgreSQL Wire Protocol Handler
|
||||
|
||||
```go
|
||||
// PostgreSQL message types
|
||||
const (
|
||||
PG_MSG_STARTUP = 0x00 // Startup message
|
||||
PG_MSG_QUERY = 'Q' // Simple query
|
||||
PG_MSG_PARSE = 'P' // Parse (prepared statement)
|
||||
PG_MSG_BIND = 'B' // Bind parameters
|
||||
PG_MSG_EXECUTE = 'E' // Execute prepared statement
|
||||
PG_MSG_DESCRIBE = 'D' // Describe statement/portal
|
||||
PG_MSG_CLOSE = 'C' // Close statement/portal
|
||||
PG_MSG_FLUSH = 'H' // Flush
|
||||
PG_MSG_SYNC = 'S' // Sync
|
||||
PG_MSG_TERMINATE = 'X' // Terminate connection
|
||||
PG_MSG_PASSWORD = 'p' // Password message
|
||||
)
|
||||
|
||||
// PostgreSQL response types
|
||||
const (
|
||||
PG_RESP_AUTH_OK = 'R' // Authentication OK
|
||||
PG_RESP_AUTH_REQ = 'R' // Authentication request
|
||||
PG_RESP_BACKEND_KEY = 'K' // Backend key data
|
||||
PG_RESP_PARAMETER = 'S' // Parameter status
|
||||
PG_RESP_READY = 'Z' // Ready for query
|
||||
PG_RESP_COMMAND = 'C' // Command complete
|
||||
PG_RESP_DATA_ROW = 'D' // Data row
|
||||
PG_RESP_ROW_DESC = 'T' // Row description
|
||||
PG_RESP_PARSE_COMPLETE = '1' // Parse complete
|
||||
PG_RESP_BIND_COMPLETE = '2' // Bind complete
|
||||
PG_RESP_CLOSE_COMPLETE = '3' // Close complete
|
||||
PG_RESP_ERROR = 'E' // Error response
|
||||
PG_RESP_NOTICE = 'N' // Notice response
|
||||
)
|
||||
```
|
||||
|
||||
### 2. Session Management
|
||||
|
||||
```go
|
||||
type PostgreSQLSession struct {
|
||||
conn net.Conn
|
||||
reader *bufio.Reader
|
||||
writer *bufio.Writer
|
||||
authenticated bool
|
||||
username string
|
||||
database string
|
||||
parameters map[string]string
|
||||
preparedStmts map[string]*PreparedStatement
|
||||
portals map[string]*Portal
|
||||
transactionState TransactionState
|
||||
processID uint32
|
||||
secretKey uint32
|
||||
}
|
||||
|
||||
type PreparedStatement struct {
|
||||
name string
|
||||
query string
|
||||
paramTypes []uint32
|
||||
fields []FieldDescription
|
||||
}
|
||||
|
||||
type Portal struct {
|
||||
name string
|
||||
statement string
|
||||
parameters [][]byte
|
||||
suspended bool
|
||||
}
|
||||
```
|
||||
|
||||
### 3. SQL Translation Layer
|
||||
|
||||
```go
|
||||
type PostgreSQLTranslator struct {
|
||||
dialectMap map[string]string
|
||||
}
|
||||
|
||||
// Translates PostgreSQL-specific SQL to SeaweedFS SQL
|
||||
func (t *PostgreSQLTranslator) TranslateQuery(pgSQL string) (string, error) {
|
||||
// Handle PostgreSQL-specific syntax:
|
||||
// - SELECT version() -> SELECT 'SeaweedFS 1.0'
|
||||
// - SELECT current_database() -> SELECT 'default'
|
||||
// - SELECT current_user -> SELECT 'seaweedfs'
|
||||
// - \d commands -> SHOW TABLES/DESCRIBE equivalents
|
||||
// - PostgreSQL system catalogs -> SeaweedFS equivalents
|
||||
}
|
||||
```
|
||||
|
||||
### 4. Data Type Mapping
|
||||
|
||||
```go
|
||||
var PostgreSQLTypeMap = map[string]uint32{
|
||||
"TEXT": 25, // PostgreSQL TEXT type
|
||||
"VARCHAR": 1043, // PostgreSQL VARCHAR type
|
||||
"INTEGER": 23, // PostgreSQL INTEGER type
|
||||
"BIGINT": 20, // PostgreSQL BIGINT type
|
||||
"FLOAT": 701, // PostgreSQL FLOAT8 type
|
||||
"BOOLEAN": 16, // PostgreSQL BOOLEAN type
|
||||
"TIMESTAMP": 1114, // PostgreSQL TIMESTAMP type
|
||||
"JSON": 114, // PostgreSQL JSON type
|
||||
}
|
||||
|
||||
func SeaweedToPostgreSQLType(seaweedType string) uint32 {
|
||||
if pgType, exists := PostgreSQLTypeMap[strings.ToUpper(seaweedType)]; exists {
|
||||
return pgType
|
||||
}
|
||||
return 25 // Default to TEXT
|
||||
}
|
||||
```
|
||||
|
||||
## Protocol Implementation
|
||||
|
||||
### 1. Connection Flow
|
||||
|
||||
```
|
||||
Client Server
|
||||
│ │
|
||||
├─ StartupMessage ────────────►│
|
||||
│ ├─ AuthenticationOk
|
||||
│ ├─ ParameterStatus (multiple)
|
||||
│ ├─ BackendKeyData
|
||||
│ └─ ReadyForQuery
|
||||
│ │
|
||||
├─ Query('SELECT 1') ─────────►│
|
||||
│ ├─ RowDescription
|
||||
│ ├─ DataRow
|
||||
│ ├─ CommandComplete
|
||||
│ └─ ReadyForQuery
|
||||
│ │
|
||||
├─ Parse('stmt1', 'SELECT $1')►│
|
||||
│ └─ ParseComplete
|
||||
├─ Bind('portal1', 'stmt1')───►│
|
||||
│ └─ BindComplete
|
||||
├─ Execute('portal1')─────────►│
|
||||
│ ├─ DataRow (multiple)
|
||||
│ └─ CommandComplete
|
||||
├─ Sync ──────────────────────►│
|
||||
│ └─ ReadyForQuery
|
||||
│ │
|
||||
├─ Terminate ─────────────────►│
|
||||
│ └─ [Connection closed]
|
||||
```
|
||||
|
||||
### 2. Authentication
|
||||
|
||||
```go
|
||||
type AuthMethod int
|
||||
|
||||
const (
|
||||
AuthTrust AuthMethod = iota
|
||||
AuthPassword
|
||||
AuthMD5
|
||||
AuthSASL
|
||||
)
|
||||
|
||||
func (s *PostgreSQLServer) handleAuthentication(session *PostgreSQLSession) error {
|
||||
switch s.authMethod {
|
||||
case AuthTrust:
|
||||
return s.sendAuthenticationOk(session)
|
||||
case AuthPassword:
|
||||
return s.handlePasswordAuth(session)
|
||||
case AuthMD5:
|
||||
return s.handleMD5Auth(session)
|
||||
default:
|
||||
return fmt.Errorf("unsupported auth method")
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Query Processing
|
||||
|
||||
```go
|
||||
func (s *PostgreSQLServer) handleSimpleQuery(session *PostgreSQLSession, query string) error {
|
||||
// 1. Translate PostgreSQL SQL to SeaweedFS SQL
|
||||
translatedQuery, err := s.translator.TranslateQuery(query)
|
||||
if err != nil {
|
||||
return s.sendError(session, err)
|
||||
}
|
||||
|
||||
// 2. Execute using existing SQL engine
|
||||
result, err := s.sqlEngine.ExecuteSQL(context.Background(), translatedQuery)
|
||||
if err != nil {
|
||||
return s.sendError(session, err)
|
||||
}
|
||||
|
||||
// 3. Send results in PostgreSQL format
|
||||
err = s.sendRowDescription(session, result.Columns)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, row := range result.Rows {
|
||||
err = s.sendDataRow(session, row)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return s.sendCommandComplete(session, fmt.Sprintf("SELECT %d", len(result.Rows)))
|
||||
}
|
||||
```
|
||||
|
||||
## System Catalogs Support
|
||||
|
||||
PostgreSQL clients expect certain system catalogs. We'll implement views for key ones:
|
||||
|
||||
```sql
|
||||
-- pg_tables equivalent
|
||||
SELECT
|
||||
'default' as schemaname,
|
||||
table_name as tablename,
|
||||
'seaweedfs' as tableowner,
|
||||
NULL as tablespace,
|
||||
false as hasindexes,
|
||||
false as hasrules,
|
||||
false as hastriggers
|
||||
FROM information_schema.tables;
|
||||
|
||||
-- pg_database equivalent
|
||||
SELECT
|
||||
database_name as datname,
|
||||
'seaweedfs' as datdba,
|
||||
'UTF8' as encoding,
|
||||
'C' as datcollate,
|
||||
'C' as datctype
|
||||
FROM information_schema.schemata;
|
||||
|
||||
-- pg_version equivalent
|
||||
SELECT 'SeaweedFS 1.0 (PostgreSQL 14.0 compatible)' as version;
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
### Server Configuration
|
||||
```go
|
||||
type PostgreSQLServerConfig struct {
|
||||
Host string
|
||||
Port int
|
||||
Database string
|
||||
AuthMethod AuthMethod
|
||||
Users map[string]string // username -> password
|
||||
TLSConfig *tls.Config
|
||||
MaxConns int
|
||||
IdleTimeout time.Duration
|
||||
}
|
||||
```
|
||||
|
||||
### Client Connection String
|
||||
```bash
|
||||
# Standard PostgreSQL connection strings work
|
||||
psql "host=localhost port=5432 dbname=default user=seaweedfs"
|
||||
PGPASSWORD=secret psql -h localhost -p 5432 -U seaweedfs -d default
|
||||
|
||||
# JDBC URL
|
||||
jdbc:postgresql://localhost:5432/default?user=seaweedfs&password=secret
|
||||
```
|
||||
|
||||
## Command Line Interface
|
||||
|
||||
```bash
|
||||
# Start PostgreSQL protocol server
|
||||
weed-db -port=5432 -auth=trust
|
||||
weed-db -port=5432 -auth=password -users="admin:secret;readonly:pass"
|
||||
weed-db -port=5432 -tls-cert=server.crt -tls-key=server.key
|
||||
|
||||
# Configuration options
|
||||
-host=localhost # Listen host
|
||||
-port=5432 # PostgreSQL standard port
|
||||
-auth=trust|password|md5 # Authentication method
|
||||
-users=user:pass;user2:pass2 # User credentials (password/md5 auth) - use semicolons to separate users
|
||||
-database=default # Default database name
|
||||
-max-connections=100 # Maximum concurrent connections
|
||||
-idle-timeout=1h # Connection idle timeout
|
||||
-tls-cert="" # TLS certificate file
|
||||
-tls-key="" # TLS private key file
|
||||
```
|
||||
|
||||
## Client Compatibility Testing
|
||||
|
||||
### Essential Clients
|
||||
- **psql**: PostgreSQL command line client
|
||||
- **pgAdmin**: Web-based administration tool
|
||||
- **DBeaver**: Universal database tool
|
||||
- **DataGrip**: JetBrains database IDE
|
||||
|
||||
### Programming Language Drivers
|
||||
- **Python**: psycopg2, asyncpg
|
||||
- **Java**: PostgreSQL JDBC driver
|
||||
- **Node.js**: pg, node-postgres
|
||||
- **Go**: lib/pq, pgx
|
||||
- **.NET**: Npgsql
|
||||
|
||||
### BI Tools
|
||||
- **Grafana**: PostgreSQL data source
|
||||
- **Superset**: PostgreSQL connector
|
||||
- **Tableau**: PostgreSQL native connector
|
||||
- **Power BI**: PostgreSQL connector
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
1. **Phase 1**: Basic wire protocol and simple queries
|
||||
2. **Phase 2**: Extended query protocol (prepared statements)
|
||||
3. **Phase 3**: System catalog views
|
||||
4. **Phase 4**: Advanced features (transactions, notifications)
|
||||
5. **Phase 5**: Performance optimization and caching
|
||||
|
||||
## Limitations
|
||||
|
||||
### Read-Only Access
|
||||
- INSERT/UPDATE/DELETE operations not supported
|
||||
- Returns appropriate error messages for write operations
|
||||
|
||||
### Partial SQL Compatibility
|
||||
- Subset of PostgreSQL SQL features
|
||||
- SeaweedFS-specific limitations apply
|
||||
|
||||
### System Features
|
||||
- No stored procedures/functions
|
||||
- No triggers or constraints
|
||||
- No user-defined types
|
||||
- Limited transaction support (mostly no-op)
|
||||
|
||||
## Security Considerations
|
||||
|
||||
### Authentication
|
||||
- Support for trust, password, and MD5 authentication
|
||||
- TLS encryption support
|
||||
- User access control
|
||||
|
||||
### SQL Injection Prevention
|
||||
- Prepared statements with parameter binding
|
||||
- Input validation and sanitization
|
||||
- Query complexity limits
|
||||
|
||||
## Performance Optimizations
|
||||
|
||||
### Connection Pooling
|
||||
- Configurable maximum connections
|
||||
- Connection reuse and idle timeout
|
||||
- Memory efficient session management
|
||||
|
||||
### Query Caching
|
||||
- Prepared statement caching
|
||||
- Result set caching for repeated queries
|
||||
- Metadata caching
|
||||
|
||||
### Protocol Efficiency
|
||||
- Binary result format support
|
||||
- Batch query processing
|
||||
- Streaming large result sets
|
||||
|
||||
This design provides a comprehensive PostgreSQL wire protocol implementation that makes SeaweedFS accessible to the entire PostgreSQL ecosystem while maintaining compatibility and performance.
|
||||
284
weed/server/postgres/README.md
Normal file
284
weed/server/postgres/README.md
Normal file
@@ -0,0 +1,284 @@
|
||||
# PostgreSQL Wire Protocol Package
|
||||
|
||||
This package implements PostgreSQL wire protocol support for SeaweedFS, enabling universal compatibility with PostgreSQL clients, tools, and applications.
|
||||
|
||||
## Package Structure
|
||||
|
||||
```
|
||||
weed/server/postgres/
|
||||
├── README.md # This documentation
|
||||
├── server.go # Main PostgreSQL server implementation
|
||||
├── protocol.go # Wire protocol message handlers with MQ integration
|
||||
├── DESIGN.md # Architecture and design documentation
|
||||
└── IMPLEMENTATION.md # Complete implementation guide
|
||||
```
|
||||
|
||||
## Core Components
|
||||
|
||||
### `server.go`
|
||||
- **PostgreSQLServer**: Main server structure with connection management
|
||||
- **PostgreSQLSession**: Individual client session handling
|
||||
- **PostgreSQLServerConfig**: Server configuration options
|
||||
- **Authentication System**: Trust, password, and MD5 authentication
|
||||
- **TLS Support**: Encrypted connections with custom certificates
|
||||
- **Connection Pooling**: Resource management and cleanup
|
||||
|
||||
### `protocol.go`
|
||||
- **Wire Protocol Implementation**: Full PostgreSQL 3.0 protocol support
|
||||
- **Message Handlers**: Startup, query, parse/bind/execute sequences
|
||||
- **Response Generation**: Row descriptions, data rows, command completion
|
||||
- **Data Type Mapping**: SeaweedFS to PostgreSQL type conversion
|
||||
- **SQL Parser**: Uses PostgreSQL-native parser for full dialect compatibility
|
||||
- **Error Handling**: PostgreSQL-compliant error responses
|
||||
- **MQ Integration**: Direct integration with SeaweedFS SQL engine for real topic data
|
||||
- **System Query Support**: Essential PostgreSQL system queries (version, current_user, etc.)
|
||||
- **Database Context**: Session-based database switching with USE commands
|
||||
|
||||
## Key Features
|
||||
|
||||
### Real MQ Topic Integration
|
||||
The PostgreSQL server now directly integrates with SeaweedFS Message Queue topics, providing:
|
||||
|
||||
- **Live Topic Discovery**: Automatically discovers MQ namespaces and topics from the filer
|
||||
- **Real Schema Information**: Reads actual topic schemas from broker configuration
|
||||
- **Actual Data Access**: Queries real MQ data stored in Parquet and log files
|
||||
- **Dynamic Updates**: Reflects topic additions and schema changes automatically
|
||||
- **Consistent SQL Engine**: Uses the same SQL engine as `weed-sql` command
|
||||
|
||||
### Database Context Management
|
||||
- **Session Isolation**: Each PostgreSQL connection has its own database context
|
||||
- **USE Command Support**: Switch between namespaces using standard `USE database` syntax
|
||||
- **Auto-Discovery**: Topics are discovered and registered on first access
|
||||
- **Schema Caching**: Efficient caching of topic schemas and metadata
|
||||
|
||||
## Usage
|
||||
|
||||
### Import the Package
|
||||
```go
|
||||
import "github.com/seaweedfs/seaweedfs/weed/server/postgres"
|
||||
```
|
||||
|
||||
### Create and Start Server
|
||||
```go
|
||||
config := &postgres.PostgreSQLServerConfig{
|
||||
Host: "localhost",
|
||||
Port: 5432,
|
||||
AuthMethod: postgres.AuthMD5,
|
||||
Users: map[string]string{"admin": "secret"},
|
||||
Database: "default",
|
||||
MaxConns: 100,
|
||||
IdleTimeout: time.Hour,
|
||||
}
|
||||
|
||||
server, err := postgres.NewPostgreSQLServer(config, "localhost:9333")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = server.Start()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Server is now accepting PostgreSQL connections
|
||||
```
|
||||
|
||||
## Authentication Methods
|
||||
|
||||
The package supports three authentication methods:
|
||||
|
||||
### Trust Authentication
|
||||
```go
|
||||
AuthMethod: postgres.AuthTrust
|
||||
```
|
||||
- No password required
|
||||
- Suitable for development/testing
|
||||
- Not recommended for production
|
||||
|
||||
### Password Authentication
|
||||
```go
|
||||
AuthMethod: postgres.AuthPassword,
|
||||
Users: map[string]string{"user": "password"}
|
||||
```
|
||||
- Clear text password transmission
|
||||
- Simple but less secure
|
||||
- Requires TLS for production use
|
||||
|
||||
### MD5 Authentication
|
||||
```go
|
||||
AuthMethod: postgres.AuthMD5,
|
||||
Users: map[string]string{"user": "password"}
|
||||
```
|
||||
- Secure hashed authentication with salt
|
||||
- **Recommended for production**
|
||||
- Compatible with all PostgreSQL clients
|
||||
|
||||
## TLS Configuration
|
||||
|
||||
Enable TLS encryption for secure connections:
|
||||
|
||||
```go
|
||||
cert, err := tls.LoadX509KeyPair("server.crt", "server.key")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
config.TLSConfig = &tls.Config{
|
||||
Certificates: []tls.Certificate{cert},
|
||||
}
|
||||
```
|
||||
|
||||
## Client Compatibility
|
||||
|
||||
This implementation is compatible with:
|
||||
|
||||
### Command Line Tools
|
||||
- `psql` - PostgreSQL command line client
|
||||
- `pgcli` - Enhanced command line with auto-completion
|
||||
- Database IDEs (DataGrip, DBeaver)
|
||||
|
||||
### Programming Languages
|
||||
- **Python**: psycopg2, asyncpg
|
||||
- **Java**: PostgreSQL JDBC driver
|
||||
- **JavaScript**: pg (node-postgres)
|
||||
- **Go**: lib/pq, pgx
|
||||
- **.NET**: Npgsql
|
||||
- **PHP**: pdo_pgsql
|
||||
- **Ruby**: pg gem
|
||||
|
||||
### BI Tools
|
||||
- Tableau (native PostgreSQL connector)
|
||||
- Power BI (PostgreSQL data source)
|
||||
- Grafana (PostgreSQL plugin)
|
||||
- Apache Superset
|
||||
|
||||
## Supported SQL Operations
|
||||
|
||||
### Data Queries
|
||||
```sql
|
||||
SELECT * FROM topic_name;
|
||||
SELECT id, message FROM topic_name WHERE condition;
|
||||
SELECT COUNT(*) FROM topic_name;
|
||||
SELECT MIN(id), MAX(id), AVG(amount) FROM topic_name;
|
||||
```
|
||||
|
||||
### Schema Information
|
||||
```sql
|
||||
SHOW DATABASES;
|
||||
SHOW TABLES;
|
||||
DESCRIBE topic_name;
|
||||
DESC topic_name;
|
||||
```
|
||||
|
||||
### System Information
|
||||
```sql
|
||||
SELECT version();
|
||||
SELECT current_database();
|
||||
SELECT current_user;
|
||||
```
|
||||
|
||||
### System Columns
|
||||
```sql
|
||||
SELECT id, message, _timestamp_ns, _key, _source FROM topic_name;
|
||||
```
|
||||
|
||||
## Configuration Options
|
||||
|
||||
### Server Configuration
|
||||
- **Host/Port**: Server binding address and port
|
||||
- **Authentication**: Method and user credentials
|
||||
- **Database**: Default database/namespace name
|
||||
- **Connections**: Maximum concurrent connections
|
||||
- **Timeouts**: Idle connection timeout
|
||||
- **TLS**: Certificate and encryption settings
|
||||
|
||||
### Performance Tuning
|
||||
- **Connection Limits**: Prevent resource exhaustion
|
||||
- **Idle Timeout**: Automatic cleanup of unused connections
|
||||
- **Memory Management**: Efficient session handling
|
||||
- **Query Streaming**: Large result set support
|
||||
|
||||
## Error Handling
|
||||
|
||||
The package provides PostgreSQL-compliant error responses:
|
||||
|
||||
- **Connection Errors**: Authentication failures, network issues
|
||||
- **SQL Errors**: Invalid syntax, missing tables
|
||||
- **Resource Errors**: Connection limits, timeouts
|
||||
- **Security Errors**: Permission denied, invalid credentials
|
||||
|
||||
## Development and Testing
|
||||
|
||||
### Unit Tests
|
||||
Run PostgreSQL package tests:
|
||||
```bash
|
||||
go test ./weed/server/postgres
|
||||
```
|
||||
|
||||
### Integration Testing
|
||||
Use the provided Python test client:
|
||||
```bash
|
||||
python postgres-examples/test_client.py --host localhost --port 5432
|
||||
```
|
||||
|
||||
### Manual Testing
|
||||
Connect with psql:
|
||||
```bash
|
||||
psql -h localhost -p 5432 -U seaweedfs -d default
|
||||
```
|
||||
|
||||
## Documentation
|
||||
|
||||
- **DESIGN.md**: Complete architecture and design overview
|
||||
- **IMPLEMENTATION.md**: Detailed implementation guide
|
||||
- **postgres-examples/**: Client examples and test scripts
|
||||
- **Command Documentation**: `weed-db -help`
|
||||
|
||||
## Security Considerations
|
||||
|
||||
### Production Deployment
|
||||
- Use MD5 or stronger authentication
|
||||
- Enable TLS encryption
|
||||
- Configure appropriate connection limits
|
||||
- Monitor for suspicious activity
|
||||
- Use strong passwords
|
||||
- Implement proper firewall rules
|
||||
|
||||
### Access Control
|
||||
- Create dedicated read-only users
|
||||
- Use principle of least privilege
|
||||
- Monitor connection patterns
|
||||
- Log authentication attempts
|
||||
|
||||
## Architecture Notes
|
||||
|
||||
### SQL Parser Dialect Considerations
|
||||
|
||||
**✅ POSTGRESQL ONLY**: SeaweedFS SQL engine exclusively supports PostgreSQL syntax:
|
||||
|
||||
- **✅ Core Engine**: `engine.go` uses custom PostgreSQL parser for proper dialect support
|
||||
- **PostgreSQL Server**: Uses PostgreSQL parser for optimal wire protocol compatibility
|
||||
- **Parser**: Custom lightweight PostgreSQL parser for full PostgreSQL compatibility
|
||||
- **Support Status**: Only PostgreSQL syntax is supported - MySQL parsing has been removed
|
||||
|
||||
**Key Benefits of PostgreSQL Parser**:
|
||||
- **Native Dialect Support**: Correctly handles PostgreSQL-specific syntax and semantics
|
||||
- **System Catalog Compatibility**: Supports `pg_catalog`, `information_schema` queries
|
||||
- **Operator Compatibility**: Handles `||` string concatenation, PostgreSQL-specific operators
|
||||
- **Type System Alignment**: Better PostgreSQL type inference and coercion
|
||||
- **Reduced Translation Overhead**: Eliminates need for dialect translation layer
|
||||
|
||||
**PostgreSQL Syntax Support**:
|
||||
- **Identifier Quoting**: Uses PostgreSQL double quotes (`"`) for identifiers
|
||||
- **String Concatenation**: Supports PostgreSQL `||` operator
|
||||
- **System Functions**: Full support for PostgreSQL system catalogs (`pg_catalog`) and functions
|
||||
- **Standard Compliance**: Follows PostgreSQL SQL standard and dialect
|
||||
|
||||
**Implementation Features**:
|
||||
- Native PostgreSQL query processing in `protocol.go`
|
||||
- System query support (`SELECT version()`, `BEGIN`, etc.)
|
||||
- Type mapping between PostgreSQL and SeaweedFS schema types
|
||||
- Error code mapping to PostgreSQL standards
|
||||
- Comprehensive PostgreSQL wire protocol support
|
||||
|
||||
This package provides enterprise-grade PostgreSQL compatibility, enabling seamless integration of SeaweedFS with the entire PostgreSQL ecosystem.
|
||||
917
weed/server/postgres/protocol.go
Normal file
917
weed/server/postgres/protocol.go
Normal file
@@ -0,0 +1,917 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/query/engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/query/sqltypes"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/sqlutil"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/version"
|
||||
)
|
||||
|
||||
const (
|
||||
// maxStartupMessageSize limits the startup handshake body to 10 KiB.
|
||||
// Startup parameters (username, database, etc.) are always small.
|
||||
maxStartupMessageSize = 10000
|
||||
|
||||
// maxMessageSize limits runtime message bodies to 100 MiB to prevent OOM.
|
||||
// PostgreSQL protocol allows up to 1 GiB per message in theory, but real
|
||||
// queries are rarely larger than a few megabytes.
|
||||
maxMessageSize = 100 * 1024 * 1024 // 100 MiB
|
||||
|
||||
// maxAuthMessageSize caps password/MD5 payloads, which arrive before the
|
||||
// client is authenticated. Credentials are tiny, so the pre-auth allocation
|
||||
// stays far below maxMessageSize.
|
||||
maxAuthMessageSize = 10 * 1024 // 10 KiB
|
||||
)
|
||||
|
||||
// mapErrorToPostgreSQLCode maps SeaweedFS SQL engine errors to appropriate PostgreSQL error codes
|
||||
func mapErrorToPostgreSQLCode(err error) string {
|
||||
if err == nil {
|
||||
return "00000" // Success
|
||||
}
|
||||
|
||||
// Use typed errors for robust error mapping
|
||||
switch err.(type) {
|
||||
case engine.ParseError:
|
||||
return "42601" // Syntax error
|
||||
|
||||
case engine.TableNotFoundError:
|
||||
return "42P01" // Undefined table
|
||||
|
||||
case engine.ColumnNotFoundError:
|
||||
return "42703" // Undefined column
|
||||
|
||||
case engine.UnsupportedFeatureError:
|
||||
return "0A000" // Feature not supported
|
||||
|
||||
case engine.AggregationError:
|
||||
// Aggregation errors are usually function-related issues
|
||||
return "42883" // Undefined function (aggregation function issues)
|
||||
|
||||
case engine.DataSourceError:
|
||||
// Data source errors are usually access or connection issues
|
||||
return "08000" // Connection exception
|
||||
|
||||
case engine.OptimizationError:
|
||||
// Optimization failures are usually feature limitations
|
||||
return "0A000" // Feature not supported
|
||||
|
||||
case engine.NoSchemaError:
|
||||
// Topic exists but no schema available
|
||||
return "42P01" // Undefined table (treat as table not found)
|
||||
}
|
||||
|
||||
// Fallback: analyze error message for backward compatibility with non-typed errors
|
||||
errLower := strings.ToLower(err.Error())
|
||||
|
||||
// Parsing and syntax errors
|
||||
if strings.Contains(errLower, "parse error") || strings.Contains(errLower, "syntax") {
|
||||
return "42601" // Syntax error
|
||||
}
|
||||
|
||||
// Unsupported features
|
||||
if strings.Contains(errLower, "unsupported") || strings.Contains(errLower, "not supported") {
|
||||
return "0A000" // Feature not supported
|
||||
}
|
||||
|
||||
// Table/topic not found
|
||||
if strings.Contains(errLower, "not found") ||
|
||||
(strings.Contains(errLower, "topic") && strings.Contains(errLower, "available")) {
|
||||
return "42P01" // Undefined table
|
||||
}
|
||||
|
||||
// Column-related errors
|
||||
if strings.Contains(errLower, "column") || strings.Contains(errLower, "field") {
|
||||
return "42703" // Undefined column
|
||||
}
|
||||
|
||||
// Multi-table or complex query limitations
|
||||
if strings.Contains(errLower, "single table") || strings.Contains(errLower, "join") {
|
||||
return "0A000" // Feature not supported
|
||||
}
|
||||
|
||||
// Default to generic syntax/access error
|
||||
return "42000" // Syntax error or access rule violation
|
||||
}
|
||||
|
||||
// handleMessage processes a single PostgreSQL protocol message
|
||||
func (s *PostgreSQLServer) handleMessage(session *PostgreSQLSession) error {
|
||||
// Read message type
|
||||
msgType := make([]byte, 1)
|
||||
_, err := io.ReadFull(session.reader, msgType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Read message length
|
||||
length := make([]byte, 4)
|
||||
_, err = io.ReadFull(session.reader, length)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rawLength := binary.BigEndian.Uint32(length)
|
||||
if rawLength < 4 {
|
||||
return fmt.Errorf("invalid message length: %d (must be >= 4)", rawLength)
|
||||
}
|
||||
msgLength := rawLength - 4
|
||||
if msgLength > maxMessageSize {
|
||||
return fmt.Errorf("message too large: %d bytes", msgLength)
|
||||
}
|
||||
msgBody := make([]byte, msgLength)
|
||||
if msgLength > 0 {
|
||||
_, err = io.ReadFull(session.reader, msgBody)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Process message based on type
|
||||
switch msgType[0] {
|
||||
case PG_MSG_QUERY:
|
||||
query := strings.TrimSuffix(string(msgBody), "\x00") // Remove null terminator if present
|
||||
return s.handleSimpleQuery(session, query)
|
||||
case PG_MSG_PARSE:
|
||||
return s.handleParse(session, msgBody)
|
||||
case PG_MSG_BIND:
|
||||
return s.handleBind(session, msgBody)
|
||||
case PG_MSG_EXECUTE:
|
||||
return s.handleExecute(session, msgBody)
|
||||
case PG_MSG_DESCRIBE:
|
||||
return s.handleDescribe(session, msgBody)
|
||||
case PG_MSG_CLOSE:
|
||||
return s.handleClose(session, msgBody)
|
||||
case PG_MSG_FLUSH:
|
||||
return s.handleFlush(session)
|
||||
case PG_MSG_SYNC:
|
||||
return s.handleSync(session)
|
||||
case PG_MSG_TERMINATE:
|
||||
return io.EOF // Signal connection termination
|
||||
default:
|
||||
return s.sendError(session, "08P01", fmt.Sprintf("unknown message type: %c", msgType[0]))
|
||||
}
|
||||
}
|
||||
|
||||
// handleSimpleQuery processes a simple query message
|
||||
func (s *PostgreSQLServer) handleSimpleQuery(session *PostgreSQLSession, query string) error {
|
||||
glog.V(2).Infof("PostgreSQL Query (ID: %d): %s", session.processID, query)
|
||||
|
||||
// Add comprehensive error recovery to prevent crashes
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
glog.Errorf("Panic in handleSimpleQuery (ID: %d): %v", session.processID, r)
|
||||
// Try to send error message
|
||||
s.sendError(session, "XX000", fmt.Sprintf("Internal error: %v", r))
|
||||
// Try to send ReadyForQuery to keep connection alive
|
||||
s.sendReadyForQuery(session)
|
||||
}
|
||||
}()
|
||||
|
||||
// Handle USE database commands for session context
|
||||
parts := strings.Fields(strings.TrimSpace(query))
|
||||
if len(parts) >= 2 && strings.ToUpper(parts[0]) == "USE" {
|
||||
// Re-join the parts after "USE" to handle names with spaces, then trim.
|
||||
dbName := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(query), parts[0]))
|
||||
|
||||
// Unquote if necessary (handle quoted identifiers like "my-database")
|
||||
if len(dbName) > 1 && dbName[0] == '"' && dbName[len(dbName)-1] == '"' {
|
||||
dbName = dbName[1 : len(dbName)-1]
|
||||
} else if len(dbName) > 1 && dbName[0] == '`' && dbName[len(dbName)-1] == '`' {
|
||||
// Also handle backtick quotes for MySQL/other client compatibility
|
||||
dbName = dbName[1 : len(dbName)-1]
|
||||
}
|
||||
|
||||
session.database = dbName
|
||||
s.sqlEngine.GetCatalog().SetCurrentDatabase(dbName)
|
||||
|
||||
// Send command complete for USE
|
||||
err := s.sendCommandComplete(session, "USE")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Send ReadyForQuery and exit (don't continue processing)
|
||||
return s.sendReadyForQuery(session)
|
||||
}
|
||||
|
||||
// Set database context in SQL engine if session database is different from current
|
||||
if session.database != "" && session.database != s.sqlEngine.GetCatalog().GetCurrentDatabase() {
|
||||
s.sqlEngine.GetCatalog().SetCurrentDatabase(session.database)
|
||||
}
|
||||
|
||||
// Split query string into individual statements to handle multi-statement queries
|
||||
queries := sqlutil.SplitStatements(query)
|
||||
|
||||
// Execute each statement sequentially
|
||||
for _, singleQuery := range queries {
|
||||
cleanQuery := strings.TrimSpace(singleQuery)
|
||||
if cleanQuery == "" {
|
||||
continue // Skip empty statements
|
||||
}
|
||||
|
||||
// Handle PostgreSQL-specific system queries directly
|
||||
if systemResult := s.handleSystemQuery(session, cleanQuery); systemResult != nil {
|
||||
err := s.sendSystemQueryResult(session, systemResult, cleanQuery)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
continue // Continue with next statement
|
||||
}
|
||||
|
||||
// Execute using PostgreSQL-compatible SQL engine for proper dialect support
|
||||
ctx := context.Background()
|
||||
var result *engine.QueryResult
|
||||
var err error
|
||||
|
||||
// Execute SQL query with panic recovery to prevent crashes
|
||||
func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
glog.Errorf("Panic in SQL execution (ID: %d, Query: %s): %v", session.processID, cleanQuery, r)
|
||||
err = fmt.Errorf("internal error during SQL execution: %v", r)
|
||||
}
|
||||
}()
|
||||
|
||||
// Use the main sqlEngine (now uses CockroachDB parser for PostgreSQL compatibility)
|
||||
result, err = s.sqlEngine.ExecuteSQL(ctx, cleanQuery)
|
||||
}()
|
||||
|
||||
if err != nil {
|
||||
// Send error message but keep connection alive
|
||||
errorCode := mapErrorToPostgreSQLCode(err)
|
||||
sendErr := s.sendError(session, errorCode, err.Error())
|
||||
if sendErr != nil {
|
||||
return sendErr
|
||||
}
|
||||
// Send ReadyForQuery to keep connection alive
|
||||
return s.sendReadyForQuery(session)
|
||||
}
|
||||
|
||||
if result.Error != nil {
|
||||
// Send error message but keep connection alive
|
||||
errorCode := mapErrorToPostgreSQLCode(result.Error)
|
||||
sendErr := s.sendError(session, errorCode, result.Error.Error())
|
||||
if sendErr != nil {
|
||||
return sendErr
|
||||
}
|
||||
// Send ReadyForQuery to keep connection alive
|
||||
return s.sendReadyForQuery(session)
|
||||
}
|
||||
|
||||
// Send results for this statement
|
||||
if len(result.Columns) > 0 {
|
||||
// Send row description
|
||||
err = s.sendRowDescription(session, result)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Send data rows
|
||||
for _, row := range result.Rows {
|
||||
err = s.sendDataRow(session, row)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Send command complete for this statement
|
||||
tag := s.getCommandTag(cleanQuery, len(result.Rows))
|
||||
err = s.sendCommandComplete(session, tag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Send ready for query after all statements are processed
|
||||
return s.sendReadyForQuery(session)
|
||||
}
|
||||
|
||||
// SystemQueryResult represents the result of a system query
|
||||
type SystemQueryResult struct {
|
||||
Columns []string
|
||||
Rows [][]string
|
||||
}
|
||||
|
||||
// handleSystemQuery handles PostgreSQL system queries directly
|
||||
func (s *PostgreSQLServer) handleSystemQuery(session *PostgreSQLSession, query string) *SystemQueryResult {
|
||||
// Trim and normalize query
|
||||
query = strings.TrimSpace(query)
|
||||
query = strings.TrimSuffix(query, ";")
|
||||
queryLower := strings.ToLower(query)
|
||||
|
||||
// Handle essential PostgreSQL system queries
|
||||
switch queryLower {
|
||||
case "select version()":
|
||||
return &SystemQueryResult{
|
||||
Columns: []string{"version"},
|
||||
Rows: [][]string{{fmt.Sprintf("SeaweedFS %s (PostgreSQL 14.0 compatible)", version.VERSION_NUMBER)}},
|
||||
}
|
||||
case "select current_database()":
|
||||
return &SystemQueryResult{
|
||||
Columns: []string{"current_database"},
|
||||
Rows: [][]string{{s.config.Database}},
|
||||
}
|
||||
case "select current_user":
|
||||
return &SystemQueryResult{
|
||||
Columns: []string{"current_user"},
|
||||
Rows: [][]string{{"seaweedfs"}},
|
||||
}
|
||||
case "select current_setting('server_version')":
|
||||
return &SystemQueryResult{
|
||||
Columns: []string{"server_version"},
|
||||
Rows: [][]string{{fmt.Sprintf("%s (SeaweedFS)", version.VERSION_NUMBER)}},
|
||||
}
|
||||
case "select current_setting('server_encoding')":
|
||||
return &SystemQueryResult{
|
||||
Columns: []string{"server_encoding"},
|
||||
Rows: [][]string{{"UTF8"}},
|
||||
}
|
||||
case "select current_setting('client_encoding')":
|
||||
return &SystemQueryResult{
|
||||
Columns: []string{"client_encoding"},
|
||||
Rows: [][]string{{"UTF8"}},
|
||||
}
|
||||
}
|
||||
|
||||
// Handle transaction commands (no-op for read-only)
|
||||
switch queryLower {
|
||||
case "begin", "start transaction":
|
||||
return &SystemQueryResult{
|
||||
Columns: []string{"status"},
|
||||
Rows: [][]string{{"BEGIN"}},
|
||||
}
|
||||
case "commit":
|
||||
return &SystemQueryResult{
|
||||
Columns: []string{"status"},
|
||||
Rows: [][]string{{"COMMIT"}},
|
||||
}
|
||||
case "rollback":
|
||||
return &SystemQueryResult{
|
||||
Columns: []string{"status"},
|
||||
Rows: [][]string{{"ROLLBACK"}},
|
||||
}
|
||||
}
|
||||
|
||||
// If starts with SET, return a no-op
|
||||
if strings.HasPrefix(queryLower, "set ") {
|
||||
return &SystemQueryResult{
|
||||
Columns: []string{"status"},
|
||||
Rows: [][]string{{"SET"}},
|
||||
}
|
||||
}
|
||||
|
||||
// Return nil to use SQL engine
|
||||
return nil
|
||||
}
|
||||
|
||||
// sendSystemQueryResult sends the result of a system query
|
||||
func (s *PostgreSQLServer) sendSystemQueryResult(session *PostgreSQLSession, result *SystemQueryResult, query string) error {
|
||||
// Add panic recovery to prevent crashes in system query results
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
glog.Errorf("Panic in sendSystemQueryResult (ID: %d, Query: %s): %v", session.processID, query, r)
|
||||
// Try to send error and continue
|
||||
s.sendError(session, "XX000", fmt.Sprintf("Internal error in system query: %v", r))
|
||||
}
|
||||
}()
|
||||
|
||||
// Create column descriptions for system query results
|
||||
columns := make([]string, len(result.Columns))
|
||||
for i, col := range result.Columns {
|
||||
columns[i] = col
|
||||
}
|
||||
|
||||
// Convert to sqltypes.Value format
|
||||
var sqlRows [][]sqltypes.Value
|
||||
for _, row := range result.Rows {
|
||||
sqlRow := make([]sqltypes.Value, len(row))
|
||||
for i, cell := range row {
|
||||
sqlRow[i] = sqltypes.NewVarChar(cell)
|
||||
}
|
||||
sqlRows = append(sqlRows, sqlRow)
|
||||
}
|
||||
|
||||
// Send row description (create a temporary QueryResult for consistency)
|
||||
tempResult := &engine.QueryResult{
|
||||
Columns: columns,
|
||||
Rows: sqlRows,
|
||||
}
|
||||
err := s.sendRowDescription(session, tempResult)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Send data rows
|
||||
for _, row := range sqlRows {
|
||||
err = s.sendDataRow(session, row)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Send command complete
|
||||
tag := s.getCommandTag(query, len(result.Rows))
|
||||
err = s.sendCommandComplete(session, tag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Send ready for query
|
||||
return s.sendReadyForQuery(session)
|
||||
}
|
||||
|
||||
// handleParse processes a Parse message (prepared statement)
|
||||
func (s *PostgreSQLServer) handleParse(session *PostgreSQLSession, msgBody []byte) error {
|
||||
// Parse message format: statement_name\0query\0param_count(int16)[param_type(int32)...]
|
||||
parts := strings.Split(string(msgBody), "\x00")
|
||||
if len(parts) < 2 {
|
||||
return s.sendError(session, "08P01", "invalid Parse message format")
|
||||
}
|
||||
|
||||
stmtName := parts[0]
|
||||
query := parts[1]
|
||||
|
||||
// Create prepared statement
|
||||
stmt := &PreparedStatement{
|
||||
Name: stmtName,
|
||||
Query: query,
|
||||
ParamTypes: []uint32{},
|
||||
Fields: []FieldDescription{},
|
||||
}
|
||||
|
||||
session.preparedStmts[stmtName] = stmt
|
||||
|
||||
// Send parse complete
|
||||
return s.sendParseComplete(session)
|
||||
}
|
||||
|
||||
// handleBind processes a Bind message
|
||||
func (s *PostgreSQLServer) handleBind(session *PostgreSQLSession, msgBody []byte) error {
|
||||
// For now, simple implementation
|
||||
// In full implementation, would parse parameters and create portal
|
||||
|
||||
// Send bind complete
|
||||
return s.sendBindComplete(session)
|
||||
}
|
||||
|
||||
// handleExecute processes an Execute message
|
||||
func (s *PostgreSQLServer) handleExecute(session *PostgreSQLSession, msgBody []byte) error {
|
||||
// Parse portal name
|
||||
parts := strings.Split(string(msgBody), "\x00")
|
||||
if len(parts) == 0 {
|
||||
return s.sendError(session, "08P01", "invalid Execute message format")
|
||||
}
|
||||
|
||||
portalName := parts[0]
|
||||
|
||||
// For now, execute as simple query
|
||||
// In full implementation, would use portal with parameters
|
||||
glog.V(2).Infof("PostgreSQL Execute portal (ID: %d): %s", session.processID, portalName)
|
||||
|
||||
// Send command complete
|
||||
err := s.sendCommandComplete(session, "SELECT 0")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleDescribe processes a Describe message
|
||||
func (s *PostgreSQLServer) handleDescribe(session *PostgreSQLSession, msgBody []byte) error {
|
||||
if len(msgBody) < 2 {
|
||||
return s.sendError(session, "08P01", "invalid Describe message format")
|
||||
}
|
||||
|
||||
objectType := msgBody[0] // 'S' for statement, 'P' for portal
|
||||
objectName := string(msgBody[1:])
|
||||
|
||||
glog.V(2).Infof("PostgreSQL Describe %c (ID: %d): %s", objectType, session.processID, objectName)
|
||||
|
||||
// For now, send empty row description
|
||||
tempResult := &engine.QueryResult{
|
||||
Columns: []string{},
|
||||
Rows: [][]sqltypes.Value{},
|
||||
}
|
||||
return s.sendRowDescription(session, tempResult)
|
||||
}
|
||||
|
||||
// handleClose processes a Close message
|
||||
func (s *PostgreSQLServer) handleClose(session *PostgreSQLSession, msgBody []byte) error {
|
||||
if len(msgBody) < 2 {
|
||||
return s.sendError(session, "08P01", "invalid Close message format")
|
||||
}
|
||||
|
||||
objectType := msgBody[0] // 'S' for statement, 'P' for portal
|
||||
objectName := string(msgBody[1:])
|
||||
|
||||
switch objectType {
|
||||
case 'S':
|
||||
delete(session.preparedStmts, objectName)
|
||||
case 'P':
|
||||
delete(session.portals, objectName)
|
||||
}
|
||||
|
||||
// Send close complete
|
||||
return s.sendCloseComplete(session)
|
||||
}
|
||||
|
||||
// handleFlush processes a Flush message
|
||||
func (s *PostgreSQLServer) handleFlush(session *PostgreSQLSession) error {
|
||||
return session.writer.Flush()
|
||||
}
|
||||
|
||||
// handleSync processes a Sync message
|
||||
func (s *PostgreSQLServer) handleSync(session *PostgreSQLSession) error {
|
||||
// Reset transaction state if needed
|
||||
session.transactionState = PG_TRANS_IDLE
|
||||
|
||||
// Send ready for query
|
||||
return s.sendReadyForQuery(session)
|
||||
}
|
||||
|
||||
// sendParameterStatus sends a parameter status message
|
||||
func (s *PostgreSQLServer) sendParameterStatus(session *PostgreSQLSession, name, value string) error {
|
||||
msg := make([]byte, 0)
|
||||
msg = append(msg, PG_RESP_PARAMETER)
|
||||
|
||||
// Calculate length
|
||||
length := 4 + len(name) + 1 + len(value) + 1
|
||||
lengthBytes := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(lengthBytes, uint32(length))
|
||||
msg = append(msg, lengthBytes...)
|
||||
|
||||
// Add name and value
|
||||
msg = append(msg, []byte(name)...)
|
||||
msg = append(msg, 0) // null terminator
|
||||
msg = append(msg, []byte(value)...)
|
||||
msg = append(msg, 0) // null terminator
|
||||
|
||||
_, err := session.writer.Write(msg)
|
||||
if err == nil {
|
||||
err = session.writer.Flush()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// sendBackendKeyData sends backend key data
|
||||
func (s *PostgreSQLServer) sendBackendKeyData(session *PostgreSQLSession) error {
|
||||
msg := make([]byte, 13)
|
||||
msg[0] = PG_RESP_BACKEND_KEY
|
||||
binary.BigEndian.PutUint32(msg[1:5], 12)
|
||||
binary.BigEndian.PutUint32(msg[5:9], session.processID)
|
||||
binary.BigEndian.PutUint32(msg[9:13], session.secretKey)
|
||||
|
||||
_, err := session.writer.Write(msg)
|
||||
if err == nil {
|
||||
err = session.writer.Flush()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// sendReadyForQuery sends ready for query message
|
||||
func (s *PostgreSQLServer) sendReadyForQuery(session *PostgreSQLSession) error {
|
||||
msg := make([]byte, 6)
|
||||
msg[0] = PG_RESP_READY
|
||||
binary.BigEndian.PutUint32(msg[1:5], 5)
|
||||
msg[5] = session.transactionState
|
||||
|
||||
_, err := session.writer.Write(msg)
|
||||
if err == nil {
|
||||
err = session.writer.Flush()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// sendRowDescription sends row description message
|
||||
func (s *PostgreSQLServer) sendRowDescription(session *PostgreSQLSession, result *engine.QueryResult) error {
|
||||
msg := make([]byte, 0)
|
||||
msg = append(msg, PG_RESP_ROW_DESC)
|
||||
|
||||
// Calculate message length
|
||||
length := 4 + 2 // length + field count
|
||||
for _, col := range result.Columns {
|
||||
length += len(col) + 1 + 4 + 2 + 4 + 2 + 4 + 2 // name + null + tableOID + attrNum + typeOID + typeSize + typeMod + format
|
||||
}
|
||||
|
||||
lengthBytes := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(lengthBytes, uint32(length))
|
||||
msg = append(msg, lengthBytes...)
|
||||
|
||||
// Field count
|
||||
fieldCountBytes := make([]byte, 2)
|
||||
binary.BigEndian.PutUint16(fieldCountBytes, uint16(len(result.Columns)))
|
||||
msg = append(msg, fieldCountBytes...)
|
||||
|
||||
// Field descriptions
|
||||
for i, col := range result.Columns {
|
||||
// Field name
|
||||
msg = append(msg, []byte(col)...)
|
||||
msg = append(msg, 0) // null terminator
|
||||
|
||||
// Table OID (0 for no table)
|
||||
tableOID := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(tableOID, 0)
|
||||
msg = append(msg, tableOID...)
|
||||
|
||||
// Attribute number
|
||||
attrNum := make([]byte, 2)
|
||||
binary.BigEndian.PutUint16(attrNum, uint16(i+1))
|
||||
msg = append(msg, attrNum...)
|
||||
|
||||
// Type OID (determine from schema if available, fallback to data inference)
|
||||
typeOID := s.getPostgreSQLTypeFromSchema(result, col, i)
|
||||
typeOIDBytes := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(typeOIDBytes, typeOID)
|
||||
msg = append(msg, typeOIDBytes...)
|
||||
|
||||
// Type size (-1 for variable length)
|
||||
typeSize := make([]byte, 2)
|
||||
binary.BigEndian.PutUint16(typeSize, 0xFFFF) // -1 as uint16
|
||||
msg = append(msg, typeSize...)
|
||||
|
||||
// Type modifier (-1 for default)
|
||||
typeMod := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(typeMod, 0xFFFFFFFF) // -1 as uint32
|
||||
msg = append(msg, typeMod...)
|
||||
|
||||
// Format (0 for text)
|
||||
format := make([]byte, 2)
|
||||
binary.BigEndian.PutUint16(format, 0)
|
||||
msg = append(msg, format...)
|
||||
}
|
||||
|
||||
_, err := session.writer.Write(msg)
|
||||
if err == nil {
|
||||
err = session.writer.Flush()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// sendDataRow sends a data row message
|
||||
func (s *PostgreSQLServer) sendDataRow(session *PostgreSQLSession, row []sqltypes.Value) error {
|
||||
msg := make([]byte, 0)
|
||||
msg = append(msg, PG_RESP_DATA_ROW)
|
||||
|
||||
// Calculate message length
|
||||
length := 4 + 2 // length + field count
|
||||
for _, value := range row {
|
||||
if value.IsNull() {
|
||||
length += 4 // null value length (-1)
|
||||
} else {
|
||||
valueStr := value.ToString()
|
||||
length += 4 + len(valueStr) // field length + data
|
||||
}
|
||||
}
|
||||
|
||||
lengthBytes := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(lengthBytes, uint32(length))
|
||||
msg = append(msg, lengthBytes...)
|
||||
|
||||
// Field count
|
||||
fieldCountBytes := make([]byte, 2)
|
||||
binary.BigEndian.PutUint16(fieldCountBytes, uint16(len(row)))
|
||||
msg = append(msg, fieldCountBytes...)
|
||||
|
||||
// Field values
|
||||
for _, value := range row {
|
||||
if value.IsNull() {
|
||||
// Null value
|
||||
nullLength := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(nullLength, 0xFFFFFFFF) // -1 as uint32
|
||||
msg = append(msg, nullLength...)
|
||||
} else {
|
||||
valueStr := value.ToString()
|
||||
valueLength := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(valueLength, uint32(len(valueStr)))
|
||||
msg = append(msg, valueLength...)
|
||||
msg = append(msg, []byte(valueStr)...)
|
||||
}
|
||||
}
|
||||
|
||||
_, err := session.writer.Write(msg)
|
||||
if err == nil {
|
||||
err = session.writer.Flush()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// sendCommandComplete sends command complete message
|
||||
func (s *PostgreSQLServer) sendCommandComplete(session *PostgreSQLSession, tag string) error {
|
||||
msg := make([]byte, 0)
|
||||
msg = append(msg, PG_RESP_COMMAND)
|
||||
|
||||
length := 4 + len(tag) + 1
|
||||
lengthBytes := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(lengthBytes, uint32(length))
|
||||
msg = append(msg, lengthBytes...)
|
||||
|
||||
msg = append(msg, []byte(tag)...)
|
||||
msg = append(msg, 0) // null terminator
|
||||
|
||||
_, err := session.writer.Write(msg)
|
||||
if err == nil {
|
||||
err = session.writer.Flush()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// sendParseComplete sends parse complete message
|
||||
func (s *PostgreSQLServer) sendParseComplete(session *PostgreSQLSession) error {
|
||||
msg := make([]byte, 5)
|
||||
msg[0] = PG_RESP_PARSE_COMPLETE
|
||||
binary.BigEndian.PutUint32(msg[1:5], 4)
|
||||
|
||||
_, err := session.writer.Write(msg)
|
||||
if err == nil {
|
||||
err = session.writer.Flush()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// sendBindComplete sends bind complete message
|
||||
func (s *PostgreSQLServer) sendBindComplete(session *PostgreSQLSession) error {
|
||||
msg := make([]byte, 5)
|
||||
msg[0] = PG_RESP_BIND_COMPLETE
|
||||
binary.BigEndian.PutUint32(msg[1:5], 4)
|
||||
|
||||
_, err := session.writer.Write(msg)
|
||||
if err == nil {
|
||||
err = session.writer.Flush()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// sendCloseComplete sends close complete message
|
||||
func (s *PostgreSQLServer) sendCloseComplete(session *PostgreSQLSession) error {
|
||||
msg := make([]byte, 5)
|
||||
msg[0] = PG_RESP_CLOSE_COMPLETE
|
||||
binary.BigEndian.PutUint32(msg[1:5], 4)
|
||||
|
||||
_, err := session.writer.Write(msg)
|
||||
if err == nil {
|
||||
err = session.writer.Flush()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// sendError sends an error message
|
||||
func (s *PostgreSQLServer) sendError(session *PostgreSQLSession, code, message string) error {
|
||||
msg := make([]byte, 0)
|
||||
msg = append(msg, PG_RESP_ERROR)
|
||||
|
||||
// Build error fields
|
||||
fields := fmt.Sprintf("S%s\x00C%s\x00M%s\x00\x00", "ERROR", code, message)
|
||||
length := 4 + len(fields)
|
||||
|
||||
lengthBytes := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(lengthBytes, uint32(length))
|
||||
msg = append(msg, lengthBytes...)
|
||||
msg = append(msg, []byte(fields)...)
|
||||
|
||||
_, err := session.writer.Write(msg)
|
||||
if err == nil {
|
||||
err = session.writer.Flush()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// getCommandTag generates appropriate command tag for query
|
||||
func (s *PostgreSQLServer) getCommandTag(query string, rowCount int) string {
|
||||
queryUpper := strings.ToUpper(strings.TrimSpace(query))
|
||||
|
||||
if strings.HasPrefix(queryUpper, "SELECT") {
|
||||
return fmt.Sprintf("SELECT %d", rowCount)
|
||||
} else if strings.HasPrefix(queryUpper, "INSERT") {
|
||||
return fmt.Sprintf("INSERT 0 %d", rowCount)
|
||||
} else if strings.HasPrefix(queryUpper, "UPDATE") {
|
||||
return fmt.Sprintf("UPDATE %d", rowCount)
|
||||
} else if strings.HasPrefix(queryUpper, "DELETE") {
|
||||
return fmt.Sprintf("DELETE %d", rowCount)
|
||||
} else if strings.HasPrefix(queryUpper, "SHOW") {
|
||||
return fmt.Sprintf("SELECT %d", rowCount)
|
||||
} else if strings.HasPrefix(queryUpper, "DESCRIBE") || strings.HasPrefix(queryUpper, "DESC") {
|
||||
return fmt.Sprintf("SELECT %d", rowCount)
|
||||
}
|
||||
|
||||
return "SELECT 0"
|
||||
}
|
||||
|
||||
// getPostgreSQLTypeFromSchema determines PostgreSQL type OID from schema information first, fallback to data
|
||||
func (s *PostgreSQLServer) getPostgreSQLTypeFromSchema(result *engine.QueryResult, columnName string, colIndex int) uint32 {
|
||||
// Try to get type from schema if database and table are available
|
||||
if result.Database != "" && result.Table != "" {
|
||||
if tableInfo, err := s.sqlEngine.GetCatalog().GetTableInfo(result.Database, result.Table); err == nil {
|
||||
if tableInfo.Schema != nil && tableInfo.Schema.RecordType != nil {
|
||||
// Look for the field in the schema
|
||||
for _, field := range tableInfo.Schema.RecordType.Fields {
|
||||
if field.Name == columnName {
|
||||
return s.mapSchemaTypeToPostgreSQL(field.Type)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle system columns
|
||||
switch columnName {
|
||||
case "_timestamp_ns":
|
||||
return PG_TYPE_INT8 // PostgreSQL BIGINT for nanosecond timestamps
|
||||
case "_key":
|
||||
return PG_TYPE_BYTEA // PostgreSQL BYTEA for binary keys
|
||||
case "_source":
|
||||
return PG_TYPE_TEXT // PostgreSQL TEXT for source information
|
||||
}
|
||||
|
||||
// Fallback to data-based inference if schema is not available
|
||||
return s.getPostgreSQLTypeFromData(result.Columns, result.Rows, colIndex)
|
||||
}
|
||||
|
||||
// mapSchemaTypeToPostgreSQL maps SeaweedFS schema types to PostgreSQL type OIDs
|
||||
func (s *PostgreSQLServer) mapSchemaTypeToPostgreSQL(fieldType *schema_pb.Type) uint32 {
|
||||
if fieldType == nil {
|
||||
return PG_TYPE_TEXT
|
||||
}
|
||||
|
||||
switch kind := fieldType.Kind.(type) {
|
||||
case *schema_pb.Type_ScalarType:
|
||||
switch kind.ScalarType {
|
||||
case schema_pb.ScalarType_BOOL:
|
||||
return PG_TYPE_BOOL
|
||||
case schema_pb.ScalarType_INT32:
|
||||
return PG_TYPE_INT4
|
||||
case schema_pb.ScalarType_INT64:
|
||||
return PG_TYPE_INT8
|
||||
case schema_pb.ScalarType_FLOAT:
|
||||
return PG_TYPE_FLOAT4
|
||||
case schema_pb.ScalarType_DOUBLE:
|
||||
return PG_TYPE_FLOAT8
|
||||
case schema_pb.ScalarType_BYTES:
|
||||
return PG_TYPE_BYTEA
|
||||
case schema_pb.ScalarType_STRING:
|
||||
return PG_TYPE_TEXT
|
||||
default:
|
||||
return PG_TYPE_TEXT
|
||||
}
|
||||
case *schema_pb.Type_ListType:
|
||||
// For list types, we'll represent them as JSON text
|
||||
return PG_TYPE_JSONB
|
||||
case *schema_pb.Type_RecordType:
|
||||
// For nested record types, we'll represent them as JSON text
|
||||
return PG_TYPE_JSONB
|
||||
default:
|
||||
return PG_TYPE_TEXT
|
||||
}
|
||||
}
|
||||
|
||||
// getPostgreSQLTypeFromData determines PostgreSQL type OID from data (legacy fallback method)
|
||||
func (s *PostgreSQLServer) getPostgreSQLTypeFromData(columns []string, rows [][]sqltypes.Value, colIndex int) uint32 {
|
||||
if len(rows) == 0 || colIndex >= len(rows[0]) {
|
||||
return PG_TYPE_TEXT // Default to text
|
||||
}
|
||||
|
||||
// Sample first non-null value to determine type
|
||||
for _, row := range rows {
|
||||
if colIndex < len(row) && !row[colIndex].IsNull() {
|
||||
value := row[colIndex]
|
||||
switch value.Type() {
|
||||
case sqltypes.Int8, sqltypes.Int16, sqltypes.Int32:
|
||||
return PG_TYPE_INT4
|
||||
case sqltypes.Int64:
|
||||
return PG_TYPE_INT8
|
||||
case sqltypes.Float32, sqltypes.Float64:
|
||||
return PG_TYPE_FLOAT8
|
||||
case sqltypes.Bit:
|
||||
return PG_TYPE_BOOL
|
||||
case sqltypes.Timestamp, sqltypes.Datetime:
|
||||
return PG_TYPE_TIMESTAMP
|
||||
default:
|
||||
// Try to infer from string content
|
||||
valueStr := value.ToString()
|
||||
if _, err := strconv.ParseInt(valueStr, 10, 32); err == nil {
|
||||
return PG_TYPE_INT4
|
||||
}
|
||||
if _, err := strconv.ParseInt(valueStr, 10, 64); err == nil {
|
||||
return PG_TYPE_INT8
|
||||
}
|
||||
if _, err := strconv.ParseFloat(valueStr, 64); err == nil {
|
||||
return PG_TYPE_FLOAT8
|
||||
}
|
||||
if valueStr == "true" || valueStr == "false" {
|
||||
return PG_TYPE_BOOL
|
||||
}
|
||||
return PG_TYPE_TEXT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return PG_TYPE_TEXT // Default to text
|
||||
}
|
||||
726
weed/server/postgres/server.go
Normal file
726
weed/server/postgres/server.go
Normal file
@@ -0,0 +1,726 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/md5"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/query/engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/version"
|
||||
)
|
||||
|
||||
// PostgreSQL protocol constants
|
||||
const (
|
||||
// Protocol versions
|
||||
PG_PROTOCOL_VERSION_3 = 196608 // PostgreSQL 3.0 protocol (0x00030000)
|
||||
PG_SSL_REQUEST = 80877103 // SSL request (0x04d2162f)
|
||||
PG_GSSAPI_REQUEST = 80877104 // GSSAPI request (0x04d21630)
|
||||
|
||||
// Message types from client
|
||||
PG_MSG_STARTUP = 0x00
|
||||
PG_MSG_QUERY = 'Q'
|
||||
PG_MSG_PARSE = 'P'
|
||||
PG_MSG_BIND = 'B'
|
||||
PG_MSG_EXECUTE = 'E'
|
||||
PG_MSG_DESCRIBE = 'D'
|
||||
PG_MSG_CLOSE = 'C'
|
||||
PG_MSG_FLUSH = 'H'
|
||||
PG_MSG_SYNC = 'S'
|
||||
PG_MSG_TERMINATE = 'X'
|
||||
PG_MSG_PASSWORD = 'p'
|
||||
|
||||
// Response types to client
|
||||
PG_RESP_AUTH_OK = 'R'
|
||||
PG_RESP_BACKEND_KEY = 'K'
|
||||
PG_RESP_PARAMETER = 'S'
|
||||
PG_RESP_READY = 'Z'
|
||||
PG_RESP_COMMAND = 'C'
|
||||
PG_RESP_DATA_ROW = 'D'
|
||||
PG_RESP_ROW_DESC = 'T'
|
||||
PG_RESP_PARSE_COMPLETE = '1'
|
||||
PG_RESP_BIND_COMPLETE = '2'
|
||||
PG_RESP_CLOSE_COMPLETE = '3'
|
||||
PG_RESP_ERROR = 'E'
|
||||
PG_RESP_NOTICE = 'N'
|
||||
|
||||
// Transaction states
|
||||
PG_TRANS_IDLE = 'I'
|
||||
PG_TRANS_INTRANS = 'T'
|
||||
PG_TRANS_ERROR = 'E'
|
||||
|
||||
// Authentication methods
|
||||
AUTH_OK = 0
|
||||
AUTH_CLEAR = 3
|
||||
AUTH_MD5 = 5
|
||||
AUTH_TRUST = 10
|
||||
|
||||
// PostgreSQL data types
|
||||
PG_TYPE_BOOL = 16
|
||||
PG_TYPE_BYTEA = 17
|
||||
PG_TYPE_INT8 = 20
|
||||
PG_TYPE_INT4 = 23
|
||||
PG_TYPE_TEXT = 25
|
||||
PG_TYPE_FLOAT4 = 700
|
||||
PG_TYPE_FLOAT8 = 701
|
||||
PG_TYPE_VARCHAR = 1043
|
||||
PG_TYPE_TIMESTAMP = 1114
|
||||
PG_TYPE_JSON = 114
|
||||
PG_TYPE_JSONB = 3802
|
||||
|
||||
// Default values
|
||||
DEFAULT_POSTGRES_PORT = 5432
|
||||
)
|
||||
|
||||
// Authentication method type
|
||||
type AuthMethod int
|
||||
|
||||
const (
|
||||
AuthTrust AuthMethod = iota
|
||||
AuthPassword
|
||||
AuthMD5
|
||||
)
|
||||
|
||||
// PostgreSQL server configuration
|
||||
type PostgreSQLServerConfig struct {
|
||||
Host string
|
||||
Port int
|
||||
AuthMethod AuthMethod
|
||||
Users map[string]string
|
||||
TLSConfig *tls.Config
|
||||
MaxConns int
|
||||
IdleTimeout time.Duration
|
||||
StartupTimeout time.Duration // Timeout for client startup handshake
|
||||
Database string
|
||||
}
|
||||
|
||||
// PostgreSQL server
|
||||
type PostgreSQLServer struct {
|
||||
config *PostgreSQLServerConfig
|
||||
listener net.Listener
|
||||
sqlEngine *engine.SQLEngine
|
||||
sessions map[uint32]*PostgreSQLSession
|
||||
sessionMux sync.RWMutex
|
||||
shutdown chan struct{}
|
||||
wg sync.WaitGroup
|
||||
nextConnID uint32
|
||||
}
|
||||
|
||||
// PostgreSQL session
|
||||
type PostgreSQLSession struct {
|
||||
conn net.Conn
|
||||
reader *bufio.Reader
|
||||
writer *bufio.Writer
|
||||
authenticated bool
|
||||
username string
|
||||
database string
|
||||
parameters map[string]string
|
||||
preparedStmts map[string]*PreparedStatement
|
||||
portals map[string]*Portal
|
||||
transactionState byte
|
||||
processID uint32
|
||||
secretKey uint32
|
||||
created time.Time
|
||||
lastActivity time.Time
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
// Prepared statement
|
||||
type PreparedStatement struct {
|
||||
Name string
|
||||
Query string
|
||||
ParamTypes []uint32
|
||||
Fields []FieldDescription
|
||||
}
|
||||
|
||||
// Portal (cursor)
|
||||
type Portal struct {
|
||||
Name string
|
||||
Statement string
|
||||
Parameters [][]byte
|
||||
Suspended bool
|
||||
}
|
||||
|
||||
// Field description
|
||||
type FieldDescription struct {
|
||||
Name string
|
||||
TableOID uint32
|
||||
AttrNum int16
|
||||
TypeOID uint32
|
||||
TypeSize int16
|
||||
TypeMod int32
|
||||
Format int16
|
||||
}
|
||||
|
||||
// NewPostgreSQLServer creates a new PostgreSQL protocol server
|
||||
func NewPostgreSQLServer(config *PostgreSQLServerConfig, masterAddr string) (*PostgreSQLServer, error) {
|
||||
if config.Port <= 0 {
|
||||
config.Port = DEFAULT_POSTGRES_PORT
|
||||
}
|
||||
if config.Host == "" {
|
||||
config.Host = "localhost"
|
||||
}
|
||||
if config.Database == "" {
|
||||
config.Database = "default"
|
||||
}
|
||||
if config.MaxConns <= 0 {
|
||||
config.MaxConns = 100
|
||||
}
|
||||
if config.IdleTimeout <= 0 {
|
||||
config.IdleTimeout = time.Hour
|
||||
}
|
||||
if config.StartupTimeout <= 0 {
|
||||
config.StartupTimeout = 30 * time.Second
|
||||
}
|
||||
|
||||
// Create SQL engine (now uses CockroachDB parser for PostgreSQL compatibility)
|
||||
sqlEngine := engine.NewSQLEngine(masterAddr)
|
||||
|
||||
server := &PostgreSQLServer{
|
||||
config: config,
|
||||
sqlEngine: sqlEngine,
|
||||
sessions: make(map[uint32]*PostgreSQLSession),
|
||||
shutdown: make(chan struct{}),
|
||||
nextConnID: 1,
|
||||
}
|
||||
|
||||
return server, nil
|
||||
}
|
||||
|
||||
// Start begins listening for PostgreSQL connections
|
||||
func (s *PostgreSQLServer) Start() error {
|
||||
addr := fmt.Sprintf("%s:%d", s.config.Host, s.config.Port)
|
||||
|
||||
var listener net.Listener
|
||||
var err error
|
||||
|
||||
if s.config.TLSConfig != nil {
|
||||
listener, err = tls.Listen("tcp", addr, s.config.TLSConfig)
|
||||
glog.Infof("PostgreSQL Server with TLS listening on %s", addr)
|
||||
} else {
|
||||
listener, err = net.Listen("tcp", addr)
|
||||
glog.Infof("PostgreSQL Server listening on %s", addr)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start PostgreSQL server on %s: %v", addr, err)
|
||||
}
|
||||
|
||||
s.listener = listener
|
||||
|
||||
// Start accepting connections
|
||||
s.wg.Add(1)
|
||||
go s.acceptConnections()
|
||||
|
||||
// Start cleanup routine
|
||||
s.wg.Add(1)
|
||||
go s.cleanupSessions()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the PostgreSQL server
|
||||
func (s *PostgreSQLServer) Stop() error {
|
||||
close(s.shutdown)
|
||||
|
||||
if s.listener != nil {
|
||||
s.listener.Close()
|
||||
}
|
||||
|
||||
// Close all sessions
|
||||
s.sessionMux.Lock()
|
||||
for _, session := range s.sessions {
|
||||
session.close()
|
||||
}
|
||||
s.sessions = make(map[uint32]*PostgreSQLSession)
|
||||
s.sessionMux.Unlock()
|
||||
|
||||
s.wg.Wait()
|
||||
glog.Infof("PostgreSQL Server stopped")
|
||||
return nil
|
||||
}
|
||||
|
||||
// acceptConnections handles incoming PostgreSQL connections
|
||||
func (s *PostgreSQLServer) acceptConnections() {
|
||||
defer s.wg.Done()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-s.shutdown:
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
conn, err := s.listener.Accept()
|
||||
if err != nil {
|
||||
select {
|
||||
case <-s.shutdown:
|
||||
return
|
||||
default:
|
||||
glog.Errorf("Failed to accept PostgreSQL connection: %v", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Check connection limit
|
||||
s.sessionMux.RLock()
|
||||
sessionCount := len(s.sessions)
|
||||
s.sessionMux.RUnlock()
|
||||
|
||||
if sessionCount >= s.config.MaxConns {
|
||||
glog.Warningf("Maximum connections reached (%d), rejecting connection from %s",
|
||||
s.config.MaxConns, conn.RemoteAddr())
|
||||
conn.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
s.wg.Add(1)
|
||||
go s.handleConnection(conn)
|
||||
}
|
||||
}
|
||||
|
||||
// handleConnection processes a single PostgreSQL connection
|
||||
func (s *PostgreSQLServer) handleConnection(conn net.Conn) {
|
||||
defer s.wg.Done()
|
||||
defer conn.Close()
|
||||
|
||||
// Generate unique connection ID
|
||||
connID := s.generateConnectionID()
|
||||
secretKey, err := s.generateSecretKey()
|
||||
if err != nil {
|
||||
glog.Errorf("handleConnection: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Create session
|
||||
session := &PostgreSQLSession{
|
||||
conn: conn,
|
||||
reader: bufio.NewReader(conn),
|
||||
writer: bufio.NewWriter(conn),
|
||||
authenticated: false,
|
||||
database: s.config.Database,
|
||||
parameters: make(map[string]string),
|
||||
preparedStmts: make(map[string]*PreparedStatement),
|
||||
portals: make(map[string]*Portal),
|
||||
transactionState: PG_TRANS_IDLE,
|
||||
processID: connID,
|
||||
secretKey: secretKey,
|
||||
created: time.Now(),
|
||||
lastActivity: time.Now(),
|
||||
}
|
||||
|
||||
// Register session
|
||||
s.sessionMux.Lock()
|
||||
s.sessions[connID] = session
|
||||
s.sessionMux.Unlock()
|
||||
|
||||
// Clean up on exit
|
||||
defer func() {
|
||||
s.sessionMux.Lock()
|
||||
delete(s.sessions, connID)
|
||||
s.sessionMux.Unlock()
|
||||
}()
|
||||
|
||||
glog.V(2).Infof("New PostgreSQL connection from %s (ID: %d)", conn.RemoteAddr(), connID)
|
||||
|
||||
// Handle startup
|
||||
err = s.handleStartup(session)
|
||||
if err != nil {
|
||||
// Handle common disconnection scenarios more gracefully
|
||||
if strings.Contains(err.Error(), "client disconnected") {
|
||||
glog.V(1).Infof("Client startup disconnected from %s (ID: %d): %v", conn.RemoteAddr(), connID, err)
|
||||
} else if strings.Contains(err.Error(), "timeout") {
|
||||
glog.Warningf("Startup timeout for connection %d from %s: %v", connID, conn.RemoteAddr(), err)
|
||||
} else {
|
||||
glog.Errorf("Startup failed for connection %d from %s: %v", connID, conn.RemoteAddr(), err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Handle messages
|
||||
for {
|
||||
select {
|
||||
case <-s.shutdown:
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
// Set read timeout
|
||||
conn.SetReadDeadline(time.Now().Add(30 * time.Second))
|
||||
|
||||
err := s.handleMessage(session)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
glog.Infof("PostgreSQL client disconnected (ID: %d)", connID)
|
||||
} else {
|
||||
glog.Errorf("Error handling PostgreSQL message (ID: %d): %v", connID, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
session.lastActivity = time.Now()
|
||||
}
|
||||
}
|
||||
|
||||
// handleStartup processes the PostgreSQL startup sequence
|
||||
func (s *PostgreSQLServer) handleStartup(session *PostgreSQLSession) error {
|
||||
// Set a startup timeout to prevent hanging connections
|
||||
startupTimeout := s.config.StartupTimeout
|
||||
session.conn.SetReadDeadline(time.Now().Add(startupTimeout))
|
||||
defer session.conn.SetReadDeadline(time.Time{}) // Clear timeout
|
||||
|
||||
for {
|
||||
// Read startup message length
|
||||
length := make([]byte, 4)
|
||||
_, err := io.ReadFull(session.reader, length)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
// Client disconnected during startup - this is common for health checks
|
||||
return fmt.Errorf("client disconnected during startup handshake")
|
||||
}
|
||||
if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
|
||||
return fmt.Errorf("startup handshake timeout after %v", startupTimeout)
|
||||
}
|
||||
return fmt.Errorf("failed to read message length during startup: %v", err)
|
||||
}
|
||||
|
||||
msgTotalLen := binary.BigEndian.Uint32(length)
|
||||
// Prevent unsigned underflow and OOM by checking total message length first
|
||||
if msgTotalLen < 8 {
|
||||
return fmt.Errorf("startup message too short: %d bytes", msgTotalLen)
|
||||
}
|
||||
|
||||
msgLength := msgTotalLen - 4
|
||||
if msgLength > maxStartupMessageSize {
|
||||
return fmt.Errorf("startup message too large: %d bytes", msgLength)
|
||||
}
|
||||
|
||||
// Read startup message content
|
||||
msg := make([]byte, msgLength)
|
||||
_, err = io.ReadFull(session.reader, msg)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return fmt.Errorf("client disconnected while reading startup message")
|
||||
}
|
||||
if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
|
||||
return fmt.Errorf("startup message read timeout")
|
||||
}
|
||||
return fmt.Errorf("failed to read startup message: %v", err)
|
||||
}
|
||||
|
||||
// Parse protocol version (msgTotalLen >= 8 above guarantees msgLength >= 4)
|
||||
protocolVersion := binary.BigEndian.Uint32(msg[0:4])
|
||||
|
||||
switch protocolVersion {
|
||||
case PG_SSL_REQUEST:
|
||||
// Reject SSL request - send 'N' to indicate SSL not supported
|
||||
_, err = session.conn.Write([]byte{'N'})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to reject SSL request: %v", err)
|
||||
}
|
||||
// Continue loop to read the actual startup message
|
||||
continue
|
||||
|
||||
case PG_GSSAPI_REQUEST:
|
||||
// Reject GSSAPI request - send 'N' to indicate GSSAPI not supported
|
||||
_, err = session.conn.Write([]byte{'N'})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to reject GSSAPI request: %v", err)
|
||||
}
|
||||
// Continue loop to read the actual startup message
|
||||
continue
|
||||
|
||||
case PG_PROTOCOL_VERSION_3:
|
||||
// This is the actual startup message, break out of loop
|
||||
break
|
||||
|
||||
default:
|
||||
return fmt.Errorf("unsupported protocol version: %d", protocolVersion)
|
||||
}
|
||||
|
||||
// Parse parameters
|
||||
params := strings.Split(string(msg[4:]), "\x00")
|
||||
for i := 0; i < len(params)-1; i += 2 {
|
||||
if params[i] == "user" {
|
||||
session.username = params[i+1]
|
||||
} else if params[i] == "database" {
|
||||
session.database = params[i+1]
|
||||
}
|
||||
session.parameters[params[i]] = params[i+1]
|
||||
}
|
||||
|
||||
// Break out of the main loop - we have the startup message
|
||||
break
|
||||
}
|
||||
|
||||
// Handle authentication
|
||||
err := s.handleAuthentication(session)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Send parameter status messages
|
||||
err = s.sendParameterStatus(session, "server_version", fmt.Sprintf("%s (SeaweedFS)", version.VERSION_NUMBER))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = s.sendParameterStatus(session, "server_encoding", "UTF8")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = s.sendParameterStatus(session, "client_encoding", "UTF8")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = s.sendParameterStatus(session, "DateStyle", "ISO, MDY")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = s.sendParameterStatus(session, "integer_datetimes", "on")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Send backend key data
|
||||
err = s.sendBackendKeyData(session)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Send ready for query
|
||||
err = s.sendReadyForQuery(session)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
session.authenticated = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleAuthentication processes authentication
|
||||
func (s *PostgreSQLServer) handleAuthentication(session *PostgreSQLSession) error {
|
||||
switch s.config.AuthMethod {
|
||||
case AuthTrust:
|
||||
return s.sendAuthenticationOk(session)
|
||||
case AuthPassword:
|
||||
return s.handlePasswordAuth(session)
|
||||
case AuthMD5:
|
||||
return s.handleMD5Auth(session)
|
||||
default:
|
||||
return fmt.Errorf("unsupported authentication method")
|
||||
}
|
||||
}
|
||||
|
||||
// sendAuthenticationOk sends authentication OK message
|
||||
func (s *PostgreSQLServer) sendAuthenticationOk(session *PostgreSQLSession) error {
|
||||
msg := make([]byte, 9)
|
||||
msg[0] = PG_RESP_AUTH_OK
|
||||
binary.BigEndian.PutUint32(msg[1:5], 8)
|
||||
binary.BigEndian.PutUint32(msg[5:9], AUTH_OK)
|
||||
|
||||
_, err := session.writer.Write(msg)
|
||||
if err == nil {
|
||||
err = session.writer.Flush()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// handlePasswordAuth handles clear password authentication
|
||||
func (s *PostgreSQLServer) handlePasswordAuth(session *PostgreSQLSession) error {
|
||||
// Send password request
|
||||
msg := make([]byte, 9)
|
||||
msg[0] = PG_RESP_AUTH_OK
|
||||
binary.BigEndian.PutUint32(msg[1:5], 8)
|
||||
binary.BigEndian.PutUint32(msg[5:9], AUTH_CLEAR)
|
||||
|
||||
_, err := session.writer.Write(msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = session.writer.Flush()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Read password response
|
||||
msgType := make([]byte, 1)
|
||||
_, err = io.ReadFull(session.reader, msgType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if msgType[0] != PG_MSG_PASSWORD {
|
||||
return fmt.Errorf("expected password message, got %c", msgType[0])
|
||||
}
|
||||
|
||||
length := make([]byte, 4)
|
||||
_, err = io.ReadFull(session.reader, length)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rawLength := binary.BigEndian.Uint32(length)
|
||||
if rawLength < 4 {
|
||||
return fmt.Errorf("password message too short: %d bytes", rawLength)
|
||||
}
|
||||
msgLength := rawLength - 4
|
||||
if msgLength > maxAuthMessageSize {
|
||||
return fmt.Errorf("password message too large: %d bytes", msgLength)
|
||||
}
|
||||
password := make([]byte, msgLength)
|
||||
_, err = io.ReadFull(session.reader, password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Verify password (safe null-terminator removal)
|
||||
expectedPassword, exists := s.config.Users[session.username]
|
||||
passwordStr := strings.TrimSuffix(string(password), "\x00")
|
||||
if !exists || passwordStr != expectedPassword {
|
||||
return s.sendError(session, "28P01", "authentication failed for user \""+session.username+"\"")
|
||||
}
|
||||
|
||||
return s.sendAuthenticationOk(session)
|
||||
}
|
||||
|
||||
// handleMD5Auth handles MD5 password authentication
|
||||
func (s *PostgreSQLServer) handleMD5Auth(session *PostgreSQLSession) error {
|
||||
// Generate salt
|
||||
salt := make([]byte, 4)
|
||||
_, err := rand.Read(salt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Send MD5 request
|
||||
msg := make([]byte, 13)
|
||||
msg[0] = PG_RESP_AUTH_OK
|
||||
binary.BigEndian.PutUint32(msg[1:5], 12)
|
||||
binary.BigEndian.PutUint32(msg[5:9], AUTH_MD5)
|
||||
copy(msg[9:13], salt)
|
||||
|
||||
_, err = session.writer.Write(msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = session.writer.Flush()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Read password response
|
||||
msgType := make([]byte, 1)
|
||||
_, err = io.ReadFull(session.reader, msgType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if msgType[0] != PG_MSG_PASSWORD {
|
||||
return fmt.Errorf("expected password message, got %c", msgType[0])
|
||||
}
|
||||
|
||||
length := make([]byte, 4)
|
||||
_, err = io.ReadFull(session.reader, length)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rawLength := binary.BigEndian.Uint32(length)
|
||||
if rawLength < 4 {
|
||||
return fmt.Errorf("MD5 response too short: %d bytes", rawLength)
|
||||
}
|
||||
msgLength := rawLength - 4
|
||||
if msgLength > maxAuthMessageSize {
|
||||
return fmt.Errorf("MD5 response too large: %d bytes", msgLength)
|
||||
}
|
||||
response := make([]byte, msgLength)
|
||||
_, err = io.ReadFull(session.reader, response)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Verify MD5 hash
|
||||
expectedPassword, exists := s.config.Users[session.username]
|
||||
if !exists {
|
||||
return s.sendError(session, "28P01", "authentication failed for user \""+session.username+"\"")
|
||||
}
|
||||
|
||||
// Calculate expected hash: md5(md5(password + username) + salt)
|
||||
inner := md5.Sum([]byte(expectedPassword + session.username))
|
||||
expected := fmt.Sprintf("md5%x", md5.Sum(append([]byte(fmt.Sprintf("%x", inner)), salt...)))
|
||||
|
||||
if strings.TrimSuffix(string(response), "\x00") != expected {
|
||||
return s.sendError(session, "28P01", "authentication failed for user \""+session.username+"\"")
|
||||
}
|
||||
|
||||
return s.sendAuthenticationOk(session)
|
||||
}
|
||||
|
||||
// generateConnectionID generates a unique connection ID
|
||||
func (s *PostgreSQLServer) generateConnectionID() uint32 {
|
||||
s.sessionMux.Lock()
|
||||
defer s.sessionMux.Unlock()
|
||||
id := s.nextConnID
|
||||
s.nextConnID++
|
||||
return id
|
||||
}
|
||||
|
||||
// generateSecretKey generates a secret key for the connection
|
||||
func (s *PostgreSQLServer) generateSecretKey() (uint32, error) {
|
||||
key := make([]byte, 4)
|
||||
if _, err := rand.Read(key); err != nil {
|
||||
return 0, fmt.Errorf("generate secret key: %w", err)
|
||||
}
|
||||
return binary.BigEndian.Uint32(key), nil
|
||||
}
|
||||
|
||||
// close marks the session as closed
|
||||
func (s *PostgreSQLSession) close() {
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
if s.conn != nil {
|
||||
s.conn.Close()
|
||||
s.conn = nil
|
||||
}
|
||||
}
|
||||
|
||||
// cleanupSessions periodically cleans up idle sessions
|
||||
func (s *PostgreSQLServer) cleanupSessions() {
|
||||
defer s.wg.Done()
|
||||
|
||||
ticker := time.NewTicker(time.Minute)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-s.shutdown:
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.cleanupIdleSessions()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cleanupIdleSessions removes sessions that have been idle too long
|
||||
func (s *PostgreSQLServer) cleanupIdleSessions() {
|
||||
now := time.Now()
|
||||
|
||||
s.sessionMux.Lock()
|
||||
defer s.sessionMux.Unlock()
|
||||
|
||||
for id, session := range s.sessions {
|
||||
if now.Sub(session.lastActivity) > s.config.IdleTimeout {
|
||||
glog.Infof("Closing idle PostgreSQL session %d", id)
|
||||
session.close()
|
||||
delete(s.sessions, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user