snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 03:23:42 +05:00
commit 89f5be005d
3997 changed files with 961970 additions and 0 deletions

191
weed/glog/LICENSE Normal file
View File

@@ -0,0 +1,191 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction, and
distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by the copyright
owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all other entities
that control, are controlled by, or are under common control with that entity.
For the purposes of this definition, "control" means (i) the power, direct or
indirect, to cause the direction or management of such entity, whether by
contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity exercising
permissions granted by this License.
"Source" form shall mean the preferred form for making modifications, including
but not limited to software source code, documentation source, and configuration
files.
"Object" form shall mean any form resulting from mechanical transformation or
translation of a Source form, including but not limited to compiled object code,
generated documentation, and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or Object form, made
available under the License, as indicated by a copyright notice that is included
in or attached to the work (an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object form, that
is based on (or derived from) the Work and for which the editorial revisions,
annotations, elaborations, or other modifications represent, as a whole, an
original work of authorship. For the purposes of this License, Derivative Works
shall not include works that remain separable from, or merely link (or bind by
name) to the interfaces of, the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including the original version
of the Work and any modifications or additions to that Work or Derivative Works
thereof, that is intentionally submitted to Licensor for inclusion in the Work
by the copyright owner or by an individual or Legal Entity authorized to submit
on behalf of the copyright owner. For the purposes of this definition,
"submitted" means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems, and
issue tracking systems that are managed by, or on behalf of, the Licensor for
the purpose of discussing and improving the Work, but excluding communication
that is conspicuously marked or otherwise designated in writing by the copyright
owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
of whom a Contribution has been received by Licensor and subsequently
incorporated within the Work.
2. Grant of Copyright License.
Subject to the terms and conditions of this License, each Contributor hereby
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
irrevocable copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the Work and such
Derivative Works in Source or Object form.
3. Grant of Patent License.
Subject to the terms and conditions of this License, each Contributor hereby
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to make, have
made, use, offer to sell, sell, import, and otherwise transfer the Work, where
such license applies only to those patent claims licensable by such Contributor
that are necessarily infringed by their Contribution(s) alone or by combination
of their Contribution(s) with the Work to which such Contribution(s) was
submitted. If You institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work or a
Contribution incorporated within the Work constitutes direct or contributory
patent infringement, then any patent licenses granted to You under this License
for that Work shall terminate as of the date such litigation is filed.
4. Redistribution.
You may reproduce and distribute copies of the Work or Derivative Works thereof
in any medium, with or without modifications, and in Source or Object form,
provided that You meet the following conditions:
You must give any other recipients of the Work or Derivative Works a copy of
this License; and
You must cause any modified files to carry prominent notices stating that You
changed the files; and
You must retain, in the Source form of any Derivative Works that You distribute,
all copyright, patent, trademark, and attribution notices from the Source form
of the Work, excluding those notices that do not pertain to any part of the
Derivative Works; and
If the Work includes a "NOTICE" text file as part of its distribution, then any
Derivative Works that You distribute must include a readable copy of the
attribution notices contained within such NOTICE file, excluding those notices
that do not pertain to any part of the Derivative Works, in at least one of the
following places: within a NOTICE text file distributed as part of the
Derivative Works; within the Source form or documentation, if provided along
with the Derivative Works; or, within a display generated by the Derivative
Works, if and wherever such third-party notices normally appear. The contents of
the NOTICE file are for informational purposes only and do not modify the
License. You may add Your own attribution notices within Derivative Works that
You distribute, alongside or as an addendum to the NOTICE text from the Work,
provided that such additional attribution notices cannot be construed as
modifying the License.
You may add Your own copyright statement to Your modifications and may provide
additional or different license terms and conditions for use, reproduction, or
distribution of Your modifications, or for any such Derivative Works as a whole,
provided Your use, reproduction, and distribution of the Work otherwise complies
with the conditions stated in this License.
5. Submission of Contributions.
Unless You explicitly state otherwise, any Contribution intentionally submitted
for inclusion in the Work by You to the Licensor shall be under the terms and
conditions of this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify the terms of
any separate license agreement you may have executed with Licensor regarding
such Contributions.
6. Trademarks.
This License does not grant permission to use the trade names, trademarks,
service marks, or product names of the Licensor, except as required for
reasonable and customary use in describing the origin of the Work and
reproducing the content of the NOTICE file.
7. Disclaimer of Warranty.
Unless required by applicable law or agreed to in writing, Licensor provides the
Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
including, without limitation, any warranties or conditions of TITLE,
NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
solely responsible for determining the appropriateness of using or
redistributing the Work and assume any risks associated with Your exercise of
permissions under this License.
8. Limitation of Liability.
In no event and under no legal theory, whether in tort (including negligence),
contract, or otherwise, unless required by applicable law (such as deliberate
and grossly negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special, incidental,
or consequential damages of any character arising as a result of this License or
out of the use or inability to use the Work (including but not limited to
damages for loss of goodwill, work stoppage, computer failure or malfunction, or
any and all other commercial damages or losses), even if such Contributor has
been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability.
While redistributing the Work or Derivative Works thereof, You may choose to
offer, and charge a fee for, acceptance of support, warranty, indemnity, or
other liability obligations and/or rights consistent with this License. However,
in accepting such obligations, You may act only on Your own behalf and on Your
sole responsibility, not on behalf of any other Contributor, and only if You
agree to indemnify, defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason of your
accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work
To apply the Apache License to your work, attach the following boilerplate
notice, with the fields enclosed by brackets "[]" replaced with your own
identifying information. (Don't include the brackets!) The text should be
enclosed in the appropriate comment syntax for the file format. We also
recommend that a file or class name and description of purpose be included on
the same "printed page" as the copyright notice for easier identification within
third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

44
weed/glog/README Normal file
View File

@@ -0,0 +1,44 @@
glog
====
Leveled execution logs for Go.
This is an efficient pure Go implementation of leveled logs in the
manner of the open source C++ package
http://code.google.com/p/google-glog
By binding methods to booleans it is possible to use the log package
without paying the expense of evaluating the arguments to the log.
Through the -vmodule flag, the package also provides fine-grained
control over logging at the file level.
The comment from glog.go introduces the ideas:
Package glog implements logging analogous to the Google-internal
C++ INFO/ERROR/V setup. It provides functions Info, Warning,
Error, Fatal, plus formatting variants such as Infof. It
also provides V-style logging controlled by the -v and
-vmodule=file=2 flags.
Basic examples:
glog.Info("Prepare to repel boarders")
glog.Fatalf("Initialization failed: %s", err)
See the documentation for the V function for an explanation
of these examples:
if glog.V(2) {
glog.Info("Starting transaction...")
}
glog.V(2).Infoln("Processed", nItems, "elements")
The repository contains an open source version of the log package
used inside Google. The master copy of the source lives inside
Google, not here. The code in this repo is for export only and is not itself
under development. Feature requests will be ignored.
Send bug reports to golang-nuts@googlegroups.com.

View File

@@ -0,0 +1,6 @@
package glog
/*
Copying the original glog because it is missing several convenient methods.
1. remove nano time in log format
*/

1274
weed/glog/glog.go Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,87 @@
package glog
import (
"compress/gzip"
"io"
"os"
"strings"
"sync/atomic"
)
// CompressRotated controls whether rotated log files are gzip-compressed.
// 0 = disabled (default), 1 = enabled.
var compressRotated int32
// SetCompressRotated enables or disables gzip compression of rotated log files.
// When enabled, after a log file is rotated the old file is compressed in a
// background goroutine (non-blocking to the logging path).
func SetCompressRotated(enabled bool) {
if enabled {
atomic.StoreInt32(&compressRotated, 1)
} else {
atomic.StoreInt32(&compressRotated, 0)
}
}
// IsCompressRotated returns whether compression of rotated files is active.
func IsCompressRotated() bool {
return atomic.LoadInt32(&compressRotated) == 1
}
// compressFile compresses the given file with gzip and removes the original.
// This runs in a background goroutine to avoid blocking the log write path.
// If any step fails the original file is kept intact.
func compressFile(path string) {
// Skip if already compressed
if strings.HasSuffix(path, ".gz") {
return
}
src, err := os.Open(path)
if err != nil {
return
}
defer src.Close()
dstPath := path + ".gz"
tmpPath := dstPath + ".tmp"
dst, err := os.Create(tmpPath)
if err != nil {
return
}
var success bool
defer func() {
dst.Close()
if !success {
os.Remove(tmpPath)
}
}()
gz, err := gzip.NewWriterLevel(dst, gzip.BestSpeed)
if err != nil {
return
}
if _, err = io.Copy(gz, src); err != nil {
gz.Close()
return
}
if err = gz.Close(); err != nil {
return
}
if err = dst.Close(); err != nil {
return
}
if err = os.Rename(tmpPath, dstPath); err != nil {
return
}
success = true
// Only remove original after successful compression
os.Remove(path)
}

View File

@@ -0,0 +1,151 @@
package glog
import (
"compress/gzip"
"io"
"os"
"path/filepath"
"strings"
"testing"
)
func TestCompressRotated_Toggle(t *testing.T) {
if IsCompressRotated() {
t.Error("compression should be off by default")
}
SetCompressRotated(true)
if !IsCompressRotated() {
t.Error("compression should be on after SetCompressRotated(true)")
}
SetCompressRotated(false)
if IsCompressRotated() {
t.Error("compression should be off after SetCompressRotated(false)")
}
}
func TestCompressFile(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "test.log")
content := "line 1\nline 2\nline 3\n"
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
t.Fatalf("failed to write test file: %v", err)
}
compressFile(path)
// Original should be deleted
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Error("original file should be deleted after compression")
}
// .gz should exist
gzPath := path + ".gz"
if _, err := os.Stat(gzPath); err != nil {
t.Fatalf(".gz file should exist: %v", err)
}
// Verify .gz content
f, err := os.Open(gzPath)
if err != nil {
t.Fatalf("failed to open .gz: %v", err)
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
t.Fatalf("failed to create gzip reader: %v", err)
}
defer gz.Close()
decompressed, err := io.ReadAll(gz)
if err != nil {
t.Fatalf("failed to decompress: %v", err)
}
if string(decompressed) != content {
t.Errorf("decompressed content mismatch: got %q, want %q", string(decompressed), content)
}
}
func TestCompressFile_AlreadyGz(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "test.log.gz")
if err := os.WriteFile(path, []byte("fake gz"), 0644); err != nil {
t.Fatal(err)
}
// Should skip .gz files
compressFile(path)
// File should still exist unchanged
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if string(data) != "fake gz" {
t.Error("should not modify .gz files")
}
// No .gz.gz should exist
if _, err := os.Stat(path + ".gz"); !os.IsNotExist(err) {
t.Error(".gz.gz should not be created")
}
}
func TestCompressFile_NonExistent(t *testing.T) {
// Should not panic on missing file
missing := filepath.Join(t.TempDir(), "does-not-exist.log")
compressFile(missing)
}
func TestCompressFile_EmptyFile(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "empty.log")
if err := os.WriteFile(path, []byte{}, 0644); err != nil {
t.Fatal(err)
}
compressFile(path)
// .gz should exist even for empty files
gzPath := path + ".gz"
if _, err := os.Stat(gzPath); err != nil {
t.Errorf(".gz should exist for empty file: %v", err)
}
// Original should be removed
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Error("original empty file should be deleted")
}
}
func TestCompressFile_LargeContent(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "large.log")
// Write 1MB of repetitive content (compresses well)
content := strings.Repeat("I0319 12:00:00.000000 12345 server.go:42] repeated log line\n", 15000)
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
t.Fatal(err)
}
origSize := int64(len(content))
compressFile(path)
gzPath := path + ".gz"
info, err := os.Stat(gzPath)
if err != nil {
t.Fatalf(".gz missing: %v", err)
}
// Compressed should be significantly smaller
ratio := float64(info.Size()) / float64(origSize) * 100
if ratio > 20 {
t.Errorf("compression ratio too high: %.1f%% (expected < 20%% for repetitive logs)", ratio)
}
}

246
weed/glog/glog_ctx.go Normal file
View File

@@ -0,0 +1,246 @@
package glog
import (
"context"
"fmt"
"sync/atomic"
reqid "github.com/seaweedfs/seaweedfs/weed/util/request_id"
)
const requestIDField = "request_id"
// formatMetaTag returns a formatted request ID tag from the context,
// like "request_id:abc123". Returns an empty string if no request ID is found.
func formatMetaTag(ctx context.Context) string {
if requestID := reqid.Get(ctx); requestID != "" {
return fmt.Sprintf("%s:%s", requestIDField, requestID)
}
return ""
}
// InfoCtx is a context-aware alternative to Verbose.Info.
// Logs to the INFO log, guarded by the value of v, and prepends a request ID from the context if present.
// Arguments are handled in the manner of fmt.Print.
func (v Verbose) InfoCtx(ctx context.Context, args ...interface{}) {
if !v {
return
}
if metaTag := formatMetaTag(ctx); metaTag != "" {
args = append([]interface{}{metaTag}, args...)
}
logging.print(infoLog, args...)
}
// InfolnCtx is a context-aware alternative to Verbose.Infoln.
// Logs to the INFO log, prepending a request ID from the context if it exists.
// Arguments are handled in the manner of fmt.Println.
func (v Verbose) InfolnCtx(ctx context.Context, args ...interface{}) {
if !v {
return
}
if metaTag := formatMetaTag(ctx); metaTag != "" {
args = append([]interface{}{metaTag}, args...)
}
logging.println(infoLog, args...)
}
// InfofCtx is a context-aware alternative to Verbose.Infof.
// Logs to the INFO log, guarded by the value of v, and prepends a request ID from the context if present.
// Arguments are handled in the manner of fmt.Printf.
func (v Verbose) InfofCtx(ctx context.Context, format string, args ...interface{}) {
if !v {
return
}
if metaTag := formatMetaTag(ctx); metaTag != "" {
format = metaTag + " " + format
}
logging.printf(infoLog, format, args...)
}
// InfofCtx logs a formatted message at info level, prepending a request ID from
// the context if it exists. This is a context-aware alternative to Infof.
func InfofCtx(ctx context.Context, format string, args ...interface{}) {
if metaTag := formatMetaTag(ctx); metaTag != "" {
format = metaTag + " " + format
}
logging.printf(infoLog, format, args...)
}
// InfoCtx logs a message at info level, prepending a request ID from the context
// if it exists. This is a context-aware alternative to Info.
func InfoCtx(ctx context.Context, args ...interface{}) {
if metaTag := formatMetaTag(ctx); metaTag != "" {
args = append([]interface{}{metaTag}, args...)
}
logging.print(infoLog, args...)
}
// WarningCtx logs to the WARNING and INFO logs.
// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Print.
// This is a context-aware alternative to Warning.
func WarningCtx(ctx context.Context, args ...interface{}) {
if metaTag := formatMetaTag(ctx); metaTag != "" {
args = append([]interface{}{metaTag}, args...)
}
logging.print(warningLog, args...)
}
// WarningDepthCtx logs to the WARNING and INFO logs with a custom call depth.
// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Print.
// This is a context-aware alternative to WarningDepth.
func WarningDepthCtx(ctx context.Context, depth int, args ...interface{}) {
if metaTag := formatMetaTag(ctx); metaTag != "" {
args = append([]interface{}{metaTag}, args...)
}
logging.printDepth(warningLog, depth, args...)
}
// WarninglnCtx logs to the WARNING and INFO logs.
// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Println.
// This is a context-aware alternative to Warningln.
func WarninglnCtx(ctx context.Context, args ...interface{}) {
if metaTag := formatMetaTag(ctx); metaTag != "" {
args = append([]interface{}{metaTag}, args...)
}
logging.println(warningLog, args...)
}
// WarningfCtx logs to the WARNING and INFO logs.
// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Printf.
// This is a context-aware alternative to Warningf.
func WarningfCtx(ctx context.Context, format string, args ...interface{}) {
if metaTag := formatMetaTag(ctx); metaTag != "" {
format = metaTag + " " + format
}
logging.printf(warningLog, format, args...)
}
// ErrorCtx logs to the ERROR, WARNING, and INFO logs.
// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Print.
// This is a context-aware alternative to Error.
func ErrorCtx(ctx context.Context, args ...interface{}) {
if metaTag := formatMetaTag(ctx); metaTag != "" {
args = append([]interface{}{metaTag}, args...)
}
logging.print(errorLog, args...)
}
// ErrorDepthCtx logs to the ERROR, WARNING, and INFO logs with a custom call depth.
// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Print.
// This is a context-aware alternative to ErrorDepth.
func ErrorDepthCtx(ctx context.Context, depth int, args ...interface{}) {
if metaTag := formatMetaTag(ctx); metaTag != "" {
args = append([]interface{}{metaTag}, args...)
}
logging.printDepth(errorLog, depth, args...)
}
// ErrorlnCtx logs to the ERROR, WARNING, and INFO logs.
// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Println.
// This is a context-aware alternative to Errorln.
func ErrorlnCtx(ctx context.Context, args ...interface{}) {
if metaTag := formatMetaTag(ctx); metaTag != "" {
args = append([]interface{}{metaTag}, args...)
}
logging.println(errorLog, args...)
}
// ErrorfCtx logs to the ERROR, WARNING, and INFO logs.
// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Printf.
// This is a context-aware alternative to Errorf.
func ErrorfCtx(ctx context.Context, format string, args ...interface{}) {
if metaTag := formatMetaTag(ctx); metaTag != "" {
format = metaTag + " " + format
}
logging.printf(errorLog, format, args...)
}
// FatalCtx logs to the FATAL, ERROR, WARNING, and INFO logs,
// including a stack trace of all running goroutines, then calls os.Exit(255).
// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Print.
// This is a context-aware alternative to Fatal.
func FatalCtx(ctx context.Context, args ...interface{}) {
if metaTag := formatMetaTag(ctx); metaTag != "" {
args = append([]interface{}{metaTag}, args...)
}
logging.print(fatalLog, args...)
}
// FatalDepthCtx logs to the FATAL, ERROR, WARNING, and INFO logs with a custom call depth,
// including a stack trace of all running goroutines, then calls os.Exit(255).
// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Print.
// This is a context-aware alternative to FatalDepth.
func FatalDepthCtx(ctx context.Context, depth int, args ...interface{}) {
if metaTag := formatMetaTag(ctx); metaTag != "" {
args = append([]interface{}{metaTag}, args...)
}
logging.printDepth(fatalLog, depth, args...)
}
// FatallnCtx logs to the FATAL, ERROR, WARNING, and INFO logs,
// including a stack trace of all running goroutines, then calls os.Exit(255).
// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Println.
// This is a context-aware alternative to Fatalln.
func FatallnCtx(ctx context.Context, args ...interface{}) {
if metaTag := formatMetaTag(ctx); metaTag != "" {
args = append([]interface{}{metaTag}, args...)
}
logging.println(fatalLog, args...)
}
// FatalfCtx logs to the FATAL, ERROR, WARNING, and INFO logs,
// including a stack trace of all running goroutines, then calls os.Exit(255).
// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Printf.
// This is a context-aware alternative to Fatalf.
func FatalfCtx(ctx context.Context, format string, args ...interface{}) {
if metaTag := formatMetaTag(ctx); metaTag != "" {
format = metaTag + " " + format
}
logging.printf(fatalLog, format, args...)
}
// ExitCtx logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1).
// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Print.
// This is a context-aware alternative to ExitCtx
func ExitCtx(ctx context.Context, args ...interface{}) {
atomic.StoreUint32(&fatalNoStacks, 1)
if metaTag := formatMetaTag(ctx); metaTag != "" {
args = append([]interface{}{metaTag}, args...)
}
logging.print(fatalLog, args...)
}
// ExitDepthCtx logs to the FATAL, ERROR, WARNING, and INFO logs with a custom call depth,
// then calls os.Exit(1). Prepends a request ID from the context if it exists.
// Arguments are handled in the manner of fmt.Print.
// This is a context-aware alternative to ExitDepth.
func ExitDepthCtx(ctx context.Context, depth int, args ...interface{}) {
atomic.StoreUint32(&fatalNoStacks, 1)
if metaTag := formatMetaTag(ctx); metaTag != "" {
args = append([]interface{}{metaTag}, args...)
}
logging.printDepth(fatalLog, depth, args...)
}
// ExitlnCtx logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1).
// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Println.
// This is a context-aware alternative to Exitln.
func ExitlnCtx(ctx context.Context, args ...interface{}) {
atomic.StoreUint32(&fatalNoStacks, 1)
if metaTag := formatMetaTag(ctx); metaTag != "" {
args = append([]interface{}{metaTag}, args...)
}
logging.println(fatalLog, args...)
}
// ExitfCtx logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1).
// Prepends a request ID from the context if it exists. Arguments are handled in the manner of fmt.Printf.
// This is a context-aware alternative to Exitf.
func ExitfCtx(ctx context.Context, format string, args ...interface{}) {
atomic.StoreUint32(&fatalNoStacks, 1)
if metaTag := formatMetaTag(ctx); metaTag != "" {
format = metaTag + " " + format
}
logging.printf(fatalLog, format, args...)
}

215
weed/glog/glog_file.go Normal file
View File

@@ -0,0 +1,215 @@
// Go support for leveled logs, analogous to https://code.google.com/p/google-glog/
//
// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// File I/O for logs.
package glog
import (
"errors"
"fmt"
"os"
"os/user"
"path/filepath"
"sort"
"strings"
"sync"
"time"
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
)
// MaxSize is the maximum size of a log file in bytes.
// It is initialized from the --log_max_size_mb flag when the first log file is created.
var MaxSize uint64 = 1024 * 1024 * 1800
// MaxFileCount is the maximum number of log files retained per severity level.
// It is initialized from the --log_max_files flag when the first log file is created.
var MaxFileCount = 5
// logDirs lists the candidate directories for new log files.
var logDirs []string
// If non-empty, overrides the choice of directory in which to write logs.
// See createLogDirs for the full list of possible destinations.
var logDir = flag.String("logdir", "", "If non-empty, write log files in this directory")
// logMaxSizeMB controls the maximum size of each log file in megabytes.
// When a log file reaches this size it is closed and a new file is created.
// Defaults to 1800 MB. Set to 0 to use the compiled-in default.
var logMaxSizeMB = flag.Uint64("log_max_size_mb", 1800, "Maximum size in megabytes of each log file before it is rotated (0 = use default of 1800 MB)")
// logMaxFiles controls how many log files are kept per severity level.
// Older files are deleted when the limit is exceeded.
// Defaults to 5.
var logMaxFiles = flag.Int("log_max_files", 5, "Maximum number of log files to keep per severity level before older ones are deleted (0 = use default of 5)")
// logRotateHours controls time-based log rotation.
// When non-zero, each log file is rotated after the given number of hours
// regardless of its size. This prevents log files from accumulating in
// long-running deployments even when log volume is low.
// The default is 168 hours (7 days). Set to 0 to disable time-based rotation.
var logRotateHours = flag.Int("log_rotate_hours", 168, "Rotate log files after this many hours (default: 168 = 7 days, 0 = disabled)")
// logJSON enables JSON-formatted log output (one JSON object per line).
// Useful for integration with ELK, Loki, Datadog, and other log aggregation systems.
var logJSON = flag.Bool("log_json", false, "Output logs in JSON format instead of glog text format")
// logCompress enables gzip compression of rotated log files.
// Compressed files get a .gz suffix. Compression runs in the background.
var logCompress = flag.Bool("log_compress", false, "Gzip-compress rotated log files to save disk space")
func createLogDirs() {
// Apply flag values now that flags have been parsed.
if *logMaxSizeMB > 0 {
MaxSize = *logMaxSizeMB * 1024 * 1024
}
if *logMaxFiles > 0 {
MaxFileCount = *logMaxFiles
}
if *logCompress {
SetCompressRotated(true)
}
if *logDir != "" {
logDirs = append(logDirs, *logDir)
} else {
logDirs = append(logDirs, os.TempDir())
}
}
// LogRotateHours returns the configured time-based rotation interval.
// This is used by syncBuffer to decide when to rotate open log files.
func LogRotateHours() int {
return *logRotateHours
}
var (
pid = os.Getpid()
program = filepath.Base(os.Args[0])
host = "unknownhost"
userName = "unknownuser"
)
func init() {
h, err := os.Hostname()
if err == nil {
host = shortHostname(h)
}
current, err := user.Current()
if err == nil {
userName = current.Username
}
// Sanitize userName since it may contain filepath separators on Windows.
userName = strings.Replace(userName, `\`, "_", -1)
}
// shortHostname returns its argument, truncating at the first period.
// For instance, given "www.google.com" it returns "www".
func shortHostname(hostname string) string {
if i := strings.Index(hostname, "."); i >= 0 {
return hostname[:i]
}
return hostname
}
// logName returns a new log file name containing tag, with start time t, and
// the name for the symlink for tag.
func logName(tag string, t time.Time) (name, link string) {
name = fmt.Sprintf("%s.%s.%s.log.%s.%04d%02d%02d-%02d%02d%02d.%d",
program,
host,
userName,
tag,
t.Year(),
t.Month(),
t.Day(),
t.Hour(),
t.Minute(),
t.Second(),
pid)
return name, program + "." + tag
}
func prefix(tag string) string {
return fmt.Sprintf("%s.%s.%s.log.%s.",
program,
host,
userName,
tag,
)
}
var onceLogDirs sync.Once
// create creates a new log file and returns the file and its filename, which
// contains tag ("INFO", "FATAL", etc.) and t. If the file is created
// successfully, create also attempts to update the symlink for that tag, ignoring
// errors.
func create(tag string, t time.Time) (f *os.File, filename string, err error) {
onceLogDirs.Do(createLogDirs)
if len(logDirs) == 0 {
return nil, "", errors.New("log: no log dirs")
}
name, link := logName(tag, t)
logPrefix := prefix(tag)
var lastErr error
for _, dir := range logDirs {
// remove old logs (including .gz compressed rotated files)
// Deduplicate .log/.log.gz pairs so concurrent compression
// doesn't cause double-counting against MaxFileCount.
entries, _ := os.ReadDir(dir)
previousLogs := make(map[string][]string) // bare name -> actual file names
for _, entry := range entries {
name := entry.Name()
bare := strings.TrimSuffix(name, ".gz")
if strings.HasPrefix(bare, logPrefix) {
previousLogs[bare] = append(previousLogs[bare], name)
}
}
if len(previousLogs) >= MaxFileCount {
var keys []string
for bare := range previousLogs {
keys = append(keys, bare)
}
sort.Strings(keys)
for i, bare := range keys {
if i > len(keys)-MaxFileCount {
break
}
for _, name := range previousLogs[bare] {
os.Remove(filepath.Join(dir, name))
}
}
}
// create new log file
fname := filepath.Join(dir, name)
f, err := os.Create(fname)
if err == nil {
symlink := filepath.Join(dir, link)
os.Remove(symlink) // ignore err
os.Symlink(name, symlink) // ignore err
return f, fname, nil
}
lastErr = err
}
return nil, "", fmt.Errorf("log: cannot create log: %w", lastErr)
}

157
weed/glog/glog_json.go Normal file
View File

@@ -0,0 +1,157 @@
package glog
import (
"fmt"
"runtime"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"unicode/utf8"
)
// JSONMode controls whether log output is in JSON format.
// 0 = classic glog text (default), 1 = JSON lines.
// Safe for concurrent access.
var jsonMode int32
// jsonFlagOnce ensures the --log_json flag is applied on first use,
// even when -logtostderr prevents createLogDirs() from running.
var jsonFlagOnce sync.Once
// SetJSONMode enables or disables JSON-formatted log output.
// When enabled, each log line is a single JSON object:
//
// {"ts":"2006-01-02T15:04:05.000000Z","level":"INFO","file":"server.go","line":42,"msg":"..."}
//
// This is useful for log aggregation systems (ELK, Loki, Datadog, etc).
func SetJSONMode(enabled bool) {
// Prevent lazy flag initialization from overriding this explicit call.
jsonFlagOnce.Do(func() {})
if enabled {
atomic.StoreInt32(&jsonMode, 1)
} else {
atomic.StoreInt32(&jsonMode, 0)
}
}
// IsJSONMode returns whether JSON mode is currently active.
// On first call, it applies the --log_json flag value if set.
func IsJSONMode() bool {
jsonFlagOnce.Do(func() {
if logJSON != nil && *logJSON {
atomic.StoreInt32(&jsonMode, 1)
}
})
return atomic.LoadInt32(&jsonMode) == 1
}
// formatJSON builds a JSON log line without using encoding/json
// to avoid allocations and keep it as fast as the text path.
// Output: {"ts":"...","level":"...","file":"...","line":N,"msg":"..."}\n
func (l *loggingT) formatJSON(s severity, depth int) (*buffer, string, int) {
_, file, line, ok := runtime.Caller(3 + depth)
if !ok {
file = "???"
line = 1
} else {
slash := strings.LastIndex(file, "/")
if slash >= 0 {
file = file[slash+1:]
}
}
buf := l.getBuffer()
now := timeNow()
buf.WriteString(`{"ts":"`)
buf.WriteString(now.UTC().Format(time.RFC3339Nano))
buf.WriteString(`","level":"`)
switch {
case s == infoLog:
buf.WriteString("INFO")
case s == warningLog:
buf.WriteString("WARNING")
case s == errorLog:
buf.WriteString("ERROR")
case s >= fatalLog:
buf.WriteString("FATAL")
}
buf.WriteString(`","file":"`)
buf.WriteString(jsonEscapeString(file))
buf.WriteString(`","line":`)
// Write line number without fmt.Sprintf
buf.WriteString(itoa(line))
buf.WriteString(`,"msg":"`)
return buf, file, line
}
// finishJSON closes the JSON object and adds a newline.
func finishJSON(buf *buffer) {
buf.WriteString("\"}\n")
}
// jsonEscapeString escapes a string for safe inclusion in JSON (RFC 8259).
// Handles: \, ", \n, \r, \t, control characters, and invalid UTF-8 sequences.
func jsonEscapeString(s string) string {
// Fast path: no special chars and valid UTF-8
needsEscape := false
for i := 0; i < len(s); i++ {
c := s[i]
if c == '"' || c == '\\' || c < 0x20 || c > 0x7e {
needsEscape = true
break
}
}
if !needsEscape {
return s
}
var b strings.Builder
b.Grow(len(s) + 10)
for i := 0; i < len(s); {
c := s[i]
switch {
case c == '"':
b.WriteString(`\"`)
i++
case c == '\\':
b.WriteString(`\\`)
i++
case c == '\n':
b.WriteString(`\n`)
i++
case c == '\r':
b.WriteString(`\r`)
i++
case c == '\t':
b.WriteString(`\t`)
i++
case c < 0x20:
fmt.Fprintf(&b, `\u%04x`, c)
i++
case c < utf8.RuneSelf:
b.WriteByte(c)
i++
default:
r, size := utf8.DecodeRuneInString(s[i:])
if r == utf8.RuneError && size == 1 {
b.WriteString(`\ufffd`)
i++
} else {
b.WriteString(s[i : i+size])
i += size
}
}
}
return b.String()
}
// itoa converts an integer to a string.
func itoa(i int) string {
return strconv.Itoa(i)
}

188
weed/glog/glog_json_test.go Normal file
View File

@@ -0,0 +1,188 @@
package glog
import (
"encoding/json"
"strings"
"testing"
"time"
)
func TestJSONMode_Toggle(t *testing.T) {
// Default is off
if IsJSONMode() {
t.Error("JSON mode should be off by default")
}
SetJSONMode(true)
if !IsJSONMode() {
t.Error("JSON mode should be on after SetJSONMode(true)")
}
SetJSONMode(false)
if IsJSONMode() {
t.Error("JSON mode should be off after SetJSONMode(false)")
}
}
func TestJSONMode_Output(t *testing.T) {
setFlags()
defer logging.swap(logging.newBuffers())
defer SetJSONMode(false)
SetJSONMode(true)
defer func(previous func() time.Time) { timeNow = previous }(timeNow)
timeNow = func() time.Time {
return time.Date(2026, 3, 19, 15, 30, 0, 0, time.UTC)
}
Info("hello json")
output := contents(infoLog)
if !strings.HasPrefix(output, "{") {
t.Fatalf("JSON output should start with '{', got: %q", output)
}
// Parse as JSON
var parsed map[string]interface{}
// Trim trailing newline
line := strings.TrimSpace(output)
if err := json.Unmarshal([]byte(line), &parsed); err != nil {
t.Fatalf("output is not valid JSON: %v\noutput: %q", err, line)
}
// Check required fields
if _, ok := parsed["ts"]; !ok {
t.Error("JSON missing 'ts' field")
}
if level, ok := parsed["level"]; !ok || level != "INFO" {
t.Errorf("expected level=INFO, got %v", level)
}
if _, ok := parsed["file"]; !ok {
t.Error("JSON missing 'file' field")
}
if _, ok := parsed["line"]; !ok {
t.Error("JSON missing 'line' field")
}
if msg, ok := parsed["msg"]; !ok || msg != "hello json" {
t.Errorf("expected msg='hello json', got %v", msg)
}
}
func TestJSONMode_AllLevels(t *testing.T) {
setFlags()
defer logging.swap(logging.newBuffers())
defer SetJSONMode(false)
SetJSONMode(true)
Info("info msg")
Warning("warn msg")
Error("error msg")
for _, sev := range []severity{infoLog, warningLog, errorLog} {
output := contents(sev)
if output == "" {
continue
}
// Each line should be valid JSON
for _, line := range strings.Split(strings.TrimSpace(output), "\n") {
if line == "" {
continue
}
var parsed map[string]interface{}
if err := json.Unmarshal([]byte(line), &parsed); err != nil {
t.Errorf("severity %d: invalid JSON: %v\nline: %q", sev, err, line)
}
}
}
}
func TestJSONMode_Infof(t *testing.T) {
setFlags()
defer logging.swap(logging.newBuffers())
defer SetJSONMode(false)
SetJSONMode(true)
Infof("count=%d name=%s", 42, "test")
output := strings.TrimSpace(contents(infoLog))
var parsed map[string]interface{}
if err := json.Unmarshal([]byte(output), &parsed); err != nil {
t.Fatalf("Infof JSON invalid: %v\noutput: %q", err, output)
}
msg := parsed["msg"].(string)
if !strings.Contains(msg, "count=42") || !strings.Contains(msg, "name=test") {
t.Errorf("Infof message wrong: %q", msg)
}
}
func TestJSONEscapeString(t *testing.T) {
tests := []struct {
input string
want string
}{
{"hello", "hello"},
{`say "hi"`, `say \"hi\"`},
{"line1\nline2", `line1\nline2`},
{"tab\there", `tab\there`},
{`back\slash`, `back\\slash`},
{"ctrl\x00char", `ctrl\u0000char`},
{"", ""},
}
for _, tt := range tests {
got := jsonEscapeString(tt.input)
if got != tt.want {
t.Errorf("jsonEscapeString(%q) = %q, want %q", tt.input, got, tt.want)
}
}
}
func TestItoa(t *testing.T) {
tests := []struct {
input int
want string
}{
{0, "0"},
{5, "5"},
{9, "9"},
{10, "10"},
{42, "42"},
{12345, "12345"},
}
for _, tt := range tests {
got := itoa(tt.input)
if got != tt.want {
t.Errorf("itoa(%d) = %q, want %q", tt.input, got, tt.want)
}
}
}
func TestJSONMode_TextFallback(t *testing.T) {
setFlags()
defer logging.swap(logging.newBuffers())
// With JSON mode OFF, output should NOT be JSON
SetJSONMode(false)
Info("text mode")
output := contents(infoLog)
if strings.HasPrefix(output, "{") {
t.Error("text mode should not produce JSON output")
}
if !strings.Contains(output, "text mode") {
t.Error("text mode output missing message")
}
}
func BenchmarkJSONMode(b *testing.B) {
setFlags()
defer logging.swap(logging.newBuffers())
defer SetJSONMode(false)
SetJSONMode(true)
b.ResetTimer()
for i := 0; i < b.N; i++ {
Info("benchmark json message")
}
}

468
weed/glog/glog_test.go Normal file
View File

@@ -0,0 +1,468 @@
// Go support for leveled logs, analogous to https://code.google.com/p/google-glog/
//
// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package glog
import (
"bytes"
"fmt"
stdLog "log"
"path/filepath"
"runtime"
"strconv"
"strings"
"testing"
"time"
)
// Test that shortHostname works as advertised.
func TestShortHostname(t *testing.T) {
for hostname, expect := range map[string]string{
"": "",
"host": "host",
"host.google.com": "host",
} {
if got := shortHostname(hostname); expect != got {
t.Errorf("shortHostname(%q): expected %q, got %q", hostname, expect, got)
}
}
}
// flushBuffer wraps a bytes.Buffer to satisfy flushSyncWriter.
type flushBuffer struct {
bytes.Buffer
}
func (f *flushBuffer) Flush() error {
return nil
}
func (f *flushBuffer) Sync() error {
return nil
}
// swap sets the log writers and returns the old array.
func (l *loggingT) swap(writers [numSeverity]flushSyncWriter) (old [numSeverity]flushSyncWriter) {
l.mu.Lock()
defer l.mu.Unlock()
old = l.file
for i, w := range writers {
logging.file[i] = w
}
return
}
// newBuffers sets the log writers to all new byte buffers and returns the old array.
func (l *loggingT) newBuffers() [numSeverity]flushSyncWriter {
return l.swap([numSeverity]flushSyncWriter{new(flushBuffer), new(flushBuffer), new(flushBuffer), new(flushBuffer)})
}
// contents returns the specified log value as a string.
func contents(s severity) string {
return logging.file[s].(*flushBuffer).String()
}
// contains reports whether the string is contained in the log.
func contains(s severity, str string, t *testing.T) bool {
return strings.Contains(contents(s), str)
}
// setFlags configures the logging flags how the test expects them.
func setFlags() {
logging.toStderr = false
}
// Test that Info works as advertised.
func TestInfo(t *testing.T) {
setFlags()
defer logging.swap(logging.newBuffers())
Info("test")
if !contains(infoLog, "I", t) {
t.Errorf("Info has wrong character: %q", contents(infoLog))
}
if !contains(infoLog, "test", t) {
t.Error("Info failed")
}
}
func TestInfoDepth(t *testing.T) {
setFlags()
defer logging.swap(logging.newBuffers())
f := func() { InfoDepth(1, "depth-test1") }
// The next three lines must stay together
_, _, wantLine, _ := runtime.Caller(0)
InfoDepth(0, "depth-test0")
f()
msgs := strings.Split(strings.TrimSuffix(contents(infoLog), "\n"), "\n")
if len(msgs) != 2 {
t.Fatalf("Got %d lines, expected 2", len(msgs))
}
for i, m := range msgs {
if !strings.HasPrefix(m, "I") {
t.Errorf("InfoDepth[%d] has wrong character: %q", i, m)
}
w := fmt.Sprintf("depth-test%d", i)
if !strings.Contains(m, w) {
t.Errorf("InfoDepth[%d] missing %q: %q", i, w, m)
}
// pull out the line number (between : and ])
msg := m[strings.LastIndex(m, ":")+1:]
x := strings.Index(msg, " ")
if x < 0 {
t.Errorf("InfoDepth[%d]: missing ' ': %q", i, m)
continue
}
line, err := strconv.Atoi(msg[:x])
if err != nil {
t.Errorf("InfoDepth[%d]: bad line number: %q", i, m)
continue
}
wantLine++
if wantLine != line {
t.Errorf("InfoDepth[%d]: got line %d, want %d", i, line, wantLine)
}
}
}
func init() {
CopyStandardLogTo("INFO")
}
// Test that CopyStandardLogTo panics on bad input.
func TestCopyStandardLogToPanic(t *testing.T) {
defer func() {
if s, ok := recover().(string); !ok || !strings.Contains(s, "LOG") {
t.Errorf(`CopyStandardLogTo("LOG") should have panicked: %v`, s)
}
}()
CopyStandardLogTo("LOG")
}
// Test that using the standard log package logs to INFO.
func TestStandardLog(t *testing.T) {
setFlags()
defer logging.swap(logging.newBuffers())
stdLog.Print("test")
if !contains(infoLog, "I", t) {
t.Errorf("Info has wrong character: %q", contents(infoLog))
}
if !contains(infoLog, "test", t) {
t.Error("Info failed")
}
}
// Test that the header has the correct format.
func TestHeader(t *testing.T) {
setFlags()
defer logging.swap(logging.newBuffers())
defer func(previous func() time.Time) { timeNow = previous }(timeNow)
timeNow = func() time.Time {
return time.Date(2006, 1, 2, 15, 4, 5, .067890e9, time.Local)
}
pid = 1234
Info("test")
var line int
format := "I0102 15:04:05.067890 glog_test.go:%d test\n"
n, err := fmt.Sscanf(contents(infoLog), format, &line)
if n != 1 || err != nil {
t.Errorf("log format error: %d elements, error %s:\n%s", n, err, contents(infoLog))
}
// Scanf treats multiple spaces as equivalent to a single space,
// so check for correct space-padding also.
want := fmt.Sprintf(format, line)
if contents(infoLog) != want {
t.Errorf("log format error: got:\n\t%q\nwant:\t%q", contents(infoLog), want)
}
}
// Test that an Error log goes to Warning and Info.
// Even in the Info log, the source character will be E, so the data should
// all be identical.
func TestError(t *testing.T) {
setFlags()
defer logging.swap(logging.newBuffers())
Error("test")
if !contains(errorLog, "E", t) {
t.Errorf("Error has wrong character: %q", contents(errorLog))
}
if !contains(errorLog, "test", t) {
t.Error("Error failed")
}
str := contents(errorLog)
if !contains(warningLog, str, t) {
t.Error("Warning failed")
}
if !contains(infoLog, str, t) {
t.Error("Info failed")
}
}
// Test that a Warning log goes to Info.
// Even in the Info log, the source character will be W, so the data should
// all be identical.
func TestWarning(t *testing.T) {
setFlags()
defer logging.swap(logging.newBuffers())
Warning("test")
if !contains(warningLog, "W", t) {
t.Errorf("Warning has wrong character: %q", contents(warningLog))
}
if !contains(warningLog, "test", t) {
t.Error("Warning failed")
}
str := contents(warningLog)
if !contains(infoLog, str, t) {
t.Error("Info failed")
}
}
// Test that a V log goes to Info.
func TestV(t *testing.T) {
setFlags()
defer logging.swap(logging.newBuffers())
logging.verbosity.Set("2")
defer logging.verbosity.Set("0")
V(2).Info("test")
if !contains(infoLog, "I", t) {
t.Errorf("Info has wrong character: %q", contents(infoLog))
}
if !contains(infoLog, "test", t) {
t.Error("Info failed")
}
}
// Test that a vmodule enables a log in this file.
func TestVmoduleOn(t *testing.T) {
setFlags()
defer logging.swap(logging.newBuffers())
logging.vmodule.Set("glog_test=2")
defer logging.vmodule.Set("")
if !V(1) {
t.Error("V not enabled for 1")
}
if !V(2) {
t.Error("V not enabled for 2")
}
if V(3) {
t.Error("V enabled for 3")
}
V(2).Info("test")
if !contains(infoLog, "I", t) {
t.Errorf("Info has wrong character: %q", contents(infoLog))
}
if !contains(infoLog, "test", t) {
t.Error("Info failed")
}
}
// Test that a vmodule of another file does not enable a log in this file.
func TestVmoduleOff(t *testing.T) {
setFlags()
defer logging.swap(logging.newBuffers())
logging.vmodule.Set("notthisfile=2")
defer logging.vmodule.Set("")
for i := 1; i <= 3; i++ {
if V(Level(i)) {
t.Errorf("V enabled for %d", i)
}
}
V(2).Info("test")
if contents(infoLog) != "" {
t.Error("V logged incorrectly")
}
}
// vGlobs are patterns that match/don't match this file at V=2.
var vGlobs = map[string]bool{
// Easy to test the numeric match here.
"glog_test=1": false, // If -vmodule sets V to 1, V(2) will fail.
"glog_test=2": true,
"glog_test=3": true, // If -vmodule sets V to 1, V(3) will succeed.
// These all use 2 and check the patterns. All are true.
"*=2": true,
"?l*=2": true,
"????_*=2": true,
"??[mno]?_*t=2": true,
// These all use 2 and check the patterns. All are false.
"*x=2": false,
"m*=2": false,
"??_*=2": false,
"?[abc]?_*t=2": false,
}
// Test that vmodule globbing works as advertised.
func testVmoduleGlob(pat string, match bool, t *testing.T) {
setFlags()
defer logging.swap(logging.newBuffers())
defer logging.vmodule.Set("")
logging.vmodule.Set(pat)
if V(2) != Verbose(match) {
t.Errorf("incorrect match for %q: got %t expected %t", pat, V(2), match)
}
}
// Test that a vmodule globbing works as advertised.
func TestVmoduleGlob(t *testing.T) {
for glob, match := range vGlobs {
testVmoduleGlob(glob, match, t)
}
}
func TestRollover(t *testing.T) {
setFlags()
var err error
defer func(previous func(error)) { logExitFunc = previous }(logExitFunc)
logExitFunc = func(e error) {
err = e
}
Info("x") // Be sure we have a file (also triggers createLogDirs via sync.Once).
// Set MaxSize after the first Info call so that createLogDirs (which
// overwrites MaxSize from the flag default) has already executed.
defer func(previous uint64) { MaxSize = previous }(MaxSize)
MaxSize = 512
info, ok := logging.file[infoLog].(*syncBuffer)
if !ok {
t.Fatal("info wasn't created")
}
if err != nil {
t.Fatalf("info has initial error: %v", err)
}
fname0 := info.file.Name()
Info(strings.Repeat("x", int(MaxSize))) // force a rollover
if err != nil {
t.Fatalf("info has error after big write: %v", err)
}
// Make sure the next log file gets a file name with a different
// time stamp.
//
// TODO: determine whether we need to support subsecond log
// rotation. C++ does not appear to handle this case (nor does it
// handle Daylight Savings Time properly).
time.Sleep(1 * time.Second)
Info("x") // create a new file
if err != nil {
t.Fatalf("error after rotation: %v", err)
}
fname1 := info.file.Name()
if fname0 == fname1 {
t.Errorf("info.f.Name did not change: %v", fname0)
}
if info.nbytes >= MaxSize {
t.Errorf("file size was not reset: %d", info.nbytes)
}
}
func TestTimeBasedRollover(t *testing.T) {
setFlags()
var err error
defer func(previous func(error)) { logExitFunc = previous }(logExitFunc)
logExitFunc = func(e error) {
err = e
}
// Disable size-based rotation by setting a very large MaxSize.
defer func(previous uint64) { MaxSize = previous }(MaxSize)
MaxSize = 1024 * 1024 * 1024
// Enable time-based rotation with a 1-hour interval.
defer func(previous int) { *logRotateHours = previous }(*logRotateHours)
*logRotateHours = 1
Info("x") // Create initial file.
info, ok := logging.file[infoLog].(*syncBuffer)
if !ok {
t.Fatal("info wasn't created")
}
if err != nil {
t.Fatalf("info has initial error: %v", err)
}
fname0 := info.file.Name()
createdAt := info.createdAt
// Mock time to 30 minutes after file creation — should NOT rotate.
defer func(previous func() time.Time) { timeNow = previous }(timeNow)
timeNow = func() time.Time { return createdAt.Add(30 * time.Minute) }
Info("still within interval")
if err != nil {
t.Fatalf("error after write within interval: %v", err)
}
if info.file.Name() != fname0 {
t.Error("file rotated before interval elapsed")
}
// Advance mock time past the 1-hour interval — should rotate.
timeNow = func() time.Time { return createdAt.Add(61 * time.Minute) }
Info("past interval")
if err != nil {
t.Fatalf("error after time-based rotation: %v", err)
}
fname1 := info.file.Name()
if fname0 == fname1 {
t.Error("file did not rotate after interval elapsed")
}
}
func TestLogBacktraceAt(t *testing.T) {
setFlags()
defer logging.swap(logging.newBuffers())
// The peculiar style of this code simplifies line counting and maintenance of the
// tracing block below.
var infoLine string
setTraceLocation := func(file string, line int, ok bool, delta int) {
if !ok {
t.Fatal("could not get file:line")
}
_, file = filepath.Split(file)
infoLine = fmt.Sprintf("%s:%d", file, line+delta)
err := logging.traceLocation.Set(infoLine)
if err != nil {
t.Fatal("error setting log_backtrace_at: ", err)
}
}
{
// Start of tracing block. These lines know about each other's relative position.
_, file, line, ok := runtime.Caller(0)
setTraceLocation(file, line, ok, +2) // Two lines between Caller and Info calls.
Info("we want a stack trace here")
}
numAppearances := strings.Count(contents(infoLog), infoLine)
if numAppearances < 2 {
// Need 2 appearances, one in the log header and one in the trace:
// log_test.go:281: I0511 16:36:06.952398 02238 log_test.go:280] we want a stack trace here
// ...
// github.com/glog/glog_test.go:280 (0x41ba91)
// ...
// We could be more precise but that would require knowing the details
// of the traceback format, which may not be dependable.
t.Fatal("got no trace back; log is ", contents(infoLog))
}
}
func BenchmarkHeader(b *testing.B) {
for i := 0; i < b.N; i++ {
buf, _, _ := logging.header(infoLog, 0)
logging.putBuffer(buf)
}
}