upstream-seaweedfs/weed/command/filer_meta_tail_elastic.go

//go:build elastic

package command

import (
	"context"
	"encoding/base64"
	"strings"
	"unicode/utf8"

	jsoniter "github.com/json-iterator/go"
	elastic "github.com/olivere/elastic/v7"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

type EsDocument struct {
	Dir         string            `json:"dir,omitempty"`
	Name        string            `json:"name,omitempty"`
	IsDirectory bool              `json:"isDir,omitempty"`
	Size        uint64            `json:"size,omitempty"`
	Uid         uint32            `json:"uid,omitempty"`
	Gid         uint32            `json:"gid,omitempty"`
	UserName    string            `json:"userName,omitempty"`
	Crtime      int64             `json:"crtime,omitempty"`
	Mtime       int64             `json:"mtime,omitempty"`
	Mime        string            `json:"mime,omitempty"`
	Extended    map[string]string `json:"extended,omitempty"`
}

func toEsEntry(event *filer_pb.EventNotification) (*EsDocument, string) {
	entry := event.NewEntry
	dir, name := event.NewParentPath, entry.Name
	id := util.Md5String([]byte(util.NewFullPath(dir, name)))
	esEntry := &EsDocument{
		Dir:         dir,
		Name:        name,
		IsDirectory: entry.IsDirectory,
		Size:        entry.Attributes.FileSize,
		Uid:         entry.Attributes.Uid,
		Gid:         entry.Attributes.Gid,
		UserName:    entry.Attributes.UserName,
		Crtime:      entry.Attributes.Crtime,
		Mtime:       entry.Attributes.Mtime,
		Mime:        entry.Attributes.Mime,
		Extended:    toExtendedStrings(entry.Extended),
	}
	return esEntry, id
}

// toExtendedStrings converts the xattr map (e.g. S3 user metadata like
// X-Amz-Meta-*) to string values so Elasticsearch indexes them as text
// instead of base64-encoding the raw bytes. Non-UTF-8 values are prefixed
// with "base64:" so consumers can distinguish encoded bytes from plain
// strings that happen to look like base64.
func toExtendedStrings(extended map[string][]byte) map[string]string {
	if len(extended) == 0 {
		return nil
	}
	result := make(map[string]string, len(extended))
	for k, v := range extended {
		if utf8.Valid(v) {
			result[k] = string(v)
		} else {
			result[k] = "base64:" + base64.StdEncoding.EncodeToString(v)
		}
	}
	return result
}

func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) {
	options := []elastic.ClientOptionFunc{}
	options = append(options, elastic.SetURL(strings.Split(servers, ",")...))
	options = append(options, elastic.SetSniff(false))
	options = append(options, elastic.SetHealthcheck(false))
	client, err := elastic.NewClient(options...)
	if err != nil {
		return nil, err
	}
	return func(resp *filer_pb.SubscribeMetadataResponse) error {
		event := resp.EventNotification
		if event.OldEntry != nil &&
			(event.NewEntry == nil || resp.Directory != event.NewParentPath || event.OldEntry.Name != event.NewEntry.Name) {
			// delete or not update the same file
			dir, name := resp.Directory, event.OldEntry.Name
			id := util.Md5String([]byte(util.NewFullPath(dir, name)))
			println("delete", id)
			_, err := client.Delete().Index(esIndex).Id(id).Do(context.Background())
			return err
		}
		if event.NewEntry != nil {
			// add a new file or update the same file
			esEntry, id := toEsEntry(event)
			value, err := jsoniter.Marshal(esEntry)
			if err != nil {
				return err
			}
			println(string(value))
			_, err = client.Index().Index(esIndex).Id(id).BodyJson(string(value)).Do(context.Background())
			return err
		}
		return nil
	}, nil
}