upstream-bleve/index_meta.go

//  Copyright (c) 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bleve

import (
	"encoding/binary"
	"fmt"
	"os"
	"path/filepath"

	"github.com/blevesearch/bleve/v2/index/upsidedown"
	"github.com/blevesearch/bleve/v2/util"
	index "github.com/blevesearch/bleve_index_api"
)

const metaFilename = "index_meta.json"

type indexMeta struct {
	Storage    string                 `json:"storage"`
	IndexType  string                 `json:"index_type"`
	Config     map[string]interface{} `json:"config,omitempty"`
	fileWriter util.FileWriter
	fileReader util.FileReader
}

func newIndexMeta(indexType string, storage string, config map[string]interface{}, path string) (*indexMeta, error) {
	indexMetaPath := indexMetaPath(path)
	fileWriter, err := util.NewFileWriter([]byte(indexMetaPath))
	if err != nil {
		return nil, fmt.Errorf("failed to create file writer for index meta: %w", err)
	}
	fileReader, err := util.NewFileReader(fileWriter.Id(), []byte(indexMetaPath))
	if err != nil {
		return nil, fmt.Errorf("failed to create file reader for index meta: %w", err)
	}
	return &indexMeta{
		IndexType:  indexType,
		Storage:    storage,
		Config:     config,
		fileWriter: fileWriter,
		fileReader: fileReader,
	}, nil
}

func openIndexMeta(path string) (*indexMeta, error) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return nil, ErrorIndexPathDoesNotExist
	}
	indexMetaPath := indexMetaPath(path)
	metaBytes, err := os.ReadFile(indexMetaPath)
	if err != nil {
		return nil, ErrorIndexMetaMissing
	}

	// check if indexMetaPath+_temp exists, if so, this means a writer update was in progress
	// and we should attempt to recover using the temp file
	if _, err := os.Stat(indexMetaPath + "_temp"); err == nil {
		tempBytes, err := os.ReadFile(indexMetaPath + "_temp")
		if err == nil {
			err = os.Rename(indexMetaPath+"_temp", indexMetaPath)
			if err != nil {
				return nil, err
			}
			metaBytes = tempBytes
		}
	}

	var im indexMeta
	var fileReader util.FileReader
	// attempt to unmarshal metabytes directly. If this succeeds,
	// then we know there was no file callback writer used and we can
	// proceed as normal.
	err = util.UnmarshalJSON(metaBytes, &im)
	if err != nil {
		// on failure, we expect the last 4 bytes to be the length of the file
		// callback id and the preceding bytes to be the file callback id, which
		// we can use to obtain the file reader to read the actual meta data bytes
		if len(metaBytes) < 4 {
			return nil, ErrorIndexMetaCorrupt
		}

		// read the length of the file callback id from the last 4 bytes
		pos := len(metaBytes) - 4
		fileWriterIDLen := int(binary.BigEndian.Uint32(metaBytes[pos:]))
		pos -= fileWriterIDLen
		if pos < 0 {
			return nil, ErrorIndexMetaCorrupt
		}

		// read and initialize the file reader using the file callback id
		fileWriterID := metaBytes[pos : pos+fileWriterIDLen]
		fileReader, err = util.NewFileReader(string(fileWriterID), []byte(indexMetaPath))
		if err != nil {
			return nil, err
		}

		buf, err := fileReader.Process(metaBytes[0:pos])
		if err != nil {
			return nil, err
		}
		err = util.UnmarshalJSON(buf, &im)
		if err != nil {
			return nil, ErrorIndexMetaCorrupt
		}
	} else {
		fileReader, err = util.NewFileReader("", []byte(indexMetaPath))
		if err != nil {
			return nil, err
		}
	}
	im.fileReader = fileReader

	if im.IndexType == "" {
		im.IndexType = upsidedown.Name
	}
	return &im, nil
}

func (i *indexMeta) Save(path string) (err error) {
	indexMetaPath := indexMetaPath(path)
	// ensure any necessary parent directories exist
	err = os.MkdirAll(path, 0700)
	if err != nil {
		if os.IsExist(err) {
			return ErrorIndexPathExists
		}
		return err
	}
	metaBytes, err := util.MarshalJSON(i)
	if err != nil {
		return err
	}
	indexMetaFile, err := os.OpenFile(indexMetaPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
	if err != nil {
		if os.IsExist(err) {
			return ErrorIndexPathExists
		}
		return err
	}
	defer func() {
		if ierr := indexMetaFile.Close(); err == nil && ierr != nil {
			err = ierr
		}
	}()

	metaBytes = i.fileWriter.Process(metaBytes)

	_, err = indexMetaFile.Write(metaBytes)
	if err != nil {
		return err
	}

	_, err = indexMetaFile.Write([]byte(i.fileWriter.Id()))
	if err != nil {
		return err
	}

	err = binary.Write(indexMetaFile, binary.BigEndian, uint32(len(i.fileWriter.Id())))
	if err != nil {
		return err
	}

	return nil
}

func (i *indexMeta) CopyTo(path string, d index.Directory) (err error) {
	metaBytes, err := os.ReadFile(indexMetaPath(path))
	if err != nil {
		return err
	}

	w, err := d.GetWriter(metaFilename)
	if w == nil || err != nil {
		return fmt.Errorf("invalid writer for file: %s, err: %v",
			metaFilename, err)
	}
	defer w.Close()

	_, err = w.Write(metaBytes)
	return err
}

// updates the file callback writer id in the index meta,
// and re-processes data with the latest file callback writer
// returns the new file callback writer and reader to be used for
// future processing of index meta data
func (i *indexMeta) UpdateWriter(path string) error {
	indexMetaPath := indexMetaPath(path)
	metaBytes, err := util.MarshalJSON(i)
	if err != nil {
		return err
	}

	i.fileWriter, err = util.NewFileWriter([]byte(indexMetaPath))
	if err != nil {
		return err
	}
	metaBytes = i.fileWriter.Process(metaBytes)

	// write out new meta with new writer id, using temp file and rename to ensure atomicity
	// if we crash in the middle of this, on next open we will see the temp file and recover using it
	tempMetaPath := indexMetaPath + "_temp"
	tempMetaFile, err := os.OpenFile(tempMetaPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
	if err != nil {
		if os.IsExist(err) {
			return ErrorIndexPathExists
		}
		return err
	}

	// write the meta bytes
	_, err = tempMetaFile.Write(metaBytes)
	if err != nil {
		return err
	}
	// write the file callback id
	_, err = tempMetaFile.Write([]byte(i.fileWriter.Id()))
	if err != nil {
		return err
	}
	// write the length of the file callback id
	err = binary.Write(tempMetaFile, binary.BigEndian, uint32(len(i.fileWriter.Id())))
	if err != nil {
		return err
	}
	// close file before renaming
	err = tempMetaFile.Close()
	if err != nil {
		return err
	}
	// atomically rename temp file to index meta file
	err = os.Rename(tempMetaPath, indexMetaPath)
	if err != nil {
		return err
	}

	// initialize the new file reader for index meta
	i.fileReader, err = util.NewFileReader(string(i.fileWriter.Id()), []byte(indexMetaPath))
	if err != nil {
		return err
	}

	return nil
}

func indexMetaPath(path string) string {
	return filepath.Join(path, metaFilename)
}