snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/pkg/id/conv.go

@@ -0,0 +1,31 @@
+package id
+
+import "strconv"
+
+func Strings(ii ...uint64) []string {
+	ss := make([]string, len(ii))
+	for i, v := range ii {
+		ss[i] = String(v)
+	}
+	return ss
+}
+
+func String(i uint64) string {
+	return strconv.FormatUint(i, 10)
+}
+
+func Uints(ss ...string) []uint64 {
+	uu := make([]uint64, len(ss))
+	for i, s := range ss {
+		uu[i] = Uint(s)
+	}
+	return uu
+}
+
+func Uint(s string) uint64 {
+	if s == "" {
+		return 0
+	}
+	i, _ := strconv.ParseUint(s, 10, 64)
+	return i
+}

server/pkg/id/http.go

@@ -0,0 +1,27 @@
+package id
+
+func ParseID(ss string) (p ID, err error) {
+	if len(ss) == 0 {
+		return
+	}
+
+	return ByteID([]byte(ss))
+}
+
+func ParseIDs(ss []string) (out []ID, err error) {
+	if len(ss) == 0 {
+		return
+	}
+
+	var aux ID
+	for _, s := range ss {
+		aux, err = ByteID([]byte(s))
+		if err != nil {
+			return
+		}
+
+		out = append(out, aux)
+	}
+
+	return
+}

server/pkg/id/id.go

@@ -0,0 +1,199 @@
+package id
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+)
+
+type (
+	ID struct {
+		Num uint64
+		Str [maxStrIdentifierLength]rune
+	}
+)
+
+const (
+	maxStrIdentifierLength = 36
+)
+
+var (
+	isNumberRegex = regexp.MustCompile(`^(0|[1-9][0-9]{0,19})$`)
+)
+
+func MustNumID(n uint64) ID {
+	id, err := NumID(n)
+	if err != nil {
+		panic(fmt.Sprintf("failed to create NumID: %v", err))
+	}
+	return id
+}
+
+func NumID(n uint64) (ID, error) {
+	return ID{Num: n}, nil
+}
+
+func MustStrID(s string) ID {
+	id, err := StrID(s)
+	if err != nil {
+		panic(fmt.Sprintf("failed to create StrID: %v", err))
+	}
+	return id
+}
+
+func StrID(s string) (ID, error) {
+	if len(s) > maxStrIdentifierLength {
+		return ID{}, fmt.Errorf("string too long: %s", s)
+	}
+
+	var str [maxStrIdentifierLength]rune
+	for i, r := range s {
+		if i < maxStrIdentifierLength {
+			str[i] = r
+		}
+	}
+	return ID{Str: str}, nil
+}
+
+func MustByteID(data []byte) (id ID) {
+	id, err := ByteID(data)
+	if err != nil {
+		panic(fmt.Sprintf("failed to parse ID: %v", err))
+	}
+
+	return id
+}
+
+func ByteID(data []byte) (id ID, err error) {
+	// numeric
+	if isNumberRegex.MatchString(string(data)) {
+		num, err := strconv.ParseUint(string(data), 10, 64)
+		if err != nil {
+			return id, err
+		}
+		id, err = NumID(num)
+		return id, err
+	}
+
+	// check if strings fit
+	if len(data) > maxStrIdentifierLength {
+		return id, fmt.Errorf("string too long: %s", data)
+	}
+
+	id, err = StrID(string(data))
+	return
+}
+
+func (a ID) Equal(b ID) bool {
+	return a == b
+}
+
+func (id ID) IsZero() bool {
+	return !id.isNum() && !id.isStr()
+}
+
+func (id ID) isNum() bool {
+	return id.Num != 0
+}
+
+func (id ID) isStr() bool {
+	return id.Str != [maxStrIdentifierLength]rune{}
+}
+
+func InSlice(needle ID, slice ...ID) bool {
+	for _, v := range slice {
+		if v.Equal(needle) {
+			return true
+		}
+	}
+	return false
+}
+
+func RemoveFromSlice(needle ID, slice ...ID) (out []ID) {
+	if len(slice) == 0 {
+		return
+	}
+
+	out = make([]ID, 0, len(slice)-1)
+
+	for _, i := range slice {
+		if i.Equal(needle) {
+			continue
+		}
+
+		out = append(out, i)
+	}
+
+	return
+}
+
+func (id ID) Value() string {
+	if id.isStr() {
+		return `"` + string(id.String()) + `"`
+	}
+
+	if id.isNum() {
+		return `"` + String(id.Number()) + `"`
+	}
+
+	// Backward compatibility for empty ID
+	return `"0"`
+}
+
+func (id ID) String() string {
+	hasValid := false
+	i := 0
+
+	for _, r := range id.Str {
+		if !hasValid && r != 0 {
+			hasValid = true
+			continue
+		}
+
+		if hasValid && r == 0 {
+			return string(id.Str[:i+1])
+		}
+
+		i++
+	}
+
+	if !hasValid {
+		return ""
+	}
+
+	return string(id.Str[:])
+}
+
+func (id ID) Number() uint64 {
+	return id.Num
+}
+
+// marshal/unmarshal JSON
+func (id ID) MarshalJSON() ([]byte, error) {
+	return []byte(id.Value()), nil
+}
+
+func (id *ID) UnmarshalJSON(data []byte) (err error) {
+	// regex to check if data contains a uint64 or smaller number
+	if len(data) < 2 || data[0] != '"' || data[len(data)-1] != '"' {
+		return nil
+	}
+
+	data = data[1 : len(data)-1]
+
+	auxID, err := ByteID(data)
+	if err != nil {
+		return
+	}
+
+	*id = auxID
+	return
+}
+
+func StringifySlice(bb ...ID) (out []string) {
+	for _, b := range bb {
+		out = append(out, b.Value())
+	}
+
+	return
+}

server/pkg/id/id_bench_test.go

@@ -0,0 +1,70 @@
+package id
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+// goos: darwin
+// goarch: arm64
+// pkg: github.com/cortezaproject/corteza/server/pkg/id
+// cpu: Apple M3 Pro
+// BenchmarkIDCastingBase-12    	 2561474	       465.7 ns/op	     312 B/op	       8 allocs/op
+// PASS
+func BenchmarkIDCastingBase(b *testing.B) {
+	b.ReportAllocs()
+
+	type TestStruct struct {
+		K1, K2 string
+	}
+
+	aux := TestStruct{
+		K1: "K1 Value",
+		K2: "K2 Value",
+	}
+	var dst TestStruct
+
+	b.ResetTimer()
+	for n := 0; n < b.N; n++ {
+		bb, err := json.Marshal(aux)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if err := json.Unmarshal(bb, &dst); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+// goos: darwin
+// goarch: arm64
+// pkg: github.com/cortezaproject/corteza/server/pkg/id
+// cpu: Apple M3 Pro
+// BenchmarkIDCasting-12    	 1284855	       866.6 ns/op	     528 B/op	      12 allocs/op
+// PASS
+func BenchmarkIDCasting(b *testing.B) {
+	b.ReportAllocs()
+
+	type TestStruct struct {
+		SomethingID ID
+		K1, K2      string
+	}
+
+	aux := TestStruct{
+		SomethingID: MustNumID(3617289),
+		K1:          "K1 Value",
+		K2:          "K2 Value",
+	}
+	var dst TestStruct
+
+	b.ResetTimer()
+	for n := 0; n < b.N; n++ {
+		bb, err := json.Marshal(aux)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if err := json.Unmarshal(bb, &dst); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

server/pkg/id/id_test.go

@@ -0,0 +1,166 @@
+package id
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type (
+	TestRes struct {
+		Id ID
+	}
+)
+
+func TestNumericIdentifiers(t *testing.T) {
+	ints := []uint64{
+		1,
+		100,
+		231,
+		1000,
+		1234567890,
+		18446744073709551615, // max uint64
+	}
+
+	for _, i := range ints {
+		id, err := NumID(i)
+		require.NoError(t, err)
+		require.Equal(t, i, id.Number())
+		require.True(t, id.isNum())
+		require.False(t, id.isStr())
+
+		tmpRes := fmt.Sprintf(`{"Id":"%d"}`, i)
+		var tr TestRes
+		err = json.Unmarshal([]byte(tmpRes), &tr)
+		require.NoError(t, err)
+		require.Equal(t, i, tr.Id.Number())
+		require.True(t, tr.Id.isNum())
+		require.False(t, tr.Id.isStr())
+
+		bb, err := json.Marshal(&tr)
+		require.NoError(t, err)
+		require.Equal(t, tmpRes, string(bb))
+	}
+}
+
+func TestStringIdentifiers(t *testing.T) {
+	strs := []string{
+		"-1",
+		"asdf",
+		"019864d2-3d93-7a86-9012-ec15f6358ab0", // uuid v7
+		"0e04ae61-6887-4dda-bcef-c514240d1cd3", // uuid v4
+		"b0a372c4-6eb4-11f0-8de9-0242ac120002", // uuid v1
+		"00000000-0000-0000-0000-000000000000", // nil uuid
+	}
+
+	for _, s := range strs {
+		id, err := StrID(s)
+		require.NoError(t, err)
+		require.Equal(t, s, id.String())
+		require.False(t, id.isNum())
+		require.True(t, id.isStr())
+
+		tmpRes := fmt.Sprintf(`{"Id":"%s"}`, s)
+		var tr TestRes
+		err = json.Unmarshal([]byte(tmpRes), &tr)
+		require.NoError(t, err)
+		require.Equal(t, s, tr.Id.String())
+		require.False(t, tr.Id.isNum())
+		require.True(t, tr.Id.isStr())
+
+		bb, err := json.Marshal(&tr)
+		require.NoError(t, err)
+		require.Equal(t, tmpRes, string(bb))
+	}
+}
+
+func TestByteIdentifiers(t *testing.T) {
+	tcc := []struct {
+		in []byte
+
+		outNum uint64
+		outStr string
+	}{
+		{
+			in:     []byte("123"),
+			outNum: 123,
+			outStr: "",
+		}, {
+			in:     []byte("asdf"),
+			outNum: 0,
+			outStr: "asdf",
+		}, {
+			in:     []byte("019864d2-3d93-7a86-9012-ec15f6358ab0"),
+			outNum: 0,
+			outStr: "019864d2-3d93-7a86-9012-ec15f6358ab0",
+		},
+	}
+
+	for _, tc := range tcc {
+		t.Run(string(tc.in), func(t *testing.T) {
+			id, err := ByteID(tc.in)
+			require.NoError(t, err)
+
+			require.Equal(t, tc.outNum, id.Number())
+			require.Equal(t, tc.outStr, id.String())
+		})
+	}
+}
+
+func TestInSlice(t *testing.T) {
+	t.Run("empty", func(t *testing.T) {
+		require.False(t, InSlice(MustNumID(1)))
+	})
+
+	t.Run("yes", func(t *testing.T) {
+		require.True(t, InSlice(MustNumID(2), MustNumID(1), MustNumID(2), MustNumID(3)))
+	})
+
+	t.Run("no", func(t *testing.T) {
+		require.False(t, InSlice(MustNumID(99), MustNumID(1), MustNumID(2), MustNumID(3)))
+	})
+}
+
+func TestRemoveFromSlice(t *testing.T) {
+	t.Run("empty", func(t *testing.T) {
+		require.Len(t, RemoveFromSlice(MustNumID(1)), 0)
+	})
+
+	t.Run("found", func(t *testing.T) {
+		slc := []ID{MustNumID(1), MustNumID(2)}
+		slc = RemoveFromSlice(MustNumID(1), slc...)
+
+		require.Len(t, slc, 1)
+		require.Equal(t, MustNumID(2), slc[0])
+	})
+
+	t.Run("not found", func(t *testing.T) {
+		slc := []ID{MustNumID(1), MustNumID(2)}
+		slc = RemoveFromSlice(MustNumID(999), slc...)
+
+		require.Len(t, slc, 2)
+	})
+}
+
+// func TestIdentifierAssignment(t *testing.T) {
+// 	ints := []uint64{
+// 		1,
+// 		100,
+// 		231,
+// 		1000,
+// 		1234567890,
+// 		18446744073709551615, // max uint64
+// 	}
+
+// 	strs := []string{
+// 		"1",
+// 		"asdf",
+// 		"019864d2-3d93-7a86-9012-ec15f6358ab0", // uuid v7
+// 		"0e04ae61-6887-4dda-bcef-c514240d1cd3", // uuid v4
+// 		"b0a372c4-6eb4-11f0-8de9-0242ac120002", // uuid v1
+// 		"00000000-0000-0000-0000-000000000000", // nil uuid
+// 	}
+
+// }

server/pkg/id/sonyflake.go

@@ -0,0 +1,133 @@
+package id
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/sony/sonyflake"
+)
+
+const (
+	// Midnight, January 1st 2017
+	jan1st2017 = 1483228800
+
+	// Env key for custom machine ID
+	//
+	// Sonyflake assumes that your private IP falls under the following ranges:
+	//     10.0.0.0 - 10.255.255.255
+	//   172.16.0.0 - 172.31.255.255
+	//  192.168.0.0 - 192.168.255.255
+	//
+	// When running in a container, this could not always the case.
+	// To override this, set SONYFLAKE_MACHINE_ID to a valid uint16
+	// (0..65535) and sonyflake will use that as machine ID.
+	envKeyMachineID = "SONYFLAKE_MACHINE_ID"
+
+	// When set to uint32 value, we no longer use sonyflake
+	// but a simple atomic counter
+	//
+	// This is useful for testing and debugging
+	// and should not be used in production
+	// @todo implement
+	// envKeySimpleIncrement = "SONYFLAKE_SIMPLE_INCREMENT"
+)
+
+var (
+	initialized bool
+	idQueue     = make(chan uint64, 8000)
+	// Keep this to 1 for now as we need to enforce that any later ID is larger
+	// than any previous one.
+	// If we use different deviceIDs for routines, this isn't guaranteed to hold.
+	makerCount = 1
+)
+
+func Init(ctx context.Context) {
+	if initialized {
+		return
+	}
+	initialized = true
+
+	wg := &sync.WaitGroup{}
+	wg.Add(makerCount)
+
+	for i := 0; i < makerCount; i++ {
+		makeIDer(ctx, wg, idQueue, uint64(i+1))
+	}
+
+	wg.Wait()
+
+	// Give it some time to warm up with some ids
+	// Arbitrarily picked a good timeout based on benchmark runs
+	time.Sleep(time.Second * 1)
+}
+
+func makeIDer(ctx context.Context, wg *sync.WaitGroup, qq chan uint64, thr uint64) {
+	go func() {
+		ss, err := initSonyflake(thr)
+		if err != nil {
+			panic(fmt.Errorf("sonyflake init failed: %w", err))
+		}
+
+		wg.Done()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+
+			default:
+				id, err := ss.NextID()
+				if err != nil {
+					panic(err)
+				}
+				qq <- id
+			}
+		}
+	}()
+}
+
+func initSonyflake(thr uint64) (out *sonyflake.Sonyflake, err error) {
+	settings := sonyflake.Settings{
+		StartTime: time.Unix(jan1st2017, 0),
+	}
+
+	// check if SONYFLAKE_MACHINE_ID is set and create
+	// MachineID function that returns it
+	var id uint64
+	if machineID := os.Getenv(envKeyMachineID); machineID != "" {
+		// check if machineID is a valid uint16
+		if id, err = strconv.ParseUint(machineID, 10, 16); err != nil {
+			// should crash right away
+			err = fmt.Errorf(
+				"could not use %s (%s), expecting uint16 (0..65535): %w",
+				envKeyMachineID,
+				machineID,
+				err,
+			)
+			return
+		} else {
+			settings.MachineID = func() (uint16, error) {
+				return uint16(id + thr), nil
+			}
+		}
+	} else if thr > 0 {
+		settings.MachineID = func() (uint16, error) {
+			return uint16(thr), nil
+		}
+	}
+
+	out = sonyflake.NewSonyflake(settings)
+	_, err = out.NextID()
+
+	return
+}
+
+func Next() uint64 {
+	if !initialized {
+		panic("ID generator not initialized: call pkg/id.Init")
+	}
+	return <-idQueue
+}

server/pkg/id/sonyflake_test.go

@@ -0,0 +1,21 @@
+package id
+
+import (
+	"testing"
+
+	"github.com/cortezaproject/corteza/server/pkg/cli"
+)
+
+// goos: darwin
+// goarch: arm64
+// pkg: github.com/cortezaproject/corteza/server/pkg/id
+// BenchmarkGenerator-12    	  162234	     39011 ns/op	       0 B/op	       0 allocs/op
+func BenchmarkGenerator(b *testing.B) {
+	ctx := cli.Context()
+	Init(ctx)
+
+	b.ResetTimer()
+	for n := 0; n < b.N; n++ {
+		Next()
+	}
+}