Skip to content

Commit

Permalink
Merge branch 'main' into refactor/lbgw/search-aggregation-algos
Browse files Browse the repository at this point in the history 
Signed-off-by: kpango <[email protected]>
  • Loading branch information
@kpango
kpango committed Jun 8, 2023
2 parents e6bd0e2 + 96df080 commit 8660b9f
Show file tree
Hide file tree
Showing 29 changed files with 848 additions and 111 deletions.
3 changes: 2 additions & 1 deletion .devcontainer/devcontainer.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,6 @@
"moby": "true"
}
},
"postCreateCommand": "go version"
"postCreateCommand": "go version",
"postAttachCommand": "sudo ln -s $(pwd)/cmd/agent/core/ngt/sample.yaml /etc/server/config.yaml"
}
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,4 +51,7 @@ hack/go.mod.default3
# for nvim
.nvimlog

# for vscode
.vscode/

telepresence.log
3 changes: 1 addition & 2 deletions Makefile.d/proto.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,7 @@ proto/deps: \
$(GOPATH)/src/github.com/planetscale/vtprotobuf \
$(GOPATH)/src/github.com/protocolbuffers/protobuf \
$(GOPATH)/src/google.golang.org/genproto \
$(GOPATH)/src/google.golang.org/protobuf \
$(ROOTDIR)/apis/proto/v1
$(GOPATH)/src/google.golang.org/protobuf

$(GOPATH)/src/github.com/protocolbuffers/protobuf:
git clone \
Expand Down
2 changes: 2 additions & 0 deletions charts/vald-helm-operator/crds/valdrelease.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,8 @@ spec:
type: integer
auto_save_index_duration:
type: string
broken_index_history_limit:
type: integer
bulk_insert_chunk_size:
type: integer
creation_edge_size:
Expand Down
1 change: 1 addition & 0 deletions charts/vald/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ Run the following command to install the chart,
| agent.ngt.auto_index_duration_limit | string | `"24h"` | limit duration of automatic indexing |
| agent.ngt.auto_index_length | int | `100` | number of cache to trigger automatic indexing |
| agent.ngt.auto_save_index_duration | string | `"35m"` | duration of automatic save index |
| agent.ngt.broken_index_history_limit | int | `0` | maximum number of broken index generations to backup |
| agent.ngt.bulk_insert_chunk_size | int | `10` | bulk insert chunk size |
| agent.ngt.creation_edge_size | int | `20` | creation edge size |
| agent.ngt.default_epsilon | float | `0.1` | default epsilon used for search |
Expand Down
4 changes: 4 additions & 0 deletions charts/vald/values.schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,10 @@
"type": "string",
"description": "duration of automatic save index"
},
"broken_index_history_limit": {
"type": "integer",
"description": "maximum number of broken index generations to backup"
},
"bulk_insert_chunk_size": {
"type": "integer",
"description": "bulk insert chunk size"
Expand Down
3 changes: 3 additions & 0 deletions charts/vald/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1714,6 +1714,9 @@ agent:
# @schema {"name": "agent.ngt.kvsdb.concurrency", "type": "integer"}
# agent.ngt.kvsdb.concurrency -- kvsdb processing concurrency
concurrency: 6
# @schema {"name": "agent.ngt.broken_index_history_limit", "type": "integer"}
# agent.ngt.broken_index_history_limit -- maximum number of broken index generations to backup
broken_index_history_limit: 0
# @schema {"name": "agent.sidecar", "type": "object"}
sidecar:
# @schema {"name": "agent.sidecar.enabled", "type": "boolean"}
Expand Down
1 change: 1 addition & 0 deletions cmd/agent/core/ngt/sample.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,3 +126,4 @@ ngt:
object_type: float
search_edge_size: 10
enable_copy_on_write: false
broken_index_history_limit: 0
5 changes: 5 additions & 0 deletions dockers/dev/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,11 @@ RUN make ngt/install \
&& make kubectl/install \
&& make k3d/install

# change GOPATH owner to vscode to run commands as user vscode without root permission
RUN chown -R vscode:vscode "${GOPATH}" \
# mkdir for agent configuration install
&& mkdir -p /etc/server

# k9s installs the binary to the current user which is root for devcontainer
# so change the current user to vscode to user afterwards
USER vscode
Expand Down
6 changes: 3 additions & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ replace (
github.com/armon/go-radix => github.com/armon/go-radix v1.0.0
github.com/armon/go-socks5 => github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
github.com/asaskevich/govalidator => github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.44.277
github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.44.278
github.com/aws/aws-sdk-go-v2 => github.com/aws/aws-sdk-go-v2 v1.18.0
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10
github.com/aws/aws-sdk-go-v2/config => github.com/aws/aws-sdk-go-v2/config v1.18.25
Expand Down Expand Up @@ -433,7 +433,7 @@ replace (
go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v1.11.1
go.opentelemetry.io/otel/sdk/metric => go.opentelemetry.io/otel/sdk/metric v0.33.0
go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.11.1
go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp v0.19.0
go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp v0.20.0
go.starlark.net => go.starlark.net v0.0.0-20230525235612-a134d8f9ddca
go.uber.org/atomic => go.uber.org/atomic v1.11.0
go.uber.org/automaxprocs => go.uber.org/automaxprocs v1.5.2
Expand Down Expand Up @@ -611,7 +611,7 @@ require (
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/googleapis/gax-go/v2 v2.10.0 // indirect
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
github.com/huandu/xstrings v1.4.0 // indirect
github.com/imdario/mergo v0.3.6 // indirect
Expand Down
8 changes: 4 additions & 4 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,8 +202,8 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg=
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/aws/aws-sdk-go v1.44.277 h1:YHmyzBPARTJ7LLYV1fxbfEbQOaUh3kh52hb7nBvX3BQ=
github.com/aws/aws-sdk-go v1.44.277/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go v1.44.278 h1:jJFDO/unYFI48WQk7UGSyO3rBA/gnmRpNYNuAw/fPgE=
github.com/aws/aws-sdk-go v1.44.278/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY=
github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
Expand Down Expand Up @@ -667,8 +667,8 @@ go.opentelemetry.io/otel/sdk/metric v0.33.0 h1:oTqyWfksgKoJmbrs2q7O7ahkJzt+Ipeki
go.opentelemetry.io/otel/sdk/metric v0.33.0/go.mod h1:xdypMeA21JBOvjjzDUtD0kzIcHO/SPez+a8HOzJPGp0=
go.opentelemetry.io/otel/trace v1.11.1 h1:ofxdnzsNrGBYXbP7t7zpUK281+go5rF7dvdIZXF8gdQ=
go.opentelemetry.io/otel/trace v1.11.1/go.mod h1:f/Q9G7vzk5u91PhbmKbg1Qn0rzH1LJ4vbPHFGkTPtOk=
go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw=
go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
go.opentelemetry.io/proto/otlp v0.20.0 h1:BLOA1cZBAGSbRiNuGCCKiFrCdYB7deeHDeD1SueyOfA=
go.opentelemetry.io/proto/otlp v0.20.0/go.mod h1:3QgjzPALBIv9pcknj2EXGPXjYPFdUh/RQfF8Lz3+Vnw=
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY=
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
Expand Down
3 changes: 3 additions & 0 deletions internal/config/ngt.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@ type NGT struct {

// KVSDB represent the ngt bidirectional kv store configuration
KVSDB *KVSDB `json:"kvsdb,omitempty" yaml:"kvsdb"`

// BrokenIndexHistoryLimit represents the maximum number of broken index generations that will be backed up
BrokenIndexHistoryLimit int `yaml:"broken_index_history_limit" json:"broken_index_history_limit,omitempty"`
}

// KVSDB represent the ngt vector bidirectional kv store configuration
Expand Down
13 changes: 0 additions & 13 deletions internal/core/algorithm/ngt/ngt.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ package ngt
import "C"

import (
"os"
"reflect"
"sync"
"unsafe"
Expand Down Expand Up @@ -307,18 +306,6 @@ func (n *ngt) loadOptions(opts ...Option) (err error) {
}

func (n *ngt) create() (err error) {
files, err := file.ListInDir(n.idxPath)
if err == nil && len(files) != 0 {
log.Warnf("index path exists, will remove the directories: %v", files)
for _, f := range files {
err = os.RemoveAll(f)
if err != nil {
return err
}
}
} else if err != nil {
log.Debug(err)
}
path := C.CString(n.idxPath)
defer C.free(unsafe.Pointer(path))

Expand Down
5 changes: 5 additions & 0 deletions internal/errors/errors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@ var (
return Wrapf(err, "failed to output %s logs", str)
}

// ErrIndexPathNotExists represents a function to generate an error that the index path is not exists.
ErrIndexPathNotExists = func(path string) error {
return Errorf("index path %s not exists", path)
}

// New represents a function to generate the new error with a message.
// When the message is nil, it will return nil instead of an error.
New = func(msg string) error {
Expand Down
1 change: 1 addition & 0 deletions internal/test/data/agent/ngt/validIndex/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
This directory holds an outcome of inserting `datafashion-mnist-784-euclidean` to Vald agent by using `example/client/agent/main.go` with `insertCount = 100`.
Binary file added internal/test/data/agent/ngt/validIndex/grp
View file
Binary file not shown.
1 change: 1 addition & 0 deletions internal/test/data/agent/ngt/validIndex/metadata.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "is_invalid": false, "ngt": { "index_count": 100 } }
Binary file added internal/test/data/agent/ngt/validIndex/ngt-meta.kvsdb
View file
Binary file not shown.
Binary file added internal/test/data/agent/ngt/validIndex/ngt-timestamp.kvsdb
View file
Binary file not shown.
Binary file added internal/test/data/agent/ngt/validIndex/obj
View file
Binary file not shown.
26 changes: 26 additions & 0 deletions internal/test/data/agent/ngt/validIndex/prf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
AccuracyTable
BatchSizeForCreation 200
BuildTimeLimit 0
DatabaseType Memory
Dimension 784
DistanceType L2
DynamicEdgeSizeBase 30
DynamicEdgeSizeRate 20
EdgeSizeForCreation 20
EdgeSizeForSearch 10
EdgeSizeLimitForCreation 5
EpsilonForCreation 0.1
GraphType ANNG
IncomingEdge 80
IncrimentalEdgeSizeLimitForTruncation 0
IndexType GraphAndTree
ObjectAlignment False
ObjectType Float-4
OutgoingEdge 10
PathAdjustmentInterval 0
PrefetchOffset 1
PrefetchSize 3136
SeedSize 10
SeedType None
ThreadPoolSize 32
TruncationThreadPoolSize 8
Binary file added internal/test/data/agent/ngt/validIndex/tre
View file
Binary file not shown.
4 changes: 1 addition & 3 deletions internal/test/data/strings/strings_benchmark_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,5 @@ func benchmark(b *testing.B, f func(l int) string) {
}

func BenchmarkRandom(b *testing.B) {
benchmark(b, func(cnt int) string {
return Random(cnt)
})
benchmark(b, Random)
}
4 changes: 4 additions & 0 deletions internal/test/testdata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ import (
"github.com/vdaas/vald/internal/strings"
)

const (
ValidIndex = "agent/ngt/validIndex"
)

// GetTestdataPath returns the test data file path under `internal/test/data`.
func GetTestdataPath(filename string) string {
return file.Join(baseDir(), "/internal/test/data/", filename)
Expand Down
1 change: 1 addition & 0 deletions k8s/agent/configmap.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ data:
auto_index_duration_limit: 24h
auto_index_length: 100
auto_save_index_duration: 35m
broken_index_history_limit: 0
bulk_insert_chunk_size: 10
creation_edge_size: 20
default_epsilon: 0.1
Expand Down
2 changes: 2 additions & 0 deletions k8s/operator/helm/crds/valdrelease.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,8 @@ spec:
type: integer
auto_save_index_duration:
type: string
broken_index_history_limit:
type: integer
bulk_insert_chunk_size:
type: integer
creation_edge_size:
Expand Down
Loading

0 comments on commit 8660b9f

Please sign in to comment.