Skip to content

Commit

Permalink
update SegmenterDict to GseDict
Browse files Browse the repository at this point in the history
  • Loading branch information
@vcaesar
vcaesar committed May 30, 2018
1 parent 23aa580 commit 98023de
Show file tree
Hide file tree
Showing 18 changed files with 76 additions and 76 deletions.
4 changes: 2 additions & 2 deletions README_zh.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,8 @@ func main() {
// 初始化
searcher.Init(types.EngineOpts{
Using: 3,
SegmenterDict: "zh",
// SegmenterDict: "your gopath"+"/src/github.com/go-ego/riot/data/dict/dictionary.txt",
GseDict: "zh",
// GseDict: "your gopath"+"/src/github.com/go-ego/riot/data/dict/dictionary.txt",
})
defer searcher.Close()

Expand Down
2 changes: 1 addition & 1 deletion data/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func main() {
// Init searcher
searcher.Init(types.EngineOpts{
Using: 4,
SegmenterDict: "./dict/dictionary.txt"})
GseDict: "./dict/dictionary.txt"})
defer searcher.Close()

text := "Google Is Experimenting With Virtual Reality Advertising"
Expand Down
2 changes: 1 addition & 1 deletion docs/en/codelab.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ The first package defines the engine function, the second package defines the co
```go
var searcher riot.Engine
searcher.Init(types.EngineOpts{
SegmenterDict: "../../data/dict/dictionary.txt",
GseDict: "../../data/dict/dictionary.txt",
StopTokenFile: "../../data/dict/stop_tokens.txt",
IndexerOpts: &types.IndexerOpts{
IndexType: types.LocsIndex,
Expand Down
2 changes: 1 addition & 1 deletion docs/zh/codelab.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ import (
```go
var searcher riot.Engine
searcher.Init(types.EngineOpts{
SegmenterDict: "../../data/dict/dictionary.txt",
GseDict: "../../data/dict/dictionary.txt",
StopTokenFile: "../../data/dict/stop_tokens.txt",
IndexerOpts: &types.IndexerOpts{
IndexType: types.LocsIndex,
Expand Down
78 changes: 39 additions & 39 deletions engine_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ func TestTry(t *testing.T) {
func TestEngineIndexDoc(t *testing.T) {
var engine Engine
engine.Init(types.EngineOpts{
Using: 1,
SegmenterDict: "./testdata/test_dict.txt",
Using: 1,
GseDict: "./testdata/test_dict.txt",
DefaultRankOpts: &types.RankOpts{
OutputOffset: 0,
MaxOutputs: 10,
Expand Down Expand Up @@ -149,8 +149,8 @@ func TestEngineIndexDoc(t *testing.T) {
func TestReverseOrder(t *testing.T) {
var engine Engine
engine.Init(types.EngineOpts{
Using: 1,
SegmenterDict: "./testdata/test_dict.txt",
Using: 1,
GseDict: "./testdata/test_dict.txt",
DefaultRankOpts: &types.RankOpts{
ReverseOrder: true,
OutputOffset: 0,
Expand Down Expand Up @@ -179,8 +179,8 @@ func TestReverseOrder(t *testing.T) {
func TestOffsetAndMaxOutputs(t *testing.T) {
var engine Engine
engine.Init(types.EngineOpts{
Using: 1,
SegmenterDict: "./testdata/test_dict.txt",
Using: 1,
GseDict: "./testdata/test_dict.txt",
DefaultRankOpts: &types.RankOpts{
ReverseOrder: true,
OutputOffset: 1,
Expand Down Expand Up @@ -220,8 +220,8 @@ func (criteria TestScoringCriteria) Score(
func TestSearchWithCriteria(t *testing.T) {
var engine Engine
engine.Init(types.EngineOpts{
Using: 1,
SegmenterDict: "./testdata/test_dict.txt",
Using: 1,
GseDict: "./testdata/test_dict.txt",
DefaultRankOpts: &types.RankOpts{
ScoringCriteria: TestScoringCriteria{},
},
Expand Down Expand Up @@ -250,8 +250,8 @@ func TestSearchWithCriteria(t *testing.T) {
func TestCompactIndex(t *testing.T) {
var engine Engine
engine.Init(types.EngineOpts{
Using: 1,
SegmenterDict: "./testdata/test_dict.txt",
Using: 1,
GseDict: "./testdata/test_dict.txt",
DefaultRankOpts: &types.RankOpts{
ScoringCriteria: TestScoringCriteria{},
},
Expand Down Expand Up @@ -287,8 +287,8 @@ func (criteria BM25ScoringCriteria) Score(
func TestFrequenciesIndex(t *testing.T) {
var engine Engine
engine.Init(types.EngineOpts{
Using: 1,
SegmenterDict: "./testdata/test_dict.txt",
Using: 1,
GseDict: "./testdata/test_dict.txt",
DefaultRankOpts: &types.RankOpts{
ScoringCriteria: BM25ScoringCriteria{},
},
Expand Down Expand Up @@ -316,8 +316,8 @@ func TestFrequenciesIndex(t *testing.T) {
func TestRemoveDoc(t *testing.T) {
var engine Engine
engine.Init(types.EngineOpts{
Using: 1,
SegmenterDict: "./testdata/test_dict.txt",
Using: 1,
GseDict: "./testdata/test_dict.txt",
DefaultRankOpts: &types.RankOpts{
ScoringCriteria: TestScoringCriteria{},
},
Expand Down Expand Up @@ -351,8 +351,8 @@ func TestRemoveDoc(t *testing.T) {
func TestEngineIndexDocWithTokens(t *testing.T) {
var engine Engine
engine.Init(types.EngineOpts{
Using: 1,
SegmenterDict: "./testdata/test_dict.txt",
Using: 1,
GseDict: "./testdata/test_dict.txt",
DefaultRankOpts: &types.RankOpts{
OutputOffset: 0,
MaxOutputs: 10,
Expand Down Expand Up @@ -417,13 +417,13 @@ func TestEngineIndexDocWithTokens(t *testing.T) {
func TestEngineIndexDocWithContentAndLabels(t *testing.T) {
var engine1, engine2 Engine
engine1.Init(types.EngineOpts{
SegmenterDict: "./data/dict/dictionary.txt",
GseDict: "./data/dict/dictionary.txt",
IndexerOpts: &types.IndexerOpts{
IndexType: types.LocsIndex,
},
})
engine2.Init(types.EngineOpts{
SegmenterDict: "./data/dict/dictionary.txt",
GseDict: "./data/dict/dictionary.txt",
IndexerOpts: &types.IndexerOpts{
IndexType: types.DocIdsIndex,
},
Expand All @@ -448,7 +448,7 @@ func TestIndexDocWithLabelsStopTokenFile(t *testing.T) {
var engine1 Engine

engine1.Init(types.EngineOpts{
SegmenterDict: "./data/dict/dictionary.txt",
GseDict: "./data/dict/dictionary.txt",
StopTokenFile: "./testdata/test_stop_dict.txt",
IndexerOpts: &types.IndexerOpts{
IndexType: types.LocsIndex,
Expand All @@ -470,8 +470,8 @@ func TestEngineIndexDocWithPersistentStorage(t *testing.T) {

var engine Engine
engine.Init(types.EngineOpts{
Using: 1,
SegmenterDict: "./testdata/test_dict.txt",
Using: 1,
GseDict: "./testdata/test_dict.txt",
DefaultRankOpts: &types.RankOpts{
OutputOffset: 0,
MaxOutputs: 10,
Expand All @@ -494,8 +494,8 @@ func TestEngineIndexDocWithPersistentStorage(t *testing.T) {

var engine1 Engine
engine1.Init(types.EngineOpts{
Using: 1,
SegmenterDict: "./testdata/test_dict.txt",
Using: 1,
GseDict: "./testdata/test_dict.txt",
DefaultRankOpts: &types.RankOpts{
OutputOffset: 0,
MaxOutputs: 10,
Expand Down Expand Up @@ -533,8 +533,8 @@ func TestEngineIndexDocWithPersistentStorage(t *testing.T) {
func TestCountDocsOnly(t *testing.T) {
var engine Engine
engine.Init(types.EngineOpts{
Using: 1,
SegmenterDict: "./testdata/test_dict.txt",
Using: 1,
GseDict: "./testdata/test_dict.txt",
DefaultRankOpts: &types.RankOpts{
ReverseOrder: true,
OutputOffset: 0,
Expand Down Expand Up @@ -566,8 +566,8 @@ func TestCountDocsOnly(t *testing.T) {
func TestDocOrderless(t *testing.T) {
var engine, engine1 Engine
engine.Init(types.EngineOpts{
Using: 1,
SegmenterDict: "./testdata/test_dict.txt",
Using: 1,
GseDict: "./testdata/test_dict.txt",
})

AddDocs(&engine)
Expand All @@ -585,9 +585,9 @@ func TestDocOrderless(t *testing.T) {
tt.Expect(t, "2", outputs.NumDocs)

engine1.Init(types.EngineOpts{
Using: 1,
IDOnly: true,
SegmenterDict: "./testdata/test_dict.txt",
Using: 1,
IDOnly: true,
GseDict: "./testdata/test_dict.txt",
})

AddDocs(&engine1)
Expand All @@ -610,9 +610,9 @@ func TestDocOrderless(t *testing.T) {
func TestDocOnlyID(t *testing.T) {
var engine Engine
engine.Init(types.EngineOpts{
Using: 1,
IDOnly: true,
SegmenterDict: "./testdata/test_dict.txt",
Using: 1,
IDOnly: true,
GseDict: "./testdata/test_dict.txt",
DefaultRankOpts: &types.RankOpts{
ReverseOrder: true,
OutputOffset: 0,
Expand Down Expand Up @@ -662,8 +662,8 @@ func TestDocOnlyID(t *testing.T) {
func TestSearchWithin(t *testing.T) {
var engine Engine
engine.Init(types.EngineOpts{
Using: 1,
SegmenterDict: "./testdata/test_dict.txt",
Using: 1,
GseDict: "./testdata/test_dict.txt",
DefaultRankOpts: &types.RankOpts{
ReverseOrder: true,
OutputOffset: 0,
Expand Down Expand Up @@ -706,8 +706,8 @@ func TestSearchWithin(t *testing.T) {
func TestSearchJp(t *testing.T) {
var engine Engine
engine.Init(types.EngineOpts{
Using: 1,
SegmenterDict: "./testdata/test_dict_jp.txt",
Using: 1,
GseDict: "./testdata/test_dict_jp.txt",
DefaultRankOpts: &types.RankOpts{
ReverseOrder: true,
OutputOffset: 0,
Expand Down Expand Up @@ -756,7 +756,7 @@ func TestSearchGse(t *testing.T) {
var engine Engine
engine.Init(types.EngineOpts{
// Using: 1,
SegmenterDict: "./testdata/test_dict_jp.txt",
GseDict: "./testdata/test_dict_jp.txt",
DefaultRankOpts: &types.RankOpts{
ReverseOrder: true,
OutputOffset: 0,
Expand Down Expand Up @@ -858,7 +858,7 @@ func TestSearchNotUseGse(t *testing.T) {
func TestSearchLogic(t *testing.T) {
var engine Engine
engine.Init(types.EngineOpts{
SegmenterDict: "./testdata/test_dict_jp.txt",
GseDict: "./testdata/test_dict_jp.txt",
DefaultRankOpts: &types.RankOpts{
ReverseOrder: true,
OutputOffset: 0,
Expand Down
4 changes: 2 additions & 2 deletions examples/benchmark/benchmark.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ var (

func initEngine() {
searcher.Init(types.EngineOpts{
SegmenterDict: *dictionaries,
GseDict: *dictionaries,
StopTokenFile: *stopTokenFile,
IndexerOpts: &types.IndexerOpts{
IndexType: *indexType,
Expand Down Expand Up @@ -188,7 +188,7 @@ func useStore(tBeginInit, tEndInit time.Time) {
t6 := time.Now()
searcher1 := riot.Engine{}
searcher1.Init(types.EngineOpts{
SegmenterDict: *dictionaries,
GseDict: *dictionaries,
StopTokenFile: *stopTokenFile,
IndexerOpts: &types.IndexerOpts{
IndexType: *indexType,
Expand Down
2 changes: 1 addition & 1 deletion examples/codelab/search_server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ func main() {
log.Print("引擎开始初始化")
searcher.Init(types.EngineOpts{
Using: 1,
SegmenterDict: *dictFile,
GseDict: *dictFile,
StopTokenFile: *stopTokenFile,
IndexerOpts: &types.IndexerOpts{
IndexType: types.LocsIndex,
Expand Down
8 changes: 4 additions & 4 deletions examples/dict/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ func dictZh() {
// 初始化
searcher.Init(types.EngineOpts{
// Using: 3,
SegmenterDict: "zh",
// SegmenterDict: "your gopath"+"/src/github.com/go-ego/riot/data/dict/dictionary.txt",
GseDict: "zh",
// GseDict: "your gopath"+"/src/github.com/go-ego/riot/data/dict/dictionary.txt",
})
defer searcher.Close()

Expand All @@ -43,8 +43,8 @@ func dictJp() {
// 初始化
searcher2.Init(types.EngineOpts{
// Using: 3,
SegmenterDict: "jp",
// SegmenterDict: "your gopath"+"/src/github.com/go-ego/riot/data/dict/jp/dict.txt",
GseDict: "jp",
// GseDict: "your gopath"+"/src/github.com/go-ego/riot/data/dict/jp/dict.txt",
})
defer searcher2.Close()

Expand Down
2 changes: 1 addition & 1 deletion examples/pinyin/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func initEngine() {
},
UseStorage: true,
StorageFolder: path,
SegmenterDict: "../../data/dict/dictionary.txt",
GseDict: "../../data/dict/dictionary.txt",
// StopTokenFile: "../../riot/data/dict/stop_tokens.txt",
})
defer searcher.Close()
Expand Down
2 changes: 1 addition & 1 deletion examples/pinyin_weibo/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ func main() {

searcher.Init(types.EngineOpts{
Using: 3,
SegmenterDict: *dictFile,
GseDict: *dictFile,
StopTokenFile: *stopTokenFile,
IndexerOpts: &types.IndexerOpts{
//IndexType: types.LocsIndex,
Expand Down
2 changes: 1 addition & 1 deletion examples/simple/zh/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func main() {
searcher.Init(types.EngineOpts{
Using: 3,
// IDOnly: true,
SegmenterDict: "../../../data/dict/dictionary.txt"})
GseDict: "../../../data/dict/dictionary.txt"})
defer searcher.Close()

text := "此次百度收购将成中国互联网最大并购"
Expand Down
6 changes: 3 additions & 3 deletions examples/store/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ func initEngine() {
UseStorage: true,
StorageFolder: path,
StorageEngine: "bg", // bg: badger, lbd: leveldb, bolt: bolt
// SegmenterDict: "../../data/dict/dictionary.txt",
SegmenterDict: "../../testdata/test_dict.txt",
// GseDict: "../../data/dict/dictionary.txt",
GseDict: "../../testdata/test_dict.txt",
StopTokenFile: "../../data/dict/stop_tokens.txt",
})
defer searcher.Close()
Expand Down Expand Up @@ -78,7 +78,7 @@ func restoreIndex() {
UseStorage: true,
StorageFolder: path,
StorageEngine: "bg", // bg: badger, lbd: leveldb, bolt: bolt
SegmenterDict: "../../data/dict/dictionary.txt",
GseDict: "../../data/dict/dictionary.txt",
StopTokenFile: "../../data/dict/stop_tokens.txt",
})
defer searcher.Close()
Expand Down
2 changes: 1 addition & 1 deletion examples/weibo/custom_scoring_criteria.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ func main() {
gob.Register(WeiboScoringFields{})
searcher.Init(types.EngineOpts{
Using: 1,
SegmenterDict: *dictionaries,
GseDict: *dictionaries,
StopTokenFile: *stopTokenFile,
IndexerOpts: &types.IndexerOpts{
IndexType: types.LocsIndex,
Expand Down
2 changes: 1 addition & 1 deletion net/com/conf.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ type Engine struct {
NumShards int `toml:"num_shards"`
OutputOffset int `toml:"output_offset"`
MaxOutputs int `toml:"max_outputs"`
SegmenterDict string `toml:"segmenter_dict"`
GseDict string `toml:"segmenter_dict"`
StopTokenFile string `toml:"stop_token_file"`
Relation string
Time string
Expand Down
10 changes: 5 additions & 5 deletions net/com/search.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,10 @@ func InitEngine(conf Config) {
numShards = conf.Engine.NumShards
}

// var segmenterDict string
segmenterDict := "../dict/dictionary.txt"
if conf.Engine.SegmenterDict != "" {
segmenterDict = conf.Engine.SegmenterDict
// var GseDict string
GseDict := "../dict/dictionary.txt"
if conf.Engine.GseDict != "" {
GseDict = conf.Engine.GseDict
}
using := conf.Engine.Using

Expand All @@ -76,7 +76,7 @@ func InitEngine(conf Config) {
UseStorage: true,
StorageFolder: path,
StorageEngine: storageEngine,
SegmenterDict: segmenterDict,
GseDict: GseDict,
StopTokenFile: stopTokenFile,
})

Expand Down
Loading

0 comments on commit 98023de

Please sign in to comment.