fumiama/jieba
1
0
Fork 0
mirror of https://github.com/fumiama/jieba.git synced 2026-06-11 20:50:29 +08:00
Code Activity

优化 dict, add fs.File 支持

Browse Source
This commit is contained in:
源文雨
2022-11-30 14:14:48 +08:00
parent c8785c7994
commit f3da9e6420
22 changed files with 190 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tokenizers/tokenizer_test.go
View File

@@ -5219,7 +5219,7 @@ func TestJiebaTokenizerDefaultModeWithHMM(t *testing.T) {
},
}
tokenizer, _ := NewJiebaTokenizer("../dict.txt", true, false)
tokenizer, _ := NewJiebaTokenizerAt("../dict.txt", true, false)
for _, test := range tests {
actual := tokenizer.Tokenize(test.input)
if !reflect.DeepEqual(actual, test.output) {
@@ -11057,7 +11057,7 @@ func TestJiebaTokenizerSearchModeWithHMM(t *testing.T) {
},
}
tokenizer, _ := NewJiebaTokenizer("../dict.txt", true, true)
tokenizer, _ := NewJiebaTokenizerAt("../dict.txt", true, true)
for _, test := range tests {
actual := tokenizer.Tokenize(test.input)
if !reflect.DeepEqual(actual, test.output) {
@@ -16474,7 +16474,7 @@ func TestJiebaTokenizerDefaultModeWithoutHMM(t *testing.T) {
},
}
tokenizer, _ := NewJiebaTokenizer("../dict.txt", false, false)
tokenizer, _ := NewJiebaTokenizerAt("../dict.txt", false, false)
for _, test := range tests {
actual := tokenizer.Tokenize(test.input)
if !reflect.DeepEqual(actual, test.output) {
@@ -22506,7 +22506,7 @@ func TestJiebaTokenizerSearchModeWithoutHMM(t *testing.T) {
},
}
tokenizer, _ := NewJiebaTokenizer("../dict.txt", false, true)
tokenizer, _ := NewJiebaTokenizerAt("../dict.txt", false, true)
for _, test := range tests {
actual := tokenizer.Tokenize(test.input)
if !reflect.DeepEqual(actual, test.output) {