fumiama/jieba
1
0
Fork 0
mirror of https://github.com/fumiama/jieba.git synced 2026-06-10 11:40:26 +08:00
Code Activity

moved tokenizers to a seperated module

Browse Source
This commit is contained in:
Wang Bin
2015-05-07 18:52:29 +08:00
parent 7440fa00df
commit 3d91f615cf
6 changed files with 62 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
example_test.go
View File

@@ -57,41 +57,3 @@ func Example_loadUserDictionary() {
// Before: 李小福 / 是 / 创新 / 办 / 主任 / 也 / 是 / 云 / 计算 / 方面 / 的 / 专家 /
// After: 李小福 / 是 / 创新办 / 主任 / 也 / 是 / 云计算 / 方面 / 的 / 专家 /
}
func Example_tokenize() {
var seg jiebago.Segmenter
seg.LoadDictionary("dict.txt")
sentence := []byte("永和服装饰品有限公司")
// default mode
tokenizer, _ := jiebago.NewJiebaTokenizer("dict.txt", true, false)
fmt.Println("Default Mode:")
for _, token := range tokenizer.Tokenize(sentence) {
fmt.Printf(
"Term: %s Start: %d End: %d Position: %d Type: %d\n",
token.Term, token.Start, token.End, token.Position, token.Type)
}
//search mode
tokenizer, _ = jiebago.NewJiebaTokenizer("dict.txt", true, true)
fmt.Println("Search Mode:")
for _, token := range tokenizer.Tokenize(sentence) {
fmt.Printf(
"Term: %s Start: %d End: %d Position: %d Type: %d\n",
token.Term, token.Start, token.End, token.Position, token.Type)
}
// Output:
// Default Mode:
// Term: 永和 Start: 0 End: 6 Position: 1 Type: 1
// Term: 服装 Start: 6 End: 12 Position: 2 Type: 1
// Term: 饰品 Start: 12 End: 18 Position: 3 Type: 1
// Term: 有限公司 Start: 18 End: 30 Position: 4 Type: 1
// Search Mode:
// Term: 永和 Start: 0 End: 6 Position: 1 Type: 1
// Term: 服装 Start: 6 End: 12 Position: 2 Type: 1
// Term: 饰品 Start: 12 End: 18 Position: 3 Type: 1
// Term: 有限 Start: 18 End: 24 Position: 4 Type: 1
// Term: 公司 Start: 24 End: 30 Position: 5 Type: 1
// Term: 有限公司 Start: 18 End: 30 Position: 6 Type: 1
}