fumiama/jieba
1
0
Fork 0
mirror of https://github.com/fumiama/jieba.git synced 2026-06-09 02:50:24 +08:00
Code Activity

refactor Cut function, make CutAll a seperate function, to simplify the logic of Cut function

Browse Source
This commit is contained in:
Wang Bin
2015-03-30 15:18:36 +08:00
parent 556b96b137
commit c4c3a5f9ad
5 changed files with 44 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tokenizers/jieba.go
View File

@@ -35,7 +35,7 @@ func (jt *JiebaTokenizer) Tokenize(input []byte) analysis.TokenStream {
pos := 1
var width int
var gram string
for word := range jt.j.Cut(string(input), false, jt.hmm) {
for word := range jt.j.Cut(string(input), jt.hmm) {
if jt.searchMode {
runes := []rune(word)
width = len(runes)