fumiama/jieba
1
0
Fork 0
mirror of https://github.com/fumiama/jieba.git synced 2026-06-22 20:20:44 +08:00
Code Activity

make struct Jieba's fields private

Browse Source
This commit is contained in:
Wang Bin
2015-03-30 16:12:02 +08:00
parent c4c3a5f9ad
commit 68fed7e250
6 changed files with 109 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tokenizers/jieba.go
View File

@@ -19,7 +19,7 @@ type JiebaTokenizer struct {
}
func NewJiebaTokenizer(dictFileName string, hmm, searchMode bool) (analysis.Tokenizer, error) {
j, err := jiebago.NewJieba(dictFileName)
j, err := jiebago.Open(dictFileName)
return &JiebaTokenizer{
j: j,
hmm: hmm,
@@ -44,7 +44,7 @@ func (jt *JiebaTokenizer) Tokenize(input []byte) analysis.TokenStream {
for i := 0; i < width-step+1; i++ {
gram = string(runes[i : i+step])
gramLen := len(gram)
if value, ok := jt.j.Freq[gram]; ok && value > 0 {
if value, ok := jt.j.Freq(gram); ok && value > 0 {
gramStart := start + len(string(runes[:i]))
token := analysis.Token{
Term: []byte(gram),