fumiama/jieba
1
0
Fork 0
mirror of https://github.com/fumiama/jieba.git synced 2026-06-05 00:32:51 +08:00
Code Activity

small code refactor

Browse Source
This commit is contained in:
Wang Bin
2015-02-04 14:47:59 +08:00
parent c3dce0d4a1
commit 0f7c56b4ef
5 changed files with 20 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@@ -1,3 +1,3 @@
language: go
go:
- 1.3
- 1.4.1

22
jieba.go
View File

@@ -9,7 +9,6 @@ import (
var (
Dictionary = "dict.txt"
trie *Trie
UserWordTagTab = make(map[string]string)
)
@@ -77,10 +76,10 @@ func GetDAG(sentence string) map[int][]int {
i = k
frag = string(runes[k])
for {
if !trie.Nodes.Contains(frag) {
if !T.Nodes.Contains(frag) {
break
}
if _, ok := trie.Freq[frag]; ok {
if _, ok := T.Freq[frag]; ok {
tmpList = append(tmpList, i)
}
i += 1
@@ -112,10 +111,10 @@ func Calc(sentence string, dag map[int][]int, idx int) map[int]*Route {
word = string(runes[idx : i+1])
}
var route *Route
if _, ok := trie.Freq[word]; ok {
route = &Route{trie.Freq[word] + routes[i+1].Freq, i}
if _, ok := T.Freq[word]; ok {
route = &Route{T.Freq[word] + routes[i+1].Freq, i}
} else {
route = &Route{trie.MinFreq + routes[i+1].Freq, i}
route = &Route{T.MinFreq + routes[i+1].Freq, i}
}
candidates = append(candidates, route)
}
@@ -151,7 +150,7 @@ func cut_DAG(sentence string) []string {
buf = make([]rune, 0)
} else {
bufString := string(buf)
if _, ok := trie.Freq[bufString]; !ok {
if _, ok := T.Freq[bufString]; !ok {
recognized := finalseg.Cut(bufString)
for _, t := range recognized {
result = append(result, t)
@@ -174,7 +173,7 @@ func cut_DAG(sentence string) []string {
result = append(result, string(buf))
} else {
bufString := string(buf)
if _, ok := trie.Freq[bufString]; !ok {
if _, ok := T.Freq[bufString]; !ok {
recognized := finalseg.Cut(bufString)
for _, t := range recognized {
result = append(result, t)
@@ -318,7 +317,7 @@ func CutForSearch(sentence string, hmm bool) []string {
var gram2 string
for i := 0; i < len(runes)-increment+1; i++ {
gram2 = string(runes[i : i+increment])
if _, ok := trie.Freq[gram2]; ok {
if _, ok := T.Freq[gram2]; ok {
result = append(result, gram2)
}
}
@@ -328,8 +327,3 @@ func CutForSearch(sentence string, hmm bool) []string {
}
return result
}
func SetDictionary(dict_path string) (err error) {
trie, err = newTrie(dict_path)
return
}

4
posseg/posseg.go
View File

@@ -142,7 +142,7 @@ func cut_DAG(sentence string) []WordTag {
buf = make([]rune, 0)
} else {
bufString := string(buf)
if _, ok := jiebago.TT.Freq[bufString]; !ok {
if _, ok := jiebago.T.Freq[bufString]; !ok {
recognized := cutDetail(bufString)
for _, t := range recognized {
result = append(result, t)
@@ -181,7 +181,7 @@ func cut_DAG(sentence string) []WordTag {
}
} else {
bufString := string(buf)
if _, ok := jiebago.TT.Freq[bufString]; !ok {
if _, ok := jiebago.T.Freq[bufString]; !ok {
recognized := cutDetail(bufString)
for _, t := range recognized {
result = append(result, t)

2
tokenize.go
View File

@@ -24,7 +24,7 @@ func Tokenize(sentence string, mode string, HMM bool) []Token {
if width > step {
for i := 0; i < width-step+1; i++ {
gram := string(runes[i : i+step])
if _, ok := trie.Freq[gram]; ok {
if _, ok := T.Freq[gram]; ok {
tokens = append(tokens, Token{gram, start + i, start + i + step})
}
}

9
trie_node.go
View File

@@ -15,6 +15,8 @@ import (
"strings"
)
var T *Trie
type Trie struct {
Nodes mapset.Set
MinFreq float64
@@ -181,7 +183,7 @@ func addWord(word string, freq float64, tag string) {
if len(tag) > 0 {
UserWordTagTab[word] = strings.TrimSpace(tag)
}
trie.addWord(word, freq)
T.addWord(word, freq)
}
func LoadUserDict(filePath string) error {
@@ -210,3 +212,8 @@ func LoadUserDict(filePath string) error {
return scanner.Err()
}
func SetDictionary(dict_path string) (err error) {
T, err = newTrie(dict_path)
return
}