fumiama/jieba
1
0
Fork 0
mirror of https://github.com/fumiama/jieba.git synced 2026-07-02 01:50:29 +08:00
Code Activity

unify Cut method, return channel instead of array

Browse Source
This commit is contained in:
Wang Bin
2015-02-27 17:56:26 +08:00
parent 43480db509
commit 00fae2358d
1 changed files with 171 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
posseg/posseg.go
View File

@@ -37,8 +37,10 @@ func SetDictionary(dictFileName string) error {
return nil
}
func cutDetailInternal(sentence string) []WordTag {
result := make([]WordTag, 0)
func cutDetailInternal(sentence string) chan WordTag {
result := make(chan WordTag)
go func() {
runes := []rune(sentence)
_, posList := Viterbi(runes)
begin := 0
@@ -49,26 +51,30 @@ func cutDetailInternal(sentence string) []WordTag {
case 'B':
begin = i
case 'E':
result = append(result, WordTag{string(runes[begin : i+1]), posList[i].Tag})
result <- WordTag{string(runes[begin : i+1]), posList[i].Tag}
next = i + 1
case 'S':
result = append(result, WordTag{string(char), posList[i].Tag})
result <- WordTag{string(char), posList[i].Tag}
next = i + 1
}
}
if next < len(runes) {
result = append(result, WordTag{string(runes[next:]), posList[next].Tag})
result <- WordTag{string(runes[next:]), posList[next].Tag}
}
close(result)
}()
return result
}
func cutDetail(sentence string) []WordTag {
result := make([]WordTag, 0)
func cutDetail(sentence string) chan WordTag {
result := make(chan WordTag)
go func() {
blocks := jiebago.RegexpSplit(reHanDetail, sentence)
for _, blk := range blocks {
if reHanDetail.MatchString(blk) {
for _, wordTag := range cutDetailInternal(blk) {
result = append(result, wordTag)
for wordTag := range cutDetailInternal(blk) {
result <- wordTag
}
} else {
for _, x := range jiebago.RegexpSplit(reSkipDetail, blk) {
@@ -77,29 +83,32 @@ func cutDetail(sentence string) []WordTag {
}
switch {
case reNum.MatchString(x):
result = append(result, WordTag{x, "m"})
result <- WordTag{x, "m"}
case reEng.MatchString(x):
result = append(result, WordTag{x, "eng"})
result <- WordTag{x, "eng"}
default:
result = append(result, WordTag{x, "x"})
result <- WordTag{x, "x"}
}
}
}
}
close(result)
}()
return result
}
type cutFunc func(sentence string) []WordTag
type cutFunc func(sentence string) chan WordTag
func cutDAG(sentence string) []WordTag {
func cutDAG(sentence string) chan WordTag {
result := make(chan WordTag)
go func() {
dag := jiebago.DAG(sentence)
routes := jiebago.Calc(sentence, dag)
x := 0
var y int
runes := []rune(sentence)
length := len(runes)
result := make([]WordTag, 0)
buf := make([]rune, 0)
for {
if x >= length {
@@ -114,25 +123,24 @@ func cutDAG(sentence string) []WordTag {
if len(buf) == 1 {
sbuf := string(buf)
if tag, ok := wordTagMap[sbuf]; ok {
result = append(result, WordTag{sbuf, tag})
result <- WordTag{sbuf, tag}
} else {
result = append(result, WordTag{sbuf, "x"})
result <- WordTag{sbuf, "x"}
}
buf = make([]rune, 0)
} else {
bufString := string(buf)
if v, ok := jiebago.Trie.Freq[bufString]; !ok || v == 0.0 {
recognized := cutDetail(bufString)
for _, t := range recognized {
result = append(result, t)
for t := range cutDetail(bufString) {
result <- t
}
} else {
for _, elem := range buf {
selem := string(elem)
if tag, ok := wordTagMap[selem]; ok {
result = append(result, WordTag{string(elem), tag})
result <- WordTag{string(elem), tag}
} else {
result = append(result, WordTag{string(elem), "x"})
result <- WordTag{string(elem), "x"}
}
}
@@ -142,9 +150,9 @@ func cutDAG(sentence string) []WordTag {
}
sl_word := string(l_word)
if tag, ok := wordTagMap[sl_word]; ok {
result = append(result, WordTag{sl_word, tag})
result <- WordTag{sl_word, tag}
} else {
result = append(result, WordTag{sl_word, "x"})
result <- WordTag{sl_word, "x"}
}
}
x = y
@@ -154,34 +162,37 @@ func cutDAG(sentence string) []WordTag {
if len(buf) == 1 {
sbuf := string(buf)
if tag, ok := wordTagMap[sbuf]; ok {
result = append(result, WordTag{sbuf, tag})
result <- WordTag{sbuf, tag}
} else {
result = append(result, WordTag{sbuf, "x"})
result <- WordTag{sbuf, "x"}
}
} else {
bufString := string(buf)
if v, ok := jiebago.Trie.Freq[bufString]; !ok || v == 0.0 {
recognized := cutDetail(bufString)
for _, t := range recognized {
result = append(result, t)
for t := range cutDetail(bufString) {
result <- t
}
} else {
for _, elem := range buf {
selem := string(elem)
if tag, ok := wordTagMap[selem]; ok {
result = append(result, WordTag{selem, tag})
result <- WordTag{selem, tag}
} else {
result = append(result, WordTag{selem, "x"})
result <- WordTag{selem, "x"}
}
}
}
}
}
close(result)
}()
return result
}
func cutDAGNoHMM(sentence string) []WordTag {
result := make([]WordTag, 0)
func cutDAGNoHMM(sentence string) chan WordTag {
result := make(chan WordTag)
go func() {
dag := jiebago.DAG(sentence)
routes := jiebago.Calc(sentence, dag)
x := 0
@@ -200,22 +211,24 @@ func cutDAGNoHMM(sentence string) []WordTag {
x = y
} else {
if len(buf) > 0 {
result = append(result, WordTag{string(buf), "eng"})
result <- WordTag{string(buf), "eng"}
buf = make([]rune, 0)
}
sl_word := string(l_word)
if tag, ok := wordTagMap[sl_word]; ok {
result = append(result, WordTag{sl_word, tag})
result <- WordTag{sl_word, tag}
} else {
result = append(result, WordTag{sl_word, "x"})
result <- WordTag{sl_word, "x"}
}
x = y
}
}
if len(buf) > 0 {
result = append(result, WordTag{string(buf), "eng"})
result <- WordTag{string(buf), "eng"}
buf = make([]rune, 0)
}
close(result)
}()
return result
}
@@ -235,7 +248,7 @@ func Cut(sentence string, HMM bool) chan WordTag {
go func() {
for _, blk := range blocks {
if reHanInternal.MatchString(blk) {
for _, wordTag := range cut(blk) {
for wordTag := range cut(blk) {
result <- wordTag
}
} else {