1
0
mirror of https://github.com/fumiama/jieba.git synced 2026-06-30 09:00:30 +08:00

small code refactor

This commit is contained in:
Wang Bin
2015-05-07 10:59:01 +08:00
parent c068670e9b
commit 939e903f31

View File

@@ -186,7 +186,9 @@ func (seg *Segmenter) cutDAG(sentence string) <-chan Segment {
frag := runes[x:y]
if y-x == 1 {
buf = append(buf, frag...)
} else {
x = y
continue
}
if len(buf) > 0 {
bufString := string(buf)
if len(buf) == 1 {
@@ -196,7 +198,8 @@ func (seg *Segmenter) cutDAG(sentence string) <-chan Segment {
result <- Segment{bufString, "x"}
}
buf = make([]rune, 0)
} else {
continue
}
if v, ok := seg.dict.Frequency(bufString); !ok || v == 0.0 {
for t := range seg.cutDetail(bufString) {
result <- t
@@ -214,14 +217,12 @@ func (seg *Segmenter) cutDAG(sentence string) <-chan Segment {
}
buf = make([]rune, 0)
}
}
word := string(frag)
if tag, ok := seg.dict.Pos(word); ok {
result <- Segment{word, tag}
} else {
result <- Segment{word, "x"}
}
}
x = y
}
@@ -270,7 +271,8 @@ func (seg *Segmenter) cutDAGNoHMM(sentence string) <-chan Segment {
if reEng1.MatchString(string(frag)) && len(frag) == 1 {
buf = append(buf, frag...)
x = y
} else {
continue
}
if len(buf) > 0 {
result <- Segment{string(buf), "eng"}
buf = make([]rune, 0)
@@ -282,7 +284,7 @@ func (seg *Segmenter) cutDAGNoHMM(sentence string) <-chan Segment {
result <- Segment{word, "x"}
}
x = y
}
}
if len(buf) > 0 {
result <- Segment{string(buf), "eng"}
@@ -309,11 +311,13 @@ func (seg *Segmenter) Cut(sentence string, hmm bool) <-chan Segment {
for wordTag := range cut(blk) {
result <- wordTag
}
} else {
continue
}
for _, x := range util.RegexpSplit(reSkipInternal, blk, -1) {
if reSkipInternal.MatchString(x) {
result <- Segment{x, "x"}
} else {
continue
}
for _, xx := range x {
s := string(xx)
switch {
@@ -327,8 +331,6 @@ func (seg *Segmenter) Cut(sentence string, hmm bool) <-chan Segment {
}
}
}
}
}
close(result)
}()
return result