fumiama/jieba
1
0
Fork 0
mirror of https://github.com/fumiama/jieba.git synced 2026-06-28 08:02:45 +08:00
Code Activity

small code refactor

Browse Source
This commit is contained in:
Wang Bin
2015-05-07 10:59:01 +08:00
parent c068670e9b
commit 939e903f31
1 changed files with 60 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
posseg/posseg.go
View File

@@ -186,7 +186,9 @@ func (seg *Segmenter) cutDAG(sentence string) <-chan Segment {
frag := runes[x:y] frag := runes[x:y]
if y-x == 1 { if y-x == 1 {
buf = append(buf, frag...) buf = append(buf, frag...)
} else { x = y
continue
}
if len(buf) > 0 { if len(buf) > 0 {
bufString := string(buf) bufString := string(buf)
if len(buf) == 1 { if len(buf) == 1 {
@@ -196,7 +198,8 @@ func (seg *Segmenter) cutDAG(sentence string) <-chan Segment {
result <- Segment{bufString, "x"} result <- Segment{bufString, "x"}
} }
buf = make([]rune, 0) buf = make([]rune, 0)
} else { continue
}
if v, ok := seg.dict.Frequency(bufString); !ok || v == 0.0 { if v, ok := seg.dict.Frequency(bufString); !ok || v == 0.0 {
for t := range seg.cutDetail(bufString) { for t := range seg.cutDetail(bufString) {
result <- t result <- t
@@ -214,14 +217,12 @@ func (seg *Segmenter) cutDAG(sentence string) <-chan Segment {
} }
buf = make([]rune, 0) buf = make([]rune, 0)
} }
}
word := string(frag) word := string(frag)
if tag, ok := seg.dict.Pos(word); ok { if tag, ok := seg.dict.Pos(word); ok {
result <- Segment{word, tag} result <- Segment{word, tag}
} else { } else {
result <- Segment{word, "x"} result <- Segment{word, "x"}
} }
}
x = y x = y
} }
@@ -270,7 +271,8 @@ func (seg *Segmenter) cutDAGNoHMM(sentence string) <-chan Segment {
if reEng1.MatchString(string(frag)) && len(frag) == 1 { if reEng1.MatchString(string(frag)) && len(frag) == 1 {
buf = append(buf, frag...) buf = append(buf, frag...)
x = y x = y
} else { continue
}
if len(buf) > 0 { if len(buf) > 0 {
result <- Segment{string(buf), "eng"} result <- Segment{string(buf), "eng"}
buf = make([]rune, 0) buf = make([]rune, 0)
@@ -282,7 +284,7 @@ func (seg *Segmenter) cutDAGNoHMM(sentence string) <-chan Segment {
result <- Segment{word, "x"} result <- Segment{word, "x"}
} }
x = y x = y
}
} }
if len(buf) > 0 { if len(buf) > 0 {
result <- Segment{string(buf), "eng"} result <- Segment{string(buf), "eng"}
@@ -309,11 +311,13 @@ func (seg *Segmenter) Cut(sentence string, hmm bool) <-chan Segment {
for wordTag := range cut(blk) { for wordTag := range cut(blk) {
result <- wordTag result <- wordTag
} }
} else { continue
}
for _, x := range util.RegexpSplit(reSkipInternal, blk, -1) { for _, x := range util.RegexpSplit(reSkipInternal, blk, -1) {
if reSkipInternal.MatchString(x) { if reSkipInternal.MatchString(x) {
result <- Segment{x, "x"} result <- Segment{x, "x"}
} else { continue
}
for _, xx := range x { for _, xx := range x {
s := string(xx) s := string(xx)
switch { switch {
@@ -327,8 +331,6 @@ func (seg *Segmenter) Cut(sentence string, hmm bool) <-chan Segment {
} }
} }
} }
}
}
close(result) close(result)
}() }()
return result return result