unify Cut method, return channel instead of array

posseg/posseg.go

@@ -37,8 +37,10 @@ func SetDictionary(dictFileName string) error {
 	return nil
 }
 
-func cutDetailInternal(sentence string) []WordTag {
+func cutDetailInternal(sentence string) chan WordTag {
-	result := make([]WordTag, 0)
+	result := make(chan WordTag)
+
+	go func() {
 		runes := []rune(sentence)
 		_, posList := Viterbi(runes)
 		begin := 0
@@ -49,26 +51,30 @@ func cutDetailInternal(sentence string) []WordTag {
 			case 'B':
 				begin = i
 			case 'E':
-			result = append(result, WordTag{string(runes[begin : i+1]), posList[i].Tag})
+				result <- WordTag{string(runes[begin : i+1]), posList[i].Tag}
 				next = i + 1
 			case 'S':
-			result = append(result, WordTag{string(char), posList[i].Tag})
+				result <- WordTag{string(char), posList[i].Tag}
 				next = i + 1
 			}
 		}
 		if next < len(runes) {
-		result = append(result, WordTag{string(runes[next:]), posList[next].Tag})
+			result <- WordTag{string(runes[next:]), posList[next].Tag}
 		}
+		close(result)
+	}()
 	return result
 }
 
-func cutDetail(sentence string) []WordTag {
+func cutDetail(sentence string) chan WordTag {
-	result := make([]WordTag, 0)
+	result := make(chan WordTag)
+
+	go func() {
 		blocks := jiebago.RegexpSplit(reHanDetail, sentence)
 		for _, blk := range blocks {
 			if reHanDetail.MatchString(blk) {
-			for _, wordTag := range cutDetailInternal(blk) {
+				for wordTag := range cutDetailInternal(blk) {
-				result = append(result, wordTag)
+					result <- wordTag
 				}
 			} else {
 				for _, x := range jiebago.RegexpSplit(reSkipDetail, blk) {
@@ -77,29 +83,32 @@ func cutDetail(sentence string) []WordTag {
 					}
 					switch {
 					case reNum.MatchString(x):
-					result = append(result, WordTag{x, "m"})
+						result <- WordTag{x, "m"}
 					case reEng.MatchString(x):
-					result = append(result, WordTag{x, "eng"})
+						result <- WordTag{x, "eng"}
 					default:
-					result = append(result, WordTag{x, "x"})
+						result <- WordTag{x, "x"}
 					}
 				}
 			}
 		}
-
+		close(result)
+	}()
 	return result
 }
 
-type cutFunc func(sentence string) []WordTag
+type cutFunc func(sentence string) chan WordTag
 
-func cutDAG(sentence string) []WordTag {
+func cutDAG(sentence string) chan WordTag {
+	result := make(chan WordTag)
+
+	go func() {
 		dag := jiebago.DAG(sentence)
 		routes := jiebago.Calc(sentence, dag)
 		x := 0
 		var y int
 		runes := []rune(sentence)
 		length := len(runes)
-	result := make([]WordTag, 0)
 		buf := make([]rune, 0)
 		for {
 			if x >= length {
@@ -114,25 +123,24 @@ func cutDAG(sentence string) []WordTag {
 					if len(buf) == 1 {
 						sbuf := string(buf)
 						if tag, ok := wordTagMap[sbuf]; ok {
-						result = append(result, WordTag{sbuf, tag})
+							result <- WordTag{sbuf, tag}
 						} else {
-						result = append(result, WordTag{sbuf, "x"})
+							result <- WordTag{sbuf, "x"}
 						}
 						buf = make([]rune, 0)
 					} else {
 						bufString := string(buf)
 						if v, ok := jiebago.Trie.Freq[bufString]; !ok || v == 0.0 {
-						recognized := cutDetail(bufString)
+							for t := range cutDetail(bufString) {
-						for _, t := range recognized {
+								result <- t
-							result = append(result, t)
 							}
 						} else {
 							for _, elem := range buf {
 								selem := string(elem)
 								if tag, ok := wordTagMap[selem]; ok {
-								result = append(result, WordTag{string(elem), tag})
+									result <- WordTag{string(elem), tag}
 								} else {
-								result = append(result, WordTag{string(elem), "x"})
+									result <- WordTag{string(elem), "x"}
 								}
 
 							}
@@ -142,9 +150,9 @@ func cutDAG(sentence string) []WordTag {
 				}
 				sl_word := string(l_word)
 				if tag, ok := wordTagMap[sl_word]; ok {
-				result = append(result, WordTag{sl_word, tag})
+					result <- WordTag{sl_word, tag}
 				} else {
-				result = append(result, WordTag{sl_word, "x"})
+					result <- WordTag{sl_word, "x"}
 				}
 			}
 			x = y
@@ -154,34 +162,37 @@ func cutDAG(sentence string) []WordTag {
 			if len(buf) == 1 {
 				sbuf := string(buf)
 				if tag, ok := wordTagMap[sbuf]; ok {
-				result = append(result, WordTag{sbuf, tag})
+					result <- WordTag{sbuf, tag}
 				} else {
-				result = append(result, WordTag{sbuf, "x"})
+					result <- WordTag{sbuf, "x"}
 				}
 			} else {
 				bufString := string(buf)
 				if v, ok := jiebago.Trie.Freq[bufString]; !ok || v == 0.0 {
-				recognized := cutDetail(bufString)
+					for t := range cutDetail(bufString) {
-				for _, t := range recognized {
+						result <- t
-					result = append(result, t)
 					}
 				} else {
 					for _, elem := range buf {
 						selem := string(elem)
 						if tag, ok := wordTagMap[selem]; ok {
-						result = append(result, WordTag{selem, tag})
+							result <- WordTag{selem, tag}
 						} else {
-						result = append(result, WordTag{selem, "x"})
+							result <- WordTag{selem, "x"}
 						}
 					}
 				}
 			}
 		}
+		close(result)
+	}()
 	return result
 }
 
-func cutDAGNoHMM(sentence string) []WordTag {
+func cutDAGNoHMM(sentence string) chan WordTag {
-	result := make([]WordTag, 0)
+	result := make(chan WordTag)
+
+	go func() {
 		dag := jiebago.DAG(sentence)
 		routes := jiebago.Calc(sentence, dag)
 		x := 0
@@ -200,22 +211,24 @@ func cutDAGNoHMM(sentence string) []WordTag {
 				x = y
 			} else {
 				if len(buf) > 0 {
-				result = append(result, WordTag{string(buf), "eng"})
+					result <- WordTag{string(buf), "eng"}
 					buf = make([]rune, 0)
 				}
 				sl_word := string(l_word)
 				if tag, ok := wordTagMap[sl_word]; ok {
-				result = append(result, WordTag{sl_word, tag})
+					result <- WordTag{sl_word, tag}
 				} else {
-				result = append(result, WordTag{sl_word, "x"})
+					result <- WordTag{sl_word, "x"}
 				}
 				x = y
 			}
 		}
 		if len(buf) > 0 {
-		result = append(result, WordTag{string(buf), "eng"})
+			result <- WordTag{string(buf), "eng"}
 			buf = make([]rune, 0)
 		}
+		close(result)
+	}()
 	return result
 }
 
@@ -235,7 +248,7 @@ func Cut(sentence string, HMM bool) chan WordTag {
 	go func() {
 		for _, blk := range blocks {
 			if reHanInternal.MatchString(blk) {
-				for _, wordTag := range cut(blk) {
+				for wordTag := range cut(blk) {
 					result <- wordTag
 				}
 			} else {