refactor, removed duplicated codes

analyse/stopwords.go

@@ -1,10 +1,7 @@
 package analyse
 
 import (
-	"bufio"
+	"github.com/wangbin/jiebago"
-	"os"
-	"path/filepath"
-	"strings"
 )
 
 var stopWords map[string]string
@@ -45,26 +42,15 @@ func init() {
 	}
 }
 
-func SetStopWords(stopWordsFilePath string) error {
+func SetStopWords(stopWordsFileName string) error {
-	if !filepath.IsAbs(stopWordsFilePath) {
+	stopWordsFilePath, err := jiebago.DictPath(stopWordsFileName)
-		pwd, err := os.Getwd()
-		if err != nil {
-			return err
-		}
-		stopWordsFilePath = filepath.Clean(filepath.Join(pwd, stopWordsFilePath))
-	}
-	stopWordsFile, err := os.Open(stopWordsFilePath)
 	if err != nil {
 		return err
 	}
-	scanner := bufio.NewScanner(stopWordsFile)
+
-	for scanner.Scan() {
+	wtfs, err := jiebago.ParseDictFile(stopWordsFilePath)
-		stopWord := scanner.Text()
+	for _, wtf := range wtfs {
-		stopWord = strings.TrimSpace(stopWord)
+		stopWords[wtf.Word] = wtf.Word
-		stopWords[stopWord] = stopWord
-	}
-	if err := scanner.Err(); err != nil {
-		return err
 	}
 	return nil
 }