refactor, removed duplicated codes

analyse/idf.go

@@ -1,12 +1,8 @@
 package analyse
 
 import (
-	"bufio"
+	"github.com/wangbin/jiebago"
-	"os"
-	"path/filepath"
 	"sort"
-	"strconv"
-	"strings"
 )
 
 var (
@@ -33,39 +29,27 @@ func (loader *IDFLoader) newPath(idfFilePath string) error {
 	if loader.Path == idfFilePath {
 		return nil
 	}
-	idfFile, err := os.Open(idfFilePath)
+	wtfs, err := jiebago.ParseDictFile(idfFilePath)
 	if err != nil {
 		return err
 	}
-	scanner := bufio.NewScanner(idfFile)
+
 	freqs := make([]float64, 0)
-	for scanner.Scan() {
+
-		line := scanner.Text()
+	for _, wtf := range wtfs {
-		words := strings.Split(line, " ")
+		loader.Freq[wtf.Word] = wtf.Freq
-		word, freqStr := words[0], words[1]
+		freqs = append(freqs, wtf.Freq)
-		freq, err := strconv.ParseFloat(freqStr, 64)
-		if err != nil {
-			continue
-		}
-		loader.Freq[word] = freq
-		freqs = append(freqs, freq)
-	}
-	if err := scanner.Err(); err != nil {
-		return err
 	}
+
 	sort.Float64s(freqs)
 	loader.Median = freqs[len(freqs)/2]
 	return nil
-
 }
 
-func SetIdf(idfFilePath string) error {
+func SetIdf(idfFileName string) error {
-	if !filepath.IsAbs(idfFilePath) {
+	idfFilePath, err := jiebago.DictPath(idfFileName)
-		pwd, err := os.Getwd()
+	if err != nil {
-		if err != nil {
+		return err
-			return err
-		}
-		idfFilePath = filepath.Clean(filepath.Join(pwd, idfFilePath))
 	}
 	return idfLoader.newPath(idfFilePath)
 }

dict.go

@@ -32,7 +32,7 @@ func ParseDictFile(dictFilePath string) (wtfs []*WordTagFreq, err error) {
 	if err != nil {
 		return
 	}
-
+	defer dictFile.Close()
 	scanner := bufio.NewScanner(dictFile)
 	for scanner.Scan() {
 		line := scanner.Text()