diff --git a/analyse/idf.go b/analyse/idf.go index ae03169..961afef 100644 --- a/analyse/idf.go +++ b/analyse/idf.go @@ -1,12 +1,8 @@ package analyse import ( - "bufio" - "os" - "path/filepath" + "github.com/wangbin/jiebago" "sort" - "strconv" - "strings" ) var ( @@ -33,39 +29,27 @@ func (loader *IDFLoader) newPath(idfFilePath string) error { if loader.Path == idfFilePath { return nil } - idfFile, err := os.Open(idfFilePath) + wtfs, err := jiebago.ParseDictFile(idfFilePath) if err != nil { return err } - scanner := bufio.NewScanner(idfFile) + freqs := make([]float64, 0) - for scanner.Scan() { - line := scanner.Text() - words := strings.Split(line, " ") - word, freqStr := words[0], words[1] - freq, err := strconv.ParseFloat(freqStr, 64) - if err != nil { - continue - } - loader.Freq[word] = freq - freqs = append(freqs, freq) - } - if err := scanner.Err(); err != nil { - return err + + for _, wtf := range wtfs { + loader.Freq[wtf.Word] = wtf.Freq + freqs = append(freqs, wtf.Freq) } + sort.Float64s(freqs) loader.Median = freqs[len(freqs)/2] return nil - } -func SetIdf(idfFilePath string) error { - if !filepath.IsAbs(idfFilePath) { - pwd, err := os.Getwd() - if err != nil { - return err - } - idfFilePath = filepath.Clean(filepath.Join(pwd, idfFilePath)) +func SetIdf(idfFileName string) error { + idfFilePath, err := jiebago.DictPath(idfFileName) + if err != nil { + return err } return idfLoader.newPath(idfFilePath) } diff --git a/dict.go b/dict.go index c01006f..50f7e0e 100644 --- a/dict.go +++ b/dict.go @@ -32,7 +32,7 @@ func ParseDictFile(dictFilePath string) (wtfs []*WordTagFreq, err error) { if err != nil { return } - + defer dictFile.Close() scanner := bufio.NewScanner(dictFile) for scanner.Scan() { line := scanner.Text()