jieba/analyse/idf.go

package analyse

import (
	"github.com/wangbin/jiebago"
	"sort"
)

type IDFLoader struct {
	IDFFreq map[string]float64
	Median  float64
	freqs   []float64
}

func (l *IDFLoader) AddEntry(entry *jiebago.Entry) {
	l.IDFFreq[entry.Word] = entry.Freq
	l.freqs = append(l.freqs, entry.Freq)
}

func NewIDFLoader(IDFFileName string) (*IDFLoader, error) {
	IDFFilePath, err := jiebago.DictPath(IDFFileName)
	if err != nil {
		return nil, err
	}
	loader := &IDFLoader{make(map[string]float64), 0.0, make([]float64, 0)}
	err = jiebago.LoadDict(loader, IDFFilePath, false)
	if err != nil {
		return nil, err
	}

	sort.Float64s(loader.freqs)
	loader.Median = loader.freqs[len(loader.freqs)/2]
	loader.freqs = []float64{}
	return loader, nil
}