diff --git a/dictionary/dictionary.go b/dictionary/dictionary.go index 7fa7625..083d072 100644 --- a/dictionary/dictionary.go +++ b/dictionary/dictionary.go @@ -16,6 +16,10 @@ type Dictionary struct { sync.RWMutex } +func New() *Dictionary { + return &Dictionary{freqMap: make(map[string]float64)} +} + func (d *Dictionary) addToken(token Token) { d.freqMap[token.text] = token.frequency d.total += token.frequency @@ -84,6 +88,8 @@ func (d *Dictionary) loadDictionary(fileName string, isUserDictionary bool) erro if !isUserDictionary && len(d.freqMap) > 0 { d.freqMap = make(map[string]float64) + d.total = 0.0 + d.logTotal = 0.0 } for scanner.Scan() { line = scanner.Text() diff --git a/dictionary/dictionary_test.go b/dictionary/dictionary_test.go new file mode 100644 index 0000000..1bd82af --- /dev/null +++ b/dictionary/dictionary_test.go @@ -0,0 +1,45 @@ +package dictionary + +import ( + "math" + "testing" +) + +var d *Dictionary + +func init() { + d = New() +} + +func TestLoadDictionary(t *testing.T) { + if err := d.LoadDictionary("../dict.txt"); err != nil { + t.Fatalf("Failed to load dict.txt, err = %s", err) + } + n := len(d.freqMap) + + d.LoadDictionary("../foobar.txt") + if len(d.freqMap) == n { + t.Fatalf("Failed to load foobar.txt") + } +} + +func TestLoadUserDictionary(t *testing.T) { + err := d.LoadUserDictionary("../userdict.txt") + if err != nil { + t.Fatalf("Failed to load userdict.txt, err = %s", err) + } + if f, _ := d.Frequency("八一双鹿"); f != 3.0 { + t.Fatalf("Wrong frequency for word \"八一双鹿\", expect 3.0, got %f", f) + } +} + +func TestTotal(t *testing.T) { + d.LoadDictionary("../userdict.txt") + + if d.Total() != 319.0 { + t.Fatalf("Wrong total for userdict.txt, expect 319.0, got %f", d.Total()) + } + if d.LogTotal() != math.Log(319.0) { + t.Fatalf("Wrong total for userdict.txt, expect %f, got %f", math.Log(319.0), d.LogTotal()) + } +}