code refactor, added more documents

dictionary/dictionary.go

@@ -8,6 +8,8 @@ import (
 	"strings"
 )
 
+// DictLoader represents a interface that could add one token or load bunch of
+// tokens from channel.
 type DictLoader interface {
 	Load(<-chan Token)
 	AddToken(Token)
@@ -49,6 +51,7 @@ func loadDictionary(file *os.File) (<-chan Token, <-chan error) {
 
 }
 
+// LoadDictionary reads the given file and passes all tokens to a DictLoader.
 func LoadDictionary(dl DictLoader, fileName string) error {
 	filePath, err := dictPath(fileName)
 	if err != nil {

dictionary/token.go

@@ -1,23 +1,28 @@
 package dictionary
 
+// Token represents a Chinese word with (optional) frequency and POS.
 type Token struct {
 	text      string
 	frequency float64
 	pos       string
 }
 
+//Text returns token's text.
 func (t Token) Text() string {
 	return t.text
 }
 
+// Frequency returns token's frequency.
 func (t Token) Frequency() float64 {
 	return t.frequency
 }
 
+// Pos returns token's POS.
 func (t Token) Pos() string {
 	return t.pos
 }
 
+// NewToken creates a new token.
 func NewToken(text string, frequency float64, pos string) Token {
 	return Token{text: text, frequency: frequency, pos: pos}
 }