diff --git a/dictionary.go b/dictionary.go index 630ad3d..be71c64 100644 --- a/dictionary.go +++ b/dictionary.go @@ -9,7 +9,3 @@ type Entry struct { type DictLoader interface { AddEntry(Entry) } - -type Cacher interface { - CacheNameFormat() string -} diff --git a/jieba.go b/jieba.go index d0b606e..1f5e09c 100644 --- a/jieba.go +++ b/jieba.go @@ -85,7 +85,7 @@ func (j *Jieba) LoadUserDict(dictFilePath string) error { // sentence. func NewJieba(dictFileName string) (*Jieba, error) { j := &Jieba{Total: 0.0, Freq: make(map[string]float64)} - err := SetDict(j, dictFileName, false) + err := LoadDict(j, dictFileName, false) return j, err } diff --git a/util.go b/util.go index c2b9716..7ff9e3f 100644 --- a/util.go +++ b/util.go @@ -2,10 +2,6 @@ package jiebago import ( "bufio" - "crypto/md5" - "encoding/gob" - "fmt" - "log" "os" "path/filepath" "regexp" @@ -32,13 +28,12 @@ func LoadDict(l DictLoader, dictFileName string, usingFlag bool) error { return err } - log.Printf("Building Trie..., from %s\n", dictFilePath) - dictFile, err := os.Open(dictFilePath) if err != nil { return err } defer dictFile.Close() + scanner := bufio.NewScanner(dictFile) var entry Entry var line string @@ -61,80 +56,6 @@ func LoadDict(l DictLoader, dictFileName string, usingFlag bool) error { return scanner.Err() } -func cacheFilePath(c Cacher, dictPath string) string { - return filepath.Join(os.TempDir(), - fmt.Sprintf(c.CacheNameFormat(), md5.Sum([]byte(dictPath)))) -} - -func cached(dictPath, cachePath string) (bool, error) { - dictFileInfo, err := os.Stat(dictPath) - if err != nil { - return false, err - } - cacheFileInfo, err := os.Stat(cachePath) - if err != nil { - return false, nil - } - return cacheFileInfo.ModTime().After(dictFileInfo.ModTime()), nil -} - -func load(l DictLoader, cachePath string) error { - cacheFile, err := os.Open(cachePath) - if err != nil { - return err - } - defer cacheFile.Close() - - dec := gob.NewDecoder(cacheFile) - return dec.Decode(l) -} - -func dump(c Cacher, cachePath string) error { - cacheFile, err := os.OpenFile(cachePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - return err - } - defer cacheFile.Close() - enc := gob.NewEncoder(cacheFile) - return enc.Encode(c) -} - -func SetDict(l DictLoader, dictName string, pos bool) error { - dictPath, err := dictPath(dictName) - if err != nil { - return err - } - - var cachePath string - if c, ok := l.(Cacher); ok { - cachePath = cacheFilePath(c, dictPath) - cached, err := cached(dictPath, cachePath) - if err != nil { - return err - } - - if cached { - err = load(l, cachePath) - if err == nil { - log.Printf("loaded model from cache %s\n", cachePath) - return nil - } - } - } - err = LoadDict(l, dictPath, pos) - if err != nil { - return err - } - if c, ok := l.(Cacher); ok { - err = dump(c, cachePath) - if err == nil { - log.Printf("dumped model from cache %s\n", cachePath) - return nil - } - } - return err -} - // Split sentence using regular expression. func RegexpSplit(r *regexp.Regexp, sentence string) chan string { result := make(chan string)