fumiama/jieba
1
0
Fork 0
mirror of https://github.com/fumiama/jieba.git synced 2026-06-20 02:30:32 +08:00
Code Activity

using bufio.Scanner to reading files, this the recommended way since Go v1.1

Browse Source
This commit is contained in:
Wang Bin
2014-08-11 16:43:38 +08:00
parent b46a8dd73d
commit bd12e7682d
1 changed files with 7 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
trie_node.go
View File

@@ -45,18 +45,19 @@ func newTopTrie(filename string) (*TopTrie, error) {
} }
defer file.Close() defer file.Close()
reader := bufio.NewReader(file) scanner := bufio.NewScanner(file)
for { for scanner.Scan() {
line, readError := reader.ReadString('\n') line := scanner.Text()
if readError != nil && len(line) == 0 {
break
}
words := strings.Split(line, " ") words := strings.Split(line, " ")
word, freqStr := words[0], words[1] word, freqStr := words[0], words[1]
freq, _ := strconv.ParseFloat(freqStr, 64) freq, _ := strconv.ParseFloat(freqStr, 64)
topTrie.Total += freq topTrie.Total += freq
topTrie.addWord(word, freq) topTrie.addWord(word, freq)
} }
if scanErr := scanner.Err(); scanErr != nil {
return nil, scanErr
}
var val float64 var val float64
for key := range topTrie.Freq { for key := range topTrie.Freq {
val = math.Log(topTrie.Freq[key] / topTrie.Total) val = math.Log(topTrie.Freq[key] / topTrie.Total)