fumiama/jieba
1
0
Fork 0
mirror of https://github.com/fumiama/jieba.git synced 2026-06-28 16:10:32 +08:00
Code Activity

make some public variable/function to private

Browse Source
This commit is contained in:
Wang Bin
2015-02-28 17:51:03 +08:00
parent 84bd2e14e2
commit 30b40c34df
4 changed files with 31 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
finalseg/prob_emit.go
View File

@@ -1,9 +1,9 @@
package finalseg package finalseg
var ProbEmit = make(map[byte]map[rune]float64) var probEmit = make(map[byte]map[rune]float64)
func init() { func init() {
ProbEmit['B'] = map[rune]float64{'\u4e00': -3.6544978750449433, probEmit['B'] = map[rune]float64{'\u4e00': -3.6544978750449433,
'\u4e01': -8.125041941842026, '\u4e01': -8.125041941842026,
'\u4e03': -7.817392401429855, '\u4e03': -7.817392401429855,
'\u4e07': -6.3096425804013165, '\u4e07': -6.3096425804013165,
@@ -6860,7 +6860,7 @@ func init() {
'\u9f9a': -9.557108305917183, '\u9f9a': -9.557108305917183,
'\u9f9c': -10.895131537474946, '\u9f9c': -10.895131537474946,
'\u9f9f': -10.895131537474946} '\u9f9f': -10.895131537474946}
ProbEmit['E'] = map[rune]float64{'\u4e00': -6.044987536255073, probEmit['E'] = map[rune]float64{'\u4e00': -6.044987536255073,
'\u4e01': -9.075800412310807, '\u4e01': -9.075800412310807,
'\u4e03': -9.198842005220659, '\u4e03': -9.198842005220659,
'\u4e07': -7.655326112989935, '\u4e07': -7.655326112989935,
@@ -14299,7 +14299,7 @@ func init() {
'\u9f9b': -12.729311722586953, '\u9f9b': -12.729311722586953,
'\u9f9c': -10.574067217491615, '\u9f9c': -10.574067217491615,
'\u9f9f': -10.574067217491615} '\u9f9f': -10.574067217491615}
ProbEmit['M'] = map[rune]float64{'\u4e00': -4.428158526435913, probEmit['M'] = map[rune]float64{'\u4e00': -4.428158526435913,
'\u4e01': -7.932945687598502, '\u4e01': -7.932945687598502,
'\u4e03': -6.559715525951586, '\u4e03': -6.559715525951586,
'\u4e07': -6.139922374120667, '\u4e07': -6.139922374120667,
@@ -20708,7 +20708,7 @@ func init() {
'\u9f9a': -14.14915250738439, '\u9f9a': -14.14915250738439,
'\u9f9c': -11.058110054026073, '\u9f9c': -11.058110054026073,
'\u9f9f': -11.058110054026073} '\u9f9f': -11.058110054026073}
ProbEmit['S'] = map[rune]float64{'\u2236': -15.828865681131282, probEmit['S'] = map[rune]float64{'\u2236': -15.828865681131282,
'\u4e00': -4.92368982120877, '\u4e00': -4.92368982120877,
'\u4e01': -9.024528361347633, '\u4e01': -9.024528361347633,
'\u4e02': -16.522012861691227, '\u4e02': -16.522012861691227,

10
finalseg/prob_start.go
View File

@@ -1,10 +0,0 @@
package finalseg
var ProbStart = make(map[byte]float64)
func init() {
ProbStart['B'] = -0.26268660809250016
ProbStart['E'] = -3.14e+100
ProbStart['M'] = -3.14e+100
ProbStart['S'] = -1.4652633398537678
}

10
finalseg/prob_trans.go
View File

@@ -1,14 +1,14 @@
package finalseg package finalseg
var ProbTrans = make(map[byte]map[byte]float64) var probTrans = make(map[byte]map[byte]float64)
func init() { func init() {
ProbTrans['B'] = map[byte]float64{'E': -0.510825623765990, probTrans['B'] = map[byte]float64{'E': -0.510825623765990,
'M': -0.916290731874155} 'M': -0.916290731874155}
ProbTrans['E'] = map[byte]float64{'B': -0.5897149736854513, probTrans['E'] = map[byte]float64{'B': -0.5897149736854513,
'S': -0.8085250474669937} 'S': -0.8085250474669937}
ProbTrans['M'] = map[byte]float64{'E': -0.33344856811948514, probTrans['M'] = map[byte]float64{'E': -0.33344856811948514,
'M': -1.2603623820268226} 'M': -1.2603623820268226}
ProbTrans['S'] = map[byte]float64{'B': -0.7211965654669841, probTrans['S'] = map[byte]float64{'B': -0.7211965654669841,
'S': -0.6658631448798212} 'S': -0.6658631448798212}
} }

35
finalseg/viterbi.go
View File

@@ -5,15 +5,22 @@ import (
"sort" "sort"
) )
const MIN_FLOAT = -3.14e100 const MinFloat = -3.14e100
var PrevStatus = make(map[byte][]byte) var (
prevStatus = make(map[byte][]byte)
probStart = make(map[byte]float64)
)
func init() { func init() {
PrevStatus['B'] = []byte{'E', 'S'} prevStatus['B'] = []byte{'E', 'S'}
PrevStatus['M'] = []byte{'M', 'B'} prevStatus['M'] = []byte{'M', 'B'}
PrevStatus['S'] = []byte{'S', 'E'} prevStatus['S'] = []byte{'S', 'E'}
PrevStatus['E'] = []byte{'B', 'M'} prevStatus['E'] = []byte{'B', 'M'}
probStart['B'] = -0.26268660809250016
probStart['E'] = -3.14e+100
probStart['M'] = -3.14e+100
probStart['S'] = -1.4652633398537678
} }
type Viterbi struct { type Viterbi struct {
@@ -47,10 +54,10 @@ func viterbi(obs []rune, states []byte) (float64, []byte) {
V := make([]map[byte]float64, len(obs)) V := make([]map[byte]float64, len(obs))
V[0] = make(map[byte]float64) V[0] = make(map[byte]float64)
for _, y := range states { for _, y := range states {
if val, ok := ProbEmit[y][obs[0]]; ok { if val, ok := probEmit[y][obs[0]]; ok {
V[0][y] = val + ProbStart[y] V[0][y] = val + probStart[y]
} else { } else {
V[0][y] = MIN_FLOAT + ProbStart[y] V[0][y] = MinFloat + probStart[y]
} }
path[y] = []byte{y} path[y] = []byte{y}
} }
@@ -61,17 +68,17 @@ func viterbi(obs []rune, states []byte) (float64, []byte) {
for _, y := range states { for _, y := range states {
vs0 := make(Viterbis, 0) vs0 := make(Viterbis, 0)
var em_p float64 var em_p float64
if val, ok := ProbEmit[y][obs[t]]; ok { if val, ok := probEmit[y][obs[t]]; ok {
em_p = val em_p = val
} else { } else {
em_p = MIN_FLOAT em_p = MinFloat
} }
for _, y0 := range PrevStatus[y] { for _, y0 := range prevStatus[y] {
var transP float64 var transP float64
if tp, ok := ProbTrans[y0][y]; ok { if tp, ok := probTrans[y0][y]; ok {
transP = tp transP = tp
} else { } else {
transP = MIN_FLOAT transP = MinFloat
} }
prob0 := V[t-1][y0] + transP + em_p prob0 := V[t-1][y0] + transP + em_p
vs0 = append(vs0, &Viterbi{prob: prob0, state: y0}) vs0 = append(vs0, &Viterbi{prob: prob0, state: y0})