mirror of
https://github.com/fumiama/jieba.git
synced 2026-06-05 00:32:51 +08:00
make some public variable/function to private
This commit is contained in:
@@ -62,10 +62,10 @@ func ExtractTags(sentence string, topK int) (tags TfIdfs) {
|
||||
tis := make(TfIdfs, 0)
|
||||
for k, v := range freq {
|
||||
var ti TfIdf
|
||||
if freq_, ok := idfLoader.Freq[k]; ok {
|
||||
if freq_, ok := loader.Freq[k]; ok {
|
||||
ti = TfIdf{Word: k, Freq: freq_ * v}
|
||||
} else {
|
||||
ti = TfIdf{Word: k, Freq: idfLoader.Median * v}
|
||||
ti = TfIdf{Word: k, Freq: loader.Median * v}
|
||||
}
|
||||
tis = append(tis, ti)
|
||||
}
|
||||
|
||||
@@ -6,26 +6,26 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
idfLoader *IDFLoader
|
||||
loader *idfLoader
|
||||
)
|
||||
|
||||
func init() {
|
||||
idfLoader = NewIDFLoader()
|
||||
loader = newIDFLoader()
|
||||
}
|
||||
|
||||
type IDFLoader struct {
|
||||
type idfLoader struct {
|
||||
Path string
|
||||
Freq map[string]float64
|
||||
Median float64
|
||||
}
|
||||
|
||||
func NewIDFLoader() *IDFLoader {
|
||||
loader := new(IDFLoader)
|
||||
func newIDFLoader() *idfLoader {
|
||||
loader := new(idfLoader)
|
||||
loader.Freq = make(map[string]float64)
|
||||
return loader
|
||||
}
|
||||
|
||||
func (loader *IDFLoader) newPath(idfFilePath string) error {
|
||||
func (loader *idfLoader) newPath(idfFilePath string) error {
|
||||
if loader.Path == idfFilePath {
|
||||
return nil
|
||||
}
|
||||
@@ -46,10 +46,12 @@ func (loader *IDFLoader) newPath(idfFilePath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Set the IDF file path, could be absolute path of IDF file, or IDF file
|
||||
// name in current directory.
|
||||
func SetIdf(idfFileName string) error {
|
||||
idfFilePath, err := jiebago.DictPath(idfFileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return idfLoader.newPath(idfFilePath)
|
||||
return loader.newPath(idfFilePath)
|
||||
}
|
||||
|
||||
@@ -42,6 +42,8 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
// Set the stop words file path, could be absolute path of stop words file, or
|
||||
// file name in current directory.
|
||||
func SetStopWords(stopWordsFileName string) error {
|
||||
stopWordsFilePath, err := jiebago.DictPath(stopWordsFileName)
|
||||
if err != nil {
|
||||
|
||||
@@ -7,9 +7,7 @@ import (
|
||||
"sort"
|
||||
)
|
||||
|
||||
const (
|
||||
DampingFactor = 0.85
|
||||
)
|
||||
const dampingFactor = 0.85
|
||||
|
||||
var (
|
||||
defaultAllowPOS = []string{"ns", "n", "vn", "v"}
|
||||
@@ -95,7 +93,7 @@ func (u *undirectWeightedGraph) rank() TfIdfs {
|
||||
for _, e := range inedges {
|
||||
s += e.weight / outSum[e.end] * ws[e.end]
|
||||
}
|
||||
ws[n] = (1 - DampingFactor) + DampingFactor*s
|
||||
ws[n] = (1 - dampingFactor) + dampingFactor*s
|
||||
}
|
||||
}
|
||||
minRank := math.MaxFloat64
|
||||
@@ -158,6 +156,9 @@ func TextRank(sentence string, topK int) TfIdfs {
|
||||
return TextRankWithPOS(sentence, topK, defaultAllowPOS)
|
||||
}
|
||||
|
||||
// Set the dictionary, could be absolute path of dictionary file, or dictionary
|
||||
// name in current directory. This function must be called before cut any
|
||||
// sentence.
|
||||
func SetDictionary(dictFileName string) error {
|
||||
return posseg.SetDictionary(dictFileName)
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"sort"
|
||||
)
|
||||
|
||||
const MinFloat = -3.14e100
|
||||
const minFloat = -3.14e100
|
||||
|
||||
var (
|
||||
prevStatus = make(map[byte][]byte)
|
||||
@@ -57,7 +57,7 @@ func viterbi(obs []rune, states []byte) (float64, []byte) {
|
||||
if val, ok := probEmit[y][obs[0]]; ok {
|
||||
V[0][y] = val + probStart[y]
|
||||
} else {
|
||||
V[0][y] = MinFloat + probStart[y]
|
||||
V[0][y] = minFloat + probStart[y]
|
||||
}
|
||||
path[y] = []byte{y}
|
||||
}
|
||||
@@ -71,14 +71,14 @@ func viterbi(obs []rune, states []byte) (float64, []byte) {
|
||||
if val, ok := probEmit[y][obs[t]]; ok {
|
||||
em_p = val
|
||||
} else {
|
||||
em_p = MinFloat
|
||||
em_p = minFloat
|
||||
}
|
||||
for _, y0 := range prevStatus[y] {
|
||||
var transP float64
|
||||
if tp, ok := probTrans[y0][y]; ok {
|
||||
transP = tp
|
||||
} else {
|
||||
transP = MinFloat
|
||||
transP = minFloat
|
||||
}
|
||||
prob0 := V[t-1][y0] + transP + em_p
|
||||
ps0 = append(ps0, &probState{prob: prob0, state: y0})
|
||||
|
||||
Reference in New Issue
Block a user