fumiama/jieba
1
0
Fork 0
mirror of https://github.com/fumiama/jieba.git synced 2026-06-05 00:32:51 +08:00
Code Activity

优化 tag_extracker

Browse Source
This commit is contained in:
源文雨
2022-11-30 13:35:21 +08:00
parent ae85ccb20a
commit d487545eb5
14 changed files with 60 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@@ -1,7 +1,7 @@
#结巴分词 Go 语言版：Jiebago #结巴分词 Go 语言版：jieba
[![Build Status](https://travis-ci.org/wangbin/jiebago.png?branch=master)](https://travis-ci.org/wangbin/jiebago) [![GoDoc](https://godoc.org/github.com/fumiama/jieba?status.svg)](https://godoc.org/github.com/fumiama/jieba) [![Build Status](https://travis-ci.org/wangbin/jieba.png?branch=master)](https://travis-ci.org/wangbin/jieba) [![GoDoc](https://godoc.org/github.com/fumiama/jieba?status.svg)](https://godoc.org/github.com/fumiama/jieba)
[结巴分词](https://github.com/fxsjy/jieba) 是由 [@fxsjy](https://github.com/fxsjy) 使用 Python 编写的中文分词组件，Iiebago 是结巴分词的 Golang 语言实现。 [结巴分词](https://github.com/fxsjy/jieba) 是由 [@fxsjy](https://github.com/fxsjy) 使用 Python 编写的中文分词组件，Iiebago 是结巴分词的 Golang 语言实现。
@@ -23,7 +23,7 @@ import (
"github.com/fumiama/jieba" "github.com/fumiama/jieba"
) )
var seg jiebago.Segmenter var seg jieba.Segmenter
func init() { func init() {
seg.LoadDictionary("dict.txt") seg.LoadDictionary("dict.txt")

27
analyse/tag_extracker.go
View File

@@ -6,7 +6,7 @@ import (
"strings" "strings"
"unicode/utf8" "unicode/utf8"
jiebago "github.com/fumiama/jieba" jieba "github.com/fumiama/jieba"
) )
// Segment represents a word with weight. // Segment represents a word with weight.
@@ -26,7 +26,7 @@ func (s Segment) Weight() float64 {
} }
// Segments represents a slice of Segment. // Segments represents a slice of Segment.
type Segments []*Segment type Segments []Segment
func (ss Segments) Len() int { func (ss Segments) Len() int {
return len(ss) return len(ss)
@@ -46,7 +46,7 @@ func (ss Segments) Swap(i, j int) {
// TagExtracter is used to extract tags from sentence. // TagExtracter is used to extract tags from sentence.
type TagExtracter struct { type TagExtracter struct {
seg *jiebago.Segmenter seg *jieba.Segmenter
idf *Idf idf *Idf
stopWord *StopWord stopWord *StopWord
} }
@@ -54,7 +54,7 @@ type TagExtracter struct {
// LoadDictionary reads the given filename and create a new dictionary. // LoadDictionary reads the given filename and create a new dictionary.
func (t *TagExtracter) LoadDictionary(fileName string) error { func (t *TagExtracter) LoadDictionary(fileName string) error {
t.stopWord = NewStopWord() t.stopWord = NewStopWord()
t.seg = new(jiebago.Segmenter) t.seg = new(jieba.Segmenter)
return t.seg.LoadDictionary(fileName) return t.seg.LoadDictionary(fileName)
} }
@@ -72,7 +72,7 @@ func (t *TagExtracter) LoadStopWords(fileName string) error {
// ExtractTags extracts the topK key words from sentence. // ExtractTags extracts the topK key words from sentence.
func (t *TagExtracter) ExtractTags(sentence string, topK int) (tags Segments) { func (t *TagExtracter) ExtractTags(sentence string, topK int) (tags Segments) {
freqMap := make(map[string]float64) freqMap := make(map[string]uint64, 256)
for _, w := range t.seg.Cut(sentence, true) { for _, w := range t.seg.Cut(sentence, true) {
w = strings.TrimSpace(w) w = strings.TrimSpace(w)
@@ -82,28 +82,25 @@ func (t *TagExtracter) ExtractTags(sentence string, topK int) (tags Segments) {
if t.stopWord.IsStopWord(w) { if t.stopWord.IsStopWord(w) {
continue continue
} }
if f, ok := freqMap[w]; ok { if v, ok := freqMap[w]; ok {
freqMap[w] = f + 1.0 freqMap[w] = v + 1
} else { } else {
freqMap[w] = 1.0 freqMap[w] = 1
} }
} }
total := 0.0 total := uint64(0)
for _, freq := range freqMap { for _, freq := range freqMap {
total += freq total += freq
} }
for k, v := range freqMap {
freqMap[k] = v / total
}
ws := make(Segments, 0) ws := make(Segments, 0)
var s Segment var s Segment
for k, v := range freqMap { for k, v := range freqMap {
if freq, ok := t.idf.Frequency(k); ok { if freq, ok := t.idf.Frequency(k); ok {
s = Segment{text: k, weight: freq * v} s = Segment{text: k, weight: freq * float64(v) / float64(total)}
} else { } else {
s = Segment{text: k, weight: t.idf.median * v} s = Segment{text: k, weight: t.idf.median * float64(v) / float64(total)}
} }
ws = append(ws, &s) ws = append(ws, s)
} }
sort.Sort(sort.Reverse(ws)) sort.Sort(sort.Reverse(ws))
if len(ws) > topK { if len(ws) > topK {

40
analyse/tag_extracker_test.go
View File

@@ -228,29 +228,29 @@ var (
雖然沒有藉口 雖然沒有藉口
` `
LyciWeight = Segments{ LyciWeight = Segments{
&Segment{text: "所謂", weight: 1.010262}, Segment{text: "所謂", weight: 1.010262},
&Segment{text: "是否", weight: 0.738650}, Segment{text: "是否", weight: 0.738650},
&Segment{text: "一般", weight: 0.607600}, Segment{text: "一般", weight: 0.607600},
&Segment{text: "雖然", weight: 0.336754}, Segment{text: "雖然", weight: 0.336754},
&Segment{text: "退縮", weight: 0.336754}, Segment{text: "退縮", weight: 0.336754},
&Segment{text: "肌迫", weight: 0.336754}, Segment{text: "肌迫", weight: 0.336754},
&Segment{text: "矯作", weight: 0.336754}, Segment{text: "矯作", weight: 0.336754},
&Segment{text: "沒有", weight: 0.336754}, Segment{text: "沒有", weight: 0.336754},
&Segment{text: "怯懦", weight: 0.271099}, Segment{text: "怯懦", weight: 0.271099},
&Segment{text: "隨便", weight: 0.168377}, Segment{text: "隨便", weight: 0.168377},
} }
LyciWeight2 = Segments{ LyciWeight2 = Segments{
&Segment{text: "所謂", weight: 1.215739}, Segment{text: "所謂", weight: 1.215739},
&Segment{text: "一般", weight: 0.731179}, Segment{text: "一般", weight: 0.731179},
&Segment{text: "雖然", weight: 0.405246}, Segment{text: "雖然", weight: 0.405246},
&Segment{text: "退縮", weight: 0.405246}, Segment{text: "退縮", weight: 0.405246},
&Segment{text: "肌迫", weight: 0.405246}, Segment{text: "肌迫", weight: 0.405246},
&Segment{text: "矯作", weight: 0.405246}, Segment{text: "矯作", weight: 0.405246},
&Segment{text: "怯懦", weight: 0.326238}, Segment{text: "怯懦", weight: 0.326238},
&Segment{text: "逼不得已", weight: 0.202623}, Segment{text: "逼不得已", weight: 0.202623},
&Segment{text: "右銘", weight: 0.202623}, Segment{text: "右銘", weight: 0.202623},
&Segment{text: "寬闊", weight: 0.202623}, Segment{text: "寬闊", weight: 0.202623},
} }
) )

3
analyse/textrank.go
View File

@@ -106,7 +106,8 @@ func (u *undirectWeightedGraph) rank() Segments {
result := make(Segments, len(ws)) result := make(Segments, len(ws))
i := 0 i := 0
for n, w := range ws { for n, w := range ws {
result[i] = &Segment{text: n, weight: (w - minRank/10.0) / (maxRank - minRank/10.0)} result[i].text = n
result[i].weight = (w - minRank/10.0) / (maxRank - minRank/10.0)
i++ i++
} }
sort.Sort(sort.Reverse(result)) sort.Sort(sort.Reverse(result))

20
analyse/textrank_test.go
View File

@@ -9,16 +9,16 @@ var (
sentence = "此外，公司拟对全资子公司吉林欧亚置业有限公司增资4.3亿元，增资后，吉林欧亚置业注册资本由7000万元增加到5亿元。吉林欧亚置业主要经营范围为房地产开发及百货零售等业务。目前在建吉林欧亚城市商业综合体项目。2013年，实现营业收入0万元，实现净利润-139.13万元。" sentence = "此外，公司拟对全资子公司吉林欧亚置业有限公司增资4.3亿元，增资后，吉林欧亚置业注册资本由7000万元增加到5亿元。吉林欧亚置业主要经营范围为房地产开发及百货零售等业务。目前在建吉林欧亚城市商业综合体项目。2013年，实现营业收入0万元，实现净利润-139.13万元。"
tagRanks = Segments{ tagRanks = Segments{
&Segment{text: "吉林", weight: 1.0}, Segment{text: "吉林", weight: 1.0},
&Segment{text: "欧亚", weight: 0.87807810644}, Segment{text: "欧亚", weight: 0.87807810644},
&Segment{text: "置业", weight: 0.562048250306}, Segment{text: "置业", weight: 0.562048250306},
&Segment{text: "实现", weight: 0.520905743929}, Segment{text: "实现", weight: 0.520905743929},
&Segment{text: "收入", weight: 0.384283870648}, Segment{text: "收入", weight: 0.384283870648},
&Segment{text: "增资", weight: 0.360590945312}, Segment{text: "增资", weight: 0.360590945312},
&Segment{text: "子公司", weight: 0.353131980904}, Segment{text: "子公司", weight: 0.353131980904},
&Segment{text: "城市", weight: 0.307509449283}, Segment{text: "城市", weight: 0.307509449283},
&Segment{text: "全资", weight: 0.306324426665}, Segment{text: "全资", weight: 0.306324426665},
&Segment{text: "商业", weight: 0.306138241063}, Segment{text: "商业", weight: 0.306138241063},
} }
) )

2
dictionary.go
View File

@@ -1,4 +1,4 @@
package jiebago package jieba
import ( import (
"math" "math"

2
dictionary/dictionary.go
View File

@@ -1,5 +1,5 @@
// Package dictionary contains a interface and wraps all io related work. // Package dictionary contains a interface and wraps all io related work.
// It is used by jiebago module to read/write files. // It is used by jieba module to read/write files.
package dictionary package dictionary
import ( import (

2
example_parallel_cut_test.go
View File

@@ -1,4 +1,4 @@
package jiebago package jieba
import ( import (
"bufio" "bufio"

2
example_test.go
View File

@@ -1,4 +1,4 @@
package jiebago package jieba
import ( import (
"fmt" "fmt"

2
finalseg/finalseg.go
View File

@@ -37,7 +37,7 @@ func cutHan(sentence string) []string {
} }
// Cut cuts sentence into words using Hidden Markov Model with Viterbi // Cut cuts sentence into words using Hidden Markov Model with Viterbi
// algorithm. It is used by Jiebago for unknonw words. // algorithm. It is used by jieba for unknonw words.
func Cut(sentence string) []string { func Cut(sentence string) []string {
result := make([]string, 0, 10) result := make([]string, 0, 10)
s := sentence s := sentence

4
jieba.go
View File

@@ -1,5 +1,5 @@
// Package jiebago is the Golang implemention of [Jieba](https://github.com/fxsjy/jieba), Python Chinese text segmentation module. // Package jieba is the Golang implemention of [Jieba](https://github.com/fxsjy/jieba), Python Chinese text segmentation module.
package jiebago package jieba
import ( import (
"math" "math"

2
jieba_test.go
View File

@@ -1,4 +1,4 @@
package jiebago package jieba
import "testing" import "testing"

8
tokenizers/tokenizer.go
View File

@@ -7,7 +7,7 @@ import (
"github.com/blevesearch/bleve/analysis" "github.com/blevesearch/bleve/analysis"
"github.com/blevesearch/bleve/registry" "github.com/blevesearch/bleve/registry"
jiebago "github.com/fumiama/jieba" jieba "github.com/fumiama/jieba"
) )
// Name is the jieba tokenizer name. // Name is the jieba tokenizer name.
@@ -15,9 +15,9 @@ const Name = "jieba"
var ideographRegexp = regexp.MustCompile(`\p{Han}+`) var ideographRegexp = regexp.MustCompile(`\p{Han}+`)
// JiebaTokenizer is the beleve tokenizer for jiebago. // JiebaTokenizer is the beleve tokenizer for jieba.
type JiebaTokenizer struct { type JiebaTokenizer struct {
seg jiebago.Segmenter seg jieba.Segmenter
hmm, searchMode bool hmm, searchMode bool
} }
@@ -42,7 +42,7 @@ Parameters:
this word into "交换", "换机", which are valid Chinese words. this word into "交换", "换机", which are valid Chinese words.
*/ */
func NewJiebaTokenizer(dictFilePath string, hmm, searchMode bool) (analysis.Tokenizer, error) { func NewJiebaTokenizer(dictFilePath string, hmm, searchMode bool) (analysis.Tokenizer, error) {
var seg jiebago.Segmenter var seg jieba.Segmenter
err := seg.LoadDictionary(dictFilePath) err := seg.LoadDictionary(dictFilePath)
return &JiebaTokenizer{ return &JiebaTokenizer{
seg: seg, seg: seg,

2
util/util.go
View File

@@ -1,4 +1,4 @@
// Package util contains some util functions used by jiebago. // Package util contains some util functions used by jieba.
package util package util
import "regexp" import "regexp"