fumiama/jieba
1
0
Fork 0
mirror of https://github.com/fumiama/jieba.git synced 2026-06-28 16:10:32 +08:00
Code Activity

优化 textrank

Browse Source
This commit is contained in:
源文雨
2022-11-30 12:38:47 +08:00
parent 8bbc755ed4
commit 21cdb2e863
5 changed files with 176 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
analyse/tag_extracker.go
View File

@@ -26,7 +26,7 @@ func (s Segment) Weight() float64 {
} }
// Segments represents a slice of Segment. // Segments represents a slice of Segment.
type Segments []Segment type Segments []*Segment
func (ss Segments) Len() int { func (ss Segments) Len() int {
return len(ss) return len(ss)
@@ -103,7 +103,7 @@ func (t *TagExtracter) ExtractTags(sentence string, topK int) (tags Segments) {
} else { } else {
s = Segment{text: k, weight: t.idf.median * v} s = Segment{text: k, weight: t.idf.median * v}
} }
ws = append(ws, s) ws = append(ws, &s)
} }
sort.Sort(sort.Reverse(ws)) sort.Sort(sort.Reverse(ws))
if len(ws) > topK { if len(ws) > topK {

212
analyse/tag_extracker_test.go
View File

@@ -94,91 +94,91 @@ var (
"枪杆子中出政权"} "枪杆子中出政权"}
Tags = [][]string{ Tags = [][]string{
[]string{"Python", "C++", "伸手不见五指", "孙悟空", "黑夜", "北京", "这是", "一个"}, {"Python", "C++", "伸手不见五指", "孙悟空", "黑夜", "北京", "这是", "一个"},
[]string{"和服", "喜欢", "日本"}, {"和服", "喜欢", "日本"},
[]string{"雷猴", "人间", "回归"}, {"雷猴", "人间", "回归"},
[]string{"工信处", "女干事", "24", "交换机", "科室", "亲口", "器件", "技术性", "下属", "交代", "每月", "安装", "经过", "工作"}, {"工信处", "女干事", "24", "交换机", "科室", "亲口", "器件", "技术性", "下属", "交代", "每月", "安装", "经过", "工作"},
[]string{"廉租房", "需要"}, {"廉租房", "需要"},
[]string{"饰品", "永和", "服装", "有限公司"}, {"饰品", "永和", "服装", "有限公司"},
[]string{"天安门", "北京"}, {"天安门", "北京"},
[]string{"abc"}, {"abc"},
[]string{"马尔可夫"}, {"马尔可夫"},
[]string{"雷猴", "网站"}, {"雷猴", "网站"},
[]string{"SOFTware", "Microsoft", "MICROcomputer", "微型", "一词", "软件", "计算机", "组成", "部分"}, {"SOFTware", "Microsoft", "MICROcomputer", "微型", "一词", "软件", "计算机", "组成", "部分"},
[]string{"草泥马", "欺实", "词汇", "流行", "今年"}, {"草泥马", "欺实", "词汇", "流行", "今年"},
[]string{"洋华堂", "总府", "伊藤"}, {"洋华堂", "总府", "伊藤"},
[]string{"中国科学院计算技术研究所"}, {"中国科学院计算技术研究所"},
[]string{"朱丽叶", "罗密欧"}, {"朱丽叶", "罗密欧"},
[]string{"道具", "服装", "购买"}, {"道具", "服装", "购买"},
[]string{"自珍", "敞帚", "PS", "开源", "不断改进", "敦促", "好处", "避免", "能够", "觉得", "就是", "自己", "一个"}, {"自珍", "敞帚", "PS", "开源", "不断改进", "敦促", "好处", "避免", "能够", "觉得", "就是", "自己", "一个"},
[]string{"石首市", "湖北省"}, {"石首市", "湖北省"},
[]string{"十堰市", "湖北省"}, {"十堰市", "湖北省"},
[]string{"总经理", "这件", "完成", "事情"}, {"总经理", "这件", "完成", "事情"},
[]string{"修好", "电脑"}, {"修好", "电脑"},
[]string{"一了百了", "做好", "这件", "事情"}, {"一了百了", "做好", "这件", "事情"},
[]string{"审美", "观点", "人们", "不同"}, {"审美", "观点", "人们", "不同"},
[]string{"美的", "空调", "我们", "一个"}, {"美的", "空调", "我们", "一个"},
[]string{"线程", "初始化", "注意", "我们"}, {"线程", "初始化", "注意", "我们"},
[]string{"好多", "原子", "分子", "组织", "一个"}, {"好多", "原子", "分子", "组织", "一个"},
[]string{"马到功成"}, {"马到功成"},
[]string{"无底洞"}, {"无底洞"},
[]string{"首都", "北京", "中国"}, {"首都", "北京", "中国"},
[]string{"孙君意"}, {"孙君意"},
[]string{"马朝旭", "外交部", "发言人"}, {"马朝旭", "外交部", "发言人"},
[]string{"第四届", "东亚", "峰会", "领导人", "会议"}, {"第四届", "东亚", "峰会", "领导人", "会议"},
[]string{"五年", "过去"}, {"五年", "过去"},
[]string{"很长", "需要"}, {"很长", "需要"},
[]string{"60", "阅兵", "周年", "首都"}, {"60", "阅兵", "周年", "首都"},
[]string{"审美", "你好", "观点", "人们", "不同"}, {"审美", "你好", "观点", "人们", "不同"},
[]string{"世博园", "水果", "然后"}, {"世博园", "水果", "然后"},
[]string{"世博园", "水果", "然后"}, {"世博园", "水果", "然后"},
[]string{"后来", "但是", "知道"}, {"后来", "但是", "知道"},
[]string{"合理", "存在"}, {"合理", "存在"},
[]string{}, {},
[]string{"rong", "love", "不以为耻", "以为"}, {"rong", "love", "不以为耻", "以为"},
[]string{}, {},
[]string{}, {},
[]string{"hello", "审美", "你好", "观点", "人们", "不同"}, {"hello", "审美", "你好", "观点", "人们", "不同"},
[]string{"网页", "基于", "形式", "主要"}, {"网页", "基于", "形式", "主要"},
[]string{"hello", "审美", "你好", "观点", "人们", "不同"}, {"hello", "审美", "你好", "观点", "人们", "不同"},
[]string{"想要", "拥有", "为什么", "生活", "不能"}, {"想要", "拥有", "为什么", "生活", "不能"},
[]string{"后来"}, {"后来"},
[]string{"此次", "为了", "中国"}, {"此次", "为了", "中国"},
[]string{"解决", "使用", "一些", "问题", "可以"}, {"解决", "使用", "一些", "问题", "可以"},
[]string{"解决", "使用", "一些", "问题", "可以"}, {"解决", "使用", "一些", "问题", "可以"},
[]string{"解决", "其实", "使用", "一些", "问题", "可以"}, {"解决", "其实", "使用", "一些", "问题", "可以"},
[]string{"好人", "解决", "使用", "一些", "问题", "可以"}, {"好人", "解决", "使用", "一些", "问题", "可以"},
[]string{"是因为", "国家"}, {"是因为", "国家"},
[]string{"老年", "搜索", "支持"}, {"老年", "搜索", "支持"},
[]string{"闲法", "中本", "laoshipukong", "RT", "27", "责任法", "蒙人", "万劫不复", "举证", "倒置", "医患", "那部", "拉倒", "侵权", "全国人大常委会", "草案", "境地", "纠纷", "删除", "弱势"}, {"闲法", "中本", "laoshipukong", "RT", "27", "责任法", "蒙人", "万劫不复", "举证", "倒置", "医患", "那部", "拉倒", "侵权", "全国人大常委会", "草案", "境地", "纠纷", "删除", "弱势"},
[]string{}, {},
[]string{}, {},
[]string{"在理", "确实"}, {"在理", "确实"},
[]string{"长春", "春节", "讲话", "市长"}, {"长春", "春节", "讲话", "市长"},
[]string{"结婚", "尚未"}, {"结婚", "尚未"},
[]string{"分子", "结合"}, {"分子", "结合"},
[]string{"旅游", "最好", "服务"}, {"旅游", "最好", "服务"},
[]string{"的确", "这件", "事情"}, {"的确", "这件", "事情"},
[]string{"指正", "参考", "大家"}, {"指正", "参考", "大家"},
[]string{"塌桥", "哈尔滨", "公布", "原因", "政府"}, {"塌桥", "哈尔滨", "公布", "原因", "政府"},
[]string{"入口处", "机场"}, {"入口处", "机场"},
[]string{"邢永臣", "摄影", "报道"}, {"邢永臣", "摄影", "报道"},
[]string{"区分度", "BP", "神经网络", "训练", "分类", "才能", "如何", "增加"}, {"区分度", "BP", "神经网络", "训练", "分类", "才能", "如何", "增加"},
[]string{"长江大桥", "南京市"}, {"长江大桥", "南京市"},
[]string{"SMT", "NiuTrans", "使用者", "便于", "用于", "建议", "利用", "为了", "研究", "一些"}, {"SMT", "NiuTrans", "使用者", "便于", "用于", "建议", "利用", "为了", "研究", "一些"},
[]string{"长春市", "药店", "长春"}, {"长春市", "药店", "长春"},
[]string{"邓颖超", "生前", "衣服", "喜欢"}, {"邓颖超", "生前", "衣服", "喜欢"},
[]string{"政治局", "热爱", "常委", "胡锦涛", "和平", "世界"}, {"政治局", "热爱", "常委", "胡锦涛", "和平", "世界"},
[]string{"右面", "孙健", "范凯", "李松洪", "朱会震", "海林", "左面", "程序员", "再往"}, {"右面", "孙健", "范凯", "李松洪", "朱会震", "海林", "左面", "程序员", "再往"},
[]string{"一次性", "多少"}, {"一次性", "多少"},
[]string{"四块", "五块", "三块", "一斤", "两块", "一本", "一套", "一条"}, {"四块", "五块", "三块", "一斤", "两块", "一本", "一套", "一条"},
[]string{"和尚", "和尚头", "一样", "一个"}, {"和尚", "和尚头", "一样", "一个"},
[]string{"和平门", "共和党", "地铁", "党员", "公民", "爸爸", "中华人民共和国"}, {"和平门", "共和党", "地铁", "党员", "公民", "爸爸", "中华人民共和国"},
[]string{"张晓梅", "T恤", "B超", "医院", "人民", "然后"}, {"张晓梅", "T恤", "B超", "医院", "人民", "然后"},
[]string{"offer", "AT&T", "不错", "一件", "公司"}, {"offer", "AT&T", "不错", "一件", "公司"},
[]string{"c#", "PI", "C++", "3.14159", "133", "122", "11", "关系", "什么"}, {"c#", "PI", "C++", "3.14159", "133", "122", "11", "关系", "什么"},
[]string{"的士", "的哥", "他开", "握手", "一辆", "黑色", "主席", "认识", "那个"}, {"的士", "的哥", "他开", "握手", "一辆", "黑色", "主席", "认识", "那个"},
[]string{"枪杆子", "政权"}, {"枪杆子", "政权"},
} }
Lyric = ` Lyric = `
@@ -228,29 +228,29 @@ var (
雖然沒有藉口 雖然沒有藉口
` `
LyciWeight = Segments{ LyciWeight = Segments{
Segment{text: "所謂", weight: 1.010262}, &Segment{text: "所謂", weight: 1.010262},
Segment{text: "是否", weight: 0.738650}, &Segment{text: "是否", weight: 0.738650},
Segment{text: "一般", weight: 0.607600}, &Segment{text: "一般", weight: 0.607600},
Segment{text: "雖然", weight: 0.336754}, &Segment{text: "雖然", weight: 0.336754},
Segment{text: "退縮", weight: 0.336754}, &Segment{text: "退縮", weight: 0.336754},
Segment{text: "肌迫", weight: 0.336754}, &Segment{text: "肌迫", weight: 0.336754},
Segment{text: "矯作", weight: 0.336754}, &Segment{text: "矯作", weight: 0.336754},
Segment{text: "沒有", weight: 0.336754}, &Segment{text: "沒有", weight: 0.336754},
Segment{text: "怯懦", weight: 0.271099}, &Segment{text: "怯懦", weight: 0.271099},
Segment{text: "隨便", weight: 0.168377}, &Segment{text: "隨便", weight: 0.168377},
} }
LyciWeight2 = Segments{ LyciWeight2 = Segments{
Segment{text: "所謂", weight: 1.215739}, &Segment{text: "所謂", weight: 1.215739},
Segment{text: "一般", weight: 0.731179}, &Segment{text: "一般", weight: 0.731179},
Segment{text: "雖然", weight: 0.405246}, &Segment{text: "雖然", weight: 0.405246},
Segment{text: "退縮", weight: 0.405246}, &Segment{text: "退縮", weight: 0.405246},
Segment{text: "肌迫", weight: 0.405246}, &Segment{text: "肌迫", weight: 0.405246},
Segment{text: "矯作", weight: 0.405246}, &Segment{text: "矯作", weight: 0.405246},
Segment{text: "怯懦", weight: 0.326238}, &Segment{text: "怯懦", weight: 0.326238},
Segment{text: "逼不得已", weight: 0.202623}, &Segment{text: "逼不得已", weight: 0.202623},
Segment{text: "右銘", weight: 0.202623}, &Segment{text: "右銘", weight: 0.202623},
Segment{text: "寬闊", weight: 0.202623}, &Segment{text: "寬闊", weight: 0.202623},
} }
) )
@@ -266,7 +266,7 @@ func TestExtractTags(t *testing.T) {
} }
for i, tag := range result { for i, tag := range result {
if tag.text != Tags[index][i] { if tag.text != Tags[index][i] {
t.Fatalf("%s != %s", tag, Tags[index][i]) t.Fatalf("%v != %v", tag, Tags[index][i])
} }
} }
} }

59
analyse/textrank.go
View File

@@ -1,25 +1,27 @@
package analyse package analyse
import ( import (
"hash/crc64"
"math" "math"
"sort" "sort"
"github.com/fumiama/jieba/posseg" "github.com/fumiama/jieba/posseg"
"github.com/fumiama/jieba/util/helper"
) )
const dampingFactor = 0.85 const dampingFactor = 0.85
var ( var (
defaultAllowPOS = []string{"ns", "n", "vn", "v"} defaultAllowPOS = [...]string{"ns", "n", "vn", "v"}
) )
type edge struct { type edge struct {
weight float64
start string start string
end string end string
weight float64
} }
type edges []edge type edges []*edge
func (es edges) Len() int { func (es edges) Len() int {
return len(es) return len(es)
@@ -39,25 +41,25 @@ type undirectWeightedGraph struct {
} }
func newUndirectWeightedGraph() *undirectWeightedGraph { func newUndirectWeightedGraph() *undirectWeightedGraph {
u := new(undirectWeightedGraph) return &undirectWeightedGraph{
u.graph = make(map[string]edges) graph: make(map[string]edges, 256),
u.keys = make(sort.StringSlice, 0) keys: make(sort.StringSlice, 0, 256),
return u }
} }
func (u *undirectWeightedGraph) addEdge(start, end string, weight float64) { func (u *undirectWeightedGraph) addEdge(start, end string, weight float64) {
if _, ok := u.graph[start]; !ok { if _, ok := u.graph[start]; !ok {
u.keys = append(u.keys, start) u.keys = append(u.keys, start)
u.graph[start] = edges{edge{start: start, end: end, weight: weight}} u.graph[start] = edges{&edge{start: start, end: end, weight: weight}}
} else { } else {
u.graph[start] = append(u.graph[start], edge{start: start, end: end, weight: weight}) u.graph[start] = append(u.graph[start], &edge{start: start, end: end, weight: weight})
} }
if _, ok := u.graph[end]; !ok { if _, ok := u.graph[end]; !ok {
u.keys = append(u.keys, end) u.keys = append(u.keys, end)
u.graph[end] = edges{edge{start: end, end: start, weight: weight}} u.graph[end] = edges{&edge{start: end, end: start, weight: weight}}
} else { } else {
u.graph[end] = append(u.graph[end], edge{start: end, end: start, weight: weight}) u.graph[end] = append(u.graph[end], &edge{start: end, end: start, weight: weight})
} }
} }
@@ -66,8 +68,8 @@ func (u *undirectWeightedGraph) rank() Segments {
sort.Sort(u.keys) sort.Sort(u.keys)
} }
ws := make(map[string]float64) ws := make(map[string]float64, len(u.graph)*2)
outSum := make(map[string]float64) outSum := make(map[string]float64, len(u.graph)*2)
wsdef := 1.0 wsdef := 1.0
if len(u.graph) > 0 { if len(u.graph) > 0 {
@@ -101,9 +103,11 @@ func (u *undirectWeightedGraph) rank() Segments {
maxRank = w maxRank = w
} }
} }
result := make(Segments, 0) result := make(Segments, len(ws))
i := 0
for n, w := range ws { for n, w := range ws {
result = append(result, Segment{text: n, weight: (w - minRank/10.0) / (maxRank - minRank/10.0)}) result[i] = &Segment{text: n, weight: (w - minRank/10.0) / (maxRank - minRank/10.0)}
i++
} }
sort.Sort(sort.Reverse(result)) sort.Sort(sort.Reverse(result))
return result return result
@@ -112,12 +116,20 @@ func (u *undirectWeightedGraph) rank() Segments {
// TextRankWithPOS extracts keywords from sentence using TextRank algorithm. // TextRankWithPOS extracts keywords from sentence using TextRank algorithm.
// Parameter allowPOS allows a customized pos list. // Parameter allowPOS allows a customized pos list.
func (t *TextRanker) TextRankWithPOS(sentence string, topK int, allowPOS []string) Segments { func (t *TextRanker) TextRankWithPOS(sentence string, topK int, allowPOS []string) Segments {
posFilt := make(map[string]int) posFilt := make(map[string]int, len(allowPOS)*2)
for _, pos := range allowPOS { for _, pos := range allowPOS {
posFilt[pos] = 1 posFilt[pos] = 1
} }
g := newUndirectWeightedGraph() g := newUndirectWeightedGraph()
cm := make(map[[2]string]float64) cm := make(map[uint64]float64, 256)
hm := make(map[uint64][2]string, 256)
gethash := func(a, b string) uint64 {
h := crc64.New(crc64.MakeTable(crc64.ISO))
h.Write(helper.StringToBytes(a))
h.Write([]byte("\t"))
h.Write(helper.StringToBytes(b))
return h.Sum64()
}
span := 5 span := 5
var pairs []posseg.Segment var pairs []posseg.Segment
for pair := range t.seg.Cut(sentence, true) { for pair := range t.seg.Cut(sentence, true) {
@@ -129,15 +141,18 @@ func (t *TextRanker) TextRankWithPOS(sentence string, topK int, allowPOS []strin
if _, ok := posFilt[pairs[j].Pos()]; !ok { if _, ok := posFilt[pairs[j].Pos()]; !ok {
continue continue
} }
if _, ok := cm[[2]string{pairs[i].Text(), pairs[j].Text()}]; !ok { h := gethash(pairs[i].Text(), pairs[j].Text())
cm[[2]string{pairs[i].Text(), pairs[j].Text()}] = 1.0 if _, ok := cm[h]; !ok {
cm[h] = 1.0
hm[h] = [2]string{pairs[i].Text(), pairs[j].Text()}
} else { } else {
cm[[2]string{pairs[i].Text(), pairs[j].Text()}] += 1.0 cm[h] += 1.0
} }
} }
} }
} }
for startEnd, weight := range cm { for h, weight := range cm {
startEnd := hm[h]
g.addEdge(startEnd[0], startEnd[1], weight) g.addEdge(startEnd[0], startEnd[1], weight)
} }
tags := g.rank() tags := g.rank()
@@ -150,7 +165,7 @@ func (t *TextRanker) TextRankWithPOS(sentence string, topK int, allowPOS []strin
// TextRank extract keywords from sentence using TextRank algorithm. // TextRank extract keywords from sentence using TextRank algorithm.
// Parameter topK specify how many top keywords to be returned at most. // Parameter topK specify how many top keywords to be returned at most.
func (t *TextRanker) TextRank(sentence string, topK int) Segments { func (t *TextRanker) TextRank(sentence string, topK int) Segments {
return t.TextRankWithPOS(sentence, topK, defaultAllowPOS) return t.TextRankWithPOS(sentence, topK, defaultAllowPOS[:])
} }
// TextRanker is used to extract tags from sentence. // TextRanker is used to extract tags from sentence.

20
analyse/textrank_test.go
View File

@@ -9,16 +9,16 @@ var (
sentence = "此外，公司拟对全资子公司吉林欧亚置业有限公司增资4.3亿元，增资后，吉林欧亚置业注册资本由7000万元增加到5亿元。吉林欧亚置业主要经营范围为房地产开发及百货零售等业务。目前在建吉林欧亚城市商业综合体项目。2013年，实现营业收入0万元，实现净利润-139.13万元。" sentence = "此外，公司拟对全资子公司吉林欧亚置业有限公司增资4.3亿元，增资后，吉林欧亚置业注册资本由7000万元增加到5亿元。吉林欧亚置业主要经营范围为房地产开发及百货零售等业务。目前在建吉林欧亚城市商业综合体项目。2013年，实现营业收入0万元，实现净利润-139.13万元。"
tagRanks = Segments{ tagRanks = Segments{
Segment{text: "吉林", weight: 1.0}, &Segment{text: "吉林", weight: 1.0},
Segment{text: "欧亚", weight: 0.87807810644}, &Segment{text: "欧亚", weight: 0.87807810644},
Segment{text: "置业", weight: 0.562048250306}, &Segment{text: "置业", weight: 0.562048250306},
Segment{text: "实现", weight: 0.520905743929}, &Segment{text: "实现", weight: 0.520905743929},
Segment{text: "收入", weight: 0.384283870648}, &Segment{text: "收入", weight: 0.384283870648},
Segment{text: "增资", weight: 0.360590945312}, &Segment{text: "增资", weight: 0.360590945312},
Segment{text: "子公司", weight: 0.353131980904}, &Segment{text: "子公司", weight: 0.353131980904},
Segment{text: "城市", weight: 0.307509449283}, &Segment{text: "城市", weight: 0.307509449283},
Segment{text: "全资", weight: 0.306324426665}, &Segment{text: "全资", weight: 0.306324426665},
Segment{text: "商业", weight: 0.306138241063}, &Segment{text: "商业", weight: 0.306138241063},
} }
) )

21
util/helper/helper.go Normal file
View File

@@ -0,0 +1,21 @@
package helper
import (
"reflect"
"unsafe"
)
// BytesToString 没有内存开销的转换
func BytesToString(b []byte) string {
return *(*string)(unsafe.Pointer(&b))
}
// StringToBytes 没有内存开销的转换
func StringToBytes(s string) (b []byte) {
bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
sh := (*reflect.StringHeader)(unsafe.Pointer(&s))
bh.Data = sh.Data
bh.Len = sh.Len
bh.Cap = sh.Len
return b
}