优化 textrank

2026-06-05 00:32:51 +08:00 · 2022-11-30 12:38:47 +08:00
parent 8bbc755ed4
commit 21cdb2e863
5 changed files with 176 additions and 140 deletions
--- a/analyse/tag_extracker.go
+++ b/analyse/tag_extracker.go
@@ -26,7 +26,7 @@ func (s Segment) Weight() float64 {
 }

 // Segments represents a slice of Segment.
-type Segments []Segment
+type Segments []*Segment

 func (ss Segments) Len() int {
 	return len(ss)
@@ -103,7 +103,7 @@ func (t *TagExtracter) ExtractTags(sentence string, topK int) (tags Segments) {
 		} else {
 			s = Segment{text: k, weight: t.idf.median * v}
 		}
-		ws = append(ws, s)
+		ws = append(ws, &s)
 	}
 	sort.Sort(sort.Reverse(ws))
 	if len(ws) > topK {
--- a/analyse/tag_extracker_test.go
+++ b/analyse/tag_extracker_test.go
@@ -94,91 +94,91 @@ var (
 		"枪杆子中出政权"}

 	Tags = [][]string{
-		[]string{"Python", "C++", "伸手不见五指", "孙悟空", "黑夜", "北京", "这是", "一个"},
-		[]string{"和服", "喜欢", "日本"},
-		[]string{"雷猴", "人间", "回归"},
-		[]string{"工信处", "女干事", "24", "交换机", "科室", "亲口", "器件", "技术性", "下属", "交代", "每月", "安装", "经过", "工作"},
-		[]string{"廉租房", "需要"},
-		[]string{"饰品", "永和", "服装", "有限公司"},
-		[]string{"天安门", "北京"},
-		[]string{"abc"},
-		[]string{"马尔可夫"},
-		[]string{"雷猴", "网站"},
-		[]string{"SOFTware", "Microsoft", "MICROcomputer", "微型", "一词", "软件", "计算机", "组成", "部分"},
-		[]string{"草泥马", "欺实", "词汇", "流行", "今年"},
-		[]string{"洋华堂", "总府", "伊藤"},
-		[]string{"中国科学院计算技术研究所"},
-		[]string{"朱丽叶", "罗密欧"},
-		[]string{"道具", "服装", "购买"},
-		[]string{"自珍", "敞帚", "PS", "开源", "不断改进", "敦促", "好处", "避免", "能够", "觉得", "就是", "自己", "一个"},
-		[]string{"石首市", "湖北省"},
-		[]string{"十堰市", "湖北省"},
-		[]string{"总经理", "这件", "完成", "事情"},
-		[]string{"修好", "电脑"},
-		[]string{"一了百了", "做好", "这件", "事情"},
-		[]string{"审美", "观点", "人们", "不同"},
-		[]string{"美的", "空调", "我们", "一个"},
-		[]string{"线程", "初始化", "注意", "我们"},
-		[]string{"好多", "原子", "分子", "组织", "一个"},
-		[]string{"马到功成"},
-		[]string{"无底洞"},
-		[]string{"首都", "北京", "中国"},
-		[]string{"孙君意"},
-		[]string{"马朝旭", "外交部", "发言人"},
-		[]string{"第四届", "东亚", "峰会", "领导人", "会议"},
-		[]string{"五年", "过去"},
-		[]string{"很长", "需要"},
-		[]string{"60", "阅兵", "周年", "首都"},
-		[]string{"审美", "你好", "观点", "人们", "不同"},
-		[]string{"世博园", "水果", "然后"},
-		[]string{"世博园", "水果", "然后"},
-		[]string{"后来", "但是", "知道"},
-		[]string{"合理", "存在"},
-		[]string{},
-		[]string{"rong", "love", "不以为耻", "以为"},
-		[]string{},
-		[]string{},
-		[]string{"hello", "审美", "你好", "观点", "人们", "不同"},
-		[]string{"网页", "基于", "形式", "主要"},
-		[]string{"hello", "审美", "你好", "观点", "人们", "不同"},
-		[]string{"想要", "拥有", "为什么", "生活", "不能"},
-		[]string{"后来"},
-		[]string{"此次", "为了", "中国"},
-		[]string{"解决", "使用", "一些", "问题", "可以"},
-		[]string{"解决", "使用", "一些", "问题", "可以"},
-		[]string{"解决", "其实", "使用", "一些", "问题", "可以"},
-		[]string{"好人", "解决", "使用", "一些", "问题", "可以"},
-		[]string{"是因为", "国家"},
-		[]string{"老年", "搜索", "支持"},
-		[]string{"闲法", "中本", "laoshipukong", "RT", "27", "责任法", "蒙人", "万劫不复", "举证", "倒置", "医患", "那部", "拉倒", "侵权", "全国人大常委会", "草案", "境地", "纠纷", "删除", "弱势"},
-		[]string{},
-		[]string{},
-		[]string{"在理", "确实"},
-		[]string{"长春", "春节", "讲话", "市长"},
-		[]string{"结婚", "尚未"},
-		[]string{"分子", "结合"},
-		[]string{"旅游", "最好", "服务"},
-		[]string{"的确", "这件", "事情"},
-		[]string{"指正", "参考", "大家"},
-		[]string{"塌桥", "哈尔滨", "公布", "原因", "政府"},
-		[]string{"入口处", "机场"},
-		[]string{"邢永臣", "摄影", "报道"},
-		[]string{"区分度", "BP", "神经网络", "训练", "分类", "才能", "如何", "增加"},
-		[]string{"长江大桥", "南京市"},
-		[]string{"SMT", "NiuTrans", "使用者", "便于", "用于", "建议", "利用", "为了", "研究", "一些"},
-		[]string{"长春市", "药店", "长春"},
-		[]string{"邓颖超", "生前", "衣服", "喜欢"},
-		[]string{"政治局", "热爱", "常委", "胡锦涛", "和平", "世界"},
-		[]string{"右面", "孙健", "范凯", "李松洪", "朱会震", "海林", "左面", "程序员", "再往"},
-		[]string{"一次性", "多少"},
-		[]string{"四块", "五块", "三块", "一斤", "两块", "一本", "一套", "一条"},
-		[]string{"和尚", "和尚头", "一样", "一个"},
-		[]string{"和平门", "共和党", "地铁", "党员", "公民", "爸爸", "中华人民共和国"},
-		[]string{"张晓梅", "T恤", "B超", "医院", "人民", "然后"},
-		[]string{"offer", "AT&T", "不错", "一件", "公司"},
-		[]string{"c#", "PI", "C++", "3.14159", "133", "122", "11", "关系", "什么"},
-		[]string{"的士", "的哥", "他开", "握手", "一辆", "黑色", "主席", "认识", "那个"},
-		[]string{"枪杆子", "政权"},
+		{"Python", "C++", "伸手不见五指", "孙悟空", "黑夜", "北京", "这是", "一个"},
+		{"和服", "喜欢", "日本"},
+		{"雷猴", "人间", "回归"},
+		{"工信处", "女干事", "24", "交换机", "科室", "亲口", "器件", "技术性", "下属", "交代", "每月", "安装", "经过", "工作"},
+		{"廉租房", "需要"},
+		{"饰品", "永和", "服装", "有限公司"},
+		{"天安门", "北京"},
+		{"abc"},
+		{"马尔可夫"},
+		{"雷猴", "网站"},
+		{"SOFTware", "Microsoft", "MICROcomputer", "微型", "一词", "软件", "计算机", "组成", "部分"},
+		{"草泥马", "欺实", "词汇", "流行", "今年"},
+		{"洋华堂", "总府", "伊藤"},
+		{"中国科学院计算技术研究所"},
+		{"朱丽叶", "罗密欧"},
+		{"道具", "服装", "购买"},
+		{"自珍", "敞帚", "PS", "开源", "不断改进", "敦促", "好处", "避免", "能够", "觉得", "就是", "自己", "一个"},
+		{"石首市", "湖北省"},
+		{"十堰市", "湖北省"},
+		{"总经理", "这件", "完成", "事情"},
+		{"修好", "电脑"},
+		{"一了百了", "做好", "这件", "事情"},
+		{"审美", "观点", "人们", "不同"},
+		{"美的", "空调", "我们", "一个"},
+		{"线程", "初始化", "注意", "我们"},
+		{"好多", "原子", "分子", "组织", "一个"},
+		{"马到功成"},
+		{"无底洞"},
+		{"首都", "北京", "中国"},
+		{"孙君意"},
+		{"马朝旭", "外交部", "发言人"},
+		{"第四届", "东亚", "峰会", "领导人", "会议"},
+		{"五年", "过去"},
+		{"很长", "需要"},
+		{"60", "阅兵", "周年", "首都"},
+		{"审美", "你好", "观点", "人们", "不同"},
+		{"世博园", "水果", "然后"},
+		{"世博园", "水果", "然后"},
+		{"后来", "但是", "知道"},
+		{"合理", "存在"},
+		{},
+		{"rong", "love", "不以为耻", "以为"},
+		{},
+		{},
+		{"hello", "审美", "你好", "观点", "人们", "不同"},
+		{"网页", "基于", "形式", "主要"},
+		{"hello", "审美", "你好", "观点", "人们", "不同"},
+		{"想要", "拥有", "为什么", "生活", "不能"},
+		{"后来"},
+		{"此次", "为了", "中国"},
+		{"解决", "使用", "一些", "问题", "可以"},
+		{"解决", "使用", "一些", "问题", "可以"},
+		{"解决", "其实", "使用", "一些", "问题", "可以"},
+		{"好人", "解决", "使用", "一些", "问题", "可以"},
+		{"是因为", "国家"},
+		{"老年", "搜索", "支持"},
+		{"闲法", "中本", "laoshipukong", "RT", "27", "责任法", "蒙人", "万劫不复", "举证", "倒置", "医患", "那部", "拉倒", "侵权", "全国人大常委会", "草案", "境地", "纠纷", "删除", "弱势"},
+		{},
+		{},
+		{"在理", "确实"},
+		{"长春", "春节", "讲话", "市长"},
+		{"结婚", "尚未"},
+		{"分子", "结合"},
+		{"旅游", "最好", "服务"},
+		{"的确", "这件", "事情"},
+		{"指正", "参考", "大家"},
+		{"塌桥", "哈尔滨", "公布", "原因", "政府"},
+		{"入口处", "机场"},
+		{"邢永臣", "摄影", "报道"},
+		{"区分度", "BP", "神经网络", "训练", "分类", "才能", "如何", "增加"},
+		{"长江大桥", "南京市"},
+		{"SMT", "NiuTrans", "使用者", "便于", "用于", "建议", "利用", "为了", "研究", "一些"},
+		{"长春市", "药店", "长春"},
+		{"邓颖超", "生前", "衣服", "喜欢"},
+		{"政治局", "热爱", "常委", "胡锦涛", "和平", "世界"},
+		{"右面", "孙健", "范凯", "李松洪", "朱会震", "海林", "左面", "程序员", "再往"},
+		{"一次性", "多少"},
+		{"四块", "五块", "三块", "一斤", "两块", "一本", "一套", "一条"},
+		{"和尚", "和尚头", "一样", "一个"},
+		{"和平门", "共和党", "地铁", "党员", "公民", "爸爸", "中华人民共和国"},
+		{"张晓梅", "T恤", "B超", "医院", "人民", "然后"},
+		{"offer", "AT&T", "不错", "一件", "公司"},
+		{"c#", "PI", "C++", "3.14159", "133", "122", "11", "关系", "什么"},
+		{"的士", "的哥", "他开", "握手", "一辆", "黑色", "主席", "认识", "那个"},
+		{"枪杆子", "政权"},
 	}

 	Lyric = `
@@ -228,29 +228,29 @@ var (
 雖然沒有藉口
 `
 	LyciWeight = Segments{
-		Segment{text: "所謂", weight: 1.010262},
-		Segment{text: "是否", weight: 0.738650},
-		Segment{text: "一般", weight: 0.607600},
-		Segment{text: "雖然", weight: 0.336754},
-		Segment{text: "退縮", weight: 0.336754},
-		Segment{text: "肌迫", weight: 0.336754},
-		Segment{text: "矯作", weight: 0.336754},
-		Segment{text: "沒有", weight: 0.336754},
-		Segment{text: "怯懦", weight: 0.271099},
-		Segment{text: "隨便", weight: 0.168377},
+		&Segment{text: "所謂", weight: 1.010262},
+		&Segment{text: "是否", weight: 0.738650},
+		&Segment{text: "一般", weight: 0.607600},
+		&Segment{text: "雖然", weight: 0.336754},
+		&Segment{text: "退縮", weight: 0.336754},
+		&Segment{text: "肌迫", weight: 0.336754},
+		&Segment{text: "矯作", weight: 0.336754},
+		&Segment{text: "沒有", weight: 0.336754},
+		&Segment{text: "怯懦", weight: 0.271099},
+		&Segment{text: "隨便", weight: 0.168377},
 	}

 	LyciWeight2 = Segments{
-		Segment{text: "所謂", weight: 1.215739},
-		Segment{text: "一般", weight: 0.731179},
-		Segment{text: "雖然", weight: 0.405246},
-		Segment{text: "退縮", weight: 0.405246},
-		Segment{text: "肌迫", weight: 0.405246},
-		Segment{text: "矯作", weight: 0.405246},
-		Segment{text: "怯懦", weight: 0.326238},
-		Segment{text: "逼不得已", weight: 0.202623},
-		Segment{text: "右銘", weight: 0.202623},
-		Segment{text: "寬闊", weight: 0.202623},
+		&Segment{text: "所謂", weight: 1.215739},
+		&Segment{text: "一般", weight: 0.731179},
+		&Segment{text: "雖然", weight: 0.405246},
+		&Segment{text: "退縮", weight: 0.405246},
+		&Segment{text: "肌迫", weight: 0.405246},
+		&Segment{text: "矯作", weight: 0.405246},
+		&Segment{text: "怯懦", weight: 0.326238},
+		&Segment{text: "逼不得已", weight: 0.202623},
+		&Segment{text: "右銘", weight: 0.202623},
+		&Segment{text: "寬闊", weight: 0.202623},
 	}
 )

@@ -266,7 +266,7 @@ func TestExtractTags(t *testing.T) {
 		}
 		for i, tag := range result {
 			if tag.text != Tags[index][i] {
-				t.Fatalf("%s != %s", tag, Tags[index][i])
+				t.Fatalf("%v != %v", tag, Tags[index][i])
 			}
 		}
 	}
--- a/analyse/textrank.go
+++ b/analyse/textrank.go
@@ -1,25 +1,27 @@
 package analyse

 import (
+	"hash/crc64"
 	"math"
 	"sort"

 	"github.com/fumiama/jieba/posseg"
+	"github.com/fumiama/jieba/util/helper"
 )

 const dampingFactor = 0.85

 var (
-	defaultAllowPOS = []string{"ns", "n", "vn", "v"}
+	defaultAllowPOS = [...]string{"ns", "n", "vn", "v"}
 )

 type edge struct {
+	weight float64
 	start  string
 	end    string
-	weight float64
 }

-type edges []edge
+type edges []*edge

 func (es edges) Len() int {
 	return len(es)
@@ -39,25 +41,25 @@ type undirectWeightedGraph struct {
 }

 func newUndirectWeightedGraph() *undirectWeightedGraph {
-	u := new(undirectWeightedGraph)
-	u.graph = make(map[string]edges)
-	u.keys = make(sort.StringSlice, 0)
-	return u
+	return &undirectWeightedGraph{
+		graph: make(map[string]edges, 256),
+		keys:  make(sort.StringSlice, 0, 256),
+	}
 }

 func (u *undirectWeightedGraph) addEdge(start, end string, weight float64) {
 	if _, ok := u.graph[start]; !ok {
 		u.keys = append(u.keys, start)
-		u.graph[start] = edges{edge{start: start, end: end, weight: weight}}
+		u.graph[start] = edges{&edge{start: start, end: end, weight: weight}}
 	} else {
-		u.graph[start] = append(u.graph[start], edge{start: start, end: end, weight: weight})
+		u.graph[start] = append(u.graph[start], &edge{start: start, end: end, weight: weight})
 	}

 	if _, ok := u.graph[end]; !ok {
 		u.keys = append(u.keys, end)
-		u.graph[end] = edges{edge{start: end, end: start, weight: weight}}
+		u.graph[end] = edges{&edge{start: end, end: start, weight: weight}}
 	} else {
-		u.graph[end] = append(u.graph[end], edge{start: end, end: start, weight: weight})
+		u.graph[end] = append(u.graph[end], &edge{start: end, end: start, weight: weight})
 	}
 }

@@ -66,8 +68,8 @@ func (u *undirectWeightedGraph) rank() Segments {
 		sort.Sort(u.keys)
 	}

-	ws := make(map[string]float64)
-	outSum := make(map[string]float64)
+	ws := make(map[string]float64, len(u.graph)*2)
+	outSum := make(map[string]float64, len(u.graph)*2)

 	wsdef := 1.0
 	if len(u.graph) > 0 {
@@ -101,9 +103,11 @@ func (u *undirectWeightedGraph) rank() Segments {
 			maxRank = w
 		}
 	}
-	result := make(Segments, 0)
+	result := make(Segments, len(ws))
+	i := 0
 	for n, w := range ws {
-		result = append(result, Segment{text: n, weight: (w - minRank/10.0) / (maxRank - minRank/10.0)})
+		result[i] = &Segment{text: n, weight: (w - minRank/10.0) / (maxRank - minRank/10.0)}
+		i++
 	}
 	sort.Sort(sort.Reverse(result))
 	return result
@@ -112,12 +116,20 @@ func (u *undirectWeightedGraph) rank() Segments {
 // TextRankWithPOS extracts keywords from sentence using TextRank algorithm.
 // Parameter allowPOS allows a customized pos list.
 func (t *TextRanker) TextRankWithPOS(sentence string, topK int, allowPOS []string) Segments {
-	posFilt := make(map[string]int)
+	posFilt := make(map[string]int, len(allowPOS)*2)
 	for _, pos := range allowPOS {
 		posFilt[pos] = 1
 	}
 	g := newUndirectWeightedGraph()
-	cm := make(map[[2]string]float64)
+	cm := make(map[uint64]float64, 256)
+	hm := make(map[uint64][2]string, 256)
+	gethash := func(a, b string) uint64 {
+		h := crc64.New(crc64.MakeTable(crc64.ISO))
+		h.Write(helper.StringToBytes(a))
+		h.Write([]byte("\t"))
+		h.Write(helper.StringToBytes(b))
+		return h.Sum64()
+	}
 	span := 5
 	var pairs []posseg.Segment
 	for pair := range t.seg.Cut(sentence, true) {
@@ -129,15 +141,18 @@ func (t *TextRanker) TextRankWithPOS(sentence string, topK int, allowPOS []strin
 				if _, ok := posFilt[pairs[j].Pos()]; !ok {
 					continue
 				}
-				if _, ok := cm[[2]string{pairs[i].Text(), pairs[j].Text()}]; !ok {
-					cm[[2]string{pairs[i].Text(), pairs[j].Text()}] = 1.0
+				h := gethash(pairs[i].Text(), pairs[j].Text())
+				if _, ok := cm[h]; !ok {
+					cm[h] = 1.0
+					hm[h] = [2]string{pairs[i].Text(), pairs[j].Text()}
 				} else {
-					cm[[2]string{pairs[i].Text(), pairs[j].Text()}] += 1.0
+					cm[h] += 1.0
 				}
 			}
 		}
 	}
-	for startEnd, weight := range cm {
+	for h, weight := range cm {
+		startEnd := hm[h]
 		g.addEdge(startEnd[0], startEnd[1], weight)
 	}
 	tags := g.rank()
@@ -150,7 +165,7 @@ func (t *TextRanker) TextRankWithPOS(sentence string, topK int, allowPOS []strin
 // TextRank extract keywords from sentence using TextRank algorithm.
 // Parameter topK specify how many top keywords to be returned at most.
 func (t *TextRanker) TextRank(sentence string, topK int) Segments {
-	return t.TextRankWithPOS(sentence, topK, defaultAllowPOS)
+	return t.TextRankWithPOS(sentence, topK, defaultAllowPOS[:])
 }

 // TextRanker is used to extract tags from sentence.
--- a/analyse/textrank_test.go
+++ b/analyse/textrank_test.go
@@ -9,16 +9,16 @@ var (
 	sentence = "此外，公司拟对全资子公司吉林欧亚置业有限公司增资4.3亿元，增资后，吉林欧亚置业注册资本由7000万元增加到5亿元。吉林欧亚置业主要经营范围为房地产开发及百货零售等业务。目前在建吉林欧亚城市商业综合体项目。2013年，实现营业收入0万元，实现净利润-139.13万元。"

 	tagRanks = Segments{
-		Segment{text: "吉林", weight: 1.0},
-		Segment{text: "欧亚", weight: 0.87807810644},
-		Segment{text: "置业", weight: 0.562048250306},
-		Segment{text: "实现", weight: 0.520905743929},
-		Segment{text: "收入", weight: 0.384283870648},
-		Segment{text: "增资", weight: 0.360590945312},
-		Segment{text: "子公司", weight: 0.353131980904},
-		Segment{text: "城市", weight: 0.307509449283},
-		Segment{text: "全资", weight: 0.306324426665},
-		Segment{text: "商业", weight: 0.306138241063},
+		&Segment{text: "吉林", weight: 1.0},
+		&Segment{text: "欧亚", weight: 0.87807810644},
+		&Segment{text: "置业", weight: 0.562048250306},
+		&Segment{text: "实现", weight: 0.520905743929},
+		&Segment{text: "收入", weight: 0.384283870648},
+		&Segment{text: "增资", weight: 0.360590945312},
+		&Segment{text: "子公司", weight: 0.353131980904},
+		&Segment{text: "城市", weight: 0.307509449283},
+		&Segment{text: "全资", weight: 0.306324426665},
+		&Segment{text: "商业", weight: 0.306138241063},
 	}
 )

--- a/util/helper/helper.go
+++ b/util/helper/helper.go
@@ -0,0 +1,21 @@
+package helper
+
+import (
+	"reflect"
+	"unsafe"
+)
+
+// BytesToString 没有内存开销的转换
+func BytesToString(b []byte) string {
+	return *(*string)(unsafe.Pointer(&b))
+}
+
+// StringToBytes 没有内存开销的转换
+func StringToBytes(s string) (b []byte) {
+	bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	sh := (*reflect.StringHeader)(unsafe.Pointer(&s))
+	bh.Data = sh.Data
+	bh.Len = sh.Len
+	bh.Cap = sh.Len
+	return b
+}