fumiama/jieba
1
0
Fork 0
mirror of https://github.com/fumiama/jieba.git synced 2026-06-05 00:32:51 +08:00
Code Activity

Merge branch 'release/v0.1'

Browse Source
This commit is contained in:
Wang Bin
2015-02-28 18:26:18 +08:00
parent 052c571015 8bf9888a1c
commit ebf4635334
31 changed files with 88124 additions and 87626 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@@ -1,3 +1,3 @@
language: go
go:
- 1.3
- 1.4.2

102
README.md
View File

@@ -19,44 +19,56 @@
import (
"fmt"
"github.com/wangbin/jiebago"
"strings"
)
var sentence = "我来到北京清华大学"
func print(ch chan string) {
for word := range ch {
fmt.Printf("%s / ", word)
}
fmt.Println()
fmt.Println()
}
func main() {
jiebago.SetDictionary("/Path/to/default/dictionary/file") // 设定字典
fmt.Printf("【全模式】: %s\n\n", strings.Join(jiebago.Cut(sentence, true, true), "/ "))
fmt.Printf("【精确模式】: %s\n\n", strings.Join(jiebago.Cut(sentence, false, true), "/ "))
fmt.Printf("【新词识别】：%s\n\n", strings.Join(jiebago.Cut("他来到了网易杭研大厦", false, true), ", "))
fmt.Printf("【搜索引擎模式】：%s\n\n", strings.Join(jiebago.CutForSearch("小明硕士毕业于中国科学院计算所，后在日本京都大学深造", true), ", "))
jiebago.SetDictionary("/Path/to/dictionary/file") // 设定字典
fmt.Print("【全模式】： ")
print(jiebago.Cut(sentence, true, true))
fmt.Print("【精确模式】： ")
print(jiebago.Cut(sentence, false, true))
fmt.Print("【新词识别】：")
print(jiebago.Cut("他来到了网易杭研大厦", false, true))
fmt.Print("【搜索引擎模式】：")
print(jiebago.CutForSearch("小明硕士毕业于中国科学院计算所，后在日本京都大学深造", true))
}
使用结巴分词自带的[词典文件](https://github.com/fxsjy/jieba/blob/master/jieba/dict.txt)，输出结果如下：
【全模式】: 我/ 来到/ 北京/ 清华/ 清华大学/ 华大/ 大学
【全模式】： 我 / 来到 / 北京 / 清华 / 清华大学 / 华大 / 大学 /
【精确模式】: 我/ 来到/ 北京/ 清华大学
【精确模式】： 我 / 来到 / 北京 / 清华大学 /
【新词识别】：他, 来到, 了, 网易, 杭研, 大厦
【新词识别】：他 / 来到 / 了 / 网易 / 杭研 / 大厦 /
【搜索引擎模式】：小明 / 硕士 / 毕业 / 于 / 中国 / 科学 / 学院 / 科学院 / 中国科学院 / 计算 / 计算所 / ， / 后 / 在 / 日本 / 京都 / 大学 / 日本京都大学 / 深造 /
【搜索引擎模式】：小明, 硕士, 毕业, 于, 中国, 科学, 学院, 科学院, 中国科学院, 计算, 计算所, ，, 后, 在, 日本, 京都, 大学, 日本京都大学, 深造
添加自定义词典
=============
var sentence = "李小福是创新办主任也是云计算方面的专家"
jiebago.SetDictionary("/Path/to/default/dictionary/file")
fmt.Printf("Before: %s\n\n", strings.Join(jiebago.Cut(sentence, false, true), "/ "))
fmt.Print("Before: ")
print(jiebago.Cut(sentence, false, true))
jiebago.LoadUserDict("/Path/to/user/dictionary/file")
fmt.Printf("After: %s\n\n", strings.Join(jiebago.Cut(sentence, false, true), "/ "))
fmt.Print("After: ")
print(jiebago.Cut(sentence, false, true))
使用结巴分词自带的[词典文件](https://github.com/fxsjy/jieba/blob/master/jieba/dict.txt)和[用户自定义词典文件](https://github.com/fxsjy/jieba/blob/master/test/userdict.txt)，结果输出如下：
Before: 李小福/ 是/ 创新/ 办/ 主任/ 也/ 是/ 云/ 计算/ 方面/ 的/ 专家
Before: 李小福 / 是 / 创新 / 办 / 主任 / 也 / 是 / 云 / 计算 / 方面 / 的 / 专家 /
After: 李小福 / 是 / 创新办 / 主任 / 也 / 是 / 云计算 / 方面 / 的 / 专家 /
After: 李小福/ 是/ 创新办/ 主任/ 也/ 是/ 云计算/ 方面/ 的/ 专家
关键词提取
========
@@ -70,23 +82,57 @@
import (
"fmt"
"github.com/wangbin/jiebago"
"github.com/wangbin/jiebago/analyse"
"strings"
)
var sentence = "这是一个伸手不见五指的黑夜。我叫孙悟空，我爱北京，我爱Python和C++。"
func main() {
jiebago.SetDictionary("/Path/to/default/dictionary/file")
analyse.SetDictionary("/Path/to/dictionary/file")
analyse.SetIdf("/Path/to/idf/file")
fmt.Println(strings.Join(analyse.ExtractTags(sentence, 20), "/ "))
for _, ww := range analyse.ExtractTags(sentence, 20) {
fmt.Printf("%s / ", ww.Word)
}
}
输出：
Python/ C++/ 伸手不见五指/ 孙悟空/ 黑夜/ 北京/ 这是/ 一个
Python / C++ / 伸手不见五指 / 孙悟空 / 黑夜 / 北京 / 这是 / 一个 /
## 基于TextRank算法的关键词抽取实现
示例代码：
package main
import (
"fmt"
"github.com/wangbin/jiebago/analyse"
)
func main() {
sentence := "此外，公司拟对全资子公司吉林欧亚置业有限公司增资4.3亿元，增资后，吉林欧亚 置业注册资本由7000万元增加到5亿元。吉林欧亚置业主要经营范围为房地产开发及百货零售等业务。目前在建吉林欧亚城市商业综合体项目。2013年，实现营业收入0万元，实现净利润-139.13万元。"
analyse.SetDictionary("/Path/to/dictionary/file")
result := analyse.TextRank(sentence, 10)
for _, wt := range result {
fmt.Printf("%s %f\n", wt.Word, wt.Freq)
}
}
输出：
吉林 1.000000
欧亚 0.878078
置业 0.562048
实现 0.520906
收入 0.384284
增资 0.360591
子公司 0.353132
城市 0.307509
全资 0.306324
商业 0.306138
词性标注
=======
@@ -107,8 +153,8 @@
var sentence = "我爱北京天安门"
func main() {
jiebago.SetDictionary("/Path/to/default/dictionary/file")
for _, wt := range posseg.Cut(sentence, true) {
posseg.SetDictionary("/Path/to/dictionary/file")
for wt := range posseg.Cut(sentence, true) {
fmt.Printf("%s %s\n", wt.Word, wt.Tag)
}
}
@@ -135,7 +181,9 @@
fileLength += len([]rune(line))
lineCount += 1
go func() {
ch <- jiebago.Cut(line, false, true)
for word := range jiebago.Cut(line, false, true) {
ch <- word
}
}()
}
if err := scanner.Err(); err != nil {
@@ -195,4 +243,4 @@ Tokenize
许可证
======
MIT: http://wangbin.mit-license.org
MIT: http://wangbin.mit-license.org

63
analyse/analyse.go
View File

@@ -1,39 +1,45 @@
package analyse
import (
"fmt"
"github.com/wangbin/jiebago"
"sort"
"strings"
"unicode/utf8"
)
type TfIdf struct {
word string
freq float64
type wordWeight struct {
Word string
Weight float64
}
type TfIdfs []TfIdf
func (tis TfIdfs) Len() int {
return len(tis)
func (w wordWeight) String() string {
return fmt.Sprintf("{%s: %f}", w.Word, w.Weight)
}
func (tis TfIdfs) Less(i, j int) bool {
if tis[i].freq == tis[j].freq {
return tis[i].word < tis[j].word
type wordWeights []wordWeight
func (ws wordWeights) Len() int {
return len(ws)
}
func (ws wordWeights) Less(i, j int) bool {
if ws[i].Weight == ws[j].Weight {
return ws[i].Word < ws[j].Word
}
return tis[i].freq < tis[j].freq
return ws[i].Weight < ws[j].Weight
}
func (tis TfIdfs) Swap(i, j int) {
tis[i], tis[j] = tis[j], tis[i]
func (ws wordWeights) Swap(i, j int) {
ws[i], ws[j] = ws[j], ws[i]
}
func ExtractTags(sentence string, topK int) []string {
words := jiebago.Cut(sentence, false, true)
// Keyword extraction.
func ExtractTags(sentence string, topK int) (tags wordWeights) {
freq := make(map[string]float64)
for _, w := range words {
for w := range jiebago.Cut(sentence, false, true) {
w = strings.TrimSpace(w)
if utf8.RuneCountInString(w) < 2 {
continue
@@ -54,26 +60,21 @@ func ExtractTags(sentence string, topK int) []string {
for k, v := range freq {
freq[k] = v / total
}
tis := make(TfIdfs, 0)
ws := make(wordWeights, 0)
for k, v := range freq {
var ti TfIdf
if freq_, ok := idfFreq[k]; ok {
ti = TfIdf{word: k, freq: freq_ * v}
var ti wordWeight
if freq_, ok := loader.Freq[k]; ok {
ti = wordWeight{Word: k, Weight: freq_ * v}
} else {
ti = TfIdf{word: k, freq: medianIdf * v}
ti = wordWeight{Word: k, Weight: loader.Median * v}
}
tis = append(tis, ti)
ws = append(ws, ti)
}
sort.Sort(sort.Reverse(tis))
var topTfIdfs TfIdfs
if len(tis) > topK {
topTfIdfs = tis[:topK]
sort.Sort(sort.Reverse(ws))
if len(ws) > topK {
tags = ws[:topK]
} else {
topTfIdfs = tis
}
tags := make([]string, len(topTfIdfs))
for index, ti := range topTfIdfs {
tags[index] = ti.word
tags = ws
}
return tags
}

262
analyse/analyse_test.go
View File

@@ -2,6 +2,7 @@ package analyse
import (
"github.com/wangbin/jiebago"
"math"
"testing"
)
@@ -92,107 +93,206 @@ var (
"C++和c#是什么关系？11+122=133，是吗？PI=3.14159",
"你认识那个和主席握手的的哥吗？他开一辆黑色的士。",
"枪杆子中出政权"}
Tags = [][]string{
[]string{"Python", "C++", "\u4f38\u624b\u4e0d\u89c1\u4e94\u6307", "\u5b59\u609f\u7a7a", "\u9ed1\u591c", "\u5317\u4eac", "\u8fd9\u662f", "\u4e00\u4e2a"},
[]string{"\u548c\u670d", "\u559c\u6b22", "\u65e5\u672c"},
[]string{"\u96f7\u7334", "\u4eba\u95f4", "\u56de\u5f52"},
[]string{"\u5de5\u4fe1\u5904", "\u5973\u5e72\u4e8b", "24", "\u4ea4\u6362\u673a", "\u79d1\u5ba4", "\u4eb2\u53e3", "\u5668\u4ef6", "\u6280\u672f\u6027", "\u4e0b\u5c5e", "\u4ea4\u4ee3", "\u6bcf\u6708", "\u5b89\u88c5", "\u7ecf\u8fc7", "\u5de5\u4f5c"},
[]string{"\u5ec9\u79df\u623f", "\u9700\u8981"},
[]string{"\u9970\u54c1", "\u6c38\u548c", "\u670d\u88c5", "\u6709\u9650\u516c\u53f8"},
[]string{"\u5929\u5b89\u95e8", "\u5317\u4eac"},
[]string{"Python", "C++", "伸手不见五指", "孙悟空", "黑夜", "北京", "这是", "一个"},
[]string{"和服", "喜欢", "日本"},
[]string{"雷猴", "人间", "回归"},
[]string{"工信处", "女干事", "24", "交换机", "科室", "亲口", "器件", "技术性", "下属", "交代", "每月", "安装", "经过", "工作"},
[]string{"廉租房", "需要"},
[]string{"饰品", "永和", "服装", "有限公司"},
[]string{"天安门", "北京"},
[]string{"abc"},
[]string{"\u9a6c\u5c14\u53ef\u592b"},
[]string{"\u96f7\u7334", "\u7f51\u7ad9"},
[]string{"SOFTware", "Microsoft", "MICROcomputer", "\u5fae\u578b", "\u4e00\u8bcd", "\u8f6f\u4ef6", "\u8ba1\u7b97\u673a", "\u7ec4\u6210", "\u90e8\u5206"},
[]string{"\u8349\u6ce5\u9a6c", "\u6b3a\u5b9e", "\u8bcd\u6c47", "\u6d41\u884c", "\u4eca\u5e74"},
[]string{"\u6d0b\u534e\u5802", "\u603b\u5e9c", "\u4f0a\u85e4"},
[]string{"\u4e2d\u56fd\u79d1\u5b66\u9662\u8ba1\u7b97\u6280\u672f\u7814\u7a76\u6240"},
[]string{"\u6731\u4e3d\u53f6", "\u7f57\u5bc6\u6b27"},
[]string{"\u9053\u5177", "\u670d\u88c5", "\u8d2d\u4e70"},
[]string{"\u81ea\u73cd", "\u655e\u5e1a", "PS", "\u5f00\u6e90", "\u4e0d\u65ad\u6539\u8fdb", "\u6566\u4fc3", "\u597d\u5904", "\u907f\u514d", "\u80fd\u591f", "\u89c9\u5f97", "\u5c31\u662f", "\u81ea\u5df1", "\u4e00\u4e2a"},
[]string{"\u77f3\u9996\u5e02", "\u6e56\u5317\u7701"},
[]string{"\u5341\u5830\u5e02", "\u6e56\u5317\u7701"},
[]string{"\u603b\u7ecf\u7406", "\u8fd9\u4ef6", "\u5b8c\u6210", "\u4e8b\u60c5"},
[]string{"\u4fee\u597d", "\u7535\u8111"},
[]string{"\u4e00\u4e86\u767e\u4e86", "\u505a\u597d", "\u8fd9\u4ef6", "\u4e8b\u60c5"},
[]string{"\u5ba1\u7f8e", "\u89c2\u70b9", "\u4eba\u4eec", "\u4e0d\u540c"},
[]string{"\u7f8e\u7684", "\u7a7a\u8c03", "\u6211\u4eec", "\u4e00\u4e2a"},
[]string{"\u7ebf\u7a0b", "\u521d\u59cb\u5316", "\u6ce8\u610f", "\u6211\u4eec"},
[]string{"\u597d\u591a", "\u539f\u5b50", "\u5206\u5b50", "\u7ec4\u7ec7", "\u4e00\u4e2a"},
[]string{"\u9a6c\u5230\u529f\u6210"},
[]string{"\u65e0\u5e95\u6d1e"},
[]string{"\u9996\u90fd", "\u5317\u4eac", "\u4e2d\u56fd"},
[]string{"\u5b59\u541b\u610f"},
[]string{"\u9a6c\u671d\u65ed", "\u5916\u4ea4\u90e8", "\u53d1\u8a00\u4eba"},
[]string{"\u7b2c\u56db\u5c4a", "\u4e1c\u4e9a", "\u5cf0\u4f1a", "\u9886\u5bfc\u4eba", "\u4f1a\u8bae"},
[]string{"\u4e94\u5e74", "\u8fc7\u53bb"},
[]string{"\u5f88\u957f", "\u9700\u8981"},
[]string{"60", "\u9605\u5175", "\u5468\u5e74", "\u9996\u90fd"},
[]string{"\u5ba1\u7f8e", "\u4f60\u597d", "\u89c2\u70b9", "\u4eba\u4eec", "\u4e0d\u540c"},
[]string{"\u4e16\u535a\u56ed", "\u6c34\u679c", "\u7136\u540e"},
[]string{"\u4e16\u535a\u56ed", "\u6c34\u679c", "\u7136\u540e"},
[]string{"\u540e\u6765", "\u4f46\u662f", "\u77e5\u9053"},
[]string{"\u5408\u7406", "\u5b58\u5728"},
[]string{"马尔可夫"},
[]string{"雷猴", "网站"},
[]string{"SOFTware", "Microsoft", "MICROcomputer", "微型", "一词", "软件", "计算机", "组成", "部分"},
[]string{"草泥马", "欺实", "词汇", "流行", "今年"},
[]string{"洋华堂", "总府", "伊藤"},
[]string{"中国科学院计算技术研究所"},
[]string{"朱丽叶", "罗密欧"},
[]string{"道具", "服装", "购买"},
[]string{"自珍", "敞帚", "PS", "开源", "不断改进", "敦促", "好处", "避免", "能够", "觉得", "就是", "自己", "一个"},
[]string{"石首市", "湖北省"},
[]string{"十堰市", "湖北省"},
[]string{"总经理", "这件", "完成", "事情"},
[]string{"修好", "电脑"},
[]string{"一了百了", "做好", "这件", "事情"},
[]string{"审美", "观点", "人们", "不同"},
[]string{"美的", "空调", "我们", "一个"},
[]string{"线程", "初始化", "注意", "我们"},
[]string{"好多", "原子", "分子", "组织", "一个"},
[]string{"马到功成"},
[]string{"无底洞"},
[]string{"首都", "北京", "中国"},
[]string{"孙君意"},
[]string{"马朝旭", "外交部", "发言人"},
[]string{"第四届", "东亚", "峰会", "领导人", "会议"},
[]string{"五年", "过去"},
[]string{"很长", "需要"},
[]string{"60", "阅兵", "周年", "首都"},
[]string{"审美", "你好", "观点", "人们", "不同"},
[]string{"世博园", "水果", "然后"},
[]string{"世博园", "水果", "然后"},
[]string{"后来", "但是", "知道"},
[]string{"合理", "存在"},
[]string{},
[]string{"rong", "love", "\u4e0d\u4ee5\u4e3a\u803b", "\u4ee5\u4e3a"},
[]string{"rong", "love", "不以为耻", "以为"},
[]string{},
[]string{},
[]string{"hello", "\u5ba1\u7f8e", "\u4f60\u597d", "\u89c2\u70b9", "\u4eba\u4eec", "\u4e0d\u540c"},
[]string{"\u7f51\u9875", "\u57fa\u4e8e", "\u5f62\u5f0f", "\u4e3b\u8981"},
[]string{"hello", "\u5ba1\u7f8e", "\u4f60\u597d", "\u89c2\u70b9", "\u4eba\u4eec", "\u4e0d\u540c"},
[]string{"\u60f3\u8981", "\u62e5\u6709", "\u4e3a\u4ec0\u4e48", "\u751f\u6d3b", "\u4e0d\u80fd"},
[]string{"\u540e\u6765"},
[]string{"\u6b64\u6b21", "\u4e3a\u4e86", "\u4e2d\u56fd"},
[]string{"\u89e3\u51b3", "\u4f7f\u7528", "\u4e00\u4e9b", "\u95ee\u9898", "\u53ef\u4ee5"},
[]string{"\u89e3\u51b3", "\u4f7f\u7528", "\u4e00\u4e9b", "\u95ee\u9898", "\u53ef\u4ee5"},
[]string{"\u89e3\u51b3", "\u5176\u5b9e", "\u4f7f\u7528", "\u4e00\u4e9b", "\u95ee\u9898", "\u53ef\u4ee5"},
[]string{"\u597d\u4eba", "\u89e3\u51b3", "\u4f7f\u7528", "\u4e00\u4e9b", "\u95ee\u9898", "\u53ef\u4ee5"},
[]string{"\u662f\u56e0\u4e3a", "\u56fd\u5bb6"},
[]string{"\u8001\u5e74", "\u641c\u7d22", "\u652f\u6301"},
[]string{"\u95f2\u6cd5", "\u4e2d\u672c", "laoshipukong", "RT", "27", "\u8d23\u4efb\u6cd5", "\u8499\u4eba", "\u4e07\u52ab\u4e0d\u590d", "\u4e3e\u8bc1", "\u5012\u7f6e", "\u533b\u60a3", "\u90a3\u90e8", "\u62c9\u5012", "\u4fb5\u6743", "\u5168\u56fd\u4eba\u5927\u5e38\u59d4\u4f1a", "\u8349\u6848", "\u5883\u5730", "\u7ea0\u7eb7", "\u5220\u9664", "\u5f31\u52bf"},
[]string{"hello", "审美", "你好", "观点", "人们", "不同"},
[]string{"网页", "基于", "形式", "主要"},
[]string{"hello", "审美", "你好", "观点", "人们", "不同"},
[]string{"想要", "拥有", "为什么", "生活", "不能"},
[]string{"后来"},
[]string{"此次", "为了", "中国"},
[]string{"解决", "使用", "一些", "问题", "可以"},
[]string{"解决", "使用", "一些", "问题", "可以"},
[]string{"解决", "其实", "使用", "一些", "问题", "可以"},
[]string{"好人", "解决", "使用", "一些", "问题", "可以"},
[]string{"是因为", "国家"},
[]string{"老年", "搜索", "支持"},
[]string{"闲法", "中本", "laoshipukong", "RT", "27", "责任法", "蒙人", "万劫不复", "举证", "倒置", "医患", "那部", "拉倒", "侵权", "全国人大常委会", "草案", "境地", "纠纷", "删除", "弱势"},
[]string{},
[]string{},
[]string{"\u5728\u7406", "\u786e\u5b9e"},
[]string{"\u957f\u6625", "\u6625\u8282", "\u8bb2\u8bdd", "\u5e02\u957f"},
[]string{"\u7ed3\u5a5a", "\u5c1a\u672a"},
[]string{"\u5206\u5b50", "\u7ed3\u5408"},
[]string{"\u65c5\u6e38", "\u6700\u597d", "\u670d\u52a1"},
[]string{"\u7684\u786e", "\u8fd9\u4ef6", "\u4e8b\u60c5"},
[]string{"\u6307\u6b63", "\u53c2\u8003", "\u5927\u5bb6"},
[]string{"\u584c\u6865", "\u54c8\u5c14\u6ee8", "\u516c\u5e03", "\u539f\u56e0", "\u653f\u5e9c"},
[]string{"\u5165\u53e3\u5904", "\u673a\u573a"},
[]string{"\u90a2\u6c38\u81e3", "\u6444\u5f71", "\u62a5\u9053"},
[]string{"\u533a\u5206\u5ea6", "BP", "\u795e\u7ecf\u7f51\u7edc", "\u8bad\u7ec3", "\u5206\u7c7b", "\u624d\u80fd", "\u5982\u4f55", "\u589e\u52a0"},
[]string{"\u957f\u6c5f\u5927\u6865", "\u5357\u4eac\u5e02"},
[]string{"SMT", "NiuTrans", "\u4f7f\u7528\u8005", "\u4fbf\u4e8e", "\u7528\u4e8e", "\u5efa\u8bae", "\u5229\u7528", "\u4e3a\u4e86", "\u7814\u7a76", "\u4e00\u4e9b"},
[]string{"\u957f\u6625\u5e02", "\u836f\u5e97", "\u957f\u6625"},
[]string{"\u9093\u9896\u8d85", "\u751f\u524d", "\u8863\u670d", "\u559c\u6b22"},
[]string{"\u653f\u6cbb\u5c40", "\u70ed\u7231", "\u5e38\u59d4", "\u80e1\u9526\u6d9b", "\u548c\u5e73", "\u4e16\u754c"},
[]string{"\u53f3\u9762", "\u5b59\u5065", "\u8303\u51ef", "\u674e\u677e\u6d2a", "\u6731\u4f1a\u9707", "\u6d77\u6797", "\u5de6\u9762", "\u7a0b\u5e8f\u5458", "\u518d\u5f80"},
[]string{"\u4e00\u6b21\u6027", "\u591a\u5c11"},
[]string{"\u56db\u5757", "\u4e94\u5757", "\u4e09\u5757", "\u4e00\u65a4", "\u4e24\u5757", "\u4e00\u672c", "\u4e00\u5957", "\u4e00\u6761"},
[]string{"\u548c\u5c1a", "\u548c\u5c1a\u5934", "\u4e00\u6837", "\u4e00\u4e2a"},
[]string{"\u548c\u5e73\u95e8", "\u5171\u548c\u515a", "\u5730\u94c1", "\u515a\u5458", "\u516c\u6c11", "\u7238\u7238", "\u4e2d\u534e\u4eba\u6c11\u5171\u548c\u56fd"},
[]string{"\u5f20\u6653\u6885", "T\u6064", "B\u8d85", "\u533b\u9662", "\u4eba\u6c11", "\u7136\u540e"},
[]string{"offer", "AT&T", "\u4e0d\u9519", "\u4e00\u4ef6", "\u516c\u53f8"},
[]string{"c#", "PI", "C++", "3.14159", "133", "122", "11", "\u5173\u7cfb", "\u4ec0\u4e48"},
[]string{"\u7684\u58eb", "\u7684\u54e5", "\u4ed6\u5f00", "\u63e1\u624b", "\u4e00\u8f86", "\u9ed1\u8272", "\u4e3b\u5e2d", "\u8ba4\u8bc6", "\u90a3\u4e2a"},
[]string{"\u67aa\u6746\u5b50", "\u653f\u6743"},
[]string{"在理", "确实"},
[]string{"长春", "春节", "讲话", "市长"},
[]string{"结婚", "尚未"},
[]string{"分子", "结合"},
[]string{"旅游", "最好", "服务"},
[]string{"的确", "这件", "事情"},
[]string{"指正", "参考", "大家"},
[]string{"塌桥", "哈尔滨", "公布", "原因", "政府"},
[]string{"入口处", "机场"},
[]string{"邢永臣", "摄影", "报道"},
[]string{"区分度", "BP", "神经网络", "训练", "分类", "才能", "如何", "增加"},
[]string{"长江大桥", "南京市"},
[]string{"SMT", "NiuTrans", "使用者", "便于", "用于", "建议", "利用", "为了", "研究", "一些"},
[]string{"长春市", "药店", "长春"},
[]string{"邓颖超", "生前", "衣服", "喜欢"},
[]string{"政治局", "热爱", "常委", "胡锦涛", "和平", "世界"},
[]string{"右面", "孙健", "范凯", "李松洪", "朱会震", "海林", "左面", "程序员", "再往"},
[]string{"一次性", "多少"},
[]string{"四块", "五块", "三块", "一斤", "两块", "一本", "一套", "一条"},
[]string{"和尚", "和尚头", "一样", "一个"},
[]string{"和平门", "共和党", "地铁", "党员", "公民", "爸爸", "中华人民共和国"},
[]string{"张晓梅", "T恤", "B超", "医院", "人民", "然后"},
[]string{"offer", "AT&T", "不错", "一件", "公司"},
[]string{"c#", "PI", "C++", "3.14159", "133", "122", "11", "关系", "什么"},
[]string{"的士", "的哥", "他开", "握手", "一辆", "黑色", "主席", "认识", "那个"},
[]string{"枪杆子", "政权"},
}
Lyric = `
我沒有心
我沒有真實的自我
我只有消瘦的臉孔
所謂軟弱
所謂的順從一向是我
的座右銘
而我
沒有那海洋的寬闊
我只要熱情的撫摸
所謂空洞
所謂不安全感是我
的墓誌銘
而你
是否和我一般怯懦
是否和我一般矯作
和我一般囉唆
而你
是否和我一般退縮
是否和我一般肌迫
一般地困惑
我沒有力
我沒有滿腔的熱火
我只有滿肚的如果
所謂勇氣
所謂的認同感是我
隨便說說
而你
是否和我一般怯懦
是否和我一般矯作
是否對你來說
只是一場遊戲
雖然沒有把握
而你
是否和我一般退縮
是否和我一般肌迫
是否對你來說
只是逼不得已
雖然沒有藉口
`
LyciWeight = []wordWeight{
wordWeight{Word: "所謂", Weight: 1.010262},
wordWeight{Word: "是否", Weight: 0.738650},
wordWeight{Word: "一般", Weight: 0.607600},
wordWeight{Word: "雖然", Weight: 0.336754},
wordWeight{Word: "退縮", Weight: 0.336754},
wordWeight{Word: "肌迫", Weight: 0.336754},
wordWeight{Word: "矯作", Weight: 0.336754},
wordWeight{Word: "沒有", Weight: 0.336754},
wordWeight{Word: "怯懦", Weight: 0.271099},
wordWeight{Word: "隨便", Weight: 0.168377},
}
LyciWeight2 = []wordWeight{
wordWeight{Word: "所謂", Weight: 1.215739},
wordWeight{Word: "一般", Weight: 0.731179},
wordWeight{Word: "雖然", Weight: 0.405246},
wordWeight{Word: "退縮", Weight: 0.405246},
wordWeight{Word: "肌迫", Weight: 0.405246},
wordWeight{Word: "矯作", Weight: 0.405246},
wordWeight{Word: "怯懦", Weight: 0.326238},
wordWeight{Word: "逼不得已", Weight: 0.202623},
wordWeight{Word: "右銘", Weight: 0.202623},
wordWeight{Word: "寬闊", Weight: 0.202623},
}
)
func TestExtractTags(t *testing.T) {
jiebago.SetDictionary("../dict.txt")
SetIdf("idf.txt")
for index, sentence := range test_contents {
result := ExtractTags(sentence, 20)
if len(result) != len(Tags[index]) {
t.Errorf("%s = %v", sentence, result)
}
for i, tag := range result {
if tag != Tags[index][i] {
t.Error(tag)
if tag.Word != Tags[index][i] {
t.Errorf("%s != %s", tag, Tags[index][i])
}
}
}
}
func TestExtratTagsWithWeight(t *testing.T) {
jiebago.SetDictionary("../dict.txt")
SetIdf("idf.txt")
result := ExtractTags(Lyric, 10)
for index, tag := range result {
if LyciWeight[index].Word != tag.Word ||
math.Abs(LyciWeight[index].Weight-tag.Weight) > 1e-6 {
t.Errorf("%v != %v", tag, LyciWeight[index])
}
}
}
func TestExtractTagsWithStopWordsFile(t *testing.T) {
jiebago.SetDictionary("../dict.txt")
SetIdf("idf.txt")
SetStopWords("stop_words.txt")
result := ExtractTags(Lyric, 7)
for index, tag := range result {
if LyciWeight2[index].Word != tag.Word ||
math.Abs(LyciWeight2[index].Weight-tag.Weight) > 1e-6 {
t.Errorf("%v != %v", tag, LyciWeight2[index])
}
}
}

85
analyse/idf.go
View File

@@ -1,80 +1,57 @@
package analyse
import (
"bufio"
"os"
"path/filepath"
"github.com/wangbin/jiebago"
"sort"
"strconv"
"strings"
)
var (
stopWords map[string]string
idfFreq map[string]float64
medianIdf float64
loader *idfLoader
)
func init() {
idfFreq = make(map[string]float64)
stopWords = map[string]string{
"the": "the", "of": "of", "is": "is", "and": "and", "to": "to", "in": "in", "that": "that", "we": "we", "for": "for", "an": "an", "are": "are", "by": "bye", "be": "be", "as": "as", "on": "on", "with": "with", "can": "can", "if": "of", "from": "from", "which": "which", "you": "you", "it": "it", "this": "this", "then": "then", "at": "at", "have": "have", "all": "all", "not": "not", "one": "one", "has": "has", "or": "or",
}
loader = newIDFLoader()
}
func SetIdf(idfFilePath string) error {
if !filepath.IsAbs(idfFilePath) {
pwd, err := os.Getwd()
if err != nil {
return err
}
idfFilePath = filepath.Clean(filepath.Join(pwd, idfFilePath))
type idfLoader struct {
Path string
Freq map[string]float64
Median float64
}
func newIDFLoader() *idfLoader {
loader := new(idfLoader)
loader.Freq = make(map[string]float64)
return loader
}
func (loader *idfLoader) newPath(idfFilePath string) error {
if loader.Path == idfFilePath {
return nil
}
idfFile, err := os.Open(idfFilePath)
wtfs, err := jiebago.ParseDictFile(idfFilePath)
if err != nil {
return err
}
scanner := bufio.NewScanner(idfFile)
freqs := make([]float64, 0)
for scanner.Scan() {
line := scanner.Text()
words := strings.Split(line, " ")
word, freqStr := words[0], words[1]
freq, err := strconv.ParseFloat(freqStr, 64)
if err != nil {
continue
}
idfFreq[word] = freq
freqs = append(freqs, freq)
}
if err := scanner.Err(); err != nil {
return err
for _, wtf := range wtfs {
loader.Freq[wtf.Word] = wtf.Freq
freqs = append(freqs, wtf.Freq)
}
sort.Float64s(freqs)
medianIdf = freqs[len(freqs)/2]
loader.Median = freqs[len(freqs)/2]
return nil
}
func SetStopWords(stopWordsFilePath string) error {
if !filepath.IsAbs(stopWordsFilePath) {
pwd, err := os.Getwd()
if err != nil {
return err
}
stopWordsFilePath = filepath.Clean(filepath.Join(pwd, stopWordsFilePath))
}
stopWordsFile, err := os.Open(stopWordsFilePath)
// Set the IDF file path, could be absolute path of IDF file, or IDF file
// name in current directory.
func SetIdf(idfFileName string) error {
idfFilePath, err := jiebago.DictPath(idfFileName)
if err != nil {
return err
}
scanner := bufio.NewScanner(stopWordsFile)
for scanner.Scan() {
stopWord := scanner.Text()
stopWord = strings.TrimSpace(stopWord)
stopWords[stopWord] = stopWord
}
if err := scanner.Err(); err != nil {
return err
}
return nil
return loader.newPath(idfFilePath)
}

51
analyse/stop_words.txt Normal file
View File

@@ -0,0 +1,51 @@
the
of
is
and
to
in
that
we
for
an
are
by
be
as
on
with
can
if
from
which
you
it
this
then
at
have
all
not
one
has
or
that
的
了
和
是
就
都
而
及
與
著
或
一個
沒有
我們
你們
妳們
他們
她們
是否

58
analyse/stopwords.go Normal file
View File

@@ -0,0 +1,58 @@
package analyse
import (
"github.com/wangbin/jiebago"
)
var stopWords map[string]int
func init() {
stopWords = map[string]int{
"the": 1,
"of": 1,
"is": 1,
"and": 1,
"to": 1,
"in": 1,
"that": 1,
"we": 1,
"for": 1,
"an": 1,
"are": 1,
"by": 1,
"be": 1,
"as": 1,
"on": 1,
"with": 1,
"can": 1,
"if": 1,
"from": 1,
"which": 1,
"you": 1,
"it": 1,
"this": 1,
"then": 1,
"at": 1,
"have": 1,
"all": 1,
"not": 1,
"one": 1,
"has": 1,
"or": 1,
}
}
// Set the stop words file path, could be absolute path of stop words file, or
// file name in current directory.
func SetStopWords(stopWordsFileName string) error {
stopWordsFilePath, err := jiebago.DictPath(stopWordsFileName)
if err != nil {
return err
}
wtfs, err := jiebago.ParseDictFile(stopWordsFilePath)
for _, wtf := range wtfs {
stopWords[wtf.Word] = 1
}
return nil
}

168
analyse/textrank.go Normal file
View File

@@ -0,0 +1,168 @@
package analyse
import (
"fmt"
"github.com/wangbin/jiebago/posseg"
"math"
"sort"
)
const dampingFactor = 0.85
var (
defaultAllowPOS = []string{"ns", "n", "vn", "v"}
)
type edge struct {
start string
end string
weight float64
}
func (e edge) String() string {
return fmt.Sprintf("(%s %s): %f", e.start, e.end, e.weight)
}
type edges []edge
func (es edges) Len() int {
return len(es)
}
func (es edges) Less(i, j int) bool {
return es[i].weight < es[j].weight
}
func (es edges) Swap(i, j int) {
es[i], es[j] = es[j], es[i]
}
type undirectWeightedGraph struct {
graph map[string]edges
keys sort.StringSlice
}
func newUndirectWeightedGraph() *undirectWeightedGraph {
u := new(undirectWeightedGraph)
u.graph = make(map[string]edges)
u.keys = make(sort.StringSlice, 0)
return u
}
func (u *undirectWeightedGraph) addEdge(start, end string, weight float64) {
if _, ok := u.graph[start]; !ok {
u.keys = append(u.keys, start)
u.graph[start] = edges{edge{start: start, end: end, weight: weight}}
} else {
u.graph[start] = append(u.graph[start], edge{start: start, end: end, weight: weight})
}
if _, ok := u.graph[end]; !ok {
u.keys = append(u.keys, end)
u.graph[end] = edges{edge{start: end, end: start, weight: weight}}
} else {
u.graph[end] = append(u.graph[end], edge{start: end, end: start, weight: weight})
}
}
func (u *undirectWeightedGraph) rank() wordWeights {
if !sort.IsSorted(u.keys) {
sort.Sort(u.keys)
}
ws := make(map[string]float64)
outSum := make(map[string]float64)
wsdef := 1.0
if len(u.graph) > 0 {
wsdef /= float64(len(u.graph))
}
for n, out := range u.graph {
ws[n] = wsdef
sum := 0.0
for _, e := range out {
sum += e.weight
}
outSum[n] = sum
}
for x := 0; x < 10; x++ {
for _, n := range u.keys {
s := 0.0
inedges := u.graph[n]
for _, e := range inedges {
s += e.weight / outSum[e.end] * ws[e.end]
}
ws[n] = (1 - dampingFactor) + dampingFactor*s
}
}
minRank := math.MaxFloat64
maxRank := math.SmallestNonzeroFloat64
for _, w := range ws {
if w < minRank {
minRank = w
} else if w > maxRank {
maxRank = w
}
}
result := make(wordWeights, 0)
for n, w := range ws {
result = append(result, wordWeight{Word: n, Weight: (w - minRank/10.0) / (maxRank - minRank/10.0)})
}
sort.Sort(sort.Reverse(result))
return result
}
// Extract keywords from sentence using TextRank algorithm. the allowed POS list
// could be manually speificed.
func TextRankWithPOS(sentence string, topK int, allowPOS []string) wordWeights {
posFilt := make(map[string]int)
for _, pos := range allowPOS {
posFilt[pos] = 1
}
g := newUndirectWeightedGraph()
cm := make(map[[2]string]float64)
span := 5
wordTags := make([]posseg.WordTag, 0)
for wordTag := range posseg.Cut(sentence, true) {
wordTags = append(wordTags, wordTag)
}
for i, _ := range wordTags {
if _, ok := posFilt[wordTags[i].Tag]; ok {
for j := i + 1; j < i+span; j++ {
if j > len(wordTags) {
break
}
if _, ok := posFilt[wordTags[j].Tag]; !ok {
continue
}
if _, ok := cm[[2]string{wordTags[i].Word, wordTags[j].Word}]; !ok {
cm[[2]string{wordTags[i].Word, wordTags[j].Word}] = 1.0
} else {
cm[[2]string{wordTags[i].Word, wordTags[j].Word}] += 1.0
}
}
}
}
for startEnd, weight := range cm {
g.addEdge(startEnd[0], startEnd[1], weight)
}
tags := g.rank()
if topK > 0 && len(tags) > topK {
tags = tags[:topK]
}
return tags
}
// Extract keywords from sentence using TextRank algorithm.
// topK specify how many top keywords to be returned at most.
func TextRank(sentence string, topK int) wordWeights {
return TextRankWithPOS(sentence, topK, defaultAllowPOS)
}
// Set the dictionary, could be absolute path of dictionary file, or dictionary
// name in current directory. This function must be called before cut any
// sentence.
func SetDictionary(dictFileName string) error {
return posseg.SetDictionary(dictFileName)
}

33
analyse/textrank_test.go Normal file
View File

@@ -0,0 +1,33 @@
package analyse
import (
"math"
"testing"
)
var (
sentence = "此外，公司拟对全资子公司吉林欧亚置业有限公司增资4.3亿元，增资后，吉林欧亚置业注册资本由7000万元增加到5亿元。吉林欧亚置业主要经营范围为房地产开发及百货零售等业务。目前在建吉林欧亚城市商业综合体项目。2013年，实现营业收入0万元，实现净利润-139.13万元。"
tagRanks = wordWeights{
wordWeight{Word: "吉林", Weight: 1.0},
wordWeight{Word: "欧亚", Weight: 0.87807810644},
wordWeight{Word: "置业", Weight: 0.562048250306},
wordWeight{Word: "实现", Weight: 0.520905743929},
wordWeight{Word: "收入", Weight: 0.384283870648},
wordWeight{Word: "增资", Weight: 0.360590945312},
wordWeight{Word: "子公司", Weight: 0.353131980904},
wordWeight{Word: "城市", Weight: 0.307509449283},
wordWeight{Word: "全资", Weight: 0.306324426665},
wordWeight{Word: "商业", Weight: 0.306138241063},
}
)
func TestTextRank(t *testing.T) {
SetDictionary("../dict.txt")
results := TextRank(sentence, 10)
for index, tw := range results {
if tw.Word != tagRanks[index].Word || math.Abs(tw.Weight-tagRanks[index].Weight) > 1e-6 {
t.Errorf("%v != %v", tw, tagRanks[index])
}
}
}

57
dict.go Normal file
View File

@@ -0,0 +1,57 @@
package jiebago
import (
"bufio"
"os"
"path/filepath"
"strconv"
"strings"
)
type WordTagFreq struct {
Word, Tag string
Freq float64
}
func DictPath(dictFileName string) (string, error) {
if filepath.IsAbs(dictFileName) {
return dictFileName, nil
}
var dictFilePath string
pwd, err := os.Getwd()
if err != nil {
return dictFilePath, err
}
dictFilePath = filepath.Clean(filepath.Join(pwd, dictFileName))
return dictFilePath, nil
}
func ParseDictFile(dictFilePath string) (wtfs []*WordTagFreq, err error) {
var dictFile *os.File
dictFile, err = os.Open(dictFilePath)
if err != nil {
return
}
defer dictFile.Close()
scanner := bufio.NewScanner(dictFile)
for scanner.Scan() {
line := scanner.Text()
fields := strings.Split(line, " ")
length := len(fields)
word := fields[0]
word = strings.Replace(word, "\ufeff", "", 1)
wtf := &WordTagFreq{Word: word}
if length > 1 {
wtf.Freq, err = strconv.ParseFloat(fields[1], 64)
if err != nil {
return
}
}
if length > 2 {
wtf.Tag = fields[2]
}
wtfs = append(wtfs, wtf)
}
err = scanner.Err()
return
}

139
finalseg/finalseg.go
View File

@@ -4,82 +4,91 @@ import (
"regexp"
)
func cutHan(sentence string) []string {
runes := []rune(sentence)
result := make([]string, 0)
_, pos_list := viterbi(runes, []byte{'B', 'M', 'E', 'S'})
begin, next := 0, 0
for i, char := range runes {
pos := pos_list[i]
switch pos {
case 'B':
begin = i
case 'E':
result = append(result, string(runes[begin:i+1]))
next = i + 1
case 'S':
result = append(result, string(char))
next = i + 1
var (
reHan = regexp.MustCompile(`\p{Han}+`)
reSkip = regexp.MustCompile(`(\d+\.\d+|[a-zA-Z0-9]+)`)
)
func cutHan(sentence string) chan string {
result := make(chan string)
go func() {
runes := []rune(sentence)
_, pos_list := viterbi(runes, []byte{'B', 'M', 'E', 'S'})
begin, next := 0, 0
for i, char := range runes {
pos := pos_list[i]
switch pos {
case 'B':
begin = i
case 'E':
result <- string(runes[begin : i+1])
next = i + 1
case 'S':
result <- string(char)
next = i + 1
}
}
}
if next < len(runes) {
result = append(result, string(runes[next:]))
}
if next < len(runes) {
result <- string(runes[next:])
}
close(result)
}()
return result
}
func Cut(sentence string) []string {
result := make([]string, 0)
re_han := regexp.MustCompile(`\p{Han}+`)
re_skip := regexp.MustCompile(`(\d+\.\d+|[a-zA-Z0-9]+)`)
func Cut(sentence string) chan string {
result := make(chan string)
s := sentence
var hans string
var hanLoc []int
var nonhanLoc []int
for {
hanLoc = re_han.FindStringIndex(s)
if hanLoc == nil {
if len(s) == 0 {
break
}
} else if hanLoc[0] == 0 {
hans = s[hanLoc[0]:hanLoc[1]]
s = s[hanLoc[1]:]
for _, han := range cutHan(hans) {
result = append(result, han)
}
continue
}
nonhanLoc = re_skip.FindStringIndex(s)
if nonhanLoc == nil {
if len(s) == 0 {
break
}
} else if nonhanLoc[0] == 0 {
nonhans := s[nonhanLoc[0]:nonhanLoc[1]]
s = s[nonhanLoc[1]:]
if nonhans != "" {
result = append(result, nonhans)
go func() {
for {
hanLoc = reHan.FindStringIndex(s)
if hanLoc == nil {
if len(s) == 0 {
break
}
} else if hanLoc[0] == 0 {
hans = s[hanLoc[0]:hanLoc[1]]
s = s[hanLoc[1]:]
for han := range cutHan(hans) {
result <- han
}
continue
}
}
var loc []int
if hanLoc == nil && nonhanLoc == nil {
if len(s) > 0 {
result = append(result, s)
break
nonhanLoc = reSkip.FindStringIndex(s)
if nonhanLoc == nil {
if len(s) == 0 {
break
}
} else if nonhanLoc[0] == 0 {
nonhans := s[nonhanLoc[0]:nonhanLoc[1]]
s = s[nonhanLoc[1]:]
if nonhans != "" {
result <- nonhans
continue
}
}
} else if hanLoc == nil {
loc = nonhanLoc
} else if nonhanLoc == nil {
loc = hanLoc
} else if hanLoc[0] < nonhanLoc[0] {
loc = hanLoc
} else {
loc = nonhanLoc
var loc []int
if hanLoc == nil && nonhanLoc == nil {
if len(s) > 0 {
result <- s
break
}
} else if hanLoc == nil {
loc = nonhanLoc
} else if nonhanLoc == nil {
loc = hanLoc
} else if hanLoc[0] < nonhanLoc[0] {
loc = hanLoc
} else {
loc = nonhanLoc
}
result <- s[:loc[0]]
s = s[loc[0]:]
}
result = append(result, s[:loc[0]])
s = s[loc[0]:]
}
close(result)
}()
return result
}

16
finalseg/finalseg_test.go
View File

@@ -5,6 +5,14 @@ import (
"testing"
)
func chanToArray(ch chan string) []string {
result := make([]string, 0)
for word := range ch {
result = append(result, word)
}
return result
}
func TestViterbi(t *testing.T) {
obs := "我们是程序员"
states := []byte{'B', 'M', 'E', 'S'}
@@ -21,7 +29,7 @@ func TestViterbi(t *testing.T) {
func TestCutHan(t *testing.T) {
obs := "我们是程序员"
result := cutHan(obs)
result := chanToArray(cutHan(obs))
if len(result) != 3 {
t.Error(result)
}
@@ -38,7 +46,7 @@ func TestCutHan(t *testing.T) {
func TestCut(t *testing.T) {
sentence := "我们是程序员"
result := Cut(sentence)
result := chanToArray(Cut(sentence))
if len(result) != 3 {
t.Error(len(result))
}
@@ -51,11 +59,11 @@ func TestCut(t *testing.T) {
if result[2] != "程序员" {
t.Error(result[2])
}
result2 := Cut("I'm a programmer!")
result2 := chanToArray(Cut("I'm a programmer!"))
if len(result2) != 8 {
t.Error(result2)
}
result3 := Cut("程序员average年龄28.6岁。")
result3 := chanToArray(Cut("程序员average年龄28.6岁。"))
if len(result3) != 6 {
t.Error(result3)
}

10
finalseg/prob_emit.go
View File

@@ -1,9 +1,9 @@
package finalseg
var ProbEmit = make(map[byte]map[rune]float64)
var probEmit = make(map[byte]map[rune]float64)
func init() {
ProbEmit['B'] = map[rune]float64{'\u4e00': -3.6544978750449433,
probEmit['B'] = map[rune]float64{'\u4e00': -3.6544978750449433,
'\u4e01': -8.125041941842026,
'\u4e03': -7.817392401429855,
'\u4e07': -6.3096425804013165,
@@ -6860,7 +6860,7 @@ func init() {
'\u9f9a': -9.557108305917183,
'\u9f9c': -10.895131537474946,
'\u9f9f': -10.895131537474946}
ProbEmit['E'] = map[rune]float64{'\u4e00': -6.044987536255073,
probEmit['E'] = map[rune]float64{'\u4e00': -6.044987536255073,
'\u4e01': -9.075800412310807,
'\u4e03': -9.198842005220659,
'\u4e07': -7.655326112989935,
@@ -14299,7 +14299,7 @@ func init() {
'\u9f9b': -12.729311722586953,
'\u9f9c': -10.574067217491615,
'\u9f9f': -10.574067217491615}
ProbEmit['M'] = map[rune]float64{'\u4e00': -4.428158526435913,
probEmit['M'] = map[rune]float64{'\u4e00': -4.428158526435913,
'\u4e01': -7.932945687598502,
'\u4e03': -6.559715525951586,
'\u4e07': -6.139922374120667,
@@ -20708,7 +20708,7 @@ func init() {
'\u9f9a': -14.14915250738439,
'\u9f9c': -11.058110054026073,
'\u9f9f': -11.058110054026073}
ProbEmit['S'] = map[rune]float64{'\u2236': -15.828865681131282,
probEmit['S'] = map[rune]float64{'\u2236': -15.828865681131282,
'\u4e00': -4.92368982120877,
'\u4e01': -9.024528361347633,
'\u4e02': -16.522012861691227,

10
finalseg/prob_start.go
View File

@@ -1,10 +0,0 @@
package finalseg
var ProbStart = make(map[byte]float64)
func init() {
ProbStart['B'] = -0.26268660809250016
ProbStart['E'] = -3.14e+100
ProbStart['M'] = -3.14e+100
ProbStart['S'] = -1.4652633398537678
}

10
finalseg/prob_trans.go
View File

@@ -1,14 +1,14 @@
package finalseg
var ProbTrans = make(map[byte]map[byte]float64)
var probTrans = make(map[byte]map[byte]float64)
func init() {
ProbTrans['B'] = map[byte]float64{'E': -0.510825623765990,
probTrans['B'] = map[byte]float64{'E': -0.510825623765990,
'M': -0.916290731874155}
ProbTrans['E'] = map[byte]float64{'B': -0.5897149736854513,
probTrans['E'] = map[byte]float64{'B': -0.5897149736854513,
'S': -0.8085250474669937}
ProbTrans['M'] = map[byte]float64{'E': -0.33344856811948514,
probTrans['M'] = map[byte]float64{'E': -0.33344856811948514,
'M': -1.2603623820268226}
ProbTrans['S'] = map[byte]float64{'B': -0.7211965654669841,
probTrans['S'] = map[byte]float64{'B': -0.7211965654669841,
'S': -0.6658631448798212}
}

79
finalseg/viterbi.go
View File

@@ -5,41 +5,48 @@ import (
"sort"
)
const MIN_FLOAT = -3.14e100
const minFloat = -3.14e100
var PrevStatus = make(map[byte][]byte)
var (
prevStatus = make(map[byte][]byte)
probStart = make(map[byte]float64)
)
func init() {
PrevStatus['B'] = []byte{'E', 'S'}
PrevStatus['M'] = []byte{'M', 'B'}
PrevStatus['S'] = []byte{'S', 'E'}
PrevStatus['E'] = []byte{'B', 'M'}
prevStatus['B'] = []byte{'E', 'S'}
prevStatus['M'] = []byte{'M', 'B'}
prevStatus['S'] = []byte{'S', 'E'}
prevStatus['E'] = []byte{'B', 'M'}
probStart['B'] = -0.26268660809250016
probStart['E'] = -3.14e+100
probStart['M'] = -3.14e+100
probStart['S'] = -1.4652633398537678
}
type Viterbi struct {
type probState struct {
prob float64
state byte
}
func (v Viterbi) String() string {
return fmt.Sprintf("(%f, %s)", v.prob, v.state)
func (p probState) String() string {
return fmt.Sprintf("(%f, %x)", p.prob, p.state)
}
type Viterbis []*Viterbi
type probStates []*probState
func (vs Viterbis) Len() int {
return len(vs)
func (ps probStates) Len() int {
return len(ps)
}
func (vs Viterbis) Less(i, j int) bool {
if vs[i].prob == vs[j].prob {
return vs[i].state < vs[j].state
func (ps probStates) Less(i, j int) bool {
if ps[i].prob == ps[j].prob {
return ps[i].state < ps[j].state
}
return vs[i].prob < vs[j].prob
return ps[i].prob < ps[j].prob
}
func (vs Viterbis) Swap(i, j int) {
vs[i], vs[j] = vs[j], vs[i]
func (ps probStates) Swap(i, j int) {
ps[i], ps[j] = ps[j], ps[i]
}
func viterbi(obs []rune, states []byte) (float64, []byte) {
@@ -47,10 +54,10 @@ func viterbi(obs []rune, states []byte) (float64, []byte) {
V := make([]map[byte]float64, len(obs))
V[0] = make(map[byte]float64)
for _, y := range states {
if val, ok := ProbEmit[y][obs[0]]; ok {
V[0][y] = val + ProbStart[y]
if val, ok := probEmit[y][obs[0]]; ok {
V[0][y] = val + probStart[y]
} else {
V[0][y] = MIN_FLOAT + ProbStart[y]
V[0][y] = minFloat + probStart[y]
}
path[y] = []byte{y}
}
@@ -59,36 +66,36 @@ func viterbi(obs []rune, states []byte) (float64, []byte) {
newPath := make(map[byte][]byte)
V[t] = make(map[byte]float64)
for _, y := range states {
vs0 := make(Viterbis, 0)
ps0 := make(probStates, 0)
var em_p float64
if val, ok := ProbEmit[y][obs[t]]; ok {
if val, ok := probEmit[y][obs[t]]; ok {
em_p = val
} else {
em_p = MIN_FLOAT
em_p = minFloat
}
for _, y0 := range PrevStatus[y] {
for _, y0 := range prevStatus[y] {
var transP float64
if tp, ok := ProbTrans[y0][y]; ok {
if tp, ok := probTrans[y0][y]; ok {
transP = tp
} else {
transP = MIN_FLOAT
transP = minFloat
}
prob0 := V[t-1][y0] + transP + em_p
vs0 = append(vs0, &Viterbi{prob: prob0, state: y0})
ps0 = append(ps0, &probState{prob: prob0, state: y0})
}
sort.Sort(sort.Reverse(vs0))
V[t][y] = vs0[0].prob
pp := make([]byte, len(path[vs0[0].state]))
copy(pp, path[vs0[0].state])
sort.Sort(sort.Reverse(ps0))
V[t][y] = ps0[0].prob
pp := make([]byte, len(path[ps0[0].state]))
copy(pp, path[ps0[0].state])
newPath[y] = append(pp, y)
}
path = newPath
}
vs := make(Viterbis, 0)
ps := make(probStates, 0)
for _, y := range []byte{'E', 'S'} {
vs = append(vs, &Viterbi{V[len(obs)-1][y], y})
ps = append(ps, &probState{V[len(obs)-1][y], y})
}
sort.Sort(sort.Reverse(vs))
v := vs[0]
sort.Sort(sort.Reverse(ps))
v := ps[0]
return v.prob, path[v.state]
}

533
jieba.go
View File

@@ -1,48 +1,54 @@
// Golang implemention of jieba (Python Chinese word segmentation module).
package jiebago
import (
"fmt"
"github.com/wangbin/jiebago/finalseg"
"math"
"regexp"
"sort"
)
var (
Dictionary = "dict.txt"
TT *TopTrie
// Word/Tag Map load from user dictionary
UserWordTagTab = make(map[string]string)
reEng = regexp.MustCompile(`[[:alnum:]]`)
reHanCutAll = regexp.MustCompile(`\p{Han}+`)
reSkipCutAll = regexp.MustCompile(`[^[:alnum:]+#\n]`)
reHanDefault = regexp.MustCompile(`([\p{Han}+[:alnum:]+#&\._]+)`)
reSkipDefault = regexp.MustCompile(`(\r\n|\s)`)
)
type Route struct {
type route struct {
Freq float64
Index int
}
func (route Route) String() string {
return fmt.Sprintf("(%f, %d)", route.Freq, route.Index)
func (r route) String() string {
return fmt.Sprintf("(%f, %d)", r.Freq, r.Index)
}
type Routes []*Route
type routes []*route
func (routes Routes) Len() int {
return len(routes)
func (rs routes) Len() int {
return len(rs)
}
func (routes Routes) Less(i, j int) bool {
routei := routes[i]
routej := routes[j]
if routei.Freq < routej.Freq {
func (rs routes) Less(i, j int) bool {
if rs[i].Freq < rs[j].Freq {
return true
} else if routei.Freq == routej.Freq {
return routei.Index < routej.Index
}
if rs[i].Freq == rs[j].Freq {
return rs[i].Index < rs[j].Index
}
return false
}
func (routes Routes) Swap(i, j int) {
routes[i], routes[j] = routes[j], routes[i]
func (rs routes) Swap(i, j int) {
rs[i], rs[j] = rs[j], rs[i]
}
// Split sentence using regular expression.
func RegexpSplit(r *regexp.Regexp, sentence string) []string {
result := make([]string, 0)
locs := r.FindAllStringIndex(sentence, -1)
@@ -66,280 +72,291 @@ func RegexpSplit(r *regexp.Regexp, sentence string) []string {
return result
}
func GetDAG(sentence string) map[int][]int {
// Build a directed acyclic graph (DAG) for sentence.
func DAG(sentence string) map[int][]int {
dag := make(map[int][]int)
runes := []rune(sentence)
n := len(runes)
p := TT.T
i, j := 0, 0
var c rune
for {
if i >= n {
break
}
c = runes[j]
if _, ok := p.Nodes[c]; ok {
p = p.Nodes[c]
if p.IsLeaf {
if _, inDag := dag[i]; !inDag {
dag[i] = []int{j}
} else {
dag[i] = append(dag[i], j)
i := 0
var frag string
for k := 0; k < n; k++ {
tmpList := make([]int, 0)
i = k
frag = string(runes[k])
for {
if freq, ok := Trie.Freq[frag]; !ok {
break
} else {
if freq > 0.0 {
tmpList = append(tmpList, i)
}
}
j += 1
if j >= n {
i += 1
j = i
p = TT.T
}
} else {
p = TT.T
i += 1
j = i
if i >= n {
break
}
frag = string(runes[k : i+1])
}
}
for i := 0; i < n; i++ {
if _, ok := dag[i]; !ok {
dag[i] = []int{i}
if len(tmpList) == 0 {
tmpList = append(tmpList, k)
}
dag[k] = tmpList
}
return dag
}
func Calc(sentence string, dag map[int][]int, idx int) map[int]*Route {
func Calc(sentence string, dag map[int][]int) map[int]*route {
runes := []rune(sentence)
number := len(runes)
routes := make(map[int]*Route)
routes[number] = &Route{0.0, 0}
rs := make(map[int]*route)
rs[number] = &route{Freq: 0.0, Index: 0}
logTotal := math.Log(Trie.Total)
for idx := number - 1; idx >= 0; idx-- {
candidates := make(Routes, 0)
candidates := make(routes, 0)
for _, i := range dag[idx] {
var word string
if i <= idx-1 {
word = string(runes[i+1 : idx])
word := string(runes[idx : i+1])
var r *route
if _, ok := Trie.Freq[word]; ok {
r = &route{Freq: math.Log(Trie.Freq[word]) - logTotal + rs[i+1].Freq, Index: i}
} else {
word = string(runes[idx : i+1])
r = &route{Freq: math.Log(1.0) - logTotal + rs[i+1].Freq, Index: i}
}
var route *Route
if _, ok := TT.Freq[word]; ok {
route = &Route{TT.Freq[word] + routes[i+1].Freq, i}
} else {
route = &Route{TT.MinFreq + routes[i+1].Freq, i}
}
candidates = append(candidates, route)
candidates = append(candidates, r)
}
sort.Sort(sort.Reverse(candidates))
routes[idx] = candidates[0]
rs[idx] = candidates[0]
}
return routes
return rs
}
type cutAction func(sentence string) []string
type cutFunc func(sentence string) chan string
func cut_DAG(sentence string) []string {
dag := GetDAG(sentence)
routes := Calc(sentence, dag, 0)
x := 0
var y int
runes := []rune(sentence)
length := len(runes)
result := make([]string, 0)
buf := make([]rune, 0)
for {
if x >= length {
break
func cutDAG(sentence string) chan string {
result := make(chan string)
go func() {
dag := DAG(sentence)
routes := Calc(sentence, dag)
x := 0
var y int
runes := []rune(sentence)
length := len(runes)
buf := make([]rune, 0)
for {
if x >= length {
break
}
y = routes[x].Index + 1
l_word := runes[x:y]
if y-x == 1 {
buf = append(buf, l_word...)
} else {
if len(buf) > 0 {
if len(buf) == 1 {
result <- string(buf)
buf = make([]rune, 0)
} else {
bufString := string(buf)
if v, ok := Trie.Freq[bufString]; !ok || v == 0.0 {
for x := range finalseg.Cut(bufString) {
result <- x
}
} else {
for _, elem := range buf {
result <- string(elem) // TODO: I don't get this?
}
}
buf = make([]rune, 0)
}
}
result <- string(l_word)
}
x = y
}
y = routes[x].Index + 1
l_word := runes[x:y]
if y-x == 1 {
buf = append(buf, l_word...)
} else {
if len(buf) > 0 {
if len(buf) == 1 {
result = append(result, string(buf))
buf = make([]rune, 0)
if len(buf) > 0 {
if len(buf) == 1 {
result <- string(buf)
} else {
bufString := string(buf)
if v, ok := Trie.Freq[bufString]; !ok || v == 0.0 {
for t := range finalseg.Cut(bufString) {
result <- t
}
} else {
bufString := string(buf)
if _, ok := TT.Freq[bufString]; !ok {
recognized := finalseg.Cut(bufString)
for _, t := range recognized {
result = append(result, t)
for _, elem := range buf {
result <- string(elem) // TODO: I don't get this?
}
}
}
}
close(result)
}()
return result
}
func cutDAGNoHMM(sentence string) chan string {
result := make(chan string)
go func() {
dag := DAG(sentence)
routes := Calc(sentence, dag)
x := 0
var y int
runes := []rune(sentence)
length := len(runes)
buf := make([]rune, 0)
for {
if x >= length {
break
}
y = routes[x].Index + 1
l_word := runes[x:y]
if reEng.MatchString(string(l_word)) && len(l_word) == 1 {
buf = append(buf, l_word...)
x = y
} else {
if len(buf) > 0 {
result <- string(buf)
buf = make([]rune, 0)
}
result <- string(l_word)
x = y
}
}
if len(buf) > 0 {
result <- string(buf)
buf = make([]rune, 0)
}
close(result)
}()
return result
}
func cutAll(sentence string) chan string {
result := make(chan string)
go func() {
runes := []rune(sentence)
dag := DAG(sentence)
old_j := -1
ks := make([]int, 0)
for k := range dag {
ks = append(ks, k)
}
sort.Ints(ks)
for k := range ks {
l := dag[k]
if len(l) == 1 && k > old_j {
result <- string(runes[k : l[0]+1])
old_j = l[0]
} else {
for _, j := range l {
if j > k {
result <- string(runes[k : j+1])
old_j = j
}
}
}
}
close(result)
}()
return result
}
/*
Cut sentence.
isCutAll controls use full cut mode or accurate mode.
Full Mode gets all the possible words from the sentence. Fast but not accurate.
Accurate Mode attempts to cut the sentence into the most accurate segmentations,
which is suitable for text analysis.
HMM contols whether to use the Hidden Markov Mode.
*/
func Cut(sentence string, isCutAll bool, HMM bool) chan string {
result := make(chan string)
go func() {
var reHan, reSkip *regexp.Regexp
if isCutAll {
reHan = reHanCutAll
reSkip = reSkipCutAll
} else {
reHan = reHanDefault
reSkip = reSkipDefault
}
blocks := RegexpSplit(reHan, sentence)
var cut cutFunc
if HMM {
cut = cutDAG
} else {
cut = cutDAGNoHMM
}
if isCutAll {
cut = cutAll
}
for _, blk := range blocks {
if len(blk) == 0 {
continue
}
if reHan.MatchString(blk) {
for x := range cut(blk) {
result <- x
}
} else {
type skipSplitFunc func(sentence string) []string
var ssf skipSplitFunc
if isCutAll {
ssf = func(sentence string) []string {
return reSkip.Split(sentence, -1)
}
} else {
ssf = func(sentence string) []string {
return RegexpSplit(reSkip, sentence)
}
}
for _, x := range ssf(blk) {
if reSkip.MatchString(x) {
result <- x
} else if !isCutAll {
for _, xx := range x {
result <- string(xx)
}
} else {
for _, elem := range buf {
result = append(result, string(elem)) // TODO: I don't get this?
result <- x
}
}
}
}
close(result)
}()
return result
}
// Cut sentence using Search Engine Mode, based on the Accurate Mode, attempts
// to cut long words into several short words, which can raise the recall rate.
// Suitable for search engines.
func CutForSearch(sentence string, hmm bool) chan string {
result := make(chan string)
go func() {
for word := range Cut(sentence, false, hmm) {
runes := []rune(word)
for _, increment := range []int{2, 3} {
if len(runes) > increment {
var gram2 string
for i := 0; i < len(runes)-increment+1; i++ {
gram2 = string(runes[i : i+increment])
if v, ok := Trie.Freq[gram2]; ok && v > 0.0 {
result <- gram2
}
}
buf = make([]rune, 0)
}
}
result = append(result, string(l_word))
result <- word
}
x = y
}
if len(buf) > 0 {
if len(buf) == 1 {
result = append(result, string(buf))
} else {
bufString := string(buf)
if _, ok := TT.Freq[bufString]; !ok {
recognized := finalseg.Cut(bufString)
for _, t := range recognized {
result = append(result, t)
}
} else {
for _, elem := range buf {
result = append(result, string(elem)) // TODO: I don't get this?
}
}
}
}
close(result)
}()
return result
}
func cut_DAG_NO_HMM(sentence string) []string {
result := make([]string, 0)
re_eng := regexp.MustCompile(`[[:alnum:]]`)
dag := GetDAG(sentence)
routes := Calc(sentence, dag, 0)
x := 0
var y int
runes := []rune(sentence)
length := len(runes)
buf := make([]rune, 0)
for {
if x >= length {
break
}
y = routes[x].Index + 1
l_word := runes[x:y]
if re_eng.MatchString(string(l_word)) && len(l_word) == 1 {
buf = append(buf, l_word...)
x = y
} else {
if len(buf) > 0 {
result = append(result, string(buf))
buf = make([]rune, 0)
}
result = append(result, string(l_word))
x = y
}
}
if len(buf) > 0 {
result = append(result, string(buf))
buf = make([]rune, 0)
}
return result
}
func cut_All(sentence string) []string {
result := make([]string, 0)
runes := []rune(sentence)
dag := GetDAG(sentence)
old_j := -1
ks := make([]int, 0)
for k := range dag {
ks = append(ks, k)
}
sort.Ints(ks)
for k := range ks {
l := dag[k]
if len(l) == 1 && k > old_j {
result = append(result, string(runes[k:l[0]+1]))
old_j = l[0]
} else {
for _, j := range l {
if j > k {
result = append(result, string(runes[k:j+1]))
old_j = j
}
}
}
}
return result
}
func Cut(sentence string, cut_all bool, HMM bool) []string {
result := make([]string, 0)
var re_han, re_skip *regexp.Regexp
if cut_all {
re_han = regexp.MustCompile(`\p{Han}+`)
re_skip = regexp.MustCompile(`[^[:alnum:]+#\n]`)
} else {
re_han = regexp.MustCompile(`([\p{Han}+[:alnum:]+#&\._]+)`)
re_skip = regexp.MustCompile(`(\r\n|\s)`)
}
blocks := RegexpSplit(re_han, sentence)
var cut_block cutAction
if HMM {
cut_block = cut_DAG
} else {
cut_block = cut_DAG_NO_HMM
}
if cut_all {
cut_block = cut_All
}
for _, blk := range blocks {
if len(blk) == 0 {
continue
}
if re_han.MatchString(blk) {
for _, word := range cut_block(blk) {
result = append(result, word)
}
} else {
type skipSplitFunc func(sentence string) []string
var ssf skipSplitFunc
if cut_all {
ssf = func(sentence string) []string {
return re_skip.Split(sentence, -1)
}
} else {
ssf = func(sentence string) []string {
return RegexpSplit(re_skip, sentence)
}
}
for _, x := range ssf(blk) {
if re_skip.MatchString(x) {
result = append(result, x)
} else if !cut_all {
for _, xx := range x {
result = append(result, string(xx))
}
} else {
result = append(result, x)
}
}
}
}
return result
}
func CutForSearch(sentence string, hmm bool) []string {
result := make([]string, 0)
words := Cut(sentence, false, hmm)
for _, word := range words {
runes := []rune(word)
for _, increment := range []int{2, 3} {
if len(runes) > increment {
var gram2 string
for i := 0; i < len(runes)-increment+1; i++ {
gram2 = string(runes[i : i+increment])
if _, ok := TT.Freq[gram2]; ok {
result = append(result, gram2)
}
}
}
}
result = append(result, word)
}
return result
}
func SetDictionary(dict_path string) (err error) {
TT, err = newTopTrie(dict_path)
return
}

1023
jieba_test.go
View File

File diff suppressed because it is too large Load Diff

158922
posseg/char_state_tab.go
View File

File diff suppressed because it is too large Load Diff

4
posseg/char_state_tab_test.go
View File

@@ -5,11 +5,11 @@ import (
)
func TestGet(t *testing.T) {
result := CharStateTab.Get('\u8000')
result := charStateTab.get('\u8000')
if len(result) != 17 {
t.FailNow()
}
result = CharStateTab.Get('\uaaaa')
result = charStateTab.get('\uaaaa')
if len(result) == 17 {
t.FailNow()
}

482
posseg/posseg.go
View File

@@ -1,292 +1,282 @@
package posseg
import (
"bufio"
"fmt"
"github.com/wangbin/jiebago"
"os"
"path/filepath"
"regexp"
"runtime"
"strings"
)
var (
WordTagTab = make(map[string]string)
wordTagMap = make(map[string]string)
reHanDetail = regexp.MustCompile(`\p{Han}+`)
reSkipDetail = regexp.MustCompile(`[[\.[:digit:]]+|[:alnum:]]+`)
reEng = regexp.MustCompile(`[[:alnum:]]`)
reNum = regexp.MustCompile(`[\.[:digit:]]+`)
reEng1 = regexp.MustCompile(`[[:alnum:]]$`)
reHanInternal = regexp.MustCompile(`([\p{Han}+[:alnum:]+#&\._]+)`)
reSkipInternal = regexp.MustCompile(`(\r\n|\s)`)
)
type WordTag struct {
Word, Tag string
}
func (wt WordTag) String() string {
return fmt.Sprintf("%s/%s", wt.Word, wt.Tag)
}
func init() {
_, filename, _, _ := runtime.Caller(1)
dict_dir := filepath.Dir(filepath.Dir(filename))
dict_path := filepath.Join(dict_dir, jiebago.Dictionary)
err := load_model(dict_path)
// Set dictionary, it could be absolute path of dictionary file, or dictionary
// name in current diectory.
func SetDictionary(dictFileName string) error {
err := jiebago.SetDictionary(dictFileName)
if err != nil {
panic(err)
}
}
func load_model(f_name string) error {
file, openError := os.Open(f_name)
if openError != nil {
return openError
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
words := strings.Split(strings.TrimSpace(line), " ")
word, tag := words[0], words[2]
WordTagTab[word] = tag
}
if err := scanner.Err(); err != nil {
return err
}
dictFilePath, err := jiebago.DictPath(dictFileName)
if err != nil {
return err
}
wtfs, err := jiebago.ParseDictFile(dictFilePath)
for _, wtf := range wtfs {
wordTagMap[wtf.Word] = wtf.Tag
}
return nil
}
func __cut(sentence string) []WordTag {
result := make([]WordTag, 0)
runes := []rune(sentence)
_, posList := Viterbi(runes)
begin := 0
next := 0
for i, char := range runes {
pos := posList[i].State
switch pos {
case 'B':
begin = i
case 'E':
result = append(result, WordTag{string(runes[begin : i+1]), posList[i].Tag})
next = i + 1
case 'S':
result = append(result, WordTag{string(char), posList[i].Tag})
next = i + 1
func cutDetailInternal(sentence string) chan WordTag {
result := make(chan WordTag)
go func() {
runes := []rune(sentence)
_, posList := viterbi(runes)
begin := 0
next := 0
for i, char := range runes {
pos := posList[i].State
switch pos {
case 'B':
begin = i
case 'E':
result <- WordTag{string(runes[begin : i+1]), posList[i].Tag}
next = i + 1
case 'S':
result <- WordTag{string(char), posList[i].Tag}
next = i + 1
}
}
}
if next < len(runes) {
result = append(result, WordTag{string(runes[next:]), posList[next].Tag})
}
if next < len(runes) {
result <- WordTag{string(runes[next:]), posList[next].Tag}
}
close(result)
}()
return result
}
func cutDetail(sentence string) []WordTag {
result := make([]WordTag, 0)
re_han := regexp.MustCompile(`\p{Han}+`)
re_skip := regexp.MustCompile(`[[\.[:digit:]]+|[:alnum:]]+`)
func cutDetail(sentence string) chan WordTag {
result := make(chan WordTag)
re_eng := regexp.MustCompile(`[[:alnum:]]`)
re_num := regexp.MustCompile(`[\.[:digit:]]+`)
blocks := jiebago.RegexpSplit(re_han, sentence)
for _, blk := range blocks {
if re_han.MatchString(blk) {
for _, wordTag := range __cut(blk) {
result = append(result, wordTag)
}
} else {
for _, x := range jiebago.RegexpSplit(re_skip, blk) {
if len(x) == 0 {
continue
go func() {
blocks := jiebago.RegexpSplit(reHanDetail, sentence)
for _, blk := range blocks {
if reHanDetail.MatchString(blk) {
for wordTag := range cutDetailInternal(blk) {
result <- wordTag
}
switch {
case re_num.MatchString(x):
result = append(result, WordTag{x, "m"})
case re_eng.MatchString(x):
result = append(result, WordTag{x, "eng"})
default:
result = append(result, WordTag{x, "x"})
}
}
}
}
return result
}
type cutAction func(sentence string) []WordTag
func cut_DAG(sentence string) []WordTag {
dag := jiebago.GetDAG(sentence)
routes := jiebago.Calc(sentence, dag, 0)
x := 0
var y int
runes := []rune(sentence)
length := len(runes)
result := make([]WordTag, 0)
buf := make([]rune, 0)
for {
if x >= length {
break
}
y = routes[x].Index + 1
l_word := runes[x:y]
if y-x == 1 {
buf = append(buf, l_word...)
} else {
if len(buf) > 0 {
if len(buf) == 1 {
sbuf := string(buf)
if tag, ok := WordTagTab[sbuf]; ok {
result = append(result, WordTag{sbuf, tag})
} else {
result = append(result, WordTag{sbuf, "x"})
} else {
for _, x := range jiebago.RegexpSplit(reSkipDetail, blk) {
if len(x) == 0 {
continue
}
buf = make([]rune, 0)
} else {
bufString := string(buf)
if _, ok := jiebago.TT.Freq[bufString]; !ok {
recognized := cutDetail(bufString)
for _, t := range recognized {
result = append(result, t)
switch {
case reNum.MatchString(x):
result <- WordTag{x, "m"}
case reEng.MatchString(x):
result <- WordTag{x, "eng"}
default:
result <- WordTag{x, "x"}
}
}
}
}
close(result)
}()
return result
}
type cutFunc func(sentence string) chan WordTag
func cutDAG(sentence string) chan WordTag {
result := make(chan WordTag)
go func() {
dag := jiebago.DAG(sentence)
routes := jiebago.Calc(sentence, dag)
x := 0
var y int
runes := []rune(sentence)
length := len(runes)
buf := make([]rune, 0)
for {
if x >= length {
break
}
y = routes[x].Index + 1
l_word := runes[x:y]
if y-x == 1 {
buf = append(buf, l_word...)
} else {
if len(buf) > 0 {
if len(buf) == 1 {
sbuf := string(buf)
if tag, ok := wordTagMap[sbuf]; ok {
result <- WordTag{sbuf, tag}
} else {
result <- WordTag{sbuf, "x"}
}
buf = make([]rune, 0)
} else {
for _, elem := range buf {
selem := string(elem)
if tag, ok := WordTagTab[selem]; ok {
result = append(result, WordTag{string(elem), tag})
} else {
result = append(result, WordTag{string(elem), "x"})
bufString := string(buf)
if v, ok := jiebago.Trie.Freq[bufString]; !ok || v == 0.0 {
for t := range cutDetail(bufString) {
result <- t
}
} else {
for _, elem := range buf {
selem := string(elem)
if tag, ok := wordTagMap[selem]; ok {
result <- WordTag{string(elem), tag}
} else {
result <- WordTag{string(elem), "x"}
}
}
}
buf = make([]rune, 0)
}
}
sl_word := string(l_word)
if tag, ok := wordTagMap[sl_word]; ok {
result <- WordTag{sl_word, tag}
} else {
result <- WordTag{sl_word, "x"}
}
}
x = y
}
if len(buf) > 0 {
if len(buf) == 1 {
sbuf := string(buf)
if tag, ok := wordTagMap[sbuf]; ok {
result <- WordTag{sbuf, tag}
} else {
result <- WordTag{sbuf, "x"}
}
} else {
bufString := string(buf)
if v, ok := jiebago.Trie.Freq[bufString]; !ok || v == 0.0 {
for t := range cutDetail(bufString) {
result <- t
}
} else {
for _, elem := range buf {
selem := string(elem)
if tag, ok := wordTagMap[selem]; ok {
result <- WordTag{selem, tag}
} else {
result <- WordTag{selem, "x"}
}
}
}
}
}
close(result)
}()
return result
}
func cutDAGNoHMM(sentence string) chan WordTag {
result := make(chan WordTag)
go func() {
dag := jiebago.DAG(sentence)
routes := jiebago.Calc(sentence, dag)
x := 0
var y int
runes := []rune(sentence)
length := len(runes)
buf := make([]rune, 0)
for {
if x >= length {
break
}
y = routes[x].Index + 1
l_word := runes[x:y]
if reEng1.MatchString(string(l_word)) && len(l_word) == 1 {
buf = append(buf, l_word...)
x = y
} else {
if len(buf) > 0 {
result <- WordTag{string(buf), "eng"}
buf = make([]rune, 0)
}
}
sl_word := string(l_word)
if tag, ok := WordTagTab[sl_word]; ok {
result = append(result, WordTag{sl_word, tag})
} else {
result = append(result, WordTag{sl_word, "x"})
}
}
x = y
}
if len(buf) > 0 {
if len(buf) == 1 {
sbuf := string(buf)
if tag, ok := WordTagTab[sbuf]; ok {
result = append(result, WordTag{sbuf, tag})
} else {
result = append(result, WordTag{sbuf, "x"})
}
} else {
bufString := string(buf)
if _, ok := jiebago.TT.Freq[bufString]; !ok {
recognized := cutDetail(bufString)
for _, t := range recognized {
result = append(result, t)
}
} else {
for _, elem := range buf {
selem := string(elem)
if tag, ok := WordTagTab[selem]; ok {
result = append(result, WordTag{selem, tag})
} else {
result = append(result, WordTag{selem, "x"})
}
}
}
}
}
return result
}
func cut_DAG_NO_HMM(sentence string) []WordTag {
result := make([]WordTag, 0)
re_eng := regexp.MustCompile(`[[:alnum:]]`)
dag := jiebago.GetDAG(sentence)
routes := jiebago.Calc(sentence, dag, 0)
x := 0
var y int
runes := []rune(sentence)
length := len(runes)
buf := make([]rune, 0)
for {
if x >= length {
break
}
y = routes[x].Index + 1
l_word := runes[x:y]
if re_eng.MatchString(string(l_word)) && len(l_word) == 1 {
buf = append(buf, l_word...)
x = y
} else {
if len(buf) > 0 {
result = append(result, WordTag{string(buf), "eng"})
buf = make([]rune, 0)
}
sl_word := string(l_word)
if tag, ok := WordTagTab[sl_word]; ok {
result = append(result, WordTag{sl_word, tag})
} else {
result = append(result, WordTag{sl_word, "x"})
}
x = y
}
}
if len(buf) > 0 {
result = append(result, WordTag{string(buf), "eng"})
buf = make([]rune, 0)
}
return result
}
func cut(sentence string, HMM bool) []WordTag {
result := make([]WordTag, 0)
re_han := regexp.MustCompile(`([\p{Han}+[:alnum:]+#&\._]+)`)
re_skip := regexp.MustCompile(`(\r\n|\s)`)
re_eng := regexp.MustCompile(`[[:alnum:]]`)
re_num := regexp.MustCompile(`[\.[:digit:]]+`)
blocks := jiebago.RegexpSplit(re_han, sentence)
var cut_block cutAction
if HMM {
cut_block = cut_DAG
} else {
cut_block = cut_DAG_NO_HMM
}
for _, blk := range blocks {
if re_han.MatchString(blk) {
for _, wordTag := range cut_block(blk) {
result = append(result, wordTag)
}
} else {
for _, x := range jiebago.RegexpSplit(re_skip, blk) {
if re_skip.MatchString(x) {
result = append(result, WordTag{x, "x"})
sl_word := string(l_word)
if tag, ok := wordTagMap[sl_word]; ok {
result <- WordTag{sl_word, tag}
} else {
for _, xx := range x {
s := string(xx)
switch {
case re_num.MatchString(s):
result = append(result, WordTag{s, "m"})
case re_eng.MatchString(x):
result = append(result, WordTag{x, "eng"})
break
default:
result = append(result, WordTag{s, "x"})
}
}
result <- WordTag{sl_word, "x"}
}
x = y
}
}
}
if len(buf) > 0 {
result <- WordTag{string(buf), "eng"}
buf = make([]rune, 0)
}
close(result)
}()
return result
}
func Cut(sentence string, HMM bool) []WordTag {
// Tags the POS of each word after segmentation, using labels compatible with
// ictclas.
func Cut(sentence string, HMM bool) chan WordTag {
for key := range jiebago.UserWordTagTab {
WordTagTab[key] = jiebago.UserWordTagTab[key]
wordTagMap[key] = jiebago.UserWordTagTab[key]
delete(jiebago.UserWordTagTab, key)
}
return cut(sentence, HMM)
result := make(chan WordTag)
blocks := jiebago.RegexpSplit(reHanInternal, sentence)
var cut cutFunc
if HMM {
cut = cutDAG
} else {
cut = cutDAGNoHMM
}
go func() {
for _, blk := range blocks {
if reHanInternal.MatchString(blk) {
for wordTag := range cut(blk) {
result <- wordTag
}
} else {
for _, x := range jiebago.RegexpSplit(reSkipInternal, blk) {
if reSkipInternal.MatchString(x) {
result <- WordTag{x, "x"}
} else {
for _, xx := range x {
s := string(xx)
switch {
case reNum.MatchString(s):
result <- WordTag{s, "m"}
case reEng.MatchString(x):
result <- WordTag{x, "eng"}
break
default:
result <- WordTag{s, "x"}
}
}
}
}
}
}
close(result)
}()
return result
}

460
posseg/posseg_test.go
View File

@@ -92,186 +92,194 @@ var (
"C++和c#是什么关系？11+122=133，是吗？PI=3.14159",
"你认识那个和主席握手的的哥吗？他开一辆黑色的士。",
"枪杆子中出政权"}
defaultCutResult = [][]WordTag{
[]WordTag{WordTag{"\u8fd9", "r"}, WordTag{"\u662f", "v"}, WordTag{"\u4e00\u4e2a", "m"}, WordTag{"\u4f38\u624b\u4e0d\u89c1\u4e94\u6307", "i"}, WordTag{"\u7684", "uj"}, WordTag{"\u9ed1\u591c", "n"}, WordTag{"\u3002", "x"}, WordTag{"\u6211", "r"}, WordTag{"\u53eb", "v"}, WordTag{"\u5b59\u609f\u7a7a", "nr"}, WordTag{"\uff0c", "x"}, WordTag{"\u6211", "r"}, WordTag{"\u7231", "v"}, WordTag{"\u5317\u4eac", "ns"}, WordTag{"\uff0c", "x"}, WordTag{"\u6211", "r"}, WordTag{"\u7231", "v"}, WordTag{"Python", "eng"}, WordTag{"\u548c", "c"}, WordTag{"C++", "nz"}, WordTag{"\u3002", "x"}},
[]WordTag{WordTag{"\u6211", "r"}, WordTag{"\u4e0d", "d"}, WordTag{"\u559c\u6b22", "v"}, WordTag{"\u65e5\u672c", "ns"}, WordTag{"\u548c\u670d", "nz"}, WordTag{"\u3002", "x"}},
[]WordTag{WordTag{"\u96f7\u7334", "n"}, WordTag{"\u56de\u5f52", "v"}, WordTag{"\u4eba\u95f4", "n"}, WordTag{"\u3002", "x"}},
[]WordTag{WordTag{"\u5de5\u4fe1\u5904", "n"}, WordTag{"\u5973\u5e72\u4e8b", "n"}, WordTag{"\u6bcf\u6708", "r"}, WordTag{"\u7ecf\u8fc7", "p"}, WordTag{"\u4e0b\u5c5e", "v"}, WordTag{"\u79d1\u5ba4", "n"}, WordTag{"\u90fd", "d"}, WordTag{"\u8981", "v"}, WordTag{"\u4eb2\u53e3", "n"}, WordTag{"\u4ea4\u4ee3", "n"}, WordTag{"24", "m"}, WordTag{"\u53e3", "n"}, WordTag{"\u4ea4\u6362\u673a", "n"}, WordTag{"\u7b49", "u"}, WordTag{"\u6280\u672f\u6027", "n"}, WordTag{"\u5668\u4ef6", "n"}, WordTag{"\u7684", "uj"}, WordTag{"\u5b89\u88c5", "v"}, WordTag{"\u5de5\u4f5c", "vn"}},
[]WordTag{WordTag{"\u6211", "r"}, WordTag{"\u9700\u8981", "v"}, WordTag{"\u5ec9\u79df\u623f", "n"}},
[]WordTag{WordTag{"\u6c38\u548c", "nz"}, WordTag{"\u670d\u88c5", "vn"}, WordTag{"\u9970\u54c1", "n"}, WordTag{"\u6709\u9650\u516c\u53f8", "n"}},
[]WordTag{WordTag{"\u6211", "r"}, WordTag{"\u7231", "v"}, WordTag{"\u5317\u4eac", "ns"}, WordTag{"\u5929\u5b89\u95e8", "ns"}},
defaultCutResult = [][]WordTag{[]WordTag{WordTag{"这", "r"}, WordTag{"是", "v"}, WordTag{"一个", "m"}, WordTag{"伸手不见五指", "i"}, WordTag{"的", "uj"}, WordTag{"黑夜", "n"}, WordTag{"。", "x"}, WordTag{"我", "r"}, WordTag{"叫", "v"}, WordTag{"孙悟空", "nr"}, WordTag{"，", "x"}, WordTag{"我", "r"}, WordTag{"爱", "v"}, WordTag{"北京", "ns"}, WordTag{"，", "x"}, WordTag{"我", "r"}, WordTag{"爱", "v"}, WordTag{"Python", "eng"}, WordTag{"和", "c"}, WordTag{"C++", "nz"}, WordTag{"。", "x"}},
[]WordTag{WordTag{"我", "r"}, WordTag{"不", "d"}, WordTag{"喜欢", "v"}, WordTag{"日本", "ns"}, WordTag{"和服", "nz"}, WordTag{"。", "x"}},
[]WordTag{WordTag{"雷猴", "n"}, WordTag{"回归", "v"}, WordTag{"人间", "n"}, WordTag{"。", "x"}},
[]WordTag{WordTag{"工信处", "n"}, WordTag{"女干事", "n"}, WordTag{"每月", "r"}, WordTag{"经过", "p"}, WordTag{"下属", "v"}, WordTag{"科室", "n"}, WordTag{"都", "d"}, WordTag{"要", "v"}, WordTag{"亲口", "n"}, WordTag{"交代", "n"}, WordTag{"24", "m"}, WordTag{"口", "n"}, WordTag{"交换机", "n"}, WordTag{"等", "u"}, WordTag{"技术性", "n"}, WordTag{"器件", "n"}, WordTag{"的", "uj"}, WordTag{"安装", "v"}, WordTag{"工作", "vn"}},
[]WordTag{WordTag{"我", "r"}, WordTag{"需要", "v"}, WordTag{"廉租房", "n"}},
[]WordTag{WordTag{"永和", "nz"}, WordTag{"服装", "vn"}, WordTag{"饰品", "n"}, WordTag{"有限公司", "n"}},
[]WordTag{WordTag{"我", "r"}, WordTag{"爱", "v"}, WordTag{"北京", "ns"}, WordTag{"天安门", "ns"}},
[]WordTag{WordTag{"abc", "eng"}},
[]WordTag{WordTag{"\u9690", "n"}, WordTag{"\u9a6c\u5c14\u53ef\u592b", "nr"}},
[]WordTag{WordTag{"\u96f7\u7334", "n"}, WordTag{"\u662f", "v"}, WordTag{"\u4e2a", "q"}, WordTag{"\u597d", "a"}, WordTag{"\u7f51\u7ad9", "n"}},
[]WordTag{WordTag{"\u201c", "x"}, WordTag{"Microsoft", "eng"}, WordTag{"\u201d", "x"}, WordTag{"\u4e00", "m"}, WordTag{"\u8bcd", "n"}, WordTag{"\u7531", "p"}, WordTag{"\u201c", "x"}, WordTag{"MICROcomputer", "eng"}, WordTag{"\uff08", "x"}, WordTag{"\u5fae\u578b", "b"}, WordTag{"\u8ba1\u7b97\u673a", "n"}, WordTag{"\uff09", "x"}, WordTag{"\u201d", "x"}, WordTag{"\u548c", "c"}, WordTag{"\u201c", "x"}, WordTag{"SOFTware", "eng"}, WordTag{"\uff08", "x"}, WordTag{"\u8f6f\u4ef6", "n"}, WordTag{"\uff09", "x"}, WordTag{"\u201d", "x"}, WordTag{"\u4e24", "m"}, WordTag{"\u90e8\u5206", "n"}, WordTag{"\u7ec4\u6210", "v"}},
[]WordTag{WordTag{"\u8349\u6ce5\u9a6c", "n"}, WordTag{"\u548c", "c"}, WordTag{"\u6b3a\u5b9e", "v"}, WordTag{"\u9a6c", "n"}, WordTag{"\u662f", "v"}, WordTag{"\u4eca\u5e74", "t"}, WordTag{"\u7684", "uj"}, WordTag{"\u6d41\u884c", "v"}, WordTag{"\u8bcd\u6c47", "n"}},
[]WordTag{WordTag{"\u4f0a\u85e4", "nr"}, WordTag{"\u6d0b\u534e\u5802", "n"}, WordTag{"\u603b\u5e9c", "n"}, WordTag{"\u5e97", "n"}},
[]WordTag{WordTag{"\u4e2d\u56fd\u79d1\u5b66\u9662\u8ba1\u7b97\u6280\u672f\u7814\u7a76\u6240", "nt"}},
[]WordTag{WordTag{"\u7f57\u5bc6\u6b27", "nr"}, WordTag{"\u4e0e", "p"}, WordTag{"\u6731\u4e3d\u53f6", "nr"}},
[]WordTag{WordTag{"\u6211", "r"}, WordTag{"\u8d2d\u4e70", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u9053\u5177", "n"}, WordTag{"\u548c", "c"}, WordTag{"\u670d\u88c5", "vn"}},
[]WordTag{WordTag{"PS", "eng"}, WordTag{":", "x"}, WordTag{" ", "x"}, WordTag{"\u6211", "r"}, WordTag{"\u89c9\u5f97", "v"}, WordTag{"\u5f00\u6e90", "n"}, WordTag{"\u6709", "v"}, WordTag{"\u4e00\u4e2a", "m"}, WordTag{"\u597d\u5904", "d"}, WordTag{"\uff0c", "x"}, WordTag{"\u5c31\u662f", "d"}, WordTag{"\u80fd\u591f", "v"}, WordTag{"\u6566\u4fc3", "v"}, WordTag{"\u81ea\u5df1", "r"}, WordTag{"\u4e0d\u65ad\u6539\u8fdb", "l"}, WordTag{"\uff0c", "x"}, WordTag{"\u907f\u514d", "v"}, WordTag{"\u655e", "v"}, WordTag{"\u5e1a", "ng"}, WordTag{"\u81ea\u73cd", "b"}},
[]WordTag{WordTag{"\u6e56\u5317\u7701", "ns"}, WordTag{"\u77f3\u9996\u5e02", "ns"}},
[]WordTag{WordTag{"\u6e56\u5317\u7701", "ns"}, WordTag{"\u5341\u5830\u5e02", "ns"}},
[]WordTag{WordTag{"\u603b\u7ecf\u7406", "n"}, WordTag{"\u5b8c\u6210", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u8fd9\u4ef6", "mq"}, WordTag{"\u4e8b\u60c5", "n"}},
[]WordTag{WordTag{"\u7535\u8111", "n"}, WordTag{"\u4fee\u597d", "v"}, WordTag{"\u4e86", "ul"}},
[]WordTag{WordTag{"\u505a\u597d", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u8fd9\u4ef6", "mq"}, WordTag{"\u4e8b\u60c5", "n"}, WordTag{"\u5c31", "d"}, WordTag{"\u4e00\u4e86\u767e\u4e86", "l"}, WordTag{"\u4e86", "ul"}},
[]WordTag{WordTag{"\u4eba\u4eec", "n"}, WordTag{"\u5ba1\u7f8e", "vn"}, WordTag{"\u7684", "uj"}, WordTag{"\u89c2\u70b9", "n"}, WordTag{"\u662f", "v"}, WordTag{"\u4e0d\u540c", "a"}, WordTag{"\u7684", "uj"}},
[]WordTag{WordTag{"\u6211\u4eec", "r"}, WordTag{"\u4e70", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u4e00\u4e2a", "m"}, WordTag{"\u7f8e\u7684", "nr"}, WordTag{"\u7a7a\u8c03", "n"}},
[]WordTag{WordTag{"\u7ebf\u7a0b", "n"}, WordTag{"\u521d\u59cb\u5316", "l"}, WordTag{"\u65f6", "n"}, WordTag{"\u6211\u4eec", "r"}, WordTag{"\u8981", "v"}, WordTag{"\u6ce8\u610f", "v"}},
[]WordTag{WordTag{"\u4e00\u4e2a", "m"}, WordTag{"\u5206\u5b50", "n"}, WordTag{"\u662f", "v"}, WordTag{"\u7531", "p"}, WordTag{"\u597d\u591a", "m"}, WordTag{"\u539f\u5b50", "n"}, WordTag{"\u7ec4\u7ec7", "v"}, WordTag{"\u6210", "v"}, WordTag{"\u7684", "uj"}},
[]WordTag{WordTag{"\u795d", "v"}, WordTag{"\u4f60", "r"}, WordTag{"\u9a6c\u5230\u529f\u6210", "i"}},
[]WordTag{WordTag{"\u4ed6", "r"}, WordTag{"\u6389", "v"}, WordTag{"\u8fdb", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u65e0\u5e95\u6d1e", "ns"}, WordTag{"\u91cc", "f"}},
[]WordTag{WordTag{"\u4e2d\u56fd", "ns"}, WordTag{"\u7684", "uj"}, WordTag{"\u9996\u90fd", "d"}, WordTag{"\u662f", "v"}, WordTag{"\u5317\u4eac", "ns"}},
[]WordTag{WordTag{"\u5b59\u541b\u610f", "nr"}},
[]WordTag{WordTag{"\u5916\u4ea4\u90e8", "nt"}, WordTag{"\u53d1\u8a00\u4eba", "l"}, WordTag{"\u9a6c\u671d\u65ed", "nr"}},
[]WordTag{WordTag{"\u9886\u5bfc\u4eba", "n"}, WordTag{"\u4f1a\u8bae", "n"}, WordTag{"\u548c", "c"}, WordTag{"\u7b2c\u56db\u5c4a", "m"}, WordTag{"\u4e1c\u4e9a", "ns"}, WordTag{"\u5cf0\u4f1a", "n"}},
[]WordTag{WordTag{"\u5728", "p"}, WordTag{"\u8fc7\u53bb", "t"}, WordTag{"\u7684", "uj"}, WordTag{"\u8fd9", "r"}, WordTag{"\u4e94\u5e74", "t"}},
[]WordTag{WordTag{"\u8fd8", "d"}, WordTag{"\u9700\u8981", "v"}, WordTag{"\u5f88", "d"}, WordTag{"\u957f", "a"}, WordTag{"\u7684", "uj"}, WordTag{"\u8def", "n"}, WordTag{"\u8981", "v"}, WordTag{"\u8d70", "v"}},
[]WordTag{WordTag{"60", "m"}, WordTag{"\u5468\u5e74", "t"}, WordTag{"\u9996\u90fd", "d"}, WordTag{"\u9605\u5175", "v"}},
[]WordTag{WordTag{"\u4f60\u597d", "l"}, WordTag{"\u4eba\u4eec", "n"}, WordTag{"\u5ba1\u7f8e", "vn"}, WordTag{"\u7684", "uj"}, WordTag{"\u89c2\u70b9", "n"}, WordTag{"\u662f", "v"}, WordTag{"\u4e0d\u540c", "a"}, WordTag{"\u7684", "uj"}},
[]WordTag{WordTag{"\u4e70", "v"}, WordTag{"\u6c34\u679c", "n"}, WordTag{"\u7136\u540e", "c"}, WordTag{"\u6765", "v"}, WordTag{"\u4e16\u535a\u56ed", "nr"}},
[]WordTag{WordTag{"\u4e70", "v"}, WordTag{"\u6c34\u679c", "n"}, WordTag{"\u7136\u540e", "c"}, WordTag{"\u53bb", "v"}, WordTag{"\u4e16\u535a\u56ed", "nr"}},
[]WordTag{WordTag{"\u4f46\u662f", "c"}, WordTag{"\u540e\u6765", "t"}, WordTag{"\u6211", "r"}, WordTag{"\u624d", "d"}, WordTag{"\u77e5\u9053", "v"}, WordTag{"\u4f60", "r"}, WordTag{"\u662f", "v"}, WordTag{"\u5bf9", "p"}, WordTag{"\u7684", "uj"}},
[]WordTag{WordTag{"\u5b58\u5728", "v"}, WordTag{"\u5373", "v"}, WordTag{"\u5408\u7406", "vn"}},
[]WordTag{WordTag{"\u7684\u7684", "u"}, WordTag{"\u7684\u7684", "u"}, WordTag{"\u7684", "uj"}, WordTag{"\u5728\u7684", "u"}, WordTag{"\u7684\u7684", "u"}, WordTag{"\u7684", "uj"}, WordTag{"\u5c31", "d"}, WordTag{"\u4ee5", "p"}, WordTag{"\u548c\u548c", "nz"}, WordTag{"\u548c", "c"}},
[]WordTag{WordTag{"I", "x"}, WordTag{" ", "x"}, WordTag{"love", "eng"}, WordTag{"\u4f60", "r"}, WordTag{"\uff0c", "x"}, WordTag{"\u4e0d\u4ee5\u4e3a\u803b", "i"}, WordTag{"\uff0c", "x"}, WordTag{"\u53cd", "zg"}, WordTag{"\u4ee5\u4e3a", "c"}, WordTag{"rong", "eng"}},
[]WordTag{WordTag{"\u56e0", "p"}},
[]WordTag{WordTag{"隐", "n"}, WordTag{"马尔可夫", "nr"}},
[]WordTag{WordTag{"雷猴", "n"}, WordTag{"是", "v"}, WordTag{"个", "q"}, WordTag{"好", "a"}, WordTag{"网站", "n"}},
[]WordTag{WordTag{"“", "x"}, WordTag{"Microsoft", "eng"}, WordTag{"”", "x"}, WordTag{"一", "m"}, WordTag{"词", "n"}, WordTag{"由", "p"}, WordTag{"“", "x"}, WordTag{"MICROcomputer", "eng"}, WordTag{"（", "x"}, WordTag{"微型", "b"}, WordTag{"计算机", "n"}, WordTag{"）", "x"}, WordTag{"”", "x"}, WordTag{"和", "c"}, WordTag{"“", "x"}, WordTag{"SOFTware", "eng"}, WordTag{"（", "x"}, WordTag{"软件", "n"}, WordTag{"）", "x"}, WordTag{"”", "x"}, WordTag{"两", "m"}, WordTag{"部分", "n"}, WordTag{"组成", "v"}},
[]WordTag{WordTag{"草泥马", "n"}, WordTag{"和", "c"}, WordTag{"欺实", "v"}, WordTag{"马", "n"}, WordTag{"是", "v"}, WordTag{"今年", "t"}, WordTag{"的", "uj"}, WordTag{"流行", "v"}, WordTag{"词汇", "n"}},
[]WordTag{WordTag{"伊藤", "nr"}, WordTag{"洋华堂", "n"}, WordTag{"总府", "n"}, WordTag{"店", "n"}},
[]WordTag{WordTag{"中国科学院计算技术研究所", "nt"}},
[]WordTag{WordTag{"罗密欧", "nr"}, WordTag{"与", "p"}, WordTag{"朱丽叶", "nr"}},
[]WordTag{WordTag{"我", "r"}, WordTag{"购买", "v"}, WordTag{"了", "ul"}, WordTag{"道具", "n"}, WordTag{"和", "c"}, WordTag{"服装", "vn"}},
[]WordTag{WordTag{"PS", "eng"}, WordTag{":", "x"}, WordTag{" ", "x"}, WordTag{"我", "r"}, WordTag{"觉得", "v"}, WordTag{"开源", "n"}, WordTag{"有", "v"}, WordTag{"一个", "m"}, WordTag{"好处", "d"}, WordTag{"，", "x"}, WordTag{"就是", "d"}, WordTag{"能够", "v"}, WordTag{"敦促", "v"}, WordTag{"自己", "r"}, WordTag{"不断改进", "l"}, WordTag{"，", "x"}, WordTag{"避免", "v"}, WordTag{"敞", "v"}, WordTag{"帚", "ng"}, WordTag{"自珍", "b"}},
[]WordTag{WordTag{"湖北省", "ns"}, WordTag{"石首市", "ns"}},
[]WordTag{WordTag{"湖北省", "ns"}, WordTag{"十堰市", "ns"}},
[]WordTag{WordTag{"总经理", "n"}, WordTag{"完成", "v"}, WordTag{"了", "ul"}, WordTag{"这件", "mq"}, WordTag{"事情", "n"}},
[]WordTag{WordTag{"电脑", "n"}, WordTag{"修好", "v"}, WordTag{"了", "ul"}},
[]WordTag{WordTag{"做好", "v"}, WordTag{"了", "ul"}, WordTag{"这件", "mq"}, WordTag{"事情", "n"}, WordTag{"就", "d"}, WordTag{"一了百了", "l"}, WordTag{"了", "ul"}},
[]WordTag{WordTag{"人们", "n"}, WordTag{"审美", "vn"}, WordTag{"的", "uj"}, WordTag{"观点", "n"}, WordTag{"是", "v"}, WordTag{"不同", "a"}, WordTag{"的", "uj"}},
[]WordTag{WordTag{"我们", "r"}, WordTag{"买", "v"}, WordTag{"了", "ul"}, WordTag{"一个", "m"}, WordTag{"美的", "nr"}, WordTag{"空调", "n"}},
[]WordTag{WordTag{"线程", "n"}, WordTag{"初始化", "l"}, WordTag{"时", "n"}, WordTag{"我们", "r"}, WordTag{"要", "v"}, WordTag{"注意", "v"}},
[]WordTag{WordTag{"一个", "m"}, WordTag{"分子", "n"}, WordTag{"是", "v"}, WordTag{"由", "p"}, WordTag{"好多", "m"}, WordTag{"原子", "n"}, WordTag{"组织", "v"}, WordTag{"成", "v"}, WordTag{"的", "uj"}},
[]WordTag{WordTag{"祝", "v"}, WordTag{"你", "r"}, WordTag{"马到功成", "i"}},
[]WordTag{WordTag{"他", "r"}, WordTag{"掉", "v"}, WordTag{"进", "v"}, WordTag{"了", "ul"}, WordTag{"无底洞", "ns"}, WordTag{"里", "f"}},
[]WordTag{WordTag{"中国", "ns"}, WordTag{"的", "uj"}, WordTag{"首都", "d"}, WordTag{"是", "v"}, WordTag{"北京", "ns"}},
[]WordTag{WordTag{"孙君意", "nr"}},
[]WordTag{WordTag{"外交部", "nt"}, WordTag{"发言人", "l"}, WordTag{"马朝旭", "nr"}},
[]WordTag{WordTag{"领导人", "n"}, WordTag{"会议", "n"}, WordTag{"和", "c"}, WordTag{"第四届", "m"}, WordTag{"东亚", "ns"}, WordTag{"峰会", "n"}},
[]WordTag{WordTag{"在", "p"}, WordTag{"过去", "t"}, WordTag{"的", "uj"}, WordTag{"这", "r"}, WordTag{"五年", "t"}},
[]WordTag{WordTag{"还", "d"}, WordTag{"需要", "v"}, WordTag{"很", "d"}, WordTag{"长", "a"}, WordTag{"的", "uj"}, WordTag{"路", "n"}, WordTag{"要", "v"}, WordTag{"走", "v"}},
[]WordTag{WordTag{"60", "m"}, WordTag{"周年", "t"}, WordTag{"首都", "d"}, WordTag{"阅兵", "v"}},
[]WordTag{WordTag{"你好", "l"}, WordTag{"人们", "n"}, WordTag{"审美", "vn"}, WordTag{"的", "uj"}, WordTag{"观点", "n"}, WordTag{"是", "v"}, WordTag{"不同", "a"}, WordTag{"的", "uj"}},
[]WordTag{WordTag{"买", "v"}, WordTag{"水果", "n"}, WordTag{"然后", "c"}, WordTag{"来", "v"}, WordTag{"世博园", "nr"}},
[]WordTag{WordTag{"买", "v"}, WordTag{"水果", "n"}, WordTag{"然后", "c"}, WordTag{"去", "v"}, WordTag{"世博园", "nr"}},
[]WordTag{WordTag{"但是", "c"}, WordTag{"后来", "t"}, WordTag{"我", "r"}, WordTag{"才", "d"}, WordTag{"知道", "v"}, WordTag{"你", "r"}, WordTag{"是", "v"}, WordTag{"对", "p"}, WordTag{"的", "uj"}},
[]WordTag{WordTag{"存在", "v"}, WordTag{"即", "v"}, WordTag{"合理", "vn"}},
[]WordTag{WordTag{"的的", "u"}, WordTag{"的的", "u"}, WordTag{"的", "uj"}, WordTag{"在的", "u"}, WordTag{"的的", "u"}, WordTag{"的", "uj"}, WordTag{"就", "d"}, WordTag{"以", "p"}, WordTag{"和和", "nz"}, WordTag{"和", "c"}},
[]WordTag{WordTag{"I", "x"}, WordTag{" ", "x"}, WordTag{"love", "eng"}, WordTag{"你", "r"}, WordTag{"，", "x"}, WordTag{"不以为耻", "i"}, WordTag{"，", "x"}, WordTag{"反", "zg"}, WordTag{"以为", "c"}, WordTag{"rong", "eng"}},
[]WordTag{WordTag{"因", "p"}},
[]WordTag{},
[]WordTag{WordTag{"hello", "eng"}, WordTag{"\u4f60\u597d", "l"}, WordTag{"\u4eba\u4eec", "n"}, WordTag{"\u5ba1\u7f8e", "vn"}, WordTag{"\u7684", "uj"}, WordTag{"\u89c2\u70b9", "n"}, WordTag{"\u662f", "v"}, WordTag{"\u4e0d\u540c", "a"}, WordTag{"\u7684", "uj"}},
[]WordTag{WordTag{"\u5f88\u597d", "a"}, WordTag{"\u4f46", "c"}, WordTag{"\u4e3b\u8981", "b"}, WordTag{"\u662f", "v"}, WordTag{"\u57fa\u4e8e", "p"}, WordTag{"\u7f51\u9875", "n"}, WordTag{"\u5f62\u5f0f", "n"}},
[]WordTag{WordTag{"hello", "eng"}, WordTag{"\u4f60\u597d", "l"}, WordTag{"\u4eba\u4eec", "n"}, WordTag{"\u5ba1\u7f8e", "vn"}, WordTag{"\u7684", "uj"}, WordTag{"\u89c2\u70b9", "n"}, WordTag{"\u662f", "v"}, WordTag{"\u4e0d\u540c", "a"}, WordTag{"\u7684", "uj"}},
[]WordTag{WordTag{"\u4e3a\u4ec0\u4e48", "r"}, WordTag{"\u6211", "r"}, WordTag{"\u4e0d\u80fd", "v"}, WordTag{"\u62e5\u6709", "v"}, WordTag{"\u60f3\u8981", "v"}, WordTag{"\u7684", "uj"}, WordTag{"\u751f\u6d3b", "vn"}},
[]WordTag{WordTag{"\u540e\u6765", "t"}, WordTag{"\u6211", "r"}, WordTag{"\u624d", "d"}},
[]WordTag{WordTag{"\u6b64\u6b21", "r"}, WordTag{"\u6765", "v"}, WordTag{"\u4e2d\u56fd", "ns"}, WordTag{"\u662f", "v"}, WordTag{"\u4e3a\u4e86", "p"}},
[]WordTag{WordTag{"\u4f7f\u7528", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u5b83", "r"}, WordTag{"\u5c31", "d"}, WordTag{"\u53ef\u4ee5", "c"}, WordTag{"\u89e3\u51b3", "v"}, WordTag{"\u4e00\u4e9b", "m"}, WordTag{"\u95ee\u9898", "n"}},
[]WordTag{WordTag{",", "x"}, WordTag{"\u4f7f\u7528", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u5b83", "r"}, WordTag{"\u5c31", "d"}, WordTag{"\u53ef\u4ee5", "c"}, WordTag{"\u89e3\u51b3", "v"}, WordTag{"\u4e00\u4e9b", "m"}, WordTag{"\u95ee\u9898", "n"}},
[]WordTag{WordTag{"\u5176\u5b9e", "d"}, WordTag{"\u4f7f\u7528", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u5b83", "r"}, WordTag{"\u5c31", "d"}, WordTag{"\u53ef\u4ee5", "c"}, WordTag{"\u89e3\u51b3", "v"}, WordTag{"\u4e00\u4e9b", "m"}, WordTag{"\u95ee\u9898", "n"}},
[]WordTag{WordTag{"\u597d\u4eba", "n"}, WordTag{"\u4f7f\u7528", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u5b83", "r"}, WordTag{"\u5c31", "d"}, WordTag{"\u53ef\u4ee5", "c"}, WordTag{"\u89e3\u51b3", "v"}, WordTag{"\u4e00\u4e9b", "m"}, WordTag{"\u95ee\u9898", "n"}},
[]WordTag{WordTag{"\u662f\u56e0\u4e3a", "c"}, WordTag{"\u548c", "c"}, WordTag{"\u56fd\u5bb6", "n"}},
[]WordTag{WordTag{"\u8001\u5e74", "t"}, WordTag{"\u641c\u7d22", "v"}, WordTag{"\u8fd8", "d"}, WordTag{"\u652f\u6301", "v"}},
[]WordTag{WordTag{"\u5e72\u8106", "d"}, WordTag{"\u5c31", "d"}, WordTag{"\u628a", "p"}, WordTag{"\u90a3\u90e8", "r"}, WordTag{"\u8499\u4eba", "n"}, WordTag{"\u7684", "uj"}, WordTag{"\u95f2\u6cd5", "n"}, WordTag{"\u7ed9", "p"}, WordTag{"\u5e9f", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u62c9\u5012", "v"}, WordTag{"\uff01", "x"}, WordTag{"RT", "eng"}, WordTag{" ", "x"}, WordTag{"@", "x"}, WordTag{"laoshipukong", "eng"}, WordTag{" ", "x"}, WordTag{":", "x"}, WordTag{" ", "x"}, WordTag{"27", "m"}, WordTag{"\u65e5", "m"}, WordTag{"\uff0c", "x"}, WordTag{"\u5168\u56fd\u4eba\u5927\u5e38\u59d4\u4f1a", "nt"}, WordTag{"\u7b2c\u4e09\u6b21", "m"}, WordTag{"\u5ba1\u8bae", "v"}, WordTag{"\u4fb5\u6743", "v"}, WordTag{"\u8d23\u4efb\u6cd5", "n"}, WordTag{"\u8349\u6848", "n"}, WordTag{"\uff0c", "x"}, WordTag{"\u5220\u9664", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u6709\u5173", "vn"}, WordTag{"\u533b\u7597", "n"}, WordTag{"\u635f\u5bb3", "v"}, WordTag{"\u8d23\u4efb", "n"}, WordTag{"\u201c", "x"}, WordTag{"\u4e3e\u8bc1", "v"}, WordTag{"\u5012\u7f6e", "v"}, WordTag{"\u201d", "x"}, WordTag{"\u7684", "uj"}, WordTag{"\u89c4\u5b9a", "n"}, WordTag{"\u3002", "x"}, WordTag{"\u5728", "p"}, WordTag{"\u533b\u60a3", "n"}, WordTag{"\u7ea0\u7eb7", "n"}, WordTag{"\u4e2d\u672c", "ns"}, WordTag{"\u5df2", "d"}, WordTag{"\u5904\u4e8e", "v"}, WordTag{"\u5f31\u52bf", "n"}, WordTag{"\u5730\u4f4d", "n"}, WordTag{"\u7684", "uj"}, WordTag{"\u6d88\u8d39\u8005", "n"}, WordTag{"\u7531\u6b64", "c"}, WordTag{"\u5c06", "d"}, WordTag{"\u9677\u5165", "v"}, WordTag{"\u4e07\u52ab\u4e0d\u590d", "i"}, WordTag{"\u7684", "uj"}, WordTag{"\u5883\u5730", "s"}, WordTag{"\u3002", "x"}, WordTag{" ", "x"}},
[]WordTag{WordTag{"\u5927", "a"}},
[]WordTag{WordTag{"hello", "eng"}, WordTag{"你好", "l"}, WordTag{"人们", "n"}, WordTag{"审美", "vn"}, WordTag{"的", "uj"}, WordTag{"观点", "n"}, WordTag{"是", "v"}, WordTag{"不同", "a"}, WordTag{"的", "uj"}},
[]WordTag{WordTag{"很好", "a"}, WordTag{"但", "c"}, WordTag{"主要", "b"}, WordTag{"是", "v"}, WordTag{"基于", "p"}, WordTag{"网页", "n"}, WordTag{"形式", "n"}},
[]WordTag{WordTag{"hello", "eng"}, WordTag{"你好", "l"}, WordTag{"人们", "n"}, WordTag{"审美", "vn"}, WordTag{"的", "uj"}, WordTag{"观点", "n"}, WordTag{"是", "v"}, WordTag{"不同", "a"}, WordTag{"的", "uj"}},
[]WordTag{WordTag{"为什么", "r"}, WordTag{"我", "r"}, WordTag{"不能", "v"}, WordTag{"拥有", "v"}, WordTag{"想要", "v"}, WordTag{"的", "uj"}, WordTag{"生活", "vn"}},
[]WordTag{WordTag{"后来", "t"}, WordTag{"我", "r"}, WordTag{"才", "d"}},
[]WordTag{WordTag{"此次", "r"}, WordTag{"来", "v"}, WordTag{"中国", "ns"}, WordTag{"是", "v"}, WordTag{"为了", "p"}},
[]WordTag{WordTag{"使用", "v"}, WordTag{"了", "ul"}, WordTag{"它", "r"}, WordTag{"就", "d"}, WordTag{"可以", "c"}, WordTag{"解决", "v"}, WordTag{"一些", "m"}, WordTag{"问题", "n"}},
[]WordTag{WordTag{",", "x"}, WordTag{"使用", "v"}, WordTag{"了", "ul"}, WordTag{"它", "r"}, WordTag{"就", "d"}, WordTag{"可以", "c"}, WordTag{"解决", "v"}, WordTag{"一些", "m"}, WordTag{"问题", "n"}},
[]WordTag{WordTag{"其实", "d"}, WordTag{"使用", "v"}, WordTag{"了", "ul"}, WordTag{"它", "r"}, WordTag{"就", "d"}, WordTag{"可以", "c"}, WordTag{"解决", "v"}, WordTag{"一些", "m"}, WordTag{"问题", "n"}},
[]WordTag{WordTag{"好人", "n"}, WordTag{"使用", "v"}, WordTag{"了", "ul"}, WordTag{"它", "r"}, WordTag{"就", "d"}, WordTag{"可以", "c"}, WordTag{"解决", "v"}, WordTag{"一些", "m"}, WordTag{"问题", "n"}},
[]WordTag{WordTag{"是因为", "c"}, WordTag{"和", "c"}, WordTag{"国家", "n"}},
[]WordTag{WordTag{"老年", "t"}, WordTag{"搜索", "v"}, WordTag{"还", "d"}, WordTag{"支持", "v"}},
[]WordTag{WordTag{"干脆", "d"}, WordTag{"就", "d"}, WordTag{"把", "p"}, WordTag{"那部", "r"}, WordTag{"蒙人", "n"}, WordTag{"的", "uj"}, WordTag{"闲法", "n"}, WordTag{"给", "p"}, WordTag{"废", "v"}, WordTag{"了", "ul"}, WordTag{"拉倒", "v"}, WordTag{"！", "x"}, WordTag{"RT", "eng"}, WordTag{" ", "x"}, WordTag{"@", "x"}, WordTag{"laoshipukong", "eng"}, WordTag{" ", "x"}, WordTag{":", "x"}, WordTag{" ", "x"}, WordTag{"27", "m"}, WordTag{"日", "m"}, WordTag{"，", "x"}, WordTag{"全国人大常委会", "nt"}, WordTag{"第三次", "m"}, WordTag{"审议", "v"}, WordTag{"侵权", "v"}, WordTag{"责任法", "n"}, WordTag{"草案", "n"}, WordTag{"，", "x"}, WordTag{"删除", "v"}, WordTag{"了", "ul"}, WordTag{"有关", "vn"}, WordTag{"医疗", "n"}, WordTag{"损害", "v"}, WordTag{"责任", "n"}, WordTag{"“", "x"}, WordTag{"举证", "v"}, WordTag{"倒置", "v"}, WordTag{"”", "x"}, WordTag{"的", "uj"}, WordTag{"规定", "n"}, WordTag{"。", "x"}, WordTag{"在", "p"}, WordTag{"医患", "n"}, WordTag{"纠纷", "n"}, WordTag{"中本", "ns"}, WordTag{"已", "d"}, WordTag{"处于", "v"}, WordTag{"弱势", "n"}, WordTag{"地位", "n"}, WordTag{"的", "uj"}, WordTag{"消费者", "n"}, WordTag{"由此", "c"}, WordTag{"将", "d"}, WordTag{"陷入", "v"}, WordTag{"万劫不复", "i"}, WordTag{"的", "uj"}, WordTag{"境地", "s"}, WordTag{"。", "x"}, WordTag{" ", "x"}},
[]WordTag{WordTag{"大", "a"}},
[]WordTag{},
[]WordTag{WordTag{"\u4ed6", "r"}, WordTag{"\u8bf4", "v"}, WordTag{"\u7684", "uj"}, WordTag{"\u786e\u5b9e", "ad"}, WordTag{"\u5728", "p"}, WordTag{"\u7406", "n"}},
[]WordTag{WordTag{"\u957f\u6625", "ns"}, WordTag{"\u5e02\u957f", "n"}, WordTag{"\u6625\u8282", "t"}, WordTag{"\u8bb2\u8bdd", "n"}},
[]WordTag{WordTag{"\u7ed3\u5a5a", "v"}, WordTag{"\u7684", "uj"}, WordTag{"\u548c", "c"}, WordTag{"\u5c1a\u672a", "d"}, WordTag{"\u7ed3\u5a5a", "v"}, WordTag{"\u7684", "uj"}},
[]WordTag{WordTag{"\u7ed3\u5408", "v"}, WordTag{"\u6210", "n"}, WordTag{"\u5206\u5b50", "n"}, WordTag{"\u65f6", "n"}},
[]WordTag{WordTag{"\u65c5\u6e38", "vn"}, WordTag{"\u548c", "c"}, WordTag{"\u670d\u52a1", "vn"}, WordTag{"\u662f", "v"}, WordTag{"\u6700\u597d", "a"}, WordTag{"\u7684", "uj"}},
[]WordTag{WordTag{"\u8fd9\u4ef6", "mq"}, WordTag{"\u4e8b\u60c5", "n"}, WordTag{"\u7684\u786e", "d"}, WordTag{"\u662f", "v"}, WordTag{"\u6211", "r"}, WordTag{"\u7684", "uj"}, WordTag{"\u9519", "n"}},
[]WordTag{WordTag{"\u4f9b", "v"}, WordTag{"\u5927\u5bb6", "n"}, WordTag{"\u53c2\u8003", "v"}, WordTag{"\u6307\u6b63", "v"}},
[]WordTag{WordTag{"\u54c8\u5c14\u6ee8", "ns"}, WordTag{"\u653f\u5e9c", "n"}, WordTag{"\u516c\u5e03", "v"}, WordTag{"\u584c", "v"}, WordTag{"\u6865", "n"}, WordTag{"\u539f\u56e0", "n"}},
[]WordTag{WordTag{"\u6211", "r"}, WordTag{"\u5728", "p"}, WordTag{"\u673a\u573a", "n"}, WordTag{"\u5165\u53e3\u5904", "i"}},
[]WordTag{WordTag{"\u90a2\u6c38\u81e3", "nr"}, WordTag{"\u6444\u5f71", "n"}, WordTag{"\u62a5\u9053", "v"}},
[]WordTag{WordTag{"BP", "eng"}, WordTag{"\u795e\u7ecf\u7f51\u7edc", "n"}, WordTag{"\u5982\u4f55", "r"}, WordTag{"\u8bad\u7ec3", "vn"}, WordTag{"\u624d\u80fd", "v"}, WordTag{"\u5728", "p"}, WordTag{"\u5206\u7c7b", "n"}, WordTag{"\u65f6", "n"}, WordTag{"\u589e\u52a0", "v"}, WordTag{"\u533a\u5206\u5ea6", "n"}, WordTag{"\uff1f", "x"}},
[]WordTag{WordTag{"\u5357\u4eac\u5e02", "ns"}, WordTag{"\u957f\u6c5f\u5927\u6865", "ns"}},
[]WordTag{WordTag{"\u5e94", "v"}, WordTag{"\u4e00\u4e9b", "m"}, WordTag{"\u4f7f\u7528\u8005", "n"}, WordTag{"\u7684", "uj"}, WordTag{"\u5efa\u8bae", "n"}, WordTag{"\uff0c", "x"}, WordTag{"\u4e5f", "d"}, WordTag{"\u4e3a\u4e86", "p"}, WordTag{"\u4fbf\u4e8e", "v"}, WordTag{"\u5229\u7528", "n"}, WordTag{"NiuTrans", "eng"}, WordTag{"\u7528\u4e8e", "v"}, WordTag{"SMT", "eng"}, WordTag{"\u7814\u7a76", "vn"}},
[]WordTag{WordTag{"\u957f\u6625\u5e02", "ns"}, WordTag{"\u957f\u6625", "ns"}, WordTag{"\u836f\u5e97", "n"}},
[]WordTag{WordTag{"\u9093\u9896\u8d85", "nr"}, WordTag{"\u751f\u524d", "t"}, WordTag{"\u6700", "d"}, WordTag{"\u559c\u6b22", "v"}, WordTag{"\u7684", "uj"}, WordTag{"\u8863\u670d", "n"}},
[]WordTag{WordTag{"\u80e1\u9526\u6d9b", "nr"}, WordTag{"\u662f", "v"}, WordTag{"\u70ed\u7231", "a"}, WordTag{"\u4e16\u754c", "n"}, WordTag{"\u548c\u5e73", "nz"}, WordTag{"\u7684", "uj"}, WordTag{"\u653f\u6cbb\u5c40", "n"}, WordTag{"\u5e38\u59d4", "j"}},
[]WordTag{WordTag{"\u7a0b\u5e8f\u5458", "n"}, WordTag{"\u795d", "v"}, WordTag{"\u6d77\u6797", "nz"}, WordTag{"\u548c", "c"}, WordTag{"\u6731\u4f1a\u9707", "nr"}, WordTag{"\u662f", "v"}, WordTag{"\u5728", "p"}, WordTag{"\u5b59\u5065", "nr"}, WordTag{"\u7684", "uj"}, WordTag{"\u5de6\u9762", "f"}, WordTag{"\u548c", "c"}, WordTag{"\u53f3\u9762", "f"}, WordTag{",", "x"}, WordTag{" ", "x"}, WordTag{"\u8303\u51ef", "nr"}, WordTag{"\u5728", "p"}, WordTag{"\u6700", "a"}, WordTag{"\u53f3\u9762", "f"}, WordTag{".", "m"}, WordTag{"\u518d\u5f80", "d"}, WordTag{"\u5de6", "f"}, WordTag{"\u662f", "v"}, WordTag{"\u674e\u677e\u6d2a", "nr"}},
[]WordTag{WordTag{"\u4e00\u6b21\u6027", "d"}, WordTag{"\u4ea4", "v"}, WordTag{"\u591a\u5c11", "m"}, WordTag{"\u94b1", "n"}},
[]WordTag{WordTag{"\u4e24\u5757", "m"}, WordTag{"\u4e94", "m"}, WordTag{"\u4e00\u5957", "m"}, WordTag{"\uff0c", "x"}, WordTag{"\u4e09\u5757", "m"}, WordTag{"\u516b", "m"}, WordTag{"\u4e00\u65a4", "m"}, WordTag{"\uff0c", "x"}, WordTag{"\u56db\u5757", "m"}, WordTag{"\u4e03", "m"}, WordTag{"\u4e00\u672c", "m"}, WordTag{"\uff0c", "x"}, WordTag{"\u4e94\u5757", "m"}, WordTag{"\u516d", "m"}, WordTag{"\u4e00\u6761", "m"}},
[]WordTag{WordTag{"\u5c0f", "a"}, WordTag{"\u548c\u5c1a", "nr"}, WordTag{"\u7559", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u4e00\u4e2a", "m"}, WordTag{"\u50cf", "v"}, WordTag{"\u5927", "a"}, WordTag{"\u548c\u5c1a", "nr"}, WordTag{"\u4e00\u6837", "r"}, WordTag{"\u7684", "uj"}, WordTag{"\u548c\u5c1a\u5934", "nr"}},
[]WordTag{WordTag{"\u6211", "r"}, WordTag{"\u662f", "v"}, WordTag{"\u4e2d\u534e\u4eba\u6c11\u5171\u548c\u56fd", "ns"}, WordTag{"\u516c\u6c11", "n"}, WordTag{";", "x"}, WordTag{"\u6211", "r"}, WordTag{"\u7238\u7238", "n"}, WordTag{"\u662f", "v"}, WordTag{"\u5171\u548c\u515a", "nt"}, WordTag{"\u515a\u5458", "n"}, WordTag{";", "x"}, WordTag{" ", "x"}, WordTag{"\u5730\u94c1", "n"}, WordTag{"\u548c\u5e73\u95e8", "ns"}, WordTag{"\u7ad9", "v"}},
[]WordTag{WordTag{"\u5f20\u6653\u6885", "nr"}, WordTag{"\u53bb", "v"}, WordTag{"\u4eba\u6c11", "n"}, WordTag{"\u533b\u9662", "n"}, WordTag{"\u505a", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u4e2a", "q"}, WordTag{"B\u8d85", "n"}, WordTag{"\u7136\u540e", "c"}, WordTag{"\u53bb", "v"}, WordTag{"\u4e70", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u4ef6", "q"}, WordTag{"T\u6064", "n"}},
[]WordTag{WordTag{"AT&T", "nz"}, WordTag{"\u662f", "v"}, WordTag{"\u4e00\u4ef6", "m"}, WordTag{"\u4e0d\u9519", "a"}, WordTag{"\u7684", "uj"}, WordTag{"\u516c\u53f8", "n"}, WordTag{"\uff0c", "x"}, WordTag{"\u7ed9", "p"}, WordTag{"\u4f60", "r"}, WordTag{"\u53d1", "v"}, WordTag{"offer", "eng"}, WordTag{"\u4e86", "ul"}, WordTag{"\u5417", "y"}, WordTag{"\uff1f", "x"}},
[]WordTag{WordTag{"C++", "nz"}, WordTag{"\u548c", "c"}, WordTag{"c#", "nz"}, WordTag{"\u662f", "v"}, WordTag{"\u4ec0\u4e48", "r"}, WordTag{"\u5173\u7cfb", "n"}, WordTag{"\uff1f", "x"}, WordTag{"11", "m"}, WordTag{"+", "x"}, WordTag{"122", "m"}, WordTag{"=", "x"}, WordTag{"133", "m"}, WordTag{"\uff0c", "x"}, WordTag{"\u662f", "v"}, WordTag{"\u5417", "y"}, WordTag{"\uff1f", "x"}, WordTag{"PI", "eng"}, WordTag{"=", "x"}, WordTag{"3.14159", "m"}},
[]WordTag{WordTag{"\u4f60", "r"}, WordTag{"\u8ba4\u8bc6", "v"}, WordTag{"\u90a3\u4e2a", "r"}, WordTag{"\u548c", "c"}, WordTag{"\u4e3b\u5e2d", "n"}, WordTag{"\u63e1\u624b", "v"}, WordTag{"\u7684", "uj"}, WordTag{"\u7684\u54e5", "n"}, WordTag{"\u5417", "y"}, WordTag{"\uff1f", "x"}, WordTag{"\u4ed6", "r"}, WordTag{"\u5f00", "v"}, WordTag{"\u4e00\u8f86", "m"}, WordTag{"\u9ed1\u8272", "n"}, WordTag{"\u7684\u58eb", "n"}, WordTag{"\u3002", "x"}},
[]WordTag{WordTag{"\u67aa\u6746\u5b50", "n"}, WordTag{"\u4e2d", "f"}, WordTag{"\u51fa", "v"}, WordTag{"\u653f\u6743", "n"}},
[]WordTag{WordTag{"他", "r"}, WordTag{"说", "v"}, WordTag{"的", "uj"}, WordTag{"确实", "ad"}, WordTag{"在", "p"}, WordTag{"理", "n"}},
[]WordTag{WordTag{"长春", "ns"}, WordTag{"市长", "n"}, WordTag{"春节", "t"}, WordTag{"讲话", "n"}},
[]WordTag{WordTag{"结婚", "v"}, WordTag{"的", "uj"}, WordTag{"和", "c"}, WordTag{"尚未", "d"}, WordTag{"结婚", "v"}, WordTag{"的", "uj"}},
[]WordTag{WordTag{"结合", "v"}, WordTag{"成", "n"}, WordTag{"分子", "n"}, WordTag{"时", "n"}},
[]WordTag{WordTag{"旅游", "vn"}, WordTag{"和", "c"}, WordTag{"服务", "vn"}, WordTag{"是", "v"}, WordTag{"最好", "a"}, WordTag{"的", "uj"}},
[]WordTag{WordTag{"这件", "mq"}, WordTag{"事情", "n"}, WordTag{"的确", "d"}, WordTag{"是", "v"}, WordTag{"我", "r"}, WordTag{"的", "uj"}, WordTag{"错", "n"}},
[]WordTag{WordTag{"供", "v"}, WordTag{"大家", "n"}, WordTag{"参考", "v"}, WordTag{"指正", "v"}},
[]WordTag{WordTag{"哈尔滨", "ns"}, WordTag{"政府", "n"}, WordTag{"公布", "v"}, WordTag{"塌", "v"}, WordTag{"桥", "n"}, WordTag{"原因", "n"}},
[]WordTag{WordTag{"我", "r"}, WordTag{"在", "p"}, WordTag{"机场", "n"}, WordTag{"入口处", "i"}},
[]WordTag{WordTag{"邢永臣", "nr"}, WordTag{"摄影", "n"}, WordTag{"报道", "v"}},
[]WordTag{WordTag{"BP", "eng"}, WordTag{"神经网络", "n"}, WordTag{"如何", "r"}, WordTag{"训练", "vn"}, WordTag{"才能", "v"}, WordTag{"在", "p"}, WordTag{"分类", "n"}, WordTag{"时", "n"}, WordTag{"增加", "v"}, WordTag{"区分度", "n"}, WordTag{"？", "x"}},
[]WordTag{WordTag{"南京市", "ns"}, WordTag{"长江大桥", "ns"}},
[]WordTag{WordTag{"应", "v"}, WordTag{"一些", "m"}, WordTag{"使用者", "n"}, WordTag{"的", "uj"}, WordTag{"建议", "n"}, WordTag{"，", "x"}, WordTag{"也", "d"}, WordTag{"为了", "p"}, WordTag{"便于", "v"}, WordTag{"利用", "n"}, WordTag{"NiuTrans", "eng"}, WordTag{"用于", "v"}, WordTag{"SMT", "eng"}, WordTag{"研究", "vn"}},
[]WordTag{WordTag{"长春市", "ns"}, WordTag{"长春", "ns"}, WordTag{"药店", "n"}},
[]WordTag{WordTag{"邓颖超", "nr"}, WordTag{"生前", "t"}, WordTag{"最", "d"}, WordTag{"喜欢", "v"}, WordTag{"的", "uj"}, WordTag{"衣服", "n"}},
[]WordTag{WordTag{"胡锦涛", "nr"}, WordTag{"是", "v"}, WordTag{"热爱", "a"}, WordTag{"世界", "n"}, WordTag{"和平", "nz"}, WordTag{"的", "uj"}, WordTag{"政治局", "n"}, WordTag{"常委", "j"}},
[]WordTag{WordTag{"程序员", "n"}, WordTag{"祝", "v"}, WordTag{"海林", "nz"}, WordTag{"和", "c"}, WordTag{"朱会震", "nr"}, WordTag{"是", "v"}, WordTag{"在", "p"}, WordTag{"孙健", "nr"}, WordTag{"的", "uj"}, WordTag{"左面", "f"}, WordTag{"和", "c"}, WordTag{"右面", "f"}, WordTag{",", "x"}, WordTag{" ", "x"}, WordTag{"范凯", "nr"}, WordTag{"在", "p"}, WordTag{"最", "a"}, WordTag{"右面", "f"}, WordTag{".", "m"}, WordTag{"再往", "d"}, WordTag{"左", "f"}, WordTag{"是", "v"}, WordTag{"李松洪", "nr"}},
[]WordTag{WordTag{"一次性", "d"}, WordTag{"交", "v"}, WordTag{"多少", "m"}, WordTag{"钱", "n"}},
[]WordTag{WordTag{"两块", "m"}, WordTag{"五", "m"}, WordTag{"一套", "m"}, WordTag{"，", "x"}, WordTag{"三块", "m"}, WordTag{"八", "m"}, WordTag{"一斤", "m"}, WordTag{"，", "x"}, WordTag{"四块", "m"}, WordTag{"七", "m"}, WordTag{"一本", "m"}, WordTag{"，", "x"}, WordTag{"五块", "m"}, WordTag{"六", "m"}, WordTag{"一条", "m"}},
[]WordTag{WordTag{"小", "a"}, WordTag{"和尚", "nr"}, WordTag{"留", "v"}, WordTag{"了", "ul"}, WordTag{"一个", "m"}, WordTag{"像", "v"}, WordTag{"大", "a"}, WordTag{"和尚", "nr"}, WordTag{"一样", "r"}, WordTag{"的", "uj"}, WordTag{"和尚头", "nr"}},
[]WordTag{WordTag{"我", "r"}, WordTag{"是", "v"}, WordTag{"中华人民共和国", "ns"}, WordTag{"公民", "n"}, WordTag{";", "x"}, WordTag{"我", "r"}, WordTag{"爸爸", "n"}, WordTag{"是", "v"}, WordTag{"共和党", "nt"}, WordTag{"党员", "n"}, WordTag{";", "x"}, WordTag{" ", "x"}, WordTag{"地铁", "n"}, WordTag{"和平门", "ns"}, WordTag{"站", "v"}},
[]WordTag{WordTag{"张晓梅", "nr"}, WordTag{"去", "v"}, WordTag{"人民", "n"}, WordTag{"医院", "n"}, WordTag{"做", "v"}, WordTag{"了", "ul"}, WordTag{"个", "q"}, WordTag{"B超", "n"}, WordTag{"然后", "c"}, WordTag{"去", "v"}, WordTag{"买", "v"}, WordTag{"了", "ul"}, WordTag{"件", "q"}, WordTag{"T恤", "n"}},
[]WordTag{WordTag{"AT&T", "nz"}, WordTag{"是", "v"}, WordTag{"一件", "m"}, WordTag{"不错", "a"}, WordTag{"的", "uj"}, WordTag{"公司", "n"}, WordTag{"，", "x"}, WordTag{"给", "p"}, WordTag{"你", "r"}, WordTag{"发", "v"}, WordTag{"offer", "eng"}, WordTag{"了", "ul"}, WordTag{"吗", "y"}, WordTag{"？", "x"}},
[]WordTag{WordTag{"C++", "nz"}, WordTag{"和", "c"}, WordTag{"c#", "nz"}, WordTag{"是", "v"}, WordTag{"什么", "r"}, WordTag{"关系", "n"}, WordTag{"？", "x"}, WordTag{"11", "m"}, WordTag{"+", "x"}, WordTag{"122", "m"}, WordTag{"=", "x"}, WordTag{"133", "m"}, WordTag{"，", "x"}, WordTag{"是", "v"}, WordTag{"吗", "y"}, WordTag{"？", "x"}, WordTag{"PI", "eng"}, WordTag{"=", "x"}, WordTag{"3.14159", "m"}},
[]WordTag{WordTag{"你", "r"}, WordTag{"认识", "v"}, WordTag{"那个", "r"}, WordTag{"和", "c"}, WordTag{"主席", "n"}, WordTag{"握手", "v"}, WordTag{"的", "uj"}, WordTag{"的哥", "n"}, WordTag{"吗", "y"}, WordTag{"？", "x"}, WordTag{"他", "r"}, WordTag{"开", "v"}, WordTag{"一辆", "m"}, WordTag{"黑色", "n"}, WordTag{"的士", "n"}, WordTag{"。", "x"}},
[]WordTag{WordTag{"枪杆子", "n"}, WordTag{"中", "f"}, WordTag{"出", "v"}, WordTag{"政权", "n"}},
}
noHMMCutResult = [][]WordTag{
[]WordTag{WordTag{"\u8fd9", "r"}, WordTag{"\u662f", "v"}, WordTag{"\u4e00\u4e2a", "m"}, WordTag{"\u4f38\u624b\u4e0d\u89c1\u4e94\u6307", "i"}, WordTag{"\u7684", "uj"}, WordTag{"\u9ed1\u591c", "n"}, WordTag{"\u3002", "x"}, WordTag{"\u6211", "r"}, WordTag{"\u53eb", "v"}, WordTag{"\u5b59\u609f\u7a7a", "nr"}, WordTag{"\uff0c", "x"}, WordTag{"\u6211", "r"}, WordTag{"\u7231", "v"}, WordTag{"\u5317\u4eac", "ns"}, WordTag{"\uff0c", "x"}, WordTag{"\u6211", "r"}, WordTag{"\u7231", "v"}, WordTag{"Python", "eng"}, WordTag{"\u548c", "c"}, WordTag{"C++", "nz"}, WordTag{"\u3002", "x"}},
[]WordTag{WordTag{"\u6211", "r"}, WordTag{"\u4e0d", "d"}, WordTag{"\u559c\u6b22", "v"}, WordTag{"\u65e5\u672c", "ns"}, WordTag{"\u548c\u670d", "nz"}, WordTag{"\u3002", "x"}},
[]WordTag{WordTag{"\u96f7\u7334", "n"}, WordTag{"\u56de\u5f52", "v"}, WordTag{"\u4eba\u95f4", "n"}, WordTag{"\u3002", "x"}},
[]WordTag{WordTag{"\u5de5\u4fe1\u5904", "n"}, WordTag{"\u5973\u5e72\u4e8b", "n"}, WordTag{"\u6bcf\u6708", "r"}, WordTag{"\u7ecf\u8fc7", "p"}, WordTag{"\u4e0b\u5c5e", "v"}, WordTag{"\u79d1\u5ba4", "n"}, WordTag{"\u90fd", "d"}, WordTag{"\u8981", "v"}, WordTag{"\u4eb2\u53e3", "n"}, WordTag{"\u4ea4\u4ee3", "n"}, WordTag{"24", "eng"}, WordTag{"\u53e3", "q"}, WordTag{"\u4ea4\u6362\u673a", "n"}, WordTag{"\u7b49", "u"}, WordTag{"\u6280\u672f\u6027", "n"}, WordTag{"\u5668\u4ef6", "n"}, WordTag{"\u7684", "uj"}, WordTag{"\u5b89\u88c5", "v"}, WordTag{"\u5de5\u4f5c", "vn"}},
[]WordTag{WordTag{"\u6211", "r"}, WordTag{"\u9700\u8981", "v"}, WordTag{"\u5ec9\u79df\u623f", "n"}},
[]WordTag{WordTag{"\u6c38\u548c", "nz"}, WordTag{"\u670d\u88c5", "vn"}, WordTag{"\u9970\u54c1", "n"}, WordTag{"\u6709\u9650\u516c\u53f8", "n"}},
[]WordTag{WordTag{"\u6211", "r"}, WordTag{"\u7231", "v"}, WordTag{"\u5317\u4eac", "ns"}, WordTag{"\u5929\u5b89\u95e8", "ns"}},
[]WordTag{WordTag{"这", "r"}, WordTag{"是", "v"}, WordTag{"一个", "m"}, WordTag{"伸手不见五指", "i"}, WordTag{"的", "uj"}, WordTag{"黑夜", "n"}, WordTag{"。", "x"}, WordTag{"我", "r"}, WordTag{"叫", "v"}, WordTag{"孙悟空", "nr"}, WordTag{"，", "x"}, WordTag{"我", "r"}, WordTag{"爱", "v"}, WordTag{"北京", "ns"}, WordTag{"，", "x"}, WordTag{"我", "r"}, WordTag{"爱", "v"}, WordTag{"Python", "eng"}, WordTag{"和", "c"}, WordTag{"C++", "nz"}, WordTag{"。", "x"}},
[]WordTag{WordTag{"我", "r"}, WordTag{"不", "d"}, WordTag{"喜欢", "v"}, WordTag{"日本", "ns"}, WordTag{"和服", "nz"}, WordTag{"。", "x"}},
[]WordTag{WordTag{"雷猴", "n"}, WordTag{"回归", "v"}, WordTag{"人间", "n"}, WordTag{"。", "x"}},
[]WordTag{WordTag{"工信处", "n"}, WordTag{"女干事", "n"}, WordTag{"每月", "r"}, WordTag{"经过", "p"}, WordTag{"下属", "v"}, WordTag{"科室", "n"}, WordTag{"都", "d"}, WordTag{"要", "v"}, WordTag{"亲口", "n"}, WordTag{"交代", "n"}, WordTag{"24", "eng"}, WordTag{"口", "q"}, WordTag{"交换机", "n"}, WordTag{"等", "u"}, WordTag{"技术性", "n"}, WordTag{"器件", "n"}, WordTag{"的", "uj"}, WordTag{"安装", "v"}, WordTag{"工作", "vn"}},
[]WordTag{WordTag{"我", "r"}, WordTag{"需要", "v"}, WordTag{"廉租房", "n"}},
[]WordTag{WordTag{"永和", "nz"}, WordTag{"服装", "vn"}, WordTag{"饰品", "n"}, WordTag{"有限公司", "n"}},
[]WordTag{WordTag{"我", "r"}, WordTag{"爱", "v"}, WordTag{"北京", "ns"}, WordTag{"天安门", "ns"}},
[]WordTag{WordTag{"abc", "eng"}},
[]WordTag{WordTag{"\u9690", "n"}, WordTag{"\u9a6c\u5c14\u53ef\u592b", "nr"}},
[]WordTag{WordTag{"\u96f7\u7334", "n"}, WordTag{"\u662f", "v"}, WordTag{"\u4e2a", "q"}, WordTag{"\u597d", "a"}, WordTag{"\u7f51\u7ad9", "n"}},
[]WordTag{WordTag{"\u201c", "x"}, WordTag{"Microsoft", "eng"}, WordTag{"\u201d", "x"}, WordTag{"\u4e00", "m"}, WordTag{"\u8bcd", "n"}, WordTag{"\u7531", "p"}, WordTag{"\u201c", "x"}, WordTag{"MICROcomputer", "eng"}, WordTag{"\uff08", "x"}, WordTag{"\u5fae\u578b", "b"}, WordTag{"\u8ba1\u7b97\u673a", "n"}, WordTag{"\uff09", "x"}, WordTag{"\u201d", "x"}, WordTag{"\u548c", "c"}, WordTag{"\u201c", "x"}, WordTag{"SOFTware", "eng"}, WordTag{"\uff08", "x"}, WordTag{"\u8f6f\u4ef6", "n"}, WordTag{"\uff09", "x"}, WordTag{"\u201d", "x"}, WordTag{"\u4e24", "m"}, WordTag{"\u90e8\u5206", "n"}, WordTag{"\u7ec4\u6210", "v"}},
[]WordTag{WordTag{"\u8349\u6ce5\u9a6c", "n"}, WordTag{"\u548c", "c"}, WordTag{"\u6b3a", "vn"}, WordTag{"\u5b9e", "n"}, WordTag{"\u9a6c", "n"}, WordTag{"\u662f", "v"}, WordTag{"\u4eca\u5e74", "t"}, WordTag{"\u7684", "uj"}, WordTag{"\u6d41\u884c", "v"}, WordTag{"\u8bcd\u6c47", "n"}},
[]WordTag{WordTag{"\u4f0a", "ns"}, WordTag{"\u85e4", "nr"}, WordTag{"\u6d0b\u534e\u5802", "n"}, WordTag{"\u603b\u5e9c", "n"}, WordTag{"\u5e97", "n"}},
[]WordTag{WordTag{"\u4e2d\u56fd\u79d1\u5b66\u9662\u8ba1\u7b97\u6280\u672f\u7814\u7a76\u6240", "nt"}},
[]WordTag{WordTag{"\u7f57\u5bc6\u6b27", "nr"}, WordTag{"\u4e0e", "p"}, WordTag{"\u6731\u4e3d\u53f6", "nr"}},
[]WordTag{WordTag{"\u6211", "r"}, WordTag{"\u8d2d\u4e70", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u9053\u5177", "n"}, WordTag{"\u548c", "c"}, WordTag{"\u670d\u88c5", "vn"}},
[]WordTag{WordTag{"PS", "eng"}, WordTag{":", "x"}, WordTag{" ", "x"}, WordTag{"\u6211", "r"}, WordTag{"\u89c9\u5f97", "v"}, WordTag{"\u5f00\u6e90", "n"}, WordTag{"\u6709", "v"}, WordTag{"\u4e00\u4e2a", "m"}, WordTag{"\u597d\u5904", "d"}, WordTag{"\uff0c", "x"}, WordTag{"\u5c31\u662f", "d"}, WordTag{"\u80fd\u591f", "v"}, WordTag{"\u6566\u4fc3", "v"}, WordTag{"\u81ea\u5df1", "r"}, WordTag{"\u4e0d\u65ad\u6539\u8fdb", "l"}, WordTag{"\uff0c", "x"}, WordTag{"\u907f\u514d", "v"}, WordTag{"\u655e", "v"}, WordTag{"\u5e1a", "ng"}, WordTag{"\u81ea\u73cd", "b"}},
[]WordTag{WordTag{"\u6e56\u5317\u7701", "ns"}, WordTag{"\u77f3\u9996\u5e02", "ns"}},
[]WordTag{WordTag{"\u6e56\u5317\u7701", "ns"}, WordTag{"\u5341\u5830\u5e02", "ns"}},
[]WordTag{WordTag{"\u603b\u7ecf\u7406", "n"}, WordTag{"\u5b8c\u6210", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u8fd9\u4ef6", "mq"}, WordTag{"\u4e8b\u60c5", "n"}},
[]WordTag{WordTag{"\u7535\u8111", "n"}, WordTag{"\u4fee\u597d", "v"}, WordTag{"\u4e86", "ul"}},
[]WordTag{WordTag{"\u505a\u597d", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u8fd9\u4ef6", "mq"}, WordTag{"\u4e8b\u60c5", "n"}, WordTag{"\u5c31", "d"}, WordTag{"\u4e00\u4e86\u767e\u4e86", "l"}, WordTag{"\u4e86", "ul"}},
[]WordTag{WordTag{"\u4eba\u4eec", "n"}, WordTag{"\u5ba1\u7f8e", "vn"}, WordTag{"\u7684", "uj"}, WordTag{"\u89c2\u70b9", "n"}, WordTag{"\u662f", "v"}, WordTag{"\u4e0d\u540c", "a"}, WordTag{"\u7684", "uj"}},
[]WordTag{WordTag{"\u6211\u4eec", "r"}, WordTag{"\u4e70", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u4e00\u4e2a", "m"}, WordTag{"\u7f8e\u7684", "nr"}, WordTag{"\u7a7a\u8c03", "n"}},
[]WordTag{WordTag{"\u7ebf\u7a0b", "n"}, WordTag{"\u521d\u59cb\u5316", "l"}, WordTag{"\u65f6", "n"}, WordTag{"\u6211\u4eec", "r"}, WordTag{"\u8981", "v"}, WordTag{"\u6ce8\u610f", "v"}},
[]WordTag{WordTag{"\u4e00\u4e2a", "m"}, WordTag{"\u5206\u5b50", "n"}, WordTag{"\u662f", "v"}, WordTag{"\u7531", "p"}, WordTag{"\u597d\u591a", "m"}, WordTag{"\u539f\u5b50", "n"}, WordTag{"\u7ec4\u7ec7", "v"}, WordTag{"\u6210", "n"}, WordTag{"\u7684", "uj"}},
[]WordTag{WordTag{"\u795d", "v"}, WordTag{"\u4f60", "r"}, WordTag{"\u9a6c\u5230\u529f\u6210", "i"}},
[]WordTag{WordTag{"\u4ed6", "r"}, WordTag{"\u6389", "zg"}, WordTag{"\u8fdb", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u65e0\u5e95\u6d1e", "ns"}, WordTag{"\u91cc", "f"}},
[]WordTag{WordTag{"\u4e2d\u56fd", "ns"}, WordTag{"\u7684", "uj"}, WordTag{"\u9996\u90fd", "d"}, WordTag{"\u662f", "v"}, WordTag{"\u5317\u4eac", "ns"}},
[]WordTag{WordTag{"\u5b59", "zg"}, WordTag{"\u541b", "nz"}, WordTag{"\u610f", "n"}},
[]WordTag{WordTag{"\u5916\u4ea4\u90e8", "nt"}, WordTag{"\u53d1\u8a00\u4eba", "l"}, WordTag{"\u9a6c\u671d\u65ed", "nr"}},
[]WordTag{WordTag{"\u9886\u5bfc\u4eba", "n"}, WordTag{"\u4f1a\u8bae", "n"}, WordTag{"\u548c", "c"}, WordTag{"\u7b2c\u56db\u5c4a", "m"}, WordTag{"\u4e1c\u4e9a", "ns"}, WordTag{"\u5cf0\u4f1a", "n"}},
[]WordTag{WordTag{"\u5728", "p"}, WordTag{"\u8fc7\u53bb", "t"}, WordTag{"\u7684", "uj"}, WordTag{"\u8fd9", "r"}, WordTag{"\u4e94\u5e74", "t"}},
[]WordTag{WordTag{"\u8fd8", "d"}, WordTag{"\u9700\u8981", "v"}, WordTag{"\u5f88", "zg"}, WordTag{"\u957f", "a"}, WordTag{"\u7684", "uj"}, WordTag{"\u8def", "n"}, WordTag{"\u8981", "v"}, WordTag{"\u8d70", "v"}},
[]WordTag{WordTag{"60", "eng"}, WordTag{"\u5468\u5e74", "t"}, WordTag{"\u9996\u90fd", "d"}, WordTag{"\u9605\u5175", "v"}},
[]WordTag{WordTag{"\u4f60\u597d", "l"}, WordTag{"\u4eba\u4eec", "n"}, WordTag{"\u5ba1\u7f8e", "vn"}, WordTag{"\u7684", "uj"}, WordTag{"\u89c2\u70b9", "n"}, WordTag{"\u662f", "v"}, WordTag{"\u4e0d\u540c", "a"}, WordTag{"\u7684", "uj"}},
[]WordTag{WordTag{"\u4e70", "v"}, WordTag{"\u6c34\u679c", "n"}, WordTag{"\u7136\u540e", "c"}, WordTag{"\u6765", "v"}, WordTag{"\u4e16\u535a\u56ed", "nr"}},
[]WordTag{WordTag{"\u4e70", "v"}, WordTag{"\u6c34\u679c", "n"}, WordTag{"\u7136\u540e", "c"}, WordTag{"\u53bb", "v"}, WordTag{"\u4e16\u535a\u56ed", "nr"}},
[]WordTag{WordTag{"\u4f46\u662f", "c"}, WordTag{"\u540e\u6765", "t"}, WordTag{"\u6211", "r"}, WordTag{"\u624d", "d"}, WordTag{"\u77e5\u9053", "v"}, WordTag{"\u4f60", "r"}, WordTag{"\u662f", "v"}, WordTag{"\u5bf9", "p"}, WordTag{"\u7684", "uj"}},
[]WordTag{WordTag{"\u5b58\u5728", "v"}, WordTag{"\u5373", "v"}, WordTag{"\u5408\u7406", "vn"}},
[]WordTag{WordTag{"\u7684", "uj"}, WordTag{"\u7684", "uj"}, WordTag{"\u7684", "uj"}, WordTag{"\u7684", "uj"}, WordTag{"\u7684", "uj"}, WordTag{"\u5728", "p"}, WordTag{"\u7684", "uj"}, WordTag{"\u7684", "uj"}, WordTag{"\u7684", "uj"}, WordTag{"\u7684", "uj"}, WordTag{"\u5c31", "d"}, WordTag{"\u4ee5", "p"}, WordTag{"\u548c", "c"}, WordTag{"\u548c", "c"}, WordTag{"\u548c", "c"}},
[]WordTag{WordTag{"I", "eng"}, WordTag{" ", "x"}, WordTag{"love", "eng"}, WordTag{"\u4f60", "r"}, WordTag{"\uff0c", "x"}, WordTag{"\u4e0d\u4ee5\u4e3a\u803b", "i"}, WordTag{"\uff0c", "x"}, WordTag{"\u53cd", "zg"}, WordTag{"\u4ee5\u4e3a", "c"}, WordTag{"rong", "eng"}},
[]WordTag{WordTag{"\u56e0", "p"}},
[]WordTag{WordTag{"隐", "n"}, WordTag{"马尔可夫", "nr"}},
[]WordTag{WordTag{"雷猴", "n"}, WordTag{"是", "v"}, WordTag{"个", "q"}, WordTag{"好", "a"}, WordTag{"网站", "n"}},
[]WordTag{WordTag{"“", "x"}, WordTag{"Microsoft", "eng"}, WordTag{"”", "x"}, WordTag{"一", "m"}, WordTag{"词", "n"}, WordTag{"由", "p"}, WordTag{"“", "x"}, WordTag{"MICROcomputer", "eng"}, WordTag{"（", "x"}, WordTag{"微型", "b"}, WordTag{"计算机", "n"}, WordTag{"）", "x"}, WordTag{"”", "x"}, WordTag{"和", "c"}, WordTag{"“", "x"}, WordTag{"SOFTware", "eng"}, WordTag{"（", "x"}, WordTag{"软件", "n"}, WordTag{"）", "x"}, WordTag{"”", "x"}, WordTag{"两", "m"}, WordTag{"部分", "n"}, WordTag{"组成", "v"}},
[]WordTag{WordTag{"草泥马", "n"}, WordTag{"和", "c"}, WordTag{"欺", "vn"}, WordTag{"实", "n"}, WordTag{"马", "n"}, WordTag{"是", "v"}, WordTag{"今年", "t"}, WordTag{"的", "uj"}, WordTag{"流行", "v"}, WordTag{"词汇", "n"}},
[]WordTag{WordTag{"伊", "ns"}, WordTag{"藤", "nr"}, WordTag{"洋华堂", "n"}, WordTag{"总府", "n"}, WordTag{"店", "n"}},
[]WordTag{WordTag{"中国科学院计算技术研究所", "nt"}},
[]WordTag{WordTag{"罗密欧", "nr"}, WordTag{"与", "p"}, WordTag{"朱丽叶", "nr"}},
[]WordTag{WordTag{"我", "r"}, WordTag{"购买", "v"}, WordTag{"了", "ul"}, WordTag{"道具", "n"}, WordTag{"和", "c"}, WordTag{"服装", "vn"}},
[]WordTag{WordTag{"PS", "eng"}, WordTag{":", "x"}, WordTag{" ", "x"}, WordTag{"我", "r"}, WordTag{"觉得", "v"}, WordTag{"开源", "n"}, WordTag{"有", "v"}, WordTag{"一个", "m"}, WordTag{"好处", "d"}, WordTag{"，", "x"}, WordTag{"就是", "d"}, WordTag{"能够", "v"}, WordTag{"敦促", "v"}, WordTag{"自己", "r"}, WordTag{"不断改进", "l"}, WordTag{"，", "x"}, WordTag{"避免", "v"}, WordTag{"敞", "v"}, WordTag{"帚", "ng"}, WordTag{"自珍", "b"}},
[]WordTag{WordTag{"湖北省", "ns"}, WordTag{"石首市", "ns"}},
[]WordTag{WordTag{"湖北省", "ns"}, WordTag{"十堰市", "ns"}},
[]WordTag{WordTag{"总经理", "n"}, WordTag{"完成", "v"}, WordTag{"了", "ul"}, WordTag{"这件", "mq"}, WordTag{"事情", "n"}},
[]WordTag{WordTag{"电脑", "n"}, WordTag{"修好", "v"}, WordTag{"了", "ul"}},
[]WordTag{WordTag{"做好", "v"}, WordTag{"了", "ul"}, WordTag{"这件", "mq"}, WordTag{"事情", "n"}, WordTag{"就", "d"}, WordTag{"一了百了", "l"}, WordTag{"了", "ul"}},
[]WordTag{WordTag{"人们", "n"}, WordTag{"审美", "vn"}, WordTag{"的", "uj"}, WordTag{"观点", "n"}, WordTag{"是", "v"}, WordTag{"不同", "a"}, WordTag{"的", "uj"}},
[]WordTag{WordTag{"我们", "r"}, WordTag{"买", "v"}, WordTag{"了", "ul"}, WordTag{"一个", "m"}, WordTag{"美的", "nr"}, WordTag{"空调", "n"}},
[]WordTag{WordTag{"线程", "n"}, WordTag{"初始化", "l"}, WordTag{"时", "n"}, WordTag{"我们", "r"}, WordTag{"要", "v"}, WordTag{"注意", "v"}},
[]WordTag{WordTag{"一个", "m"}, WordTag{"分子", "n"}, WordTag{"是", "v"}, WordTag{"由", "p"}, WordTag{"好多", "m"}, WordTag{"原子", "n"}, WordTag{"组织", "v"}, WordTag{"成", "n"}, WordTag{"的", "uj"}},
[]WordTag{WordTag{"祝", "v"}, WordTag{"你", "r"}, WordTag{"马到功成", "i"}},
[]WordTag{WordTag{"他", "r"}, WordTag{"掉", "zg"}, WordTag{"进", "v"}, WordTag{"了", "ul"}, WordTag{"无底洞", "ns"}, WordTag{"里", "f"}},
[]WordTag{WordTag{"中国", "ns"}, WordTag{"的", "uj"}, WordTag{"首都", "d"}, WordTag{"是", "v"}, WordTag{"北京", "ns"}},
[]WordTag{WordTag{"孙", "zg"}, WordTag{"君", "nz"}, WordTag{"意", "n"}},
[]WordTag{WordTag{"外交部", "nt"}, WordTag{"发言人", "l"}, WordTag{"马朝旭", "nr"}},
[]WordTag{WordTag{"领导人", "n"}, WordTag{"会议", "n"}, WordTag{"和", "c"}, WordTag{"第四届", "m"}, WordTag{"东亚", "ns"}, WordTag{"峰会", "n"}},
[]WordTag{WordTag{"在", "p"}, WordTag{"过去", "t"}, WordTag{"的", "uj"}, WordTag{"这", "r"}, WordTag{"五年", "t"}},
[]WordTag{WordTag{"还", "d"}, WordTag{"需要", "v"}, WordTag{"很", "zg"}, WordTag{"长", "a"}, WordTag{"的", "uj"}, WordTag{"路", "n"}, WordTag{"要", "v"}, WordTag{"走", "v"}},
[]WordTag{WordTag{"60", "eng"}, WordTag{"周年", "t"}, WordTag{"首都", "d"}, WordTag{"阅兵", "v"}},
[]WordTag{WordTag{"你好", "l"}, WordTag{"人们", "n"}, WordTag{"审美", "vn"}, WordTag{"的", "uj"}, WordTag{"观点", "n"}, WordTag{"是", "v"}, WordTag{"不同", "a"}, WordTag{"的", "uj"}},
[]WordTag{WordTag{"买", "v"}, WordTag{"水果", "n"}, WordTag{"然后", "c"}, WordTag{"来", "v"}, WordTag{"世博园", "nr"}},
[]WordTag{WordTag{"买", "v"}, WordTag{"水果", "n"}, WordTag{"然后", "c"}, WordTag{"去", "v"}, WordTag{"世博园", "nr"}},
[]WordTag{WordTag{"但是", "c"}, WordTag{"后来", "t"}, WordTag{"我", "r"}, WordTag{"才", "d"}, WordTag{"知道", "v"}, WordTag{"你", "r"}, WordTag{"是", "v"}, WordTag{"对", "p"}, WordTag{"的", "uj"}},
[]WordTag{WordTag{"存在", "v"}, WordTag{"即", "v"}, WordTag{"合理", "vn"}},
[]WordTag{WordTag{"的", "uj"}, WordTag{"的", "uj"}, WordTag{"的", "uj"}, WordTag{"的", "uj"}, WordTag{"的", "uj"}, WordTag{"在", "p"}, WordTag{"的", "uj"}, WordTag{"的", "uj"}, WordTag{"的", "uj"}, WordTag{"的", "uj"}, WordTag{"就", "d"}, WordTag{"以", "p"}, WordTag{"和", "c"}, WordTag{"和", "c"}, WordTag{"和", "c"}},
[]WordTag{WordTag{"I", "eng"}, WordTag{" ", "x"}, WordTag{"love", "eng"}, WordTag{"你", "r"}, WordTag{"，", "x"}, WordTag{"不以为耻", "i"}, WordTag{"，", "x"}, WordTag{"反", "zg"}, WordTag{"以为", "c"}, WordTag{"rong", "eng"}},
[]WordTag{WordTag{"因", "p"}},
[]WordTag{},
[]WordTag{WordTag{"hello", "eng"}, WordTag{"\u4f60\u597d", "l"}, WordTag{"\u4eba\u4eec", "n"}, WordTag{"\u5ba1\u7f8e", "vn"}, WordTag{"\u7684", "uj"}, WordTag{"\u89c2\u70b9", "n"}, WordTag{"\u662f", "v"}, WordTag{"\u4e0d\u540c", "a"}, WordTag{"\u7684", "uj"}},
[]WordTag{WordTag{"\u5f88", "zg"}, WordTag{"\u597d", "a"}, WordTag{"\u4f46", "c"}, WordTag{"\u4e3b\u8981", "b"}, WordTag{"\u662f", "v"}, WordTag{"\u57fa\u4e8e", "p"}, WordTag{"\u7f51\u9875", "n"}, WordTag{"\u5f62\u5f0f", "n"}},
[]WordTag{WordTag{"hello", "eng"}, WordTag{"\u4f60\u597d", "l"}, WordTag{"\u4eba\u4eec", "n"}, WordTag{"\u5ba1\u7f8e", "vn"}, WordTag{"\u7684", "uj"}, WordTag{"\u89c2\u70b9", "n"}, WordTag{"\u662f", "v"}, WordTag{"\u4e0d\u540c", "a"}, WordTag{"\u7684", "uj"}},
[]WordTag{WordTag{"\u4e3a\u4ec0\u4e48", "r"}, WordTag{"\u6211", "r"}, WordTag{"\u4e0d\u80fd", "v"}, WordTag{"\u62e5\u6709", "v"}, WordTag{"\u60f3\u8981", "v"}, WordTag{"\u7684", "uj"}, WordTag{"\u751f\u6d3b", "vn"}},
[]WordTag{WordTag{"\u540e\u6765", "t"}, WordTag{"\u6211", "r"}, WordTag{"\u624d", "d"}},
[]WordTag{WordTag{"\u6b64\u6b21", "r"}, WordTag{"\u6765", "v"}, WordTag{"\u4e2d\u56fd", "ns"}, WordTag{"\u662f", "v"}, WordTag{"\u4e3a\u4e86", "p"}},
[]WordTag{WordTag{"\u4f7f\u7528", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u5b83", "r"}, WordTag{"\u5c31", "d"}, WordTag{"\u53ef\u4ee5", "c"}, WordTag{"\u89e3\u51b3", "v"}, WordTag{"\u4e00\u4e9b", "m"}, WordTag{"\u95ee\u9898", "n"}},
[]WordTag{WordTag{",", "x"}, WordTag{"\u4f7f\u7528", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u5b83", "r"}, WordTag{"\u5c31", "d"}, WordTag{"\u53ef\u4ee5", "c"}, WordTag{"\u89e3\u51b3", "v"}, WordTag{"\u4e00\u4e9b", "m"}, WordTag{"\u95ee\u9898", "n"}},
[]WordTag{WordTag{"\u5176\u5b9e", "d"}, WordTag{"\u4f7f\u7528", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u5b83", "r"}, WordTag{"\u5c31", "d"}, WordTag{"\u53ef\u4ee5", "c"}, WordTag{"\u89e3\u51b3", "v"}, WordTag{"\u4e00\u4e9b", "m"}, WordTag{"\u95ee\u9898", "n"}},
[]WordTag{WordTag{"\u597d\u4eba", "n"}, WordTag{"\u4f7f\u7528", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u5b83", "r"}, WordTag{"\u5c31", "d"}, WordTag{"\u53ef\u4ee5", "c"}, WordTag{"\u89e3\u51b3", "v"}, WordTag{"\u4e00\u4e9b", "m"}, WordTag{"\u95ee\u9898", "n"}},
[]WordTag{WordTag{"\u662f\u56e0\u4e3a", "c"}, WordTag{"\u548c", "c"}, WordTag{"\u56fd\u5bb6", "n"}},
[]WordTag{WordTag{"\u8001\u5e74", "t"}, WordTag{"\u641c\u7d22", "v"}, WordTag{"\u8fd8", "d"}, WordTag{"\u652f\u6301", "v"}},
[]WordTag{WordTag{"\u5e72\u8106", "d"}, WordTag{"\u5c31", "d"}, WordTag{"\u628a", "p"}, WordTag{"\u90a3", "r"}, WordTag{"\u90e8", "n"}, WordTag{"\u8499", "v"}, WordTag{"\u4eba", "n"}, WordTag{"\u7684", "uj"}, WordTag{"\u95f2", "n"}, WordTag{"\u6cd5", "j"}, WordTag{"\u7ed9", "p"}, WordTag{"\u5e9f", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u62c9\u5012", "v"}, WordTag{"\uff01", "x"}, WordTag{"RT", "eng"}, WordTag{" ", "x"}, WordTag{"@", "x"}, WordTag{"laoshipukong", "eng"}, WordTag{" ", "x"}, WordTag{":", "x"}, WordTag{" ", "x"}, WordTag{"27", "eng"}, WordTag{"\u65e5", "m"}, WordTag{"\uff0c", "x"}, WordTag{"\u5168\u56fd\u4eba\u5927\u5e38\u59d4\u4f1a", "nt"}, WordTag{"\u7b2c\u4e09\u6b21", "m"}, WordTag{"\u5ba1\u8bae", "v"}, WordTag{"\u4fb5\u6743", "v"}, WordTag{"\u8d23\u4efb\u6cd5", "n"}, WordTag{"\u8349\u6848", "n"}, WordTag{"\uff0c", "x"}, WordTag{"\u5220\u9664", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u6709\u5173", "vn"}, WordTag{"\u533b\u7597", "n"}, WordTag{"\u635f\u5bb3", "v"}, WordTag{"\u8d23\u4efb", "n"}, WordTag{"\u201c", "x"}, WordTag{"\u4e3e\u8bc1", "v"}, WordTag{"\u5012\u7f6e", "v"}, WordTag{"\u201d", "x"}, WordTag{"\u7684", "uj"}, WordTag{"\u89c4\u5b9a", "n"}, WordTag{"\u3002", "x"}, WordTag{"\u5728", "p"}, WordTag{"\u533b\u60a3", "n"}, WordTag{"\u7ea0\u7eb7", "n"}, WordTag{"\u4e2d", "f"}, WordTag{"\u672c", "r"}, WordTag{"\u5df2", "d"}, WordTag{"\u5904\u4e8e", "v"}, WordTag{"\u5f31\u52bf", "n"}, WordTag{"\u5730\u4f4d", "n"}, WordTag{"\u7684", "uj"}, WordTag{"\u6d88\u8d39\u8005", "n"}, WordTag{"\u7531\u6b64", "c"}, WordTag{"\u5c06", "d"}, WordTag{"\u9677\u5165", "v"}, WordTag{"\u4e07\u52ab\u4e0d\u590d", "i"}, WordTag{"\u7684", "uj"}, WordTag{"\u5883\u5730", "s"}, WordTag{"\u3002", "x"}, WordTag{" ", "x"}},
[]WordTag{WordTag{"\u5927", "a"}},
[]WordTag{WordTag{"hello", "eng"}, WordTag{"你好", "l"}, WordTag{"人们", "n"}, WordTag{"审美", "vn"}, WordTag{"的", "uj"}, WordTag{"观点", "n"}, WordTag{"是", "v"}, WordTag{"不同", "a"}, WordTag{"的", "uj"}},
[]WordTag{WordTag{"很", "zg"}, WordTag{"好", "a"}, WordTag{"但", "c"}, WordTag{"主要", "b"}, WordTag{"是", "v"}, WordTag{"基于", "p"}, WordTag{"网页", "n"}, WordTag{"形式", "n"}},
[]WordTag{WordTag{"hello", "eng"}, WordTag{"你好", "l"}, WordTag{"人们", "n"}, WordTag{"审美", "vn"}, WordTag{"的", "uj"}, WordTag{"观点", "n"}, WordTag{"是", "v"}, WordTag{"不同", "a"}, WordTag{"的", "uj"}},
[]WordTag{WordTag{"为什么", "r"}, WordTag{"我", "r"}, WordTag{"不能", "v"}, WordTag{"拥有", "v"}, WordTag{"想要", "v"}, WordTag{"的", "uj"}, WordTag{"生活", "vn"}},
[]WordTag{WordTag{"后来", "t"}, WordTag{"我", "r"}, WordTag{"才", "d"}},
[]WordTag{WordTag{"此次", "r"}, WordTag{"来", "v"}, WordTag{"中国", "ns"}, WordTag{"是", "v"}, WordTag{"为了", "p"}},
[]WordTag{WordTag{"使用", "v"}, WordTag{"了", "ul"}, WordTag{"它", "r"}, WordTag{"就", "d"}, WordTag{"可以", "c"}, WordTag{"解决", "v"}, WordTag{"一些", "m"}, WordTag{"问题", "n"}},
[]WordTag{WordTag{",", "x"}, WordTag{"使用", "v"}, WordTag{"了", "ul"}, WordTag{"它", "r"}, WordTag{"就", "d"}, WordTag{"可以", "c"}, WordTag{"解决", "v"}, WordTag{"一些", "m"}, WordTag{"问题", "n"}},
[]WordTag{WordTag{"其实", "d"}, WordTag{"使用", "v"}, WordTag{"了", "ul"}, WordTag{"它", "r"}, WordTag{"就", "d"}, WordTag{"可以", "c"}, WordTag{"解决", "v"}, WordTag{"一些", "m"}, WordTag{"问题", "n"}},
[]WordTag{WordTag{"好人", "n"}, WordTag{"使用", "v"}, WordTag{"了", "ul"}, WordTag{"它", "r"}, WordTag{"就", "d"}, WordTag{"可以", "c"}, WordTag{"解决", "v"}, WordTag{"一些", "m"}, WordTag{"问题", "n"}},
[]WordTag{WordTag{"是因为", "c"}, WordTag{"和", "c"}, WordTag{"国家", "n"}},
[]WordTag{WordTag{"老年", "t"}, WordTag{"搜索", "v"}, WordTag{"还", "d"}, WordTag{"支持", "v"}},
[]WordTag{WordTag{"干脆", "d"}, WordTag{"就", "d"}, WordTag{"把", "p"}, WordTag{"那", "r"}, WordTag{"部", "n"}, WordTag{"蒙", "v"}, WordTag{"人", "n"}, WordTag{"的", "uj"}, WordTag{"闲", "n"}, WordTag{"法", "j"}, WordTag{"给", "p"}, WordTag{"废", "v"}, WordTag{"了", "ul"}, WordTag{"拉倒", "v"}, WordTag{"！", "x"}, WordTag{"RT", "eng"}, WordTag{" ", "x"}, WordTag{"@", "x"}, WordTag{"laoshipukong", "eng"}, WordTag{" ", "x"}, WordTag{":", "x"}, WordTag{" ", "x"}, WordTag{"27", "eng"}, WordTag{"日", "m"}, WordTag{"，", "x"}, WordTag{"全国人大常委会", "nt"}, WordTag{"第三次", "m"}, WordTag{"审议", "v"}, WordTag{"侵权", "v"}, WordTag{"责任法", "n"}, WordTag{"草案", "n"}, WordTag{"，", "x"}, WordTag{"删除", "v"}, WordTag{"了", "ul"}, WordTag{"有关", "vn"}, WordTag{"医疗", "n"}, WordTag{"损害", "v"}, WordTag{"责任", "n"}, WordTag{"“", "x"}, WordTag{"举证", "v"}, WordTag{"倒置", "v"}, WordTag{"”", "x"}, WordTag{"的", "uj"}, WordTag{"规定", "n"}, WordTag{"。", "x"}, WordTag{"在", "p"}, WordTag{"医患", "n"}, WordTag{"纠纷", "n"}, WordTag{"中", "f"}, WordTag{"本", "r"}, WordTag{"已", "d"}, WordTag{"处于", "v"}, WordTag{"弱势", "n"}, WordTag{"地位", "n"}, WordTag{"的", "uj"}, WordTag{"消费者", "n"}, WordTag{"由此", "c"}, WordTag{"将", "d"}, WordTag{"陷入", "v"}, WordTag{"万劫不复", "i"}, WordTag{"的", "uj"}, WordTag{"境地", "s"}, WordTag{"。", "x"}, WordTag{" ", "x"}},
[]WordTag{WordTag{"大", "a"}},
[]WordTag{},
[]WordTag{WordTag{"\u4ed6", "r"}, WordTag{"\u8bf4", "v"}, WordTag{"\u7684", "uj"}, WordTag{"\u786e\u5b9e", "ad"}, WordTag{"\u5728", "p"}, WordTag{"\u7406", "n"}},
[]WordTag{WordTag{"\u957f\u6625", "ns"}, WordTag{"\u5e02\u957f", "n"}, WordTag{"\u6625\u8282", "t"}, WordTag{"\u8bb2\u8bdd", "n"}},
[]WordTag{WordTag{"\u7ed3\u5a5a", "v"}, WordTag{"\u7684", "uj"}, WordTag{"\u548c", "c"}, WordTag{"\u5c1a\u672a", "d"}, WordTag{"\u7ed3\u5a5a", "v"}, WordTag{"\u7684", "uj"}},
[]WordTag{WordTag{"\u7ed3\u5408", "v"}, WordTag{"\u6210", "n"}, WordTag{"\u5206\u5b50", "n"}, WordTag{"\u65f6", "n"}},
[]WordTag{WordTag{"\u65c5\u6e38", "vn"}, WordTag{"\u548c", "c"}, WordTag{"\u670d\u52a1", "vn"}, WordTag{"\u662f", "v"}, WordTag{"\u6700\u597d", "a"}, WordTag{"\u7684", "uj"}},
[]WordTag{WordTag{"\u8fd9\u4ef6", "mq"}, WordTag{"\u4e8b\u60c5", "n"}, WordTag{"\u7684\u786e", "d"}, WordTag{"\u662f", "v"}, WordTag{"\u6211", "r"}, WordTag{"\u7684", "uj"}, WordTag{"\u9519", "v"}},
[]WordTag{WordTag{"\u4f9b", "v"}, WordTag{"\u5927\u5bb6", "n"}, WordTag{"\u53c2\u8003", "v"}, WordTag{"\u6307\u6b63", "v"}},
[]WordTag{WordTag{"\u54c8\u5c14\u6ee8", "ns"}, WordTag{"\u653f\u5e9c", "n"}, WordTag{"\u516c\u5e03", "v"}, WordTag{"\u584c", "v"}, WordTag{"\u6865", "n"}, WordTag{"\u539f\u56e0", "n"}},
[]WordTag{WordTag{"\u6211", "r"}, WordTag{"\u5728", "p"}, WordTag{"\u673a\u573a", "n"}, WordTag{"\u5165\u53e3\u5904", "i"}},
[]WordTag{WordTag{"\u90a2", "nr"}, WordTag{"\u6c38", "ns"}, WordTag{"\u81e3", "n"}, WordTag{"\u6444\u5f71", "n"}, WordTag{"\u62a5\u9053", "v"}},
[]WordTag{WordTag{"BP", "eng"}, WordTag{"\u795e\u7ecf\u7f51\u7edc", "n"}, WordTag{"\u5982\u4f55", "r"}, WordTag{"\u8bad\u7ec3", "vn"}, WordTag{"\u624d\u80fd", "v"}, WordTag{"\u5728", "p"}, WordTag{"\u5206\u7c7b", "n"}, WordTag{"\u65f6", "n"}, WordTag{"\u589e\u52a0", "v"}, WordTag{"\u533a\u5206\u5ea6", "n"}, WordTag{"\uff1f", "x"}},
[]WordTag{WordTag{"\u5357\u4eac\u5e02", "ns"}, WordTag{"\u957f\u6c5f\u5927\u6865", "ns"}},
[]WordTag{WordTag{"\u5e94", "v"}, WordTag{"\u4e00\u4e9b", "m"}, WordTag{"\u4f7f\u7528\u8005", "n"}, WordTag{"\u7684", "uj"}, WordTag{"\u5efa\u8bae", "n"}, WordTag{"\uff0c", "x"}, WordTag{"\u4e5f", "d"}, WordTag{"\u4e3a\u4e86", "p"}, WordTag{"\u4fbf\u4e8e", "v"}, WordTag{"\u5229\u7528", "n"}, WordTag{"NiuTrans", "eng"}, WordTag{"\u7528\u4e8e", "v"}, WordTag{"SMT", "eng"}, WordTag{"\u7814\u7a76", "vn"}},
[]WordTag{WordTag{"\u957f\u6625\u5e02", "ns"}, WordTag{"\u957f\u6625", "ns"}, WordTag{"\u836f\u5e97", "n"}},
[]WordTag{WordTag{"\u9093\u9896\u8d85", "nr"}, WordTag{"\u751f\u524d", "t"}, WordTag{"\u6700", "d"}, WordTag{"\u559c\u6b22", "v"}, WordTag{"\u7684", "uj"}, WordTag{"\u8863\u670d", "n"}},
[]WordTag{WordTag{"\u80e1\u9526\u6d9b", "nr"}, WordTag{"\u662f", "v"}, WordTag{"\u70ed\u7231", "a"}, WordTag{"\u4e16\u754c", "n"}, WordTag{"\u548c\u5e73", "nz"}, WordTag{"\u7684", "uj"}, WordTag{"\u653f\u6cbb\u5c40", "n"}, WordTag{"\u5e38\u59d4", "j"}},
[]WordTag{WordTag{"\u7a0b\u5e8f\u5458", "n"}, WordTag{"\u795d", "v"}, WordTag{"\u6d77\u6797", "nz"}, WordTag{"\u548c", "c"}, WordTag{"\u6731", "nr"}, WordTag{"\u4f1a", "v"}, WordTag{"\u9707", "v"}, WordTag{"\u662f", "v"}, WordTag{"\u5728", "p"}, WordTag{"\u5b59", "zg"}, WordTag{"\u5065", "a"}, WordTag{"\u7684", "uj"}, WordTag{"\u5de6\u9762", "f"}, WordTag{"\u548c", "c"}, WordTag{"\u53f3\u9762", "f"}, WordTag{",", "x"}, WordTag{" ", "x"}, WordTag{"\u8303", "nr"}, WordTag{"\u51ef", "nr"}, WordTag{"\u5728", "p"}, WordTag{"\u6700", "d"}, WordTag{"\u53f3\u9762", "f"}, WordTag{".", "x"}, WordTag{"\u518d", "d"}, WordTag{"\u5f80", "zg"}, WordTag{"\u5de6", "m"}, WordTag{"\u662f", "v"}, WordTag{"\u674e", "nr"}, WordTag{"\u677e", "v"}, WordTag{"\u6d2a", "nr"}},
[]WordTag{WordTag{"\u4e00\u6b21\u6027", "d"}, WordTag{"\u4ea4", "v"}, WordTag{"\u591a\u5c11", "m"}, WordTag{"\u94b1", "n"}},
[]WordTag{WordTag{"\u4e24\u5757", "m"}, WordTag{"\u4e94", "m"}, WordTag{"\u4e00\u5957", "m"}, WordTag{"\uff0c", "x"}, WordTag{"\u4e09\u5757", "m"}, WordTag{"\u516b", "m"}, WordTag{"\u4e00\u65a4", "m"}, WordTag{"\uff0c", "x"}, WordTag{"\u56db\u5757", "m"}, WordTag{"\u4e03", "m"}, WordTag{"\u4e00\u672c", "m"}, WordTag{"\uff0c", "x"}, WordTag{"\u4e94\u5757", "m"}, WordTag{"\u516d", "m"}, WordTag{"\u4e00\u6761", "m"}},
[]WordTag{WordTag{"\u5c0f", "a"}, WordTag{"\u548c\u5c1a", "nr"}, WordTag{"\u7559", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u4e00\u4e2a", "m"}, WordTag{"\u50cf", "v"}, WordTag{"\u5927", "a"}, WordTag{"\u548c\u5c1a", "nr"}, WordTag{"\u4e00\u6837", "r"}, WordTag{"\u7684", "uj"}, WordTag{"\u548c\u5c1a\u5934", "nr"}},
[]WordTag{WordTag{"\u6211", "r"}, WordTag{"\u662f", "v"}, WordTag{"\u4e2d\u534e\u4eba\u6c11\u5171\u548c\u56fd", "ns"}, WordTag{"\u516c\u6c11", "n"}, WordTag{";", "x"}, WordTag{"\u6211", "r"}, WordTag{"\u7238\u7238", "n"}, WordTag{"\u662f", "v"}, WordTag{"\u5171\u548c\u515a", "nt"}, WordTag{"\u515a\u5458", "n"}, WordTag{";", "x"}, WordTag{" ", "x"}, WordTag{"\u5730\u94c1", "n"}, WordTag{"\u548c\u5e73\u95e8", "ns"}, WordTag{"\u7ad9", "v"}},
[]WordTag{WordTag{"\u5f20\u6653\u6885", "nr"}, WordTag{"\u53bb", "v"}, WordTag{"\u4eba\u6c11", "n"}, WordTag{"\u533b\u9662", "n"}, WordTag{"\u505a", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u4e2a", "q"}, WordTag{"B\u8d85", "n"}, WordTag{"\u7136\u540e", "c"}, WordTag{"\u53bb", "v"}, WordTag{"\u4e70", "v"}, WordTag{"\u4e86", "ul"}, WordTag{"\u4ef6", "zg"}, WordTag{"T\u6064", "n"}},
[]WordTag{WordTag{"AT&T", "nz"}, WordTag{"\u662f", "v"}, WordTag{"\u4e00\u4ef6", "m"}, WordTag{"\u4e0d\u9519", "a"}, WordTag{"\u7684", "uj"}, WordTag{"\u516c\u53f8", "n"}, WordTag{"\uff0c", "x"}, WordTag{"\u7ed9", "p"}, WordTag{"\u4f60", "r"}, WordTag{"\u53d1", "v"}, WordTag{"offer", "eng"}, WordTag{"\u4e86", "ul"}, WordTag{"\u5417", "y"}, WordTag{"\uff1f", "x"}},
[]WordTag{WordTag{"C++", "nz"}, WordTag{"\u548c", "c"}, WordTag{"c#", "nz"}, WordTag{"\u662f", "v"}, WordTag{"\u4ec0\u4e48", "r"}, WordTag{"\u5173\u7cfb", "n"}, WordTag{"\uff1f", "x"}, WordTag{"11", "eng"}, WordTag{"+", "x"}, WordTag{"122", "eng"}, WordTag{"=", "x"}, WordTag{"133", "eng"}, WordTag{"\uff0c", "x"}, WordTag{"\u662f", "v"}, WordTag{"\u5417", "y"}, WordTag{"\uff1f", "x"}, WordTag{"PI", "eng"}, WordTag{"=", "x"}, WordTag{"3", "eng"}, WordTag{".", "x"}, WordTag{"14159", "eng"}},
[]WordTag{WordTag{"\u4f60", "r"}, WordTag{"\u8ba4\u8bc6", "v"}, WordTag{"\u90a3\u4e2a", "r"}, WordTag{"\u548c", "c"}, WordTag{"\u4e3b\u5e2d", "n"}, WordTag{"\u63e1\u624b", "v"}, WordTag{"\u7684", "uj"}, WordTag{"\u7684\u54e5", "n"}, WordTag{"\u5417", "y"}, WordTag{"\uff1f", "x"}, WordTag{"\u4ed6", "r"}, WordTag{"\u5f00", "v"}, WordTag{"\u4e00\u8f86", "m"}, WordTag{"\u9ed1\u8272", "n"}, WordTag{"\u7684\u58eb", "n"}, WordTag{"\u3002", "x"}},
[]WordTag{WordTag{"\u67aa\u6746\u5b50", "n"}, WordTag{"\u4e2d", "f"}, WordTag{"\u51fa", "v"}, WordTag{"\u653f\u6743", "n"}},
[]WordTag{WordTag{"他", "r"}, WordTag{"说", "v"}, WordTag{"的", "uj"}, WordTag{"确实", "ad"}, WordTag{"在", "p"}, WordTag{"理", "n"}},
[]WordTag{WordTag{"长春", "ns"}, WordTag{"市长", "n"}, WordTag{"春节", "t"}, WordTag{"讲话", "n"}},
[]WordTag{WordTag{"结婚", "v"}, WordTag{"的", "uj"}, WordTag{"和", "c"}, WordTag{"尚未", "d"}, WordTag{"结婚", "v"}, WordTag{"的", "uj"}},
[]WordTag{WordTag{"结合", "v"}, WordTag{"成", "n"}, WordTag{"分子", "n"}, WordTag{"时", "n"}},
[]WordTag{WordTag{"旅游", "vn"}, WordTag{"和", "c"}, WordTag{"服务", "vn"}, WordTag{"是", "v"}, WordTag{"最好", "a"}, WordTag{"的", "uj"}},
[]WordTag{WordTag{"这件", "mq"}, WordTag{"事情", "n"}, WordTag{"的确", "d"}, WordTag{"是", "v"}, WordTag{"我", "r"}, WordTag{"的", "uj"}, WordTag{"错", "v"}},
[]WordTag{WordTag{"供", "v"}, WordTag{"大家", "n"}, WordTag{"参考", "v"}, WordTag{"指正", "v"}},
[]WordTag{WordTag{"哈尔滨", "ns"}, WordTag{"政府", "n"}, WordTag{"公布", "v"}, WordTag{"塌", "v"}, WordTag{"桥", "n"}, WordTag{"原因", "n"}},
[]WordTag{WordTag{"我", "r"}, WordTag{"在", "p"}, WordTag{"机场", "n"}, WordTag{"入口处", "i"}},
[]WordTag{WordTag{"邢", "nr"}, WordTag{"永", "ns"}, WordTag{"臣", "n"}, WordTag{"摄影", "n"}, WordTag{"报道", "v"}},
[]WordTag{WordTag{"BP", "eng"}, WordTag{"神经网络", "n"}, WordTag{"如何", "r"}, WordTag{"训练", "vn"}, WordTag{"才能", "v"}, WordTag{"在", "p"}, WordTag{"分类", "n"}, WordTag{"时", "n"}, WordTag{"增加", "v"}, WordTag{"区分度", "n"}, WordTag{"？", "x"}},
[]WordTag{WordTag{"南京市", "ns"}, WordTag{"长江大桥", "ns"}},
[]WordTag{WordTag{"应", "v"}, WordTag{"一些", "m"}, WordTag{"使用者", "n"}, WordTag{"的", "uj"}, WordTag{"建议", "n"}, WordTag{"，", "x"}, WordTag{"也", "d"}, WordTag{"为了", "p"}, WordTag{"便于", "v"}, WordTag{"利用", "n"}, WordTag{"NiuTrans", "eng"}, WordTag{"用于", "v"}, WordTag{"SMT", "eng"}, WordTag{"研究", "vn"}},
[]WordTag{WordTag{"长春市", "ns"}, WordTag{"长春", "ns"}, WordTag{"药店", "n"}},
[]WordTag{WordTag{"邓颖超", "nr"}, WordTag{"生前", "t"}, WordTag{"最", "d"}, WordTag{"喜欢", "v"}, WordTag{"的", "uj"}, WordTag{"衣服", "n"}},
[]WordTag{WordTag{"胡锦涛", "nr"}, WordTag{"是", "v"}, WordTag{"热爱", "a"}, WordTag{"世界", "n"}, WordTag{"和平", "nz"}, WordTag{"的", "uj"}, WordTag{"政治局", "n"}, WordTag{"常委", "j"}},
[]WordTag{WordTag{"程序员", "n"}, WordTag{"祝", "v"}, WordTag{"海林", "nz"}, WordTag{"和", "c"}, WordTag{"朱", "nr"}, WordTag{"会", "v"}, WordTag{"震", "v"}, WordTag{"是", "v"}, WordTag{"在", "p"}, WordTag{"孙", "zg"}, WordTag{"健", "a"}, WordTag{"的", "uj"}, WordTag{"左面", "f"}, WordTag{"和", "c"}, WordTag{"右面", "f"}, WordTag{",", "x"}, WordTag{" ", "x"}, WordTag{"范", "nr"}, WordTag{"凯", "nr"}, WordTag{"在", "p"}, WordTag{"最", "d"}, WordTag{"右面", "f"}, WordTag{".", "x"}, WordTag{"再", "d"}, WordTag{"往", "zg"}, WordTag{"左", "m"}, WordTag{"是", "v"}, WordTag{"李", "nr"}, WordTag{"松", "v"}, WordTag{"洪", "nr"}},
[]WordTag{WordTag{"一次性", "d"}, WordTag{"交", "v"}, WordTag{"多少", "m"}, WordTag{"钱", "n"}},
[]WordTag{WordTag{"两块", "m"}, WordTag{"五", "m"}, WordTag{"一套", "m"}, WordTag{"，", "x"}, WordTag{"三块", "m"}, WordTag{"八", "m"}, WordTag{"一斤", "m"}, WordTag{"，", "x"}, WordTag{"四块", "m"}, WordTag{"七", "m"}, WordTag{"一本", "m"}, WordTag{"，", "x"}, WordTag{"五块", "m"}, WordTag{"六", "m"}, WordTag{"一条", "m"}},
[]WordTag{WordTag{"小", "a"}, WordTag{"和尚", "nr"}, WordTag{"留", "v"}, WordTag{"了", "ul"}, WordTag{"一个", "m"}, WordTag{"像", "v"}, WordTag{"大", "a"}, WordTag{"和尚", "nr"}, WordTag{"一样", "r"}, WordTag{"的", "uj"}, WordTag{"和尚头", "nr"}},
[]WordTag{WordTag{"我", "r"}, WordTag{"是", "v"}, WordTag{"中华人民共和国", "ns"}, WordTag{"公民", "n"}, WordTag{";", "x"}, WordTag{"我", "r"}, WordTag{"爸爸", "n"}, WordTag{"是", "v"}, WordTag{"共和党", "nt"}, WordTag{"党员", "n"}, WordTag{";", "x"}, WordTag{" ", "x"}, WordTag{"地铁", "n"}, WordTag{"和平门", "ns"}, WordTag{"站", "v"}},
[]WordTag{WordTag{"张晓梅", "nr"}, WordTag{"去", "v"}, WordTag{"人民", "n"}, WordTag{"医院", "n"}, WordTag{"做", "v"}, WordTag{"了", "ul"}, WordTag{"个", "q"}, WordTag{"B超", "n"}, WordTag{"然后", "c"}, WordTag{"去", "v"}, WordTag{"买", "v"}, WordTag{"了", "ul"}, WordTag{"件", "zg"}, WordTag{"T恤", "n"}},
[]WordTag{WordTag{"AT&T", "nz"}, WordTag{"是", "v"}, WordTag{"一件", "m"}, WordTag{"不错", "a"}, WordTag{"的", "uj"}, WordTag{"公司", "n"}, WordTag{"，", "x"}, WordTag{"给", "p"}, WordTag{"你", "r"}, WordTag{"发", "v"}, WordTag{"offer", "eng"}, WordTag{"了", "ul"}, WordTag{"吗", "y"}, WordTag{"？", "x"}},
[]WordTag{WordTag{"C++", "nz"}, WordTag{"和", "c"}, WordTag{"c#", "nz"}, WordTag{"是", "v"}, WordTag{"什么", "r"}, WordTag{"关系", "n"}, WordTag{"？", "x"}, WordTag{"11", "eng"}, WordTag{"+", "x"}, WordTag{"122", "eng"}, WordTag{"=", "x"}, WordTag{"133", "eng"}, WordTag{"，", "x"}, WordTag{"是", "v"}, WordTag{"吗", "y"}, WordTag{"？", "x"}, WordTag{"PI", "eng"}, WordTag{"=", "x"}, WordTag{"3", "eng"}, WordTag{".", "x"}, WordTag{"14159", "eng"}},
[]WordTag{WordTag{"你", "r"}, WordTag{"认识", "v"}, WordTag{"那个", "r"}, WordTag{"和", "c"}, WordTag{"主席", "n"}, WordTag{"握手", "v"}, WordTag{"的", "uj"}, WordTag{"的哥", "n"}, WordTag{"吗", "y"}, WordTag{"？", "x"}, WordTag{"他", "r"}, WordTag{"开", "v"}, WordTag{"一辆", "m"}, WordTag{"黑色", "n"}, WordTag{"的士", "n"}, WordTag{"。", "x"}},
[]WordTag{WordTag{"枪杆子", "n"}, WordTag{"中", "f"}, WordTag{"出", "v"}, WordTag{"政权", "n"}},
}
)
func chanToArray(ch chan WordTag) []WordTag {
result := make([]WordTag, 0)
for word := range ch {
result = append(result, word)
}
return result
}
func TestCut(t *testing.T) {
jiebago.SetDictionary("../dict.txt")
SetDictionary("../dict.txt")
for index, content := range test_contents {
result := Cut(content, true)
result := chanToArray(Cut(content, true))
if len(defaultCutResult[index]) != len(result) {
t.Error(content)
}
@@ -280,7 +288,7 @@ func TestCut(t *testing.T) {
t.Error(content)
}
}
result = Cut(content, false)
result = chanToArray(Cut(content, false))
if len(noHMMCutResult[index]) != len(result) {
t.Error(content)
}
@@ -297,15 +305,15 @@ func TestBug132(t *testing.T) {
/*
https://github.com/fxsjy/jieba/issues/132
*/
jiebago.SetDictionary("../dict.txt")
SetDictionary("../dict.txt")
sentence := "又跛又啞"
cutResult := []WordTag{
WordTag{"\u53c8", "d"},
WordTag{"\u8ddb", "a"},
WordTag{"\u53c8", "d"},
WordTag{"\u555e", "v"},
WordTag{"又", "d"},
WordTag{"跛", "a"},
WordTag{"又", "d"},
WordTag{"啞", "v"},
}
result := Cut(sentence, true)
result := chanToArray(Cut(sentence, true))
if len(cutResult) != len(result) {
t.Error(result)
}
@@ -320,24 +328,24 @@ func TestBug137(t *testing.T) {
/*
https://github.com/fxsjy/jieba/issues/137
*/
jiebago.SetDictionary("../dict.txt")
SetDictionary("../dict.txt")
sentence := "前港督衛奕信在八八年十月宣布成立中央政策研究組"
cutResult := []WordTag{
WordTag{"\u524d", "f"},
WordTag{"\u6e2f\u7763", "n"},
WordTag{"\u885b\u5955", "z"},
WordTag{"\u4fe1", "n"},
WordTag{"\u5728", "p"},
WordTag{"\u516b\u516b\u5e74", "m"},
WordTag{"\u5341\u6708", "t"},
WordTag{"\u5ba3\u5e03", "v"},
WordTag{"\u6210\u7acb", "v"},
WordTag{"\u4e2d\u592e", "n"},
WordTag{"\u653f\u7b56", "n"},
WordTag{"\u7814\u7a76", "vn"},
WordTag{"\u7d44", "x"},
WordTag{"前", "f"},
WordTag{"港督", "n"},
WordTag{"衛奕", "z"},
WordTag{"信", "n"},
WordTag{"在", "p"},
WordTag{"八八年", "m"},
WordTag{"十月", "t"},
WordTag{"宣布", "v"},
WordTag{"成立", "v"},
WordTag{"中央", "n"},
WordTag{"政策", "n"},
WordTag{"研究", "vn"},
WordTag{"組", "x"},
}
result := Cut(sentence, true)
result := chanToArray(Cut(sentence, true))
if len(cutResult) != len(result) {
t.Error(result)
}
@@ -349,50 +357,50 @@ func TestBug137(t *testing.T) {
}
func TestUserDict(t *testing.T) {
jiebago.SetDictionary("../dict.txt")
SetDictionary("../dict.txt")
jiebago.LoadUserDict("../userdict.txt")
sentence := "李小福是创新办主任也是云计算方面的专家; 什么是八一双鹿例如我输入一个带“韩玉赏鉴”的标题，在自定义词库中也增加了此词为N类型"
cutResult := []WordTag{
WordTag{"\u674e\u5c0f\u798f", "nr"},
WordTag{"\u662f", "v"},
WordTag{"\u521b\u65b0\u529e", "i"},
WordTag{"\u4e3b\u4efb", "b"},
WordTag{"\u4e5f", "d"},
WordTag{"\u662f", "v"},
WordTag{"\u4e91\u8ba1\u7b97", "x"},
WordTag{"\u65b9\u9762", "n"},
WordTag{"\u7684", "uj"},
WordTag{"\u4e13\u5bb6", "n"},
WordTag{"李小福", "nr"},
WordTag{"是", "v"},
WordTag{"创新办", "i"},
WordTag{"主任", "b"},
WordTag{"也", "d"},
WordTag{"是", "v"},
WordTag{"云计算", "x"},
WordTag{"方面", "n"},
WordTag{"的", "uj"},
WordTag{"专家", "n"},
WordTag{";", "x"},
WordTag{" ", "x"},
WordTag{"\u4ec0\u4e48", "r"},
WordTag{"\u662f", "v"},
WordTag{"\u516b\u4e00\u53cc\u9e7f", "nz"},
WordTag{"\u4f8b\u5982", "v"},
WordTag{"\u6211", "r"},
WordTag{"\u8f93\u5165", "v"},
WordTag{"\u4e00\u4e2a", "m"},
WordTag{"\u5e26", "v"},
WordTag{"\u201c", "x"},
WordTag{"\u97e9\u7389\u8d4f\u9274", "nz"},
WordTag{"\u201d", "x"},
WordTag{"\u7684", "uj"},
WordTag{"\u6807\u9898", "n"},
WordTag{"\uff0c", "x"},
WordTag{"\u5728", "p"},
WordTag{"\u81ea\u5b9a\u4e49\u8bcd", "n"},
WordTag{"\u5e93\u4e2d", "nrt"},
WordTag{"\u4e5f", "d"},
WordTag{"\u589e\u52a0", "v"},
WordTag{"\u4e86", "ul"},
WordTag{"\u6b64", "r"},
WordTag{"\u8bcd", "n"},
WordTag{"\u4e3a", "p"},
WordTag{"什么", "r"},
WordTag{"是", "v"},
WordTag{"八一双鹿", "nz"},
WordTag{"例如", "v"},
WordTag{"我", "r"},
WordTag{"输入", "v"},
WordTag{"一个", "m"},
WordTag{"带", "v"},
WordTag{"“", "x"},
WordTag{"韩玉赏鉴", "nz"},
WordTag{"”", "x"},
WordTag{"的", "uj"},
WordTag{"标题", "n"},
WordTag{"，", "x"},
WordTag{"在", "p"},
WordTag{"自定义词", "n"},
WordTag{"库中", "nrt"},
WordTag{"也", "d"},
WordTag{"增加", "v"},
WordTag{"了", "ul"},
WordTag{"此", "r"},
WordTag{"词", "n"},
WordTag{"为", "p"},
WordTag{"N", "eng"},
WordTag{"\u7c7b\u578b", "n"},
}
WordTag{"类型", "n"}}
result := Cut(sentence, true)
result := chanToArray(Cut(sentence, true))
if len(cutResult) != len(result) {
t.Error(result)
}

522
posseg/prob_emit.go
View File

File diff suppressed because it is too large Load Diff

514
posseg/prob_start.go
View File

@@ -1,264 +1,264 @@
package posseg
var (
ProbStart = make(map[StateTag]float64)
probStart = make(map[stateTag]float64)
)
func init() {
ProbStart[StateTag{'B', "a"}] = -4.762305214596967
ProbStart[StateTag{'B', "ad"}] = -6.680066036784177
ProbStart[StateTag{'B', "ag"}] = -3.14e+100
ProbStart[StateTag{'B', "an"}] = -8.697083223018778
ProbStart[StateTag{'B', "b"}] = -5.018374362109218
ProbStart[StateTag{'B', "bg"}] = -3.14e+100
ProbStart[StateTag{'B', "c"}] = -3.423880184954888
ProbStart[StateTag{'B', "d"}] = -3.9750475297585357
ProbStart[StateTag{'B', "df"}] = -8.888974230828882
ProbStart[StateTag{'B', "dg"}] = -3.14e+100
ProbStart[StateTag{'B', "e"}] = -8.563551830394255
ProbStart[StateTag{'B', "en"}] = -3.14e+100
ProbStart[StateTag{'B', "f"}] = -5.491630418482717
ProbStart[StateTag{'B', "g"}] = -3.14e+100
ProbStart[StateTag{'B', "h"}] = -13.533365129970255
ProbStart[StateTag{'B', "i"}] = -6.1157847275557105
ProbStart[StateTag{'B', "in"}] = -3.14e+100
ProbStart[StateTag{'B', "j"}] = -5.0576191284681915
ProbStart[StateTag{'B', "jn"}] = -3.14e+100
ProbStart[StateTag{'B', "k"}] = -3.14e+100
ProbStart[StateTag{'B', "l"}] = -4.905883584659895
ProbStart[StateTag{'B', "ln"}] = -3.14e+100
ProbStart[StateTag{'B', "m"}] = -3.6524299819046386
ProbStart[StateTag{'B', "mg"}] = -3.14e+100
ProbStart[StateTag{'B', "mq"}] = -6.78695300139688
ProbStart[StateTag{'B', "n"}] = -1.6966257797548328
ProbStart[StateTag{'B', "ng"}] = -3.14e+100
ProbStart[StateTag{'B', "nr"}] = -2.2310495913769506
ProbStart[StateTag{'B', "nrfg"}] = -5.873722175405573
ProbStart[StateTag{'B', "nrt"}] = -4.985642733519195
ProbStart[StateTag{'B', "ns"}] = -2.8228438314969213
ProbStart[StateTag{'B', "nt"}] = -4.846091668182416
ProbStart[StateTag{'B', "nz"}] = -3.94698846057672
ProbStart[StateTag{'B', "o"}] = -8.433498702146057
ProbStart[StateTag{'B', "p"}] = -4.200984132085048
ProbStart[StateTag{'B', "q"}] = -6.998123858956596
ProbStart[StateTag{'B', "qe"}] = -3.14e+100
ProbStart[StateTag{'B', "qg"}] = -3.14e+100
ProbStart[StateTag{'B', "r"}] = -3.4098187790818413
ProbStart[StateTag{'B', "rg"}] = -3.14e+100
ProbStart[StateTag{'B', "rr"}] = -12.434752841302146
ProbStart[StateTag{'B', "rz"}] = -7.946116471570005
ProbStart[StateTag{'B', "s"}] = -5.522673590839954
ProbStart[StateTag{'B', "t"}] = -3.3647479094528574
ProbStart[StateTag{'B', "tg"}] = -3.14e+100
ProbStart[StateTag{'B', "u"}] = -9.163917277503234
ProbStart[StateTag{'B', "ud"}] = -3.14e+100
ProbStart[StateTag{'B', "ug"}] = -3.14e+100
ProbStart[StateTag{'B', "uj"}] = -3.14e+100
ProbStart[StateTag{'B', "ul"}] = -3.14e+100
ProbStart[StateTag{'B', "uv"}] = -3.14e+100
ProbStart[StateTag{'B', "uz"}] = -3.14e+100
ProbStart[StateTag{'B', "v"}] = -2.6740584874265685
ProbStart[StateTag{'B', "vd"}] = -9.044728760238115
ProbStart[StateTag{'B', "vg"}] = -3.14e+100
ProbStart[StateTag{'B', "vi"}] = -12.434752841302146
ProbStart[StateTag{'B', "vn"}] = -4.3315610890163585
ProbStart[StateTag{'B', "vq"}] = -12.147070768850364
ProbStart[StateTag{'B', "w"}] = -3.14e+100
ProbStart[StateTag{'B', "x"}] = -3.14e+100
ProbStart[StateTag{'B', "y"}] = -9.844485675856319
ProbStart[StateTag{'B', "yg"}] = -3.14e+100
ProbStart[StateTag{'B', "z"}] = -7.045681111485645
ProbStart[StateTag{'B', "zg"}] = -3.14e+100
ProbStart[StateTag{'E', "a"}] = -3.14e+100
ProbStart[StateTag{'E', "ad"}] = -3.14e+100
ProbStart[StateTag{'E', "ag"}] = -3.14e+100
ProbStart[StateTag{'E', "an"}] = -3.14e+100
ProbStart[StateTag{'E', "b"}] = -3.14e+100
ProbStart[StateTag{'E', "bg"}] = -3.14e+100
ProbStart[StateTag{'E', "c"}] = -3.14e+100
ProbStart[StateTag{'E', "d"}] = -3.14e+100
ProbStart[StateTag{'E', "df"}] = -3.14e+100
ProbStart[StateTag{'E', "dg"}] = -3.14e+100
ProbStart[StateTag{'E', "e"}] = -3.14e+100
ProbStart[StateTag{'E', "en"}] = -3.14e+100
ProbStart[StateTag{'E', "f"}] = -3.14e+100
ProbStart[StateTag{'E', "g"}] = -3.14e+100
ProbStart[StateTag{'E', "h"}] = -3.14e+100
ProbStart[StateTag{'E', "i"}] = -3.14e+100
ProbStart[StateTag{'E', "in"}] = -3.14e+100
ProbStart[StateTag{'E', "j"}] = -3.14e+100
ProbStart[StateTag{'E', "jn"}] = -3.14e+100
ProbStart[StateTag{'E', "k"}] = -3.14e+100
ProbStart[StateTag{'E', "l"}] = -3.14e+100
ProbStart[StateTag{'E', "ln"}] = -3.14e+100
ProbStart[StateTag{'E', "m"}] = -3.14e+100
ProbStart[StateTag{'E', "mg"}] = -3.14e+100
ProbStart[StateTag{'E', "mq"}] = -3.14e+100
ProbStart[StateTag{'E', "n"}] = -3.14e+100
ProbStart[StateTag{'E', "ng"}] = -3.14e+100
ProbStart[StateTag{'E', "nr"}] = -3.14e+100
ProbStart[StateTag{'E', "nrfg"}] = -3.14e+100
ProbStart[StateTag{'E', "nrt"}] = -3.14e+100
ProbStart[StateTag{'E', "ns"}] = -3.14e+100
ProbStart[StateTag{'E', "nt"}] = -3.14e+100
ProbStart[StateTag{'E', "nz"}] = -3.14e+100
ProbStart[StateTag{'E', "o"}] = -3.14e+100
ProbStart[StateTag{'E', "p"}] = -3.14e+100
ProbStart[StateTag{'E', "q"}] = -3.14e+100
ProbStart[StateTag{'E', "qe"}] = -3.14e+100
ProbStart[StateTag{'E', "qg"}] = -3.14e+100
ProbStart[StateTag{'E', "r"}] = -3.14e+100
ProbStart[StateTag{'E', "rg"}] = -3.14e+100
ProbStart[StateTag{'E', "rr"}] = -3.14e+100
ProbStart[StateTag{'E', "rz"}] = -3.14e+100
ProbStart[StateTag{'E', "s"}] = -3.14e+100
ProbStart[StateTag{'E', "t"}] = -3.14e+100
ProbStart[StateTag{'E', "tg"}] = -3.14e+100
ProbStart[StateTag{'E', "u"}] = -3.14e+100
ProbStart[StateTag{'E', "ud"}] = -3.14e+100
ProbStart[StateTag{'E', "ug"}] = -3.14e+100
ProbStart[StateTag{'E', "uj"}] = -3.14e+100
ProbStart[StateTag{'E', "ul"}] = -3.14e+100
ProbStart[StateTag{'E', "uv"}] = -3.14e+100
ProbStart[StateTag{'E', "uz"}] = -3.14e+100
ProbStart[StateTag{'E', "v"}] = -3.14e+100
ProbStart[StateTag{'E', "vd"}] = -3.14e+100
ProbStart[StateTag{'E', "vg"}] = -3.14e+100
ProbStart[StateTag{'E', "vi"}] = -3.14e+100
ProbStart[StateTag{'E', "vn"}] = -3.14e+100
ProbStart[StateTag{'E', "vq"}] = -3.14e+100
ProbStart[StateTag{'E', "w"}] = -3.14e+100
ProbStart[StateTag{'E', "x"}] = -3.14e+100
ProbStart[StateTag{'E', "y"}] = -3.14e+100
ProbStart[StateTag{'E', "yg"}] = -3.14e+100
ProbStart[StateTag{'E', "z"}] = -3.14e+100
ProbStart[StateTag{'E', "zg"}] = -3.14e+100
ProbStart[StateTag{'M', "a"}] = -3.14e+100
ProbStart[StateTag{'M', "ad"}] = -3.14e+100
ProbStart[StateTag{'M', "ag"}] = -3.14e+100
ProbStart[StateTag{'M', "an"}] = -3.14e+100
ProbStart[StateTag{'M', "b"}] = -3.14e+100
ProbStart[StateTag{'M', "bg"}] = -3.14e+100
ProbStart[StateTag{'M', "c"}] = -3.14e+100
ProbStart[StateTag{'M', "d"}] = -3.14e+100
ProbStart[StateTag{'M', "df"}] = -3.14e+100
ProbStart[StateTag{'M', "dg"}] = -3.14e+100
ProbStart[StateTag{'M', "e"}] = -3.14e+100
ProbStart[StateTag{'M', "en"}] = -3.14e+100
ProbStart[StateTag{'M', "f"}] = -3.14e+100
ProbStart[StateTag{'M', "g"}] = -3.14e+100
ProbStart[StateTag{'M', "h"}] = -3.14e+100
ProbStart[StateTag{'M', "i"}] = -3.14e+100
ProbStart[StateTag{'M', "in"}] = -3.14e+100
ProbStart[StateTag{'M', "j"}] = -3.14e+100
ProbStart[StateTag{'M', "jn"}] = -3.14e+100
ProbStart[StateTag{'M', "k"}] = -3.14e+100
ProbStart[StateTag{'M', "l"}] = -3.14e+100
ProbStart[StateTag{'M', "ln"}] = -3.14e+100
ProbStart[StateTag{'M', "m"}] = -3.14e+100
ProbStart[StateTag{'M', "mg"}] = -3.14e+100
ProbStart[StateTag{'M', "mq"}] = -3.14e+100
ProbStart[StateTag{'M', "n"}] = -3.14e+100
ProbStart[StateTag{'M', "ng"}] = -3.14e+100
ProbStart[StateTag{'M', "nr"}] = -3.14e+100
ProbStart[StateTag{'M', "nrfg"}] = -3.14e+100
ProbStart[StateTag{'M', "nrt"}] = -3.14e+100
ProbStart[StateTag{'M', "ns"}] = -3.14e+100
ProbStart[StateTag{'M', "nt"}] = -3.14e+100
ProbStart[StateTag{'M', "nz"}] = -3.14e+100
ProbStart[StateTag{'M', "o"}] = -3.14e+100
ProbStart[StateTag{'M', "p"}] = -3.14e+100
ProbStart[StateTag{'M', "q"}] = -3.14e+100
ProbStart[StateTag{'M', "qe"}] = -3.14e+100
ProbStart[StateTag{'M', "qg"}] = -3.14e+100
ProbStart[StateTag{'M', "r"}] = -3.14e+100
ProbStart[StateTag{'M', "rg"}] = -3.14e+100
ProbStart[StateTag{'M', "rr"}] = -3.14e+100
ProbStart[StateTag{'M', "rz"}] = -3.14e+100
ProbStart[StateTag{'M', "s"}] = -3.14e+100
ProbStart[StateTag{'M', "t"}] = -3.14e+100
ProbStart[StateTag{'M', "tg"}] = -3.14e+100
ProbStart[StateTag{'M', "u"}] = -3.14e+100
ProbStart[StateTag{'M', "ud"}] = -3.14e+100
ProbStart[StateTag{'M', "ug"}] = -3.14e+100
ProbStart[StateTag{'M', "uj"}] = -3.14e+100
ProbStart[StateTag{'M', "ul"}] = -3.14e+100
ProbStart[StateTag{'M', "uv"}] = -3.14e+100
ProbStart[StateTag{'M', "uz"}] = -3.14e+100
ProbStart[StateTag{'M', "v"}] = -3.14e+100
ProbStart[StateTag{'M', "vd"}] = -3.14e+100
ProbStart[StateTag{'M', "vg"}] = -3.14e+100
ProbStart[StateTag{'M', "vi"}] = -3.14e+100
ProbStart[StateTag{'M', "vn"}] = -3.14e+100
ProbStart[StateTag{'M', "vq"}] = -3.14e+100
ProbStart[StateTag{'M', "w"}] = -3.14e+100
ProbStart[StateTag{'M', "x"}] = -3.14e+100
ProbStart[StateTag{'M', "y"}] = -3.14e+100
ProbStart[StateTag{'M', "yg"}] = -3.14e+100
ProbStart[StateTag{'M', "z"}] = -3.14e+100
ProbStart[StateTag{'M', "zg"}] = -3.14e+100
ProbStart[StateTag{'S', "a"}] = -3.9025396831295227
ProbStart[StateTag{'S', "ad"}] = -11.048458480182255
ProbStart[StateTag{'S', "ag"}] = -6.954113917960154
ProbStart[StateTag{'S', "an"}] = -12.84021794941031
ProbStart[StateTag{'S', "b"}] = -6.472888763970454
ProbStart[StateTag{'S', "bg"}] = -3.14e+100
ProbStart[StateTag{'S', "c"}] = -4.786966795861212
ProbStart[StateTag{'S', "d"}] = -3.903919764181873
ProbStart[StateTag{'S', "df"}] = -3.14e+100
ProbStart[StateTag{'S', "dg"}] = -8.948397651299683
ProbStart[StateTag{'S', "e"}] = -5.942513006281674
ProbStart[StateTag{'S', "en"}] = -3.14e+100
ProbStart[StateTag{'S', "f"}] = -5.194820249981676
ProbStart[StateTag{'S', "g"}] = -6.507826815331734
ProbStart[StateTag{'S', "h"}] = -8.650563207383884
ProbStart[StateTag{'S', "i"}] = -3.14e+100
ProbStart[StateTag{'S', "in"}] = -3.14e+100
ProbStart[StateTag{'S', "j"}] = -4.911992119644354
ProbStart[StateTag{'S', "jn"}] = -3.14e+100
ProbStart[StateTag{'S', "k"}] = -6.940320595827818
ProbStart[StateTag{'S', "l"}] = -3.14e+100
ProbStart[StateTag{'S', "ln"}] = -3.14e+100
ProbStart[StateTag{'S', "m"}] = -3.269200652116097
ProbStart[StateTag{'S', "mg"}] = -10.825314928868044
ProbStart[StateTag{'S', "mq"}] = -3.14e+100
ProbStart[StateTag{'S', "n"}] = -3.8551483897645107
ProbStart[StateTag{'S', "ng"}] = -4.913434861102905
ProbStart[StateTag{'S', "nr"}] = -4.483663103956885
ProbStart[StateTag{'S', "nrfg"}] = -3.14e+100
ProbStart[StateTag{'S', "nrt"}] = -3.14e+100
ProbStart[StateTag{'S', "ns"}] = -3.14e+100
ProbStart[StateTag{'S', "nt"}] = -12.147070768850364
ProbStart[StateTag{'S', "nz"}] = -3.14e+100
ProbStart[StateTag{'S', "o"}] = -8.464460927750023
ProbStart[StateTag{'S', "p"}] = -2.9868401813596317
ProbStart[StateTag{'S', "q"}] = -4.888658618255058
ProbStart[StateTag{'S', "qe"}] = -3.14e+100
ProbStart[StateTag{'S', "qg"}] = -3.14e+100
ProbStart[StateTag{'S', "r"}] = -2.7635336784127853
ProbStart[StateTag{'S', "rg"}] = -10.275268591948773
ProbStart[StateTag{'S', "rr"}] = -3.14e+100
ProbStart[StateTag{'S', "rz"}] = -3.14e+100
ProbStart[StateTag{'S', "s"}] = -3.14e+100
ProbStart[StateTag{'S', "t"}] = -3.14e+100
ProbStart[StateTag{'S', "tg"}] = -6.272842531880403
ProbStart[StateTag{'S', "u"}] = -6.940320595827818
ProbStart[StateTag{'S', "ud"}] = -7.728230161053767
ProbStart[StateTag{'S', "ug"}] = -7.5394037026636855
ProbStart[StateTag{'S', "uj"}] = -6.85251045118004
ProbStart[StateTag{'S', "ul"}] = -8.4153713175535
ProbStart[StateTag{'S', "uv"}] = -8.15808672228609
ProbStart[StateTag{'S', "uz"}] = -9.299258625372996
ProbStart[StateTag{'S', "v"}] = -3.053292303412302
ProbStart[StateTag{'S', "vd"}] = -3.14e+100
ProbStart[StateTag{'S', "vg"}] = -5.9430181843676895
ProbStart[StateTag{'S', "vi"}] = -3.14e+100
ProbStart[StateTag{'S', "vn"}] = -11.453923588290419
ProbStart[StateTag{'S', "vq"}] = -3.14e+100
ProbStart[StateTag{'S', "w"}] = -3.14e+100
ProbStart[StateTag{'S', "x"}] = -8.427419656069674
ProbStart[StateTag{'S', "y"}] = -6.1970794699489575
ProbStart[StateTag{'S', "yg"}] = -13.533365129970255
ProbStart[StateTag{'S', "z"}] = -3.14e+100
ProbStart[StateTag{'S', "zg"}] = -3.14e+100
probStart[stateTag{'B', "a"}] = -4.762305214596967
probStart[stateTag{'B', "ad"}] = -6.680066036784177
probStart[stateTag{'B', "ag"}] = -3.14e+100
probStart[stateTag{'B', "an"}] = -8.697083223018778
probStart[stateTag{'B', "b"}] = -5.018374362109218
probStart[stateTag{'B', "bg"}] = -3.14e+100
probStart[stateTag{'B', "c"}] = -3.423880184954888
probStart[stateTag{'B', "d"}] = -3.9750475297585357
probStart[stateTag{'B', "df"}] = -8.888974230828882
probStart[stateTag{'B', "dg"}] = -3.14e+100
probStart[stateTag{'B', "e"}] = -8.563551830394255
probStart[stateTag{'B', "en"}] = -3.14e+100
probStart[stateTag{'B', "f"}] = -5.491630418482717
probStart[stateTag{'B', "g"}] = -3.14e+100
probStart[stateTag{'B', "h"}] = -13.533365129970255
probStart[stateTag{'B', "i"}] = -6.1157847275557105
probStart[stateTag{'B', "in"}] = -3.14e+100
probStart[stateTag{'B', "j"}] = -5.0576191284681915
probStart[stateTag{'B', "jn"}] = -3.14e+100
probStart[stateTag{'B', "k"}] = -3.14e+100
probStart[stateTag{'B', "l"}] = -4.905883584659895
probStart[stateTag{'B', "ln"}] = -3.14e+100
probStart[stateTag{'B', "m"}] = -3.6524299819046386
probStart[stateTag{'B', "mg"}] = -3.14e+100
probStart[stateTag{'B', "mq"}] = -6.78695300139688
probStart[stateTag{'B', "n"}] = -1.6966257797548328
probStart[stateTag{'B', "ng"}] = -3.14e+100
probStart[stateTag{'B', "nr"}] = -2.2310495913769506
probStart[stateTag{'B', "nrfg"}] = -5.873722175405573
probStart[stateTag{'B', "nrt"}] = -4.985642733519195
probStart[stateTag{'B', "ns"}] = -2.8228438314969213
probStart[stateTag{'B', "nt"}] = -4.846091668182416
probStart[stateTag{'B', "nz"}] = -3.94698846057672
probStart[stateTag{'B', "o"}] = -8.433498702146057
probStart[stateTag{'B', "p"}] = -4.200984132085048
probStart[stateTag{'B', "q"}] = -6.998123858956596
probStart[stateTag{'B', "qe"}] = -3.14e+100
probStart[stateTag{'B', "qg"}] = -3.14e+100
probStart[stateTag{'B', "r"}] = -3.4098187790818413
probStart[stateTag{'B', "rg"}] = -3.14e+100
probStart[stateTag{'B', "rr"}] = -12.434752841302146
probStart[stateTag{'B', "rz"}] = -7.946116471570005
probStart[stateTag{'B', "s"}] = -5.522673590839954
probStart[stateTag{'B', "t"}] = -3.3647479094528574
probStart[stateTag{'B', "tg"}] = -3.14e+100
probStart[stateTag{'B', "u"}] = -9.163917277503234
probStart[stateTag{'B', "ud"}] = -3.14e+100
probStart[stateTag{'B', "ug"}] = -3.14e+100
probStart[stateTag{'B', "uj"}] = -3.14e+100
probStart[stateTag{'B', "ul"}] = -3.14e+100
probStart[stateTag{'B', "uv"}] = -3.14e+100
probStart[stateTag{'B', "uz"}] = -3.14e+100
probStart[stateTag{'B', "v"}] = -2.6740584874265685
probStart[stateTag{'B', "vd"}] = -9.044728760238115
probStart[stateTag{'B', "vg"}] = -3.14e+100
probStart[stateTag{'B', "vi"}] = -12.434752841302146
probStart[stateTag{'B', "vn"}] = -4.3315610890163585
probStart[stateTag{'B', "vq"}] = -12.147070768850364
probStart[stateTag{'B', "w"}] = -3.14e+100
probStart[stateTag{'B', "x"}] = -3.14e+100
probStart[stateTag{'B', "y"}] = -9.844485675856319
probStart[stateTag{'B', "yg"}] = -3.14e+100
probStart[stateTag{'B', "z"}] = -7.045681111485645
probStart[stateTag{'B', "zg"}] = -3.14e+100
probStart[stateTag{'E', "a"}] = -3.14e+100
probStart[stateTag{'E', "ad"}] = -3.14e+100
probStart[stateTag{'E', "ag"}] = -3.14e+100
probStart[stateTag{'E', "an"}] = -3.14e+100
probStart[stateTag{'E', "b"}] = -3.14e+100
probStart[stateTag{'E', "bg"}] = -3.14e+100
probStart[stateTag{'E', "c"}] = -3.14e+100
probStart[stateTag{'E', "d"}] = -3.14e+100
probStart[stateTag{'E', "df"}] = -3.14e+100
probStart[stateTag{'E', "dg"}] = -3.14e+100
probStart[stateTag{'E', "e"}] = -3.14e+100
probStart[stateTag{'E', "en"}] = -3.14e+100
probStart[stateTag{'E', "f"}] = -3.14e+100
probStart[stateTag{'E', "g"}] = -3.14e+100
probStart[stateTag{'E', "h"}] = -3.14e+100
probStart[stateTag{'E', "i"}] = -3.14e+100
probStart[stateTag{'E', "in"}] = -3.14e+100
probStart[stateTag{'E', "j"}] = -3.14e+100
probStart[stateTag{'E', "jn"}] = -3.14e+100
probStart[stateTag{'E', "k"}] = -3.14e+100
probStart[stateTag{'E', "l"}] = -3.14e+100
probStart[stateTag{'E', "ln"}] = -3.14e+100
probStart[stateTag{'E', "m"}] = -3.14e+100
probStart[stateTag{'E', "mg"}] = -3.14e+100
probStart[stateTag{'E', "mq"}] = -3.14e+100
probStart[stateTag{'E', "n"}] = -3.14e+100
probStart[stateTag{'E', "ng"}] = -3.14e+100
probStart[stateTag{'E', "nr"}] = -3.14e+100
probStart[stateTag{'E', "nrfg"}] = -3.14e+100
probStart[stateTag{'E', "nrt"}] = -3.14e+100
probStart[stateTag{'E', "ns"}] = -3.14e+100
probStart[stateTag{'E', "nt"}] = -3.14e+100
probStart[stateTag{'E', "nz"}] = -3.14e+100
probStart[stateTag{'E', "o"}] = -3.14e+100
probStart[stateTag{'E', "p"}] = -3.14e+100
probStart[stateTag{'E', "q"}] = -3.14e+100
probStart[stateTag{'E', "qe"}] = -3.14e+100
probStart[stateTag{'E', "qg"}] = -3.14e+100
probStart[stateTag{'E', "r"}] = -3.14e+100
probStart[stateTag{'E', "rg"}] = -3.14e+100
probStart[stateTag{'E', "rr"}] = -3.14e+100
probStart[stateTag{'E', "rz"}] = -3.14e+100
probStart[stateTag{'E', "s"}] = -3.14e+100
probStart[stateTag{'E', "t"}] = -3.14e+100
probStart[stateTag{'E', "tg"}] = -3.14e+100
probStart[stateTag{'E', "u"}] = -3.14e+100
probStart[stateTag{'E', "ud"}] = -3.14e+100
probStart[stateTag{'E', "ug"}] = -3.14e+100
probStart[stateTag{'E', "uj"}] = -3.14e+100
probStart[stateTag{'E', "ul"}] = -3.14e+100
probStart[stateTag{'E', "uv"}] = -3.14e+100
probStart[stateTag{'E', "uz"}] = -3.14e+100
probStart[stateTag{'E', "v"}] = -3.14e+100
probStart[stateTag{'E', "vd"}] = -3.14e+100
probStart[stateTag{'E', "vg"}] = -3.14e+100
probStart[stateTag{'E', "vi"}] = -3.14e+100
probStart[stateTag{'E', "vn"}] = -3.14e+100
probStart[stateTag{'E', "vq"}] = -3.14e+100
probStart[stateTag{'E', "w"}] = -3.14e+100
probStart[stateTag{'E', "x"}] = -3.14e+100
probStart[stateTag{'E', "y"}] = -3.14e+100
probStart[stateTag{'E', "yg"}] = -3.14e+100
probStart[stateTag{'E', "z"}] = -3.14e+100
probStart[stateTag{'E', "zg"}] = -3.14e+100
probStart[stateTag{'M', "a"}] = -3.14e+100
probStart[stateTag{'M', "ad"}] = -3.14e+100
probStart[stateTag{'M', "ag"}] = -3.14e+100
probStart[stateTag{'M', "an"}] = -3.14e+100
probStart[stateTag{'M', "b"}] = -3.14e+100
probStart[stateTag{'M', "bg"}] = -3.14e+100
probStart[stateTag{'M', "c"}] = -3.14e+100
probStart[stateTag{'M', "d"}] = -3.14e+100
probStart[stateTag{'M', "df"}] = -3.14e+100
probStart[stateTag{'M', "dg"}] = -3.14e+100
probStart[stateTag{'M', "e"}] = -3.14e+100
probStart[stateTag{'M', "en"}] = -3.14e+100
probStart[stateTag{'M', "f"}] = -3.14e+100
probStart[stateTag{'M', "g"}] = -3.14e+100
probStart[stateTag{'M', "h"}] = -3.14e+100
probStart[stateTag{'M', "i"}] = -3.14e+100
probStart[stateTag{'M', "in"}] = -3.14e+100
probStart[stateTag{'M', "j"}] = -3.14e+100
probStart[stateTag{'M', "jn"}] = -3.14e+100
probStart[stateTag{'M', "k"}] = -3.14e+100
probStart[stateTag{'M', "l"}] = -3.14e+100
probStart[stateTag{'M', "ln"}] = -3.14e+100
probStart[stateTag{'M', "m"}] = -3.14e+100
probStart[stateTag{'M', "mg"}] = -3.14e+100
probStart[stateTag{'M', "mq"}] = -3.14e+100
probStart[stateTag{'M', "n"}] = -3.14e+100
probStart[stateTag{'M', "ng"}] = -3.14e+100
probStart[stateTag{'M', "nr"}] = -3.14e+100
probStart[stateTag{'M', "nrfg"}] = -3.14e+100
probStart[stateTag{'M', "nrt"}] = -3.14e+100
probStart[stateTag{'M', "ns"}] = -3.14e+100
probStart[stateTag{'M', "nt"}] = -3.14e+100
probStart[stateTag{'M', "nz"}] = -3.14e+100
probStart[stateTag{'M', "o"}] = -3.14e+100
probStart[stateTag{'M', "p"}] = -3.14e+100
probStart[stateTag{'M', "q"}] = -3.14e+100
probStart[stateTag{'M', "qe"}] = -3.14e+100
probStart[stateTag{'M', "qg"}] = -3.14e+100
probStart[stateTag{'M', "r"}] = -3.14e+100
probStart[stateTag{'M', "rg"}] = -3.14e+100
probStart[stateTag{'M', "rr"}] = -3.14e+100
probStart[stateTag{'M', "rz"}] = -3.14e+100
probStart[stateTag{'M', "s"}] = -3.14e+100
probStart[stateTag{'M', "t"}] = -3.14e+100
probStart[stateTag{'M', "tg"}] = -3.14e+100
probStart[stateTag{'M', "u"}] = -3.14e+100
probStart[stateTag{'M', "ud"}] = -3.14e+100
probStart[stateTag{'M', "ug"}] = -3.14e+100
probStart[stateTag{'M', "uj"}] = -3.14e+100
probStart[stateTag{'M', "ul"}] = -3.14e+100
probStart[stateTag{'M', "uv"}] = -3.14e+100
probStart[stateTag{'M', "uz"}] = -3.14e+100
probStart[stateTag{'M', "v"}] = -3.14e+100
probStart[stateTag{'M', "vd"}] = -3.14e+100
probStart[stateTag{'M', "vg"}] = -3.14e+100
probStart[stateTag{'M', "vi"}] = -3.14e+100
probStart[stateTag{'M', "vn"}] = -3.14e+100
probStart[stateTag{'M', "vq"}] = -3.14e+100
probStart[stateTag{'M', "w"}] = -3.14e+100
probStart[stateTag{'M', "x"}] = -3.14e+100
probStart[stateTag{'M', "y"}] = -3.14e+100
probStart[stateTag{'M', "yg"}] = -3.14e+100
probStart[stateTag{'M', "z"}] = -3.14e+100
probStart[stateTag{'M', "zg"}] = -3.14e+100
probStart[stateTag{'S', "a"}] = -3.9025396831295227
probStart[stateTag{'S', "ad"}] = -11.048458480182255
probStart[stateTag{'S', "ag"}] = -6.954113917960154
probStart[stateTag{'S', "an"}] = -12.84021794941031
probStart[stateTag{'S', "b"}] = -6.472888763970454
probStart[stateTag{'S', "bg"}] = -3.14e+100
probStart[stateTag{'S', "c"}] = -4.786966795861212
probStart[stateTag{'S', "d"}] = -3.903919764181873
probStart[stateTag{'S', "df"}] = -3.14e+100
probStart[stateTag{'S', "dg"}] = -8.948397651299683
probStart[stateTag{'S', "e"}] = -5.942513006281674
probStart[stateTag{'S', "en"}] = -3.14e+100
probStart[stateTag{'S', "f"}] = -5.194820249981676
probStart[stateTag{'S', "g"}] = -6.507826815331734
probStart[stateTag{'S', "h"}] = -8.650563207383884
probStart[stateTag{'S', "i"}] = -3.14e+100
probStart[stateTag{'S', "in"}] = -3.14e+100
probStart[stateTag{'S', "j"}] = -4.911992119644354
probStart[stateTag{'S', "jn"}] = -3.14e+100
probStart[stateTag{'S', "k"}] = -6.940320595827818
probStart[stateTag{'S', "l"}] = -3.14e+100
probStart[stateTag{'S', "ln"}] = -3.14e+100
probStart[stateTag{'S', "m"}] = -3.269200652116097
probStart[stateTag{'S', "mg"}] = -10.825314928868044
probStart[stateTag{'S', "mq"}] = -3.14e+100
probStart[stateTag{'S', "n"}] = -3.8551483897645107
probStart[stateTag{'S', "ng"}] = -4.913434861102905
probStart[stateTag{'S', "nr"}] = -4.483663103956885
probStart[stateTag{'S', "nrfg"}] = -3.14e+100
probStart[stateTag{'S', "nrt"}] = -3.14e+100
probStart[stateTag{'S', "ns"}] = -3.14e+100
probStart[stateTag{'S', "nt"}] = -12.147070768850364
probStart[stateTag{'S', "nz"}] = -3.14e+100
probStart[stateTag{'S', "o"}] = -8.464460927750023
probStart[stateTag{'S', "p"}] = -2.9868401813596317
probStart[stateTag{'S', "q"}] = -4.888658618255058
probStart[stateTag{'S', "qe"}] = -3.14e+100
probStart[stateTag{'S', "qg"}] = -3.14e+100
probStart[stateTag{'S', "r"}] = -2.7635336784127853
probStart[stateTag{'S', "rg"}] = -10.275268591948773
probStart[stateTag{'S', "rr"}] = -3.14e+100
probStart[stateTag{'S', "rz"}] = -3.14e+100
probStart[stateTag{'S', "s"}] = -3.14e+100
probStart[stateTag{'S', "t"}] = -3.14e+100
probStart[stateTag{'S', "tg"}] = -6.272842531880403
probStart[stateTag{'S', "u"}] = -6.940320595827818
probStart[stateTag{'S', "ud"}] = -7.728230161053767
probStart[stateTag{'S', "ug"}] = -7.5394037026636855
probStart[stateTag{'S', "uj"}] = -6.85251045118004
probStart[stateTag{'S', "ul"}] = -8.4153713175535
probStart[stateTag{'S', "uv"}] = -8.15808672228609
probStart[stateTag{'S', "uz"}] = -9.299258625372996
probStart[stateTag{'S', "v"}] = -3.053292303412302
probStart[stateTag{'S', "vd"}] = -3.14e+100
probStart[stateTag{'S', "vg"}] = -5.9430181843676895
probStart[stateTag{'S', "vi"}] = -3.14e+100
probStart[stateTag{'S', "vn"}] = -11.453923588290419
probStart[stateTag{'S', "vq"}] = -3.14e+100
probStart[stateTag{'S', "w"}] = -3.14e+100
probStart[stateTag{'S', "x"}] = -8.427419656069674
probStart[stateTag{'S', "y"}] = -6.1970794699489575
probStart[stateTag{'S', "yg"}] = -13.533365129970255
probStart[stateTag{'S', "z"}] = -3.14e+100
probStart[stateTag{'S', "zg"}] = -3.14e+100
}

10958
posseg/prob_trans.go
View File

File diff suppressed because it is too large Load Diff

69
posseg/viterbi.go
View File

@@ -5,37 +5,35 @@ import (
"sort"
)
const MIN_FLOAT = -3.14e100
type StateTag struct {
type stateTag struct {
State byte
Tag string
}
func (st StateTag) String() string {
func (st stateTag) String() string {
return fmt.Sprintf("(%q, %s)", st.State, st.Tag)
}
func emptyStateTag() StateTag {
return StateTag{' ', ""}
func emptyStateTag() stateTag {
return stateTag{' ', ""}
}
type ProbState struct {
type probState struct {
Prob float64
ST StateTag
ST stateTag
}
func (ps ProbState) String() string {
func (ps probState) String() string {
return fmt.Sprintf("(%v: %f)", ps.ST, ps.Prob)
}
type ProbStates []ProbState
type probStates []probState
func (pss ProbStates) Len() int {
func (pss probStates) Len() int {
return len(pss)
}
func (pss ProbStates) Less(i, j int) bool {
func (pss probStates) Less(i, j int) bool {
if pss[i].Prob == pss[j].Prob {
if pss[i].ST.State == pss[j].ST.State {
return pss[i].ST.Tag < pss[j].ST.Tag
@@ -45,39 +43,38 @@ func (pss ProbStates) Less(i, j int) bool {
return pss[i].Prob < pss[j].Prob
}
func (pss ProbStates) Swap(i, j int) {
func (pss probStates) Swap(i, j int) {
pss[i], pss[j] = pss[j], pss[i]
}
func Viterbi(obs []rune) (float64, []StateTag) {
func viterbi(obs []rune) (float64, []stateTag) {
obsLength := len(obs)
V := make([]map[StateTag]float64, obsLength)
V[0] = make(map[StateTag]float64)
mem_path := make([]map[StateTag]StateTag, obsLength)
mem_path[0] = make(map[StateTag]StateTag)
// all_states := ProbTransKeys
ys := CharStateTab.Get(obs[0]) // default is all_states
V := make([]map[stateTag]float64, obsLength)
V[0] = make(map[stateTag]float64)
mem_path := make([]map[stateTag]stateTag, obsLength)
mem_path[0] = make(map[stateTag]stateTag)
ys := charStateTab.get(obs[0]) // default is all_states
for _, y := range ys {
V[0][y] = ProbEmit[y].Get(obs[0]) + ProbStart[y]
V[0][y] = probEmit[y].get(obs[0]) + probStart[y]
mem_path[0][y] = emptyStateTag()
}
for t := 1; t < obsLength; t++ {
prev_states := make([]StateTag, 0)
prev_states := make([]stateTag, 0)
for x, _ := range mem_path[t-1] {
if len(ProbTrans[x]) > 0 {
if len(probTrans[x]) > 0 {
prev_states = append(prev_states, x)
}
}
//use Go's map to implement Python's Set()
prev_states_expect_next := make(map[StateTag]StateTag)
prev_states_expect_next := make(map[stateTag]stateTag)
for _, x := range prev_states {
for y, _ := range ProbTrans[x] {
for y, _ := range probTrans[x] {
prev_states_expect_next[y] = y
}
}
tmp_obs_states := CharStateTab.Get(obs[t])
tmp_obs_states := charStateTab.get(obs[t])
obs_states := make([]StateTag, 0)
obs_states := make([]stateTag, 0)
for index, _ := range tmp_obs_states {
if _, ok := prev_states_expect_next[tmp_obs_states[index]]; ok {
obs_states = append(obs_states, tmp_obs_states[index])
@@ -89,15 +86,15 @@ func Viterbi(obs []rune) (float64, []StateTag) {
}
}
if len(obs_states) == 0 {
obs_states = ProbTransKeys
obs_states = probTransKeys
}
mem_path[t] = make(map[StateTag]StateTag)
V[t] = make(map[StateTag]float64)
mem_path[t] = make(map[stateTag]stateTag) // TODO: value needed or not?
V[t] = make(map[stateTag]float64)
for _, y := range obs_states {
pss := make(ProbStates, 0)
pss := make(probStates, 0)
for _, y0 := range prev_states {
ps := ProbState{
Prob: V[t-1][y0] + ProbTrans[y0].Get(y) + ProbEmit[y].Get(obs[t]),
ps := probState{
Prob: V[t-1][y0] + probTrans[y0].Get(y) + probEmit[y].get(obs[t]),
ST: y0}
pss = append(pss, ps)
}
@@ -106,17 +103,17 @@ func Viterbi(obs []rune) (float64, []StateTag) {
mem_path[t][y] = pss[0].ST
}
}
last := make(ProbStates, 0)
last := make(probStates, 0)
length := len(mem_path)
vlength := len(V)
for y, _ := range mem_path[length-1] {
ps := ProbState{Prob: V[vlength-1][y], ST: y}
ps := probState{Prob: V[vlength-1][y], ST: y}
last = append(last, ps)
}
sort.Sort(sort.Reverse(last))
prob := last[0].Prob
state := last[0].ST
route := make([]StateTag, len(obs))
route := make([]stateTag, len(obs))
i := obsLength - 1
for {
if i < 0 {

47
posseg/viterbi_test.go
View File

@@ -5,33 +5,33 @@ import (
)
var (
route1 = []StateTag{
StateTag{'B', "nr"},
StateTag{'M', "nr"},
StateTag{'E', "nr"},
StateTag{'S', "v"},
StateTag{'B', "v"},
StateTag{'E', "v"},
StateTag{'B', "n"},
StateTag{'M', "n"},
StateTag{'E', "n"},
StateTag{'S', "d"},
StateTag{'S', "v"},
StateTag{'S', "n"},
StateTag{'B', "v"},
StateTag{'E', "v"},
StateTag{'B', "nr"},
StateTag{'M', "nr"},
StateTag{'M', "nr"},
StateTag{'M', "nr"},
StateTag{'E', "nr"},
StateTag{'S', "zg"}}
route1 = []stateTag{
stateTag{'B', "nr"},
stateTag{'M', "nr"},
stateTag{'E', "nr"},
stateTag{'S', "v"},
stateTag{'B', "v"},
stateTag{'E', "v"},
stateTag{'B', "n"},
stateTag{'M', "n"},
stateTag{'E', "n"},
stateTag{'S', "d"},
stateTag{'S', "v"},
stateTag{'S', "n"},
stateTag{'B', "v"},
stateTag{'E', "v"},
stateTag{'B', "nr"},
stateTag{'M', "nr"},
stateTag{'M', "nr"},
stateTag{'M', "nr"},
stateTag{'E', "nr"},
stateTag{'S', "zg"}}
)
func TestViterbi(t *testing.T) {
ss := "李小福是创新办主任也是云计算方面的专家;"
prob, route := Viterbi([]rune(ss))
if prob != MIN_FLOAT {
prob, route := viterbi([]rune(ss))
if prob != MinFloat {
t.Error(prob)
}
if len(route) != len(route1) {
@@ -42,5 +42,4 @@ func TestViterbi(t *testing.T) {
t.Error(route[index])
}
}
}

24
tokenize.go
View File

@@ -1,36 +1,36 @@
package jiebago
type Token struct {
type token struct {
Word string
Start int
End int
}
func Tokenize(sentence string, mode string, HMM bool) []Token {
tokens := make([]Token, 0)
// Return words with position.
func Tokenize(sentence string, mode string, HMM bool) []token {
tokens := make([]token, 0)
start := 0
var width int
if mode == "default" {
for _, word := range Cut(sentence, false, HMM) {
for word := range Cut(sentence, false, HMM) {
if mode == "default" {
width = len([]rune(word))
tokens = append(tokens, Token{word, start, start + width})
tokens = append(tokens, token{word, start, start + width})
start += width
}
} else {
for _, word := range Cut(sentence, false, HMM) {
} else {
runes := []rune(word)
width = len(runes)
for _, step := range []int{2, 3} {
if width > step {
for i := 0; i < width-step+1; i++ {
gram := string(runes[i : i+step])
if _, ok := TT.Freq[gram]; ok {
tokens = append(tokens, Token{gram, start + i, start + i + step})
if _, ok := Trie.Freq[gram]; ok {
tokens = append(tokens, token{gram, start + i, start + i + step})
}
}
}
}
tokens = append(tokens, Token{word, start, start + width})
tokens = append(tokens, token{word, start, start + width})
}
}
return tokens

712
tokenize_test.go
View File

@@ -3,365 +3,365 @@ package jiebago
import "testing"
var (
result = [][]Token{
[]Token{Token{"\u8fd9\u662f", 0, 2}, Token{"\u4e00\u4e2a", 2, 4}, Token{"\u4f38\u624b\u4e0d\u89c1\u4e94\u6307", 4, 10}, Token{"\u7684", 10, 11}, Token{"\u9ed1\u591c", 11, 13}, Token{"\u3002", 13, 14}, Token{"\u6211", 14, 15}, Token{"\u53eb", 15, 16}, Token{"\u5b59\u609f\u7a7a", 16, 19}, Token{"\uff0c", 19, 20}, Token{"\u6211", 20, 21}, Token{"\u7231", 21, 22}, Token{"\u5317\u4eac", 22, 24}, Token{"\uff0c", 24, 25}, Token{"\u6211", 25, 26}, Token{"\u7231", 26, 27}, Token{"Python", 27, 33}, Token{"\u548c", 33, 34}, Token{"C++", 34, 37}, Token{"\u3002", 37, 38}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u4e0d", 1, 2}, Token{"\u559c\u6b22", 2, 4}, Token{"\u65e5\u672c", 4, 6}, Token{"\u548c\u670d", 6, 8}, Token{"\u3002", 8, 9}},
[]Token{Token{"\u96f7\u7334", 0, 2}, Token{"\u56de\u5f52", 2, 4}, Token{"\u4eba\u95f4", 4, 6}, Token{"\u3002", 6, 7}},
[]Token{Token{"\u5de5\u4fe1\u5904", 0, 3}, Token{"\u5973\u5e72\u4e8b", 3, 6}, Token{"\u6bcf\u6708", 6, 8}, Token{"\u7ecf\u8fc7", 8, 10}, Token{"\u4e0b\u5c5e", 10, 12}, Token{"\u79d1\u5ba4", 12, 14}, Token{"\u90fd", 14, 15}, Token{"\u8981", 15, 16}, Token{"\u4eb2\u53e3", 16, 18}, Token{"\u4ea4\u4ee3", 18, 20}, Token{"24", 20, 22}, Token{"\u53e3", 22, 23}, Token{"\u4ea4\u6362\u673a", 23, 26}, Token{"\u7b49", 26, 27}, Token{"\u6280\u672f\u6027", 27, 30}, Token{"\u5668\u4ef6", 30, 32}, Token{"\u7684", 32, 33}, Token{"\u5b89\u88c5", 33, 35}, Token{"\u5de5\u4f5c", 35, 37}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u9700\u8981", 1, 3}, Token{"\u5ec9\u79df\u623f", 3, 6}},
[]Token{Token{"\u6c38\u548c", 0, 2}, Token{"\u670d\u88c5", 2, 4}, Token{"\u9970\u54c1", 4, 6}, Token{"\u6709\u9650\u516c\u53f8", 6, 10}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u7231", 1, 2}, Token{"\u5317\u4eac", 2, 4}, Token{"\u5929\u5b89\u95e8", 4, 7}},
[]Token{Token{"abc", 0, 3}},
[]Token{Token{"\u9690", 0, 1}, Token{"\u9a6c\u5c14\u53ef\u592b", 1, 5}},
[]Token{Token{"\u96f7\u7334", 0, 2}, Token{"\u662f", 2, 3}, Token{"\u4e2a", 3, 4}, Token{"\u597d", 4, 5}, Token{"\u7f51\u7ad9", 5, 7}},
[]Token{Token{"\u201c", 0, 1}, Token{"Microsoft", 1, 10}, Token{"\u201d", 10, 11}, Token{"\u4e00\u8bcd", 11, 13}, Token{"\u7531", 13, 14}, Token{"\u201c", 14, 15}, Token{"MICROcomputer", 15, 28}, Token{"\uff08", 28, 29}, Token{"\u5fae\u578b", 29, 31}, Token{"\u8ba1\u7b97\u673a", 31, 34}, Token{"\uff09", 34, 35}, Token{"\u201d", 35, 36}, Token{"\u548c", 36, 37}, Token{"\u201c", 37, 38}, Token{"SOFTware", 38, 46}, Token{"\uff08", 46, 47}, Token{"\u8f6f\u4ef6", 47, 49}, Token{"\uff09", 49, 50}, Token{"\u201d", 50, 51}, Token{"\u4e24", 51, 52}, Token{"\u90e8\u5206", 52, 54}, Token{"\u7ec4\u6210", 54, 56}},
[]Token{Token{"\u8349\u6ce5\u9a6c", 0, 3}, Token{"\u548c", 3, 4}, Token{"\u6b3a\u5b9e", 4, 6}, Token{"\u9a6c", 6, 7}, Token{"\u662f", 7, 8}, Token{"\u4eca\u5e74", 8, 10}, Token{"\u7684", 10, 11}, Token{"\u6d41\u884c", 11, 13}, Token{"\u8bcd\u6c47", 13, 15}},
[]Token{Token{"\u4f0a\u85e4", 0, 2}, Token{"\u6d0b\u534e\u5802", 2, 5}, Token{"\u603b\u5e9c", 5, 7}, Token{"\u5e97", 7, 8}},
[]Token{Token{"\u4e2d\u56fd\u79d1\u5b66\u9662\u8ba1\u7b97\u6280\u672f\u7814\u7a76\u6240", 0, 12}},
[]Token{Token{"\u7f57\u5bc6\u6b27", 0, 3}, Token{"\u4e0e", 3, 4}, Token{"\u6731\u4e3d\u53f6", 4, 7}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u8d2d\u4e70", 1, 3}, Token{"\u4e86", 3, 4}, Token{"\u9053\u5177", 4, 6}, Token{"\u548c", 6, 7}, Token{"\u670d\u88c5", 7, 9}},
[]Token{Token{"PS", 0, 2}, Token{":", 2, 3}, Token{" ", 3, 4}, Token{"\u6211", 4, 5}, Token{"\u89c9\u5f97", 5, 7}, Token{"\u5f00\u6e90", 7, 9}, Token{"\u6709", 9, 10}, Token{"\u4e00\u4e2a", 10, 12}, Token{"\u597d\u5904", 12, 14}, Token{"\uff0c", 14, 15}, Token{"\u5c31\u662f", 15, 17}, Token{"\u80fd\u591f", 17, 19}, Token{"\u6566\u4fc3", 19, 21}, Token{"\u81ea\u5df1", 21, 23}, Token{"\u4e0d\u65ad\u6539\u8fdb", 23, 27}, Token{"\uff0c", 27, 28}, Token{"\u907f\u514d", 28, 30}, Token{"\u655e\u5e1a", 30, 32}, Token{"\u81ea\u73cd", 32, 34}},
[]Token{Token{"\u6e56\u5317\u7701", 0, 3}, Token{"\u77f3\u9996\u5e02", 3, 6}},
[]Token{Token{"\u6e56\u5317\u7701", 0, 3}, Token{"\u5341\u5830\u5e02", 3, 6}},
[]Token{Token{"\u603b\u7ecf\u7406", 0, 3}, Token{"\u5b8c\u6210", 3, 5}, Token{"\u4e86", 5, 6}, Token{"\u8fd9\u4ef6", 6, 8}, Token{"\u4e8b\u60c5", 8, 10}},
[]Token{Token{"\u7535\u8111", 0, 2}, Token{"\u4fee\u597d", 2, 4}, Token{"\u4e86", 4, 5}},
[]Token{Token{"\u505a\u597d", 0, 2}, Token{"\u4e86", 2, 3}, Token{"\u8fd9\u4ef6", 3, 5}, Token{"\u4e8b\u60c5", 5, 7}, Token{"\u5c31", 7, 8}, Token{"\u4e00\u4e86\u767e\u4e86", 8, 12}, Token{"\u4e86", 12, 13}},
[]Token{Token{"\u4eba\u4eec", 0, 2}, Token{"\u5ba1\u7f8e", 2, 4}, Token{"\u7684", 4, 5}, Token{"\u89c2\u70b9", 5, 7}, Token{"\u662f", 7, 8}, Token{"\u4e0d\u540c", 8, 10}, Token{"\u7684", 10, 11}},
[]Token{Token{"\u6211\u4eec", 0, 2}, Token{"\u4e70", 2, 3}, Token{"\u4e86", 3, 4}, Token{"\u4e00\u4e2a", 4, 6}, Token{"\u7f8e\u7684", 6, 8}, Token{"\u7a7a\u8c03", 8, 10}},
[]Token{Token{"\u7ebf\u7a0b", 0, 2}, Token{"\u521d\u59cb\u5316", 2, 5}, Token{"\u65f6", 5, 6}, Token{"\u6211\u4eec", 6, 8}, Token{"\u8981", 8, 9}, Token{"\u6ce8\u610f", 9, 11}},
[]Token{Token{"\u4e00\u4e2a", 0, 2}, Token{"\u5206\u5b50", 2, 4}, Token{"\u662f", 4, 5}, Token{"\u7531", 5, 6}, Token{"\u597d\u591a", 6, 8}, Token{"\u539f\u5b50", 8, 10}, Token{"\u7ec4\u7ec7", 10, 12}, Token{"\u6210", 12, 13}, Token{"\u7684", 13, 14}},
[]Token{Token{"\u795d", 0, 1}, Token{"\u4f60", 1, 2}, Token{"\u9a6c\u5230\u529f\u6210", 2, 6}},
[]Token{Token{"\u4ed6", 0, 1}, Token{"\u6389", 1, 2}, Token{"\u8fdb", 2, 3}, Token{"\u4e86", 3, 4}, Token{"\u65e0\u5e95\u6d1e", 4, 7}, Token{"\u91cc", 7, 8}},
[]Token{Token{"\u4e2d\u56fd", 0, 2}, Token{"\u7684", 2, 3}, Token{"\u9996\u90fd", 3, 5}, Token{"\u662f", 5, 6}, Token{"\u5317\u4eac", 6, 8}},
[]Token{Token{"\u5b59\u541b\u610f", 0, 3}},
[]Token{Token{"\u5916\u4ea4\u90e8", 0, 3}, Token{"\u53d1\u8a00\u4eba", 3, 6}, Token{"\u9a6c\u671d\u65ed", 6, 9}},
[]Token{Token{"\u9886\u5bfc\u4eba", 0, 3}, Token{"\u4f1a\u8bae", 3, 5}, Token{"\u548c", 5, 6}, Token{"\u7b2c\u56db\u5c4a", 6, 9}, Token{"\u4e1c\u4e9a", 9, 11}, Token{"\u5cf0\u4f1a", 11, 13}},
[]Token{Token{"\u5728", 0, 1}, Token{"\u8fc7\u53bb", 1, 3}, Token{"\u7684", 3, 4}, Token{"\u8fd9", 4, 5}, Token{"\u4e94\u5e74", 5, 7}},
[]Token{Token{"\u8fd8", 0, 1}, Token{"\u9700\u8981", 1, 3}, Token{"\u5f88\u957f", 3, 5}, Token{"\u7684", 5, 6}, Token{"\u8def", 6, 7}, Token{"\u8981", 7, 8}, Token{"\u8d70", 8, 9}},
[]Token{Token{"60", 0, 2}, Token{"\u5468\u5e74", 2, 4}, Token{"\u9996\u90fd", 4, 6}, Token{"\u9605\u5175", 6, 8}},
[]Token{Token{"\u4f60\u597d", 0, 2}, Token{"\u4eba\u4eec", 2, 4}, Token{"\u5ba1\u7f8e", 4, 6}, Token{"\u7684", 6, 7}, Token{"\u89c2\u70b9", 7, 9}, Token{"\u662f", 9, 10}, Token{"\u4e0d\u540c", 10, 12}, Token{"\u7684", 12, 13}},
[]Token{Token{"\u4e70", 0, 1}, Token{"\u6c34\u679c", 1, 3}, Token{"\u7136\u540e", 3, 5}, Token{"\u6765", 5, 6}, Token{"\u4e16\u535a\u56ed", 6, 9}},
[]Token{Token{"\u4e70", 0, 1}, Token{"\u6c34\u679c", 1, 3}, Token{"\u7136\u540e", 3, 5}, Token{"\u53bb", 5, 6}, Token{"\u4e16\u535a\u56ed", 6, 9}},
[]Token{Token{"\u4f46\u662f", 0, 2}, Token{"\u540e\u6765", 2, 4}, Token{"\u6211", 4, 5}, Token{"\u624d", 5, 6}, Token{"\u77e5\u9053", 6, 8}, Token{"\u4f60", 8, 9}, Token{"\u662f", 9, 10}, Token{"\u5bf9", 10, 11}, Token{"\u7684", 11, 12}},
[]Token{Token{"\u5b58\u5728", 0, 2}, Token{"\u5373", 2, 3}, Token{"\u5408\u7406", 3, 5}},
[]Token{Token{"\u7684", 0, 1}, Token{"\u7684", 1, 2}, Token{"\u7684", 2, 3}, Token{"\u7684", 3, 4}, Token{"\u7684", 4, 5}, Token{"\u5728", 5, 6}, Token{"\u7684", 6, 7}, Token{"\u7684", 7, 8}, Token{"\u7684", 8, 9}, Token{"\u7684", 9, 10}, Token{"\u5c31", 10, 11}, Token{"\u4ee5", 11, 12}, Token{"\u548c", 12, 13}, Token{"\u548c", 13, 14}, Token{"\u548c", 14, 15}},
[]Token{Token{"I", 0, 1}, Token{" ", 1, 2}, Token{"love", 2, 6}, Token{"\u4f60", 6, 7}, Token{"\uff0c", 7, 8}, Token{"\u4e0d\u4ee5\u4e3a\u803b", 8, 12}, Token{"\uff0c", 12, 13}, Token{"\u53cd", 13, 14}, Token{"\u4ee5\u4e3a", 14, 16}, Token{"rong", 16, 20}},
[]Token{Token{"\u56e0", 0, 1}},
[]Token{},
[]Token{Token{"hello", 0, 5}, Token{"\u4f60\u597d", 5, 7}, Token{"\u4eba\u4eec", 7, 9}, Token{"\u5ba1\u7f8e", 9, 11}, Token{"\u7684", 11, 12}, Token{"\u89c2\u70b9", 12, 14}, Token{"\u662f", 14, 15}, Token{"\u4e0d\u540c", 15, 17}, Token{"\u7684", 17, 18}},
[]Token{Token{"\u5f88", 0, 1}, Token{"\u597d", 1, 2}, Token{"\u4f46", 2, 3}, Token{"\u4e3b\u8981", 3, 5}, Token{"\u662f", 5, 6}, Token{"\u57fa\u4e8e", 6, 8}, Token{"\u7f51\u9875", 8, 10}, Token{"\u5f62\u5f0f", 10, 12}},
[]Token{Token{"hello", 0, 5}, Token{"\u4f60\u597d", 5, 7}, Token{"\u4eba\u4eec", 7, 9}, Token{"\u5ba1\u7f8e", 9, 11}, Token{"\u7684", 11, 12}, Token{"\u89c2\u70b9", 12, 14}, Token{"\u662f", 14, 15}, Token{"\u4e0d\u540c", 15, 17}, Token{"\u7684", 17, 18}},
[]Token{Token{"\u4e3a\u4ec0\u4e48", 0, 3}, Token{"\u6211", 3, 4}, Token{"\u4e0d\u80fd", 4, 6}, Token{"\u62e5\u6709", 6, 8}, Token{"\u60f3\u8981", 8, 10}, Token{"\u7684", 10, 11}, Token{"\u751f\u6d3b", 11, 13}},
[]Token{Token{"\u540e\u6765", 0, 2}, Token{"\u6211", 2, 3}, Token{"\u624d", 3, 4}},
[]Token{Token{"\u6b64\u6b21", 0, 2}, Token{"\u6765", 2, 3}, Token{"\u4e2d\u56fd", 3, 5}, Token{"\u662f", 5, 6}, Token{"\u4e3a\u4e86", 6, 8}},
[]Token{Token{"\u4f7f\u7528", 0, 2}, Token{"\u4e86", 2, 3}, Token{"\u5b83", 3, 4}, Token{"\u5c31", 4, 5}, Token{"\u53ef\u4ee5", 5, 7}, Token{"\u89e3\u51b3", 7, 9}, Token{"\u4e00\u4e9b", 9, 11}, Token{"\u95ee\u9898", 11, 13}},
[]Token{Token{",", 0, 1}, Token{"\u4f7f\u7528", 1, 3}, Token{"\u4e86", 3, 4}, Token{"\u5b83", 4, 5}, Token{"\u5c31", 5, 6}, Token{"\u53ef\u4ee5", 6, 8}, Token{"\u89e3\u51b3", 8, 10}, Token{"\u4e00\u4e9b", 10, 12}, Token{"\u95ee\u9898", 12, 14}},
[]Token{Token{"\u5176\u5b9e", 0, 2}, Token{"\u4f7f\u7528", 2, 4}, Token{"\u4e86", 4, 5}, Token{"\u5b83", 5, 6}, Token{"\u5c31", 6, 7}, Token{"\u53ef\u4ee5", 7, 9}, Token{"\u89e3\u51b3", 9, 11}, Token{"\u4e00\u4e9b", 11, 13}, Token{"\u95ee\u9898", 13, 15}},
[]Token{Token{"\u597d\u4eba", 0, 2}, Token{"\u4f7f\u7528", 2, 4}, Token{"\u4e86", 4, 5}, Token{"\u5b83", 5, 6}, Token{"\u5c31", 6, 7}, Token{"\u53ef\u4ee5", 7, 9}, Token{"\u89e3\u51b3", 9, 11}, Token{"\u4e00\u4e9b", 11, 13}, Token{"\u95ee\u9898", 13, 15}},
[]Token{Token{"\u662f\u56e0\u4e3a", 0, 3}, Token{"\u548c", 3, 4}, Token{"\u56fd\u5bb6", 4, 6}},
[]Token{Token{"\u8001\u5e74", 0, 2}, Token{"\u641c\u7d22", 2, 4}, Token{"\u8fd8", 4, 5}, Token{"\u652f\u6301", 5, 7}},
[]Token{Token{"\u5e72\u8106", 0, 2}, Token{"\u5c31", 2, 3}, Token{"\u628a", 3, 4}, Token{"\u90a3\u90e8", 4, 6}, Token{"\u8499\u4eba", 6, 8}, Token{"\u7684", 8, 9}, Token{"\u95f2\u6cd5", 9, 11}, Token{"\u7ed9", 11, 12}, Token{"\u5e9f", 12, 13}, Token{"\u4e86", 13, 14}, Token{"\u62c9\u5012", 14, 16}, Token{"\uff01", 16, 17}, Token{"RT", 17, 19}, Token{" ", 19, 20}, Token{"@", 20, 21}, Token{"laoshipukong", 21, 33}, Token{" ", 33, 34}, Token{":", 34, 35}, Token{" ", 35, 36}, Token{"27", 36, 38}, Token{"\u65e5", 38, 39}, Token{"\uff0c", 39, 40}, Token{"\u5168\u56fd\u4eba\u5927\u5e38\u59d4\u4f1a", 40, 47}, Token{"\u7b2c\u4e09\u6b21", 47, 50}, Token{"\u5ba1\u8bae", 50, 52}, Token{"\u4fb5\u6743", 52, 54}, Token{"\u8d23\u4efb\u6cd5", 54, 57}, Token{"\u8349\u6848", 57, 59}, Token{"\uff0c", 59, 60}, Token{"\u5220\u9664", 60, 62}, Token{"\u4e86", 62, 63}, Token{"\u6709\u5173", 63, 65}, Token{"\u533b\u7597", 65, 67}, Token{"\u635f\u5bb3", 67, 69}, Token{"\u8d23\u4efb", 69, 71}, Token{"\u201c", 71, 72}, Token{"\u4e3e\u8bc1", 72, 74}, Token{"\u5012\u7f6e", 74, 76}, Token{"\u201d", 76, 77}, Token{"\u7684", 77, 78}, Token{"\u89c4\u5b9a", 78, 80}, Token{"\u3002", 80, 81}, Token{"\u5728", 81, 82}, Token{"\u533b\u60a3", 82, 84}, Token{"\u7ea0\u7eb7", 84, 86}, Token{"\u4e2d\u672c", 86, 88}, Token{"\u5df2", 88, 89}, Token{"\u5904\u4e8e", 89, 91}, Token{"\u5f31\u52bf", 91, 93}, Token{"\u5730\u4f4d", 93, 95}, Token{"\u7684", 95, 96}, Token{"\u6d88\u8d39\u8005", 96, 99}, Token{"\u7531\u6b64", 99, 101}, Token{"\u5c06", 101, 102}, Token{"\u9677\u5165", 102, 104}, Token{"\u4e07\u52ab\u4e0d\u590d", 104, 108}, Token{"\u7684", 108, 109}, Token{"\u5883\u5730", 109, 111}, Token{"\u3002", 111, 112}, Token{" ", 112, 113}},
[]Token{Token{"\u5927", 0, 1}},
[]Token{},
[]Token{Token{"\u4ed6", 0, 1}, Token{"\u8bf4", 1, 2}, Token{"\u7684", 2, 3}, Token{"\u786e\u5b9e", 3, 5}, Token{"\u5728\u7406", 5, 7}},
[]Token{Token{"\u957f\u6625", 0, 2}, Token{"\u5e02\u957f", 2, 4}, Token{"\u6625\u8282", 4, 6}, Token{"\u8bb2\u8bdd", 6, 8}},
[]Token{Token{"\u7ed3\u5a5a", 0, 2}, Token{"\u7684", 2, 3}, Token{"\u548c", 3, 4}, Token{"\u5c1a\u672a", 4, 6}, Token{"\u7ed3\u5a5a", 6, 8}, Token{"\u7684", 8, 9}},
[]Token{Token{"\u7ed3\u5408", 0, 2}, Token{"\u6210", 2, 3}, Token{"\u5206\u5b50", 3, 5}, Token{"\u65f6", 5, 6}},
[]Token{Token{"\u65c5\u6e38", 0, 2}, Token{"\u548c", 2, 3}, Token{"\u670d\u52a1", 3, 5}, Token{"\u662f", 5, 6}, Token{"\u6700\u597d", 6, 8}, Token{"\u7684", 8, 9}},
[]Token{Token{"\u8fd9\u4ef6", 0, 2}, Token{"\u4e8b\u60c5", 2, 4}, Token{"\u7684\u786e", 4, 6}, Token{"\u662f", 6, 7}, Token{"\u6211", 7, 8}, Token{"\u7684", 8, 9}, Token{"\u9519", 9, 10}},
[]Token{Token{"\u4f9b", 0, 1}, Token{"\u5927\u5bb6", 1, 3}, Token{"\u53c2\u8003", 3, 5}, Token{"\u6307\u6b63", 5, 7}},
[]Token{Token{"\u54c8\u5c14\u6ee8", 0, 3}, Token{"\u653f\u5e9c", 3, 5}, Token{"\u516c\u5e03", 5, 7}, Token{"\u584c\u6865", 7, 9}, Token{"\u539f\u56e0", 9, 11}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u5728", 1, 2}, Token{"\u673a\u573a", 2, 4}, Token{"\u5165\u53e3\u5904", 4, 7}},
[]Token{Token{"\u90a2\u6c38\u81e3", 0, 3}, Token{"\u6444\u5f71", 3, 5}, Token{"\u62a5\u9053", 5, 7}},
[]Token{Token{"BP", 0, 2}, Token{"\u795e\u7ecf\u7f51\u7edc", 2, 6}, Token{"\u5982\u4f55", 6, 8}, Token{"\u8bad\u7ec3", 8, 10}, Token{"\u624d\u80fd", 10, 12}, Token{"\u5728", 12, 13}, Token{"\u5206\u7c7b", 13, 15}, Token{"\u65f6", 15, 16}, Token{"\u589e\u52a0", 16, 18}, Token{"\u533a\u5206\u5ea6", 18, 21}, Token{"\uff1f", 21, 22}},
[]Token{Token{"\u5357\u4eac\u5e02", 0, 3}, Token{"\u957f\u6c5f\u5927\u6865", 3, 7}},
[]Token{Token{"\u5e94", 0, 1}, Token{"\u4e00\u4e9b", 1, 3}, Token{"\u4f7f\u7528\u8005", 3, 6}, Token{"\u7684", 6, 7}, Token{"\u5efa\u8bae", 7, 9}, Token{"\uff0c", 9, 10}, Token{"\u4e5f", 10, 11}, Token{"\u4e3a\u4e86", 11, 13}, Token{"\u4fbf\u4e8e", 13, 15}, Token{"\u5229\u7528", 15, 17}, Token{"NiuTrans", 17, 25}, Token{"\u7528\u4e8e", 25, 27}, Token{"SMT", 27, 30}, Token{"\u7814\u7a76", 30, 32}},
[]Token{Token{"\u957f\u6625\u5e02", 0, 3}, Token{"\u957f\u6625", 3, 5}, Token{"\u836f\u5e97", 5, 7}},
[]Token{Token{"\u9093\u9896\u8d85", 0, 3}, Token{"\u751f\u524d", 3, 5}, Token{"\u6700", 5, 6}, Token{"\u559c\u6b22", 6, 8}, Token{"\u7684", 8, 9}, Token{"\u8863\u670d", 9, 11}},
[]Token{Token{"\u80e1\u9526\u6d9b", 0, 3}, Token{"\u662f", 3, 4}, Token{"\u70ed\u7231", 4, 6}, Token{"\u4e16\u754c", 6, 8}, Token{"\u548c\u5e73", 8, 10}, Token{"\u7684", 10, 11}, Token{"\u653f\u6cbb\u5c40", 11, 14}, Token{"\u5e38\u59d4", 14, 16}},
[]Token{Token{"\u7a0b\u5e8f\u5458", 0, 3}, Token{"\u795d", 3, 4}, Token{"\u6d77\u6797", 4, 6}, Token{"\u548c", 6, 7}, Token{"\u6731\u4f1a\u9707", 7, 10}, Token{"\u662f", 10, 11}, Token{"\u5728", 11, 12}, Token{"\u5b59\u5065", 12, 14}, Token{"\u7684", 14, 15}, Token{"\u5de6\u9762", 15, 17}, Token{"\u548c", 17, 18}, Token{"\u53f3\u9762", 18, 20}, Token{",", 20, 21}, Token{" ", 21, 22}, Token{"\u8303\u51ef", 22, 24}, Token{"\u5728", 24, 25}, Token{"\u6700", 25, 26}, Token{"\u53f3\u9762", 26, 28}, Token{".", 28, 29}, Token{"\u518d\u5f80", 29, 31}, Token{"\u5de6", 31, 32}, Token{"\u662f", 32, 33}, Token{"\u674e\u677e\u6d2a", 33, 36}},
[]Token{Token{"\u4e00\u6b21\u6027", 0, 3}, Token{"\u4ea4", 3, 4}, Token{"\u591a\u5c11", 4, 6}, Token{"\u94b1", 6, 7}},
[]Token{Token{"\u4e24\u5757", 0, 2}, Token{"\u4e94", 2, 3}, Token{"\u4e00\u5957", 3, 5}, Token{"\uff0c", 5, 6}, Token{"\u4e09\u5757", 6, 8}, Token{"\u516b", 8, 9}, Token{"\u4e00\u65a4", 9, 11}, Token{"\uff0c", 11, 12}, Token{"\u56db\u5757", 12, 14}, Token{"\u4e03", 14, 15}, Token{"\u4e00\u672c", 15, 17}, Token{"\uff0c", 17, 18}, Token{"\u4e94\u5757", 18, 20}, Token{"\u516d", 20, 21}, Token{"\u4e00\u6761", 21, 23}},
[]Token{Token{"\u5c0f", 0, 1}, Token{"\u548c\u5c1a", 1, 3}, Token{"\u7559", 3, 4}, Token{"\u4e86", 4, 5}, Token{"\u4e00\u4e2a", 5, 7}, Token{"\u50cf", 7, 8}, Token{"\u5927", 8, 9}, Token{"\u548c\u5c1a", 9, 11}, Token{"\u4e00\u6837", 11, 13}, Token{"\u7684", 13, 14}, Token{"\u548c\u5c1a\u5934", 14, 17}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u662f", 1, 2}, Token{"\u4e2d\u534e\u4eba\u6c11\u5171\u548c\u56fd", 2, 9}, Token{"\u516c\u6c11", 9, 11}, Token{";", 11, 12}, Token{"\u6211", 12, 13}, Token{"\u7238\u7238", 13, 15}, Token{"\u662f", 15, 16}, Token{"\u5171\u548c\u515a", 16, 19}, Token{"\u515a\u5458", 19, 21}, Token{";", 21, 22}, Token{" ", 22, 23}, Token{"\u5730\u94c1", 23, 25}, Token{"\u548c\u5e73\u95e8", 25, 28}, Token{"\u7ad9", 28, 29}},
[]Token{Token{"\u5f20\u6653\u6885", 0, 3}, Token{"\u53bb", 3, 4}, Token{"\u4eba\u6c11", 4, 6}, Token{"\u533b\u9662", 6, 8}, Token{"\u505a", 8, 9}, Token{"\u4e86", 9, 10}, Token{"\u4e2a", 10, 11}, Token{"B\u8d85", 11, 13}, Token{"\u7136\u540e", 13, 15}, Token{"\u53bb", 15, 16}, Token{"\u4e70", 16, 17}, Token{"\u4e86", 17, 18}, Token{"\u4ef6", 18, 19}, Token{"T\u6064", 19, 21}},
[]Token{Token{"AT&T", 0, 4}, Token{"\u662f", 4, 5}, Token{"\u4e00\u4ef6", 5, 7}, Token{"\u4e0d\u9519", 7, 9}, Token{"\u7684", 9, 10}, Token{"\u516c\u53f8", 10, 12}, Token{"\uff0c", 12, 13}, Token{"\u7ed9", 13, 14}, Token{"\u4f60", 14, 15}, Token{"\u53d1", 15, 16}, Token{"offer", 16, 21}, Token{"\u4e86", 21, 22}, Token{"\u5417", 22, 23}, Token{"\uff1f", 23, 24}},
[]Token{Token{"C++", 0, 3}, Token{"\u548c", 3, 4}, Token{"c#", 4, 6}, Token{"\u662f", 6, 7}, Token{"\u4ec0\u4e48", 7, 9}, Token{"\u5173\u7cfb", 9, 11}, Token{"\uff1f", 11, 12}, Token{"11", 12, 14}, Token{"+", 14, 15}, Token{"122", 15, 18}, Token{"=", 18, 19}, Token{"133", 19, 22}, Token{"\uff0c", 22, 23}, Token{"\u662f", 23, 24}, Token{"\u5417", 24, 25}, Token{"\uff1f", 25, 26}, Token{"PI", 26, 28}, Token{"=", 28, 29}, Token{"3.14159", 29, 36}},
[]Token{Token{"\u4f60", 0, 1}, Token{"\u8ba4\u8bc6", 1, 3}, Token{"\u90a3\u4e2a", 3, 5}, Token{"\u548c", 5, 6}, Token{"\u4e3b\u5e2d", 6, 8}, Token{"\u63e1\u624b", 8, 10}, Token{"\u7684", 10, 11}, Token{"\u7684\u54e5", 11, 13}, Token{"\u5417", 13, 14}, Token{"\uff1f", 14, 15}, Token{"\u4ed6\u5f00", 15, 17}, Token{"\u4e00\u8f86", 17, 19}, Token{"\u9ed1\u8272", 19, 21}, Token{"\u7684\u58eb", 21, 23}, Token{"\u3002", 23, 24}},
[]Token{Token{"\u67aa\u6746\u5b50", 0, 3}, Token{"\u4e2d", 3, 4}, Token{"\u51fa", 4, 5}, Token{"\u653f\u6743", 5, 7}},
[]Token{Token{"\u5f20\u4e09\u98ce", 0, 3}, Token{"\u540c\u5b66", 3, 5}, Token{"\u8d70\u4e0a", 5, 7}, Token{"\u4e86", 7, 8}, Token{"\u4e0d\u5f52\u8def", 8, 11}},
[]Token{Token{"\u963fQ", 0, 2}, Token{"\u8170\u95f4", 2, 4}, Token{"\u6302", 4, 5}, Token{"\u7740", 5, 6}, Token{"BB\u673a", 6, 9}, Token{"\u624b\u91cc", 9, 11}, Token{"\u62ff", 11, 12}, Token{"\u7740", 12, 13}, Token{"\u5927\u54e5\u5927", 13, 16}, Token{"\uff0c", 16, 17}, Token{"\u8bf4", 17, 18}, Token{"\uff1a", 18, 19}, Token{"\u6211", 19, 20}, Token{"\u4e00\u822c", 20, 22}, Token{"\u5403\u996d", 22, 24}, Token{"\u4e0d", 24, 25}, Token{"AA\u5236", 25, 28}, Token{"\u7684", 28, 29}, Token{"\u3002", 29, 30}},
[]Token{Token{"\u5728", 0, 1}, Token{"1\u53f7\u5e97", 1, 4}, Token{"\u80fd", 4, 5}, Token{"\u4e70", 5, 6}, Token{"\u5230", 6, 7}, Token{"\u5c0fS", 7, 9}, Token{"\u548c", 9, 10}, Token{"\u5927S", 10, 12}, Token{"\u516b\u5366", 12, 14}, Token{"\u7684", 14, 15}, Token{"\u4e66", 15, 16}, Token{"\u3002", 16, 17}},
[]Token{Token{"\u8fd9\u662f", 0, 2}, Token{"\u4e00\u4e2a", 2, 4}, Token{"\u4f38\u624b", 4, 6}, Token{"\u4e0d\u89c1", 6, 8}, Token{"\u4e94\u6307", 8, 10}, Token{"\u4f38\u624b\u4e0d\u89c1\u4e94\u6307", 4, 10}, Token{"\u7684", 10, 11}, Token{"\u9ed1\u591c", 11, 13}, Token{"\u3002", 13, 14}, Token{"\u6211", 14, 15}, Token{"\u53eb", 15, 16}, Token{"\u609f\u7a7a", 17, 19}, Token{"\u5b59\u609f\u7a7a", 16, 19}, Token{"\uff0c", 19, 20}, Token{"\u6211", 20, 21}, Token{"\u7231", 21, 22}, Token{"\u5317\u4eac", 22, 24}, Token{"\uff0c", 24, 25}, Token{"\u6211", 25, 26}, Token{"\u7231", 26, 27}, Token{"Python", 27, 33}, Token{"\u548c", 33, 34}, Token{"C++", 34, 37}, Token{"\u3002", 37, 38}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u4e0d", 1, 2}, Token{"\u559c\u6b22", 2, 4}, Token{"\u65e5\u672c", 4, 6}, Token{"\u548c\u670d", 6, 8}, Token{"\u3002", 8, 9}},
[]Token{Token{"\u96f7\u7334", 0, 2}, Token{"\u56de\u5f52", 2, 4}, Token{"\u4eba\u95f4", 4, 6}, Token{"\u3002", 6, 7}},
[]Token{Token{"\u5de5\u4fe1\u5904", 0, 3}, Token{"\u5e72\u4e8b", 4, 6}, Token{"\u5973\u5e72\u4e8b", 3, 6}, Token{"\u6bcf\u6708", 6, 8}, Token{"\u7ecf\u8fc7", 8, 10}, Token{"\u4e0b\u5c5e", 10, 12}, Token{"\u79d1\u5ba4", 12, 14}, Token{"\u90fd", 14, 15}, Token{"\u8981", 15, 16}, Token{"\u4eb2\u53e3", 16, 18}, Token{"\u4ea4\u4ee3", 18, 20}, Token{"24", 20, 22}, Token{"\u53e3", 22, 23}, Token{"\u4ea4\u6362", 23, 25}, Token{"\u6362\u673a", 24, 26}, Token{"\u4ea4\u6362\u673a", 23, 26}, Token{"\u7b49", 26, 27}, Token{"\u6280\u672f", 27, 29}, Token{"\u6280\u672f\u6027", 27, 30}, Token{"\u5668\u4ef6", 30, 32}, Token{"\u7684", 32, 33}, Token{"\u5b89\u88c5", 33, 35}, Token{"\u5de5\u4f5c", 35, 37}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u9700\u8981", 1, 3}, Token{"\u5ec9\u79df", 3, 5}, Token{"\u79df\u623f", 4, 6}, Token{"\u5ec9\u79df\u623f", 3, 6}},
[]Token{Token{"\u6c38\u548c", 0, 2}, Token{"\u670d\u88c5", 2, 4}, Token{"\u9970\u54c1", 4, 6}, Token{"\u6709\u9650", 6, 8}, Token{"\u516c\u53f8", 8, 10}, Token{"\u6709\u9650\u516c\u53f8", 6, 10}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u7231", 1, 2}, Token{"\u5317\u4eac", 2, 4}, Token{"\u5929\u5b89", 4, 6}, Token{"\u5929\u5b89\u95e8", 4, 7}},
[]Token{Token{"abc", 0, 3}},
[]Token{Token{"\u9690", 0, 1}, Token{"\u53ef\u592b", 3, 5}, Token{"\u9a6c\u5c14\u53ef", 1, 4}, Token{"\u9a6c\u5c14\u53ef\u592b", 1, 5}},
[]Token{Token{"\u96f7\u7334", 0, 2}, Token{"\u662f", 2, 3}, Token{"\u4e2a", 3, 4}, Token{"\u597d", 4, 5}, Token{"\u7f51\u7ad9", 5, 7}},
[]Token{Token{"\u201c", 0, 1}, Token{"Microsoft", 1, 10}, Token{"\u201d", 10, 11}, Token{"\u4e00\u8bcd", 11, 13}, Token{"\u7531", 13, 14}, Token{"\u201c", 14, 15}, Token{"MICROcomputer", 15, 28}, Token{"\uff08", 28, 29}, Token{"\u5fae\u578b", 29, 31}, Token{"\u8ba1\u7b97", 31, 33}, Token{"\u7b97\u673a", 32, 34}, Token{"\u8ba1\u7b97\u673a", 31, 34}, Token{"\uff09", 34, 35}, Token{"\u201d", 35, 36}, Token{"\u548c", 36, 37}, Token{"\u201c", 37, 38}, Token{"SOFTware", 38, 46}, Token{"\uff08", 46, 47}, Token{"\u8f6f\u4ef6", 47, 49}, Token{"\uff09", 49, 50}, Token{"\u201d", 50, 51}, Token{"\u4e24", 51, 52}, Token{"\u90e8\u5206", 52, 54}, Token{"\u7ec4\u6210", 54, 56}},
[]Token{Token{"\u8349\u6ce5\u9a6c", 0, 3}, Token{"\u548c", 3, 4}, Token{"\u6b3a\u5b9e", 4, 6}, Token{"\u9a6c", 6, 7}, Token{"\u662f", 7, 8}, Token{"\u4eca\u5e74", 8, 10}, Token{"\u7684", 10, 11}, Token{"\u6d41\u884c", 11, 13}, Token{"\u8bcd\u6c47", 13, 15}},
[]Token{Token{"\u4f0a\u85e4", 0, 2}, Token{"\u6d0b\u534e\u5802", 2, 5}, Token{"\u603b\u5e9c", 5, 7}, Token{"\u5e97", 7, 8}},
[]Token{Token{"\u4e2d\u56fd", 0, 2}, Token{"\u79d1\u5b66", 2, 4}, Token{"\u5b66\u9662", 3, 5}, Token{"\u8ba1\u7b97", 5, 7}, Token{"\u6280\u672f", 7, 9}, Token{"\u7814\u7a76", 9, 11}, Token{"\u79d1\u5b66\u9662", 2, 5}, Token{"\u7814\u7a76\u6240", 9, 12}, Token{"\u4e2d\u56fd\u79d1\u5b66\u9662\u8ba1\u7b97\u6280\u672f\u7814\u7a76\u6240", 0, 12}},
[]Token{Token{"\u7f57\u5bc6\u6b27", 0, 3}, Token{"\u4e0e", 3, 4}, Token{"\u6731\u4e3d\u53f6", 4, 7}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u8d2d\u4e70", 1, 3}, Token{"\u4e86", 3, 4}, Token{"\u9053\u5177", 4, 6}, Token{"\u548c", 6, 7}, Token{"\u670d\u88c5", 7, 9}},
[]Token{Token{"PS", 0, 2}, Token{":", 2, 3}, Token{" ", 3, 4}, Token{"\u6211", 4, 5}, Token{"\u89c9\u5f97", 5, 7}, Token{"\u5f00\u6e90", 7, 9}, Token{"\u6709", 9, 10}, Token{"\u4e00\u4e2a", 10, 12}, Token{"\u597d\u5904", 12, 14}, Token{"\uff0c", 14, 15}, Token{"\u5c31\u662f", 15, 17}, Token{"\u80fd\u591f", 17, 19}, Token{"\u6566\u4fc3", 19, 21}, Token{"\u81ea\u5df1", 21, 23}, Token{"\u4e0d\u65ad", 23, 25}, Token{"\u6539\u8fdb", 25, 27}, Token{"\u4e0d\u65ad\u6539\u8fdb", 23, 27}, Token{"\uff0c", 27, 28}, Token{"\u907f\u514d", 28, 30}, Token{"\u655e\u5e1a", 30, 32}, Token{"\u81ea\u73cd", 32, 34}},
[]Token{Token{"\u6e56\u5317", 0, 2}, Token{"\u6e56\u5317\u7701", 0, 3}, Token{"\u77f3\u9996", 3, 5}, Token{"\u77f3\u9996\u5e02", 3, 6}},
[]Token{Token{"\u6e56\u5317", 0, 2}, Token{"\u6e56\u5317\u7701", 0, 3}, Token{"\u5341\u5830", 3, 5}, Token{"\u5341\u5830\u5e02", 3, 6}},
[]Token{Token{"\u7ecf\u7406", 1, 3}, Token{"\u603b\u7ecf\u7406", 0, 3}, Token{"\u5b8c\u6210", 3, 5}, Token{"\u4e86", 5, 6}, Token{"\u8fd9\u4ef6", 6, 8}, Token{"\u4e8b\u60c5", 8, 10}},
[]Token{Token{"\u7535\u8111", 0, 2}, Token{"\u4fee\u597d", 2, 4}, Token{"\u4e86", 4, 5}},
[]Token{Token{"\u505a\u597d", 0, 2}, Token{"\u4e86", 2, 3}, Token{"\u8fd9\u4ef6", 3, 5}, Token{"\u4e8b\u60c5", 5, 7}, Token{"\u5c31", 7, 8}, Token{"\u4e00\u4e86\u767e\u4e86", 8, 12}, Token{"\u4e86", 12, 13}},
[]Token{Token{"\u4eba\u4eec", 0, 2}, Token{"\u5ba1\u7f8e", 2, 4}, Token{"\u7684", 4, 5}, Token{"\u89c2\u70b9", 5, 7}, Token{"\u662f", 7, 8}, Token{"\u4e0d\u540c", 8, 10}, Token{"\u7684", 10, 11}},
[]Token{Token{"\u6211\u4eec", 0, 2}, Token{"\u4e70", 2, 3}, Token{"\u4e86", 3, 4}, Token{"\u4e00\u4e2a", 4, 6}, Token{"\u7f8e\u7684", 6, 8}, Token{"\u7a7a\u8c03", 8, 10}},
[]Token{Token{"\u7ebf\u7a0b", 0, 2}, Token{"\u521d\u59cb", 2, 4}, Token{"\u521d\u59cb\u5316", 2, 5}, Token{"\u65f6", 5, 6}, Token{"\u6211\u4eec", 6, 8}, Token{"\u8981", 8, 9}, Token{"\u6ce8\u610f", 9, 11}},
[]Token{Token{"\u4e00\u4e2a", 0, 2}, Token{"\u5206\u5b50", 2, 4}, Token{"\u662f", 4, 5}, Token{"\u7531", 5, 6}, Token{"\u597d\u591a", 6, 8}, Token{"\u539f\u5b50", 8, 10}, Token{"\u7ec4\u7ec7", 10, 12}, Token{"\u6210", 12, 13}, Token{"\u7684", 13, 14}},
[]Token{Token{"\u795d", 0, 1}, Token{"\u4f60", 1, 2}, Token{"\u9a6c\u5230\u529f\u6210", 2, 6}},
[]Token{Token{"\u4ed6", 0, 1}, Token{"\u6389", 1, 2}, Token{"\u8fdb", 2, 3}, Token{"\u4e86", 3, 4}, Token{"\u65e0\u5e95", 4, 6}, Token{"\u65e0\u5e95\u6d1e", 4, 7}, Token{"\u91cc", 7, 8}},
[]Token{Token{"\u4e2d\u56fd", 0, 2}, Token{"\u7684", 2, 3}, Token{"\u9996\u90fd", 3, 5}, Token{"\u662f", 5, 6}, Token{"\u5317\u4eac", 6, 8}},
[]Token{Token{"\u5b59\u541b\u610f", 0, 3}},
[]Token{Token{"\u5916\u4ea4", 0, 2}, Token{"\u5916\u4ea4\u90e8", 0, 3}, Token{"\u53d1\u8a00", 3, 5}, Token{"\u53d1\u8a00\u4eba", 3, 6}, Token{"\u9a6c\u671d\u65ed", 6, 9}},
[]Token{Token{"\u9886\u5bfc", 0, 2}, Token{"\u9886\u5bfc\u4eba", 0, 3}, Token{"\u4f1a\u8bae", 3, 5}, Token{"\u548c", 5, 6}, Token{"\u7b2c\u56db", 6, 8}, Token{"\u56db\u5c4a", 7, 9}, Token{"\u7b2c\u56db\u5c4a", 6, 9}, Token{"\u4e1c\u4e9a", 9, 11}, Token{"\u5cf0\u4f1a", 11, 13}},
[]Token{Token{"\u5728", 0, 1}, Token{"\u8fc7\u53bb", 1, 3}, Token{"\u7684", 3, 4}, Token{"\u8fd9", 4, 5}, Token{"\u4e94\u5e74", 5, 7}},
[]Token{Token{"\u8fd8", 0, 1}, Token{"\u9700\u8981", 1, 3}, Token{"\u5f88\u957f", 3, 5}, Token{"\u7684", 5, 6}, Token{"\u8def", 6, 7}, Token{"\u8981", 7, 8}, Token{"\u8d70", 8, 9}},
[]Token{Token{"60", 0, 2}, Token{"\u5468\u5e74", 2, 4}, Token{"\u9996\u90fd", 4, 6}, Token{"\u9605\u5175", 6, 8}},
[]Token{Token{"\u4f60\u597d", 0, 2}, Token{"\u4eba\u4eec", 2, 4}, Token{"\u5ba1\u7f8e", 4, 6}, Token{"\u7684", 6, 7}, Token{"\u89c2\u70b9", 7, 9}, Token{"\u662f", 9, 10}, Token{"\u4e0d\u540c", 10, 12}, Token{"\u7684", 12, 13}},
[]Token{Token{"\u4e70", 0, 1}, Token{"\u6c34\u679c", 1, 3}, Token{"\u7136\u540e", 3, 5}, Token{"\u6765", 5, 6}, Token{"\u4e16\u535a", 6, 8}, Token{"\u535a\u56ed", 7, 9}, Token{"\u4e16\u535a\u56ed", 6, 9}},
[]Token{Token{"\u4e70", 0, 1}, Token{"\u6c34\u679c", 1, 3}, Token{"\u7136\u540e", 3, 5}, Token{"\u53bb", 5, 6}, Token{"\u4e16\u535a", 6, 8}, Token{"\u535a\u56ed", 7, 9}, Token{"\u4e16\u535a\u56ed", 6, 9}},
[]Token{Token{"\u4f46\u662f", 0, 2}, Token{"\u540e\u6765", 2, 4}, Token{"\u6211", 4, 5}, Token{"\u624d", 5, 6}, Token{"\u77e5\u9053", 6, 8}, Token{"\u4f60", 8, 9}, Token{"\u662f", 9, 10}, Token{"\u5bf9", 10, 11}, Token{"\u7684", 11, 12}},
[]Token{Token{"\u5b58\u5728", 0, 2}, Token{"\u5373", 2, 3}, Token{"\u5408\u7406", 3, 5}},
[]Token{Token{"\u7684", 0, 1}, Token{"\u7684", 1, 2}, Token{"\u7684", 2, 3}, Token{"\u7684", 3, 4}, Token{"\u7684", 4, 5}, Token{"\u5728", 5, 6}, Token{"\u7684", 6, 7}, Token{"\u7684", 7, 8}, Token{"\u7684", 8, 9}, Token{"\u7684", 9, 10}, Token{"\u5c31", 10, 11}, Token{"\u4ee5", 11, 12}, Token{"\u548c", 12, 13}, Token{"\u548c", 13, 14}, Token{"\u548c", 14, 15}},
[]Token{Token{"I", 0, 1}, Token{" ", 1, 2}, Token{"love", 2, 6}, Token{"\u4f60", 6, 7}, Token{"\uff0c", 7, 8}, Token{"\u4e0d\u4ee5", 8, 10}, Token{"\u4ee5\u4e3a", 9, 11}, Token{"\u4e0d\u4ee5\u4e3a\u803b", 8, 12}, Token{"\uff0c", 12, 13}, Token{"\u53cd", 13, 14}, Token{"\u4ee5\u4e3a", 14, 16}, Token{"rong", 16, 20}},
[]Token{Token{"\u56e0", 0, 1}},
[]Token{},
[]Token{Token{"hello", 0, 5}, Token{"\u4f60\u597d", 5, 7}, Token{"\u4eba\u4eec", 7, 9}, Token{"\u5ba1\u7f8e", 9, 11}, Token{"\u7684", 11, 12}, Token{"\u89c2\u70b9", 12, 14}, Token{"\u662f", 14, 15}, Token{"\u4e0d\u540c", 15, 17}, Token{"\u7684", 17, 18}},
[]Token{Token{"\u5f88", 0, 1}, Token{"\u597d", 1, 2}, Token{"\u4f46", 2, 3}, Token{"\u4e3b\u8981", 3, 5}, Token{"\u662f", 5, 6}, Token{"\u57fa\u4e8e", 6, 8}, Token{"\u7f51\u9875", 8, 10}, Token{"\u5f62\u5f0f", 10, 12}},
[]Token{Token{"hello", 0, 5}, Token{"\u4f60\u597d", 5, 7}, Token{"\u4eba\u4eec", 7, 9}, Token{"\u5ba1\u7f8e", 9, 11}, Token{"\u7684", 11, 12}, Token{"\u89c2\u70b9", 12, 14}, Token{"\u662f", 14, 15}, Token{"\u4e0d\u540c", 15, 17}, Token{"\u7684", 17, 18}},
[]Token{Token{"\u4ec0\u4e48", 1, 3}, Token{"\u4e3a\u4ec0\u4e48", 0, 3}, Token{"\u6211", 3, 4}, Token{"\u4e0d\u80fd", 4, 6}, Token{"\u62e5\u6709", 6, 8}, Token{"\u60f3\u8981", 8, 10}, Token{"\u7684", 10, 11}, Token{"\u751f\u6d3b", 11, 13}},
[]Token{Token{"\u540e\u6765", 0, 2}, Token{"\u6211", 2, 3}, Token{"\u624d", 3, 4}},
[]Token{Token{"\u6b64\u6b21", 0, 2}, Token{"\u6765", 2, 3}, Token{"\u4e2d\u56fd", 3, 5}, Token{"\u662f", 5, 6}, Token{"\u4e3a\u4e86", 6, 8}},
[]Token{Token{"\u4f7f\u7528", 0, 2}, Token{"\u4e86", 2, 3}, Token{"\u5b83", 3, 4}, Token{"\u5c31", 4, 5}, Token{"\u53ef\u4ee5", 5, 7}, Token{"\u89e3\u51b3", 7, 9}, Token{"\u4e00\u4e9b", 9, 11}, Token{"\u95ee\u9898", 11, 13}},
[]Token{Token{",", 0, 1}, Token{"\u4f7f\u7528", 1, 3}, Token{"\u4e86", 3, 4}, Token{"\u5b83", 4, 5}, Token{"\u5c31", 5, 6}, Token{"\u53ef\u4ee5", 6, 8}, Token{"\u89e3\u51b3", 8, 10}, Token{"\u4e00\u4e9b", 10, 12}, Token{"\u95ee\u9898", 12, 14}},
[]Token{Token{"\u5176\u5b9e", 0, 2}, Token{"\u4f7f\u7528", 2, 4}, Token{"\u4e86", 4, 5}, Token{"\u5b83", 5, 6}, Token{"\u5c31", 6, 7}, Token{"\u53ef\u4ee5", 7, 9}, Token{"\u89e3\u51b3", 9, 11}, Token{"\u4e00\u4e9b", 11, 13}, Token{"\u95ee\u9898", 13, 15}},
[]Token{Token{"\u597d\u4eba", 0, 2}, Token{"\u4f7f\u7528", 2, 4}, Token{"\u4e86", 4, 5}, Token{"\u5b83", 5, 6}, Token{"\u5c31", 6, 7}, Token{"\u53ef\u4ee5", 7, 9}, Token{"\u89e3\u51b3", 9, 11}, Token{"\u4e00\u4e9b", 11, 13}, Token{"\u95ee\u9898", 13, 15}},
[]Token{Token{"\u56e0\u4e3a", 1, 3}, Token{"\u662f\u56e0\u4e3a", 0, 3}, Token{"\u548c", 3, 4}, Token{"\u56fd\u5bb6", 4, 6}},
[]Token{Token{"\u8001\u5e74", 0, 2}, Token{"\u641c\u7d22", 2, 4}, Token{"\u8fd8", 4, 5}, Token{"\u652f\u6301", 5, 7}},
[]Token{Token{"\u5e72\u8106", 0, 2}, Token{"\u5c31", 2, 3}, Token{"\u628a", 3, 4}, Token{"\u90a3\u90e8", 4, 6}, Token{"\u8499\u4eba", 6, 8}, Token{"\u7684", 8, 9}, Token{"\u95f2\u6cd5", 9, 11}, Token{"\u7ed9", 11, 12}, Token{"\u5e9f", 12, 13}, Token{"\u4e86", 13, 14}, Token{"\u62c9\u5012", 14, 16}, Token{"\uff01", 16, 17}, Token{"RT", 17, 19}, Token{" ", 19, 20}, Token{"@", 20, 21}, Token{"laoshipukong", 21, 33}, Token{" ", 33, 34}, Token{":", 34, 35}, Token{" ", 35, 36}, Token{"27", 36, 38}, Token{"\u65e5", 38, 39}, Token{"\uff0c", 39, 40}, Token{"\u5168\u56fd", 40, 42}, Token{"\u56fd\u4eba", 41, 43}, Token{"\u4eba\u5927", 42, 44}, Token{"\u5e38\u59d4", 44, 46}, Token{"\u59d4\u4f1a", 45, 47}, Token{"\u5e38\u59d4\u4f1a", 44, 47}, Token{"\u5168\u56fd\u4eba\u5927\u5e38\u59d4\u4f1a", 40, 47}, Token{"\u7b2c\u4e09", 47, 49}, Token{"\u4e09\u6b21", 48, 50}, Token{"\u7b2c\u4e09\u6b21", 47, 50}, Token{"\u5ba1\u8bae", 50, 52}, Token{"\u4fb5\u6743", 52, 54}, Token{"\u8d23\u4efb", 54, 56}, Token{"\u8d23\u4efb\u6cd5", 54, 57}, Token{"\u8349\u6848", 57, 59}, Token{"\uff0c", 59, 60}, Token{"\u5220\u9664", 60, 62}, Token{"\u4e86", 62, 63}, Token{"\u6709\u5173", 63, 65}, Token{"\u533b\u7597", 65, 67}, Token{"\u635f\u5bb3", 67, 69}, Token{"\u8d23\u4efb", 69, 71}, Token{"\u201c", 71, 72}, Token{"\u4e3e\u8bc1", 72, 74}, Token{"\u5012\u7f6e", 74, 76}, Token{"\u201d", 76, 77}, Token{"\u7684", 77, 78}, Token{"\u89c4\u5b9a", 78, 80}, Token{"\u3002", 80, 81}, Token{"\u5728", 81, 82}, Token{"\u533b\u60a3", 82, 84}, Token{"\u7ea0\u7eb7", 84, 86}, Token{"\u4e2d\u672c", 86, 88}, Token{"\u5df2", 88, 89}, Token{"\u5904\u4e8e", 89, 91}, Token{"\u5f31\u52bf", 91, 93}, Token{"\u5730\u4f4d", 93, 95}, Token{"\u7684", 95, 96}, Token{"\u6d88\u8d39", 96, 98}, Token{"\u6d88\u8d39\u8005", 96, 99}, Token{"\u7531\u6b64", 99, 101}, Token{"\u5c06", 101, 102}, Token{"\u9677\u5165", 102, 104}, Token{"\u4e0d\u590d", 106, 108}, Token{"\u4e07\u52ab\u4e0d\u590d", 104, 108}, Token{"\u7684", 108, 109}, Token{"\u5883\u5730", 109, 111}, Token{"\u3002", 111, 112}, Token{" ", 112, 113}},
[]Token{Token{"\u5927", 0, 1}},
[]Token{},
[]Token{Token{"\u4ed6", 0, 1}, Token{"\u8bf4", 1, 2}, Token{"\u7684", 2, 3}, Token{"\u786e\u5b9e", 3, 5}, Token{"\u5728\u7406", 5, 7}},
[]Token{Token{"\u957f\u6625", 0, 2}, Token{"\u5e02\u957f", 2, 4}, Token{"\u6625\u8282", 4, 6}, Token{"\u8bb2\u8bdd", 6, 8}},
[]Token{Token{"\u7ed3\u5a5a", 0, 2}, Token{"\u7684", 2, 3}, Token{"\u548c", 3, 4}, Token{"\u5c1a\u672a", 4, 6}, Token{"\u7ed3\u5a5a", 6, 8}, Token{"\u7684", 8, 9}},
[]Token{Token{"\u7ed3\u5408", 0, 2}, Token{"\u6210", 2, 3}, Token{"\u5206\u5b50", 3, 5}, Token{"\u65f6", 5, 6}},
[]Token{Token{"\u65c5\u6e38", 0, 2}, Token{"\u548c", 2, 3}, Token{"\u670d\u52a1", 3, 5}, Token{"\u662f", 5, 6}, Token{"\u6700\u597d", 6, 8}, Token{"\u7684", 8, 9}},
[]Token{Token{"\u8fd9\u4ef6", 0, 2}, Token{"\u4e8b\u60c5", 2, 4}, Token{"\u7684\u786e", 4, 6}, Token{"\u662f", 6, 7}, Token{"\u6211", 7, 8}, Token{"\u7684", 8, 9}, Token{"\u9519", 9, 10}},
[]Token{Token{"\u4f9b", 0, 1}, Token{"\u5927\u5bb6", 1, 3}, Token{"\u53c2\u8003", 3, 5}, Token{"\u6307\u6b63", 5, 7}},
[]Token{Token{"\u54c8\u5c14", 0, 2}, Token{"\u54c8\u5c14\u6ee8", 0, 3}, Token{"\u653f\u5e9c", 3, 5}, Token{"\u516c\u5e03", 5, 7}, Token{"\u584c\u6865", 7, 9}, Token{"\u539f\u56e0", 9, 11}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u5728", 1, 2}, Token{"\u673a\u573a", 2, 4}, Token{"\u5165\u53e3", 4, 6}, Token{"\u5165\u53e3\u5904", 4, 7}},
[]Token{Token{"\u90a2\u6c38\u81e3", 0, 3}, Token{"\u6444\u5f71", 3, 5}, Token{"\u62a5\u9053", 5, 7}},
[]Token{Token{"BP", 0, 2}, Token{"\u795e\u7ecf", 2, 4}, Token{"\u7f51\u7edc", 4, 6}, Token{"\u795e\u7ecf\u7f51", 2, 5}, Token{"\u795e\u7ecf\u7f51\u7edc", 2, 6}, Token{"\u5982\u4f55", 6, 8}, Token{"\u8bad\u7ec3", 8, 10}, Token{"\u624d\u80fd", 10, 12}, Token{"\u5728", 12, 13}, Token{"\u5206\u7c7b", 13, 15}, Token{"\u65f6", 15, 16}, Token{"\u589e\u52a0", 16, 18}, Token{"\u533a\u5206", 18, 20}, Token{"\u5206\u5ea6", 19, 21}, Token{"\u533a\u5206\u5ea6", 18, 21}, Token{"\uff1f", 21, 22}},
[]Token{Token{"\u5357\u4eac", 0, 2}, Token{"\u4eac\u5e02", 1, 3}, Token{"\u5357\u4eac\u5e02", 0, 3}, Token{"\u957f\u6c5f", 3, 5}, Token{"\u5927\u6865", 5, 7}, Token{"\u957f\u6c5f\u5927\u6865", 3, 7}},
[]Token{Token{"\u5e94", 0, 1}, Token{"\u4e00\u4e9b", 1, 3}, Token{"\u4f7f\u7528", 3, 5}, Token{"\u7528\u8005", 4, 6}, Token{"\u4f7f\u7528\u8005", 3, 6}, Token{"\u7684", 6, 7}, Token{"\u5efa\u8bae", 7, 9}, Token{"\uff0c", 9, 10}, Token{"\u4e5f", 10, 11}, Token{"\u4e3a\u4e86", 11, 13}, Token{"\u4fbf\u4e8e", 13, 15}, Token{"\u5229\u7528", 15, 17}, Token{"NiuTrans", 17, 25}, Token{"\u7528\u4e8e", 25, 27}, Token{"SMT", 27, 30}, Token{"\u7814\u7a76", 30, 32}},
[]Token{Token{"\u957f\u6625", 0, 2}, Token{"\u957f\u6625\u5e02", 0, 3}, Token{"\u957f\u6625", 3, 5}, Token{"\u836f\u5e97", 5, 7}},
[]Token{Token{"\u9093\u9896\u8d85", 0, 3}, Token{"\u751f\u524d", 3, 5}, Token{"\u6700", 5, 6}, Token{"\u559c\u6b22", 6, 8}, Token{"\u7684", 8, 9}, Token{"\u8863\u670d", 9, 11}},
[]Token{Token{"\u9526\u6d9b", 1, 3}, Token{"\u80e1\u9526\u6d9b", 0, 3}, Token{"\u662f", 3, 4}, Token{"\u70ed\u7231", 4, 6}, Token{"\u4e16\u754c", 6, 8}, Token{"\u548c\u5e73", 8, 10}, Token{"\u7684", 10, 11}, Token{"\u653f\u6cbb", 11, 13}, Token{"\u653f\u6cbb\u5c40", 11, 14}, Token{"\u5e38\u59d4", 14, 16}},
[]Token{Token{"\u7a0b\u5e8f", 0, 2}, Token{"\u7a0b\u5e8f\u5458", 0, 3}, Token{"\u795d", 3, 4}, Token{"\u6d77\u6797", 4, 6}, Token{"\u548c", 6, 7}, Token{"\u6731\u4f1a\u9707", 7, 10}, Token{"\u662f", 10, 11}, Token{"\u5728", 11, 12}, Token{"\u5b59\u5065", 12, 14}, Token{"\u7684", 14, 15}, Token{"\u5de6\u9762", 15, 17}, Token{"\u548c", 17, 18}, Token{"\u53f3\u9762", 18, 20}, Token{",", 20, 21}, Token{" ", 21, 22}, Token{"\u8303\u51ef", 22, 24}, Token{"\u5728", 24, 25}, Token{"\u6700", 25, 26}, Token{"\u53f3\u9762", 26, 28}, Token{".", 28, 29}, Token{"\u518d\u5f80", 29, 31}, Token{"\u5de6", 31, 32}, Token{"\u662f", 32, 33}, Token{"\u674e\u677e\u6d2a", 33, 36}},
[]Token{Token{"\u4e00\u6b21", 0, 2}, Token{"\u4e00\u6b21\u6027", 0, 3}, Token{"\u4ea4", 3, 4}, Token{"\u591a\u5c11", 4, 6}, Token{"\u94b1", 6, 7}},
[]Token{Token{"\u4e24\u5757", 0, 2}, Token{"\u4e94", 2, 3}, Token{"\u4e00\u5957", 3, 5}, Token{"\uff0c", 5, 6}, Token{"\u4e09\u5757", 6, 8}, Token{"\u516b", 8, 9}, Token{"\u4e00\u65a4", 9, 11}, Token{"\uff0c", 11, 12}, Token{"\u56db\u5757", 12, 14}, Token{"\u4e03", 14, 15}, Token{"\u4e00\u672c", 15, 17}, Token{"\uff0c", 17, 18}, Token{"\u4e94\u5757", 18, 20}, Token{"\u516d", 20, 21}, Token{"\u4e00\u6761", 21, 23}},
[]Token{Token{"\u5c0f", 0, 1}, Token{"\u548c\u5c1a", 1, 3}, Token{"\u7559", 3, 4}, Token{"\u4e86", 4, 5}, Token{"\u4e00\u4e2a", 5, 7}, Token{"\u50cf", 7, 8}, Token{"\u5927", 8, 9}, Token{"\u548c\u5c1a", 9, 11}, Token{"\u4e00\u6837", 11, 13}, Token{"\u7684", 13, 14}, Token{"\u548c\u5c1a", 14, 16}, Token{"\u548c\u5c1a\u5934", 14, 17}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u662f", 1, 2}, Token{"\u4e2d\u534e", 2, 4}, Token{"\u534e\u4eba", 3, 5}, Token{"\u4eba\u6c11", 4, 6}, Token{"\u5171\u548c", 6, 8}, Token{"\u5171\u548c\u56fd", 6, 9}, Token{"\u4e2d\u534e\u4eba\u6c11\u5171\u548c\u56fd", 2, 9}, Token{"\u516c\u6c11", 9, 11}, Token{";", 11, 12}, Token{"\u6211", 12, 13}, Token{"\u7238\u7238", 13, 15}, Token{"\u662f", 15, 16}, Token{"\u5171\u548c", 16, 18}, Token{"\u5171\u548c\u515a", 16, 19}, Token{"\u515a\u5458", 19, 21}, Token{";", 21, 22}, Token{" ", 22, 23}, Token{"\u5730\u94c1", 23, 25}, Token{"\u548c\u5e73", 25, 27}, Token{"\u548c\u5e73\u95e8", 25, 28}, Token{"\u7ad9", 28, 29}},
[]Token{Token{"\u5f20\u6653\u6885", 0, 3}, Token{"\u53bb", 3, 4}, Token{"\u4eba\u6c11", 4, 6}, Token{"\u533b\u9662", 6, 8}, Token{"\u505a", 8, 9}, Token{"\u4e86", 9, 10}, Token{"\u4e2a", 10, 11}, Token{"B\u8d85", 11, 13}, Token{"\u7136\u540e", 13, 15}, Token{"\u53bb", 15, 16}, Token{"\u4e70", 16, 17}, Token{"\u4e86", 17, 18}, Token{"\u4ef6", 18, 19}, Token{"T\u6064", 19, 21}},
[]Token{Token{"AT&T", 0, 4}, Token{"\u662f", 4, 5}, Token{"\u4e00\u4ef6", 5, 7}, Token{"\u4e0d\u9519", 7, 9}, Token{"\u7684", 9, 10}, Token{"\u516c\u53f8", 10, 12}, Token{"\uff0c", 12, 13}, Token{"\u7ed9", 13, 14}, Token{"\u4f60", 14, 15}, Token{"\u53d1", 15, 16}, Token{"offer", 16, 21}, Token{"\u4e86", 21, 22}, Token{"\u5417", 22, 23}, Token{"\uff1f", 23, 24}},
[]Token{Token{"C++", 0, 3}, Token{"\u548c", 3, 4}, Token{"c#", 4, 6}, Token{"\u662f", 6, 7}, Token{"\u4ec0\u4e48", 7, 9}, Token{"\u5173\u7cfb", 9, 11}, Token{"\uff1f", 11, 12}, Token{"11", 12, 14}, Token{"+", 14, 15}, Token{"122", 15, 18}, Token{"=", 18, 19}, Token{"133", 19, 22}, Token{"\uff0c", 22, 23}, Token{"\u662f", 23, 24}, Token{"\u5417", 24, 25}, Token{"\uff1f", 25, 26}, Token{"PI", 26, 28}, Token{"=", 28, 29}, Token{"3.14159", 29, 36}},
[]Token{Token{"\u4f60", 0, 1}, Token{"\u8ba4\u8bc6", 1, 3}, Token{"\u90a3\u4e2a", 3, 5}, Token{"\u548c", 5, 6}, Token{"\u4e3b\u5e2d", 6, 8}, Token{"\u63e1\u624b", 8, 10}, Token{"\u7684", 10, 11}, Token{"\u7684\u54e5", 11, 13}, Token{"\u5417", 13, 14}, Token{"\uff1f", 14, 15}, Token{"\u4ed6\u5f00", 15, 17}, Token{"\u4e00\u8f86", 17, 19}, Token{"\u9ed1\u8272", 19, 21}, Token{"\u7684\u58eb", 21, 23}, Token{"\u3002", 23, 24}},
[]Token{Token{"\u67aa\u6746", 0, 2}, Token{"\u6746\u5b50", 1, 3}, Token{"\u67aa\u6746\u5b50", 0, 3}, Token{"\u4e2d", 3, 4}, Token{"\u51fa", 4, 5}, Token{"\u653f\u6743", 5, 7}},
[]Token{Token{"\u5f20\u4e09\u98ce", 0, 3}, Token{"\u540c\u5b66", 3, 5}, Token{"\u8d70\u4e0a", 5, 7}, Token{"\u4e86", 7, 8}, Token{"\u5f52\u8def", 9, 11}, Token{"\u4e0d\u5f52\u8def", 8, 11}},
[]Token{Token{"\u963fQ", 0, 2}, Token{"\u8170\u95f4", 2, 4}, Token{"\u6302", 4, 5}, Token{"\u7740", 5, 6}, Token{"BB\u673a", 6, 9}, Token{"\u624b\u91cc", 9, 11}, Token{"\u62ff", 11, 12}, Token{"\u7740", 12, 13}, Token{"\u5927\u54e5", 13, 15}, Token{"\u5927\u54e5\u5927", 13, 16}, Token{"\uff0c", 16, 17}, Token{"\u8bf4", 17, 18}, Token{"\uff1a", 18, 19}, Token{"\u6211", 19, 20}, Token{"\u4e00\u822c", 20, 22}, Token{"\u5403\u996d", 22, 24}, Token{"\u4e0d", 24, 25}, Token{"AA\u5236", 25, 28}, Token{"\u7684", 28, 29}, Token{"\u3002", 29, 30}},
[]Token{Token{"\u5728", 0, 1}, Token{"1\u53f7\u5e97", 1, 4}, Token{"\u80fd", 4, 5}, Token{"\u4e70", 5, 6}, Token{"\u5230", 6, 7}, Token{"\u5c0fS", 7, 9}, Token{"\u548c", 9, 10}, Token{"\u5927S", 10, 12}, Token{"\u516b\u5366", 12, 14}, Token{"\u7684", 14, 15}, Token{"\u4e66", 15, 16}, Token{"\u3002", 16, 17}},
result = [][]token{
[]token{token{"\u8fd9\u662f", 0, 2}, token{"\u4e00\u4e2a", 2, 4}, token{"\u4f38\u624b\u4e0d\u89c1\u4e94\u6307", 4, 10}, token{"\u7684", 10, 11}, token{"\u9ed1\u591c", 11, 13}, token{"\u3002", 13, 14}, token{"\u6211", 14, 15}, token{"\u53eb", 15, 16}, token{"\u5b59\u609f\u7a7a", 16, 19}, token{"\uff0c", 19, 20}, token{"\u6211", 20, 21}, token{"\u7231", 21, 22}, token{"\u5317\u4eac", 22, 24}, token{"\uff0c", 24, 25}, token{"\u6211", 25, 26}, token{"\u7231", 26, 27}, token{"Python", 27, 33}, token{"\u548c", 33, 34}, token{"C++", 34, 37}, token{"\u3002", 37, 38}},
[]token{token{"\u6211", 0, 1}, token{"\u4e0d", 1, 2}, token{"\u559c\u6b22", 2, 4}, token{"\u65e5\u672c", 4, 6}, token{"\u548c\u670d", 6, 8}, token{"\u3002", 8, 9}},
[]token{token{"\u96f7\u7334", 0, 2}, token{"\u56de\u5f52", 2, 4}, token{"\u4eba\u95f4", 4, 6}, token{"\u3002", 6, 7}},
[]token{token{"\u5de5\u4fe1\u5904", 0, 3}, token{"\u5973\u5e72\u4e8b", 3, 6}, token{"\u6bcf\u6708", 6, 8}, token{"\u7ecf\u8fc7", 8, 10}, token{"\u4e0b\u5c5e", 10, 12}, token{"\u79d1\u5ba4", 12, 14}, token{"\u90fd", 14, 15}, token{"\u8981", 15, 16}, token{"\u4eb2\u53e3", 16, 18}, token{"\u4ea4\u4ee3", 18, 20}, token{"24", 20, 22}, token{"\u53e3", 22, 23}, token{"\u4ea4\u6362\u673a", 23, 26}, token{"\u7b49", 26, 27}, token{"\u6280\u672f\u6027", 27, 30}, token{"\u5668\u4ef6", 30, 32}, token{"\u7684", 32, 33}, token{"\u5b89\u88c5", 33, 35}, token{"\u5de5\u4f5c", 35, 37}},
[]token{token{"\u6211", 0, 1}, token{"\u9700\u8981", 1, 3}, token{"\u5ec9\u79df\u623f", 3, 6}},
[]token{token{"\u6c38\u548c", 0, 2}, token{"\u670d\u88c5", 2, 4}, token{"\u9970\u54c1", 4, 6}, token{"\u6709\u9650\u516c\u53f8", 6, 10}},
[]token{token{"\u6211", 0, 1}, token{"\u7231", 1, 2}, token{"\u5317\u4eac", 2, 4}, token{"\u5929\u5b89\u95e8", 4, 7}},
[]token{token{"abc", 0, 3}},
[]token{token{"\u9690", 0, 1}, token{"\u9a6c\u5c14\u53ef\u592b", 1, 5}},
[]token{token{"\u96f7\u7334", 0, 2}, token{"\u662f", 2, 3}, token{"\u4e2a", 3, 4}, token{"\u597d", 4, 5}, token{"\u7f51\u7ad9", 5, 7}},
[]token{token{"\u201c", 0, 1}, token{"Microsoft", 1, 10}, token{"\u201d", 10, 11}, token{"\u4e00\u8bcd", 11, 13}, token{"\u7531", 13, 14}, token{"\u201c", 14, 15}, token{"MICROcomputer", 15, 28}, token{"\uff08", 28, 29}, token{"\u5fae\u578b", 29, 31}, token{"\u8ba1\u7b97\u673a", 31, 34}, token{"\uff09", 34, 35}, token{"\u201d", 35, 36}, token{"\u548c", 36, 37}, token{"\u201c", 37, 38}, token{"SOFTware", 38, 46}, token{"\uff08", 46, 47}, token{"\u8f6f\u4ef6", 47, 49}, token{"\uff09", 49, 50}, token{"\u201d", 50, 51}, token{"\u4e24", 51, 52}, token{"\u90e8\u5206", 52, 54}, token{"\u7ec4\u6210", 54, 56}},
[]token{token{"\u8349\u6ce5\u9a6c", 0, 3}, token{"\u548c", 3, 4}, token{"\u6b3a\u5b9e", 4, 6}, token{"\u9a6c", 6, 7}, token{"\u662f", 7, 8}, token{"\u4eca\u5e74", 8, 10}, token{"\u7684", 10, 11}, token{"\u6d41\u884c", 11, 13}, token{"\u8bcd\u6c47", 13, 15}},
[]token{token{"\u4f0a\u85e4", 0, 2}, token{"\u6d0b\u534e\u5802", 2, 5}, token{"\u603b\u5e9c", 5, 7}, token{"\u5e97", 7, 8}},
[]token{token{"\u4e2d\u56fd\u79d1\u5b66\u9662\u8ba1\u7b97\u6280\u672f\u7814\u7a76\u6240", 0, 12}},
[]token{token{"\u7f57\u5bc6\u6b27", 0, 3}, token{"\u4e0e", 3, 4}, token{"\u6731\u4e3d\u53f6", 4, 7}},
[]token{token{"\u6211", 0, 1}, token{"\u8d2d\u4e70", 1, 3}, token{"\u4e86", 3, 4}, token{"\u9053\u5177", 4, 6}, token{"\u548c", 6, 7}, token{"\u670d\u88c5", 7, 9}},
[]token{token{"PS", 0, 2}, token{":", 2, 3}, token{" ", 3, 4}, token{"\u6211", 4, 5}, token{"\u89c9\u5f97", 5, 7}, token{"\u5f00\u6e90", 7, 9}, token{"\u6709", 9, 10}, token{"\u4e00\u4e2a", 10, 12}, token{"\u597d\u5904", 12, 14}, token{"\uff0c", 14, 15}, token{"\u5c31\u662f", 15, 17}, token{"\u80fd\u591f", 17, 19}, token{"\u6566\u4fc3", 19, 21}, token{"\u81ea\u5df1", 21, 23}, token{"\u4e0d\u65ad\u6539\u8fdb", 23, 27}, token{"\uff0c", 27, 28}, token{"\u907f\u514d", 28, 30}, token{"\u655e\u5e1a", 30, 32}, token{"\u81ea\u73cd", 32, 34}},
[]token{token{"\u6e56\u5317\u7701", 0, 3}, token{"\u77f3\u9996\u5e02", 3, 6}},
[]token{token{"\u6e56\u5317\u7701", 0, 3}, token{"\u5341\u5830\u5e02", 3, 6}},
[]token{token{"\u603b\u7ecf\u7406", 0, 3}, token{"\u5b8c\u6210", 3, 5}, token{"\u4e86", 5, 6}, token{"\u8fd9\u4ef6", 6, 8}, token{"\u4e8b\u60c5", 8, 10}},
[]token{token{"\u7535\u8111", 0, 2}, token{"\u4fee\u597d", 2, 4}, token{"\u4e86", 4, 5}},
[]token{token{"\u505a\u597d", 0, 2}, token{"\u4e86", 2, 3}, token{"\u8fd9\u4ef6", 3, 5}, token{"\u4e8b\u60c5", 5, 7}, token{"\u5c31", 7, 8}, token{"\u4e00\u4e86\u767e\u4e86", 8, 12}, token{"\u4e86", 12, 13}},
[]token{token{"\u4eba\u4eec", 0, 2}, token{"\u5ba1\u7f8e", 2, 4}, token{"\u7684", 4, 5}, token{"\u89c2\u70b9", 5, 7}, token{"\u662f", 7, 8}, token{"\u4e0d\u540c", 8, 10}, token{"\u7684", 10, 11}},
[]token{token{"\u6211\u4eec", 0, 2}, token{"\u4e70", 2, 3}, token{"\u4e86", 3, 4}, token{"\u4e00\u4e2a", 4, 6}, token{"\u7f8e\u7684", 6, 8}, token{"\u7a7a\u8c03", 8, 10}},
[]token{token{"\u7ebf\u7a0b", 0, 2}, token{"\u521d\u59cb\u5316", 2, 5}, token{"\u65f6", 5, 6}, token{"\u6211\u4eec", 6, 8}, token{"\u8981", 8, 9}, token{"\u6ce8\u610f", 9, 11}},
[]token{token{"\u4e00\u4e2a", 0, 2}, token{"\u5206\u5b50", 2, 4}, token{"\u662f", 4, 5}, token{"\u7531", 5, 6}, token{"\u597d\u591a", 6, 8}, token{"\u539f\u5b50", 8, 10}, token{"\u7ec4\u7ec7", 10, 12}, token{"\u6210", 12, 13}, token{"\u7684", 13, 14}},
[]token{token{"\u795d", 0, 1}, token{"\u4f60", 1, 2}, token{"\u9a6c\u5230\u529f\u6210", 2, 6}},
[]token{token{"\u4ed6", 0, 1}, token{"\u6389", 1, 2}, token{"\u8fdb", 2, 3}, token{"\u4e86", 3, 4}, token{"\u65e0\u5e95\u6d1e", 4, 7}, token{"\u91cc", 7, 8}},
[]token{token{"\u4e2d\u56fd", 0, 2}, token{"\u7684", 2, 3}, token{"\u9996\u90fd", 3, 5}, token{"\u662f", 5, 6}, token{"\u5317\u4eac", 6, 8}},
[]token{token{"\u5b59\u541b\u610f", 0, 3}},
[]token{token{"\u5916\u4ea4\u90e8", 0, 3}, token{"\u53d1\u8a00\u4eba", 3, 6}, token{"\u9a6c\u671d\u65ed", 6, 9}},
[]token{token{"\u9886\u5bfc\u4eba", 0, 3}, token{"\u4f1a\u8bae", 3, 5}, token{"\u548c", 5, 6}, token{"\u7b2c\u56db\u5c4a", 6, 9}, token{"\u4e1c\u4e9a", 9, 11}, token{"\u5cf0\u4f1a", 11, 13}},
[]token{token{"\u5728", 0, 1}, token{"\u8fc7\u53bb", 1, 3}, token{"\u7684", 3, 4}, token{"\u8fd9", 4, 5}, token{"\u4e94\u5e74", 5, 7}},
[]token{token{"\u8fd8", 0, 1}, token{"\u9700\u8981", 1, 3}, token{"\u5f88\u957f", 3, 5}, token{"\u7684", 5, 6}, token{"\u8def", 6, 7}, token{"\u8981", 7, 8}, token{"\u8d70", 8, 9}},
[]token{token{"60", 0, 2}, token{"\u5468\u5e74", 2, 4}, token{"\u9996\u90fd", 4, 6}, token{"\u9605\u5175", 6, 8}},
[]token{token{"\u4f60\u597d", 0, 2}, token{"\u4eba\u4eec", 2, 4}, token{"\u5ba1\u7f8e", 4, 6}, token{"\u7684", 6, 7}, token{"\u89c2\u70b9", 7, 9}, token{"\u662f", 9, 10}, token{"\u4e0d\u540c", 10, 12}, token{"\u7684", 12, 13}},
[]token{token{"\u4e70", 0, 1}, token{"\u6c34\u679c", 1, 3}, token{"\u7136\u540e", 3, 5}, token{"\u6765", 5, 6}, token{"\u4e16\u535a\u56ed", 6, 9}},
[]token{token{"\u4e70", 0, 1}, token{"\u6c34\u679c", 1, 3}, token{"\u7136\u540e", 3, 5}, token{"\u53bb", 5, 6}, token{"\u4e16\u535a\u56ed", 6, 9}},
[]token{token{"\u4f46\u662f", 0, 2}, token{"\u540e\u6765", 2, 4}, token{"\u6211", 4, 5}, token{"\u624d", 5, 6}, token{"\u77e5\u9053", 6, 8}, token{"\u4f60", 8, 9}, token{"\u662f", 9, 10}, token{"\u5bf9", 10, 11}, token{"\u7684", 11, 12}},
[]token{token{"\u5b58\u5728", 0, 2}, token{"\u5373", 2, 3}, token{"\u5408\u7406", 3, 5}},
[]token{token{"\u7684", 0, 1}, token{"\u7684", 1, 2}, token{"\u7684", 2, 3}, token{"\u7684", 3, 4}, token{"\u7684", 4, 5}, token{"\u5728", 5, 6}, token{"\u7684", 6, 7}, token{"\u7684", 7, 8}, token{"\u7684", 8, 9}, token{"\u7684", 9, 10}, token{"\u5c31", 10, 11}, token{"\u4ee5", 11, 12}, token{"\u548c", 12, 13}, token{"\u548c", 13, 14}, token{"\u548c", 14, 15}},
[]token{token{"I", 0, 1}, token{" ", 1, 2}, token{"love", 2, 6}, token{"\u4f60", 6, 7}, token{"\uff0c", 7, 8}, token{"\u4e0d\u4ee5\u4e3a\u803b", 8, 12}, token{"\uff0c", 12, 13}, token{"\u53cd", 13, 14}, token{"\u4ee5\u4e3a", 14, 16}, token{"rong", 16, 20}},
[]token{token{"\u56e0", 0, 1}},
[]token{},
[]token{token{"hello", 0, 5}, token{"\u4f60\u597d", 5, 7}, token{"\u4eba\u4eec", 7, 9}, token{"\u5ba1\u7f8e", 9, 11}, token{"\u7684", 11, 12}, token{"\u89c2\u70b9", 12, 14}, token{"\u662f", 14, 15}, token{"\u4e0d\u540c", 15, 17}, token{"\u7684", 17, 18}},
[]token{token{"\u5f88", 0, 1}, token{"\u597d", 1, 2}, token{"\u4f46", 2, 3}, token{"\u4e3b\u8981", 3, 5}, token{"\u662f", 5, 6}, token{"\u57fa\u4e8e", 6, 8}, token{"\u7f51\u9875", 8, 10}, token{"\u5f62\u5f0f", 10, 12}},
[]token{token{"hello", 0, 5}, token{"\u4f60\u597d", 5, 7}, token{"\u4eba\u4eec", 7, 9}, token{"\u5ba1\u7f8e", 9, 11}, token{"\u7684", 11, 12}, token{"\u89c2\u70b9", 12, 14}, token{"\u662f", 14, 15}, token{"\u4e0d\u540c", 15, 17}, token{"\u7684", 17, 18}},
[]token{token{"\u4e3a\u4ec0\u4e48", 0, 3}, token{"\u6211", 3, 4}, token{"\u4e0d\u80fd", 4, 6}, token{"\u62e5\u6709", 6, 8}, token{"\u60f3\u8981", 8, 10}, token{"\u7684", 10, 11}, token{"\u751f\u6d3b", 11, 13}},
[]token{token{"\u540e\u6765", 0, 2}, token{"\u6211", 2, 3}, token{"\u624d", 3, 4}},
[]token{token{"\u6b64\u6b21", 0, 2}, token{"\u6765", 2, 3}, token{"\u4e2d\u56fd", 3, 5}, token{"\u662f", 5, 6}, token{"\u4e3a\u4e86", 6, 8}},
[]token{token{"\u4f7f\u7528", 0, 2}, token{"\u4e86", 2, 3}, token{"\u5b83", 3, 4}, token{"\u5c31", 4, 5}, token{"\u53ef\u4ee5", 5, 7}, token{"\u89e3\u51b3", 7, 9}, token{"\u4e00\u4e9b", 9, 11}, token{"\u95ee\u9898", 11, 13}},
[]token{token{",", 0, 1}, token{"\u4f7f\u7528", 1, 3}, token{"\u4e86", 3, 4}, token{"\u5b83", 4, 5}, token{"\u5c31", 5, 6}, token{"\u53ef\u4ee5", 6, 8}, token{"\u89e3\u51b3", 8, 10}, token{"\u4e00\u4e9b", 10, 12}, token{"\u95ee\u9898", 12, 14}},
[]token{token{"\u5176\u5b9e", 0, 2}, token{"\u4f7f\u7528", 2, 4}, token{"\u4e86", 4, 5}, token{"\u5b83", 5, 6}, token{"\u5c31", 6, 7}, token{"\u53ef\u4ee5", 7, 9}, token{"\u89e3\u51b3", 9, 11}, token{"\u4e00\u4e9b", 11, 13}, token{"\u95ee\u9898", 13, 15}},
[]token{token{"\u597d\u4eba", 0, 2}, token{"\u4f7f\u7528", 2, 4}, token{"\u4e86", 4, 5}, token{"\u5b83", 5, 6}, token{"\u5c31", 6, 7}, token{"\u53ef\u4ee5", 7, 9}, token{"\u89e3\u51b3", 9, 11}, token{"\u4e00\u4e9b", 11, 13}, token{"\u95ee\u9898", 13, 15}},
[]token{token{"\u662f\u56e0\u4e3a", 0, 3}, token{"\u548c", 3, 4}, token{"\u56fd\u5bb6", 4, 6}},
[]token{token{"\u8001\u5e74", 0, 2}, token{"\u641c\u7d22", 2, 4}, token{"\u8fd8", 4, 5}, token{"\u652f\u6301", 5, 7}},
[]token{token{"\u5e72\u8106", 0, 2}, token{"\u5c31", 2, 3}, token{"\u628a", 3, 4}, token{"\u90a3\u90e8", 4, 6}, token{"\u8499\u4eba", 6, 8}, token{"\u7684", 8, 9}, token{"\u95f2\u6cd5", 9, 11}, token{"\u7ed9", 11, 12}, token{"\u5e9f", 12, 13}, token{"\u4e86", 13, 14}, token{"\u62c9\u5012", 14, 16}, token{"\uff01", 16, 17}, token{"RT", 17, 19}, token{" ", 19, 20}, token{"@", 20, 21}, token{"laoshipukong", 21, 33}, token{" ", 33, 34}, token{":", 34, 35}, token{" ", 35, 36}, token{"27", 36, 38}, token{"\u65e5", 38, 39}, token{"\uff0c", 39, 40}, token{"\u5168\u56fd\u4eba\u5927\u5e38\u59d4\u4f1a", 40, 47}, token{"\u7b2c\u4e09\u6b21", 47, 50}, token{"\u5ba1\u8bae", 50, 52}, token{"\u4fb5\u6743", 52, 54}, token{"\u8d23\u4efb\u6cd5", 54, 57}, token{"\u8349\u6848", 57, 59}, token{"\uff0c", 59, 60}, token{"\u5220\u9664", 60, 62}, token{"\u4e86", 62, 63}, token{"\u6709\u5173", 63, 65}, token{"\u533b\u7597", 65, 67}, token{"\u635f\u5bb3", 67, 69}, token{"\u8d23\u4efb", 69, 71}, token{"\u201c", 71, 72}, token{"\u4e3e\u8bc1", 72, 74}, token{"\u5012\u7f6e", 74, 76}, token{"\u201d", 76, 77}, token{"\u7684", 77, 78}, token{"\u89c4\u5b9a", 78, 80}, token{"\u3002", 80, 81}, token{"\u5728", 81, 82}, token{"\u533b\u60a3", 82, 84}, token{"\u7ea0\u7eb7", 84, 86}, token{"\u4e2d\u672c", 86, 88}, token{"\u5df2", 88, 89}, token{"\u5904\u4e8e", 89, 91}, token{"\u5f31\u52bf", 91, 93}, token{"\u5730\u4f4d", 93, 95}, token{"\u7684", 95, 96}, token{"\u6d88\u8d39\u8005", 96, 99}, token{"\u7531\u6b64", 99, 101}, token{"\u5c06", 101, 102}, token{"\u9677\u5165", 102, 104}, token{"\u4e07\u52ab\u4e0d\u590d", 104, 108}, token{"\u7684", 108, 109}, token{"\u5883\u5730", 109, 111}, token{"\u3002", 111, 112}, token{" ", 112, 113}},
[]token{token{"\u5927", 0, 1}},
[]token{},
[]token{token{"\u4ed6", 0, 1}, token{"\u8bf4", 1, 2}, token{"\u7684", 2, 3}, token{"\u786e\u5b9e", 3, 5}, token{"\u5728\u7406", 5, 7}},
[]token{token{"\u957f\u6625", 0, 2}, token{"\u5e02\u957f", 2, 4}, token{"\u6625\u8282", 4, 6}, token{"\u8bb2\u8bdd", 6, 8}},
[]token{token{"\u7ed3\u5a5a", 0, 2}, token{"\u7684", 2, 3}, token{"\u548c", 3, 4}, token{"\u5c1a\u672a", 4, 6}, token{"\u7ed3\u5a5a", 6, 8}, token{"\u7684", 8, 9}},
[]token{token{"\u7ed3\u5408", 0, 2}, token{"\u6210", 2, 3}, token{"\u5206\u5b50", 3, 5}, token{"\u65f6", 5, 6}},
[]token{token{"\u65c5\u6e38", 0, 2}, token{"\u548c", 2, 3}, token{"\u670d\u52a1", 3, 5}, token{"\u662f", 5, 6}, token{"\u6700\u597d", 6, 8}, token{"\u7684", 8, 9}},
[]token{token{"\u8fd9\u4ef6", 0, 2}, token{"\u4e8b\u60c5", 2, 4}, token{"\u7684\u786e", 4, 6}, token{"\u662f", 6, 7}, token{"\u6211", 7, 8}, token{"\u7684", 8, 9}, token{"\u9519", 9, 10}},
[]token{token{"\u4f9b", 0, 1}, token{"\u5927\u5bb6", 1, 3}, token{"\u53c2\u8003", 3, 5}, token{"\u6307\u6b63", 5, 7}},
[]token{token{"\u54c8\u5c14\u6ee8", 0, 3}, token{"\u653f\u5e9c", 3, 5}, token{"\u516c\u5e03", 5, 7}, token{"\u584c\u6865", 7, 9}, token{"\u539f\u56e0", 9, 11}},
[]token{token{"\u6211", 0, 1}, token{"\u5728", 1, 2}, token{"\u673a\u573a", 2, 4}, token{"\u5165\u53e3\u5904", 4, 7}},
[]token{token{"\u90a2\u6c38\u81e3", 0, 3}, token{"\u6444\u5f71", 3, 5}, token{"\u62a5\u9053", 5, 7}},
[]token{token{"BP", 0, 2}, token{"\u795e\u7ecf\u7f51\u7edc", 2, 6}, token{"\u5982\u4f55", 6, 8}, token{"\u8bad\u7ec3", 8, 10}, token{"\u624d\u80fd", 10, 12}, token{"\u5728", 12, 13}, token{"\u5206\u7c7b", 13, 15}, token{"\u65f6", 15, 16}, token{"\u589e\u52a0", 16, 18}, token{"\u533a\u5206\u5ea6", 18, 21}, token{"\uff1f", 21, 22}},
[]token{token{"\u5357\u4eac\u5e02", 0, 3}, token{"\u957f\u6c5f\u5927\u6865", 3, 7}},
[]token{token{"\u5e94", 0, 1}, token{"\u4e00\u4e9b", 1, 3}, token{"\u4f7f\u7528\u8005", 3, 6}, token{"\u7684", 6, 7}, token{"\u5efa\u8bae", 7, 9}, token{"\uff0c", 9, 10}, token{"\u4e5f", 10, 11}, token{"\u4e3a\u4e86", 11, 13}, token{"\u4fbf\u4e8e", 13, 15}, token{"\u5229\u7528", 15, 17}, token{"NiuTrans", 17, 25}, token{"\u7528\u4e8e", 25, 27}, token{"SMT", 27, 30}, token{"\u7814\u7a76", 30, 32}},
[]token{token{"\u957f\u6625\u5e02", 0, 3}, token{"\u957f\u6625", 3, 5}, token{"\u836f\u5e97", 5, 7}},
[]token{token{"\u9093\u9896\u8d85", 0, 3}, token{"\u751f\u524d", 3, 5}, token{"\u6700", 5, 6}, token{"\u559c\u6b22", 6, 8}, token{"\u7684", 8, 9}, token{"\u8863\u670d", 9, 11}},
[]token{token{"\u80e1\u9526\u6d9b", 0, 3}, token{"\u662f", 3, 4}, token{"\u70ed\u7231", 4, 6}, token{"\u4e16\u754c", 6, 8}, token{"\u548c\u5e73", 8, 10}, token{"\u7684", 10, 11}, token{"\u653f\u6cbb\u5c40", 11, 14}, token{"\u5e38\u59d4", 14, 16}},
[]token{token{"\u7a0b\u5e8f\u5458", 0, 3}, token{"\u795d", 3, 4}, token{"\u6d77\u6797", 4, 6}, token{"\u548c", 6, 7}, token{"\u6731\u4f1a\u9707", 7, 10}, token{"\u662f", 10, 11}, token{"\u5728", 11, 12}, token{"\u5b59\u5065", 12, 14}, token{"\u7684", 14, 15}, token{"\u5de6\u9762", 15, 17}, token{"\u548c", 17, 18}, token{"\u53f3\u9762", 18, 20}, token{",", 20, 21}, token{" ", 21, 22}, token{"\u8303\u51ef", 22, 24}, token{"\u5728", 24, 25}, token{"\u6700", 25, 26}, token{"\u53f3\u9762", 26, 28}, token{".", 28, 29}, token{"\u518d\u5f80", 29, 31}, token{"\u5de6", 31, 32}, token{"\u662f", 32, 33}, token{"\u674e\u677e\u6d2a", 33, 36}},
[]token{token{"\u4e00\u6b21\u6027", 0, 3}, token{"\u4ea4", 3, 4}, token{"\u591a\u5c11", 4, 6}, token{"\u94b1", 6, 7}},
[]token{token{"\u4e24\u5757", 0, 2}, token{"\u4e94", 2, 3}, token{"\u4e00\u5957", 3, 5}, token{"\uff0c", 5, 6}, token{"\u4e09\u5757", 6, 8}, token{"\u516b", 8, 9}, token{"\u4e00\u65a4", 9, 11}, token{"\uff0c", 11, 12}, token{"\u56db\u5757", 12, 14}, token{"\u4e03", 14, 15}, token{"\u4e00\u672c", 15, 17}, token{"\uff0c", 17, 18}, token{"\u4e94\u5757", 18, 20}, token{"\u516d", 20, 21}, token{"\u4e00\u6761", 21, 23}},
[]token{token{"\u5c0f", 0, 1}, token{"\u548c\u5c1a", 1, 3}, token{"\u7559", 3, 4}, token{"\u4e86", 4, 5}, token{"\u4e00\u4e2a", 5, 7}, token{"\u50cf", 7, 8}, token{"\u5927", 8, 9}, token{"\u548c\u5c1a", 9, 11}, token{"\u4e00\u6837", 11, 13}, token{"\u7684", 13, 14}, token{"\u548c\u5c1a\u5934", 14, 17}},
[]token{token{"\u6211", 0, 1}, token{"\u662f", 1, 2}, token{"\u4e2d\u534e\u4eba\u6c11\u5171\u548c\u56fd", 2, 9}, token{"\u516c\u6c11", 9, 11}, token{";", 11, 12}, token{"\u6211", 12, 13}, token{"\u7238\u7238", 13, 15}, token{"\u662f", 15, 16}, token{"\u5171\u548c\u515a", 16, 19}, token{"\u515a\u5458", 19, 21}, token{";", 21, 22}, token{" ", 22, 23}, token{"\u5730\u94c1", 23, 25}, token{"\u548c\u5e73\u95e8", 25, 28}, token{"\u7ad9", 28, 29}},
[]token{token{"\u5f20\u6653\u6885", 0, 3}, token{"\u53bb", 3, 4}, token{"\u4eba\u6c11", 4, 6}, token{"\u533b\u9662", 6, 8}, token{"\u505a", 8, 9}, token{"\u4e86", 9, 10}, token{"\u4e2a", 10, 11}, token{"B\u8d85", 11, 13}, token{"\u7136\u540e", 13, 15}, token{"\u53bb", 15, 16}, token{"\u4e70", 16, 17}, token{"\u4e86", 17, 18}, token{"\u4ef6", 18, 19}, token{"T\u6064", 19, 21}},
[]token{token{"AT&T", 0, 4}, token{"\u662f", 4, 5}, token{"\u4e00\u4ef6", 5, 7}, token{"\u4e0d\u9519", 7, 9}, token{"\u7684", 9, 10}, token{"\u516c\u53f8", 10, 12}, token{"\uff0c", 12, 13}, token{"\u7ed9", 13, 14}, token{"\u4f60", 14, 15}, token{"\u53d1", 15, 16}, token{"offer", 16, 21}, token{"\u4e86", 21, 22}, token{"\u5417", 22, 23}, token{"\uff1f", 23, 24}},
[]token{token{"C++", 0, 3}, token{"\u548c", 3, 4}, token{"c#", 4, 6}, token{"\u662f", 6, 7}, token{"\u4ec0\u4e48", 7, 9}, token{"\u5173\u7cfb", 9, 11}, token{"\uff1f", 11, 12}, token{"11", 12, 14}, token{"+", 14, 15}, token{"122", 15, 18}, token{"=", 18, 19}, token{"133", 19, 22}, token{"\uff0c", 22, 23}, token{"\u662f", 23, 24}, token{"\u5417", 24, 25}, token{"\uff1f", 25, 26}, token{"PI", 26, 28}, token{"=", 28, 29}, token{"3.14159", 29, 36}},
[]token{token{"\u4f60", 0, 1}, token{"\u8ba4\u8bc6", 1, 3}, token{"\u90a3\u4e2a", 3, 5}, token{"\u548c", 5, 6}, token{"\u4e3b\u5e2d", 6, 8}, token{"\u63e1\u624b", 8, 10}, token{"\u7684", 10, 11}, token{"\u7684\u54e5", 11, 13}, token{"\u5417", 13, 14}, token{"\uff1f", 14, 15}, token{"\u4ed6\u5f00", 15, 17}, token{"\u4e00\u8f86", 17, 19}, token{"\u9ed1\u8272", 19, 21}, token{"\u7684\u58eb", 21, 23}, token{"\u3002", 23, 24}},
[]token{token{"\u67aa\u6746\u5b50", 0, 3}, token{"\u4e2d", 3, 4}, token{"\u51fa", 4, 5}, token{"\u653f\u6743", 5, 7}},
[]token{token{"\u5f20\u4e09\u98ce", 0, 3}, token{"\u540c\u5b66", 3, 5}, token{"\u8d70\u4e0a", 5, 7}, token{"\u4e86", 7, 8}, token{"\u4e0d\u5f52\u8def", 8, 11}},
[]token{token{"\u963fQ", 0, 2}, token{"\u8170\u95f4", 2, 4}, token{"\u6302", 4, 5}, token{"\u7740", 5, 6}, token{"BB\u673a", 6, 9}, token{"\u624b\u91cc", 9, 11}, token{"\u62ff", 11, 12}, token{"\u7740", 12, 13}, token{"\u5927\u54e5\u5927", 13, 16}, token{"\uff0c", 16, 17}, token{"\u8bf4", 17, 18}, token{"\uff1a", 18, 19}, token{"\u6211", 19, 20}, token{"\u4e00\u822c", 20, 22}, token{"\u5403\u996d", 22, 24}, token{"\u4e0d", 24, 25}, token{"AA\u5236", 25, 28}, token{"\u7684", 28, 29}, token{"\u3002", 29, 30}},
[]token{token{"\u5728", 0, 1}, token{"1\u53f7\u5e97", 1, 4}, token{"\u80fd", 4, 5}, token{"\u4e70", 5, 6}, token{"\u5230", 6, 7}, token{"\u5c0fS", 7, 9}, token{"\u548c", 9, 10}, token{"\u5927S", 10, 12}, token{"\u516b\u5366", 12, 14}, token{"\u7684", 14, 15}, token{"\u4e66", 15, 16}, token{"\u3002", 16, 17}},
[]token{token{"\u8fd9\u662f", 0, 2}, token{"\u4e00\u4e2a", 2, 4}, token{"\u4f38\u624b", 4, 6}, token{"\u4e0d\u89c1", 6, 8}, token{"\u4e94\u6307", 8, 10}, token{"\u4f38\u624b\u4e0d\u89c1\u4e94\u6307", 4, 10}, token{"\u7684", 10, 11}, token{"\u9ed1\u591c", 11, 13}, token{"\u3002", 13, 14}, token{"\u6211", 14, 15}, token{"\u53eb", 15, 16}, token{"\u609f\u7a7a", 17, 19}, token{"\u5b59\u609f\u7a7a", 16, 19}, token{"\uff0c", 19, 20}, token{"\u6211", 20, 21}, token{"\u7231", 21, 22}, token{"\u5317\u4eac", 22, 24}, token{"\uff0c", 24, 25}, token{"\u6211", 25, 26}, token{"\u7231", 26, 27}, token{"Python", 27, 33}, token{"\u548c", 33, 34}, token{"C++", 34, 37}, token{"\u3002", 37, 38}},
[]token{token{"\u6211", 0, 1}, token{"\u4e0d", 1, 2}, token{"\u559c\u6b22", 2, 4}, token{"\u65e5\u672c", 4, 6}, token{"\u548c\u670d", 6, 8}, token{"\u3002", 8, 9}},
[]token{token{"\u96f7\u7334", 0, 2}, token{"\u56de\u5f52", 2, 4}, token{"\u4eba\u95f4", 4, 6}, token{"\u3002", 6, 7}},
[]token{token{"\u5de5\u4fe1\u5904", 0, 3}, token{"\u5e72\u4e8b", 4, 6}, token{"\u5973\u5e72\u4e8b", 3, 6}, token{"\u6bcf\u6708", 6, 8}, token{"\u7ecf\u8fc7", 8, 10}, token{"\u4e0b\u5c5e", 10, 12}, token{"\u79d1\u5ba4", 12, 14}, token{"\u90fd", 14, 15}, token{"\u8981", 15, 16}, token{"\u4eb2\u53e3", 16, 18}, token{"\u4ea4\u4ee3", 18, 20}, token{"24", 20, 22}, token{"\u53e3", 22, 23}, token{"\u4ea4\u6362", 23, 25}, token{"\u6362\u673a", 24, 26}, token{"\u4ea4\u6362\u673a", 23, 26}, token{"\u7b49", 26, 27}, token{"\u6280\u672f", 27, 29}, token{"\u6280\u672f\u6027", 27, 30}, token{"\u5668\u4ef6", 30, 32}, token{"\u7684", 32, 33}, token{"\u5b89\u88c5", 33, 35}, token{"\u5de5\u4f5c", 35, 37}},
[]token{token{"\u6211", 0, 1}, token{"\u9700\u8981", 1, 3}, token{"\u5ec9\u79df", 3, 5}, token{"\u79df\u623f", 4, 6}, token{"\u5ec9\u79df\u623f", 3, 6}},
[]token{token{"\u6c38\u548c", 0, 2}, token{"\u670d\u88c5", 2, 4}, token{"\u9970\u54c1", 4, 6}, token{"\u6709\u9650", 6, 8}, token{"\u516c\u53f8", 8, 10}, token{"\u6709\u9650\u516c\u53f8", 6, 10}},
[]token{token{"\u6211", 0, 1}, token{"\u7231", 1, 2}, token{"\u5317\u4eac", 2, 4}, token{"\u5929\u5b89", 4, 6}, token{"\u5929\u5b89\u95e8", 4, 7}},
[]token{token{"abc", 0, 3}},
[]token{token{"\u9690", 0, 1}, token{"\u53ef\u592b", 3, 5}, token{"\u9a6c\u5c14\u53ef", 1, 4}, token{"\u9a6c\u5c14\u53ef\u592b", 1, 5}},
[]token{token{"\u96f7\u7334", 0, 2}, token{"\u662f", 2, 3}, token{"\u4e2a", 3, 4}, token{"\u597d", 4, 5}, token{"\u7f51\u7ad9", 5, 7}},
[]token{token{"\u201c", 0, 1}, token{"Microsoft", 1, 10}, token{"\u201d", 10, 11}, token{"\u4e00\u8bcd", 11, 13}, token{"\u7531", 13, 14}, token{"\u201c", 14, 15}, token{"MICROcomputer", 15, 28}, token{"\uff08", 28, 29}, token{"\u5fae\u578b", 29, 31}, token{"\u8ba1\u7b97", 31, 33}, token{"\u7b97\u673a", 32, 34}, token{"\u8ba1\u7b97\u673a", 31, 34}, token{"\uff09", 34, 35}, token{"\u201d", 35, 36}, token{"\u548c", 36, 37}, token{"\u201c", 37, 38}, token{"SOFTware", 38, 46}, token{"\uff08", 46, 47}, token{"\u8f6f\u4ef6", 47, 49}, token{"\uff09", 49, 50}, token{"\u201d", 50, 51}, token{"\u4e24", 51, 52}, token{"\u90e8\u5206", 52, 54}, token{"\u7ec4\u6210", 54, 56}},
[]token{token{"\u8349\u6ce5\u9a6c", 0, 3}, token{"\u548c", 3, 4}, token{"\u6b3a\u5b9e", 4, 6}, token{"\u9a6c", 6, 7}, token{"\u662f", 7, 8}, token{"\u4eca\u5e74", 8, 10}, token{"\u7684", 10, 11}, token{"\u6d41\u884c", 11, 13}, token{"\u8bcd\u6c47", 13, 15}},
[]token{token{"\u4f0a\u85e4", 0, 2}, token{"\u6d0b\u534e\u5802", 2, 5}, token{"\u603b\u5e9c", 5, 7}, token{"\u5e97", 7, 8}},
[]token{token{"\u4e2d\u56fd", 0, 2}, token{"\u79d1\u5b66", 2, 4}, token{"\u5b66\u9662", 3, 5}, token{"\u8ba1\u7b97", 5, 7}, token{"\u6280\u672f", 7, 9}, token{"\u7814\u7a76", 9, 11}, token{"\u79d1\u5b66\u9662", 2, 5}, token{"\u7814\u7a76\u6240", 9, 12}, token{"\u4e2d\u56fd\u79d1\u5b66\u9662\u8ba1\u7b97\u6280\u672f\u7814\u7a76\u6240", 0, 12}},
[]token{token{"\u7f57\u5bc6\u6b27", 0, 3}, token{"\u4e0e", 3, 4}, token{"\u6731\u4e3d\u53f6", 4, 7}},
[]token{token{"\u6211", 0, 1}, token{"\u8d2d\u4e70", 1, 3}, token{"\u4e86", 3, 4}, token{"\u9053\u5177", 4, 6}, token{"\u548c", 6, 7}, token{"\u670d\u88c5", 7, 9}},
[]token{token{"PS", 0, 2}, token{":", 2, 3}, token{" ", 3, 4}, token{"\u6211", 4, 5}, token{"\u89c9\u5f97", 5, 7}, token{"\u5f00\u6e90", 7, 9}, token{"\u6709", 9, 10}, token{"\u4e00\u4e2a", 10, 12}, token{"\u597d\u5904", 12, 14}, token{"\uff0c", 14, 15}, token{"\u5c31\u662f", 15, 17}, token{"\u80fd\u591f", 17, 19}, token{"\u6566\u4fc3", 19, 21}, token{"\u81ea\u5df1", 21, 23}, token{"\u4e0d\u65ad", 23, 25}, token{"\u6539\u8fdb", 25, 27}, token{"\u4e0d\u65ad\u6539\u8fdb", 23, 27}, token{"\uff0c", 27, 28}, token{"\u907f\u514d", 28, 30}, token{"\u655e\u5e1a", 30, 32}, token{"\u81ea\u73cd", 32, 34}},
[]token{token{"\u6e56\u5317", 0, 2}, token{"\u6e56\u5317\u7701", 0, 3}, token{"\u77f3\u9996", 3, 5}, token{"\u77f3\u9996\u5e02", 3, 6}},
[]token{token{"\u6e56\u5317", 0, 2}, token{"\u6e56\u5317\u7701", 0, 3}, token{"\u5341\u5830", 3, 5}, token{"\u5341\u5830\u5e02", 3, 6}},
[]token{token{"\u7ecf\u7406", 1, 3}, token{"\u603b\u7ecf\u7406", 0, 3}, token{"\u5b8c\u6210", 3, 5}, token{"\u4e86", 5, 6}, token{"\u8fd9\u4ef6", 6, 8}, token{"\u4e8b\u60c5", 8, 10}},
[]token{token{"\u7535\u8111", 0, 2}, token{"\u4fee\u597d", 2, 4}, token{"\u4e86", 4, 5}},
[]token{token{"\u505a\u597d", 0, 2}, token{"\u4e86", 2, 3}, token{"\u8fd9\u4ef6", 3, 5}, token{"\u4e8b\u60c5", 5, 7}, token{"\u5c31", 7, 8}, token{"\u4e00\u4e86\u767e\u4e86", 8, 12}, token{"\u4e86", 12, 13}},
[]token{token{"\u4eba\u4eec", 0, 2}, token{"\u5ba1\u7f8e", 2, 4}, token{"\u7684", 4, 5}, token{"\u89c2\u70b9", 5, 7}, token{"\u662f", 7, 8}, token{"\u4e0d\u540c", 8, 10}, token{"\u7684", 10, 11}},
[]token{token{"\u6211\u4eec", 0, 2}, token{"\u4e70", 2, 3}, token{"\u4e86", 3, 4}, token{"\u4e00\u4e2a", 4, 6}, token{"\u7f8e\u7684", 6, 8}, token{"\u7a7a\u8c03", 8, 10}},
[]token{token{"\u7ebf\u7a0b", 0, 2}, token{"\u521d\u59cb", 2, 4}, token{"\u521d\u59cb\u5316", 2, 5}, token{"\u65f6", 5, 6}, token{"\u6211\u4eec", 6, 8}, token{"\u8981", 8, 9}, token{"\u6ce8\u610f", 9, 11}},
[]token{token{"\u4e00\u4e2a", 0, 2}, token{"\u5206\u5b50", 2, 4}, token{"\u662f", 4, 5}, token{"\u7531", 5, 6}, token{"\u597d\u591a", 6, 8}, token{"\u539f\u5b50", 8, 10}, token{"\u7ec4\u7ec7", 10, 12}, token{"\u6210", 12, 13}, token{"\u7684", 13, 14}},
[]token{token{"\u795d", 0, 1}, token{"\u4f60", 1, 2}, token{"\u9a6c\u5230\u529f\u6210", 2, 6}},
[]token{token{"\u4ed6", 0, 1}, token{"\u6389", 1, 2}, token{"\u8fdb", 2, 3}, token{"\u4e86", 3, 4}, token{"\u65e0\u5e95", 4, 6}, token{"\u65e0\u5e95\u6d1e", 4, 7}, token{"\u91cc", 7, 8}},
[]token{token{"\u4e2d\u56fd", 0, 2}, token{"\u7684", 2, 3}, token{"\u9996\u90fd", 3, 5}, token{"\u662f", 5, 6}, token{"\u5317\u4eac", 6, 8}},
[]token{token{"\u5b59\u541b\u610f", 0, 3}},
[]token{token{"\u5916\u4ea4", 0, 2}, token{"\u5916\u4ea4\u90e8", 0, 3}, token{"\u53d1\u8a00", 3, 5}, token{"\u53d1\u8a00\u4eba", 3, 6}, token{"\u9a6c\u671d\u65ed", 6, 9}},
[]token{token{"\u9886\u5bfc", 0, 2}, token{"\u9886\u5bfc\u4eba", 0, 3}, token{"\u4f1a\u8bae", 3, 5}, token{"\u548c", 5, 6}, token{"\u7b2c\u56db", 6, 8}, token{"\u56db\u5c4a", 7, 9}, token{"\u7b2c\u56db\u5c4a", 6, 9}, token{"\u4e1c\u4e9a", 9, 11}, token{"\u5cf0\u4f1a", 11, 13}},
[]token{token{"\u5728", 0, 1}, token{"\u8fc7\u53bb", 1, 3}, token{"\u7684", 3, 4}, token{"\u8fd9", 4, 5}, token{"\u4e94\u5e74", 5, 7}},
[]token{token{"\u8fd8", 0, 1}, token{"\u9700\u8981", 1, 3}, token{"\u5f88\u957f", 3, 5}, token{"\u7684", 5, 6}, token{"\u8def", 6, 7}, token{"\u8981", 7, 8}, token{"\u8d70", 8, 9}},
[]token{token{"60", 0, 2}, token{"\u5468\u5e74", 2, 4}, token{"\u9996\u90fd", 4, 6}, token{"\u9605\u5175", 6, 8}},
[]token{token{"\u4f60\u597d", 0, 2}, token{"\u4eba\u4eec", 2, 4}, token{"\u5ba1\u7f8e", 4, 6}, token{"\u7684", 6, 7}, token{"\u89c2\u70b9", 7, 9}, token{"\u662f", 9, 10}, token{"\u4e0d\u540c", 10, 12}, token{"\u7684", 12, 13}},
[]token{token{"\u4e70", 0, 1}, token{"\u6c34\u679c", 1, 3}, token{"\u7136\u540e", 3, 5}, token{"\u6765", 5, 6}, token{"\u4e16\u535a", 6, 8}, token{"\u535a\u56ed", 7, 9}, token{"\u4e16\u535a\u56ed", 6, 9}},
[]token{token{"\u4e70", 0, 1}, token{"\u6c34\u679c", 1, 3}, token{"\u7136\u540e", 3, 5}, token{"\u53bb", 5, 6}, token{"\u4e16\u535a", 6, 8}, token{"\u535a\u56ed", 7, 9}, token{"\u4e16\u535a\u56ed", 6, 9}},
[]token{token{"\u4f46\u662f", 0, 2}, token{"\u540e\u6765", 2, 4}, token{"\u6211", 4, 5}, token{"\u624d", 5, 6}, token{"\u77e5\u9053", 6, 8}, token{"\u4f60", 8, 9}, token{"\u662f", 9, 10}, token{"\u5bf9", 10, 11}, token{"\u7684", 11, 12}},
[]token{token{"\u5b58\u5728", 0, 2}, token{"\u5373", 2, 3}, token{"\u5408\u7406", 3, 5}},
[]token{token{"\u7684", 0, 1}, token{"\u7684", 1, 2}, token{"\u7684", 2, 3}, token{"\u7684", 3, 4}, token{"\u7684", 4, 5}, token{"\u5728", 5, 6}, token{"\u7684", 6, 7}, token{"\u7684", 7, 8}, token{"\u7684", 8, 9}, token{"\u7684", 9, 10}, token{"\u5c31", 10, 11}, token{"\u4ee5", 11, 12}, token{"\u548c", 12, 13}, token{"\u548c", 13, 14}, token{"\u548c", 14, 15}},
[]token{token{"I", 0, 1}, token{" ", 1, 2}, token{"love", 2, 6}, token{"\u4f60", 6, 7}, token{"\uff0c", 7, 8}, token{"\u4e0d\u4ee5", 8, 10}, token{"\u4ee5\u4e3a", 9, 11}, token{"\u4e0d\u4ee5\u4e3a\u803b", 8, 12}, token{"\uff0c", 12, 13}, token{"\u53cd", 13, 14}, token{"\u4ee5\u4e3a", 14, 16}, token{"rong", 16, 20}},
[]token{token{"\u56e0", 0, 1}},
[]token{},
[]token{token{"hello", 0, 5}, token{"\u4f60\u597d", 5, 7}, token{"\u4eba\u4eec", 7, 9}, token{"\u5ba1\u7f8e", 9, 11}, token{"\u7684", 11, 12}, token{"\u89c2\u70b9", 12, 14}, token{"\u662f", 14, 15}, token{"\u4e0d\u540c", 15, 17}, token{"\u7684", 17, 18}},
[]token{token{"\u5f88", 0, 1}, token{"\u597d", 1, 2}, token{"\u4f46", 2, 3}, token{"\u4e3b\u8981", 3, 5}, token{"\u662f", 5, 6}, token{"\u57fa\u4e8e", 6, 8}, token{"\u7f51\u9875", 8, 10}, token{"\u5f62\u5f0f", 10, 12}},
[]token{token{"hello", 0, 5}, token{"\u4f60\u597d", 5, 7}, token{"\u4eba\u4eec", 7, 9}, token{"\u5ba1\u7f8e", 9, 11}, token{"\u7684", 11, 12}, token{"\u89c2\u70b9", 12, 14}, token{"\u662f", 14, 15}, token{"\u4e0d\u540c", 15, 17}, token{"\u7684", 17, 18}},
[]token{token{"\u4ec0\u4e48", 1, 3}, token{"\u4e3a\u4ec0\u4e48", 0, 3}, token{"\u6211", 3, 4}, token{"\u4e0d\u80fd", 4, 6}, token{"\u62e5\u6709", 6, 8}, token{"\u60f3\u8981", 8, 10}, token{"\u7684", 10, 11}, token{"\u751f\u6d3b", 11, 13}},
[]token{token{"\u540e\u6765", 0, 2}, token{"\u6211", 2, 3}, token{"\u624d", 3, 4}},
[]token{token{"\u6b64\u6b21", 0, 2}, token{"\u6765", 2, 3}, token{"\u4e2d\u56fd", 3, 5}, token{"\u662f", 5, 6}, token{"\u4e3a\u4e86", 6, 8}},
[]token{token{"\u4f7f\u7528", 0, 2}, token{"\u4e86", 2, 3}, token{"\u5b83", 3, 4}, token{"\u5c31", 4, 5}, token{"\u53ef\u4ee5", 5, 7}, token{"\u89e3\u51b3", 7, 9}, token{"\u4e00\u4e9b", 9, 11}, token{"\u95ee\u9898", 11, 13}},
[]token{token{",", 0, 1}, token{"\u4f7f\u7528", 1, 3}, token{"\u4e86", 3, 4}, token{"\u5b83", 4, 5}, token{"\u5c31", 5, 6}, token{"\u53ef\u4ee5", 6, 8}, token{"\u89e3\u51b3", 8, 10}, token{"\u4e00\u4e9b", 10, 12}, token{"\u95ee\u9898", 12, 14}},
[]token{token{"\u5176\u5b9e", 0, 2}, token{"\u4f7f\u7528", 2, 4}, token{"\u4e86", 4, 5}, token{"\u5b83", 5, 6}, token{"\u5c31", 6, 7}, token{"\u53ef\u4ee5", 7, 9}, token{"\u89e3\u51b3", 9, 11}, token{"\u4e00\u4e9b", 11, 13}, token{"\u95ee\u9898", 13, 15}},
[]token{token{"\u597d\u4eba", 0, 2}, token{"\u4f7f\u7528", 2, 4}, token{"\u4e86", 4, 5}, token{"\u5b83", 5, 6}, token{"\u5c31", 6, 7}, token{"\u53ef\u4ee5", 7, 9}, token{"\u89e3\u51b3", 9, 11}, token{"\u4e00\u4e9b", 11, 13}, token{"\u95ee\u9898", 13, 15}},
[]token{token{"\u56e0\u4e3a", 1, 3}, token{"\u662f\u56e0\u4e3a", 0, 3}, token{"\u548c", 3, 4}, token{"\u56fd\u5bb6", 4, 6}},
[]token{token{"\u8001\u5e74", 0, 2}, token{"\u641c\u7d22", 2, 4}, token{"\u8fd8", 4, 5}, token{"\u652f\u6301", 5, 7}},
[]token{token{"\u5e72\u8106", 0, 2}, token{"\u5c31", 2, 3}, token{"\u628a", 3, 4}, token{"\u90a3\u90e8", 4, 6}, token{"\u8499\u4eba", 6, 8}, token{"\u7684", 8, 9}, token{"\u95f2\u6cd5", 9, 11}, token{"\u7ed9", 11, 12}, token{"\u5e9f", 12, 13}, token{"\u4e86", 13, 14}, token{"\u62c9\u5012", 14, 16}, token{"\uff01", 16, 17}, token{"RT", 17, 19}, token{" ", 19, 20}, token{"@", 20, 21}, token{"laoshipukong", 21, 33}, token{" ", 33, 34}, token{":", 34, 35}, token{" ", 35, 36}, token{"27", 36, 38}, token{"\u65e5", 38, 39}, token{"\uff0c", 39, 40}, token{"\u5168\u56fd", 40, 42}, token{"\u56fd\u4eba", 41, 43}, token{"\u4eba\u5927", 42, 44}, token{"\u5e38\u59d4", 44, 46}, token{"\u59d4\u4f1a", 45, 47}, token{"\u5e38\u59d4\u4f1a", 44, 47}, token{"\u5168\u56fd\u4eba\u5927\u5e38\u59d4\u4f1a", 40, 47}, token{"\u7b2c\u4e09", 47, 49}, token{"\u4e09\u6b21", 48, 50}, token{"\u7b2c\u4e09\u6b21", 47, 50}, token{"\u5ba1\u8bae", 50, 52}, token{"\u4fb5\u6743", 52, 54}, token{"\u8d23\u4efb", 54, 56}, token{"\u8d23\u4efb\u6cd5", 54, 57}, token{"\u8349\u6848", 57, 59}, token{"\uff0c", 59, 60}, token{"\u5220\u9664", 60, 62}, token{"\u4e86", 62, 63}, token{"\u6709\u5173", 63, 65}, token{"\u533b\u7597", 65, 67}, token{"\u635f\u5bb3", 67, 69}, token{"\u8d23\u4efb", 69, 71}, token{"\u201c", 71, 72}, token{"\u4e3e\u8bc1", 72, 74}, token{"\u5012\u7f6e", 74, 76}, token{"\u201d", 76, 77}, token{"\u7684", 77, 78}, token{"\u89c4\u5b9a", 78, 80}, token{"\u3002", 80, 81}, token{"\u5728", 81, 82}, token{"\u533b\u60a3", 82, 84}, token{"\u7ea0\u7eb7", 84, 86}, token{"\u4e2d\u672c", 86, 88}, token{"\u5df2", 88, 89}, token{"\u5904\u4e8e", 89, 91}, token{"\u5f31\u52bf", 91, 93}, token{"\u5730\u4f4d", 93, 95}, token{"\u7684", 95, 96}, token{"\u6d88\u8d39", 96, 98}, token{"\u6d88\u8d39\u8005", 96, 99}, token{"\u7531\u6b64", 99, 101}, token{"\u5c06", 101, 102}, token{"\u9677\u5165", 102, 104}, token{"\u4e0d\u590d", 106, 108}, token{"\u4e07\u52ab\u4e0d\u590d", 104, 108}, token{"\u7684", 108, 109}, token{"\u5883\u5730", 109, 111}, token{"\u3002", 111, 112}, token{" ", 112, 113}},
[]token{token{"\u5927", 0, 1}},
[]token{},
[]token{token{"\u4ed6", 0, 1}, token{"\u8bf4", 1, 2}, token{"\u7684", 2, 3}, token{"\u786e\u5b9e", 3, 5}, token{"\u5728\u7406", 5, 7}},
[]token{token{"\u957f\u6625", 0, 2}, token{"\u5e02\u957f", 2, 4}, token{"\u6625\u8282", 4, 6}, token{"\u8bb2\u8bdd", 6, 8}},
[]token{token{"\u7ed3\u5a5a", 0, 2}, token{"\u7684", 2, 3}, token{"\u548c", 3, 4}, token{"\u5c1a\u672a", 4, 6}, token{"\u7ed3\u5a5a", 6, 8}, token{"\u7684", 8, 9}},
[]token{token{"\u7ed3\u5408", 0, 2}, token{"\u6210", 2, 3}, token{"\u5206\u5b50", 3, 5}, token{"\u65f6", 5, 6}},
[]token{token{"\u65c5\u6e38", 0, 2}, token{"\u548c", 2, 3}, token{"\u670d\u52a1", 3, 5}, token{"\u662f", 5, 6}, token{"\u6700\u597d", 6, 8}, token{"\u7684", 8, 9}},
[]token{token{"\u8fd9\u4ef6", 0, 2}, token{"\u4e8b\u60c5", 2, 4}, token{"\u7684\u786e", 4, 6}, token{"\u662f", 6, 7}, token{"\u6211", 7, 8}, token{"\u7684", 8, 9}, token{"\u9519", 9, 10}},
[]token{token{"\u4f9b", 0, 1}, token{"\u5927\u5bb6", 1, 3}, token{"\u53c2\u8003", 3, 5}, token{"\u6307\u6b63", 5, 7}},
[]token{token{"\u54c8\u5c14", 0, 2}, token{"\u54c8\u5c14\u6ee8", 0, 3}, token{"\u653f\u5e9c", 3, 5}, token{"\u516c\u5e03", 5, 7}, token{"\u584c\u6865", 7, 9}, token{"\u539f\u56e0", 9, 11}},
[]token{token{"\u6211", 0, 1}, token{"\u5728", 1, 2}, token{"\u673a\u573a", 2, 4}, token{"\u5165\u53e3", 4, 6}, token{"\u5165\u53e3\u5904", 4, 7}},
[]token{token{"\u90a2\u6c38\u81e3", 0, 3}, token{"\u6444\u5f71", 3, 5}, token{"\u62a5\u9053", 5, 7}},
[]token{token{"BP", 0, 2}, token{"\u795e\u7ecf", 2, 4}, token{"\u7f51\u7edc", 4, 6}, token{"\u795e\u7ecf\u7f51", 2, 5}, token{"\u795e\u7ecf\u7f51\u7edc", 2, 6}, token{"\u5982\u4f55", 6, 8}, token{"\u8bad\u7ec3", 8, 10}, token{"\u624d\u80fd", 10, 12}, token{"\u5728", 12, 13}, token{"\u5206\u7c7b", 13, 15}, token{"\u65f6", 15, 16}, token{"\u589e\u52a0", 16, 18}, token{"\u533a\u5206", 18, 20}, token{"\u5206\u5ea6", 19, 21}, token{"\u533a\u5206\u5ea6", 18, 21}, token{"\uff1f", 21, 22}},
[]token{token{"\u5357\u4eac", 0, 2}, token{"\u4eac\u5e02", 1, 3}, token{"\u5357\u4eac\u5e02", 0, 3}, token{"\u957f\u6c5f", 3, 5}, token{"\u5927\u6865", 5, 7}, token{"\u957f\u6c5f\u5927\u6865", 3, 7}},
[]token{token{"\u5e94", 0, 1}, token{"\u4e00\u4e9b", 1, 3}, token{"\u4f7f\u7528", 3, 5}, token{"\u7528\u8005", 4, 6}, token{"\u4f7f\u7528\u8005", 3, 6}, token{"\u7684", 6, 7}, token{"\u5efa\u8bae", 7, 9}, token{"\uff0c", 9, 10}, token{"\u4e5f", 10, 11}, token{"\u4e3a\u4e86", 11, 13}, token{"\u4fbf\u4e8e", 13, 15}, token{"\u5229\u7528", 15, 17}, token{"NiuTrans", 17, 25}, token{"\u7528\u4e8e", 25, 27}, token{"SMT", 27, 30}, token{"\u7814\u7a76", 30, 32}},
[]token{token{"\u957f\u6625", 0, 2}, token{"\u957f\u6625\u5e02", 0, 3}, token{"\u957f\u6625", 3, 5}, token{"\u836f\u5e97", 5, 7}},
[]token{token{"\u9093\u9896\u8d85", 0, 3}, token{"\u751f\u524d", 3, 5}, token{"\u6700", 5, 6}, token{"\u559c\u6b22", 6, 8}, token{"\u7684", 8, 9}, token{"\u8863\u670d", 9, 11}},
[]token{token{"\u9526\u6d9b", 1, 3}, token{"\u80e1\u9526\u6d9b", 0, 3}, token{"\u662f", 3, 4}, token{"\u70ed\u7231", 4, 6}, token{"\u4e16\u754c", 6, 8}, token{"\u548c\u5e73", 8, 10}, token{"\u7684", 10, 11}, token{"\u653f\u6cbb", 11, 13}, token{"\u653f\u6cbb\u5c40", 11, 14}, token{"\u5e38\u59d4", 14, 16}},
[]token{token{"\u7a0b\u5e8f", 0, 2}, token{"\u7a0b\u5e8f\u5458", 0, 3}, token{"\u795d", 3, 4}, token{"\u6d77\u6797", 4, 6}, token{"\u548c", 6, 7}, token{"\u6731\u4f1a\u9707", 7, 10}, token{"\u662f", 10, 11}, token{"\u5728", 11, 12}, token{"\u5b59\u5065", 12, 14}, token{"\u7684", 14, 15}, token{"\u5de6\u9762", 15, 17}, token{"\u548c", 17, 18}, token{"\u53f3\u9762", 18, 20}, token{",", 20, 21}, token{" ", 21, 22}, token{"\u8303\u51ef", 22, 24}, token{"\u5728", 24, 25}, token{"\u6700", 25, 26}, token{"\u53f3\u9762", 26, 28}, token{".", 28, 29}, token{"\u518d\u5f80", 29, 31}, token{"\u5de6", 31, 32}, token{"\u662f", 32, 33}, token{"\u674e\u677e\u6d2a", 33, 36}},
[]token{token{"\u4e00\u6b21", 0, 2}, token{"\u4e00\u6b21\u6027", 0, 3}, token{"\u4ea4", 3, 4}, token{"\u591a\u5c11", 4, 6}, token{"\u94b1", 6, 7}},
[]token{token{"\u4e24\u5757", 0, 2}, token{"\u4e94", 2, 3}, token{"\u4e00\u5957", 3, 5}, token{"\uff0c", 5, 6}, token{"\u4e09\u5757", 6, 8}, token{"\u516b", 8, 9}, token{"\u4e00\u65a4", 9, 11}, token{"\uff0c", 11, 12}, token{"\u56db\u5757", 12, 14}, token{"\u4e03", 14, 15}, token{"\u4e00\u672c", 15, 17}, token{"\uff0c", 17, 18}, token{"\u4e94\u5757", 18, 20}, token{"\u516d", 20, 21}, token{"\u4e00\u6761", 21, 23}},
[]token{token{"\u5c0f", 0, 1}, token{"\u548c\u5c1a", 1, 3}, token{"\u7559", 3, 4}, token{"\u4e86", 4, 5}, token{"\u4e00\u4e2a", 5, 7}, token{"\u50cf", 7, 8}, token{"\u5927", 8, 9}, token{"\u548c\u5c1a", 9, 11}, token{"\u4e00\u6837", 11, 13}, token{"\u7684", 13, 14}, token{"\u548c\u5c1a", 14, 16}, token{"\u548c\u5c1a\u5934", 14, 17}},
[]token{token{"\u6211", 0, 1}, token{"\u662f", 1, 2}, token{"\u4e2d\u534e", 2, 4}, token{"\u534e\u4eba", 3, 5}, token{"\u4eba\u6c11", 4, 6}, token{"\u5171\u548c", 6, 8}, token{"\u5171\u548c\u56fd", 6, 9}, token{"\u4e2d\u534e\u4eba\u6c11\u5171\u548c\u56fd", 2, 9}, token{"\u516c\u6c11", 9, 11}, token{";", 11, 12}, token{"\u6211", 12, 13}, token{"\u7238\u7238", 13, 15}, token{"\u662f", 15, 16}, token{"\u5171\u548c", 16, 18}, token{"\u5171\u548c\u515a", 16, 19}, token{"\u515a\u5458", 19, 21}, token{";", 21, 22}, token{" ", 22, 23}, token{"\u5730\u94c1", 23, 25}, token{"\u548c\u5e73", 25, 27}, token{"\u548c\u5e73\u95e8", 25, 28}, token{"\u7ad9", 28, 29}},
[]token{token{"\u5f20\u6653\u6885", 0, 3}, token{"\u53bb", 3, 4}, token{"\u4eba\u6c11", 4, 6}, token{"\u533b\u9662", 6, 8}, token{"\u505a", 8, 9}, token{"\u4e86", 9, 10}, token{"\u4e2a", 10, 11}, token{"B\u8d85", 11, 13}, token{"\u7136\u540e", 13, 15}, token{"\u53bb", 15, 16}, token{"\u4e70", 16, 17}, token{"\u4e86", 17, 18}, token{"\u4ef6", 18, 19}, token{"T\u6064", 19, 21}},
[]token{token{"AT&T", 0, 4}, token{"\u662f", 4, 5}, token{"\u4e00\u4ef6", 5, 7}, token{"\u4e0d\u9519", 7, 9}, token{"\u7684", 9, 10}, token{"\u516c\u53f8", 10, 12}, token{"\uff0c", 12, 13}, token{"\u7ed9", 13, 14}, token{"\u4f60", 14, 15}, token{"\u53d1", 15, 16}, token{"offer", 16, 21}, token{"\u4e86", 21, 22}, token{"\u5417", 22, 23}, token{"\uff1f", 23, 24}},
[]token{token{"C++", 0, 3}, token{"\u548c", 3, 4}, token{"c#", 4, 6}, token{"\u662f", 6, 7}, token{"\u4ec0\u4e48", 7, 9}, token{"\u5173\u7cfb", 9, 11}, token{"\uff1f", 11, 12}, token{"11", 12, 14}, token{"+", 14, 15}, token{"122", 15, 18}, token{"=", 18, 19}, token{"133", 19, 22}, token{"\uff0c", 22, 23}, token{"\u662f", 23, 24}, token{"\u5417", 24, 25}, token{"\uff1f", 25, 26}, token{"PI", 26, 28}, token{"=", 28, 29}, token{"3.14159", 29, 36}},
[]token{token{"\u4f60", 0, 1}, token{"\u8ba4\u8bc6", 1, 3}, token{"\u90a3\u4e2a", 3, 5}, token{"\u548c", 5, 6}, token{"\u4e3b\u5e2d", 6, 8}, token{"\u63e1\u624b", 8, 10}, token{"\u7684", 10, 11}, token{"\u7684\u54e5", 11, 13}, token{"\u5417", 13, 14}, token{"\uff1f", 14, 15}, token{"\u4ed6\u5f00", 15, 17}, token{"\u4e00\u8f86", 17, 19}, token{"\u9ed1\u8272", 19, 21}, token{"\u7684\u58eb", 21, 23}, token{"\u3002", 23, 24}},
[]token{token{"\u67aa\u6746", 0, 2}, token{"\u6746\u5b50", 1, 3}, token{"\u67aa\u6746\u5b50", 0, 3}, token{"\u4e2d", 3, 4}, token{"\u51fa", 4, 5}, token{"\u653f\u6743", 5, 7}},
[]token{token{"\u5f20\u4e09\u98ce", 0, 3}, token{"\u540c\u5b66", 3, 5}, token{"\u8d70\u4e0a", 5, 7}, token{"\u4e86", 7, 8}, token{"\u5f52\u8def", 9, 11}, token{"\u4e0d\u5f52\u8def", 8, 11}},
[]token{token{"\u963fQ", 0, 2}, token{"\u8170\u95f4", 2, 4}, token{"\u6302", 4, 5}, token{"\u7740", 5, 6}, token{"BB\u673a", 6, 9}, token{"\u624b\u91cc", 9, 11}, token{"\u62ff", 11, 12}, token{"\u7740", 12, 13}, token{"\u5927\u54e5", 13, 15}, token{"\u5927\u54e5\u5927", 13, 16}, token{"\uff0c", 16, 17}, token{"\u8bf4", 17, 18}, token{"\uff1a", 18, 19}, token{"\u6211", 19, 20}, token{"\u4e00\u822c", 20, 22}, token{"\u5403\u996d", 22, 24}, token{"\u4e0d", 24, 25}, token{"AA\u5236", 25, 28}, token{"\u7684", 28, 29}, token{"\u3002", 29, 30}},
[]token{token{"\u5728", 0, 1}, token{"1\u53f7\u5e97", 1, 4}, token{"\u80fd", 4, 5}, token{"\u4e70", 5, 6}, token{"\u5230", 6, 7}, token{"\u5c0fS", 7, 9}, token{"\u548c", 9, 10}, token{"\u5927S", 10, 12}, token{"\u516b\u5366", 12, 14}, token{"\u7684", 14, 15}, token{"\u4e66", 15, 16}, token{"\u3002", 16, 17}},
}
noHmmResult = [][]Token{
[]Token{Token{"\u8fd9", 0, 1}, Token{"\u662f", 1, 2}, Token{"\u4e00\u4e2a", 2, 4}, Token{"\u4f38\u624b\u4e0d\u89c1\u4e94\u6307", 4, 10}, Token{"\u7684", 10, 11}, Token{"\u9ed1\u591c", 11, 13}, Token{"\u3002", 13, 14}, Token{"\u6211", 14, 15}, Token{"\u53eb", 15, 16}, Token{"\u5b59\u609f\u7a7a", 16, 19}, Token{"\uff0c", 19, 20}, Token{"\u6211", 20, 21}, Token{"\u7231", 21, 22}, Token{"\u5317\u4eac", 22, 24}, Token{"\uff0c", 24, 25}, Token{"\u6211", 25, 26}, Token{"\u7231", 26, 27}, Token{"Python", 27, 33}, Token{"\u548c", 33, 34}, Token{"C++", 34, 37}, Token{"\u3002", 37, 38}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u4e0d", 1, 2}, Token{"\u559c\u6b22", 2, 4}, Token{"\u65e5\u672c", 4, 6}, Token{"\u548c\u670d", 6, 8}, Token{"\u3002", 8, 9}},
[]Token{Token{"\u96f7\u7334", 0, 2}, Token{"\u56de\u5f52", 2, 4}, Token{"\u4eba\u95f4", 4, 6}, Token{"\u3002", 6, 7}},
[]Token{Token{"\u5de5\u4fe1\u5904", 0, 3}, Token{"\u5973\u5e72\u4e8b", 3, 6}, Token{"\u6bcf\u6708", 6, 8}, Token{"\u7ecf\u8fc7", 8, 10}, Token{"\u4e0b\u5c5e", 10, 12}, Token{"\u79d1\u5ba4", 12, 14}, Token{"\u90fd", 14, 15}, Token{"\u8981", 15, 16}, Token{"\u4eb2\u53e3", 16, 18}, Token{"\u4ea4\u4ee3", 18, 20}, Token{"24", 20, 22}, Token{"\u53e3", 22, 23}, Token{"\u4ea4\u6362\u673a", 23, 26}, Token{"\u7b49", 26, 27}, Token{"\u6280\u672f\u6027", 27, 30}, Token{"\u5668\u4ef6", 30, 32}, Token{"\u7684", 32, 33}, Token{"\u5b89\u88c5", 33, 35}, Token{"\u5de5\u4f5c", 35, 37}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u9700\u8981", 1, 3}, Token{"\u5ec9\u79df\u623f", 3, 6}},
[]Token{Token{"\u6c38\u548c", 0, 2}, Token{"\u670d\u88c5", 2, 4}, Token{"\u9970\u54c1", 4, 6}, Token{"\u6709\u9650\u516c\u53f8", 6, 10}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u7231", 1, 2}, Token{"\u5317\u4eac", 2, 4}, Token{"\u5929\u5b89\u95e8", 4, 7}},
[]Token{Token{"abc", 0, 3}},
[]Token{Token{"\u9690", 0, 1}, Token{"\u9a6c\u5c14\u53ef\u592b", 1, 5}},
[]Token{Token{"\u96f7\u7334", 0, 2}, Token{"\u662f", 2, 3}, Token{"\u4e2a", 3, 4}, Token{"\u597d", 4, 5}, Token{"\u7f51\u7ad9", 5, 7}},
[]Token{Token{"\u201c", 0, 1}, Token{"Microsoft", 1, 10}, Token{"\u201d", 10, 11}, Token{"\u4e00", 11, 12}, Token{"\u8bcd", 12, 13}, Token{"\u7531", 13, 14}, Token{"\u201c", 14, 15}, Token{"MICROcomputer", 15, 28}, Token{"\uff08", 28, 29}, Token{"\u5fae\u578b", 29, 31}, Token{"\u8ba1\u7b97\u673a", 31, 34}, Token{"\uff09", 34, 35}, Token{"\u201d", 35, 36}, Token{"\u548c", 36, 37}, Token{"\u201c", 37, 38}, Token{"SOFTware", 38, 46}, Token{"\uff08", 46, 47}, Token{"\u8f6f\u4ef6", 47, 49}, Token{"\uff09", 49, 50}, Token{"\u201d", 50, 51}, Token{"\u4e24", 51, 52}, Token{"\u90e8\u5206", 52, 54}, Token{"\u7ec4\u6210", 54, 56}},
[]Token{Token{"\u8349\u6ce5\u9a6c", 0, 3}, Token{"\u548c", 3, 4}, Token{"\u6b3a", 4, 5}, Token{"\u5b9e", 5, 6}, Token{"\u9a6c", 6, 7}, Token{"\u662f", 7, 8}, Token{"\u4eca\u5e74", 8, 10}, Token{"\u7684", 10, 11}, Token{"\u6d41\u884c", 11, 13}, Token{"\u8bcd\u6c47", 13, 15}},
[]Token{Token{"\u4f0a", 0, 1}, Token{"\u85e4", 1, 2}, Token{"\u6d0b\u534e\u5802", 2, 5}, Token{"\u603b\u5e9c", 5, 7}, Token{"\u5e97", 7, 8}},
[]Token{Token{"\u4e2d\u56fd\u79d1\u5b66\u9662\u8ba1\u7b97\u6280\u672f\u7814\u7a76\u6240", 0, 12}},
[]Token{Token{"\u7f57\u5bc6\u6b27", 0, 3}, Token{"\u4e0e", 3, 4}, Token{"\u6731\u4e3d\u53f6", 4, 7}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u8d2d\u4e70", 1, 3}, Token{"\u4e86", 3, 4}, Token{"\u9053\u5177", 4, 6}, Token{"\u548c", 6, 7}, Token{"\u670d\u88c5", 7, 9}},
[]Token{Token{"PS", 0, 2}, Token{":", 2, 3}, Token{" ", 3, 4}, Token{"\u6211", 4, 5}, Token{"\u89c9\u5f97", 5, 7}, Token{"\u5f00\u6e90", 7, 9}, Token{"\u6709", 9, 10}, Token{"\u4e00\u4e2a", 10, 12}, Token{"\u597d\u5904", 12, 14}, Token{"\uff0c", 14, 15}, Token{"\u5c31\u662f", 15, 17}, Token{"\u80fd\u591f", 17, 19}, Token{"\u6566\u4fc3", 19, 21}, Token{"\u81ea\u5df1", 21, 23}, Token{"\u4e0d\u65ad\u6539\u8fdb", 23, 27}, Token{"\uff0c", 27, 28}, Token{"\u907f\u514d", 28, 30}, Token{"\u655e", 30, 31}, Token{"\u5e1a", 31, 32}, Token{"\u81ea\u73cd", 32, 34}},
[]Token{Token{"\u6e56\u5317\u7701", 0, 3}, Token{"\u77f3\u9996\u5e02", 3, 6}},
[]Token{Token{"\u6e56\u5317\u7701", 0, 3}, Token{"\u5341\u5830\u5e02", 3, 6}},
[]Token{Token{"\u603b\u7ecf\u7406", 0, 3}, Token{"\u5b8c\u6210", 3, 5}, Token{"\u4e86", 5, 6}, Token{"\u8fd9\u4ef6", 6, 8}, Token{"\u4e8b\u60c5", 8, 10}},
[]Token{Token{"\u7535\u8111", 0, 2}, Token{"\u4fee\u597d", 2, 4}, Token{"\u4e86", 4, 5}},
[]Token{Token{"\u505a\u597d", 0, 2}, Token{"\u4e86", 2, 3}, Token{"\u8fd9\u4ef6", 3, 5}, Token{"\u4e8b\u60c5", 5, 7}, Token{"\u5c31", 7, 8}, Token{"\u4e00\u4e86\u767e\u4e86", 8, 12}, Token{"\u4e86", 12, 13}},
[]Token{Token{"\u4eba\u4eec", 0, 2}, Token{"\u5ba1\u7f8e", 2, 4}, Token{"\u7684", 4, 5}, Token{"\u89c2\u70b9", 5, 7}, Token{"\u662f", 7, 8}, Token{"\u4e0d\u540c", 8, 10}, Token{"\u7684", 10, 11}},
[]Token{Token{"\u6211\u4eec", 0, 2}, Token{"\u4e70", 2, 3}, Token{"\u4e86", 3, 4}, Token{"\u4e00\u4e2a", 4, 6}, Token{"\u7f8e\u7684", 6, 8}, Token{"\u7a7a\u8c03", 8, 10}},
[]Token{Token{"\u7ebf\u7a0b", 0, 2}, Token{"\u521d\u59cb\u5316", 2, 5}, Token{"\u65f6", 5, 6}, Token{"\u6211\u4eec", 6, 8}, Token{"\u8981", 8, 9}, Token{"\u6ce8\u610f", 9, 11}},
[]Token{Token{"\u4e00\u4e2a", 0, 2}, Token{"\u5206\u5b50", 2, 4}, Token{"\u662f", 4, 5}, Token{"\u7531", 5, 6}, Token{"\u597d\u591a", 6, 8}, Token{"\u539f\u5b50", 8, 10}, Token{"\u7ec4\u7ec7", 10, 12}, Token{"\u6210", 12, 13}, Token{"\u7684", 13, 14}},
[]Token{Token{"\u795d", 0, 1}, Token{"\u4f60", 1, 2}, Token{"\u9a6c\u5230\u529f\u6210", 2, 6}},
[]Token{Token{"\u4ed6", 0, 1}, Token{"\u6389", 1, 2}, Token{"\u8fdb", 2, 3}, Token{"\u4e86", 3, 4}, Token{"\u65e0\u5e95\u6d1e", 4, 7}, Token{"\u91cc", 7, 8}},
[]Token{Token{"\u4e2d\u56fd", 0, 2}, Token{"\u7684", 2, 3}, Token{"\u9996\u90fd", 3, 5}, Token{"\u662f", 5, 6}, Token{"\u5317\u4eac", 6, 8}},
[]Token{Token{"\u5b59", 0, 1}, Token{"\u541b", 1, 2}, Token{"\u610f", 2, 3}},
[]Token{Token{"\u5916\u4ea4\u90e8", 0, 3}, Token{"\u53d1\u8a00\u4eba", 3, 6}, Token{"\u9a6c\u671d\u65ed", 6, 9}},
[]Token{Token{"\u9886\u5bfc\u4eba", 0, 3}, Token{"\u4f1a\u8bae", 3, 5}, Token{"\u548c", 5, 6}, Token{"\u7b2c\u56db\u5c4a", 6, 9}, Token{"\u4e1c\u4e9a", 9, 11}, Token{"\u5cf0\u4f1a", 11, 13}},
[]Token{Token{"\u5728", 0, 1}, Token{"\u8fc7\u53bb", 1, 3}, Token{"\u7684", 3, 4}, Token{"\u8fd9", 4, 5}, Token{"\u4e94\u5e74", 5, 7}},
[]Token{Token{"\u8fd8", 0, 1}, Token{"\u9700\u8981", 1, 3}, Token{"\u5f88", 3, 4}, Token{"\u957f", 4, 5}, Token{"\u7684", 5, 6}, Token{"\u8def", 6, 7}, Token{"\u8981", 7, 8}, Token{"\u8d70", 8, 9}},
[]Token{Token{"60", 0, 2}, Token{"\u5468\u5e74", 2, 4}, Token{"\u9996\u90fd", 4, 6}, Token{"\u9605\u5175", 6, 8}},
[]Token{Token{"\u4f60\u597d", 0, 2}, Token{"\u4eba\u4eec", 2, 4}, Token{"\u5ba1\u7f8e", 4, 6}, Token{"\u7684", 6, 7}, Token{"\u89c2\u70b9", 7, 9}, Token{"\u662f", 9, 10}, Token{"\u4e0d\u540c", 10, 12}, Token{"\u7684", 12, 13}},
[]Token{Token{"\u4e70", 0, 1}, Token{"\u6c34\u679c", 1, 3}, Token{"\u7136\u540e", 3, 5}, Token{"\u6765", 5, 6}, Token{"\u4e16\u535a\u56ed", 6, 9}},
[]Token{Token{"\u4e70", 0, 1}, Token{"\u6c34\u679c", 1, 3}, Token{"\u7136\u540e", 3, 5}, Token{"\u53bb", 5, 6}, Token{"\u4e16\u535a\u56ed", 6, 9}},
[]Token{Token{"\u4f46\u662f", 0, 2}, Token{"\u540e\u6765", 2, 4}, Token{"\u6211", 4, 5}, Token{"\u624d", 5, 6}, Token{"\u77e5\u9053", 6, 8}, Token{"\u4f60", 8, 9}, Token{"\u662f", 9, 10}, Token{"\u5bf9", 10, 11}, Token{"\u7684", 11, 12}},
[]Token{Token{"\u5b58\u5728", 0, 2}, Token{"\u5373", 2, 3}, Token{"\u5408\u7406", 3, 5}},
[]Token{Token{"\u7684", 0, 1}, Token{"\u7684", 1, 2}, Token{"\u7684", 2, 3}, Token{"\u7684", 3, 4}, Token{"\u7684", 4, 5}, Token{"\u5728", 5, 6}, Token{"\u7684", 6, 7}, Token{"\u7684", 7, 8}, Token{"\u7684", 8, 9}, Token{"\u7684", 9, 10}, Token{"\u5c31", 10, 11}, Token{"\u4ee5", 11, 12}, Token{"\u548c", 12, 13}, Token{"\u548c", 13, 14}, Token{"\u548c", 14, 15}},
[]Token{Token{"I", 0, 1}, Token{" ", 1, 2}, Token{"love", 2, 6}, Token{"\u4f60", 6, 7}, Token{"\uff0c", 7, 8}, Token{"\u4e0d\u4ee5\u4e3a\u803b", 8, 12}, Token{"\uff0c", 12, 13}, Token{"\u53cd", 13, 14}, Token{"\u4ee5\u4e3a", 14, 16}, Token{"rong", 16, 20}},
[]Token{Token{"\u56e0", 0, 1}},
[]Token{},
[]Token{Token{"hello", 0, 5}, Token{"\u4f60\u597d", 5, 7}, Token{"\u4eba\u4eec", 7, 9}, Token{"\u5ba1\u7f8e", 9, 11}, Token{"\u7684", 11, 12}, Token{"\u89c2\u70b9", 12, 14}, Token{"\u662f", 14, 15}, Token{"\u4e0d\u540c", 15, 17}, Token{"\u7684", 17, 18}},
[]Token{Token{"\u5f88", 0, 1}, Token{"\u597d", 1, 2}, Token{"\u4f46", 2, 3}, Token{"\u4e3b\u8981", 3, 5}, Token{"\u662f", 5, 6}, Token{"\u57fa\u4e8e", 6, 8}, Token{"\u7f51\u9875", 8, 10}, Token{"\u5f62\u5f0f", 10, 12}},
[]Token{Token{"hello", 0, 5}, Token{"\u4f60\u597d", 5, 7}, Token{"\u4eba\u4eec", 7, 9}, Token{"\u5ba1\u7f8e", 9, 11}, Token{"\u7684", 11, 12}, Token{"\u89c2\u70b9", 12, 14}, Token{"\u662f", 14, 15}, Token{"\u4e0d\u540c", 15, 17}, Token{"\u7684", 17, 18}},
[]Token{Token{"\u4e3a\u4ec0\u4e48", 0, 3}, Token{"\u6211", 3, 4}, Token{"\u4e0d\u80fd", 4, 6}, Token{"\u62e5\u6709", 6, 8}, Token{"\u60f3\u8981", 8, 10}, Token{"\u7684", 10, 11}, Token{"\u751f\u6d3b", 11, 13}},
[]Token{Token{"\u540e\u6765", 0, 2}, Token{"\u6211", 2, 3}, Token{"\u624d", 3, 4}},
[]Token{Token{"\u6b64\u6b21", 0, 2}, Token{"\u6765", 2, 3}, Token{"\u4e2d\u56fd", 3, 5}, Token{"\u662f", 5, 6}, Token{"\u4e3a\u4e86", 6, 8}},
[]Token{Token{"\u4f7f\u7528", 0, 2}, Token{"\u4e86", 2, 3}, Token{"\u5b83", 3, 4}, Token{"\u5c31", 4, 5}, Token{"\u53ef\u4ee5", 5, 7}, Token{"\u89e3\u51b3", 7, 9}, Token{"\u4e00\u4e9b", 9, 11}, Token{"\u95ee\u9898", 11, 13}},
[]Token{Token{",", 0, 1}, Token{"\u4f7f\u7528", 1, 3}, Token{"\u4e86", 3, 4}, Token{"\u5b83", 4, 5}, Token{"\u5c31", 5, 6}, Token{"\u53ef\u4ee5", 6, 8}, Token{"\u89e3\u51b3", 8, 10}, Token{"\u4e00\u4e9b", 10, 12}, Token{"\u95ee\u9898", 12, 14}},
[]Token{Token{"\u5176\u5b9e", 0, 2}, Token{"\u4f7f\u7528", 2, 4}, Token{"\u4e86", 4, 5}, Token{"\u5b83", 5, 6}, Token{"\u5c31", 6, 7}, Token{"\u53ef\u4ee5", 7, 9}, Token{"\u89e3\u51b3", 9, 11}, Token{"\u4e00\u4e9b", 11, 13}, Token{"\u95ee\u9898", 13, 15}},
[]Token{Token{"\u597d\u4eba", 0, 2}, Token{"\u4f7f\u7528", 2, 4}, Token{"\u4e86", 4, 5}, Token{"\u5b83", 5, 6}, Token{"\u5c31", 6, 7}, Token{"\u53ef\u4ee5", 7, 9}, Token{"\u89e3\u51b3", 9, 11}, Token{"\u4e00\u4e9b", 11, 13}, Token{"\u95ee\u9898", 13, 15}},
[]Token{Token{"\u662f\u56e0\u4e3a", 0, 3}, Token{"\u548c", 3, 4}, Token{"\u56fd\u5bb6", 4, 6}},
[]Token{Token{"\u8001\u5e74", 0, 2}, Token{"\u641c\u7d22", 2, 4}, Token{"\u8fd8", 4, 5}, Token{"\u652f\u6301", 5, 7}},
[]Token{Token{"\u5e72\u8106", 0, 2}, Token{"\u5c31", 2, 3}, Token{"\u628a", 3, 4}, Token{"\u90a3", 4, 5}, Token{"\u90e8", 5, 6}, Token{"\u8499", 6, 7}, Token{"\u4eba", 7, 8}, Token{"\u7684", 8, 9}, Token{"\u95f2", 9, 10}, Token{"\u6cd5", 10, 11}, Token{"\u7ed9", 11, 12}, Token{"\u5e9f", 12, 13}, Token{"\u4e86", 13, 14}, Token{"\u62c9\u5012", 14, 16}, Token{"\uff01", 16, 17}, Token{"RT", 17, 19}, Token{" ", 19, 20}, Token{"@", 20, 21}, Token{"laoshipukong", 21, 33}, Token{" ", 33, 34}, Token{":", 34, 35}, Token{" ", 35, 36}, Token{"27", 36, 38}, Token{"\u65e5", 38, 39}, Token{"\uff0c", 39, 40}, Token{"\u5168\u56fd\u4eba\u5927\u5e38\u59d4\u4f1a", 40, 47}, Token{"\u7b2c\u4e09\u6b21", 47, 50}, Token{"\u5ba1\u8bae", 50, 52}, Token{"\u4fb5\u6743", 52, 54}, Token{"\u8d23\u4efb\u6cd5", 54, 57}, Token{"\u8349\u6848", 57, 59}, Token{"\uff0c", 59, 60}, Token{"\u5220\u9664", 60, 62}, Token{"\u4e86", 62, 63}, Token{"\u6709\u5173", 63, 65}, Token{"\u533b\u7597", 65, 67}, Token{"\u635f\u5bb3", 67, 69}, Token{"\u8d23\u4efb", 69, 71}, Token{"\u201c", 71, 72}, Token{"\u4e3e\u8bc1", 72, 74}, Token{"\u5012\u7f6e", 74, 76}, Token{"\u201d", 76, 77}, Token{"\u7684", 77, 78}, Token{"\u89c4\u5b9a", 78, 80}, Token{"\u3002", 80, 81}, Token{"\u5728", 81, 82}, Token{"\u533b\u60a3", 82, 84}, Token{"\u7ea0\u7eb7", 84, 86}, Token{"\u4e2d", 86, 87}, Token{"\u672c", 87, 88}, Token{"\u5df2", 88, 89}, Token{"\u5904\u4e8e", 89, 91}, Token{"\u5f31\u52bf", 91, 93}, Token{"\u5730\u4f4d", 93, 95}, Token{"\u7684", 95, 96}, Token{"\u6d88\u8d39\u8005", 96, 99}, Token{"\u7531\u6b64", 99, 101}, Token{"\u5c06", 101, 102}, Token{"\u9677\u5165", 102, 104}, Token{"\u4e07\u52ab\u4e0d\u590d", 104, 108}, Token{"\u7684", 108, 109}, Token{"\u5883\u5730", 109, 111}, Token{"\u3002", 111, 112}, Token{" ", 112, 113}},
[]Token{Token{"\u5927", 0, 1}},
[]Token{},
[]Token{Token{"\u4ed6", 0, 1}, Token{"\u8bf4", 1, 2}, Token{"\u7684", 2, 3}, Token{"\u786e\u5b9e", 3, 5}, Token{"\u5728", 5, 6}, Token{"\u7406", 6, 7}},
[]Token{Token{"\u957f\u6625", 0, 2}, Token{"\u5e02\u957f", 2, 4}, Token{"\u6625\u8282", 4, 6}, Token{"\u8bb2\u8bdd", 6, 8}},
[]Token{Token{"\u7ed3\u5a5a", 0, 2}, Token{"\u7684", 2, 3}, Token{"\u548c", 3, 4}, Token{"\u5c1a\u672a", 4, 6}, Token{"\u7ed3\u5a5a", 6, 8}, Token{"\u7684", 8, 9}},
[]Token{Token{"\u7ed3\u5408", 0, 2}, Token{"\u6210", 2, 3}, Token{"\u5206\u5b50", 3, 5}, Token{"\u65f6", 5, 6}},
[]Token{Token{"\u65c5\u6e38", 0, 2}, Token{"\u548c", 2, 3}, Token{"\u670d\u52a1", 3, 5}, Token{"\u662f", 5, 6}, Token{"\u6700\u597d", 6, 8}, Token{"\u7684", 8, 9}},
[]Token{Token{"\u8fd9\u4ef6", 0, 2}, Token{"\u4e8b\u60c5", 2, 4}, Token{"\u7684\u786e", 4, 6}, Token{"\u662f", 6, 7}, Token{"\u6211", 7, 8}, Token{"\u7684", 8, 9}, Token{"\u9519", 9, 10}},
[]Token{Token{"\u4f9b", 0, 1}, Token{"\u5927\u5bb6", 1, 3}, Token{"\u53c2\u8003", 3, 5}, Token{"\u6307\u6b63", 5, 7}},
[]Token{Token{"\u54c8\u5c14\u6ee8", 0, 3}, Token{"\u653f\u5e9c", 3, 5}, Token{"\u516c\u5e03", 5, 7}, Token{"\u584c", 7, 8}, Token{"\u6865", 8, 9}, Token{"\u539f\u56e0", 9, 11}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u5728", 1, 2}, Token{"\u673a\u573a", 2, 4}, Token{"\u5165\u53e3\u5904", 4, 7}},
[]Token{Token{"\u90a2", 0, 1}, Token{"\u6c38", 1, 2}, Token{"\u81e3", 2, 3}, Token{"\u6444\u5f71", 3, 5}, Token{"\u62a5\u9053", 5, 7}},
[]Token{Token{"BP", 0, 2}, Token{"\u795e\u7ecf\u7f51\u7edc", 2, 6}, Token{"\u5982\u4f55", 6, 8}, Token{"\u8bad\u7ec3", 8, 10}, Token{"\u624d\u80fd", 10, 12}, Token{"\u5728", 12, 13}, Token{"\u5206\u7c7b", 13, 15}, Token{"\u65f6", 15, 16}, Token{"\u589e\u52a0", 16, 18}, Token{"\u533a\u5206\u5ea6", 18, 21}, Token{"\uff1f", 21, 22}},
[]Token{Token{"\u5357\u4eac\u5e02", 0, 3}, Token{"\u957f\u6c5f\u5927\u6865", 3, 7}},
[]Token{Token{"\u5e94", 0, 1}, Token{"\u4e00\u4e9b", 1, 3}, Token{"\u4f7f\u7528\u8005", 3, 6}, Token{"\u7684", 6, 7}, Token{"\u5efa\u8bae", 7, 9}, Token{"\uff0c", 9, 10}, Token{"\u4e5f", 10, 11}, Token{"\u4e3a\u4e86", 11, 13}, Token{"\u4fbf\u4e8e", 13, 15}, Token{"\u5229\u7528", 15, 17}, Token{"NiuTrans", 17, 25}, Token{"\u7528\u4e8e", 25, 27}, Token{"SMT", 27, 30}, Token{"\u7814\u7a76", 30, 32}},
[]Token{Token{"\u957f\u6625\u5e02", 0, 3}, Token{"\u957f\u6625", 3, 5}, Token{"\u836f\u5e97", 5, 7}},
[]Token{Token{"\u9093\u9896\u8d85", 0, 3}, Token{"\u751f\u524d", 3, 5}, Token{"\u6700", 5, 6}, Token{"\u559c\u6b22", 6, 8}, Token{"\u7684", 8, 9}, Token{"\u8863\u670d", 9, 11}},
[]Token{Token{"\u80e1\u9526\u6d9b", 0, 3}, Token{"\u662f", 3, 4}, Token{"\u70ed\u7231", 4, 6}, Token{"\u4e16\u754c", 6, 8}, Token{"\u548c\u5e73", 8, 10}, Token{"\u7684", 10, 11}, Token{"\u653f\u6cbb\u5c40", 11, 14}, Token{"\u5e38\u59d4", 14, 16}},
[]Token{Token{"\u7a0b\u5e8f\u5458", 0, 3}, Token{"\u795d", 3, 4}, Token{"\u6d77\u6797", 4, 6}, Token{"\u548c", 6, 7}, Token{"\u6731", 7, 8}, Token{"\u4f1a", 8, 9}, Token{"\u9707", 9, 10}, Token{"\u662f", 10, 11}, Token{"\u5728", 11, 12}, Token{"\u5b59", 12, 13}, Token{"\u5065", 13, 14}, Token{"\u7684", 14, 15}, Token{"\u5de6\u9762", 15, 17}, Token{"\u548c", 17, 18}, Token{"\u53f3\u9762", 18, 20}, Token{",", 20, 21}, Token{" ", 21, 22}, Token{"\u8303", 22, 23}, Token{"\u51ef", 23, 24}, Token{"\u5728", 24, 25}, Token{"\u6700", 25, 26}, Token{"\u53f3\u9762", 26, 28}, Token{".", 28, 29}, Token{"\u518d", 29, 30}, Token{"\u5f80", 30, 31}, Token{"\u5de6", 31, 32}, Token{"\u662f", 32, 33}, Token{"\u674e", 33, 34}, Token{"\u677e", 34, 35}, Token{"\u6d2a", 35, 36}},
[]Token{Token{"\u4e00\u6b21\u6027", 0, 3}, Token{"\u4ea4", 3, 4}, Token{"\u591a\u5c11", 4, 6}, Token{"\u94b1", 6, 7}},
[]Token{Token{"\u4e24\u5757", 0, 2}, Token{"\u4e94", 2, 3}, Token{"\u4e00\u5957", 3, 5}, Token{"\uff0c", 5, 6}, Token{"\u4e09\u5757", 6, 8}, Token{"\u516b", 8, 9}, Token{"\u4e00\u65a4", 9, 11}, Token{"\uff0c", 11, 12}, Token{"\u56db\u5757", 12, 14}, Token{"\u4e03", 14, 15}, Token{"\u4e00\u672c", 15, 17}, Token{"\uff0c", 17, 18}, Token{"\u4e94\u5757", 18, 20}, Token{"\u516d", 20, 21}, Token{"\u4e00\u6761", 21, 23}},
[]Token{Token{"\u5c0f", 0, 1}, Token{"\u548c\u5c1a", 1, 3}, Token{"\u7559", 3, 4}, Token{"\u4e86", 4, 5}, Token{"\u4e00\u4e2a", 5, 7}, Token{"\u50cf", 7, 8}, Token{"\u5927", 8, 9}, Token{"\u548c\u5c1a", 9, 11}, Token{"\u4e00\u6837", 11, 13}, Token{"\u7684", 13, 14}, Token{"\u548c\u5c1a\u5934", 14, 17}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u662f", 1, 2}, Token{"\u4e2d\u534e\u4eba\u6c11\u5171\u548c\u56fd", 2, 9}, Token{"\u516c\u6c11", 9, 11}, Token{";", 11, 12}, Token{"\u6211", 12, 13}, Token{"\u7238\u7238", 13, 15}, Token{"\u662f", 15, 16}, Token{"\u5171\u548c\u515a", 16, 19}, Token{"\u515a\u5458", 19, 21}, Token{";", 21, 22}, Token{" ", 22, 23}, Token{"\u5730\u94c1", 23, 25}, Token{"\u548c\u5e73\u95e8", 25, 28}, Token{"\u7ad9", 28, 29}},
[]Token{Token{"\u5f20\u6653\u6885", 0, 3}, Token{"\u53bb", 3, 4}, Token{"\u4eba\u6c11", 4, 6}, Token{"\u533b\u9662", 6, 8}, Token{"\u505a", 8, 9}, Token{"\u4e86", 9, 10}, Token{"\u4e2a", 10, 11}, Token{"B\u8d85", 11, 13}, Token{"\u7136\u540e", 13, 15}, Token{"\u53bb", 15, 16}, Token{"\u4e70", 16, 17}, Token{"\u4e86", 17, 18}, Token{"\u4ef6", 18, 19}, Token{"T\u6064", 19, 21}},
[]Token{Token{"AT&T", 0, 4}, Token{"\u662f", 4, 5}, Token{"\u4e00\u4ef6", 5, 7}, Token{"\u4e0d\u9519", 7, 9}, Token{"\u7684", 9, 10}, Token{"\u516c\u53f8", 10, 12}, Token{"\uff0c", 12, 13}, Token{"\u7ed9", 13, 14}, Token{"\u4f60", 14, 15}, Token{"\u53d1", 15, 16}, Token{"offer", 16, 21}, Token{"\u4e86", 21, 22}, Token{"\u5417", 22, 23}, Token{"\uff1f", 23, 24}},
[]Token{Token{"C++", 0, 3}, Token{"\u548c", 3, 4}, Token{"c#", 4, 6}, Token{"\u662f", 6, 7}, Token{"\u4ec0\u4e48", 7, 9}, Token{"\u5173\u7cfb", 9, 11}, Token{"\uff1f", 11, 12}, Token{"11", 12, 14}, Token{"+", 14, 15}, Token{"122", 15, 18}, Token{"=", 18, 19}, Token{"133", 19, 22}, Token{"\uff0c", 22, 23}, Token{"\u662f", 23, 24}, Token{"\u5417", 24, 25}, Token{"\uff1f", 25, 26}, Token{"PI", 26, 28}, Token{"=", 28, 29}, Token{"3", 29, 30}, Token{".", 30, 31}, Token{"14159", 31, 36}},
[]Token{Token{"\u4f60", 0, 1}, Token{"\u8ba4\u8bc6", 1, 3}, Token{"\u90a3\u4e2a", 3, 5}, Token{"\u548c", 5, 6}, Token{"\u4e3b\u5e2d", 6, 8}, Token{"\u63e1\u624b", 8, 10}, Token{"\u7684", 10, 11}, Token{"\u7684\u54e5", 11, 13}, Token{"\u5417", 13, 14}, Token{"\uff1f", 14, 15}, Token{"\u4ed6", 15, 16}, Token{"\u5f00", 16, 17}, Token{"\u4e00\u8f86", 17, 19}, Token{"\u9ed1\u8272", 19, 21}, Token{"\u7684\u58eb", 21, 23}, Token{"\u3002", 23, 24}},
[]Token{Token{"\u67aa\u6746\u5b50", 0, 3}, Token{"\u4e2d", 3, 4}, Token{"\u51fa", 4, 5}, Token{"\u653f\u6743", 5, 7}},
[]Token{Token{"\u5f20", 0, 1}, Token{"\u4e09", 1, 2}, Token{"\u98ce", 2, 3}, Token{"\u540c\u5b66", 3, 5}, Token{"\u8d70\u4e0a", 5, 7}, Token{"\u4e86", 7, 8}, Token{"\u4e0d\u5f52\u8def", 8, 11}},
[]Token{Token{"\u963fQ", 0, 2}, Token{"\u8170\u95f4", 2, 4}, Token{"\u6302", 4, 5}, Token{"\u7740", 5, 6}, Token{"BB\u673a", 6, 9}, Token{"\u624b\u91cc", 9, 11}, Token{"\u62ff", 11, 12}, Token{"\u7740", 12, 13}, Token{"\u5927\u54e5\u5927", 13, 16}, Token{"\uff0c", 16, 17}, Token{"\u8bf4", 17, 18}, Token{"\uff1a", 18, 19}, Token{"\u6211", 19, 20}, Token{"\u4e00\u822c", 20, 22}, Token{"\u5403\u996d", 22, 24}, Token{"\u4e0d", 24, 25}, Token{"AA\u5236", 25, 28}, Token{"\u7684", 28, 29}, Token{"\u3002", 29, 30}},
[]Token{Token{"\u5728", 0, 1}, Token{"1\u53f7\u5e97", 1, 4}, Token{"\u80fd", 4, 5}, Token{"\u4e70", 5, 6}, Token{"\u5230", 6, 7}, Token{"\u5c0fS", 7, 9}, Token{"\u548c", 9, 10}, Token{"\u5927S", 10, 12}, Token{"\u516b\u5366", 12, 14}, Token{"\u7684", 14, 15}, Token{"\u4e66", 15, 16}, Token{"\u3002", 16, 17}},
[]Token{Token{"\u8fd9", 0, 1}, Token{"\u662f", 1, 2}, Token{"\u4e00\u4e2a", 2, 4}, Token{"\u4f38\u624b", 4, 6}, Token{"\u4e0d\u89c1", 6, 8}, Token{"\u4e94\u6307", 8, 10}, Token{"\u4f38\u624b\u4e0d\u89c1\u4e94\u6307", 4, 10}, Token{"\u7684", 10, 11}, Token{"\u9ed1\u591c", 11, 13}, Token{"\u3002", 13, 14}, Token{"\u6211", 14, 15}, Token{"\u53eb", 15, 16}, Token{"\u609f\u7a7a", 17, 19}, Token{"\u5b59\u609f\u7a7a", 16, 19}, Token{"\uff0c", 19, 20}, Token{"\u6211", 20, 21}, Token{"\u7231", 21, 22}, Token{"\u5317\u4eac", 22, 24}, Token{"\uff0c", 24, 25}, Token{"\u6211", 25, 26}, Token{"\u7231", 26, 27}, Token{"Python", 27, 33}, Token{"\u548c", 33, 34}, Token{"C++", 34, 37}, Token{"\u3002", 37, 38}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u4e0d", 1, 2}, Token{"\u559c\u6b22", 2, 4}, Token{"\u65e5\u672c", 4, 6}, Token{"\u548c\u670d", 6, 8}, Token{"\u3002", 8, 9}},
[]Token{Token{"\u96f7\u7334", 0, 2}, Token{"\u56de\u5f52", 2, 4}, Token{"\u4eba\u95f4", 4, 6}, Token{"\u3002", 6, 7}},
[]Token{Token{"\u5de5\u4fe1\u5904", 0, 3}, Token{"\u5e72\u4e8b", 4, 6}, Token{"\u5973\u5e72\u4e8b", 3, 6}, Token{"\u6bcf\u6708", 6, 8}, Token{"\u7ecf\u8fc7", 8, 10}, Token{"\u4e0b\u5c5e", 10, 12}, Token{"\u79d1\u5ba4", 12, 14}, Token{"\u90fd", 14, 15}, Token{"\u8981", 15, 16}, Token{"\u4eb2\u53e3", 16, 18}, Token{"\u4ea4\u4ee3", 18, 20}, Token{"24", 20, 22}, Token{"\u53e3", 22, 23}, Token{"\u4ea4\u6362", 23, 25}, Token{"\u6362\u673a", 24, 26}, Token{"\u4ea4\u6362\u673a", 23, 26}, Token{"\u7b49", 26, 27}, Token{"\u6280\u672f", 27, 29}, Token{"\u6280\u672f\u6027", 27, 30}, Token{"\u5668\u4ef6", 30, 32}, Token{"\u7684", 32, 33}, Token{"\u5b89\u88c5", 33, 35}, Token{"\u5de5\u4f5c", 35, 37}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u9700\u8981", 1, 3}, Token{"\u5ec9\u79df", 3, 5}, Token{"\u79df\u623f", 4, 6}, Token{"\u5ec9\u79df\u623f", 3, 6}},
[]Token{Token{"\u6c38\u548c", 0, 2}, Token{"\u670d\u88c5", 2, 4}, Token{"\u9970\u54c1", 4, 6}, Token{"\u6709\u9650", 6, 8}, Token{"\u516c\u53f8", 8, 10}, Token{"\u6709\u9650\u516c\u53f8", 6, 10}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u7231", 1, 2}, Token{"\u5317\u4eac", 2, 4}, Token{"\u5929\u5b89", 4, 6}, Token{"\u5929\u5b89\u95e8", 4, 7}},
[]Token{Token{"abc", 0, 3}},
[]Token{Token{"\u9690", 0, 1}, Token{"\u53ef\u592b", 3, 5}, Token{"\u9a6c\u5c14\u53ef", 1, 4}, Token{"\u9a6c\u5c14\u53ef\u592b", 1, 5}},
[]Token{Token{"\u96f7\u7334", 0, 2}, Token{"\u662f", 2, 3}, Token{"\u4e2a", 3, 4}, Token{"\u597d", 4, 5}, Token{"\u7f51\u7ad9", 5, 7}},
[]Token{Token{"\u201c", 0, 1}, Token{"Microsoft", 1, 10}, Token{"\u201d", 10, 11}, Token{"\u4e00", 11, 12}, Token{"\u8bcd", 12, 13}, Token{"\u7531", 13, 14}, Token{"\u201c", 14, 15}, Token{"MICROcomputer", 15, 28}, Token{"\uff08", 28, 29}, Token{"\u5fae\u578b", 29, 31}, Token{"\u8ba1\u7b97", 31, 33}, Token{"\u7b97\u673a", 32, 34}, Token{"\u8ba1\u7b97\u673a", 31, 34}, Token{"\uff09", 34, 35}, Token{"\u201d", 35, 36}, Token{"\u548c", 36, 37}, Token{"\u201c", 37, 38}, Token{"SOFTware", 38, 46}, Token{"\uff08", 46, 47}, Token{"\u8f6f\u4ef6", 47, 49}, Token{"\uff09", 49, 50}, Token{"\u201d", 50, 51}, Token{"\u4e24", 51, 52}, Token{"\u90e8\u5206", 52, 54}, Token{"\u7ec4\u6210", 54, 56}},
[]Token{Token{"\u8349\u6ce5\u9a6c", 0, 3}, Token{"\u548c", 3, 4}, Token{"\u6b3a", 4, 5}, Token{"\u5b9e", 5, 6}, Token{"\u9a6c", 6, 7}, Token{"\u662f", 7, 8}, Token{"\u4eca\u5e74", 8, 10}, Token{"\u7684", 10, 11}, Token{"\u6d41\u884c", 11, 13}, Token{"\u8bcd\u6c47", 13, 15}},
[]Token{Token{"\u4f0a", 0, 1}, Token{"\u85e4", 1, 2}, Token{"\u6d0b\u534e\u5802", 2, 5}, Token{"\u603b\u5e9c", 5, 7}, Token{"\u5e97", 7, 8}},
[]Token{Token{"\u4e2d\u56fd", 0, 2}, Token{"\u79d1\u5b66", 2, 4}, Token{"\u5b66\u9662", 3, 5}, Token{"\u8ba1\u7b97", 5, 7}, Token{"\u6280\u672f", 7, 9}, Token{"\u7814\u7a76", 9, 11}, Token{"\u79d1\u5b66\u9662", 2, 5}, Token{"\u7814\u7a76\u6240", 9, 12}, Token{"\u4e2d\u56fd\u79d1\u5b66\u9662\u8ba1\u7b97\u6280\u672f\u7814\u7a76\u6240", 0, 12}},
[]Token{Token{"\u7f57\u5bc6\u6b27", 0, 3}, Token{"\u4e0e", 3, 4}, Token{"\u6731\u4e3d\u53f6", 4, 7}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u8d2d\u4e70", 1, 3}, Token{"\u4e86", 3, 4}, Token{"\u9053\u5177", 4, 6}, Token{"\u548c", 6, 7}, Token{"\u670d\u88c5", 7, 9}},
[]Token{Token{"PS", 0, 2}, Token{":", 2, 3}, Token{" ", 3, 4}, Token{"\u6211", 4, 5}, Token{"\u89c9\u5f97", 5, 7}, Token{"\u5f00\u6e90", 7, 9}, Token{"\u6709", 9, 10}, Token{"\u4e00\u4e2a", 10, 12}, Token{"\u597d\u5904", 12, 14}, Token{"\uff0c", 14, 15}, Token{"\u5c31\u662f", 15, 17}, Token{"\u80fd\u591f", 17, 19}, Token{"\u6566\u4fc3", 19, 21}, Token{"\u81ea\u5df1", 21, 23}, Token{"\u4e0d\u65ad", 23, 25}, Token{"\u6539\u8fdb", 25, 27}, Token{"\u4e0d\u65ad\u6539\u8fdb", 23, 27}, Token{"\uff0c", 27, 28}, Token{"\u907f\u514d", 28, 30}, Token{"\u655e", 30, 31}, Token{"\u5e1a", 31, 32}, Token{"\u81ea\u73cd", 32, 34}},
[]Token{Token{"\u6e56\u5317", 0, 2}, Token{"\u6e56\u5317\u7701", 0, 3}, Token{"\u77f3\u9996", 3, 5}, Token{"\u77f3\u9996\u5e02", 3, 6}},
[]Token{Token{"\u6e56\u5317", 0, 2}, Token{"\u6e56\u5317\u7701", 0, 3}, Token{"\u5341\u5830", 3, 5}, Token{"\u5341\u5830\u5e02", 3, 6}},
[]Token{Token{"\u7ecf\u7406", 1, 3}, Token{"\u603b\u7ecf\u7406", 0, 3}, Token{"\u5b8c\u6210", 3, 5}, Token{"\u4e86", 5, 6}, Token{"\u8fd9\u4ef6", 6, 8}, Token{"\u4e8b\u60c5", 8, 10}},
[]Token{Token{"\u7535\u8111", 0, 2}, Token{"\u4fee\u597d", 2, 4}, Token{"\u4e86", 4, 5}},
[]Token{Token{"\u505a\u597d", 0, 2}, Token{"\u4e86", 2, 3}, Token{"\u8fd9\u4ef6", 3, 5}, Token{"\u4e8b\u60c5", 5, 7}, Token{"\u5c31", 7, 8}, Token{"\u4e00\u4e86\u767e\u4e86", 8, 12}, Token{"\u4e86", 12, 13}},
[]Token{Token{"\u4eba\u4eec", 0, 2}, Token{"\u5ba1\u7f8e", 2, 4}, Token{"\u7684", 4, 5}, Token{"\u89c2\u70b9", 5, 7}, Token{"\u662f", 7, 8}, Token{"\u4e0d\u540c", 8, 10}, Token{"\u7684", 10, 11}},
[]Token{Token{"\u6211\u4eec", 0, 2}, Token{"\u4e70", 2, 3}, Token{"\u4e86", 3, 4}, Token{"\u4e00\u4e2a", 4, 6}, Token{"\u7f8e\u7684", 6, 8}, Token{"\u7a7a\u8c03", 8, 10}},
[]Token{Token{"\u7ebf\u7a0b", 0, 2}, Token{"\u521d\u59cb", 2, 4}, Token{"\u521d\u59cb\u5316", 2, 5}, Token{"\u65f6", 5, 6}, Token{"\u6211\u4eec", 6, 8}, Token{"\u8981", 8, 9}, Token{"\u6ce8\u610f", 9, 11}},
[]Token{Token{"\u4e00\u4e2a", 0, 2}, Token{"\u5206\u5b50", 2, 4}, Token{"\u662f", 4, 5}, Token{"\u7531", 5, 6}, Token{"\u597d\u591a", 6, 8}, Token{"\u539f\u5b50", 8, 10}, Token{"\u7ec4\u7ec7", 10, 12}, Token{"\u6210", 12, 13}, Token{"\u7684", 13, 14}},
[]Token{Token{"\u795d", 0, 1}, Token{"\u4f60", 1, 2}, Token{"\u9a6c\u5230\u529f\u6210", 2, 6}},
[]Token{Token{"\u4ed6", 0, 1}, Token{"\u6389", 1, 2}, Token{"\u8fdb", 2, 3}, Token{"\u4e86", 3, 4}, Token{"\u65e0\u5e95", 4, 6}, Token{"\u65e0\u5e95\u6d1e", 4, 7}, Token{"\u91cc", 7, 8}},
[]Token{Token{"\u4e2d\u56fd", 0, 2}, Token{"\u7684", 2, 3}, Token{"\u9996\u90fd", 3, 5}, Token{"\u662f", 5, 6}, Token{"\u5317\u4eac", 6, 8}},
[]Token{Token{"\u5b59", 0, 1}, Token{"\u541b", 1, 2}, Token{"\u610f", 2, 3}},
[]Token{Token{"\u5916\u4ea4", 0, 2}, Token{"\u5916\u4ea4\u90e8", 0, 3}, Token{"\u53d1\u8a00", 3, 5}, Token{"\u53d1\u8a00\u4eba", 3, 6}, Token{"\u9a6c\u671d\u65ed", 6, 9}},
[]Token{Token{"\u9886\u5bfc", 0, 2}, Token{"\u9886\u5bfc\u4eba", 0, 3}, Token{"\u4f1a\u8bae", 3, 5}, Token{"\u548c", 5, 6}, Token{"\u7b2c\u56db", 6, 8}, Token{"\u56db\u5c4a", 7, 9}, Token{"\u7b2c\u56db\u5c4a", 6, 9}, Token{"\u4e1c\u4e9a", 9, 11}, Token{"\u5cf0\u4f1a", 11, 13}},
[]Token{Token{"\u5728", 0, 1}, Token{"\u8fc7\u53bb", 1, 3}, Token{"\u7684", 3, 4}, Token{"\u8fd9", 4, 5}, Token{"\u4e94\u5e74", 5, 7}},
[]Token{Token{"\u8fd8", 0, 1}, Token{"\u9700\u8981", 1, 3}, Token{"\u5f88", 3, 4}, Token{"\u957f", 4, 5}, Token{"\u7684", 5, 6}, Token{"\u8def", 6, 7}, Token{"\u8981", 7, 8}, Token{"\u8d70", 8, 9}},
[]Token{Token{"60", 0, 2}, Token{"\u5468\u5e74", 2, 4}, Token{"\u9996\u90fd", 4, 6}, Token{"\u9605\u5175", 6, 8}},
[]Token{Token{"\u4f60\u597d", 0, 2}, Token{"\u4eba\u4eec", 2, 4}, Token{"\u5ba1\u7f8e", 4, 6}, Token{"\u7684", 6, 7}, Token{"\u89c2\u70b9", 7, 9}, Token{"\u662f", 9, 10}, Token{"\u4e0d\u540c", 10, 12}, Token{"\u7684", 12, 13}},
[]Token{Token{"\u4e70", 0, 1}, Token{"\u6c34\u679c", 1, 3}, Token{"\u7136\u540e", 3, 5}, Token{"\u6765", 5, 6}, Token{"\u4e16\u535a", 6, 8}, Token{"\u535a\u56ed", 7, 9}, Token{"\u4e16\u535a\u56ed", 6, 9}},
[]Token{Token{"\u4e70", 0, 1}, Token{"\u6c34\u679c", 1, 3}, Token{"\u7136\u540e", 3, 5}, Token{"\u53bb", 5, 6}, Token{"\u4e16\u535a", 6, 8}, Token{"\u535a\u56ed", 7, 9}, Token{"\u4e16\u535a\u56ed", 6, 9}},
[]Token{Token{"\u4f46\u662f", 0, 2}, Token{"\u540e\u6765", 2, 4}, Token{"\u6211", 4, 5}, Token{"\u624d", 5, 6}, Token{"\u77e5\u9053", 6, 8}, Token{"\u4f60", 8, 9}, Token{"\u662f", 9, 10}, Token{"\u5bf9", 10, 11}, Token{"\u7684", 11, 12}},
[]Token{Token{"\u5b58\u5728", 0, 2}, Token{"\u5373", 2, 3}, Token{"\u5408\u7406", 3, 5}},
[]Token{Token{"\u7684", 0, 1}, Token{"\u7684", 1, 2}, Token{"\u7684", 2, 3}, Token{"\u7684", 3, 4}, Token{"\u7684", 4, 5}, Token{"\u5728", 5, 6}, Token{"\u7684", 6, 7}, Token{"\u7684", 7, 8}, Token{"\u7684", 8, 9}, Token{"\u7684", 9, 10}, Token{"\u5c31", 10, 11}, Token{"\u4ee5", 11, 12}, Token{"\u548c", 12, 13}, Token{"\u548c", 13, 14}, Token{"\u548c", 14, 15}},
[]Token{Token{"I", 0, 1}, Token{" ", 1, 2}, Token{"love", 2, 6}, Token{"\u4f60", 6, 7}, Token{"\uff0c", 7, 8}, Token{"\u4e0d\u4ee5", 8, 10}, Token{"\u4ee5\u4e3a", 9, 11}, Token{"\u4e0d\u4ee5\u4e3a\u803b", 8, 12}, Token{"\uff0c", 12, 13}, Token{"\u53cd", 13, 14}, Token{"\u4ee5\u4e3a", 14, 16}, Token{"rong", 16, 20}},
[]Token{Token{"\u56e0", 0, 1}},
[]Token{},
[]Token{Token{"hello", 0, 5}, Token{"\u4f60\u597d", 5, 7}, Token{"\u4eba\u4eec", 7, 9}, Token{"\u5ba1\u7f8e", 9, 11}, Token{"\u7684", 11, 12}, Token{"\u89c2\u70b9", 12, 14}, Token{"\u662f", 14, 15}, Token{"\u4e0d\u540c", 15, 17}, Token{"\u7684", 17, 18}},
[]Token{Token{"\u5f88", 0, 1}, Token{"\u597d", 1, 2}, Token{"\u4f46", 2, 3}, Token{"\u4e3b\u8981", 3, 5}, Token{"\u662f", 5, 6}, Token{"\u57fa\u4e8e", 6, 8}, Token{"\u7f51\u9875", 8, 10}, Token{"\u5f62\u5f0f", 10, 12}},
[]Token{Token{"hello", 0, 5}, Token{"\u4f60\u597d", 5, 7}, Token{"\u4eba\u4eec", 7, 9}, Token{"\u5ba1\u7f8e", 9, 11}, Token{"\u7684", 11, 12}, Token{"\u89c2\u70b9", 12, 14}, Token{"\u662f", 14, 15}, Token{"\u4e0d\u540c", 15, 17}, Token{"\u7684", 17, 18}},
[]Token{Token{"\u4ec0\u4e48", 1, 3}, Token{"\u4e3a\u4ec0\u4e48", 0, 3}, Token{"\u6211", 3, 4}, Token{"\u4e0d\u80fd", 4, 6}, Token{"\u62e5\u6709", 6, 8}, Token{"\u60f3\u8981", 8, 10}, Token{"\u7684", 10, 11}, Token{"\u751f\u6d3b", 11, 13}},
[]Token{Token{"\u540e\u6765", 0, 2}, Token{"\u6211", 2, 3}, Token{"\u624d", 3, 4}},
[]Token{Token{"\u6b64\u6b21", 0, 2}, Token{"\u6765", 2, 3}, Token{"\u4e2d\u56fd", 3, 5}, Token{"\u662f", 5, 6}, Token{"\u4e3a\u4e86", 6, 8}},
[]Token{Token{"\u4f7f\u7528", 0, 2}, Token{"\u4e86", 2, 3}, Token{"\u5b83", 3, 4}, Token{"\u5c31", 4, 5}, Token{"\u53ef\u4ee5", 5, 7}, Token{"\u89e3\u51b3", 7, 9}, Token{"\u4e00\u4e9b", 9, 11}, Token{"\u95ee\u9898", 11, 13}},
[]Token{Token{",", 0, 1}, Token{"\u4f7f\u7528", 1, 3}, Token{"\u4e86", 3, 4}, Token{"\u5b83", 4, 5}, Token{"\u5c31", 5, 6}, Token{"\u53ef\u4ee5", 6, 8}, Token{"\u89e3\u51b3", 8, 10}, Token{"\u4e00\u4e9b", 10, 12}, Token{"\u95ee\u9898", 12, 14}},
[]Token{Token{"\u5176\u5b9e", 0, 2}, Token{"\u4f7f\u7528", 2, 4}, Token{"\u4e86", 4, 5}, Token{"\u5b83", 5, 6}, Token{"\u5c31", 6, 7}, Token{"\u53ef\u4ee5", 7, 9}, Token{"\u89e3\u51b3", 9, 11}, Token{"\u4e00\u4e9b", 11, 13}, Token{"\u95ee\u9898", 13, 15}},
[]Token{Token{"\u597d\u4eba", 0, 2}, Token{"\u4f7f\u7528", 2, 4}, Token{"\u4e86", 4, 5}, Token{"\u5b83", 5, 6}, Token{"\u5c31", 6, 7}, Token{"\u53ef\u4ee5", 7, 9}, Token{"\u89e3\u51b3", 9, 11}, Token{"\u4e00\u4e9b", 11, 13}, Token{"\u95ee\u9898", 13, 15}},
[]Token{Token{"\u56e0\u4e3a", 1, 3}, Token{"\u662f\u56e0\u4e3a", 0, 3}, Token{"\u548c", 3, 4}, Token{"\u56fd\u5bb6", 4, 6}},
[]Token{Token{"\u8001\u5e74", 0, 2}, Token{"\u641c\u7d22", 2, 4}, Token{"\u8fd8", 4, 5}, Token{"\u652f\u6301", 5, 7}},
[]Token{Token{"\u5e72\u8106", 0, 2}, Token{"\u5c31", 2, 3}, Token{"\u628a", 3, 4}, Token{"\u90a3", 4, 5}, Token{"\u90e8", 5, 6}, Token{"\u8499", 6, 7}, Token{"\u4eba", 7, 8}, Token{"\u7684", 8, 9}, Token{"\u95f2", 9, 10}, Token{"\u6cd5", 10, 11}, Token{"\u7ed9", 11, 12}, Token{"\u5e9f", 12, 13}, Token{"\u4e86", 13, 14}, Token{"\u62c9\u5012", 14, 16}, Token{"\uff01", 16, 17}, Token{"RT", 17, 19}, Token{" ", 19, 20}, Token{"@", 20, 21}, Token{"laoshipukong", 21, 33}, Token{" ", 33, 34}, Token{":", 34, 35}, Token{" ", 35, 36}, Token{"27", 36, 38}, Token{"\u65e5", 38, 39}, Token{"\uff0c", 39, 40}, Token{"\u5168\u56fd", 40, 42}, Token{"\u56fd\u4eba", 41, 43}, Token{"\u4eba\u5927", 42, 44}, Token{"\u5e38\u59d4", 44, 46}, Token{"\u59d4\u4f1a", 45, 47}, Token{"\u5e38\u59d4\u4f1a", 44, 47}, Token{"\u5168\u56fd\u4eba\u5927\u5e38\u59d4\u4f1a", 40, 47}, Token{"\u7b2c\u4e09", 47, 49}, Token{"\u4e09\u6b21", 48, 50}, Token{"\u7b2c\u4e09\u6b21", 47, 50}, Token{"\u5ba1\u8bae", 50, 52}, Token{"\u4fb5\u6743", 52, 54}, Token{"\u8d23\u4efb", 54, 56}, Token{"\u8d23\u4efb\u6cd5", 54, 57}, Token{"\u8349\u6848", 57, 59}, Token{"\uff0c", 59, 60}, Token{"\u5220\u9664", 60, 62}, Token{"\u4e86", 62, 63}, Token{"\u6709\u5173", 63, 65}, Token{"\u533b\u7597", 65, 67}, Token{"\u635f\u5bb3", 67, 69}, Token{"\u8d23\u4efb", 69, 71}, Token{"\u201c", 71, 72}, Token{"\u4e3e\u8bc1", 72, 74}, Token{"\u5012\u7f6e", 74, 76}, Token{"\u201d", 76, 77}, Token{"\u7684", 77, 78}, Token{"\u89c4\u5b9a", 78, 80}, Token{"\u3002", 80, 81}, Token{"\u5728", 81, 82}, Token{"\u533b\u60a3", 82, 84}, Token{"\u7ea0\u7eb7", 84, 86}, Token{"\u4e2d", 86, 87}, Token{"\u672c", 87, 88}, Token{"\u5df2", 88, 89}, Token{"\u5904\u4e8e", 89, 91}, Token{"\u5f31\u52bf", 91, 93}, Token{"\u5730\u4f4d", 93, 95}, Token{"\u7684", 95, 96}, Token{"\u6d88\u8d39", 96, 98}, Token{"\u6d88\u8d39\u8005", 96, 99}, Token{"\u7531\u6b64", 99, 101}, Token{"\u5c06", 101, 102}, Token{"\u9677\u5165", 102, 104}, Token{"\u4e0d\u590d", 106, 108}, Token{"\u4e07\u52ab\u4e0d\u590d", 104, 108}, Token{"\u7684", 108, 109}, Token{"\u5883\u5730", 109, 111}, Token{"\u3002", 111, 112}, Token{" ", 112, 113}},
[]Token{Token{"\u5927", 0, 1}},
[]Token{},
[]Token{Token{"\u4ed6", 0, 1}, Token{"\u8bf4", 1, 2}, Token{"\u7684", 2, 3}, Token{"\u786e\u5b9e", 3, 5}, Token{"\u5728", 5, 6}, Token{"\u7406", 6, 7}},
[]Token{Token{"\u957f\u6625", 0, 2}, Token{"\u5e02\u957f", 2, 4}, Token{"\u6625\u8282", 4, 6}, Token{"\u8bb2\u8bdd", 6, 8}},
[]Token{Token{"\u7ed3\u5a5a", 0, 2}, Token{"\u7684", 2, 3}, Token{"\u548c", 3, 4}, Token{"\u5c1a\u672a", 4, 6}, Token{"\u7ed3\u5a5a", 6, 8}, Token{"\u7684", 8, 9}},
[]Token{Token{"\u7ed3\u5408", 0, 2}, Token{"\u6210", 2, 3}, Token{"\u5206\u5b50", 3, 5}, Token{"\u65f6", 5, 6}},
[]Token{Token{"\u65c5\u6e38", 0, 2}, Token{"\u548c", 2, 3}, Token{"\u670d\u52a1", 3, 5}, Token{"\u662f", 5, 6}, Token{"\u6700\u597d", 6, 8}, Token{"\u7684", 8, 9}},
[]Token{Token{"\u8fd9\u4ef6", 0, 2}, Token{"\u4e8b\u60c5", 2, 4}, Token{"\u7684\u786e", 4, 6}, Token{"\u662f", 6, 7}, Token{"\u6211", 7, 8}, Token{"\u7684", 8, 9}, Token{"\u9519", 9, 10}},
[]Token{Token{"\u4f9b", 0, 1}, Token{"\u5927\u5bb6", 1, 3}, Token{"\u53c2\u8003", 3, 5}, Token{"\u6307\u6b63", 5, 7}},
[]Token{Token{"\u54c8\u5c14", 0, 2}, Token{"\u54c8\u5c14\u6ee8", 0, 3}, Token{"\u653f\u5e9c", 3, 5}, Token{"\u516c\u5e03", 5, 7}, Token{"\u584c", 7, 8}, Token{"\u6865", 8, 9}, Token{"\u539f\u56e0", 9, 11}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u5728", 1, 2}, Token{"\u673a\u573a", 2, 4}, Token{"\u5165\u53e3", 4, 6}, Token{"\u5165\u53e3\u5904", 4, 7}},
[]Token{Token{"\u90a2", 0, 1}, Token{"\u6c38", 1, 2}, Token{"\u81e3", 2, 3}, Token{"\u6444\u5f71", 3, 5}, Token{"\u62a5\u9053", 5, 7}},
[]Token{Token{"BP", 0, 2}, Token{"\u795e\u7ecf", 2, 4}, Token{"\u7f51\u7edc", 4, 6}, Token{"\u795e\u7ecf\u7f51", 2, 5}, Token{"\u795e\u7ecf\u7f51\u7edc", 2, 6}, Token{"\u5982\u4f55", 6, 8}, Token{"\u8bad\u7ec3", 8, 10}, Token{"\u624d\u80fd", 10, 12}, Token{"\u5728", 12, 13}, Token{"\u5206\u7c7b", 13, 15}, Token{"\u65f6", 15, 16}, Token{"\u589e\u52a0", 16, 18}, Token{"\u533a\u5206", 18, 20}, Token{"\u5206\u5ea6", 19, 21}, Token{"\u533a\u5206\u5ea6", 18, 21}, Token{"\uff1f", 21, 22}},
[]Token{Token{"\u5357\u4eac", 0, 2}, Token{"\u4eac\u5e02", 1, 3}, Token{"\u5357\u4eac\u5e02", 0, 3}, Token{"\u957f\u6c5f", 3, 5}, Token{"\u5927\u6865", 5, 7}, Token{"\u957f\u6c5f\u5927\u6865", 3, 7}},
[]Token{Token{"\u5e94", 0, 1}, Token{"\u4e00\u4e9b", 1, 3}, Token{"\u4f7f\u7528", 3, 5}, Token{"\u7528\u8005", 4, 6}, Token{"\u4f7f\u7528\u8005", 3, 6}, Token{"\u7684", 6, 7}, Token{"\u5efa\u8bae", 7, 9}, Token{"\uff0c", 9, 10}, Token{"\u4e5f", 10, 11}, Token{"\u4e3a\u4e86", 11, 13}, Token{"\u4fbf\u4e8e", 13, 15}, Token{"\u5229\u7528", 15, 17}, Token{"NiuTrans", 17, 25}, Token{"\u7528\u4e8e", 25, 27}, Token{"SMT", 27, 30}, Token{"\u7814\u7a76", 30, 32}},
[]Token{Token{"\u957f\u6625", 0, 2}, Token{"\u957f\u6625\u5e02", 0, 3}, Token{"\u957f\u6625", 3, 5}, Token{"\u836f\u5e97", 5, 7}},
[]Token{Token{"\u9093\u9896\u8d85", 0, 3}, Token{"\u751f\u524d", 3, 5}, Token{"\u6700", 5, 6}, Token{"\u559c\u6b22", 6, 8}, Token{"\u7684", 8, 9}, Token{"\u8863\u670d", 9, 11}},
[]Token{Token{"\u9526\u6d9b", 1, 3}, Token{"\u80e1\u9526\u6d9b", 0, 3}, Token{"\u662f", 3, 4}, Token{"\u70ed\u7231", 4, 6}, Token{"\u4e16\u754c", 6, 8}, Token{"\u548c\u5e73", 8, 10}, Token{"\u7684", 10, 11}, Token{"\u653f\u6cbb", 11, 13}, Token{"\u653f\u6cbb\u5c40", 11, 14}, Token{"\u5e38\u59d4", 14, 16}},
[]Token{Token{"\u7a0b\u5e8f", 0, 2}, Token{"\u7a0b\u5e8f\u5458", 0, 3}, Token{"\u795d", 3, 4}, Token{"\u6d77\u6797", 4, 6}, Token{"\u548c", 6, 7}, Token{"\u6731", 7, 8}, Token{"\u4f1a", 8, 9}, Token{"\u9707", 9, 10}, Token{"\u662f", 10, 11}, Token{"\u5728", 11, 12}, Token{"\u5b59", 12, 13}, Token{"\u5065", 13, 14}, Token{"\u7684", 14, 15}, Token{"\u5de6\u9762", 15, 17}, Token{"\u548c", 17, 18}, Token{"\u53f3\u9762", 18, 20}, Token{",", 20, 21}, Token{" ", 21, 22}, Token{"\u8303", 22, 23}, Token{"\u51ef", 23, 24}, Token{"\u5728", 24, 25}, Token{"\u6700", 25, 26}, Token{"\u53f3\u9762", 26, 28}, Token{".", 28, 29}, Token{"\u518d", 29, 30}, Token{"\u5f80", 30, 31}, Token{"\u5de6", 31, 32}, Token{"\u662f", 32, 33}, Token{"\u674e", 33, 34}, Token{"\u677e", 34, 35}, Token{"\u6d2a", 35, 36}},
[]Token{Token{"\u4e00\u6b21", 0, 2}, Token{"\u4e00\u6b21\u6027", 0, 3}, Token{"\u4ea4", 3, 4}, Token{"\u591a\u5c11", 4, 6}, Token{"\u94b1", 6, 7}},
[]Token{Token{"\u4e24\u5757", 0, 2}, Token{"\u4e94", 2, 3}, Token{"\u4e00\u5957", 3, 5}, Token{"\uff0c", 5, 6}, Token{"\u4e09\u5757", 6, 8}, Token{"\u516b", 8, 9}, Token{"\u4e00\u65a4", 9, 11}, Token{"\uff0c", 11, 12}, Token{"\u56db\u5757", 12, 14}, Token{"\u4e03", 14, 15}, Token{"\u4e00\u672c", 15, 17}, Token{"\uff0c", 17, 18}, Token{"\u4e94\u5757", 18, 20}, Token{"\u516d", 20, 21}, Token{"\u4e00\u6761", 21, 23}},
[]Token{Token{"\u5c0f", 0, 1}, Token{"\u548c\u5c1a", 1, 3}, Token{"\u7559", 3, 4}, Token{"\u4e86", 4, 5}, Token{"\u4e00\u4e2a", 5, 7}, Token{"\u50cf", 7, 8}, Token{"\u5927", 8, 9}, Token{"\u548c\u5c1a", 9, 11}, Token{"\u4e00\u6837", 11, 13}, Token{"\u7684", 13, 14}, Token{"\u548c\u5c1a", 14, 16}, Token{"\u548c\u5c1a\u5934", 14, 17}},
[]Token{Token{"\u6211", 0, 1}, Token{"\u662f", 1, 2}, Token{"\u4e2d\u534e", 2, 4}, Token{"\u534e\u4eba", 3, 5}, Token{"\u4eba\u6c11", 4, 6}, Token{"\u5171\u548c", 6, 8}, Token{"\u5171\u548c\u56fd", 6, 9}, Token{"\u4e2d\u534e\u4eba\u6c11\u5171\u548c\u56fd", 2, 9}, Token{"\u516c\u6c11", 9, 11}, Token{";", 11, 12}, Token{"\u6211", 12, 13}, Token{"\u7238\u7238", 13, 15}, Token{"\u662f", 15, 16}, Token{"\u5171\u548c", 16, 18}, Token{"\u5171\u548c\u515a", 16, 19}, Token{"\u515a\u5458", 19, 21}, Token{";", 21, 22}, Token{" ", 22, 23}, Token{"\u5730\u94c1", 23, 25}, Token{"\u548c\u5e73", 25, 27}, Token{"\u548c\u5e73\u95e8", 25, 28}, Token{"\u7ad9", 28, 29}},
[]Token{Token{"\u5f20\u6653\u6885", 0, 3}, Token{"\u53bb", 3, 4}, Token{"\u4eba\u6c11", 4, 6}, Token{"\u533b\u9662", 6, 8}, Token{"\u505a", 8, 9}, Token{"\u4e86", 9, 10}, Token{"\u4e2a", 10, 11}, Token{"B\u8d85", 11, 13}, Token{"\u7136\u540e", 13, 15}, Token{"\u53bb", 15, 16}, Token{"\u4e70", 16, 17}, Token{"\u4e86", 17, 18}, Token{"\u4ef6", 18, 19}, Token{"T\u6064", 19, 21}},
[]Token{Token{"AT&T", 0, 4}, Token{"\u662f", 4, 5}, Token{"\u4e00\u4ef6", 5, 7}, Token{"\u4e0d\u9519", 7, 9}, Token{"\u7684", 9, 10}, Token{"\u516c\u53f8", 10, 12}, Token{"\uff0c", 12, 13}, Token{"\u7ed9", 13, 14}, Token{"\u4f60", 14, 15}, Token{"\u53d1", 15, 16}, Token{"offer", 16, 21}, Token{"\u4e86", 21, 22}, Token{"\u5417", 22, 23}, Token{"\uff1f", 23, 24}},
[]Token{Token{"C++", 0, 3}, Token{"\u548c", 3, 4}, Token{"c#", 4, 6}, Token{"\u662f", 6, 7}, Token{"\u4ec0\u4e48", 7, 9}, Token{"\u5173\u7cfb", 9, 11}, Token{"\uff1f", 11, 12}, Token{"11", 12, 14}, Token{"+", 14, 15}, Token{"122", 15, 18}, Token{"=", 18, 19}, Token{"133", 19, 22}, Token{"\uff0c", 22, 23}, Token{"\u662f", 23, 24}, Token{"\u5417", 24, 25}, Token{"\uff1f", 25, 26}, Token{"PI", 26, 28}, Token{"=", 28, 29}, Token{"3", 29, 30}, Token{".", 30, 31}, Token{"14159", 31, 36}},
[]Token{Token{"\u4f60", 0, 1}, Token{"\u8ba4\u8bc6", 1, 3}, Token{"\u90a3\u4e2a", 3, 5}, Token{"\u548c", 5, 6}, Token{"\u4e3b\u5e2d", 6, 8}, Token{"\u63e1\u624b", 8, 10}, Token{"\u7684", 10, 11}, Token{"\u7684\u54e5", 11, 13}, Token{"\u5417", 13, 14}, Token{"\uff1f", 14, 15}, Token{"\u4ed6", 15, 16}, Token{"\u5f00", 16, 17}, Token{"\u4e00\u8f86", 17, 19}, Token{"\u9ed1\u8272", 19, 21}, Token{"\u7684\u58eb", 21, 23}, Token{"\u3002", 23, 24}},
[]Token{Token{"\u67aa\u6746", 0, 2}, Token{"\u6746\u5b50", 1, 3}, Token{"\u67aa\u6746\u5b50", 0, 3}, Token{"\u4e2d", 3, 4}, Token{"\u51fa", 4, 5}, Token{"\u653f\u6743", 5, 7}},
[]Token{Token{"\u5f20", 0, 1}, Token{"\u4e09", 1, 2}, Token{"\u98ce", 2, 3}, Token{"\u540c\u5b66", 3, 5}, Token{"\u8d70\u4e0a", 5, 7}, Token{"\u4e86", 7, 8}, Token{"\u5f52\u8def", 9, 11}, Token{"\u4e0d\u5f52\u8def", 8, 11}},
[]Token{Token{"\u963fQ", 0, 2}, Token{"\u8170\u95f4", 2, 4}, Token{"\u6302", 4, 5}, Token{"\u7740", 5, 6}, Token{"BB\u673a", 6, 9}, Token{"\u624b\u91cc", 9, 11}, Token{"\u62ff", 11, 12}, Token{"\u7740", 12, 13}, Token{"\u5927\u54e5", 13, 15}, Token{"\u5927\u54e5\u5927", 13, 16}, Token{"\uff0c", 16, 17}, Token{"\u8bf4", 17, 18}, Token{"\uff1a", 18, 19}, Token{"\u6211", 19, 20}, Token{"\u4e00\u822c", 20, 22}, Token{"\u5403\u996d", 22, 24}, Token{"\u4e0d", 24, 25}, Token{"AA\u5236", 25, 28}, Token{"\u7684", 28, 29}, Token{"\u3002", 29, 30}},
[]Token{Token{"\u5728", 0, 1}, Token{"1\u53f7\u5e97", 1, 4}, Token{"\u80fd", 4, 5}, Token{"\u4e70", 5, 6}, Token{"\u5230", 6, 7}, Token{"\u5c0fS", 7, 9}, Token{"\u548c", 9, 10}, Token{"\u5927S", 10, 12}, Token{"\u516b\u5366", 12, 14}, Token{"\u7684", 14, 15}, Token{"\u4e66", 15, 16}, Token{"\u3002", 16, 17}},
noHmmResult = [][]token{
[]token{token{"\u8fd9", 0, 1}, token{"\u662f", 1, 2}, token{"\u4e00\u4e2a", 2, 4}, token{"\u4f38\u624b\u4e0d\u89c1\u4e94\u6307", 4, 10}, token{"\u7684", 10, 11}, token{"\u9ed1\u591c", 11, 13}, token{"\u3002", 13, 14}, token{"\u6211", 14, 15}, token{"\u53eb", 15, 16}, token{"\u5b59\u609f\u7a7a", 16, 19}, token{"\uff0c", 19, 20}, token{"\u6211", 20, 21}, token{"\u7231", 21, 22}, token{"\u5317\u4eac", 22, 24}, token{"\uff0c", 24, 25}, token{"\u6211", 25, 26}, token{"\u7231", 26, 27}, token{"Python", 27, 33}, token{"\u548c", 33, 34}, token{"C++", 34, 37}, token{"\u3002", 37, 38}},
[]token{token{"\u6211", 0, 1}, token{"\u4e0d", 1, 2}, token{"\u559c\u6b22", 2, 4}, token{"\u65e5\u672c", 4, 6}, token{"\u548c\u670d", 6, 8}, token{"\u3002", 8, 9}},
[]token{token{"\u96f7\u7334", 0, 2}, token{"\u56de\u5f52", 2, 4}, token{"\u4eba\u95f4", 4, 6}, token{"\u3002", 6, 7}},
[]token{token{"\u5de5\u4fe1\u5904", 0, 3}, token{"\u5973\u5e72\u4e8b", 3, 6}, token{"\u6bcf\u6708", 6, 8}, token{"\u7ecf\u8fc7", 8, 10}, token{"\u4e0b\u5c5e", 10, 12}, token{"\u79d1\u5ba4", 12, 14}, token{"\u90fd", 14, 15}, token{"\u8981", 15, 16}, token{"\u4eb2\u53e3", 16, 18}, token{"\u4ea4\u4ee3", 18, 20}, token{"24", 20, 22}, token{"\u53e3", 22, 23}, token{"\u4ea4\u6362\u673a", 23, 26}, token{"\u7b49", 26, 27}, token{"\u6280\u672f\u6027", 27, 30}, token{"\u5668\u4ef6", 30, 32}, token{"\u7684", 32, 33}, token{"\u5b89\u88c5", 33, 35}, token{"\u5de5\u4f5c", 35, 37}},
[]token{token{"\u6211", 0, 1}, token{"\u9700\u8981", 1, 3}, token{"\u5ec9\u79df\u623f", 3, 6}},
[]token{token{"\u6c38\u548c", 0, 2}, token{"\u670d\u88c5", 2, 4}, token{"\u9970\u54c1", 4, 6}, token{"\u6709\u9650\u516c\u53f8", 6, 10}},
[]token{token{"\u6211", 0, 1}, token{"\u7231", 1, 2}, token{"\u5317\u4eac", 2, 4}, token{"\u5929\u5b89\u95e8", 4, 7}},
[]token{token{"abc", 0, 3}},
[]token{token{"\u9690", 0, 1}, token{"\u9a6c\u5c14\u53ef\u592b", 1, 5}},
[]token{token{"\u96f7\u7334", 0, 2}, token{"\u662f", 2, 3}, token{"\u4e2a", 3, 4}, token{"\u597d", 4, 5}, token{"\u7f51\u7ad9", 5, 7}},
[]token{token{"\u201c", 0, 1}, token{"Microsoft", 1, 10}, token{"\u201d", 10, 11}, token{"\u4e00", 11, 12}, token{"\u8bcd", 12, 13}, token{"\u7531", 13, 14}, token{"\u201c", 14, 15}, token{"MICROcomputer", 15, 28}, token{"\uff08", 28, 29}, token{"\u5fae\u578b", 29, 31}, token{"\u8ba1\u7b97\u673a", 31, 34}, token{"\uff09", 34, 35}, token{"\u201d", 35, 36}, token{"\u548c", 36, 37}, token{"\u201c", 37, 38}, token{"SOFTware", 38, 46}, token{"\uff08", 46, 47}, token{"\u8f6f\u4ef6", 47, 49}, token{"\uff09", 49, 50}, token{"\u201d", 50, 51}, token{"\u4e24", 51, 52}, token{"\u90e8\u5206", 52, 54}, token{"\u7ec4\u6210", 54, 56}},
[]token{token{"\u8349\u6ce5\u9a6c", 0, 3}, token{"\u548c", 3, 4}, token{"\u6b3a", 4, 5}, token{"\u5b9e", 5, 6}, token{"\u9a6c", 6, 7}, token{"\u662f", 7, 8}, token{"\u4eca\u5e74", 8, 10}, token{"\u7684", 10, 11}, token{"\u6d41\u884c", 11, 13}, token{"\u8bcd\u6c47", 13, 15}},
[]token{token{"\u4f0a", 0, 1}, token{"\u85e4", 1, 2}, token{"\u6d0b\u534e\u5802", 2, 5}, token{"\u603b\u5e9c", 5, 7}, token{"\u5e97", 7, 8}},
[]token{token{"\u4e2d\u56fd\u79d1\u5b66\u9662\u8ba1\u7b97\u6280\u672f\u7814\u7a76\u6240", 0, 12}},
[]token{token{"\u7f57\u5bc6\u6b27", 0, 3}, token{"\u4e0e", 3, 4}, token{"\u6731\u4e3d\u53f6", 4, 7}},
[]token{token{"\u6211", 0, 1}, token{"\u8d2d\u4e70", 1, 3}, token{"\u4e86", 3, 4}, token{"\u9053\u5177", 4, 6}, token{"\u548c", 6, 7}, token{"\u670d\u88c5", 7, 9}},
[]token{token{"PS", 0, 2}, token{":", 2, 3}, token{" ", 3, 4}, token{"\u6211", 4, 5}, token{"\u89c9\u5f97", 5, 7}, token{"\u5f00\u6e90", 7, 9}, token{"\u6709", 9, 10}, token{"\u4e00\u4e2a", 10, 12}, token{"\u597d\u5904", 12, 14}, token{"\uff0c", 14, 15}, token{"\u5c31\u662f", 15, 17}, token{"\u80fd\u591f", 17, 19}, token{"\u6566\u4fc3", 19, 21}, token{"\u81ea\u5df1", 21, 23}, token{"\u4e0d\u65ad\u6539\u8fdb", 23, 27}, token{"\uff0c", 27, 28}, token{"\u907f\u514d", 28, 30}, token{"\u655e", 30, 31}, token{"\u5e1a", 31, 32}, token{"\u81ea\u73cd", 32, 34}},
[]token{token{"\u6e56\u5317\u7701", 0, 3}, token{"\u77f3\u9996\u5e02", 3, 6}},
[]token{token{"\u6e56\u5317\u7701", 0, 3}, token{"\u5341\u5830\u5e02", 3, 6}},
[]token{token{"\u603b\u7ecf\u7406", 0, 3}, token{"\u5b8c\u6210", 3, 5}, token{"\u4e86", 5, 6}, token{"\u8fd9\u4ef6", 6, 8}, token{"\u4e8b\u60c5", 8, 10}},
[]token{token{"\u7535\u8111", 0, 2}, token{"\u4fee\u597d", 2, 4}, token{"\u4e86", 4, 5}},
[]token{token{"\u505a\u597d", 0, 2}, token{"\u4e86", 2, 3}, token{"\u8fd9\u4ef6", 3, 5}, token{"\u4e8b\u60c5", 5, 7}, token{"\u5c31", 7, 8}, token{"\u4e00\u4e86\u767e\u4e86", 8, 12}, token{"\u4e86", 12, 13}},
[]token{token{"\u4eba\u4eec", 0, 2}, token{"\u5ba1\u7f8e", 2, 4}, token{"\u7684", 4, 5}, token{"\u89c2\u70b9", 5, 7}, token{"\u662f", 7, 8}, token{"\u4e0d\u540c", 8, 10}, token{"\u7684", 10, 11}},
[]token{token{"\u6211\u4eec", 0, 2}, token{"\u4e70", 2, 3}, token{"\u4e86", 3, 4}, token{"\u4e00\u4e2a", 4, 6}, token{"\u7f8e\u7684", 6, 8}, token{"\u7a7a\u8c03", 8, 10}},
[]token{token{"\u7ebf\u7a0b", 0, 2}, token{"\u521d\u59cb\u5316", 2, 5}, token{"\u65f6", 5, 6}, token{"\u6211\u4eec", 6, 8}, token{"\u8981", 8, 9}, token{"\u6ce8\u610f", 9, 11}},
[]token{token{"\u4e00\u4e2a", 0, 2}, token{"\u5206\u5b50", 2, 4}, token{"\u662f", 4, 5}, token{"\u7531", 5, 6}, token{"\u597d\u591a", 6, 8}, token{"\u539f\u5b50", 8, 10}, token{"\u7ec4\u7ec7", 10, 12}, token{"\u6210", 12, 13}, token{"\u7684", 13, 14}},
[]token{token{"\u795d", 0, 1}, token{"\u4f60", 1, 2}, token{"\u9a6c\u5230\u529f\u6210", 2, 6}},
[]token{token{"\u4ed6", 0, 1}, token{"\u6389", 1, 2}, token{"\u8fdb", 2, 3}, token{"\u4e86", 3, 4}, token{"\u65e0\u5e95\u6d1e", 4, 7}, token{"\u91cc", 7, 8}},
[]token{token{"\u4e2d\u56fd", 0, 2}, token{"\u7684", 2, 3}, token{"\u9996\u90fd", 3, 5}, token{"\u662f", 5, 6}, token{"\u5317\u4eac", 6, 8}},
[]token{token{"\u5b59", 0, 1}, token{"\u541b", 1, 2}, token{"\u610f", 2, 3}},
[]token{token{"\u5916\u4ea4\u90e8", 0, 3}, token{"\u53d1\u8a00\u4eba", 3, 6}, token{"\u9a6c\u671d\u65ed", 6, 9}},
[]token{token{"\u9886\u5bfc\u4eba", 0, 3}, token{"\u4f1a\u8bae", 3, 5}, token{"\u548c", 5, 6}, token{"\u7b2c\u56db\u5c4a", 6, 9}, token{"\u4e1c\u4e9a", 9, 11}, token{"\u5cf0\u4f1a", 11, 13}},
[]token{token{"\u5728", 0, 1}, token{"\u8fc7\u53bb", 1, 3}, token{"\u7684", 3, 4}, token{"\u8fd9", 4, 5}, token{"\u4e94\u5e74", 5, 7}},
[]token{token{"\u8fd8", 0, 1}, token{"\u9700\u8981", 1, 3}, token{"\u5f88", 3, 4}, token{"\u957f", 4, 5}, token{"\u7684", 5, 6}, token{"\u8def", 6, 7}, token{"\u8981", 7, 8}, token{"\u8d70", 8, 9}},
[]token{token{"60", 0, 2}, token{"\u5468\u5e74", 2, 4}, token{"\u9996\u90fd", 4, 6}, token{"\u9605\u5175", 6, 8}},
[]token{token{"\u4f60\u597d", 0, 2}, token{"\u4eba\u4eec", 2, 4}, token{"\u5ba1\u7f8e", 4, 6}, token{"\u7684", 6, 7}, token{"\u89c2\u70b9", 7, 9}, token{"\u662f", 9, 10}, token{"\u4e0d\u540c", 10, 12}, token{"\u7684", 12, 13}},
[]token{token{"\u4e70", 0, 1}, token{"\u6c34\u679c", 1, 3}, token{"\u7136\u540e", 3, 5}, token{"\u6765", 5, 6}, token{"\u4e16\u535a\u56ed", 6, 9}},
[]token{token{"\u4e70", 0, 1}, token{"\u6c34\u679c", 1, 3}, token{"\u7136\u540e", 3, 5}, token{"\u53bb", 5, 6}, token{"\u4e16\u535a\u56ed", 6, 9}},
[]token{token{"\u4f46\u662f", 0, 2}, token{"\u540e\u6765", 2, 4}, token{"\u6211", 4, 5}, token{"\u624d", 5, 6}, token{"\u77e5\u9053", 6, 8}, token{"\u4f60", 8, 9}, token{"\u662f", 9, 10}, token{"\u5bf9", 10, 11}, token{"\u7684", 11, 12}},
[]token{token{"\u5b58\u5728", 0, 2}, token{"\u5373", 2, 3}, token{"\u5408\u7406", 3, 5}},
[]token{token{"\u7684", 0, 1}, token{"\u7684", 1, 2}, token{"\u7684", 2, 3}, token{"\u7684", 3, 4}, token{"\u7684", 4, 5}, token{"\u5728", 5, 6}, token{"\u7684", 6, 7}, token{"\u7684", 7, 8}, token{"\u7684", 8, 9}, token{"\u7684", 9, 10}, token{"\u5c31", 10, 11}, token{"\u4ee5", 11, 12}, token{"\u548c", 12, 13}, token{"\u548c", 13, 14}, token{"\u548c", 14, 15}},
[]token{token{"I", 0, 1}, token{" ", 1, 2}, token{"love", 2, 6}, token{"\u4f60", 6, 7}, token{"\uff0c", 7, 8}, token{"\u4e0d\u4ee5\u4e3a\u803b", 8, 12}, token{"\uff0c", 12, 13}, token{"\u53cd", 13, 14}, token{"\u4ee5\u4e3a", 14, 16}, token{"rong", 16, 20}},
[]token{token{"\u56e0", 0, 1}},
[]token{},
[]token{token{"hello", 0, 5}, token{"\u4f60\u597d", 5, 7}, token{"\u4eba\u4eec", 7, 9}, token{"\u5ba1\u7f8e", 9, 11}, token{"\u7684", 11, 12}, token{"\u89c2\u70b9", 12, 14}, token{"\u662f", 14, 15}, token{"\u4e0d\u540c", 15, 17}, token{"\u7684", 17, 18}},
[]token{token{"\u5f88", 0, 1}, token{"\u597d", 1, 2}, token{"\u4f46", 2, 3}, token{"\u4e3b\u8981", 3, 5}, token{"\u662f", 5, 6}, token{"\u57fa\u4e8e", 6, 8}, token{"\u7f51\u9875", 8, 10}, token{"\u5f62\u5f0f", 10, 12}},
[]token{token{"hello", 0, 5}, token{"\u4f60\u597d", 5, 7}, token{"\u4eba\u4eec", 7, 9}, token{"\u5ba1\u7f8e", 9, 11}, token{"\u7684", 11, 12}, token{"\u89c2\u70b9", 12, 14}, token{"\u662f", 14, 15}, token{"\u4e0d\u540c", 15, 17}, token{"\u7684", 17, 18}},
[]token{token{"\u4e3a\u4ec0\u4e48", 0, 3}, token{"\u6211", 3, 4}, token{"\u4e0d\u80fd", 4, 6}, token{"\u62e5\u6709", 6, 8}, token{"\u60f3\u8981", 8, 10}, token{"\u7684", 10, 11}, token{"\u751f\u6d3b", 11, 13}},
[]token{token{"\u540e\u6765", 0, 2}, token{"\u6211", 2, 3}, token{"\u624d", 3, 4}},
[]token{token{"\u6b64\u6b21", 0, 2}, token{"\u6765", 2, 3}, token{"\u4e2d\u56fd", 3, 5}, token{"\u662f", 5, 6}, token{"\u4e3a\u4e86", 6, 8}},
[]token{token{"\u4f7f\u7528", 0, 2}, token{"\u4e86", 2, 3}, token{"\u5b83", 3, 4}, token{"\u5c31", 4, 5}, token{"\u53ef\u4ee5", 5, 7}, token{"\u89e3\u51b3", 7, 9}, token{"\u4e00\u4e9b", 9, 11}, token{"\u95ee\u9898", 11, 13}},
[]token{token{",", 0, 1}, token{"\u4f7f\u7528", 1, 3}, token{"\u4e86", 3, 4}, token{"\u5b83", 4, 5}, token{"\u5c31", 5, 6}, token{"\u53ef\u4ee5", 6, 8}, token{"\u89e3\u51b3", 8, 10}, token{"\u4e00\u4e9b", 10, 12}, token{"\u95ee\u9898", 12, 14}},
[]token{token{"\u5176\u5b9e", 0, 2}, token{"\u4f7f\u7528", 2, 4}, token{"\u4e86", 4, 5}, token{"\u5b83", 5, 6}, token{"\u5c31", 6, 7}, token{"\u53ef\u4ee5", 7, 9}, token{"\u89e3\u51b3", 9, 11}, token{"\u4e00\u4e9b", 11, 13}, token{"\u95ee\u9898", 13, 15}},
[]token{token{"\u597d\u4eba", 0, 2}, token{"\u4f7f\u7528", 2, 4}, token{"\u4e86", 4, 5}, token{"\u5b83", 5, 6}, token{"\u5c31", 6, 7}, token{"\u53ef\u4ee5", 7, 9}, token{"\u89e3\u51b3", 9, 11}, token{"\u4e00\u4e9b", 11, 13}, token{"\u95ee\u9898", 13, 15}},
[]token{token{"\u662f\u56e0\u4e3a", 0, 3}, token{"\u548c", 3, 4}, token{"\u56fd\u5bb6", 4, 6}},
[]token{token{"\u8001\u5e74", 0, 2}, token{"\u641c\u7d22", 2, 4}, token{"\u8fd8", 4, 5}, token{"\u652f\u6301", 5, 7}},
[]token{token{"\u5e72\u8106", 0, 2}, token{"\u5c31", 2, 3}, token{"\u628a", 3, 4}, token{"\u90a3", 4, 5}, token{"\u90e8", 5, 6}, token{"\u8499", 6, 7}, token{"\u4eba", 7, 8}, token{"\u7684", 8, 9}, token{"\u95f2", 9, 10}, token{"\u6cd5", 10, 11}, token{"\u7ed9", 11, 12}, token{"\u5e9f", 12, 13}, token{"\u4e86", 13, 14}, token{"\u62c9\u5012", 14, 16}, token{"\uff01", 16, 17}, token{"RT", 17, 19}, token{" ", 19, 20}, token{"@", 20, 21}, token{"laoshipukong", 21, 33}, token{" ", 33, 34}, token{":", 34, 35}, token{" ", 35, 36}, token{"27", 36, 38}, token{"\u65e5", 38, 39}, token{"\uff0c", 39, 40}, token{"\u5168\u56fd\u4eba\u5927\u5e38\u59d4\u4f1a", 40, 47}, token{"\u7b2c\u4e09\u6b21", 47, 50}, token{"\u5ba1\u8bae", 50, 52}, token{"\u4fb5\u6743", 52, 54}, token{"\u8d23\u4efb\u6cd5", 54, 57}, token{"\u8349\u6848", 57, 59}, token{"\uff0c", 59, 60}, token{"\u5220\u9664", 60, 62}, token{"\u4e86", 62, 63}, token{"\u6709\u5173", 63, 65}, token{"\u533b\u7597", 65, 67}, token{"\u635f\u5bb3", 67, 69}, token{"\u8d23\u4efb", 69, 71}, token{"\u201c", 71, 72}, token{"\u4e3e\u8bc1", 72, 74}, token{"\u5012\u7f6e", 74, 76}, token{"\u201d", 76, 77}, token{"\u7684", 77, 78}, token{"\u89c4\u5b9a", 78, 80}, token{"\u3002", 80, 81}, token{"\u5728", 81, 82}, token{"\u533b\u60a3", 82, 84}, token{"\u7ea0\u7eb7", 84, 86}, token{"\u4e2d", 86, 87}, token{"\u672c", 87, 88}, token{"\u5df2", 88, 89}, token{"\u5904\u4e8e", 89, 91}, token{"\u5f31\u52bf", 91, 93}, token{"\u5730\u4f4d", 93, 95}, token{"\u7684", 95, 96}, token{"\u6d88\u8d39\u8005", 96, 99}, token{"\u7531\u6b64", 99, 101}, token{"\u5c06", 101, 102}, token{"\u9677\u5165", 102, 104}, token{"\u4e07\u52ab\u4e0d\u590d", 104, 108}, token{"\u7684", 108, 109}, token{"\u5883\u5730", 109, 111}, token{"\u3002", 111, 112}, token{" ", 112, 113}},
[]token{token{"\u5927", 0, 1}},
[]token{},
[]token{token{"\u4ed6", 0, 1}, token{"\u8bf4", 1, 2}, token{"\u7684", 2, 3}, token{"\u786e\u5b9e", 3, 5}, token{"\u5728", 5, 6}, token{"\u7406", 6, 7}},
[]token{token{"\u957f\u6625", 0, 2}, token{"\u5e02\u957f", 2, 4}, token{"\u6625\u8282", 4, 6}, token{"\u8bb2\u8bdd", 6, 8}},
[]token{token{"\u7ed3\u5a5a", 0, 2}, token{"\u7684", 2, 3}, token{"\u548c", 3, 4}, token{"\u5c1a\u672a", 4, 6}, token{"\u7ed3\u5a5a", 6, 8}, token{"\u7684", 8, 9}},
[]token{token{"\u7ed3\u5408", 0, 2}, token{"\u6210", 2, 3}, token{"\u5206\u5b50", 3, 5}, token{"\u65f6", 5, 6}},
[]token{token{"\u65c5\u6e38", 0, 2}, token{"\u548c", 2, 3}, token{"\u670d\u52a1", 3, 5}, token{"\u662f", 5, 6}, token{"\u6700\u597d", 6, 8}, token{"\u7684", 8, 9}},
[]token{token{"\u8fd9\u4ef6", 0, 2}, token{"\u4e8b\u60c5", 2, 4}, token{"\u7684\u786e", 4, 6}, token{"\u662f", 6, 7}, token{"\u6211", 7, 8}, token{"\u7684", 8, 9}, token{"\u9519", 9, 10}},
[]token{token{"\u4f9b", 0, 1}, token{"\u5927\u5bb6", 1, 3}, token{"\u53c2\u8003", 3, 5}, token{"\u6307\u6b63", 5, 7}},
[]token{token{"\u54c8\u5c14\u6ee8", 0, 3}, token{"\u653f\u5e9c", 3, 5}, token{"\u516c\u5e03", 5, 7}, token{"\u584c", 7, 8}, token{"\u6865", 8, 9}, token{"\u539f\u56e0", 9, 11}},
[]token{token{"\u6211", 0, 1}, token{"\u5728", 1, 2}, token{"\u673a\u573a", 2, 4}, token{"\u5165\u53e3\u5904", 4, 7}},
[]token{token{"\u90a2", 0, 1}, token{"\u6c38", 1, 2}, token{"\u81e3", 2, 3}, token{"\u6444\u5f71", 3, 5}, token{"\u62a5\u9053", 5, 7}},
[]token{token{"BP", 0, 2}, token{"\u795e\u7ecf\u7f51\u7edc", 2, 6}, token{"\u5982\u4f55", 6, 8}, token{"\u8bad\u7ec3", 8, 10}, token{"\u624d\u80fd", 10, 12}, token{"\u5728", 12, 13}, token{"\u5206\u7c7b", 13, 15}, token{"\u65f6", 15, 16}, token{"\u589e\u52a0", 16, 18}, token{"\u533a\u5206\u5ea6", 18, 21}, token{"\uff1f", 21, 22}},
[]token{token{"\u5357\u4eac\u5e02", 0, 3}, token{"\u957f\u6c5f\u5927\u6865", 3, 7}},
[]token{token{"\u5e94", 0, 1}, token{"\u4e00\u4e9b", 1, 3}, token{"\u4f7f\u7528\u8005", 3, 6}, token{"\u7684", 6, 7}, token{"\u5efa\u8bae", 7, 9}, token{"\uff0c", 9, 10}, token{"\u4e5f", 10, 11}, token{"\u4e3a\u4e86", 11, 13}, token{"\u4fbf\u4e8e", 13, 15}, token{"\u5229\u7528", 15, 17}, token{"NiuTrans", 17, 25}, token{"\u7528\u4e8e", 25, 27}, token{"SMT", 27, 30}, token{"\u7814\u7a76", 30, 32}},
[]token{token{"\u957f\u6625\u5e02", 0, 3}, token{"\u957f\u6625", 3, 5}, token{"\u836f\u5e97", 5, 7}},
[]token{token{"\u9093\u9896\u8d85", 0, 3}, token{"\u751f\u524d", 3, 5}, token{"\u6700", 5, 6}, token{"\u559c\u6b22", 6, 8}, token{"\u7684", 8, 9}, token{"\u8863\u670d", 9, 11}},
[]token{token{"\u80e1\u9526\u6d9b", 0, 3}, token{"\u662f", 3, 4}, token{"\u70ed\u7231", 4, 6}, token{"\u4e16\u754c", 6, 8}, token{"\u548c\u5e73", 8, 10}, token{"\u7684", 10, 11}, token{"\u653f\u6cbb\u5c40", 11, 14}, token{"\u5e38\u59d4", 14, 16}},
[]token{token{"\u7a0b\u5e8f\u5458", 0, 3}, token{"\u795d", 3, 4}, token{"\u6d77\u6797", 4, 6}, token{"\u548c", 6, 7}, token{"\u6731", 7, 8}, token{"\u4f1a", 8, 9}, token{"\u9707", 9, 10}, token{"\u662f", 10, 11}, token{"\u5728", 11, 12}, token{"\u5b59", 12, 13}, token{"\u5065", 13, 14}, token{"\u7684", 14, 15}, token{"\u5de6\u9762", 15, 17}, token{"\u548c", 17, 18}, token{"\u53f3\u9762", 18, 20}, token{",", 20, 21}, token{" ", 21, 22}, token{"\u8303", 22, 23}, token{"\u51ef", 23, 24}, token{"\u5728", 24, 25}, token{"\u6700", 25, 26}, token{"\u53f3\u9762", 26, 28}, token{".", 28, 29}, token{"\u518d", 29, 30}, token{"\u5f80", 30, 31}, token{"\u5de6", 31, 32}, token{"\u662f", 32, 33}, token{"\u674e", 33, 34}, token{"\u677e", 34, 35}, token{"\u6d2a", 35, 36}},
[]token{token{"\u4e00\u6b21\u6027", 0, 3}, token{"\u4ea4", 3, 4}, token{"\u591a\u5c11", 4, 6}, token{"\u94b1", 6, 7}},
[]token{token{"\u4e24\u5757", 0, 2}, token{"\u4e94", 2, 3}, token{"\u4e00\u5957", 3, 5}, token{"\uff0c", 5, 6}, token{"\u4e09\u5757", 6, 8}, token{"\u516b", 8, 9}, token{"\u4e00\u65a4", 9, 11}, token{"\uff0c", 11, 12}, token{"\u56db\u5757", 12, 14}, token{"\u4e03", 14, 15}, token{"\u4e00\u672c", 15, 17}, token{"\uff0c", 17, 18}, token{"\u4e94\u5757", 18, 20}, token{"\u516d", 20, 21}, token{"\u4e00\u6761", 21, 23}},
[]token{token{"\u5c0f", 0, 1}, token{"\u548c\u5c1a", 1, 3}, token{"\u7559", 3, 4}, token{"\u4e86", 4, 5}, token{"\u4e00\u4e2a", 5, 7}, token{"\u50cf", 7, 8}, token{"\u5927", 8, 9}, token{"\u548c\u5c1a", 9, 11}, token{"\u4e00\u6837", 11, 13}, token{"\u7684", 13, 14}, token{"\u548c\u5c1a\u5934", 14, 17}},
[]token{token{"\u6211", 0, 1}, token{"\u662f", 1, 2}, token{"\u4e2d\u534e\u4eba\u6c11\u5171\u548c\u56fd", 2, 9}, token{"\u516c\u6c11", 9, 11}, token{";", 11, 12}, token{"\u6211", 12, 13}, token{"\u7238\u7238", 13, 15}, token{"\u662f", 15, 16}, token{"\u5171\u548c\u515a", 16, 19}, token{"\u515a\u5458", 19, 21}, token{";", 21, 22}, token{" ", 22, 23}, token{"\u5730\u94c1", 23, 25}, token{"\u548c\u5e73\u95e8", 25, 28}, token{"\u7ad9", 28, 29}},
[]token{token{"\u5f20\u6653\u6885", 0, 3}, token{"\u53bb", 3, 4}, token{"\u4eba\u6c11", 4, 6}, token{"\u533b\u9662", 6, 8}, token{"\u505a", 8, 9}, token{"\u4e86", 9, 10}, token{"\u4e2a", 10, 11}, token{"B\u8d85", 11, 13}, token{"\u7136\u540e", 13, 15}, token{"\u53bb", 15, 16}, token{"\u4e70", 16, 17}, token{"\u4e86", 17, 18}, token{"\u4ef6", 18, 19}, token{"T\u6064", 19, 21}},
[]token{token{"AT&T", 0, 4}, token{"\u662f", 4, 5}, token{"\u4e00\u4ef6", 5, 7}, token{"\u4e0d\u9519", 7, 9}, token{"\u7684", 9, 10}, token{"\u516c\u53f8", 10, 12}, token{"\uff0c", 12, 13}, token{"\u7ed9", 13, 14}, token{"\u4f60", 14, 15}, token{"\u53d1", 15, 16}, token{"offer", 16, 21}, token{"\u4e86", 21, 22}, token{"\u5417", 22, 23}, token{"\uff1f", 23, 24}},
[]token{token{"C++", 0, 3}, token{"\u548c", 3, 4}, token{"c#", 4, 6}, token{"\u662f", 6, 7}, token{"\u4ec0\u4e48", 7, 9}, token{"\u5173\u7cfb", 9, 11}, token{"\uff1f", 11, 12}, token{"11", 12, 14}, token{"+", 14, 15}, token{"122", 15, 18}, token{"=", 18, 19}, token{"133", 19, 22}, token{"\uff0c", 22, 23}, token{"\u662f", 23, 24}, token{"\u5417", 24, 25}, token{"\uff1f", 25, 26}, token{"PI", 26, 28}, token{"=", 28, 29}, token{"3", 29, 30}, token{".", 30, 31}, token{"14159", 31, 36}},
[]token{token{"\u4f60", 0, 1}, token{"\u8ba4\u8bc6", 1, 3}, token{"\u90a3\u4e2a", 3, 5}, token{"\u548c", 5, 6}, token{"\u4e3b\u5e2d", 6, 8}, token{"\u63e1\u624b", 8, 10}, token{"\u7684", 10, 11}, token{"\u7684\u54e5", 11, 13}, token{"\u5417", 13, 14}, token{"\uff1f", 14, 15}, token{"\u4ed6", 15, 16}, token{"\u5f00", 16, 17}, token{"\u4e00\u8f86", 17, 19}, token{"\u9ed1\u8272", 19, 21}, token{"\u7684\u58eb", 21, 23}, token{"\u3002", 23, 24}},
[]token{token{"\u67aa\u6746\u5b50", 0, 3}, token{"\u4e2d", 3, 4}, token{"\u51fa", 4, 5}, token{"\u653f\u6743", 5, 7}},
[]token{token{"\u5f20", 0, 1}, token{"\u4e09", 1, 2}, token{"\u98ce", 2, 3}, token{"\u540c\u5b66", 3, 5}, token{"\u8d70\u4e0a", 5, 7}, token{"\u4e86", 7, 8}, token{"\u4e0d\u5f52\u8def", 8, 11}},
[]token{token{"\u963fQ", 0, 2}, token{"\u8170\u95f4", 2, 4}, token{"\u6302", 4, 5}, token{"\u7740", 5, 6}, token{"BB\u673a", 6, 9}, token{"\u624b\u91cc", 9, 11}, token{"\u62ff", 11, 12}, token{"\u7740", 12, 13}, token{"\u5927\u54e5\u5927", 13, 16}, token{"\uff0c", 16, 17}, token{"\u8bf4", 17, 18}, token{"\uff1a", 18, 19}, token{"\u6211", 19, 20}, token{"\u4e00\u822c", 20, 22}, token{"\u5403\u996d", 22, 24}, token{"\u4e0d", 24, 25}, token{"AA\u5236", 25, 28}, token{"\u7684", 28, 29}, token{"\u3002", 29, 30}},
[]token{token{"\u5728", 0, 1}, token{"1\u53f7\u5e97", 1, 4}, token{"\u80fd", 4, 5}, token{"\u4e70", 5, 6}, token{"\u5230", 6, 7}, token{"\u5c0fS", 7, 9}, token{"\u548c", 9, 10}, token{"\u5927S", 10, 12}, token{"\u516b\u5366", 12, 14}, token{"\u7684", 14, 15}, token{"\u4e66", 15, 16}, token{"\u3002", 16, 17}},
[]token{token{"\u8fd9", 0, 1}, token{"\u662f", 1, 2}, token{"\u4e00\u4e2a", 2, 4}, token{"\u4f38\u624b", 4, 6}, token{"\u4e0d\u89c1", 6, 8}, token{"\u4e94\u6307", 8, 10}, token{"\u4f38\u624b\u4e0d\u89c1\u4e94\u6307", 4, 10}, token{"\u7684", 10, 11}, token{"\u9ed1\u591c", 11, 13}, token{"\u3002", 13, 14}, token{"\u6211", 14, 15}, token{"\u53eb", 15, 16}, token{"\u609f\u7a7a", 17, 19}, token{"\u5b59\u609f\u7a7a", 16, 19}, token{"\uff0c", 19, 20}, token{"\u6211", 20, 21}, token{"\u7231", 21, 22}, token{"\u5317\u4eac", 22, 24}, token{"\uff0c", 24, 25}, token{"\u6211", 25, 26}, token{"\u7231", 26, 27}, token{"Python", 27, 33}, token{"\u548c", 33, 34}, token{"C++", 34, 37}, token{"\u3002", 37, 38}},
[]token{token{"\u6211", 0, 1}, token{"\u4e0d", 1, 2}, token{"\u559c\u6b22", 2, 4}, token{"\u65e5\u672c", 4, 6}, token{"\u548c\u670d", 6, 8}, token{"\u3002", 8, 9}},
[]token{token{"\u96f7\u7334", 0, 2}, token{"\u56de\u5f52", 2, 4}, token{"\u4eba\u95f4", 4, 6}, token{"\u3002", 6, 7}},
[]token{token{"\u5de5\u4fe1\u5904", 0, 3}, token{"\u5e72\u4e8b", 4, 6}, token{"\u5973\u5e72\u4e8b", 3, 6}, token{"\u6bcf\u6708", 6, 8}, token{"\u7ecf\u8fc7", 8, 10}, token{"\u4e0b\u5c5e", 10, 12}, token{"\u79d1\u5ba4", 12, 14}, token{"\u90fd", 14, 15}, token{"\u8981", 15, 16}, token{"\u4eb2\u53e3", 16, 18}, token{"\u4ea4\u4ee3", 18, 20}, token{"24", 20, 22}, token{"\u53e3", 22, 23}, token{"\u4ea4\u6362", 23, 25}, token{"\u6362\u673a", 24, 26}, token{"\u4ea4\u6362\u673a", 23, 26}, token{"\u7b49", 26, 27}, token{"\u6280\u672f", 27, 29}, token{"\u6280\u672f\u6027", 27, 30}, token{"\u5668\u4ef6", 30, 32}, token{"\u7684", 32, 33}, token{"\u5b89\u88c5", 33, 35}, token{"\u5de5\u4f5c", 35, 37}},
[]token{token{"\u6211", 0, 1}, token{"\u9700\u8981", 1, 3}, token{"\u5ec9\u79df", 3, 5}, token{"\u79df\u623f", 4, 6}, token{"\u5ec9\u79df\u623f", 3, 6}},
[]token{token{"\u6c38\u548c", 0, 2}, token{"\u670d\u88c5", 2, 4}, token{"\u9970\u54c1", 4, 6}, token{"\u6709\u9650", 6, 8}, token{"\u516c\u53f8", 8, 10}, token{"\u6709\u9650\u516c\u53f8", 6, 10}},
[]token{token{"\u6211", 0, 1}, token{"\u7231", 1, 2}, token{"\u5317\u4eac", 2, 4}, token{"\u5929\u5b89", 4, 6}, token{"\u5929\u5b89\u95e8", 4, 7}},
[]token{token{"abc", 0, 3}},
[]token{token{"\u9690", 0, 1}, token{"\u53ef\u592b", 3, 5}, token{"\u9a6c\u5c14\u53ef", 1, 4}, token{"\u9a6c\u5c14\u53ef\u592b", 1, 5}},
[]token{token{"\u96f7\u7334", 0, 2}, token{"\u662f", 2, 3}, token{"\u4e2a", 3, 4}, token{"\u597d", 4, 5}, token{"\u7f51\u7ad9", 5, 7}},
[]token{token{"\u201c", 0, 1}, token{"Microsoft", 1, 10}, token{"\u201d", 10, 11}, token{"\u4e00", 11, 12}, token{"\u8bcd", 12, 13}, token{"\u7531", 13, 14}, token{"\u201c", 14, 15}, token{"MICROcomputer", 15, 28}, token{"\uff08", 28, 29}, token{"\u5fae\u578b", 29, 31}, token{"\u8ba1\u7b97", 31, 33}, token{"\u7b97\u673a", 32, 34}, token{"\u8ba1\u7b97\u673a", 31, 34}, token{"\uff09", 34, 35}, token{"\u201d", 35, 36}, token{"\u548c", 36, 37}, token{"\u201c", 37, 38}, token{"SOFTware", 38, 46}, token{"\uff08", 46, 47}, token{"\u8f6f\u4ef6", 47, 49}, token{"\uff09", 49, 50}, token{"\u201d", 50, 51}, token{"\u4e24", 51, 52}, token{"\u90e8\u5206", 52, 54}, token{"\u7ec4\u6210", 54, 56}},
[]token{token{"\u8349\u6ce5\u9a6c", 0, 3}, token{"\u548c", 3, 4}, token{"\u6b3a", 4, 5}, token{"\u5b9e", 5, 6}, token{"\u9a6c", 6, 7}, token{"\u662f", 7, 8}, token{"\u4eca\u5e74", 8, 10}, token{"\u7684", 10, 11}, token{"\u6d41\u884c", 11, 13}, token{"\u8bcd\u6c47", 13, 15}},
[]token{token{"\u4f0a", 0, 1}, token{"\u85e4", 1, 2}, token{"\u6d0b\u534e\u5802", 2, 5}, token{"\u603b\u5e9c", 5, 7}, token{"\u5e97", 7, 8}},
[]token{token{"\u4e2d\u56fd", 0, 2}, token{"\u79d1\u5b66", 2, 4}, token{"\u5b66\u9662", 3, 5}, token{"\u8ba1\u7b97", 5, 7}, token{"\u6280\u672f", 7, 9}, token{"\u7814\u7a76", 9, 11}, token{"\u79d1\u5b66\u9662", 2, 5}, token{"\u7814\u7a76\u6240", 9, 12}, token{"\u4e2d\u56fd\u79d1\u5b66\u9662\u8ba1\u7b97\u6280\u672f\u7814\u7a76\u6240", 0, 12}},
[]token{token{"\u7f57\u5bc6\u6b27", 0, 3}, token{"\u4e0e", 3, 4}, token{"\u6731\u4e3d\u53f6", 4, 7}},
[]token{token{"\u6211", 0, 1}, token{"\u8d2d\u4e70", 1, 3}, token{"\u4e86", 3, 4}, token{"\u9053\u5177", 4, 6}, token{"\u548c", 6, 7}, token{"\u670d\u88c5", 7, 9}},
[]token{token{"PS", 0, 2}, token{":", 2, 3}, token{" ", 3, 4}, token{"\u6211", 4, 5}, token{"\u89c9\u5f97", 5, 7}, token{"\u5f00\u6e90", 7, 9}, token{"\u6709", 9, 10}, token{"\u4e00\u4e2a", 10, 12}, token{"\u597d\u5904", 12, 14}, token{"\uff0c", 14, 15}, token{"\u5c31\u662f", 15, 17}, token{"\u80fd\u591f", 17, 19}, token{"\u6566\u4fc3", 19, 21}, token{"\u81ea\u5df1", 21, 23}, token{"\u4e0d\u65ad", 23, 25}, token{"\u6539\u8fdb", 25, 27}, token{"\u4e0d\u65ad\u6539\u8fdb", 23, 27}, token{"\uff0c", 27, 28}, token{"\u907f\u514d", 28, 30}, token{"\u655e", 30, 31}, token{"\u5e1a", 31, 32}, token{"\u81ea\u73cd", 32, 34}},
[]token{token{"\u6e56\u5317", 0, 2}, token{"\u6e56\u5317\u7701", 0, 3}, token{"\u77f3\u9996", 3, 5}, token{"\u77f3\u9996\u5e02", 3, 6}},
[]token{token{"\u6e56\u5317", 0, 2}, token{"\u6e56\u5317\u7701", 0, 3}, token{"\u5341\u5830", 3, 5}, token{"\u5341\u5830\u5e02", 3, 6}},
[]token{token{"\u7ecf\u7406", 1, 3}, token{"\u603b\u7ecf\u7406", 0, 3}, token{"\u5b8c\u6210", 3, 5}, token{"\u4e86", 5, 6}, token{"\u8fd9\u4ef6", 6, 8}, token{"\u4e8b\u60c5", 8, 10}},
[]token{token{"\u7535\u8111", 0, 2}, token{"\u4fee\u597d", 2, 4}, token{"\u4e86", 4, 5}},
[]token{token{"\u505a\u597d", 0, 2}, token{"\u4e86", 2, 3}, token{"\u8fd9\u4ef6", 3, 5}, token{"\u4e8b\u60c5", 5, 7}, token{"\u5c31", 7, 8}, token{"\u4e00\u4e86\u767e\u4e86", 8, 12}, token{"\u4e86", 12, 13}},
[]token{token{"\u4eba\u4eec", 0, 2}, token{"\u5ba1\u7f8e", 2, 4}, token{"\u7684", 4, 5}, token{"\u89c2\u70b9", 5, 7}, token{"\u662f", 7, 8}, token{"\u4e0d\u540c", 8, 10}, token{"\u7684", 10, 11}},
[]token{token{"\u6211\u4eec", 0, 2}, token{"\u4e70", 2, 3}, token{"\u4e86", 3, 4}, token{"\u4e00\u4e2a", 4, 6}, token{"\u7f8e\u7684", 6, 8}, token{"\u7a7a\u8c03", 8, 10}},
[]token{token{"\u7ebf\u7a0b", 0, 2}, token{"\u521d\u59cb", 2, 4}, token{"\u521d\u59cb\u5316", 2, 5}, token{"\u65f6", 5, 6}, token{"\u6211\u4eec", 6, 8}, token{"\u8981", 8, 9}, token{"\u6ce8\u610f", 9, 11}},
[]token{token{"\u4e00\u4e2a", 0, 2}, token{"\u5206\u5b50", 2, 4}, token{"\u662f", 4, 5}, token{"\u7531", 5, 6}, token{"\u597d\u591a", 6, 8}, token{"\u539f\u5b50", 8, 10}, token{"\u7ec4\u7ec7", 10, 12}, token{"\u6210", 12, 13}, token{"\u7684", 13, 14}},
[]token{token{"\u795d", 0, 1}, token{"\u4f60", 1, 2}, token{"\u9a6c\u5230\u529f\u6210", 2, 6}},
[]token{token{"\u4ed6", 0, 1}, token{"\u6389", 1, 2}, token{"\u8fdb", 2, 3}, token{"\u4e86", 3, 4}, token{"\u65e0\u5e95", 4, 6}, token{"\u65e0\u5e95\u6d1e", 4, 7}, token{"\u91cc", 7, 8}},
[]token{token{"\u4e2d\u56fd", 0, 2}, token{"\u7684", 2, 3}, token{"\u9996\u90fd", 3, 5}, token{"\u662f", 5, 6}, token{"\u5317\u4eac", 6, 8}},
[]token{token{"\u5b59", 0, 1}, token{"\u541b", 1, 2}, token{"\u610f", 2, 3}},
[]token{token{"\u5916\u4ea4", 0, 2}, token{"\u5916\u4ea4\u90e8", 0, 3}, token{"\u53d1\u8a00", 3, 5}, token{"\u53d1\u8a00\u4eba", 3, 6}, token{"\u9a6c\u671d\u65ed", 6, 9}},
[]token{token{"\u9886\u5bfc", 0, 2}, token{"\u9886\u5bfc\u4eba", 0, 3}, token{"\u4f1a\u8bae", 3, 5}, token{"\u548c", 5, 6}, token{"\u7b2c\u56db", 6, 8}, token{"\u56db\u5c4a", 7, 9}, token{"\u7b2c\u56db\u5c4a", 6, 9}, token{"\u4e1c\u4e9a", 9, 11}, token{"\u5cf0\u4f1a", 11, 13}},
[]token{token{"\u5728", 0, 1}, token{"\u8fc7\u53bb", 1, 3}, token{"\u7684", 3, 4}, token{"\u8fd9", 4, 5}, token{"\u4e94\u5e74", 5, 7}},
[]token{token{"\u8fd8", 0, 1}, token{"\u9700\u8981", 1, 3}, token{"\u5f88", 3, 4}, token{"\u957f", 4, 5}, token{"\u7684", 5, 6}, token{"\u8def", 6, 7}, token{"\u8981", 7, 8}, token{"\u8d70", 8, 9}},
[]token{token{"60", 0, 2}, token{"\u5468\u5e74", 2, 4}, token{"\u9996\u90fd", 4, 6}, token{"\u9605\u5175", 6, 8}},
[]token{token{"\u4f60\u597d", 0, 2}, token{"\u4eba\u4eec", 2, 4}, token{"\u5ba1\u7f8e", 4, 6}, token{"\u7684", 6, 7}, token{"\u89c2\u70b9", 7, 9}, token{"\u662f", 9, 10}, token{"\u4e0d\u540c", 10, 12}, token{"\u7684", 12, 13}},
[]token{token{"\u4e70", 0, 1}, token{"\u6c34\u679c", 1, 3}, token{"\u7136\u540e", 3, 5}, token{"\u6765", 5, 6}, token{"\u4e16\u535a", 6, 8}, token{"\u535a\u56ed", 7, 9}, token{"\u4e16\u535a\u56ed", 6, 9}},
[]token{token{"\u4e70", 0, 1}, token{"\u6c34\u679c", 1, 3}, token{"\u7136\u540e", 3, 5}, token{"\u53bb", 5, 6}, token{"\u4e16\u535a", 6, 8}, token{"\u535a\u56ed", 7, 9}, token{"\u4e16\u535a\u56ed", 6, 9}},
[]token{token{"\u4f46\u662f", 0, 2}, token{"\u540e\u6765", 2, 4}, token{"\u6211", 4, 5}, token{"\u624d", 5, 6}, token{"\u77e5\u9053", 6, 8}, token{"\u4f60", 8, 9}, token{"\u662f", 9, 10}, token{"\u5bf9", 10, 11}, token{"\u7684", 11, 12}},
[]token{token{"\u5b58\u5728", 0, 2}, token{"\u5373", 2, 3}, token{"\u5408\u7406", 3, 5}},
[]token{token{"\u7684", 0, 1}, token{"\u7684", 1, 2}, token{"\u7684", 2, 3}, token{"\u7684", 3, 4}, token{"\u7684", 4, 5}, token{"\u5728", 5, 6}, token{"\u7684", 6, 7}, token{"\u7684", 7, 8}, token{"\u7684", 8, 9}, token{"\u7684", 9, 10}, token{"\u5c31", 10, 11}, token{"\u4ee5", 11, 12}, token{"\u548c", 12, 13}, token{"\u548c", 13, 14}, token{"\u548c", 14, 15}},
[]token{token{"I", 0, 1}, token{" ", 1, 2}, token{"love", 2, 6}, token{"\u4f60", 6, 7}, token{"\uff0c", 7, 8}, token{"\u4e0d\u4ee5", 8, 10}, token{"\u4ee5\u4e3a", 9, 11}, token{"\u4e0d\u4ee5\u4e3a\u803b", 8, 12}, token{"\uff0c", 12, 13}, token{"\u53cd", 13, 14}, token{"\u4ee5\u4e3a", 14, 16}, token{"rong", 16, 20}},
[]token{token{"\u56e0", 0, 1}},
[]token{},
[]token{token{"hello", 0, 5}, token{"\u4f60\u597d", 5, 7}, token{"\u4eba\u4eec", 7, 9}, token{"\u5ba1\u7f8e", 9, 11}, token{"\u7684", 11, 12}, token{"\u89c2\u70b9", 12, 14}, token{"\u662f", 14, 15}, token{"\u4e0d\u540c", 15, 17}, token{"\u7684", 17, 18}},
[]token{token{"\u5f88", 0, 1}, token{"\u597d", 1, 2}, token{"\u4f46", 2, 3}, token{"\u4e3b\u8981", 3, 5}, token{"\u662f", 5, 6}, token{"\u57fa\u4e8e", 6, 8}, token{"\u7f51\u9875", 8, 10}, token{"\u5f62\u5f0f", 10, 12}},
[]token{token{"hello", 0, 5}, token{"\u4f60\u597d", 5, 7}, token{"\u4eba\u4eec", 7, 9}, token{"\u5ba1\u7f8e", 9, 11}, token{"\u7684", 11, 12}, token{"\u89c2\u70b9", 12, 14}, token{"\u662f", 14, 15}, token{"\u4e0d\u540c", 15, 17}, token{"\u7684", 17, 18}},
[]token{token{"\u4ec0\u4e48", 1, 3}, token{"\u4e3a\u4ec0\u4e48", 0, 3}, token{"\u6211", 3, 4}, token{"\u4e0d\u80fd", 4, 6}, token{"\u62e5\u6709", 6, 8}, token{"\u60f3\u8981", 8, 10}, token{"\u7684", 10, 11}, token{"\u751f\u6d3b", 11, 13}},
[]token{token{"\u540e\u6765", 0, 2}, token{"\u6211", 2, 3}, token{"\u624d", 3, 4}},
[]token{token{"\u6b64\u6b21", 0, 2}, token{"\u6765", 2, 3}, token{"\u4e2d\u56fd", 3, 5}, token{"\u662f", 5, 6}, token{"\u4e3a\u4e86", 6, 8}},
[]token{token{"\u4f7f\u7528", 0, 2}, token{"\u4e86", 2, 3}, token{"\u5b83", 3, 4}, token{"\u5c31", 4, 5}, token{"\u53ef\u4ee5", 5, 7}, token{"\u89e3\u51b3", 7, 9}, token{"\u4e00\u4e9b", 9, 11}, token{"\u95ee\u9898", 11, 13}},
[]token{token{",", 0, 1}, token{"\u4f7f\u7528", 1, 3}, token{"\u4e86", 3, 4}, token{"\u5b83", 4, 5}, token{"\u5c31", 5, 6}, token{"\u53ef\u4ee5", 6, 8}, token{"\u89e3\u51b3", 8, 10}, token{"\u4e00\u4e9b", 10, 12}, token{"\u95ee\u9898", 12, 14}},
[]token{token{"\u5176\u5b9e", 0, 2}, token{"\u4f7f\u7528", 2, 4}, token{"\u4e86", 4, 5}, token{"\u5b83", 5, 6}, token{"\u5c31", 6, 7}, token{"\u53ef\u4ee5", 7, 9}, token{"\u89e3\u51b3", 9, 11}, token{"\u4e00\u4e9b", 11, 13}, token{"\u95ee\u9898", 13, 15}},
[]token{token{"\u597d\u4eba", 0, 2}, token{"\u4f7f\u7528", 2, 4}, token{"\u4e86", 4, 5}, token{"\u5b83", 5, 6}, token{"\u5c31", 6, 7}, token{"\u53ef\u4ee5", 7, 9}, token{"\u89e3\u51b3", 9, 11}, token{"\u4e00\u4e9b", 11, 13}, token{"\u95ee\u9898", 13, 15}},
[]token{token{"\u56e0\u4e3a", 1, 3}, token{"\u662f\u56e0\u4e3a", 0, 3}, token{"\u548c", 3, 4}, token{"\u56fd\u5bb6", 4, 6}},
[]token{token{"\u8001\u5e74", 0, 2}, token{"\u641c\u7d22", 2, 4}, token{"\u8fd8", 4, 5}, token{"\u652f\u6301", 5, 7}},
[]token{token{"\u5e72\u8106", 0, 2}, token{"\u5c31", 2, 3}, token{"\u628a", 3, 4}, token{"\u90a3", 4, 5}, token{"\u90e8", 5, 6}, token{"\u8499", 6, 7}, token{"\u4eba", 7, 8}, token{"\u7684", 8, 9}, token{"\u95f2", 9, 10}, token{"\u6cd5", 10, 11}, token{"\u7ed9", 11, 12}, token{"\u5e9f", 12, 13}, token{"\u4e86", 13, 14}, token{"\u62c9\u5012", 14, 16}, token{"\uff01", 16, 17}, token{"RT", 17, 19}, token{" ", 19, 20}, token{"@", 20, 21}, token{"laoshipukong", 21, 33}, token{" ", 33, 34}, token{":", 34, 35}, token{" ", 35, 36}, token{"27", 36, 38}, token{"\u65e5", 38, 39}, token{"\uff0c", 39, 40}, token{"\u5168\u56fd", 40, 42}, token{"\u56fd\u4eba", 41, 43}, token{"\u4eba\u5927", 42, 44}, token{"\u5e38\u59d4", 44, 46}, token{"\u59d4\u4f1a", 45, 47}, token{"\u5e38\u59d4\u4f1a", 44, 47}, token{"\u5168\u56fd\u4eba\u5927\u5e38\u59d4\u4f1a", 40, 47}, token{"\u7b2c\u4e09", 47, 49}, token{"\u4e09\u6b21", 48, 50}, token{"\u7b2c\u4e09\u6b21", 47, 50}, token{"\u5ba1\u8bae", 50, 52}, token{"\u4fb5\u6743", 52, 54}, token{"\u8d23\u4efb", 54, 56}, token{"\u8d23\u4efb\u6cd5", 54, 57}, token{"\u8349\u6848", 57, 59}, token{"\uff0c", 59, 60}, token{"\u5220\u9664", 60, 62}, token{"\u4e86", 62, 63}, token{"\u6709\u5173", 63, 65}, token{"\u533b\u7597", 65, 67}, token{"\u635f\u5bb3", 67, 69}, token{"\u8d23\u4efb", 69, 71}, token{"\u201c", 71, 72}, token{"\u4e3e\u8bc1", 72, 74}, token{"\u5012\u7f6e", 74, 76}, token{"\u201d", 76, 77}, token{"\u7684", 77, 78}, token{"\u89c4\u5b9a", 78, 80}, token{"\u3002", 80, 81}, token{"\u5728", 81, 82}, token{"\u533b\u60a3", 82, 84}, token{"\u7ea0\u7eb7", 84, 86}, token{"\u4e2d", 86, 87}, token{"\u672c", 87, 88}, token{"\u5df2", 88, 89}, token{"\u5904\u4e8e", 89, 91}, token{"\u5f31\u52bf", 91, 93}, token{"\u5730\u4f4d", 93, 95}, token{"\u7684", 95, 96}, token{"\u6d88\u8d39", 96, 98}, token{"\u6d88\u8d39\u8005", 96, 99}, token{"\u7531\u6b64", 99, 101}, token{"\u5c06", 101, 102}, token{"\u9677\u5165", 102, 104}, token{"\u4e0d\u590d", 106, 108}, token{"\u4e07\u52ab\u4e0d\u590d", 104, 108}, token{"\u7684", 108, 109}, token{"\u5883\u5730", 109, 111}, token{"\u3002", 111, 112}, token{" ", 112, 113}},
[]token{token{"\u5927", 0, 1}},
[]token{},
[]token{token{"\u4ed6", 0, 1}, token{"\u8bf4", 1, 2}, token{"\u7684", 2, 3}, token{"\u786e\u5b9e", 3, 5}, token{"\u5728", 5, 6}, token{"\u7406", 6, 7}},
[]token{token{"\u957f\u6625", 0, 2}, token{"\u5e02\u957f", 2, 4}, token{"\u6625\u8282", 4, 6}, token{"\u8bb2\u8bdd", 6, 8}},
[]token{token{"\u7ed3\u5a5a", 0, 2}, token{"\u7684", 2, 3}, token{"\u548c", 3, 4}, token{"\u5c1a\u672a", 4, 6}, token{"\u7ed3\u5a5a", 6, 8}, token{"\u7684", 8, 9}},
[]token{token{"\u7ed3\u5408", 0, 2}, token{"\u6210", 2, 3}, token{"\u5206\u5b50", 3, 5}, token{"\u65f6", 5, 6}},
[]token{token{"\u65c5\u6e38", 0, 2}, token{"\u548c", 2, 3}, token{"\u670d\u52a1", 3, 5}, token{"\u662f", 5, 6}, token{"\u6700\u597d", 6, 8}, token{"\u7684", 8, 9}},
[]token{token{"\u8fd9\u4ef6", 0, 2}, token{"\u4e8b\u60c5", 2, 4}, token{"\u7684\u786e", 4, 6}, token{"\u662f", 6, 7}, token{"\u6211", 7, 8}, token{"\u7684", 8, 9}, token{"\u9519", 9, 10}},
[]token{token{"\u4f9b", 0, 1}, token{"\u5927\u5bb6", 1, 3}, token{"\u53c2\u8003", 3, 5}, token{"\u6307\u6b63", 5, 7}},
[]token{token{"\u54c8\u5c14", 0, 2}, token{"\u54c8\u5c14\u6ee8", 0, 3}, token{"\u653f\u5e9c", 3, 5}, token{"\u516c\u5e03", 5, 7}, token{"\u584c", 7, 8}, token{"\u6865", 8, 9}, token{"\u539f\u56e0", 9, 11}},
[]token{token{"\u6211", 0, 1}, token{"\u5728", 1, 2}, token{"\u673a\u573a", 2, 4}, token{"\u5165\u53e3", 4, 6}, token{"\u5165\u53e3\u5904", 4, 7}},
[]token{token{"\u90a2", 0, 1}, token{"\u6c38", 1, 2}, token{"\u81e3", 2, 3}, token{"\u6444\u5f71", 3, 5}, token{"\u62a5\u9053", 5, 7}},
[]token{token{"BP", 0, 2}, token{"\u795e\u7ecf", 2, 4}, token{"\u7f51\u7edc", 4, 6}, token{"\u795e\u7ecf\u7f51", 2, 5}, token{"\u795e\u7ecf\u7f51\u7edc", 2, 6}, token{"\u5982\u4f55", 6, 8}, token{"\u8bad\u7ec3", 8, 10}, token{"\u624d\u80fd", 10, 12}, token{"\u5728", 12, 13}, token{"\u5206\u7c7b", 13, 15}, token{"\u65f6", 15, 16}, token{"\u589e\u52a0", 16, 18}, token{"\u533a\u5206", 18, 20}, token{"\u5206\u5ea6", 19, 21}, token{"\u533a\u5206\u5ea6", 18, 21}, token{"\uff1f", 21, 22}},
[]token{token{"\u5357\u4eac", 0, 2}, token{"\u4eac\u5e02", 1, 3}, token{"\u5357\u4eac\u5e02", 0, 3}, token{"\u957f\u6c5f", 3, 5}, token{"\u5927\u6865", 5, 7}, token{"\u957f\u6c5f\u5927\u6865", 3, 7}},
[]token{token{"\u5e94", 0, 1}, token{"\u4e00\u4e9b", 1, 3}, token{"\u4f7f\u7528", 3, 5}, token{"\u7528\u8005", 4, 6}, token{"\u4f7f\u7528\u8005", 3, 6}, token{"\u7684", 6, 7}, token{"\u5efa\u8bae", 7, 9}, token{"\uff0c", 9, 10}, token{"\u4e5f", 10, 11}, token{"\u4e3a\u4e86", 11, 13}, token{"\u4fbf\u4e8e", 13, 15}, token{"\u5229\u7528", 15, 17}, token{"NiuTrans", 17, 25}, token{"\u7528\u4e8e", 25, 27}, token{"SMT", 27, 30}, token{"\u7814\u7a76", 30, 32}},
[]token{token{"\u957f\u6625", 0, 2}, token{"\u957f\u6625\u5e02", 0, 3}, token{"\u957f\u6625", 3, 5}, token{"\u836f\u5e97", 5, 7}},
[]token{token{"\u9093\u9896\u8d85", 0, 3}, token{"\u751f\u524d", 3, 5}, token{"\u6700", 5, 6}, token{"\u559c\u6b22", 6, 8}, token{"\u7684", 8, 9}, token{"\u8863\u670d", 9, 11}},
[]token{token{"\u9526\u6d9b", 1, 3}, token{"\u80e1\u9526\u6d9b", 0, 3}, token{"\u662f", 3, 4}, token{"\u70ed\u7231", 4, 6}, token{"\u4e16\u754c", 6, 8}, token{"\u548c\u5e73", 8, 10}, token{"\u7684", 10, 11}, token{"\u653f\u6cbb", 11, 13}, token{"\u653f\u6cbb\u5c40", 11, 14}, token{"\u5e38\u59d4", 14, 16}},
[]token{token{"\u7a0b\u5e8f", 0, 2}, token{"\u7a0b\u5e8f\u5458", 0, 3}, token{"\u795d", 3, 4}, token{"\u6d77\u6797", 4, 6}, token{"\u548c", 6, 7}, token{"\u6731", 7, 8}, token{"\u4f1a", 8, 9}, token{"\u9707", 9, 10}, token{"\u662f", 10, 11}, token{"\u5728", 11, 12}, token{"\u5b59", 12, 13}, token{"\u5065", 13, 14}, token{"\u7684", 14, 15}, token{"\u5de6\u9762", 15, 17}, token{"\u548c", 17, 18}, token{"\u53f3\u9762", 18, 20}, token{",", 20, 21}, token{" ", 21, 22}, token{"\u8303", 22, 23}, token{"\u51ef", 23, 24}, token{"\u5728", 24, 25}, token{"\u6700", 25, 26}, token{"\u53f3\u9762", 26, 28}, token{".", 28, 29}, token{"\u518d", 29, 30}, token{"\u5f80", 30, 31}, token{"\u5de6", 31, 32}, token{"\u662f", 32, 33}, token{"\u674e", 33, 34}, token{"\u677e", 34, 35}, token{"\u6d2a", 35, 36}},
[]token{token{"\u4e00\u6b21", 0, 2}, token{"\u4e00\u6b21\u6027", 0, 3}, token{"\u4ea4", 3, 4}, token{"\u591a\u5c11", 4, 6}, token{"\u94b1", 6, 7}},
[]token{token{"\u4e24\u5757", 0, 2}, token{"\u4e94", 2, 3}, token{"\u4e00\u5957", 3, 5}, token{"\uff0c", 5, 6}, token{"\u4e09\u5757", 6, 8}, token{"\u516b", 8, 9}, token{"\u4e00\u65a4", 9, 11}, token{"\uff0c", 11, 12}, token{"\u56db\u5757", 12, 14}, token{"\u4e03", 14, 15}, token{"\u4e00\u672c", 15, 17}, token{"\uff0c", 17, 18}, token{"\u4e94\u5757", 18, 20}, token{"\u516d", 20, 21}, token{"\u4e00\u6761", 21, 23}},
[]token{token{"\u5c0f", 0, 1}, token{"\u548c\u5c1a", 1, 3}, token{"\u7559", 3, 4}, token{"\u4e86", 4, 5}, token{"\u4e00\u4e2a", 5, 7}, token{"\u50cf", 7, 8}, token{"\u5927", 8, 9}, token{"\u548c\u5c1a", 9, 11}, token{"\u4e00\u6837", 11, 13}, token{"\u7684", 13, 14}, token{"\u548c\u5c1a", 14, 16}, token{"\u548c\u5c1a\u5934", 14, 17}},
[]token{token{"\u6211", 0, 1}, token{"\u662f", 1, 2}, token{"\u4e2d\u534e", 2, 4}, token{"\u534e\u4eba", 3, 5}, token{"\u4eba\u6c11", 4, 6}, token{"\u5171\u548c", 6, 8}, token{"\u5171\u548c\u56fd", 6, 9}, token{"\u4e2d\u534e\u4eba\u6c11\u5171\u548c\u56fd", 2, 9}, token{"\u516c\u6c11", 9, 11}, token{";", 11, 12}, token{"\u6211", 12, 13}, token{"\u7238\u7238", 13, 15}, token{"\u662f", 15, 16}, token{"\u5171\u548c", 16, 18}, token{"\u5171\u548c\u515a", 16, 19}, token{"\u515a\u5458", 19, 21}, token{";", 21, 22}, token{" ", 22, 23}, token{"\u5730\u94c1", 23, 25}, token{"\u548c\u5e73", 25, 27}, token{"\u548c\u5e73\u95e8", 25, 28}, token{"\u7ad9", 28, 29}},
[]token{token{"\u5f20\u6653\u6885", 0, 3}, token{"\u53bb", 3, 4}, token{"\u4eba\u6c11", 4, 6}, token{"\u533b\u9662", 6, 8}, token{"\u505a", 8, 9}, token{"\u4e86", 9, 10}, token{"\u4e2a", 10, 11}, token{"B\u8d85", 11, 13}, token{"\u7136\u540e", 13, 15}, token{"\u53bb", 15, 16}, token{"\u4e70", 16, 17}, token{"\u4e86", 17, 18}, token{"\u4ef6", 18, 19}, token{"T\u6064", 19, 21}},
[]token{token{"AT&T", 0, 4}, token{"\u662f", 4, 5}, token{"\u4e00\u4ef6", 5, 7}, token{"\u4e0d\u9519", 7, 9}, token{"\u7684", 9, 10}, token{"\u516c\u53f8", 10, 12}, token{"\uff0c", 12, 13}, token{"\u7ed9", 13, 14}, token{"\u4f60", 14, 15}, token{"\u53d1", 15, 16}, token{"offer", 16, 21}, token{"\u4e86", 21, 22}, token{"\u5417", 22, 23}, token{"\uff1f", 23, 24}},
[]token{token{"C++", 0, 3}, token{"\u548c", 3, 4}, token{"c#", 4, 6}, token{"\u662f", 6, 7}, token{"\u4ec0\u4e48", 7, 9}, token{"\u5173\u7cfb", 9, 11}, token{"\uff1f", 11, 12}, token{"11", 12, 14}, token{"+", 14, 15}, token{"122", 15, 18}, token{"=", 18, 19}, token{"133", 19, 22}, token{"\uff0c", 22, 23}, token{"\u662f", 23, 24}, token{"\u5417", 24, 25}, token{"\uff1f", 25, 26}, token{"PI", 26, 28}, token{"=", 28, 29}, token{"3", 29, 30}, token{".", 30, 31}, token{"14159", 31, 36}},
[]token{token{"\u4f60", 0, 1}, token{"\u8ba4\u8bc6", 1, 3}, token{"\u90a3\u4e2a", 3, 5}, token{"\u548c", 5, 6}, token{"\u4e3b\u5e2d", 6, 8}, token{"\u63e1\u624b", 8, 10}, token{"\u7684", 10, 11}, token{"\u7684\u54e5", 11, 13}, token{"\u5417", 13, 14}, token{"\uff1f", 14, 15}, token{"\u4ed6", 15, 16}, token{"\u5f00", 16, 17}, token{"\u4e00\u8f86", 17, 19}, token{"\u9ed1\u8272", 19, 21}, token{"\u7684\u58eb", 21, 23}, token{"\u3002", 23, 24}},
[]token{token{"\u67aa\u6746", 0, 2}, token{"\u6746\u5b50", 1, 3}, token{"\u67aa\u6746\u5b50", 0, 3}, token{"\u4e2d", 3, 4}, token{"\u51fa", 4, 5}, token{"\u653f\u6743", 5, 7}},
[]token{token{"\u5f20", 0, 1}, token{"\u4e09", 1, 2}, token{"\u98ce", 2, 3}, token{"\u540c\u5b66", 3, 5}, token{"\u8d70\u4e0a", 5, 7}, token{"\u4e86", 7, 8}, token{"\u5f52\u8def", 9, 11}, token{"\u4e0d\u5f52\u8def", 8, 11}},
[]token{token{"\u963fQ", 0, 2}, token{"\u8170\u95f4", 2, 4}, token{"\u6302", 4, 5}, token{"\u7740", 5, 6}, token{"BB\u673a", 6, 9}, token{"\u624b\u91cc", 9, 11}, token{"\u62ff", 11, 12}, token{"\u7740", 12, 13}, token{"\u5927\u54e5", 13, 15}, token{"\u5927\u54e5\u5927", 13, 16}, token{"\uff0c", 16, 17}, token{"\u8bf4", 17, 18}, token{"\uff1a", 18, 19}, token{"\u6211", 19, 20}, token{"\u4e00\u822c", 20, 22}, token{"\u5403\u996d", 22, 24}, token{"\u4e0d", 24, 25}, token{"AA\u5236", 25, 28}, token{"\u7684", 28, 29}, token{"\u3002", 29, 30}},
[]token{token{"\u5728", 0, 1}, token{"1\u53f7\u5e97", 1, 4}, token{"\u80fd", 4, 5}, token{"\u4e70", 5, 6}, token{"\u5230", 6, 7}, token{"\u5c0fS", 7, 9}, token{"\u548c", 9, 10}, token{"\u5927S", 10, 12}, token{"\u516b\u5366", 12, 14}, token{"\u7684", 14, 15}, token{"\u4e66", 15, 16}, token{"\u3002", 16, 17}},
}
)
func TestTokenizeDefaultMode(t *testing.T) {
func TesttokenizeDefaultMode(t *testing.T) {
for index, sentence := range test_contents {
tokens := Tokenize(sentence, "default", true)
if len(tokens) != len(result[index]) {
@@ -375,7 +375,7 @@ func TestTokenizeDefaultMode(t *testing.T) {
}
}
func TestTokenizeNoHMM(t *testing.T) {
func TesttokenizeNoHMM(t *testing.T) {
for index, sentence := range test_contents {
tokens := Tokenize(sentence, "default", false)
if len(tokens) != len(noHmmResult[index]) {

155
trie.go Normal file
View File

@@ -0,0 +1,155 @@
package jiebago
import (
"bytes"
"crypto/md5"
"encoding/gob"
"fmt"
"log"
"os"
"path/filepath"
"strings"
)
// Trie store the total frequency and map of all words and their frequenciesb
var Trie *trie
type trie struct {
Total float64
Freq map[string]float64
}
func (t trie) MarshalBinary() ([]byte, error) {
var b bytes.Buffer
enc := gob.NewEncoder(&b)
err := enc.Encode(t.Total)
if err != nil {
return nil, err
}
err = enc.Encode(t.Freq)
if err != nil {
return nil, err
}
return b.Bytes(), nil
}
func (t *trie) UnmarshalBinary(data []byte) error {
b := bytes.NewBuffer(data)
dec := gob.NewDecoder(b)
err := dec.Decode(&t.Total)
if err != nil {
return err
}
err = dec.Decode(&t.Freq)
if err != nil {
return err
}
return nil
}
func (t *trie) load(dictFileName string) error {
dictFilePath, err := DictPath(dictFileName)
if err != nil {
return err
}
dictFileInfo, err := os.Stat(dictFilePath)
if err != nil {
return err
}
log.Printf("Building Trie..., from %s\n", dictFilePath)
h := fmt.Sprintf("%x", md5.Sum([]byte(dictFilePath)))
cacheFileName := fmt.Sprintf("jieba.%s.cache", h)
cacheFilePath := filepath.Join(os.TempDir(), cacheFileName)
isDictCached := true
cacheFileInfo, err := os.Stat(cacheFilePath)
if err != nil {
isDictCached = false
}
if isDictCached {
isDictCached = cacheFileInfo.ModTime().After(dictFileInfo.ModTime())
}
var cacheFile *os.File
if isDictCached {
cacheFile, err = os.Open(cacheFilePath)
if err != nil {
isDictCached = false
}
defer cacheFile.Close()
}
if isDictCached {
dec := gob.NewDecoder(cacheFile)
err = dec.Decode(&t)
if err != nil {
isDictCached = false
} else {
log.Printf("loaded model from cache %s\n", cacheFilePath)
}
}
if !isDictCached {
wtfs, err := ParseDictFile(dictFilePath)
if err != nil {
return err
}
for _, wtf := range wtfs {
t.addWord(wtf)
}
// dump trie
cacheFile, err = os.OpenFile(cacheFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return err
}
defer cacheFile.Close()
enc := gob.NewEncoder(cacheFile)
err = enc.Encode(t)
if err != nil {
return err
} else {
log.Printf("dumped model from cache %s\n", cacheFilePath)
}
}
return nil
}
func (t *trie) addWord(wtf *WordTagFreq) {
t.Freq[wtf.Word] = wtf.Freq
t.Total += wtf.Freq
runes := []rune(wtf.Word)
count := len(runes)
for i := 0; i < count; i++ {
wfrag := string(runes[0 : i+1])
if _, ok := t.Freq[wfrag]; !ok {
t.Freq[wfrag] = 0.0
}
}
}
// Load user specified dictionary file.
func LoadUserDict(dictFilePath string) error {
wtfs, err := ParseDictFile(dictFilePath)
if err != nil {
return err
}
for _, wtf := range wtfs {
if len(wtf.Tag) > 0 {
UserWordTagTab[wtf.Word] = strings.TrimSpace(wtf.Tag)
}
Trie.addWord(wtf)
}
return nil
}
// Set the dictionary, could be absolute path of dictionary file, or dictionary
// name in current directory. This function must be called before cut any
// sentence.
func SetDictionary(dictFileName string) error {
Trie = &trie{Total: 0.0, Freq: make(map[string]float64)}
return Trie.load(dictFileName)
}

180
trie_node.go
View File

@@ -1,180 +0,0 @@
package jiebago
import (
"bufio"
"crypto/md5"
"encoding/gob"
"fmt"
"log"
"math"
"os"
"path/filepath"
"strconv"
"strings"
)
type Trie struct {
Nodes map[rune]*Trie
IsLeaf bool
}
func NewTrie() *Trie {
return &Trie{make(map[rune]*Trie), false}
}
type TopTrie struct {
T *Trie
MinFreq float64
Total float64
Freq map[string]float64
}
func newTopTrie(filename string) (*TopTrie, error) {
var file_path string
var topTrie *TopTrie
if filepath.IsAbs(filename) {
file_path = filename
} else {
pwd, err := os.Getwd()
if err != nil {
return nil, err
}
file_path = filepath.Clean(filepath.Join(pwd, filename))
}
fi, err := os.Stat(file_path)
if err != nil {
return nil, err
}
log.Printf("Building Trie..., from %s\n", file_path)
h := fmt.Sprintf("%x", md5.Sum([]byte(file_path)))
cache_file_name := fmt.Sprintf("jieba.%s.cache", h)
cache_path := filepath.Join(os.TempDir(), cache_file_name)
isDictCached := true
cache_fi, err := os.Stat(cache_path)
if err != nil {
isDictCached = false
}
if isDictCached {
isDictCached = cache_fi.ModTime().After(fi.ModTime())
}
var cacheFile *os.File
if isDictCached {
cacheFile, err = os.Open(cache_path)
if err != nil {
isDictCached = false
}
defer cacheFile.Close()
}
if isDictCached {
dec := gob.NewDecoder(cacheFile)
err = dec.Decode(&topTrie)
if err != nil {
isDictCached = false
} else {
log.Printf("loaded model from cache %s\n", cache_path)
}
}
if !isDictCached {
topTrie = &TopTrie{T: NewTrie(), MinFreq: 100.0, Total: 0.0, Freq: make(map[string]float64)}
file, openError := os.Open(file_path)
if openError != nil {
return nil, openError
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
words := strings.Split(line, " ")
word, freqStr := words[0], words[1]
freq, _ := strconv.ParseFloat(freqStr, 64)
topTrie.Total += freq
topTrie.addWord(word, freq)
}
if scanErr := scanner.Err(); scanErr != nil {
return nil, scanErr
}
var val float64
for key := range topTrie.Freq {
val = math.Log(topTrie.Freq[key] / topTrie.Total)
if val < topTrie.MinFreq {
topTrie.MinFreq = val
}
topTrie.Freq[key] = val
}
// dump topTrie
cacheFile, err = os.OpenFile(cache_path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return topTrie, err
}
defer cacheFile.Close()
enc := gob.NewEncoder(cacheFile)
err := enc.Encode(topTrie)
if err != nil {
return topTrie, err
} else {
log.Printf("dumped model from cache %s\n", cache_path)
}
}
return topTrie, nil
}
func (tt *TopTrie) addWord(word string, freq float64) {
tt.Freq[word] = freq
var p *Trie
runes := []rune(word)
count := len(runes)
for index, key := range runes {
if index == 0 {
p = tt.T
}
if _, ok := p.Nodes[key]; !ok {
p.Nodes[key] = NewTrie()
}
if index == count-1 {
p.Nodes[key].IsLeaf = true
}
p = p.Nodes[key]
}
}
func addWord(word string, freq float64, tag string) {
if len(tag) > 0 {
UserWordTagTab[word] = strings.TrimSpace(tag)
}
TT.addWord(word, freq)
}
func LoadUserDict(file_path string) error {
file, openError := os.Open(file_path)
if openError != nil {
return openError
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
words := strings.Split(line, " ")
word, freqStr := words[0], words[1]
word = strings.Replace(word, "\ufeff", "", 1)
freq, freqErr := strconv.ParseFloat(freqStr, 64)
if freqErr != nil {
continue // TODO: how to handle wrong type of frequency?
}
tag := ""
if len(words) == 3 {
tag = words[2]
}
addWord(word, freq, tag)
}
return scanner.Err()
}