mirror of
https://github.com/fumiama/jieba.git
synced 2026-06-09 11:00:23 +08:00
Merge branch 'release/v0.2'
This commit is contained in:
Wang Bin
255
README.md
255
README.md
@@ -1,18 +1,18 @@
|
||||
结巴分词Go版 jiebago
|
||||
===================
|
||||
#结巴分词 Go 语言版：jiebago
|
||||
|
||||
|
||||
[](https://travis-ci.org/wangbin/jiebago)
|
||||
|
||||
[结巴分词](https://github.com/fxsjy/jieba)是[@fxsjy](https://github.com/fxsjy)用Python编写的中文分词组件，jiebago是结巴分词的Go语言实现，目前已经实现的功能包括：三种模式分词、自定义词典、关键词提取和词性标注。
|
||||
|
||||
|
||||
安装
|
||||
=====
|
||||
## 安装
|
||||
|
||||
go get github.com/wangbin/jiebago
|
||||
|
||||
go get github.com/wangbin/jiebago/...
|
||||
|
||||
分词
|
||||
=====
|
||||
## 分词
|
||||
|
||||
|
||||
package main
|
||||
|
||||
@@ -53,8 +53,8 @@
|
||||
|
||||
【搜索引擎模式】：小明 / 硕士 / 毕业 / 于 / 中国 / 科学 / 学院 / 科学院 / 中国科学院 / 计算 / 计算所 / ， / 后 / 在 / 日本 / 京都 / 大学 / 日本京都大学 / 深造 /
|
||||
|
||||
添加自定义词典
|
||||
=============
|
||||
## 添加自定义词典
|
||||
|
||||
|
||||
var sentence = "李小福是创新办主任也是云计算方面的专家"
|
||||
fmt.Print("Before: ")
|
||||
@@ -69,12 +69,7 @@
|
||||
|
||||
After: 李小福 / 是 / 创新办 / 主任 / 也 / 是 / 云计算 / 方面 / 的 / 专家 /
|
||||
|
||||
关键词提取
|
||||
========
|
||||
|
||||
需要先安装analyse模块：
|
||||
|
||||
go get github.com/wangbin/jiebago/analyse
|
||||
## 关键词提取
|
||||
|
||||
示例代码：
|
||||
|
||||
@@ -133,12 +128,7 @@
|
||||
全资 0.306324
|
||||
商业 0.306138
|
||||
|
||||
词性标注
|
||||
=======
|
||||
|
||||
需要先安装posseg模块：
|
||||
|
||||
go get github.com/wangbin/jiebago/posseg
|
||||
## 词性标注
|
||||
|
||||
示例代码：
|
||||
|
||||
@@ -166,8 +156,8 @@
|
||||
北京 ns
|
||||
天安门 ns
|
||||
|
||||
并行分词
|
||||
=======
|
||||
|
||||
## 并行分词
|
||||
|
||||
因为Go有强大的goroutine特性，并行分词实现起来非常简单，所以并没有内置到jiebaogo中，而是由使用者自己实现，下面是一个简单的例子：
|
||||
|
||||
@@ -207,40 +197,213 @@
|
||||
writer.Flush()
|
||||
|
||||
|
||||
Tokenize
|
||||
=========
|
||||
## Tokenize：返回词语在原文的起始位置
|
||||
|
||||
var sentence = "永和服装饰品有限公司"
|
||||
// 默认模式
|
||||
for _, token := range jiebago.Tokenize(sentence, "default", true) {
|
||||
fmt.Printf("word %s\t\t start: %d \t\t end:%d\n", token.Word, token.Start, token.End)
|
||||
|
||||
注意新版的 Jiebago Tokenizer 实现了 Bleve 的 Tokenizer 接口，跟之前的实现有很大的变化：
|
||||
|
||||
1. 接受的参数必须是 []byte。
|
||||
2. 输出的 Token 的起始和终止位置是 byte 的位置，不是之前的 rune 的位置，所以和 Python 版的 Jieba.tokenize 输出不一致。
|
||||
|
||||
```
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/wangbin/jiebago/tokenizers"
|
||||
)
|
||||
|
||||
const DictPath = "/path/to/dict.txt"
|
||||
|
||||
var sentence = []byte("永和服装饰品有限公司")
|
||||
|
||||
func main() {
|
||||
// default mode
|
||||
tokenizer, _ := tokenizers.NewJiebaTokenizer(DictPath, true, false) for _, token := range tokenizer.Tokenize(sentence) {
|
||||
fmt.Printf(
|
||||
"Term: %s\t Start: %d \t End: %d\t Position: %d\t Type: %d\n",
|
||||
token.Term, token.Start, token.End, token.Position, token.Type)
|
||||
}
|
||||
// 搜索模式
|
||||
for _, token := range jiebago.Tokenize(sentence, "search", true) {
|
||||
fmt.Printf("word %s\t\t start: %d \t\t end:%d\n", token.Word, token.Start, token.End)
|
||||
|
||||
//search mode
|
||||
tokenizer, _ = tokenizers.NewJiebaTokenizer(DictPath, true, true)
|
||||
for _, token := range tokenizer.Tokenize(sentence) {
|
||||
fmt.Printf(
|
||||
"Term: %s\t Start: %d \t End: %d\t Position: %d\t Type: %d\n",
|
||||
token.Term, token.Start, token.End, token.Position, token.Type)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
```
|
||||
默认模式输出：
|
||||
|
||||
```
|
||||
Term: 永和 Start: 0 End: 6 Position: 1 Type: 1
|
||||
Term: 服装 Start: 6 End: 12 Position: 2 Type: 1
|
||||
Term: 饰品 Start: 12 End: 18 Position: 3 Type: 1
|
||||
Term: 有限公司 Start: 18 End: 30 Position: 4 Type: 1
|
||||
```
|
||||
搜索模式输出：
|
||||
|
||||
```
|
||||
Term: 永和 Start: 0 End: 6 Position: 1 Type: 1
|
||||
Term: 服装 Start: 6 End: 12 Position: 2 Type: 1
|
||||
Term: 饰品 Start: 12 End: 18 Position: 3 Type: 1
|
||||
Term: 有限 Start: 18 End: 24 Position: 4 Type: 1
|
||||
Term: 公司 Start: 24 End: 30 Position: 5 Type: 1
|
||||
Term: 有限公司 Start: 18 End: 30 Position: 6 Type: 1
|
||||
```
|
||||
### 配合 bleve 进行中文全文检索
|
||||
|
||||
[bleve](http://www.blevesearch.com/) 是一个 Go 语言实现的全文索引系统，jiebago 可以配合 bleve 使用实现中文的全文检索。一个简单的用法示例：
|
||||
|
||||
```
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/blevesearch/bleve"
|
||||
_ "github.com/wangbin/jiebago/analyse/tokenizers"
|
||||
"log"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// open a new index
|
||||
indexMapping := bleve.NewIndexMapping()
|
||||
|
||||
err := indexMapping.AddCustomTokenizer("jieba",
|
||||
map[string]interface{}{
|
||||
"file": "/Users/wangbin/mygo/src/github.com/wangbin/jiebago/dict.txt",
|
||||
"type": "jieba",
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
err = indexMapping.AddCustomAnalyzer("jieba",
|
||||
map[string]interface{}{
|
||||
"type": "custom",
|
||||
"tokenizer": "jieba",
|
||||
"token_filters": []string{
|
||||
"possessive_en",
|
||||
"to_lower",
|
||||
"stop_en",
|
||||
},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
indexMapping.DefaultAnalyzer = "jieba"
|
||||
|
||||
index, err := bleve.New("example.bleve", indexMapping)
|
||||
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
indexMapping.DefaultAnalyzer = "jieba"
|
||||
|
||||
index, err := bleve.New("example.bleve", indexMapping)
|
||||
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
docs := []struct {
|
||||
Title string
|
||||
Name string
|
||||
}{
|
||||
{
|
||||
Title: "Doc 1",
|
||||
Name: "This is the first document we’ve added",
|
||||
},
|
||||
{
|
||||
Title: "Doc 2",
|
||||
Name: "The second one 你 中文测试中文 is even more interesting! 吃水果",
|
||||
},
|
||||
{
|
||||
Title: "Doc 3",
|
||||
Name: "买水果然后来世博园。",
|
||||
},
|
||||
{
|
||||
Title: "Doc 4",
|
||||
Name: "工信处女干事每月经过下属科室都要亲口交代24口交换机等技术性器件的安装工作",
|
||||
},
|
||||
{
|
||||
Title: "Doc 5",
|
||||
Name: "咱俩交换一下吧。",
|
||||
},
|
||||
}
|
||||
// index docs
|
||||
for _, doc := range docs {
|
||||
index.Index(doc.Title, doc)
|
||||
}
|
||||
|
||||
// search for some text
|
||||
for _, keyword := range []string{"水果世博园", "你", "first", "中文", "交换机", "交换"} {
|
||||
query := bleve.NewMatchQuery(keyword)
|
||||
search := bleve.NewSearchRequest(query)
|
||||
search.Highlight = bleve.NewHighlight()
|
||||
searchResults, err := index.Search(search)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Printf("Result of %s: %s\n", keyword, searchResults)
|
||||
}
|
||||
}
|
||||
```
|
||||
输出结果：
|
||||
|
||||
word 永和 start: 0 end:2
|
||||
word 服装 start: 2 end:4
|
||||
word 饰品 start: 4 end:6
|
||||
word 有限公司 start: 6 end:10
|
||||
```
|
||||
Result of 水果世博园: 2 matches, showing 1 through 2, took 377.988µs
|
||||
1. Doc 3 (1.099550)
|
||||
Name
|
||||
买<span class="highlight">水果</span>然后来<span class="highlight">世博</span>园。
|
||||
2. Doc 2 (0.031941)
|
||||
Name
|
||||
The second one 你 中文测试中文 is even more interesting! 吃<span class="highlight">水果</span>
|
||||
|
||||
word 永和 start: 0 end:2
|
||||
word 服装 start: 0 end:2
|
||||
word 饰品 start: 0 end:2
|
||||
word 有限 start: 0 end:2
|
||||
word 公司 start: 2 end:4
|
||||
word 有限公司 start: 0 end:4
|
||||
Result of 你: 1 matches, showing 1 through 1, took 103.367µs
|
||||
1. Doc 2 (0.391161)
|
||||
Name
|
||||
The second one <span class="highlight">你</span> 中文测试中文 is even more interesting! 吃水果
|
||||
|
||||
分词速度
|
||||
=======
|
||||
Result of first: 1 matches, showing 1 through 1, took 373.317µs
|
||||
1. Doc 1 (0.512150)
|
||||
Name
|
||||
This is the <span class="highlight">first</span> document we’ve added
|
||||
|
||||
Result of 中文: 1 matches, showing 1 through 1, took 106.433µs
|
||||
1. Doc 2 (0.553186)
|
||||
Name
|
||||
The second one 你 <span class="highlight">中文</span>测试<span class="highlight">中文</span> is even more interesting! 吃水果
|
||||
|
||||
Result of 交换机: 2 matches, showing 1 through 2, took 188.235µs
|
||||
1. Doc 4 (0.608495)
|
||||
Name
|
||||
工信处女干事每月经过下属科室都要亲口交代24口<span class="highlight">交换</span>机等技术性器件的安装工作
|
||||
2. Doc 5 (0.086700)
|
||||
Name
|
||||
咱俩<span class="highlight">交换</span>一下吧。
|
||||
|
||||
Result of 交换: 2 matches, showing 1 through 2, took 148.822µs
|
||||
1. Doc 5 (0.534158)
|
||||
Name
|
||||
咱俩<span class="highlight">交换</span>一下吧。
|
||||
2. Doc 4 (0.296297)
|
||||
Name
|
||||
工信处女干事每月经过下属科室都要亲口交代24口<span class="highlight">交换</span>机等技术性器件的安装工作
|
||||
```
|
||||
|
||||
## 分词速度
|
||||
|
||||
- 2MB / Second in Full Mode
|
||||
- 700KB / Second in Default Mode
|
||||
- Test Env: AMD Phenom(tm) II X6 1055T CPU @ 2.8GHz; 《金庸全集》
|
||||
|
||||
许可证
|
||||
======
|
||||
## 许可证
|
||||
|
||||
MIT: http://wangbin.mit-license.org
|
||||
|
||||
37
tokenize.go
37
tokenize.go
@@ -1,37 +0,0 @@
|
||||
package jiebago
|
||||
|
||||
type token struct {
|
||||
Word string
|
||||
Start int
|
||||
End int
|
||||
}
|
||||
|
||||
// Return words with position.
|
||||
func Tokenize(sentence string, mode string, HMM bool) []token {
|
||||
tokens := make([]token, 0)
|
||||
start := 0
|
||||
var width int
|
||||
for word := range Cut(sentence, false, HMM) {
|
||||
if mode == "default" {
|
||||
width = len([]rune(word))
|
||||
tokens = append(tokens, token{word, start, start + width})
|
||||
start += width
|
||||
|
||||
} else {
|
||||
runes := []rune(word)
|
||||
width = len(runes)
|
||||
for _, step := range []int{2, 3} {
|
||||
if width > step {
|
||||
for i := 0; i < width-step+1; i++ {
|
||||
gram := string(runes[i : i+step])
|
||||
if _, ok := Trie.Freq[gram]; ok {
|
||||
tokens = append(tokens, token{gram, start + i, start + i + step})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
tokens = append(tokens, token{word, start, start + width})
|
||||
}
|
||||
}
|
||||
return tokens
|
||||
}
|
||||
390
tokenize_test.go
390
tokenize_test.go
@@ -1,390 +0,0 @@
|
||||
package jiebago
|
||||
|
||||
import "testing"
|
||||
|
||||
var (
|
||||
result = [][]token{
|
||||
[]token{token{"\u8fd9\u662f", 0, 2}, token{"\u4e00\u4e2a", 2, 4}, token{"\u4f38\u624b\u4e0d\u89c1\u4e94\u6307", 4, 10}, token{"\u7684", 10, 11}, token{"\u9ed1\u591c", 11, 13}, token{"\u3002", 13, 14}, token{"\u6211", 14, 15}, token{"\u53eb", 15, 16}, token{"\u5b59\u609f\u7a7a", 16, 19}, token{"\uff0c", 19, 20}, token{"\u6211", 20, 21}, token{"\u7231", 21, 22}, token{"\u5317\u4eac", 22, 24}, token{"\uff0c", 24, 25}, token{"\u6211", 25, 26}, token{"\u7231", 26, 27}, token{"Python", 27, 33}, token{"\u548c", 33, 34}, token{"C++", 34, 37}, token{"\u3002", 37, 38}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u4e0d", 1, 2}, token{"\u559c\u6b22", 2, 4}, token{"\u65e5\u672c", 4, 6}, token{"\u548c\u670d", 6, 8}, token{"\u3002", 8, 9}},
|
||||
[]token{token{"\u96f7\u7334", 0, 2}, token{"\u56de\u5f52", 2, 4}, token{"\u4eba\u95f4", 4, 6}, token{"\u3002", 6, 7}},
|
||||
[]token{token{"\u5de5\u4fe1\u5904", 0, 3}, token{"\u5973\u5e72\u4e8b", 3, 6}, token{"\u6bcf\u6708", 6, 8}, token{"\u7ecf\u8fc7", 8, 10}, token{"\u4e0b\u5c5e", 10, 12}, token{"\u79d1\u5ba4", 12, 14}, token{"\u90fd", 14, 15}, token{"\u8981", 15, 16}, token{"\u4eb2\u53e3", 16, 18}, token{"\u4ea4\u4ee3", 18, 20}, token{"24", 20, 22}, token{"\u53e3", 22, 23}, token{"\u4ea4\u6362\u673a", 23, 26}, token{"\u7b49", 26, 27}, token{"\u6280\u672f\u6027", 27, 30}, token{"\u5668\u4ef6", 30, 32}, token{"\u7684", 32, 33}, token{"\u5b89\u88c5", 33, 35}, token{"\u5de5\u4f5c", 35, 37}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u9700\u8981", 1, 3}, token{"\u5ec9\u79df\u623f", 3, 6}},
|
||||
[]token{token{"\u6c38\u548c", 0, 2}, token{"\u670d\u88c5", 2, 4}, token{"\u9970\u54c1", 4, 6}, token{"\u6709\u9650\u516c\u53f8", 6, 10}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u7231", 1, 2}, token{"\u5317\u4eac", 2, 4}, token{"\u5929\u5b89\u95e8", 4, 7}},
|
||||
[]token{token{"abc", 0, 3}},
|
||||
[]token{token{"\u9690", 0, 1}, token{"\u9a6c\u5c14\u53ef\u592b", 1, 5}},
|
||||
[]token{token{"\u96f7\u7334", 0, 2}, token{"\u662f", 2, 3}, token{"\u4e2a", 3, 4}, token{"\u597d", 4, 5}, token{"\u7f51\u7ad9", 5, 7}},
|
||||
[]token{token{"\u201c", 0, 1}, token{"Microsoft", 1, 10}, token{"\u201d", 10, 11}, token{"\u4e00\u8bcd", 11, 13}, token{"\u7531", 13, 14}, token{"\u201c", 14, 15}, token{"MICROcomputer", 15, 28}, token{"\uff08", 28, 29}, token{"\u5fae\u578b", 29, 31}, token{"\u8ba1\u7b97\u673a", 31, 34}, token{"\uff09", 34, 35}, token{"\u201d", 35, 36}, token{"\u548c", 36, 37}, token{"\u201c", 37, 38}, token{"SOFTware", 38, 46}, token{"\uff08", 46, 47}, token{"\u8f6f\u4ef6", 47, 49}, token{"\uff09", 49, 50}, token{"\u201d", 50, 51}, token{"\u4e24", 51, 52}, token{"\u90e8\u5206", 52, 54}, token{"\u7ec4\u6210", 54, 56}},
|
||||
[]token{token{"\u8349\u6ce5\u9a6c", 0, 3}, token{"\u548c", 3, 4}, token{"\u6b3a\u5b9e", 4, 6}, token{"\u9a6c", 6, 7}, token{"\u662f", 7, 8}, token{"\u4eca\u5e74", 8, 10}, token{"\u7684", 10, 11}, token{"\u6d41\u884c", 11, 13}, token{"\u8bcd\u6c47", 13, 15}},
|
||||
[]token{token{"\u4f0a\u85e4", 0, 2}, token{"\u6d0b\u534e\u5802", 2, 5}, token{"\u603b\u5e9c", 5, 7}, token{"\u5e97", 7, 8}},
|
||||
[]token{token{"\u4e2d\u56fd\u79d1\u5b66\u9662\u8ba1\u7b97\u6280\u672f\u7814\u7a76\u6240", 0, 12}},
|
||||
[]token{token{"\u7f57\u5bc6\u6b27", 0, 3}, token{"\u4e0e", 3, 4}, token{"\u6731\u4e3d\u53f6", 4, 7}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u8d2d\u4e70", 1, 3}, token{"\u4e86", 3, 4}, token{"\u9053\u5177", 4, 6}, token{"\u548c", 6, 7}, token{"\u670d\u88c5", 7, 9}},
|
||||
[]token{token{"PS", 0, 2}, token{":", 2, 3}, token{" ", 3, 4}, token{"\u6211", 4, 5}, token{"\u89c9\u5f97", 5, 7}, token{"\u5f00\u6e90", 7, 9}, token{"\u6709", 9, 10}, token{"\u4e00\u4e2a", 10, 12}, token{"\u597d\u5904", 12, 14}, token{"\uff0c", 14, 15}, token{"\u5c31\u662f", 15, 17}, token{"\u80fd\u591f", 17, 19}, token{"\u6566\u4fc3", 19, 21}, token{"\u81ea\u5df1", 21, 23}, token{"\u4e0d\u65ad\u6539\u8fdb", 23, 27}, token{"\uff0c", 27, 28}, token{"\u907f\u514d", 28, 30}, token{"\u655e\u5e1a", 30, 32}, token{"\u81ea\u73cd", 32, 34}},
|
||||
[]token{token{"\u6e56\u5317\u7701", 0, 3}, token{"\u77f3\u9996\u5e02", 3, 6}},
|
||||
[]token{token{"\u6e56\u5317\u7701", 0, 3}, token{"\u5341\u5830\u5e02", 3, 6}},
|
||||
[]token{token{"\u603b\u7ecf\u7406", 0, 3}, token{"\u5b8c\u6210", 3, 5}, token{"\u4e86", 5, 6}, token{"\u8fd9\u4ef6", 6, 8}, token{"\u4e8b\u60c5", 8, 10}},
|
||||
[]token{token{"\u7535\u8111", 0, 2}, token{"\u4fee\u597d", 2, 4}, token{"\u4e86", 4, 5}},
|
||||
[]token{token{"\u505a\u597d", 0, 2}, token{"\u4e86", 2, 3}, token{"\u8fd9\u4ef6", 3, 5}, token{"\u4e8b\u60c5", 5, 7}, token{"\u5c31", 7, 8}, token{"\u4e00\u4e86\u767e\u4e86", 8, 12}, token{"\u4e86", 12, 13}},
|
||||
[]token{token{"\u4eba\u4eec", 0, 2}, token{"\u5ba1\u7f8e", 2, 4}, token{"\u7684", 4, 5}, token{"\u89c2\u70b9", 5, 7}, token{"\u662f", 7, 8}, token{"\u4e0d\u540c", 8, 10}, token{"\u7684", 10, 11}},
|
||||
[]token{token{"\u6211\u4eec", 0, 2}, token{"\u4e70", 2, 3}, token{"\u4e86", 3, 4}, token{"\u4e00\u4e2a", 4, 6}, token{"\u7f8e\u7684", 6, 8}, token{"\u7a7a\u8c03", 8, 10}},
|
||||
[]token{token{"\u7ebf\u7a0b", 0, 2}, token{"\u521d\u59cb\u5316", 2, 5}, token{"\u65f6", 5, 6}, token{"\u6211\u4eec", 6, 8}, token{"\u8981", 8, 9}, token{"\u6ce8\u610f", 9, 11}},
|
||||
[]token{token{"\u4e00\u4e2a", 0, 2}, token{"\u5206\u5b50", 2, 4}, token{"\u662f", 4, 5}, token{"\u7531", 5, 6}, token{"\u597d\u591a", 6, 8}, token{"\u539f\u5b50", 8, 10}, token{"\u7ec4\u7ec7", 10, 12}, token{"\u6210", 12, 13}, token{"\u7684", 13, 14}},
|
||||
[]token{token{"\u795d", 0, 1}, token{"\u4f60", 1, 2}, token{"\u9a6c\u5230\u529f\u6210", 2, 6}},
|
||||
[]token{token{"\u4ed6", 0, 1}, token{"\u6389", 1, 2}, token{"\u8fdb", 2, 3}, token{"\u4e86", 3, 4}, token{"\u65e0\u5e95\u6d1e", 4, 7}, token{"\u91cc", 7, 8}},
|
||||
[]token{token{"\u4e2d\u56fd", 0, 2}, token{"\u7684", 2, 3}, token{"\u9996\u90fd", 3, 5}, token{"\u662f", 5, 6}, token{"\u5317\u4eac", 6, 8}},
|
||||
[]token{token{"\u5b59\u541b\u610f", 0, 3}},
|
||||
[]token{token{"\u5916\u4ea4\u90e8", 0, 3}, token{"\u53d1\u8a00\u4eba", 3, 6}, token{"\u9a6c\u671d\u65ed", 6, 9}},
|
||||
[]token{token{"\u9886\u5bfc\u4eba", 0, 3}, token{"\u4f1a\u8bae", 3, 5}, token{"\u548c", 5, 6}, token{"\u7b2c\u56db\u5c4a", 6, 9}, token{"\u4e1c\u4e9a", 9, 11}, token{"\u5cf0\u4f1a", 11, 13}},
|
||||
[]token{token{"\u5728", 0, 1}, token{"\u8fc7\u53bb", 1, 3}, token{"\u7684", 3, 4}, token{"\u8fd9", 4, 5}, token{"\u4e94\u5e74", 5, 7}},
|
||||
[]token{token{"\u8fd8", 0, 1}, token{"\u9700\u8981", 1, 3}, token{"\u5f88\u957f", 3, 5}, token{"\u7684", 5, 6}, token{"\u8def", 6, 7}, token{"\u8981", 7, 8}, token{"\u8d70", 8, 9}},
|
||||
[]token{token{"60", 0, 2}, token{"\u5468\u5e74", 2, 4}, token{"\u9996\u90fd", 4, 6}, token{"\u9605\u5175", 6, 8}},
|
||||
[]token{token{"\u4f60\u597d", 0, 2}, token{"\u4eba\u4eec", 2, 4}, token{"\u5ba1\u7f8e", 4, 6}, token{"\u7684", 6, 7}, token{"\u89c2\u70b9", 7, 9}, token{"\u662f", 9, 10}, token{"\u4e0d\u540c", 10, 12}, token{"\u7684", 12, 13}},
|
||||
[]token{token{"\u4e70", 0, 1}, token{"\u6c34\u679c", 1, 3}, token{"\u7136\u540e", 3, 5}, token{"\u6765", 5, 6}, token{"\u4e16\u535a\u56ed", 6, 9}},
|
||||
[]token{token{"\u4e70", 0, 1}, token{"\u6c34\u679c", 1, 3}, token{"\u7136\u540e", 3, 5}, token{"\u53bb", 5, 6}, token{"\u4e16\u535a\u56ed", 6, 9}},
|
||||
[]token{token{"\u4f46\u662f", 0, 2}, token{"\u540e\u6765", 2, 4}, token{"\u6211", 4, 5}, token{"\u624d", 5, 6}, token{"\u77e5\u9053", 6, 8}, token{"\u4f60", 8, 9}, token{"\u662f", 9, 10}, token{"\u5bf9", 10, 11}, token{"\u7684", 11, 12}},
|
||||
[]token{token{"\u5b58\u5728", 0, 2}, token{"\u5373", 2, 3}, token{"\u5408\u7406", 3, 5}},
|
||||
[]token{token{"\u7684", 0, 1}, token{"\u7684", 1, 2}, token{"\u7684", 2, 3}, token{"\u7684", 3, 4}, token{"\u7684", 4, 5}, token{"\u5728", 5, 6}, token{"\u7684", 6, 7}, token{"\u7684", 7, 8}, token{"\u7684", 8, 9}, token{"\u7684", 9, 10}, token{"\u5c31", 10, 11}, token{"\u4ee5", 11, 12}, token{"\u548c", 12, 13}, token{"\u548c", 13, 14}, token{"\u548c", 14, 15}},
|
||||
[]token{token{"I", 0, 1}, token{" ", 1, 2}, token{"love", 2, 6}, token{"\u4f60", 6, 7}, token{"\uff0c", 7, 8}, token{"\u4e0d\u4ee5\u4e3a\u803b", 8, 12}, token{"\uff0c", 12, 13}, token{"\u53cd", 13, 14}, token{"\u4ee5\u4e3a", 14, 16}, token{"rong", 16, 20}},
|
||||
[]token{token{"\u56e0", 0, 1}},
|
||||
[]token{},
|
||||
[]token{token{"hello", 0, 5}, token{"\u4f60\u597d", 5, 7}, token{"\u4eba\u4eec", 7, 9}, token{"\u5ba1\u7f8e", 9, 11}, token{"\u7684", 11, 12}, token{"\u89c2\u70b9", 12, 14}, token{"\u662f", 14, 15}, token{"\u4e0d\u540c", 15, 17}, token{"\u7684", 17, 18}},
|
||||
[]token{token{"\u5f88", 0, 1}, token{"\u597d", 1, 2}, token{"\u4f46", 2, 3}, token{"\u4e3b\u8981", 3, 5}, token{"\u662f", 5, 6}, token{"\u57fa\u4e8e", 6, 8}, token{"\u7f51\u9875", 8, 10}, token{"\u5f62\u5f0f", 10, 12}},
|
||||
[]token{token{"hello", 0, 5}, token{"\u4f60\u597d", 5, 7}, token{"\u4eba\u4eec", 7, 9}, token{"\u5ba1\u7f8e", 9, 11}, token{"\u7684", 11, 12}, token{"\u89c2\u70b9", 12, 14}, token{"\u662f", 14, 15}, token{"\u4e0d\u540c", 15, 17}, token{"\u7684", 17, 18}},
|
||||
[]token{token{"\u4e3a\u4ec0\u4e48", 0, 3}, token{"\u6211", 3, 4}, token{"\u4e0d\u80fd", 4, 6}, token{"\u62e5\u6709", 6, 8}, token{"\u60f3\u8981", 8, 10}, token{"\u7684", 10, 11}, token{"\u751f\u6d3b", 11, 13}},
|
||||
[]token{token{"\u540e\u6765", 0, 2}, token{"\u6211", 2, 3}, token{"\u624d", 3, 4}},
|
||||
[]token{token{"\u6b64\u6b21", 0, 2}, token{"\u6765", 2, 3}, token{"\u4e2d\u56fd", 3, 5}, token{"\u662f", 5, 6}, token{"\u4e3a\u4e86", 6, 8}},
|
||||
[]token{token{"\u4f7f\u7528", 0, 2}, token{"\u4e86", 2, 3}, token{"\u5b83", 3, 4}, token{"\u5c31", 4, 5}, token{"\u53ef\u4ee5", 5, 7}, token{"\u89e3\u51b3", 7, 9}, token{"\u4e00\u4e9b", 9, 11}, token{"\u95ee\u9898", 11, 13}},
|
||||
[]token{token{",", 0, 1}, token{"\u4f7f\u7528", 1, 3}, token{"\u4e86", 3, 4}, token{"\u5b83", 4, 5}, token{"\u5c31", 5, 6}, token{"\u53ef\u4ee5", 6, 8}, token{"\u89e3\u51b3", 8, 10}, token{"\u4e00\u4e9b", 10, 12}, token{"\u95ee\u9898", 12, 14}},
|
||||
[]token{token{"\u5176\u5b9e", 0, 2}, token{"\u4f7f\u7528", 2, 4}, token{"\u4e86", 4, 5}, token{"\u5b83", 5, 6}, token{"\u5c31", 6, 7}, token{"\u53ef\u4ee5", 7, 9}, token{"\u89e3\u51b3", 9, 11}, token{"\u4e00\u4e9b", 11, 13}, token{"\u95ee\u9898", 13, 15}},
|
||||
[]token{token{"\u597d\u4eba", 0, 2}, token{"\u4f7f\u7528", 2, 4}, token{"\u4e86", 4, 5}, token{"\u5b83", 5, 6}, token{"\u5c31", 6, 7}, token{"\u53ef\u4ee5", 7, 9}, token{"\u89e3\u51b3", 9, 11}, token{"\u4e00\u4e9b", 11, 13}, token{"\u95ee\u9898", 13, 15}},
|
||||
[]token{token{"\u662f\u56e0\u4e3a", 0, 3}, token{"\u548c", 3, 4}, token{"\u56fd\u5bb6", 4, 6}},
|
||||
[]token{token{"\u8001\u5e74", 0, 2}, token{"\u641c\u7d22", 2, 4}, token{"\u8fd8", 4, 5}, token{"\u652f\u6301", 5, 7}},
|
||||
[]token{token{"\u5e72\u8106", 0, 2}, token{"\u5c31", 2, 3}, token{"\u628a", 3, 4}, token{"\u90a3\u90e8", 4, 6}, token{"\u8499\u4eba", 6, 8}, token{"\u7684", 8, 9}, token{"\u95f2\u6cd5", 9, 11}, token{"\u7ed9", 11, 12}, token{"\u5e9f", 12, 13}, token{"\u4e86", 13, 14}, token{"\u62c9\u5012", 14, 16}, token{"\uff01", 16, 17}, token{"RT", 17, 19}, token{" ", 19, 20}, token{"@", 20, 21}, token{"laoshipukong", 21, 33}, token{" ", 33, 34}, token{":", 34, 35}, token{" ", 35, 36}, token{"27", 36, 38}, token{"\u65e5", 38, 39}, token{"\uff0c", 39, 40}, token{"\u5168\u56fd\u4eba\u5927\u5e38\u59d4\u4f1a", 40, 47}, token{"\u7b2c\u4e09\u6b21", 47, 50}, token{"\u5ba1\u8bae", 50, 52}, token{"\u4fb5\u6743", 52, 54}, token{"\u8d23\u4efb\u6cd5", 54, 57}, token{"\u8349\u6848", 57, 59}, token{"\uff0c", 59, 60}, token{"\u5220\u9664", 60, 62}, token{"\u4e86", 62, 63}, token{"\u6709\u5173", 63, 65}, token{"\u533b\u7597", 65, 67}, token{"\u635f\u5bb3", 67, 69}, token{"\u8d23\u4efb", 69, 71}, token{"\u201c", 71, 72}, token{"\u4e3e\u8bc1", 72, 74}, token{"\u5012\u7f6e", 74, 76}, token{"\u201d", 76, 77}, token{"\u7684", 77, 78}, token{"\u89c4\u5b9a", 78, 80}, token{"\u3002", 80, 81}, token{"\u5728", 81, 82}, token{"\u533b\u60a3", 82, 84}, token{"\u7ea0\u7eb7", 84, 86}, token{"\u4e2d\u672c", 86, 88}, token{"\u5df2", 88, 89}, token{"\u5904\u4e8e", 89, 91}, token{"\u5f31\u52bf", 91, 93}, token{"\u5730\u4f4d", 93, 95}, token{"\u7684", 95, 96}, token{"\u6d88\u8d39\u8005", 96, 99}, token{"\u7531\u6b64", 99, 101}, token{"\u5c06", 101, 102}, token{"\u9677\u5165", 102, 104}, token{"\u4e07\u52ab\u4e0d\u590d", 104, 108}, token{"\u7684", 108, 109}, token{"\u5883\u5730", 109, 111}, token{"\u3002", 111, 112}, token{" ", 112, 113}},
|
||||
[]token{token{"\u5927", 0, 1}},
|
||||
[]token{},
|
||||
[]token{token{"\u4ed6", 0, 1}, token{"\u8bf4", 1, 2}, token{"\u7684", 2, 3}, token{"\u786e\u5b9e", 3, 5}, token{"\u5728\u7406", 5, 7}},
|
||||
[]token{token{"\u957f\u6625", 0, 2}, token{"\u5e02\u957f", 2, 4}, token{"\u6625\u8282", 4, 6}, token{"\u8bb2\u8bdd", 6, 8}},
|
||||
[]token{token{"\u7ed3\u5a5a", 0, 2}, token{"\u7684", 2, 3}, token{"\u548c", 3, 4}, token{"\u5c1a\u672a", 4, 6}, token{"\u7ed3\u5a5a", 6, 8}, token{"\u7684", 8, 9}},
|
||||
[]token{token{"\u7ed3\u5408", 0, 2}, token{"\u6210", 2, 3}, token{"\u5206\u5b50", 3, 5}, token{"\u65f6", 5, 6}},
|
||||
[]token{token{"\u65c5\u6e38", 0, 2}, token{"\u548c", 2, 3}, token{"\u670d\u52a1", 3, 5}, token{"\u662f", 5, 6}, token{"\u6700\u597d", 6, 8}, token{"\u7684", 8, 9}},
|
||||
[]token{token{"\u8fd9\u4ef6", 0, 2}, token{"\u4e8b\u60c5", 2, 4}, token{"\u7684\u786e", 4, 6}, token{"\u662f", 6, 7}, token{"\u6211", 7, 8}, token{"\u7684", 8, 9}, token{"\u9519", 9, 10}},
|
||||
[]token{token{"\u4f9b", 0, 1}, token{"\u5927\u5bb6", 1, 3}, token{"\u53c2\u8003", 3, 5}, token{"\u6307\u6b63", 5, 7}},
|
||||
[]token{token{"\u54c8\u5c14\u6ee8", 0, 3}, token{"\u653f\u5e9c", 3, 5}, token{"\u516c\u5e03", 5, 7}, token{"\u584c\u6865", 7, 9}, token{"\u539f\u56e0", 9, 11}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u5728", 1, 2}, token{"\u673a\u573a", 2, 4}, token{"\u5165\u53e3\u5904", 4, 7}},
|
||||
[]token{token{"\u90a2\u6c38\u81e3", 0, 3}, token{"\u6444\u5f71", 3, 5}, token{"\u62a5\u9053", 5, 7}},
|
||||
[]token{token{"BP", 0, 2}, token{"\u795e\u7ecf\u7f51\u7edc", 2, 6}, token{"\u5982\u4f55", 6, 8}, token{"\u8bad\u7ec3", 8, 10}, token{"\u624d\u80fd", 10, 12}, token{"\u5728", 12, 13}, token{"\u5206\u7c7b", 13, 15}, token{"\u65f6", 15, 16}, token{"\u589e\u52a0", 16, 18}, token{"\u533a\u5206\u5ea6", 18, 21}, token{"\uff1f", 21, 22}},
|
||||
[]token{token{"\u5357\u4eac\u5e02", 0, 3}, token{"\u957f\u6c5f\u5927\u6865", 3, 7}},
|
||||
[]token{token{"\u5e94", 0, 1}, token{"\u4e00\u4e9b", 1, 3}, token{"\u4f7f\u7528\u8005", 3, 6}, token{"\u7684", 6, 7}, token{"\u5efa\u8bae", 7, 9}, token{"\uff0c", 9, 10}, token{"\u4e5f", 10, 11}, token{"\u4e3a\u4e86", 11, 13}, token{"\u4fbf\u4e8e", 13, 15}, token{"\u5229\u7528", 15, 17}, token{"NiuTrans", 17, 25}, token{"\u7528\u4e8e", 25, 27}, token{"SMT", 27, 30}, token{"\u7814\u7a76", 30, 32}},
|
||||
[]token{token{"\u957f\u6625\u5e02", 0, 3}, token{"\u957f\u6625", 3, 5}, token{"\u836f\u5e97", 5, 7}},
|
||||
[]token{token{"\u9093\u9896\u8d85", 0, 3}, token{"\u751f\u524d", 3, 5}, token{"\u6700", 5, 6}, token{"\u559c\u6b22", 6, 8}, token{"\u7684", 8, 9}, token{"\u8863\u670d", 9, 11}},
|
||||
[]token{token{"\u80e1\u9526\u6d9b", 0, 3}, token{"\u662f", 3, 4}, token{"\u70ed\u7231", 4, 6}, token{"\u4e16\u754c", 6, 8}, token{"\u548c\u5e73", 8, 10}, token{"\u7684", 10, 11}, token{"\u653f\u6cbb\u5c40", 11, 14}, token{"\u5e38\u59d4", 14, 16}},
|
||||
[]token{token{"\u7a0b\u5e8f\u5458", 0, 3}, token{"\u795d", 3, 4}, token{"\u6d77\u6797", 4, 6}, token{"\u548c", 6, 7}, token{"\u6731\u4f1a\u9707", 7, 10}, token{"\u662f", 10, 11}, token{"\u5728", 11, 12}, token{"\u5b59\u5065", 12, 14}, token{"\u7684", 14, 15}, token{"\u5de6\u9762", 15, 17}, token{"\u548c", 17, 18}, token{"\u53f3\u9762", 18, 20}, token{",", 20, 21}, token{" ", 21, 22}, token{"\u8303\u51ef", 22, 24}, token{"\u5728", 24, 25}, token{"\u6700", 25, 26}, token{"\u53f3\u9762", 26, 28}, token{".", 28, 29}, token{"\u518d\u5f80", 29, 31}, token{"\u5de6", 31, 32}, token{"\u662f", 32, 33}, token{"\u674e\u677e\u6d2a", 33, 36}},
|
||||
[]token{token{"\u4e00\u6b21\u6027", 0, 3}, token{"\u4ea4", 3, 4}, token{"\u591a\u5c11", 4, 6}, token{"\u94b1", 6, 7}},
|
||||
[]token{token{"\u4e24\u5757", 0, 2}, token{"\u4e94", 2, 3}, token{"\u4e00\u5957", 3, 5}, token{"\uff0c", 5, 6}, token{"\u4e09\u5757", 6, 8}, token{"\u516b", 8, 9}, token{"\u4e00\u65a4", 9, 11}, token{"\uff0c", 11, 12}, token{"\u56db\u5757", 12, 14}, token{"\u4e03", 14, 15}, token{"\u4e00\u672c", 15, 17}, token{"\uff0c", 17, 18}, token{"\u4e94\u5757", 18, 20}, token{"\u516d", 20, 21}, token{"\u4e00\u6761", 21, 23}},
|
||||
[]token{token{"\u5c0f", 0, 1}, token{"\u548c\u5c1a", 1, 3}, token{"\u7559", 3, 4}, token{"\u4e86", 4, 5}, token{"\u4e00\u4e2a", 5, 7}, token{"\u50cf", 7, 8}, token{"\u5927", 8, 9}, token{"\u548c\u5c1a", 9, 11}, token{"\u4e00\u6837", 11, 13}, token{"\u7684", 13, 14}, token{"\u548c\u5c1a\u5934", 14, 17}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u662f", 1, 2}, token{"\u4e2d\u534e\u4eba\u6c11\u5171\u548c\u56fd", 2, 9}, token{"\u516c\u6c11", 9, 11}, token{";", 11, 12}, token{"\u6211", 12, 13}, token{"\u7238\u7238", 13, 15}, token{"\u662f", 15, 16}, token{"\u5171\u548c\u515a", 16, 19}, token{"\u515a\u5458", 19, 21}, token{";", 21, 22}, token{" ", 22, 23}, token{"\u5730\u94c1", 23, 25}, token{"\u548c\u5e73\u95e8", 25, 28}, token{"\u7ad9", 28, 29}},
|
||||
[]token{token{"\u5f20\u6653\u6885", 0, 3}, token{"\u53bb", 3, 4}, token{"\u4eba\u6c11", 4, 6}, token{"\u533b\u9662", 6, 8}, token{"\u505a", 8, 9}, token{"\u4e86", 9, 10}, token{"\u4e2a", 10, 11}, token{"B\u8d85", 11, 13}, token{"\u7136\u540e", 13, 15}, token{"\u53bb", 15, 16}, token{"\u4e70", 16, 17}, token{"\u4e86", 17, 18}, token{"\u4ef6", 18, 19}, token{"T\u6064", 19, 21}},
|
||||
[]token{token{"AT&T", 0, 4}, token{"\u662f", 4, 5}, token{"\u4e00\u4ef6", 5, 7}, token{"\u4e0d\u9519", 7, 9}, token{"\u7684", 9, 10}, token{"\u516c\u53f8", 10, 12}, token{"\uff0c", 12, 13}, token{"\u7ed9", 13, 14}, token{"\u4f60", 14, 15}, token{"\u53d1", 15, 16}, token{"offer", 16, 21}, token{"\u4e86", 21, 22}, token{"\u5417", 22, 23}, token{"\uff1f", 23, 24}},
|
||||
[]token{token{"C++", 0, 3}, token{"\u548c", 3, 4}, token{"c#", 4, 6}, token{"\u662f", 6, 7}, token{"\u4ec0\u4e48", 7, 9}, token{"\u5173\u7cfb", 9, 11}, token{"\uff1f", 11, 12}, token{"11", 12, 14}, token{"+", 14, 15}, token{"122", 15, 18}, token{"=", 18, 19}, token{"133", 19, 22}, token{"\uff0c", 22, 23}, token{"\u662f", 23, 24}, token{"\u5417", 24, 25}, token{"\uff1f", 25, 26}, token{"PI", 26, 28}, token{"=", 28, 29}, token{"3.14159", 29, 36}},
|
||||
[]token{token{"\u4f60", 0, 1}, token{"\u8ba4\u8bc6", 1, 3}, token{"\u90a3\u4e2a", 3, 5}, token{"\u548c", 5, 6}, token{"\u4e3b\u5e2d", 6, 8}, token{"\u63e1\u624b", 8, 10}, token{"\u7684", 10, 11}, token{"\u7684\u54e5", 11, 13}, token{"\u5417", 13, 14}, token{"\uff1f", 14, 15}, token{"\u4ed6\u5f00", 15, 17}, token{"\u4e00\u8f86", 17, 19}, token{"\u9ed1\u8272", 19, 21}, token{"\u7684\u58eb", 21, 23}, token{"\u3002", 23, 24}},
|
||||
[]token{token{"\u67aa\u6746\u5b50", 0, 3}, token{"\u4e2d", 3, 4}, token{"\u51fa", 4, 5}, token{"\u653f\u6743", 5, 7}},
|
||||
[]token{token{"\u5f20\u4e09\u98ce", 0, 3}, token{"\u540c\u5b66", 3, 5}, token{"\u8d70\u4e0a", 5, 7}, token{"\u4e86", 7, 8}, token{"\u4e0d\u5f52\u8def", 8, 11}},
|
||||
[]token{token{"\u963fQ", 0, 2}, token{"\u8170\u95f4", 2, 4}, token{"\u6302", 4, 5}, token{"\u7740", 5, 6}, token{"BB\u673a", 6, 9}, token{"\u624b\u91cc", 9, 11}, token{"\u62ff", 11, 12}, token{"\u7740", 12, 13}, token{"\u5927\u54e5\u5927", 13, 16}, token{"\uff0c", 16, 17}, token{"\u8bf4", 17, 18}, token{"\uff1a", 18, 19}, token{"\u6211", 19, 20}, token{"\u4e00\u822c", 20, 22}, token{"\u5403\u996d", 22, 24}, token{"\u4e0d", 24, 25}, token{"AA\u5236", 25, 28}, token{"\u7684", 28, 29}, token{"\u3002", 29, 30}},
|
||||
[]token{token{"\u5728", 0, 1}, token{"1\u53f7\u5e97", 1, 4}, token{"\u80fd", 4, 5}, token{"\u4e70", 5, 6}, token{"\u5230", 6, 7}, token{"\u5c0fS", 7, 9}, token{"\u548c", 9, 10}, token{"\u5927S", 10, 12}, token{"\u516b\u5366", 12, 14}, token{"\u7684", 14, 15}, token{"\u4e66", 15, 16}, token{"\u3002", 16, 17}},
|
||||
[]token{token{"\u8fd9\u662f", 0, 2}, token{"\u4e00\u4e2a", 2, 4}, token{"\u4f38\u624b", 4, 6}, token{"\u4e0d\u89c1", 6, 8}, token{"\u4e94\u6307", 8, 10}, token{"\u4f38\u624b\u4e0d\u89c1\u4e94\u6307", 4, 10}, token{"\u7684", 10, 11}, token{"\u9ed1\u591c", 11, 13}, token{"\u3002", 13, 14}, token{"\u6211", 14, 15}, token{"\u53eb", 15, 16}, token{"\u609f\u7a7a", 17, 19}, token{"\u5b59\u609f\u7a7a", 16, 19}, token{"\uff0c", 19, 20}, token{"\u6211", 20, 21}, token{"\u7231", 21, 22}, token{"\u5317\u4eac", 22, 24}, token{"\uff0c", 24, 25}, token{"\u6211", 25, 26}, token{"\u7231", 26, 27}, token{"Python", 27, 33}, token{"\u548c", 33, 34}, token{"C++", 34, 37}, token{"\u3002", 37, 38}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u4e0d", 1, 2}, token{"\u559c\u6b22", 2, 4}, token{"\u65e5\u672c", 4, 6}, token{"\u548c\u670d", 6, 8}, token{"\u3002", 8, 9}},
|
||||
[]token{token{"\u96f7\u7334", 0, 2}, token{"\u56de\u5f52", 2, 4}, token{"\u4eba\u95f4", 4, 6}, token{"\u3002", 6, 7}},
|
||||
[]token{token{"\u5de5\u4fe1\u5904", 0, 3}, token{"\u5e72\u4e8b", 4, 6}, token{"\u5973\u5e72\u4e8b", 3, 6}, token{"\u6bcf\u6708", 6, 8}, token{"\u7ecf\u8fc7", 8, 10}, token{"\u4e0b\u5c5e", 10, 12}, token{"\u79d1\u5ba4", 12, 14}, token{"\u90fd", 14, 15}, token{"\u8981", 15, 16}, token{"\u4eb2\u53e3", 16, 18}, token{"\u4ea4\u4ee3", 18, 20}, token{"24", 20, 22}, token{"\u53e3", 22, 23}, token{"\u4ea4\u6362", 23, 25}, token{"\u6362\u673a", 24, 26}, token{"\u4ea4\u6362\u673a", 23, 26}, token{"\u7b49", 26, 27}, token{"\u6280\u672f", 27, 29}, token{"\u6280\u672f\u6027", 27, 30}, token{"\u5668\u4ef6", 30, 32}, token{"\u7684", 32, 33}, token{"\u5b89\u88c5", 33, 35}, token{"\u5de5\u4f5c", 35, 37}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u9700\u8981", 1, 3}, token{"\u5ec9\u79df", 3, 5}, token{"\u79df\u623f", 4, 6}, token{"\u5ec9\u79df\u623f", 3, 6}},
|
||||
[]token{token{"\u6c38\u548c", 0, 2}, token{"\u670d\u88c5", 2, 4}, token{"\u9970\u54c1", 4, 6}, token{"\u6709\u9650", 6, 8}, token{"\u516c\u53f8", 8, 10}, token{"\u6709\u9650\u516c\u53f8", 6, 10}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u7231", 1, 2}, token{"\u5317\u4eac", 2, 4}, token{"\u5929\u5b89", 4, 6}, token{"\u5929\u5b89\u95e8", 4, 7}},
|
||||
[]token{token{"abc", 0, 3}},
|
||||
[]token{token{"\u9690", 0, 1}, token{"\u53ef\u592b", 3, 5}, token{"\u9a6c\u5c14\u53ef", 1, 4}, token{"\u9a6c\u5c14\u53ef\u592b", 1, 5}},
|
||||
[]token{token{"\u96f7\u7334", 0, 2}, token{"\u662f", 2, 3}, token{"\u4e2a", 3, 4}, token{"\u597d", 4, 5}, token{"\u7f51\u7ad9", 5, 7}},
|
||||
[]token{token{"\u201c", 0, 1}, token{"Microsoft", 1, 10}, token{"\u201d", 10, 11}, token{"\u4e00\u8bcd", 11, 13}, token{"\u7531", 13, 14}, token{"\u201c", 14, 15}, token{"MICROcomputer", 15, 28}, token{"\uff08", 28, 29}, token{"\u5fae\u578b", 29, 31}, token{"\u8ba1\u7b97", 31, 33}, token{"\u7b97\u673a", 32, 34}, token{"\u8ba1\u7b97\u673a", 31, 34}, token{"\uff09", 34, 35}, token{"\u201d", 35, 36}, token{"\u548c", 36, 37}, token{"\u201c", 37, 38}, token{"SOFTware", 38, 46}, token{"\uff08", 46, 47}, token{"\u8f6f\u4ef6", 47, 49}, token{"\uff09", 49, 50}, token{"\u201d", 50, 51}, token{"\u4e24", 51, 52}, token{"\u90e8\u5206", 52, 54}, token{"\u7ec4\u6210", 54, 56}},
|
||||
[]token{token{"\u8349\u6ce5\u9a6c", 0, 3}, token{"\u548c", 3, 4}, token{"\u6b3a\u5b9e", 4, 6}, token{"\u9a6c", 6, 7}, token{"\u662f", 7, 8}, token{"\u4eca\u5e74", 8, 10}, token{"\u7684", 10, 11}, token{"\u6d41\u884c", 11, 13}, token{"\u8bcd\u6c47", 13, 15}},
|
||||
[]token{token{"\u4f0a\u85e4", 0, 2}, token{"\u6d0b\u534e\u5802", 2, 5}, token{"\u603b\u5e9c", 5, 7}, token{"\u5e97", 7, 8}},
|
||||
[]token{token{"\u4e2d\u56fd", 0, 2}, token{"\u79d1\u5b66", 2, 4}, token{"\u5b66\u9662", 3, 5}, token{"\u8ba1\u7b97", 5, 7}, token{"\u6280\u672f", 7, 9}, token{"\u7814\u7a76", 9, 11}, token{"\u79d1\u5b66\u9662", 2, 5}, token{"\u7814\u7a76\u6240", 9, 12}, token{"\u4e2d\u56fd\u79d1\u5b66\u9662\u8ba1\u7b97\u6280\u672f\u7814\u7a76\u6240", 0, 12}},
|
||||
[]token{token{"\u7f57\u5bc6\u6b27", 0, 3}, token{"\u4e0e", 3, 4}, token{"\u6731\u4e3d\u53f6", 4, 7}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u8d2d\u4e70", 1, 3}, token{"\u4e86", 3, 4}, token{"\u9053\u5177", 4, 6}, token{"\u548c", 6, 7}, token{"\u670d\u88c5", 7, 9}},
|
||||
[]token{token{"PS", 0, 2}, token{":", 2, 3}, token{" ", 3, 4}, token{"\u6211", 4, 5}, token{"\u89c9\u5f97", 5, 7}, token{"\u5f00\u6e90", 7, 9}, token{"\u6709", 9, 10}, token{"\u4e00\u4e2a", 10, 12}, token{"\u597d\u5904", 12, 14}, token{"\uff0c", 14, 15}, token{"\u5c31\u662f", 15, 17}, token{"\u80fd\u591f", 17, 19}, token{"\u6566\u4fc3", 19, 21}, token{"\u81ea\u5df1", 21, 23}, token{"\u4e0d\u65ad", 23, 25}, token{"\u6539\u8fdb", 25, 27}, token{"\u4e0d\u65ad\u6539\u8fdb", 23, 27}, token{"\uff0c", 27, 28}, token{"\u907f\u514d", 28, 30}, token{"\u655e\u5e1a", 30, 32}, token{"\u81ea\u73cd", 32, 34}},
|
||||
[]token{token{"\u6e56\u5317", 0, 2}, token{"\u6e56\u5317\u7701", 0, 3}, token{"\u77f3\u9996", 3, 5}, token{"\u77f3\u9996\u5e02", 3, 6}},
|
||||
[]token{token{"\u6e56\u5317", 0, 2}, token{"\u6e56\u5317\u7701", 0, 3}, token{"\u5341\u5830", 3, 5}, token{"\u5341\u5830\u5e02", 3, 6}},
|
||||
[]token{token{"\u7ecf\u7406", 1, 3}, token{"\u603b\u7ecf\u7406", 0, 3}, token{"\u5b8c\u6210", 3, 5}, token{"\u4e86", 5, 6}, token{"\u8fd9\u4ef6", 6, 8}, token{"\u4e8b\u60c5", 8, 10}},
|
||||
[]token{token{"\u7535\u8111", 0, 2}, token{"\u4fee\u597d", 2, 4}, token{"\u4e86", 4, 5}},
|
||||
[]token{token{"\u505a\u597d", 0, 2}, token{"\u4e86", 2, 3}, token{"\u8fd9\u4ef6", 3, 5}, token{"\u4e8b\u60c5", 5, 7}, token{"\u5c31", 7, 8}, token{"\u4e00\u4e86\u767e\u4e86", 8, 12}, token{"\u4e86", 12, 13}},
|
||||
[]token{token{"\u4eba\u4eec", 0, 2}, token{"\u5ba1\u7f8e", 2, 4}, token{"\u7684", 4, 5}, token{"\u89c2\u70b9", 5, 7}, token{"\u662f", 7, 8}, token{"\u4e0d\u540c", 8, 10}, token{"\u7684", 10, 11}},
|
||||
[]token{token{"\u6211\u4eec", 0, 2}, token{"\u4e70", 2, 3}, token{"\u4e86", 3, 4}, token{"\u4e00\u4e2a", 4, 6}, token{"\u7f8e\u7684", 6, 8}, token{"\u7a7a\u8c03", 8, 10}},
|
||||
[]token{token{"\u7ebf\u7a0b", 0, 2}, token{"\u521d\u59cb", 2, 4}, token{"\u521d\u59cb\u5316", 2, 5}, token{"\u65f6", 5, 6}, token{"\u6211\u4eec", 6, 8}, token{"\u8981", 8, 9}, token{"\u6ce8\u610f", 9, 11}},
|
||||
[]token{token{"\u4e00\u4e2a", 0, 2}, token{"\u5206\u5b50", 2, 4}, token{"\u662f", 4, 5}, token{"\u7531", 5, 6}, token{"\u597d\u591a", 6, 8}, token{"\u539f\u5b50", 8, 10}, token{"\u7ec4\u7ec7", 10, 12}, token{"\u6210", 12, 13}, token{"\u7684", 13, 14}},
|
||||
[]token{token{"\u795d", 0, 1}, token{"\u4f60", 1, 2}, token{"\u9a6c\u5230\u529f\u6210", 2, 6}},
|
||||
[]token{token{"\u4ed6", 0, 1}, token{"\u6389", 1, 2}, token{"\u8fdb", 2, 3}, token{"\u4e86", 3, 4}, token{"\u65e0\u5e95", 4, 6}, token{"\u65e0\u5e95\u6d1e", 4, 7}, token{"\u91cc", 7, 8}},
|
||||
[]token{token{"\u4e2d\u56fd", 0, 2}, token{"\u7684", 2, 3}, token{"\u9996\u90fd", 3, 5}, token{"\u662f", 5, 6}, token{"\u5317\u4eac", 6, 8}},
|
||||
[]token{token{"\u5b59\u541b\u610f", 0, 3}},
|
||||
[]token{token{"\u5916\u4ea4", 0, 2}, token{"\u5916\u4ea4\u90e8", 0, 3}, token{"\u53d1\u8a00", 3, 5}, token{"\u53d1\u8a00\u4eba", 3, 6}, token{"\u9a6c\u671d\u65ed", 6, 9}},
|
||||
[]token{token{"\u9886\u5bfc", 0, 2}, token{"\u9886\u5bfc\u4eba", 0, 3}, token{"\u4f1a\u8bae", 3, 5}, token{"\u548c", 5, 6}, token{"\u7b2c\u56db", 6, 8}, token{"\u56db\u5c4a", 7, 9}, token{"\u7b2c\u56db\u5c4a", 6, 9}, token{"\u4e1c\u4e9a", 9, 11}, token{"\u5cf0\u4f1a", 11, 13}},
|
||||
[]token{token{"\u5728", 0, 1}, token{"\u8fc7\u53bb", 1, 3}, token{"\u7684", 3, 4}, token{"\u8fd9", 4, 5}, token{"\u4e94\u5e74", 5, 7}},
|
||||
[]token{token{"\u8fd8", 0, 1}, token{"\u9700\u8981", 1, 3}, token{"\u5f88\u957f", 3, 5}, token{"\u7684", 5, 6}, token{"\u8def", 6, 7}, token{"\u8981", 7, 8}, token{"\u8d70", 8, 9}},
|
||||
[]token{token{"60", 0, 2}, token{"\u5468\u5e74", 2, 4}, token{"\u9996\u90fd", 4, 6}, token{"\u9605\u5175", 6, 8}},
|
||||
[]token{token{"\u4f60\u597d", 0, 2}, token{"\u4eba\u4eec", 2, 4}, token{"\u5ba1\u7f8e", 4, 6}, token{"\u7684", 6, 7}, token{"\u89c2\u70b9", 7, 9}, token{"\u662f", 9, 10}, token{"\u4e0d\u540c", 10, 12}, token{"\u7684", 12, 13}},
|
||||
[]token{token{"\u4e70", 0, 1}, token{"\u6c34\u679c", 1, 3}, token{"\u7136\u540e", 3, 5}, token{"\u6765", 5, 6}, token{"\u4e16\u535a", 6, 8}, token{"\u535a\u56ed", 7, 9}, token{"\u4e16\u535a\u56ed", 6, 9}},
|
||||
[]token{token{"\u4e70", 0, 1}, token{"\u6c34\u679c", 1, 3}, token{"\u7136\u540e", 3, 5}, token{"\u53bb", 5, 6}, token{"\u4e16\u535a", 6, 8}, token{"\u535a\u56ed", 7, 9}, token{"\u4e16\u535a\u56ed", 6, 9}},
|
||||
[]token{token{"\u4f46\u662f", 0, 2}, token{"\u540e\u6765", 2, 4}, token{"\u6211", 4, 5}, token{"\u624d", 5, 6}, token{"\u77e5\u9053", 6, 8}, token{"\u4f60", 8, 9}, token{"\u662f", 9, 10}, token{"\u5bf9", 10, 11}, token{"\u7684", 11, 12}},
|
||||
[]token{token{"\u5b58\u5728", 0, 2}, token{"\u5373", 2, 3}, token{"\u5408\u7406", 3, 5}},
|
||||
[]token{token{"\u7684", 0, 1}, token{"\u7684", 1, 2}, token{"\u7684", 2, 3}, token{"\u7684", 3, 4}, token{"\u7684", 4, 5}, token{"\u5728", 5, 6}, token{"\u7684", 6, 7}, token{"\u7684", 7, 8}, token{"\u7684", 8, 9}, token{"\u7684", 9, 10}, token{"\u5c31", 10, 11}, token{"\u4ee5", 11, 12}, token{"\u548c", 12, 13}, token{"\u548c", 13, 14}, token{"\u548c", 14, 15}},
|
||||
[]token{token{"I", 0, 1}, token{" ", 1, 2}, token{"love", 2, 6}, token{"\u4f60", 6, 7}, token{"\uff0c", 7, 8}, token{"\u4e0d\u4ee5", 8, 10}, token{"\u4ee5\u4e3a", 9, 11}, token{"\u4e0d\u4ee5\u4e3a\u803b", 8, 12}, token{"\uff0c", 12, 13}, token{"\u53cd", 13, 14}, token{"\u4ee5\u4e3a", 14, 16}, token{"rong", 16, 20}},
|
||||
[]token{token{"\u56e0", 0, 1}},
|
||||
[]token{},
|
||||
[]token{token{"hello", 0, 5}, token{"\u4f60\u597d", 5, 7}, token{"\u4eba\u4eec", 7, 9}, token{"\u5ba1\u7f8e", 9, 11}, token{"\u7684", 11, 12}, token{"\u89c2\u70b9", 12, 14}, token{"\u662f", 14, 15}, token{"\u4e0d\u540c", 15, 17}, token{"\u7684", 17, 18}},
|
||||
[]token{token{"\u5f88", 0, 1}, token{"\u597d", 1, 2}, token{"\u4f46", 2, 3}, token{"\u4e3b\u8981", 3, 5}, token{"\u662f", 5, 6}, token{"\u57fa\u4e8e", 6, 8}, token{"\u7f51\u9875", 8, 10}, token{"\u5f62\u5f0f", 10, 12}},
|
||||
[]token{token{"hello", 0, 5}, token{"\u4f60\u597d", 5, 7}, token{"\u4eba\u4eec", 7, 9}, token{"\u5ba1\u7f8e", 9, 11}, token{"\u7684", 11, 12}, token{"\u89c2\u70b9", 12, 14}, token{"\u662f", 14, 15}, token{"\u4e0d\u540c", 15, 17}, token{"\u7684", 17, 18}},
|
||||
[]token{token{"\u4ec0\u4e48", 1, 3}, token{"\u4e3a\u4ec0\u4e48", 0, 3}, token{"\u6211", 3, 4}, token{"\u4e0d\u80fd", 4, 6}, token{"\u62e5\u6709", 6, 8}, token{"\u60f3\u8981", 8, 10}, token{"\u7684", 10, 11}, token{"\u751f\u6d3b", 11, 13}},
|
||||
[]token{token{"\u540e\u6765", 0, 2}, token{"\u6211", 2, 3}, token{"\u624d", 3, 4}},
|
||||
[]token{token{"\u6b64\u6b21", 0, 2}, token{"\u6765", 2, 3}, token{"\u4e2d\u56fd", 3, 5}, token{"\u662f", 5, 6}, token{"\u4e3a\u4e86", 6, 8}},
|
||||
[]token{token{"\u4f7f\u7528", 0, 2}, token{"\u4e86", 2, 3}, token{"\u5b83", 3, 4}, token{"\u5c31", 4, 5}, token{"\u53ef\u4ee5", 5, 7}, token{"\u89e3\u51b3", 7, 9}, token{"\u4e00\u4e9b", 9, 11}, token{"\u95ee\u9898", 11, 13}},
|
||||
[]token{token{",", 0, 1}, token{"\u4f7f\u7528", 1, 3}, token{"\u4e86", 3, 4}, token{"\u5b83", 4, 5}, token{"\u5c31", 5, 6}, token{"\u53ef\u4ee5", 6, 8}, token{"\u89e3\u51b3", 8, 10}, token{"\u4e00\u4e9b", 10, 12}, token{"\u95ee\u9898", 12, 14}},
|
||||
[]token{token{"\u5176\u5b9e", 0, 2}, token{"\u4f7f\u7528", 2, 4}, token{"\u4e86", 4, 5}, token{"\u5b83", 5, 6}, token{"\u5c31", 6, 7}, token{"\u53ef\u4ee5", 7, 9}, token{"\u89e3\u51b3", 9, 11}, token{"\u4e00\u4e9b", 11, 13}, token{"\u95ee\u9898", 13, 15}},
|
||||
[]token{token{"\u597d\u4eba", 0, 2}, token{"\u4f7f\u7528", 2, 4}, token{"\u4e86", 4, 5}, token{"\u5b83", 5, 6}, token{"\u5c31", 6, 7}, token{"\u53ef\u4ee5", 7, 9}, token{"\u89e3\u51b3", 9, 11}, token{"\u4e00\u4e9b", 11, 13}, token{"\u95ee\u9898", 13, 15}},
|
||||
[]token{token{"\u56e0\u4e3a", 1, 3}, token{"\u662f\u56e0\u4e3a", 0, 3}, token{"\u548c", 3, 4}, token{"\u56fd\u5bb6", 4, 6}},
|
||||
[]token{token{"\u8001\u5e74", 0, 2}, token{"\u641c\u7d22", 2, 4}, token{"\u8fd8", 4, 5}, token{"\u652f\u6301", 5, 7}},
|
||||
[]token{token{"\u5e72\u8106", 0, 2}, token{"\u5c31", 2, 3}, token{"\u628a", 3, 4}, token{"\u90a3\u90e8", 4, 6}, token{"\u8499\u4eba", 6, 8}, token{"\u7684", 8, 9}, token{"\u95f2\u6cd5", 9, 11}, token{"\u7ed9", 11, 12}, token{"\u5e9f", 12, 13}, token{"\u4e86", 13, 14}, token{"\u62c9\u5012", 14, 16}, token{"\uff01", 16, 17}, token{"RT", 17, 19}, token{" ", 19, 20}, token{"@", 20, 21}, token{"laoshipukong", 21, 33}, token{" ", 33, 34}, token{":", 34, 35}, token{" ", 35, 36}, token{"27", 36, 38}, token{"\u65e5", 38, 39}, token{"\uff0c", 39, 40}, token{"\u5168\u56fd", 40, 42}, token{"\u56fd\u4eba", 41, 43}, token{"\u4eba\u5927", 42, 44}, token{"\u5e38\u59d4", 44, 46}, token{"\u59d4\u4f1a", 45, 47}, token{"\u5e38\u59d4\u4f1a", 44, 47}, token{"\u5168\u56fd\u4eba\u5927\u5e38\u59d4\u4f1a", 40, 47}, token{"\u7b2c\u4e09", 47, 49}, token{"\u4e09\u6b21", 48, 50}, token{"\u7b2c\u4e09\u6b21", 47, 50}, token{"\u5ba1\u8bae", 50, 52}, token{"\u4fb5\u6743", 52, 54}, token{"\u8d23\u4efb", 54, 56}, token{"\u8d23\u4efb\u6cd5", 54, 57}, token{"\u8349\u6848", 57, 59}, token{"\uff0c", 59, 60}, token{"\u5220\u9664", 60, 62}, token{"\u4e86", 62, 63}, token{"\u6709\u5173", 63, 65}, token{"\u533b\u7597", 65, 67}, token{"\u635f\u5bb3", 67, 69}, token{"\u8d23\u4efb", 69, 71}, token{"\u201c", 71, 72}, token{"\u4e3e\u8bc1", 72, 74}, token{"\u5012\u7f6e", 74, 76}, token{"\u201d", 76, 77}, token{"\u7684", 77, 78}, token{"\u89c4\u5b9a", 78, 80}, token{"\u3002", 80, 81}, token{"\u5728", 81, 82}, token{"\u533b\u60a3", 82, 84}, token{"\u7ea0\u7eb7", 84, 86}, token{"\u4e2d\u672c", 86, 88}, token{"\u5df2", 88, 89}, token{"\u5904\u4e8e", 89, 91}, token{"\u5f31\u52bf", 91, 93}, token{"\u5730\u4f4d", 93, 95}, token{"\u7684", 95, 96}, token{"\u6d88\u8d39", 96, 98}, token{"\u6d88\u8d39\u8005", 96, 99}, token{"\u7531\u6b64", 99, 101}, token{"\u5c06", 101, 102}, token{"\u9677\u5165", 102, 104}, token{"\u4e0d\u590d", 106, 108}, token{"\u4e07\u52ab\u4e0d\u590d", 104, 108}, token{"\u7684", 108, 109}, token{"\u5883\u5730", 109, 111}, token{"\u3002", 111, 112}, token{" ", 112, 113}},
|
||||
[]token{token{"\u5927", 0, 1}},
|
||||
[]token{},
|
||||
[]token{token{"\u4ed6", 0, 1}, token{"\u8bf4", 1, 2}, token{"\u7684", 2, 3}, token{"\u786e\u5b9e", 3, 5}, token{"\u5728\u7406", 5, 7}},
|
||||
[]token{token{"\u957f\u6625", 0, 2}, token{"\u5e02\u957f", 2, 4}, token{"\u6625\u8282", 4, 6}, token{"\u8bb2\u8bdd", 6, 8}},
|
||||
[]token{token{"\u7ed3\u5a5a", 0, 2}, token{"\u7684", 2, 3}, token{"\u548c", 3, 4}, token{"\u5c1a\u672a", 4, 6}, token{"\u7ed3\u5a5a", 6, 8}, token{"\u7684", 8, 9}},
|
||||
[]token{token{"\u7ed3\u5408", 0, 2}, token{"\u6210", 2, 3}, token{"\u5206\u5b50", 3, 5}, token{"\u65f6", 5, 6}},
|
||||
[]token{token{"\u65c5\u6e38", 0, 2}, token{"\u548c", 2, 3}, token{"\u670d\u52a1", 3, 5}, token{"\u662f", 5, 6}, token{"\u6700\u597d", 6, 8}, token{"\u7684", 8, 9}},
|
||||
[]token{token{"\u8fd9\u4ef6", 0, 2}, token{"\u4e8b\u60c5", 2, 4}, token{"\u7684\u786e", 4, 6}, token{"\u662f", 6, 7}, token{"\u6211", 7, 8}, token{"\u7684", 8, 9}, token{"\u9519", 9, 10}},
|
||||
[]token{token{"\u4f9b", 0, 1}, token{"\u5927\u5bb6", 1, 3}, token{"\u53c2\u8003", 3, 5}, token{"\u6307\u6b63", 5, 7}},
|
||||
[]token{token{"\u54c8\u5c14", 0, 2}, token{"\u54c8\u5c14\u6ee8", 0, 3}, token{"\u653f\u5e9c", 3, 5}, token{"\u516c\u5e03", 5, 7}, token{"\u584c\u6865", 7, 9}, token{"\u539f\u56e0", 9, 11}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u5728", 1, 2}, token{"\u673a\u573a", 2, 4}, token{"\u5165\u53e3", 4, 6}, token{"\u5165\u53e3\u5904", 4, 7}},
|
||||
[]token{token{"\u90a2\u6c38\u81e3", 0, 3}, token{"\u6444\u5f71", 3, 5}, token{"\u62a5\u9053", 5, 7}},
|
||||
[]token{token{"BP", 0, 2}, token{"\u795e\u7ecf", 2, 4}, token{"\u7f51\u7edc", 4, 6}, token{"\u795e\u7ecf\u7f51", 2, 5}, token{"\u795e\u7ecf\u7f51\u7edc", 2, 6}, token{"\u5982\u4f55", 6, 8}, token{"\u8bad\u7ec3", 8, 10}, token{"\u624d\u80fd", 10, 12}, token{"\u5728", 12, 13}, token{"\u5206\u7c7b", 13, 15}, token{"\u65f6", 15, 16}, token{"\u589e\u52a0", 16, 18}, token{"\u533a\u5206", 18, 20}, token{"\u5206\u5ea6", 19, 21}, token{"\u533a\u5206\u5ea6", 18, 21}, token{"\uff1f", 21, 22}},
|
||||
[]token{token{"\u5357\u4eac", 0, 2}, token{"\u4eac\u5e02", 1, 3}, token{"\u5357\u4eac\u5e02", 0, 3}, token{"\u957f\u6c5f", 3, 5}, token{"\u5927\u6865", 5, 7}, token{"\u957f\u6c5f\u5927\u6865", 3, 7}},
|
||||
[]token{token{"\u5e94", 0, 1}, token{"\u4e00\u4e9b", 1, 3}, token{"\u4f7f\u7528", 3, 5}, token{"\u7528\u8005", 4, 6}, token{"\u4f7f\u7528\u8005", 3, 6}, token{"\u7684", 6, 7}, token{"\u5efa\u8bae", 7, 9}, token{"\uff0c", 9, 10}, token{"\u4e5f", 10, 11}, token{"\u4e3a\u4e86", 11, 13}, token{"\u4fbf\u4e8e", 13, 15}, token{"\u5229\u7528", 15, 17}, token{"NiuTrans", 17, 25}, token{"\u7528\u4e8e", 25, 27}, token{"SMT", 27, 30}, token{"\u7814\u7a76", 30, 32}},
|
||||
[]token{token{"\u957f\u6625", 0, 2}, token{"\u957f\u6625\u5e02", 0, 3}, token{"\u957f\u6625", 3, 5}, token{"\u836f\u5e97", 5, 7}},
|
||||
[]token{token{"\u9093\u9896\u8d85", 0, 3}, token{"\u751f\u524d", 3, 5}, token{"\u6700", 5, 6}, token{"\u559c\u6b22", 6, 8}, token{"\u7684", 8, 9}, token{"\u8863\u670d", 9, 11}},
|
||||
[]token{token{"\u9526\u6d9b", 1, 3}, token{"\u80e1\u9526\u6d9b", 0, 3}, token{"\u662f", 3, 4}, token{"\u70ed\u7231", 4, 6}, token{"\u4e16\u754c", 6, 8}, token{"\u548c\u5e73", 8, 10}, token{"\u7684", 10, 11}, token{"\u653f\u6cbb", 11, 13}, token{"\u653f\u6cbb\u5c40", 11, 14}, token{"\u5e38\u59d4", 14, 16}},
|
||||
[]token{token{"\u7a0b\u5e8f", 0, 2}, token{"\u7a0b\u5e8f\u5458", 0, 3}, token{"\u795d", 3, 4}, token{"\u6d77\u6797", 4, 6}, token{"\u548c", 6, 7}, token{"\u6731\u4f1a\u9707", 7, 10}, token{"\u662f", 10, 11}, token{"\u5728", 11, 12}, token{"\u5b59\u5065", 12, 14}, token{"\u7684", 14, 15}, token{"\u5de6\u9762", 15, 17}, token{"\u548c", 17, 18}, token{"\u53f3\u9762", 18, 20}, token{",", 20, 21}, token{" ", 21, 22}, token{"\u8303\u51ef", 22, 24}, token{"\u5728", 24, 25}, token{"\u6700", 25, 26}, token{"\u53f3\u9762", 26, 28}, token{".", 28, 29}, token{"\u518d\u5f80", 29, 31}, token{"\u5de6", 31, 32}, token{"\u662f", 32, 33}, token{"\u674e\u677e\u6d2a", 33, 36}},
|
||||
[]token{token{"\u4e00\u6b21", 0, 2}, token{"\u4e00\u6b21\u6027", 0, 3}, token{"\u4ea4", 3, 4}, token{"\u591a\u5c11", 4, 6}, token{"\u94b1", 6, 7}},
|
||||
[]token{token{"\u4e24\u5757", 0, 2}, token{"\u4e94", 2, 3}, token{"\u4e00\u5957", 3, 5}, token{"\uff0c", 5, 6}, token{"\u4e09\u5757", 6, 8}, token{"\u516b", 8, 9}, token{"\u4e00\u65a4", 9, 11}, token{"\uff0c", 11, 12}, token{"\u56db\u5757", 12, 14}, token{"\u4e03", 14, 15}, token{"\u4e00\u672c", 15, 17}, token{"\uff0c", 17, 18}, token{"\u4e94\u5757", 18, 20}, token{"\u516d", 20, 21}, token{"\u4e00\u6761", 21, 23}},
|
||||
[]token{token{"\u5c0f", 0, 1}, token{"\u548c\u5c1a", 1, 3}, token{"\u7559", 3, 4}, token{"\u4e86", 4, 5}, token{"\u4e00\u4e2a", 5, 7}, token{"\u50cf", 7, 8}, token{"\u5927", 8, 9}, token{"\u548c\u5c1a", 9, 11}, token{"\u4e00\u6837", 11, 13}, token{"\u7684", 13, 14}, token{"\u548c\u5c1a", 14, 16}, token{"\u548c\u5c1a\u5934", 14, 17}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u662f", 1, 2}, token{"\u4e2d\u534e", 2, 4}, token{"\u534e\u4eba", 3, 5}, token{"\u4eba\u6c11", 4, 6}, token{"\u5171\u548c", 6, 8}, token{"\u5171\u548c\u56fd", 6, 9}, token{"\u4e2d\u534e\u4eba\u6c11\u5171\u548c\u56fd", 2, 9}, token{"\u516c\u6c11", 9, 11}, token{";", 11, 12}, token{"\u6211", 12, 13}, token{"\u7238\u7238", 13, 15}, token{"\u662f", 15, 16}, token{"\u5171\u548c", 16, 18}, token{"\u5171\u548c\u515a", 16, 19}, token{"\u515a\u5458", 19, 21}, token{";", 21, 22}, token{" ", 22, 23}, token{"\u5730\u94c1", 23, 25}, token{"\u548c\u5e73", 25, 27}, token{"\u548c\u5e73\u95e8", 25, 28}, token{"\u7ad9", 28, 29}},
|
||||
[]token{token{"\u5f20\u6653\u6885", 0, 3}, token{"\u53bb", 3, 4}, token{"\u4eba\u6c11", 4, 6}, token{"\u533b\u9662", 6, 8}, token{"\u505a", 8, 9}, token{"\u4e86", 9, 10}, token{"\u4e2a", 10, 11}, token{"B\u8d85", 11, 13}, token{"\u7136\u540e", 13, 15}, token{"\u53bb", 15, 16}, token{"\u4e70", 16, 17}, token{"\u4e86", 17, 18}, token{"\u4ef6", 18, 19}, token{"T\u6064", 19, 21}},
|
||||
[]token{token{"AT&T", 0, 4}, token{"\u662f", 4, 5}, token{"\u4e00\u4ef6", 5, 7}, token{"\u4e0d\u9519", 7, 9}, token{"\u7684", 9, 10}, token{"\u516c\u53f8", 10, 12}, token{"\uff0c", 12, 13}, token{"\u7ed9", 13, 14}, token{"\u4f60", 14, 15}, token{"\u53d1", 15, 16}, token{"offer", 16, 21}, token{"\u4e86", 21, 22}, token{"\u5417", 22, 23}, token{"\uff1f", 23, 24}},
|
||||
[]token{token{"C++", 0, 3}, token{"\u548c", 3, 4}, token{"c#", 4, 6}, token{"\u662f", 6, 7}, token{"\u4ec0\u4e48", 7, 9}, token{"\u5173\u7cfb", 9, 11}, token{"\uff1f", 11, 12}, token{"11", 12, 14}, token{"+", 14, 15}, token{"122", 15, 18}, token{"=", 18, 19}, token{"133", 19, 22}, token{"\uff0c", 22, 23}, token{"\u662f", 23, 24}, token{"\u5417", 24, 25}, token{"\uff1f", 25, 26}, token{"PI", 26, 28}, token{"=", 28, 29}, token{"3.14159", 29, 36}},
|
||||
[]token{token{"\u4f60", 0, 1}, token{"\u8ba4\u8bc6", 1, 3}, token{"\u90a3\u4e2a", 3, 5}, token{"\u548c", 5, 6}, token{"\u4e3b\u5e2d", 6, 8}, token{"\u63e1\u624b", 8, 10}, token{"\u7684", 10, 11}, token{"\u7684\u54e5", 11, 13}, token{"\u5417", 13, 14}, token{"\uff1f", 14, 15}, token{"\u4ed6\u5f00", 15, 17}, token{"\u4e00\u8f86", 17, 19}, token{"\u9ed1\u8272", 19, 21}, token{"\u7684\u58eb", 21, 23}, token{"\u3002", 23, 24}},
|
||||
[]token{token{"\u67aa\u6746", 0, 2}, token{"\u6746\u5b50", 1, 3}, token{"\u67aa\u6746\u5b50", 0, 3}, token{"\u4e2d", 3, 4}, token{"\u51fa", 4, 5}, token{"\u653f\u6743", 5, 7}},
|
||||
[]token{token{"\u5f20\u4e09\u98ce", 0, 3}, token{"\u540c\u5b66", 3, 5}, token{"\u8d70\u4e0a", 5, 7}, token{"\u4e86", 7, 8}, token{"\u5f52\u8def", 9, 11}, token{"\u4e0d\u5f52\u8def", 8, 11}},
|
||||
[]token{token{"\u963fQ", 0, 2}, token{"\u8170\u95f4", 2, 4}, token{"\u6302", 4, 5}, token{"\u7740", 5, 6}, token{"BB\u673a", 6, 9}, token{"\u624b\u91cc", 9, 11}, token{"\u62ff", 11, 12}, token{"\u7740", 12, 13}, token{"\u5927\u54e5", 13, 15}, token{"\u5927\u54e5\u5927", 13, 16}, token{"\uff0c", 16, 17}, token{"\u8bf4", 17, 18}, token{"\uff1a", 18, 19}, token{"\u6211", 19, 20}, token{"\u4e00\u822c", 20, 22}, token{"\u5403\u996d", 22, 24}, token{"\u4e0d", 24, 25}, token{"AA\u5236", 25, 28}, token{"\u7684", 28, 29}, token{"\u3002", 29, 30}},
|
||||
[]token{token{"\u5728", 0, 1}, token{"1\u53f7\u5e97", 1, 4}, token{"\u80fd", 4, 5}, token{"\u4e70", 5, 6}, token{"\u5230", 6, 7}, token{"\u5c0fS", 7, 9}, token{"\u548c", 9, 10}, token{"\u5927S", 10, 12}, token{"\u516b\u5366", 12, 14}, token{"\u7684", 14, 15}, token{"\u4e66", 15, 16}, token{"\u3002", 16, 17}},
|
||||
}
|
||||
noHmmResult = [][]token{
|
||||
[]token{token{"\u8fd9", 0, 1}, token{"\u662f", 1, 2}, token{"\u4e00\u4e2a", 2, 4}, token{"\u4f38\u624b\u4e0d\u89c1\u4e94\u6307", 4, 10}, token{"\u7684", 10, 11}, token{"\u9ed1\u591c", 11, 13}, token{"\u3002", 13, 14}, token{"\u6211", 14, 15}, token{"\u53eb", 15, 16}, token{"\u5b59\u609f\u7a7a", 16, 19}, token{"\uff0c", 19, 20}, token{"\u6211", 20, 21}, token{"\u7231", 21, 22}, token{"\u5317\u4eac", 22, 24}, token{"\uff0c", 24, 25}, token{"\u6211", 25, 26}, token{"\u7231", 26, 27}, token{"Python", 27, 33}, token{"\u548c", 33, 34}, token{"C++", 34, 37}, token{"\u3002", 37, 38}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u4e0d", 1, 2}, token{"\u559c\u6b22", 2, 4}, token{"\u65e5\u672c", 4, 6}, token{"\u548c\u670d", 6, 8}, token{"\u3002", 8, 9}},
|
||||
[]token{token{"\u96f7\u7334", 0, 2}, token{"\u56de\u5f52", 2, 4}, token{"\u4eba\u95f4", 4, 6}, token{"\u3002", 6, 7}},
|
||||
[]token{token{"\u5de5\u4fe1\u5904", 0, 3}, token{"\u5973\u5e72\u4e8b", 3, 6}, token{"\u6bcf\u6708", 6, 8}, token{"\u7ecf\u8fc7", 8, 10}, token{"\u4e0b\u5c5e", 10, 12}, token{"\u79d1\u5ba4", 12, 14}, token{"\u90fd", 14, 15}, token{"\u8981", 15, 16}, token{"\u4eb2\u53e3", 16, 18}, token{"\u4ea4\u4ee3", 18, 20}, token{"24", 20, 22}, token{"\u53e3", 22, 23}, token{"\u4ea4\u6362\u673a", 23, 26}, token{"\u7b49", 26, 27}, token{"\u6280\u672f\u6027", 27, 30}, token{"\u5668\u4ef6", 30, 32}, token{"\u7684", 32, 33}, token{"\u5b89\u88c5", 33, 35}, token{"\u5de5\u4f5c", 35, 37}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u9700\u8981", 1, 3}, token{"\u5ec9\u79df\u623f", 3, 6}},
|
||||
[]token{token{"\u6c38\u548c", 0, 2}, token{"\u670d\u88c5", 2, 4}, token{"\u9970\u54c1", 4, 6}, token{"\u6709\u9650\u516c\u53f8", 6, 10}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u7231", 1, 2}, token{"\u5317\u4eac", 2, 4}, token{"\u5929\u5b89\u95e8", 4, 7}},
|
||||
[]token{token{"abc", 0, 3}},
|
||||
[]token{token{"\u9690", 0, 1}, token{"\u9a6c\u5c14\u53ef\u592b", 1, 5}},
|
||||
[]token{token{"\u96f7\u7334", 0, 2}, token{"\u662f", 2, 3}, token{"\u4e2a", 3, 4}, token{"\u597d", 4, 5}, token{"\u7f51\u7ad9", 5, 7}},
|
||||
[]token{token{"\u201c", 0, 1}, token{"Microsoft", 1, 10}, token{"\u201d", 10, 11}, token{"\u4e00", 11, 12}, token{"\u8bcd", 12, 13}, token{"\u7531", 13, 14}, token{"\u201c", 14, 15}, token{"MICROcomputer", 15, 28}, token{"\uff08", 28, 29}, token{"\u5fae\u578b", 29, 31}, token{"\u8ba1\u7b97\u673a", 31, 34}, token{"\uff09", 34, 35}, token{"\u201d", 35, 36}, token{"\u548c", 36, 37}, token{"\u201c", 37, 38}, token{"SOFTware", 38, 46}, token{"\uff08", 46, 47}, token{"\u8f6f\u4ef6", 47, 49}, token{"\uff09", 49, 50}, token{"\u201d", 50, 51}, token{"\u4e24", 51, 52}, token{"\u90e8\u5206", 52, 54}, token{"\u7ec4\u6210", 54, 56}},
|
||||
[]token{token{"\u8349\u6ce5\u9a6c", 0, 3}, token{"\u548c", 3, 4}, token{"\u6b3a", 4, 5}, token{"\u5b9e", 5, 6}, token{"\u9a6c", 6, 7}, token{"\u662f", 7, 8}, token{"\u4eca\u5e74", 8, 10}, token{"\u7684", 10, 11}, token{"\u6d41\u884c", 11, 13}, token{"\u8bcd\u6c47", 13, 15}},
|
||||
[]token{token{"\u4f0a", 0, 1}, token{"\u85e4", 1, 2}, token{"\u6d0b\u534e\u5802", 2, 5}, token{"\u603b\u5e9c", 5, 7}, token{"\u5e97", 7, 8}},
|
||||
[]token{token{"\u4e2d\u56fd\u79d1\u5b66\u9662\u8ba1\u7b97\u6280\u672f\u7814\u7a76\u6240", 0, 12}},
|
||||
[]token{token{"\u7f57\u5bc6\u6b27", 0, 3}, token{"\u4e0e", 3, 4}, token{"\u6731\u4e3d\u53f6", 4, 7}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u8d2d\u4e70", 1, 3}, token{"\u4e86", 3, 4}, token{"\u9053\u5177", 4, 6}, token{"\u548c", 6, 7}, token{"\u670d\u88c5", 7, 9}},
|
||||
[]token{token{"PS", 0, 2}, token{":", 2, 3}, token{" ", 3, 4}, token{"\u6211", 4, 5}, token{"\u89c9\u5f97", 5, 7}, token{"\u5f00\u6e90", 7, 9}, token{"\u6709", 9, 10}, token{"\u4e00\u4e2a", 10, 12}, token{"\u597d\u5904", 12, 14}, token{"\uff0c", 14, 15}, token{"\u5c31\u662f", 15, 17}, token{"\u80fd\u591f", 17, 19}, token{"\u6566\u4fc3", 19, 21}, token{"\u81ea\u5df1", 21, 23}, token{"\u4e0d\u65ad\u6539\u8fdb", 23, 27}, token{"\uff0c", 27, 28}, token{"\u907f\u514d", 28, 30}, token{"\u655e", 30, 31}, token{"\u5e1a", 31, 32}, token{"\u81ea\u73cd", 32, 34}},
|
||||
[]token{token{"\u6e56\u5317\u7701", 0, 3}, token{"\u77f3\u9996\u5e02", 3, 6}},
|
||||
[]token{token{"\u6e56\u5317\u7701", 0, 3}, token{"\u5341\u5830\u5e02", 3, 6}},
|
||||
[]token{token{"\u603b\u7ecf\u7406", 0, 3}, token{"\u5b8c\u6210", 3, 5}, token{"\u4e86", 5, 6}, token{"\u8fd9\u4ef6", 6, 8}, token{"\u4e8b\u60c5", 8, 10}},
|
||||
[]token{token{"\u7535\u8111", 0, 2}, token{"\u4fee\u597d", 2, 4}, token{"\u4e86", 4, 5}},
|
||||
[]token{token{"\u505a\u597d", 0, 2}, token{"\u4e86", 2, 3}, token{"\u8fd9\u4ef6", 3, 5}, token{"\u4e8b\u60c5", 5, 7}, token{"\u5c31", 7, 8}, token{"\u4e00\u4e86\u767e\u4e86", 8, 12}, token{"\u4e86", 12, 13}},
|
||||
[]token{token{"\u4eba\u4eec", 0, 2}, token{"\u5ba1\u7f8e", 2, 4}, token{"\u7684", 4, 5}, token{"\u89c2\u70b9", 5, 7}, token{"\u662f", 7, 8}, token{"\u4e0d\u540c", 8, 10}, token{"\u7684", 10, 11}},
|
||||
[]token{token{"\u6211\u4eec", 0, 2}, token{"\u4e70", 2, 3}, token{"\u4e86", 3, 4}, token{"\u4e00\u4e2a", 4, 6}, token{"\u7f8e\u7684", 6, 8}, token{"\u7a7a\u8c03", 8, 10}},
|
||||
[]token{token{"\u7ebf\u7a0b", 0, 2}, token{"\u521d\u59cb\u5316", 2, 5}, token{"\u65f6", 5, 6}, token{"\u6211\u4eec", 6, 8}, token{"\u8981", 8, 9}, token{"\u6ce8\u610f", 9, 11}},
|
||||
[]token{token{"\u4e00\u4e2a", 0, 2}, token{"\u5206\u5b50", 2, 4}, token{"\u662f", 4, 5}, token{"\u7531", 5, 6}, token{"\u597d\u591a", 6, 8}, token{"\u539f\u5b50", 8, 10}, token{"\u7ec4\u7ec7", 10, 12}, token{"\u6210", 12, 13}, token{"\u7684", 13, 14}},
|
||||
[]token{token{"\u795d", 0, 1}, token{"\u4f60", 1, 2}, token{"\u9a6c\u5230\u529f\u6210", 2, 6}},
|
||||
[]token{token{"\u4ed6", 0, 1}, token{"\u6389", 1, 2}, token{"\u8fdb", 2, 3}, token{"\u4e86", 3, 4}, token{"\u65e0\u5e95\u6d1e", 4, 7}, token{"\u91cc", 7, 8}},
|
||||
[]token{token{"\u4e2d\u56fd", 0, 2}, token{"\u7684", 2, 3}, token{"\u9996\u90fd", 3, 5}, token{"\u662f", 5, 6}, token{"\u5317\u4eac", 6, 8}},
|
||||
[]token{token{"\u5b59", 0, 1}, token{"\u541b", 1, 2}, token{"\u610f", 2, 3}},
|
||||
[]token{token{"\u5916\u4ea4\u90e8", 0, 3}, token{"\u53d1\u8a00\u4eba", 3, 6}, token{"\u9a6c\u671d\u65ed", 6, 9}},
|
||||
[]token{token{"\u9886\u5bfc\u4eba", 0, 3}, token{"\u4f1a\u8bae", 3, 5}, token{"\u548c", 5, 6}, token{"\u7b2c\u56db\u5c4a", 6, 9}, token{"\u4e1c\u4e9a", 9, 11}, token{"\u5cf0\u4f1a", 11, 13}},
|
||||
[]token{token{"\u5728", 0, 1}, token{"\u8fc7\u53bb", 1, 3}, token{"\u7684", 3, 4}, token{"\u8fd9", 4, 5}, token{"\u4e94\u5e74", 5, 7}},
|
||||
[]token{token{"\u8fd8", 0, 1}, token{"\u9700\u8981", 1, 3}, token{"\u5f88", 3, 4}, token{"\u957f", 4, 5}, token{"\u7684", 5, 6}, token{"\u8def", 6, 7}, token{"\u8981", 7, 8}, token{"\u8d70", 8, 9}},
|
||||
[]token{token{"60", 0, 2}, token{"\u5468\u5e74", 2, 4}, token{"\u9996\u90fd", 4, 6}, token{"\u9605\u5175", 6, 8}},
|
||||
[]token{token{"\u4f60\u597d", 0, 2}, token{"\u4eba\u4eec", 2, 4}, token{"\u5ba1\u7f8e", 4, 6}, token{"\u7684", 6, 7}, token{"\u89c2\u70b9", 7, 9}, token{"\u662f", 9, 10}, token{"\u4e0d\u540c", 10, 12}, token{"\u7684", 12, 13}},
|
||||
[]token{token{"\u4e70", 0, 1}, token{"\u6c34\u679c", 1, 3}, token{"\u7136\u540e", 3, 5}, token{"\u6765", 5, 6}, token{"\u4e16\u535a\u56ed", 6, 9}},
|
||||
[]token{token{"\u4e70", 0, 1}, token{"\u6c34\u679c", 1, 3}, token{"\u7136\u540e", 3, 5}, token{"\u53bb", 5, 6}, token{"\u4e16\u535a\u56ed", 6, 9}},
|
||||
[]token{token{"\u4f46\u662f", 0, 2}, token{"\u540e\u6765", 2, 4}, token{"\u6211", 4, 5}, token{"\u624d", 5, 6}, token{"\u77e5\u9053", 6, 8}, token{"\u4f60", 8, 9}, token{"\u662f", 9, 10}, token{"\u5bf9", 10, 11}, token{"\u7684", 11, 12}},
|
||||
[]token{token{"\u5b58\u5728", 0, 2}, token{"\u5373", 2, 3}, token{"\u5408\u7406", 3, 5}},
|
||||
[]token{token{"\u7684", 0, 1}, token{"\u7684", 1, 2}, token{"\u7684", 2, 3}, token{"\u7684", 3, 4}, token{"\u7684", 4, 5}, token{"\u5728", 5, 6}, token{"\u7684", 6, 7}, token{"\u7684", 7, 8}, token{"\u7684", 8, 9}, token{"\u7684", 9, 10}, token{"\u5c31", 10, 11}, token{"\u4ee5", 11, 12}, token{"\u548c", 12, 13}, token{"\u548c", 13, 14}, token{"\u548c", 14, 15}},
|
||||
[]token{token{"I", 0, 1}, token{" ", 1, 2}, token{"love", 2, 6}, token{"\u4f60", 6, 7}, token{"\uff0c", 7, 8}, token{"\u4e0d\u4ee5\u4e3a\u803b", 8, 12}, token{"\uff0c", 12, 13}, token{"\u53cd", 13, 14}, token{"\u4ee5\u4e3a", 14, 16}, token{"rong", 16, 20}},
|
||||
[]token{token{"\u56e0", 0, 1}},
|
||||
[]token{},
|
||||
[]token{token{"hello", 0, 5}, token{"\u4f60\u597d", 5, 7}, token{"\u4eba\u4eec", 7, 9}, token{"\u5ba1\u7f8e", 9, 11}, token{"\u7684", 11, 12}, token{"\u89c2\u70b9", 12, 14}, token{"\u662f", 14, 15}, token{"\u4e0d\u540c", 15, 17}, token{"\u7684", 17, 18}},
|
||||
[]token{token{"\u5f88", 0, 1}, token{"\u597d", 1, 2}, token{"\u4f46", 2, 3}, token{"\u4e3b\u8981", 3, 5}, token{"\u662f", 5, 6}, token{"\u57fa\u4e8e", 6, 8}, token{"\u7f51\u9875", 8, 10}, token{"\u5f62\u5f0f", 10, 12}},
|
||||
[]token{token{"hello", 0, 5}, token{"\u4f60\u597d", 5, 7}, token{"\u4eba\u4eec", 7, 9}, token{"\u5ba1\u7f8e", 9, 11}, token{"\u7684", 11, 12}, token{"\u89c2\u70b9", 12, 14}, token{"\u662f", 14, 15}, token{"\u4e0d\u540c", 15, 17}, token{"\u7684", 17, 18}},
|
||||
[]token{token{"\u4e3a\u4ec0\u4e48", 0, 3}, token{"\u6211", 3, 4}, token{"\u4e0d\u80fd", 4, 6}, token{"\u62e5\u6709", 6, 8}, token{"\u60f3\u8981", 8, 10}, token{"\u7684", 10, 11}, token{"\u751f\u6d3b", 11, 13}},
|
||||
[]token{token{"\u540e\u6765", 0, 2}, token{"\u6211", 2, 3}, token{"\u624d", 3, 4}},
|
||||
[]token{token{"\u6b64\u6b21", 0, 2}, token{"\u6765", 2, 3}, token{"\u4e2d\u56fd", 3, 5}, token{"\u662f", 5, 6}, token{"\u4e3a\u4e86", 6, 8}},
|
||||
[]token{token{"\u4f7f\u7528", 0, 2}, token{"\u4e86", 2, 3}, token{"\u5b83", 3, 4}, token{"\u5c31", 4, 5}, token{"\u53ef\u4ee5", 5, 7}, token{"\u89e3\u51b3", 7, 9}, token{"\u4e00\u4e9b", 9, 11}, token{"\u95ee\u9898", 11, 13}},
|
||||
[]token{token{",", 0, 1}, token{"\u4f7f\u7528", 1, 3}, token{"\u4e86", 3, 4}, token{"\u5b83", 4, 5}, token{"\u5c31", 5, 6}, token{"\u53ef\u4ee5", 6, 8}, token{"\u89e3\u51b3", 8, 10}, token{"\u4e00\u4e9b", 10, 12}, token{"\u95ee\u9898", 12, 14}},
|
||||
[]token{token{"\u5176\u5b9e", 0, 2}, token{"\u4f7f\u7528", 2, 4}, token{"\u4e86", 4, 5}, token{"\u5b83", 5, 6}, token{"\u5c31", 6, 7}, token{"\u53ef\u4ee5", 7, 9}, token{"\u89e3\u51b3", 9, 11}, token{"\u4e00\u4e9b", 11, 13}, token{"\u95ee\u9898", 13, 15}},
|
||||
[]token{token{"\u597d\u4eba", 0, 2}, token{"\u4f7f\u7528", 2, 4}, token{"\u4e86", 4, 5}, token{"\u5b83", 5, 6}, token{"\u5c31", 6, 7}, token{"\u53ef\u4ee5", 7, 9}, token{"\u89e3\u51b3", 9, 11}, token{"\u4e00\u4e9b", 11, 13}, token{"\u95ee\u9898", 13, 15}},
|
||||
[]token{token{"\u662f\u56e0\u4e3a", 0, 3}, token{"\u548c", 3, 4}, token{"\u56fd\u5bb6", 4, 6}},
|
||||
[]token{token{"\u8001\u5e74", 0, 2}, token{"\u641c\u7d22", 2, 4}, token{"\u8fd8", 4, 5}, token{"\u652f\u6301", 5, 7}},
|
||||
[]token{token{"\u5e72\u8106", 0, 2}, token{"\u5c31", 2, 3}, token{"\u628a", 3, 4}, token{"\u90a3", 4, 5}, token{"\u90e8", 5, 6}, token{"\u8499", 6, 7}, token{"\u4eba", 7, 8}, token{"\u7684", 8, 9}, token{"\u95f2", 9, 10}, token{"\u6cd5", 10, 11}, token{"\u7ed9", 11, 12}, token{"\u5e9f", 12, 13}, token{"\u4e86", 13, 14}, token{"\u62c9\u5012", 14, 16}, token{"\uff01", 16, 17}, token{"RT", 17, 19}, token{" ", 19, 20}, token{"@", 20, 21}, token{"laoshipukong", 21, 33}, token{" ", 33, 34}, token{":", 34, 35}, token{" ", 35, 36}, token{"27", 36, 38}, token{"\u65e5", 38, 39}, token{"\uff0c", 39, 40}, token{"\u5168\u56fd\u4eba\u5927\u5e38\u59d4\u4f1a", 40, 47}, token{"\u7b2c\u4e09\u6b21", 47, 50}, token{"\u5ba1\u8bae", 50, 52}, token{"\u4fb5\u6743", 52, 54}, token{"\u8d23\u4efb\u6cd5", 54, 57}, token{"\u8349\u6848", 57, 59}, token{"\uff0c", 59, 60}, token{"\u5220\u9664", 60, 62}, token{"\u4e86", 62, 63}, token{"\u6709\u5173", 63, 65}, token{"\u533b\u7597", 65, 67}, token{"\u635f\u5bb3", 67, 69}, token{"\u8d23\u4efb", 69, 71}, token{"\u201c", 71, 72}, token{"\u4e3e\u8bc1", 72, 74}, token{"\u5012\u7f6e", 74, 76}, token{"\u201d", 76, 77}, token{"\u7684", 77, 78}, token{"\u89c4\u5b9a", 78, 80}, token{"\u3002", 80, 81}, token{"\u5728", 81, 82}, token{"\u533b\u60a3", 82, 84}, token{"\u7ea0\u7eb7", 84, 86}, token{"\u4e2d", 86, 87}, token{"\u672c", 87, 88}, token{"\u5df2", 88, 89}, token{"\u5904\u4e8e", 89, 91}, token{"\u5f31\u52bf", 91, 93}, token{"\u5730\u4f4d", 93, 95}, token{"\u7684", 95, 96}, token{"\u6d88\u8d39\u8005", 96, 99}, token{"\u7531\u6b64", 99, 101}, token{"\u5c06", 101, 102}, token{"\u9677\u5165", 102, 104}, token{"\u4e07\u52ab\u4e0d\u590d", 104, 108}, token{"\u7684", 108, 109}, token{"\u5883\u5730", 109, 111}, token{"\u3002", 111, 112}, token{" ", 112, 113}},
|
||||
[]token{token{"\u5927", 0, 1}},
|
||||
[]token{},
|
||||
[]token{token{"\u4ed6", 0, 1}, token{"\u8bf4", 1, 2}, token{"\u7684", 2, 3}, token{"\u786e\u5b9e", 3, 5}, token{"\u5728", 5, 6}, token{"\u7406", 6, 7}},
|
||||
[]token{token{"\u957f\u6625", 0, 2}, token{"\u5e02\u957f", 2, 4}, token{"\u6625\u8282", 4, 6}, token{"\u8bb2\u8bdd", 6, 8}},
|
||||
[]token{token{"\u7ed3\u5a5a", 0, 2}, token{"\u7684", 2, 3}, token{"\u548c", 3, 4}, token{"\u5c1a\u672a", 4, 6}, token{"\u7ed3\u5a5a", 6, 8}, token{"\u7684", 8, 9}},
|
||||
[]token{token{"\u7ed3\u5408", 0, 2}, token{"\u6210", 2, 3}, token{"\u5206\u5b50", 3, 5}, token{"\u65f6", 5, 6}},
|
||||
[]token{token{"\u65c5\u6e38", 0, 2}, token{"\u548c", 2, 3}, token{"\u670d\u52a1", 3, 5}, token{"\u662f", 5, 6}, token{"\u6700\u597d", 6, 8}, token{"\u7684", 8, 9}},
|
||||
[]token{token{"\u8fd9\u4ef6", 0, 2}, token{"\u4e8b\u60c5", 2, 4}, token{"\u7684\u786e", 4, 6}, token{"\u662f", 6, 7}, token{"\u6211", 7, 8}, token{"\u7684", 8, 9}, token{"\u9519", 9, 10}},
|
||||
[]token{token{"\u4f9b", 0, 1}, token{"\u5927\u5bb6", 1, 3}, token{"\u53c2\u8003", 3, 5}, token{"\u6307\u6b63", 5, 7}},
|
||||
[]token{token{"\u54c8\u5c14\u6ee8", 0, 3}, token{"\u653f\u5e9c", 3, 5}, token{"\u516c\u5e03", 5, 7}, token{"\u584c", 7, 8}, token{"\u6865", 8, 9}, token{"\u539f\u56e0", 9, 11}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u5728", 1, 2}, token{"\u673a\u573a", 2, 4}, token{"\u5165\u53e3\u5904", 4, 7}},
|
||||
[]token{token{"\u90a2", 0, 1}, token{"\u6c38", 1, 2}, token{"\u81e3", 2, 3}, token{"\u6444\u5f71", 3, 5}, token{"\u62a5\u9053", 5, 7}},
|
||||
[]token{token{"BP", 0, 2}, token{"\u795e\u7ecf\u7f51\u7edc", 2, 6}, token{"\u5982\u4f55", 6, 8}, token{"\u8bad\u7ec3", 8, 10}, token{"\u624d\u80fd", 10, 12}, token{"\u5728", 12, 13}, token{"\u5206\u7c7b", 13, 15}, token{"\u65f6", 15, 16}, token{"\u589e\u52a0", 16, 18}, token{"\u533a\u5206\u5ea6", 18, 21}, token{"\uff1f", 21, 22}},
|
||||
[]token{token{"\u5357\u4eac\u5e02", 0, 3}, token{"\u957f\u6c5f\u5927\u6865", 3, 7}},
|
||||
[]token{token{"\u5e94", 0, 1}, token{"\u4e00\u4e9b", 1, 3}, token{"\u4f7f\u7528\u8005", 3, 6}, token{"\u7684", 6, 7}, token{"\u5efa\u8bae", 7, 9}, token{"\uff0c", 9, 10}, token{"\u4e5f", 10, 11}, token{"\u4e3a\u4e86", 11, 13}, token{"\u4fbf\u4e8e", 13, 15}, token{"\u5229\u7528", 15, 17}, token{"NiuTrans", 17, 25}, token{"\u7528\u4e8e", 25, 27}, token{"SMT", 27, 30}, token{"\u7814\u7a76", 30, 32}},
|
||||
[]token{token{"\u957f\u6625\u5e02", 0, 3}, token{"\u957f\u6625", 3, 5}, token{"\u836f\u5e97", 5, 7}},
|
||||
[]token{token{"\u9093\u9896\u8d85", 0, 3}, token{"\u751f\u524d", 3, 5}, token{"\u6700", 5, 6}, token{"\u559c\u6b22", 6, 8}, token{"\u7684", 8, 9}, token{"\u8863\u670d", 9, 11}},
|
||||
[]token{token{"\u80e1\u9526\u6d9b", 0, 3}, token{"\u662f", 3, 4}, token{"\u70ed\u7231", 4, 6}, token{"\u4e16\u754c", 6, 8}, token{"\u548c\u5e73", 8, 10}, token{"\u7684", 10, 11}, token{"\u653f\u6cbb\u5c40", 11, 14}, token{"\u5e38\u59d4", 14, 16}},
|
||||
[]token{token{"\u7a0b\u5e8f\u5458", 0, 3}, token{"\u795d", 3, 4}, token{"\u6d77\u6797", 4, 6}, token{"\u548c", 6, 7}, token{"\u6731", 7, 8}, token{"\u4f1a", 8, 9}, token{"\u9707", 9, 10}, token{"\u662f", 10, 11}, token{"\u5728", 11, 12}, token{"\u5b59", 12, 13}, token{"\u5065", 13, 14}, token{"\u7684", 14, 15}, token{"\u5de6\u9762", 15, 17}, token{"\u548c", 17, 18}, token{"\u53f3\u9762", 18, 20}, token{",", 20, 21}, token{" ", 21, 22}, token{"\u8303", 22, 23}, token{"\u51ef", 23, 24}, token{"\u5728", 24, 25}, token{"\u6700", 25, 26}, token{"\u53f3\u9762", 26, 28}, token{".", 28, 29}, token{"\u518d", 29, 30}, token{"\u5f80", 30, 31}, token{"\u5de6", 31, 32}, token{"\u662f", 32, 33}, token{"\u674e", 33, 34}, token{"\u677e", 34, 35}, token{"\u6d2a", 35, 36}},
|
||||
[]token{token{"\u4e00\u6b21\u6027", 0, 3}, token{"\u4ea4", 3, 4}, token{"\u591a\u5c11", 4, 6}, token{"\u94b1", 6, 7}},
|
||||
[]token{token{"\u4e24\u5757", 0, 2}, token{"\u4e94", 2, 3}, token{"\u4e00\u5957", 3, 5}, token{"\uff0c", 5, 6}, token{"\u4e09\u5757", 6, 8}, token{"\u516b", 8, 9}, token{"\u4e00\u65a4", 9, 11}, token{"\uff0c", 11, 12}, token{"\u56db\u5757", 12, 14}, token{"\u4e03", 14, 15}, token{"\u4e00\u672c", 15, 17}, token{"\uff0c", 17, 18}, token{"\u4e94\u5757", 18, 20}, token{"\u516d", 20, 21}, token{"\u4e00\u6761", 21, 23}},
|
||||
[]token{token{"\u5c0f", 0, 1}, token{"\u548c\u5c1a", 1, 3}, token{"\u7559", 3, 4}, token{"\u4e86", 4, 5}, token{"\u4e00\u4e2a", 5, 7}, token{"\u50cf", 7, 8}, token{"\u5927", 8, 9}, token{"\u548c\u5c1a", 9, 11}, token{"\u4e00\u6837", 11, 13}, token{"\u7684", 13, 14}, token{"\u548c\u5c1a\u5934", 14, 17}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u662f", 1, 2}, token{"\u4e2d\u534e\u4eba\u6c11\u5171\u548c\u56fd", 2, 9}, token{"\u516c\u6c11", 9, 11}, token{";", 11, 12}, token{"\u6211", 12, 13}, token{"\u7238\u7238", 13, 15}, token{"\u662f", 15, 16}, token{"\u5171\u548c\u515a", 16, 19}, token{"\u515a\u5458", 19, 21}, token{";", 21, 22}, token{" ", 22, 23}, token{"\u5730\u94c1", 23, 25}, token{"\u548c\u5e73\u95e8", 25, 28}, token{"\u7ad9", 28, 29}},
|
||||
[]token{token{"\u5f20\u6653\u6885", 0, 3}, token{"\u53bb", 3, 4}, token{"\u4eba\u6c11", 4, 6}, token{"\u533b\u9662", 6, 8}, token{"\u505a", 8, 9}, token{"\u4e86", 9, 10}, token{"\u4e2a", 10, 11}, token{"B\u8d85", 11, 13}, token{"\u7136\u540e", 13, 15}, token{"\u53bb", 15, 16}, token{"\u4e70", 16, 17}, token{"\u4e86", 17, 18}, token{"\u4ef6", 18, 19}, token{"T\u6064", 19, 21}},
|
||||
[]token{token{"AT&T", 0, 4}, token{"\u662f", 4, 5}, token{"\u4e00\u4ef6", 5, 7}, token{"\u4e0d\u9519", 7, 9}, token{"\u7684", 9, 10}, token{"\u516c\u53f8", 10, 12}, token{"\uff0c", 12, 13}, token{"\u7ed9", 13, 14}, token{"\u4f60", 14, 15}, token{"\u53d1", 15, 16}, token{"offer", 16, 21}, token{"\u4e86", 21, 22}, token{"\u5417", 22, 23}, token{"\uff1f", 23, 24}},
|
||||
[]token{token{"C++", 0, 3}, token{"\u548c", 3, 4}, token{"c#", 4, 6}, token{"\u662f", 6, 7}, token{"\u4ec0\u4e48", 7, 9}, token{"\u5173\u7cfb", 9, 11}, token{"\uff1f", 11, 12}, token{"11", 12, 14}, token{"+", 14, 15}, token{"122", 15, 18}, token{"=", 18, 19}, token{"133", 19, 22}, token{"\uff0c", 22, 23}, token{"\u662f", 23, 24}, token{"\u5417", 24, 25}, token{"\uff1f", 25, 26}, token{"PI", 26, 28}, token{"=", 28, 29}, token{"3", 29, 30}, token{".", 30, 31}, token{"14159", 31, 36}},
|
||||
[]token{token{"\u4f60", 0, 1}, token{"\u8ba4\u8bc6", 1, 3}, token{"\u90a3\u4e2a", 3, 5}, token{"\u548c", 5, 6}, token{"\u4e3b\u5e2d", 6, 8}, token{"\u63e1\u624b", 8, 10}, token{"\u7684", 10, 11}, token{"\u7684\u54e5", 11, 13}, token{"\u5417", 13, 14}, token{"\uff1f", 14, 15}, token{"\u4ed6", 15, 16}, token{"\u5f00", 16, 17}, token{"\u4e00\u8f86", 17, 19}, token{"\u9ed1\u8272", 19, 21}, token{"\u7684\u58eb", 21, 23}, token{"\u3002", 23, 24}},
|
||||
[]token{token{"\u67aa\u6746\u5b50", 0, 3}, token{"\u4e2d", 3, 4}, token{"\u51fa", 4, 5}, token{"\u653f\u6743", 5, 7}},
|
||||
[]token{token{"\u5f20", 0, 1}, token{"\u4e09", 1, 2}, token{"\u98ce", 2, 3}, token{"\u540c\u5b66", 3, 5}, token{"\u8d70\u4e0a", 5, 7}, token{"\u4e86", 7, 8}, token{"\u4e0d\u5f52\u8def", 8, 11}},
|
||||
[]token{token{"\u963fQ", 0, 2}, token{"\u8170\u95f4", 2, 4}, token{"\u6302", 4, 5}, token{"\u7740", 5, 6}, token{"BB\u673a", 6, 9}, token{"\u624b\u91cc", 9, 11}, token{"\u62ff", 11, 12}, token{"\u7740", 12, 13}, token{"\u5927\u54e5\u5927", 13, 16}, token{"\uff0c", 16, 17}, token{"\u8bf4", 17, 18}, token{"\uff1a", 18, 19}, token{"\u6211", 19, 20}, token{"\u4e00\u822c", 20, 22}, token{"\u5403\u996d", 22, 24}, token{"\u4e0d", 24, 25}, token{"AA\u5236", 25, 28}, token{"\u7684", 28, 29}, token{"\u3002", 29, 30}},
|
||||
[]token{token{"\u5728", 0, 1}, token{"1\u53f7\u5e97", 1, 4}, token{"\u80fd", 4, 5}, token{"\u4e70", 5, 6}, token{"\u5230", 6, 7}, token{"\u5c0fS", 7, 9}, token{"\u548c", 9, 10}, token{"\u5927S", 10, 12}, token{"\u516b\u5366", 12, 14}, token{"\u7684", 14, 15}, token{"\u4e66", 15, 16}, token{"\u3002", 16, 17}},
|
||||
[]token{token{"\u8fd9", 0, 1}, token{"\u662f", 1, 2}, token{"\u4e00\u4e2a", 2, 4}, token{"\u4f38\u624b", 4, 6}, token{"\u4e0d\u89c1", 6, 8}, token{"\u4e94\u6307", 8, 10}, token{"\u4f38\u624b\u4e0d\u89c1\u4e94\u6307", 4, 10}, token{"\u7684", 10, 11}, token{"\u9ed1\u591c", 11, 13}, token{"\u3002", 13, 14}, token{"\u6211", 14, 15}, token{"\u53eb", 15, 16}, token{"\u609f\u7a7a", 17, 19}, token{"\u5b59\u609f\u7a7a", 16, 19}, token{"\uff0c", 19, 20}, token{"\u6211", 20, 21}, token{"\u7231", 21, 22}, token{"\u5317\u4eac", 22, 24}, token{"\uff0c", 24, 25}, token{"\u6211", 25, 26}, token{"\u7231", 26, 27}, token{"Python", 27, 33}, token{"\u548c", 33, 34}, token{"C++", 34, 37}, token{"\u3002", 37, 38}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u4e0d", 1, 2}, token{"\u559c\u6b22", 2, 4}, token{"\u65e5\u672c", 4, 6}, token{"\u548c\u670d", 6, 8}, token{"\u3002", 8, 9}},
|
||||
[]token{token{"\u96f7\u7334", 0, 2}, token{"\u56de\u5f52", 2, 4}, token{"\u4eba\u95f4", 4, 6}, token{"\u3002", 6, 7}},
|
||||
[]token{token{"\u5de5\u4fe1\u5904", 0, 3}, token{"\u5e72\u4e8b", 4, 6}, token{"\u5973\u5e72\u4e8b", 3, 6}, token{"\u6bcf\u6708", 6, 8}, token{"\u7ecf\u8fc7", 8, 10}, token{"\u4e0b\u5c5e", 10, 12}, token{"\u79d1\u5ba4", 12, 14}, token{"\u90fd", 14, 15}, token{"\u8981", 15, 16}, token{"\u4eb2\u53e3", 16, 18}, token{"\u4ea4\u4ee3", 18, 20}, token{"24", 20, 22}, token{"\u53e3", 22, 23}, token{"\u4ea4\u6362", 23, 25}, token{"\u6362\u673a", 24, 26}, token{"\u4ea4\u6362\u673a", 23, 26}, token{"\u7b49", 26, 27}, token{"\u6280\u672f", 27, 29}, token{"\u6280\u672f\u6027", 27, 30}, token{"\u5668\u4ef6", 30, 32}, token{"\u7684", 32, 33}, token{"\u5b89\u88c5", 33, 35}, token{"\u5de5\u4f5c", 35, 37}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u9700\u8981", 1, 3}, token{"\u5ec9\u79df", 3, 5}, token{"\u79df\u623f", 4, 6}, token{"\u5ec9\u79df\u623f", 3, 6}},
|
||||
[]token{token{"\u6c38\u548c", 0, 2}, token{"\u670d\u88c5", 2, 4}, token{"\u9970\u54c1", 4, 6}, token{"\u6709\u9650", 6, 8}, token{"\u516c\u53f8", 8, 10}, token{"\u6709\u9650\u516c\u53f8", 6, 10}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u7231", 1, 2}, token{"\u5317\u4eac", 2, 4}, token{"\u5929\u5b89", 4, 6}, token{"\u5929\u5b89\u95e8", 4, 7}},
|
||||
[]token{token{"abc", 0, 3}},
|
||||
[]token{token{"\u9690", 0, 1}, token{"\u53ef\u592b", 3, 5}, token{"\u9a6c\u5c14\u53ef", 1, 4}, token{"\u9a6c\u5c14\u53ef\u592b", 1, 5}},
|
||||
[]token{token{"\u96f7\u7334", 0, 2}, token{"\u662f", 2, 3}, token{"\u4e2a", 3, 4}, token{"\u597d", 4, 5}, token{"\u7f51\u7ad9", 5, 7}},
|
||||
[]token{token{"\u201c", 0, 1}, token{"Microsoft", 1, 10}, token{"\u201d", 10, 11}, token{"\u4e00", 11, 12}, token{"\u8bcd", 12, 13}, token{"\u7531", 13, 14}, token{"\u201c", 14, 15}, token{"MICROcomputer", 15, 28}, token{"\uff08", 28, 29}, token{"\u5fae\u578b", 29, 31}, token{"\u8ba1\u7b97", 31, 33}, token{"\u7b97\u673a", 32, 34}, token{"\u8ba1\u7b97\u673a", 31, 34}, token{"\uff09", 34, 35}, token{"\u201d", 35, 36}, token{"\u548c", 36, 37}, token{"\u201c", 37, 38}, token{"SOFTware", 38, 46}, token{"\uff08", 46, 47}, token{"\u8f6f\u4ef6", 47, 49}, token{"\uff09", 49, 50}, token{"\u201d", 50, 51}, token{"\u4e24", 51, 52}, token{"\u90e8\u5206", 52, 54}, token{"\u7ec4\u6210", 54, 56}},
|
||||
[]token{token{"\u8349\u6ce5\u9a6c", 0, 3}, token{"\u548c", 3, 4}, token{"\u6b3a", 4, 5}, token{"\u5b9e", 5, 6}, token{"\u9a6c", 6, 7}, token{"\u662f", 7, 8}, token{"\u4eca\u5e74", 8, 10}, token{"\u7684", 10, 11}, token{"\u6d41\u884c", 11, 13}, token{"\u8bcd\u6c47", 13, 15}},
|
||||
[]token{token{"\u4f0a", 0, 1}, token{"\u85e4", 1, 2}, token{"\u6d0b\u534e\u5802", 2, 5}, token{"\u603b\u5e9c", 5, 7}, token{"\u5e97", 7, 8}},
|
||||
[]token{token{"\u4e2d\u56fd", 0, 2}, token{"\u79d1\u5b66", 2, 4}, token{"\u5b66\u9662", 3, 5}, token{"\u8ba1\u7b97", 5, 7}, token{"\u6280\u672f", 7, 9}, token{"\u7814\u7a76", 9, 11}, token{"\u79d1\u5b66\u9662", 2, 5}, token{"\u7814\u7a76\u6240", 9, 12}, token{"\u4e2d\u56fd\u79d1\u5b66\u9662\u8ba1\u7b97\u6280\u672f\u7814\u7a76\u6240", 0, 12}},
|
||||
[]token{token{"\u7f57\u5bc6\u6b27", 0, 3}, token{"\u4e0e", 3, 4}, token{"\u6731\u4e3d\u53f6", 4, 7}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u8d2d\u4e70", 1, 3}, token{"\u4e86", 3, 4}, token{"\u9053\u5177", 4, 6}, token{"\u548c", 6, 7}, token{"\u670d\u88c5", 7, 9}},
|
||||
[]token{token{"PS", 0, 2}, token{":", 2, 3}, token{" ", 3, 4}, token{"\u6211", 4, 5}, token{"\u89c9\u5f97", 5, 7}, token{"\u5f00\u6e90", 7, 9}, token{"\u6709", 9, 10}, token{"\u4e00\u4e2a", 10, 12}, token{"\u597d\u5904", 12, 14}, token{"\uff0c", 14, 15}, token{"\u5c31\u662f", 15, 17}, token{"\u80fd\u591f", 17, 19}, token{"\u6566\u4fc3", 19, 21}, token{"\u81ea\u5df1", 21, 23}, token{"\u4e0d\u65ad", 23, 25}, token{"\u6539\u8fdb", 25, 27}, token{"\u4e0d\u65ad\u6539\u8fdb", 23, 27}, token{"\uff0c", 27, 28}, token{"\u907f\u514d", 28, 30}, token{"\u655e", 30, 31}, token{"\u5e1a", 31, 32}, token{"\u81ea\u73cd", 32, 34}},
|
||||
[]token{token{"\u6e56\u5317", 0, 2}, token{"\u6e56\u5317\u7701", 0, 3}, token{"\u77f3\u9996", 3, 5}, token{"\u77f3\u9996\u5e02", 3, 6}},
|
||||
[]token{token{"\u6e56\u5317", 0, 2}, token{"\u6e56\u5317\u7701", 0, 3}, token{"\u5341\u5830", 3, 5}, token{"\u5341\u5830\u5e02", 3, 6}},
|
||||
[]token{token{"\u7ecf\u7406", 1, 3}, token{"\u603b\u7ecf\u7406", 0, 3}, token{"\u5b8c\u6210", 3, 5}, token{"\u4e86", 5, 6}, token{"\u8fd9\u4ef6", 6, 8}, token{"\u4e8b\u60c5", 8, 10}},
|
||||
[]token{token{"\u7535\u8111", 0, 2}, token{"\u4fee\u597d", 2, 4}, token{"\u4e86", 4, 5}},
|
||||
[]token{token{"\u505a\u597d", 0, 2}, token{"\u4e86", 2, 3}, token{"\u8fd9\u4ef6", 3, 5}, token{"\u4e8b\u60c5", 5, 7}, token{"\u5c31", 7, 8}, token{"\u4e00\u4e86\u767e\u4e86", 8, 12}, token{"\u4e86", 12, 13}},
|
||||
[]token{token{"\u4eba\u4eec", 0, 2}, token{"\u5ba1\u7f8e", 2, 4}, token{"\u7684", 4, 5}, token{"\u89c2\u70b9", 5, 7}, token{"\u662f", 7, 8}, token{"\u4e0d\u540c", 8, 10}, token{"\u7684", 10, 11}},
|
||||
[]token{token{"\u6211\u4eec", 0, 2}, token{"\u4e70", 2, 3}, token{"\u4e86", 3, 4}, token{"\u4e00\u4e2a", 4, 6}, token{"\u7f8e\u7684", 6, 8}, token{"\u7a7a\u8c03", 8, 10}},
|
||||
[]token{token{"\u7ebf\u7a0b", 0, 2}, token{"\u521d\u59cb", 2, 4}, token{"\u521d\u59cb\u5316", 2, 5}, token{"\u65f6", 5, 6}, token{"\u6211\u4eec", 6, 8}, token{"\u8981", 8, 9}, token{"\u6ce8\u610f", 9, 11}},
|
||||
[]token{token{"\u4e00\u4e2a", 0, 2}, token{"\u5206\u5b50", 2, 4}, token{"\u662f", 4, 5}, token{"\u7531", 5, 6}, token{"\u597d\u591a", 6, 8}, token{"\u539f\u5b50", 8, 10}, token{"\u7ec4\u7ec7", 10, 12}, token{"\u6210", 12, 13}, token{"\u7684", 13, 14}},
|
||||
[]token{token{"\u795d", 0, 1}, token{"\u4f60", 1, 2}, token{"\u9a6c\u5230\u529f\u6210", 2, 6}},
|
||||
[]token{token{"\u4ed6", 0, 1}, token{"\u6389", 1, 2}, token{"\u8fdb", 2, 3}, token{"\u4e86", 3, 4}, token{"\u65e0\u5e95", 4, 6}, token{"\u65e0\u5e95\u6d1e", 4, 7}, token{"\u91cc", 7, 8}},
|
||||
[]token{token{"\u4e2d\u56fd", 0, 2}, token{"\u7684", 2, 3}, token{"\u9996\u90fd", 3, 5}, token{"\u662f", 5, 6}, token{"\u5317\u4eac", 6, 8}},
|
||||
[]token{token{"\u5b59", 0, 1}, token{"\u541b", 1, 2}, token{"\u610f", 2, 3}},
|
||||
[]token{token{"\u5916\u4ea4", 0, 2}, token{"\u5916\u4ea4\u90e8", 0, 3}, token{"\u53d1\u8a00", 3, 5}, token{"\u53d1\u8a00\u4eba", 3, 6}, token{"\u9a6c\u671d\u65ed", 6, 9}},
|
||||
[]token{token{"\u9886\u5bfc", 0, 2}, token{"\u9886\u5bfc\u4eba", 0, 3}, token{"\u4f1a\u8bae", 3, 5}, token{"\u548c", 5, 6}, token{"\u7b2c\u56db", 6, 8}, token{"\u56db\u5c4a", 7, 9}, token{"\u7b2c\u56db\u5c4a", 6, 9}, token{"\u4e1c\u4e9a", 9, 11}, token{"\u5cf0\u4f1a", 11, 13}},
|
||||
[]token{token{"\u5728", 0, 1}, token{"\u8fc7\u53bb", 1, 3}, token{"\u7684", 3, 4}, token{"\u8fd9", 4, 5}, token{"\u4e94\u5e74", 5, 7}},
|
||||
[]token{token{"\u8fd8", 0, 1}, token{"\u9700\u8981", 1, 3}, token{"\u5f88", 3, 4}, token{"\u957f", 4, 5}, token{"\u7684", 5, 6}, token{"\u8def", 6, 7}, token{"\u8981", 7, 8}, token{"\u8d70", 8, 9}},
|
||||
[]token{token{"60", 0, 2}, token{"\u5468\u5e74", 2, 4}, token{"\u9996\u90fd", 4, 6}, token{"\u9605\u5175", 6, 8}},
|
||||
[]token{token{"\u4f60\u597d", 0, 2}, token{"\u4eba\u4eec", 2, 4}, token{"\u5ba1\u7f8e", 4, 6}, token{"\u7684", 6, 7}, token{"\u89c2\u70b9", 7, 9}, token{"\u662f", 9, 10}, token{"\u4e0d\u540c", 10, 12}, token{"\u7684", 12, 13}},
|
||||
[]token{token{"\u4e70", 0, 1}, token{"\u6c34\u679c", 1, 3}, token{"\u7136\u540e", 3, 5}, token{"\u6765", 5, 6}, token{"\u4e16\u535a", 6, 8}, token{"\u535a\u56ed", 7, 9}, token{"\u4e16\u535a\u56ed", 6, 9}},
|
||||
[]token{token{"\u4e70", 0, 1}, token{"\u6c34\u679c", 1, 3}, token{"\u7136\u540e", 3, 5}, token{"\u53bb", 5, 6}, token{"\u4e16\u535a", 6, 8}, token{"\u535a\u56ed", 7, 9}, token{"\u4e16\u535a\u56ed", 6, 9}},
|
||||
[]token{token{"\u4f46\u662f", 0, 2}, token{"\u540e\u6765", 2, 4}, token{"\u6211", 4, 5}, token{"\u624d", 5, 6}, token{"\u77e5\u9053", 6, 8}, token{"\u4f60", 8, 9}, token{"\u662f", 9, 10}, token{"\u5bf9", 10, 11}, token{"\u7684", 11, 12}},
|
||||
[]token{token{"\u5b58\u5728", 0, 2}, token{"\u5373", 2, 3}, token{"\u5408\u7406", 3, 5}},
|
||||
[]token{token{"\u7684", 0, 1}, token{"\u7684", 1, 2}, token{"\u7684", 2, 3}, token{"\u7684", 3, 4}, token{"\u7684", 4, 5}, token{"\u5728", 5, 6}, token{"\u7684", 6, 7}, token{"\u7684", 7, 8}, token{"\u7684", 8, 9}, token{"\u7684", 9, 10}, token{"\u5c31", 10, 11}, token{"\u4ee5", 11, 12}, token{"\u548c", 12, 13}, token{"\u548c", 13, 14}, token{"\u548c", 14, 15}},
|
||||
[]token{token{"I", 0, 1}, token{" ", 1, 2}, token{"love", 2, 6}, token{"\u4f60", 6, 7}, token{"\uff0c", 7, 8}, token{"\u4e0d\u4ee5", 8, 10}, token{"\u4ee5\u4e3a", 9, 11}, token{"\u4e0d\u4ee5\u4e3a\u803b", 8, 12}, token{"\uff0c", 12, 13}, token{"\u53cd", 13, 14}, token{"\u4ee5\u4e3a", 14, 16}, token{"rong", 16, 20}},
|
||||
[]token{token{"\u56e0", 0, 1}},
|
||||
[]token{},
|
||||
[]token{token{"hello", 0, 5}, token{"\u4f60\u597d", 5, 7}, token{"\u4eba\u4eec", 7, 9}, token{"\u5ba1\u7f8e", 9, 11}, token{"\u7684", 11, 12}, token{"\u89c2\u70b9", 12, 14}, token{"\u662f", 14, 15}, token{"\u4e0d\u540c", 15, 17}, token{"\u7684", 17, 18}},
|
||||
[]token{token{"\u5f88", 0, 1}, token{"\u597d", 1, 2}, token{"\u4f46", 2, 3}, token{"\u4e3b\u8981", 3, 5}, token{"\u662f", 5, 6}, token{"\u57fa\u4e8e", 6, 8}, token{"\u7f51\u9875", 8, 10}, token{"\u5f62\u5f0f", 10, 12}},
|
||||
[]token{token{"hello", 0, 5}, token{"\u4f60\u597d", 5, 7}, token{"\u4eba\u4eec", 7, 9}, token{"\u5ba1\u7f8e", 9, 11}, token{"\u7684", 11, 12}, token{"\u89c2\u70b9", 12, 14}, token{"\u662f", 14, 15}, token{"\u4e0d\u540c", 15, 17}, token{"\u7684", 17, 18}},
|
||||
[]token{token{"\u4ec0\u4e48", 1, 3}, token{"\u4e3a\u4ec0\u4e48", 0, 3}, token{"\u6211", 3, 4}, token{"\u4e0d\u80fd", 4, 6}, token{"\u62e5\u6709", 6, 8}, token{"\u60f3\u8981", 8, 10}, token{"\u7684", 10, 11}, token{"\u751f\u6d3b", 11, 13}},
|
||||
[]token{token{"\u540e\u6765", 0, 2}, token{"\u6211", 2, 3}, token{"\u624d", 3, 4}},
|
||||
[]token{token{"\u6b64\u6b21", 0, 2}, token{"\u6765", 2, 3}, token{"\u4e2d\u56fd", 3, 5}, token{"\u662f", 5, 6}, token{"\u4e3a\u4e86", 6, 8}},
|
||||
[]token{token{"\u4f7f\u7528", 0, 2}, token{"\u4e86", 2, 3}, token{"\u5b83", 3, 4}, token{"\u5c31", 4, 5}, token{"\u53ef\u4ee5", 5, 7}, token{"\u89e3\u51b3", 7, 9}, token{"\u4e00\u4e9b", 9, 11}, token{"\u95ee\u9898", 11, 13}},
|
||||
[]token{token{",", 0, 1}, token{"\u4f7f\u7528", 1, 3}, token{"\u4e86", 3, 4}, token{"\u5b83", 4, 5}, token{"\u5c31", 5, 6}, token{"\u53ef\u4ee5", 6, 8}, token{"\u89e3\u51b3", 8, 10}, token{"\u4e00\u4e9b", 10, 12}, token{"\u95ee\u9898", 12, 14}},
|
||||
[]token{token{"\u5176\u5b9e", 0, 2}, token{"\u4f7f\u7528", 2, 4}, token{"\u4e86", 4, 5}, token{"\u5b83", 5, 6}, token{"\u5c31", 6, 7}, token{"\u53ef\u4ee5", 7, 9}, token{"\u89e3\u51b3", 9, 11}, token{"\u4e00\u4e9b", 11, 13}, token{"\u95ee\u9898", 13, 15}},
|
||||
[]token{token{"\u597d\u4eba", 0, 2}, token{"\u4f7f\u7528", 2, 4}, token{"\u4e86", 4, 5}, token{"\u5b83", 5, 6}, token{"\u5c31", 6, 7}, token{"\u53ef\u4ee5", 7, 9}, token{"\u89e3\u51b3", 9, 11}, token{"\u4e00\u4e9b", 11, 13}, token{"\u95ee\u9898", 13, 15}},
|
||||
[]token{token{"\u56e0\u4e3a", 1, 3}, token{"\u662f\u56e0\u4e3a", 0, 3}, token{"\u548c", 3, 4}, token{"\u56fd\u5bb6", 4, 6}},
|
||||
[]token{token{"\u8001\u5e74", 0, 2}, token{"\u641c\u7d22", 2, 4}, token{"\u8fd8", 4, 5}, token{"\u652f\u6301", 5, 7}},
|
||||
[]token{token{"\u5e72\u8106", 0, 2}, token{"\u5c31", 2, 3}, token{"\u628a", 3, 4}, token{"\u90a3", 4, 5}, token{"\u90e8", 5, 6}, token{"\u8499", 6, 7}, token{"\u4eba", 7, 8}, token{"\u7684", 8, 9}, token{"\u95f2", 9, 10}, token{"\u6cd5", 10, 11}, token{"\u7ed9", 11, 12}, token{"\u5e9f", 12, 13}, token{"\u4e86", 13, 14}, token{"\u62c9\u5012", 14, 16}, token{"\uff01", 16, 17}, token{"RT", 17, 19}, token{" ", 19, 20}, token{"@", 20, 21}, token{"laoshipukong", 21, 33}, token{" ", 33, 34}, token{":", 34, 35}, token{" ", 35, 36}, token{"27", 36, 38}, token{"\u65e5", 38, 39}, token{"\uff0c", 39, 40}, token{"\u5168\u56fd", 40, 42}, token{"\u56fd\u4eba", 41, 43}, token{"\u4eba\u5927", 42, 44}, token{"\u5e38\u59d4", 44, 46}, token{"\u59d4\u4f1a", 45, 47}, token{"\u5e38\u59d4\u4f1a", 44, 47}, token{"\u5168\u56fd\u4eba\u5927\u5e38\u59d4\u4f1a", 40, 47}, token{"\u7b2c\u4e09", 47, 49}, token{"\u4e09\u6b21", 48, 50}, token{"\u7b2c\u4e09\u6b21", 47, 50}, token{"\u5ba1\u8bae", 50, 52}, token{"\u4fb5\u6743", 52, 54}, token{"\u8d23\u4efb", 54, 56}, token{"\u8d23\u4efb\u6cd5", 54, 57}, token{"\u8349\u6848", 57, 59}, token{"\uff0c", 59, 60}, token{"\u5220\u9664", 60, 62}, token{"\u4e86", 62, 63}, token{"\u6709\u5173", 63, 65}, token{"\u533b\u7597", 65, 67}, token{"\u635f\u5bb3", 67, 69}, token{"\u8d23\u4efb", 69, 71}, token{"\u201c", 71, 72}, token{"\u4e3e\u8bc1", 72, 74}, token{"\u5012\u7f6e", 74, 76}, token{"\u201d", 76, 77}, token{"\u7684", 77, 78}, token{"\u89c4\u5b9a", 78, 80}, token{"\u3002", 80, 81}, token{"\u5728", 81, 82}, token{"\u533b\u60a3", 82, 84}, token{"\u7ea0\u7eb7", 84, 86}, token{"\u4e2d", 86, 87}, token{"\u672c", 87, 88}, token{"\u5df2", 88, 89}, token{"\u5904\u4e8e", 89, 91}, token{"\u5f31\u52bf", 91, 93}, token{"\u5730\u4f4d", 93, 95}, token{"\u7684", 95, 96}, token{"\u6d88\u8d39", 96, 98}, token{"\u6d88\u8d39\u8005", 96, 99}, token{"\u7531\u6b64", 99, 101}, token{"\u5c06", 101, 102}, token{"\u9677\u5165", 102, 104}, token{"\u4e0d\u590d", 106, 108}, token{"\u4e07\u52ab\u4e0d\u590d", 104, 108}, token{"\u7684", 108, 109}, token{"\u5883\u5730", 109, 111}, token{"\u3002", 111, 112}, token{" ", 112, 113}},
|
||||
[]token{token{"\u5927", 0, 1}},
|
||||
[]token{},
|
||||
[]token{token{"\u4ed6", 0, 1}, token{"\u8bf4", 1, 2}, token{"\u7684", 2, 3}, token{"\u786e\u5b9e", 3, 5}, token{"\u5728", 5, 6}, token{"\u7406", 6, 7}},
|
||||
[]token{token{"\u957f\u6625", 0, 2}, token{"\u5e02\u957f", 2, 4}, token{"\u6625\u8282", 4, 6}, token{"\u8bb2\u8bdd", 6, 8}},
|
||||
[]token{token{"\u7ed3\u5a5a", 0, 2}, token{"\u7684", 2, 3}, token{"\u548c", 3, 4}, token{"\u5c1a\u672a", 4, 6}, token{"\u7ed3\u5a5a", 6, 8}, token{"\u7684", 8, 9}},
|
||||
[]token{token{"\u7ed3\u5408", 0, 2}, token{"\u6210", 2, 3}, token{"\u5206\u5b50", 3, 5}, token{"\u65f6", 5, 6}},
|
||||
[]token{token{"\u65c5\u6e38", 0, 2}, token{"\u548c", 2, 3}, token{"\u670d\u52a1", 3, 5}, token{"\u662f", 5, 6}, token{"\u6700\u597d", 6, 8}, token{"\u7684", 8, 9}},
|
||||
[]token{token{"\u8fd9\u4ef6", 0, 2}, token{"\u4e8b\u60c5", 2, 4}, token{"\u7684\u786e", 4, 6}, token{"\u662f", 6, 7}, token{"\u6211", 7, 8}, token{"\u7684", 8, 9}, token{"\u9519", 9, 10}},
|
||||
[]token{token{"\u4f9b", 0, 1}, token{"\u5927\u5bb6", 1, 3}, token{"\u53c2\u8003", 3, 5}, token{"\u6307\u6b63", 5, 7}},
|
||||
[]token{token{"\u54c8\u5c14", 0, 2}, token{"\u54c8\u5c14\u6ee8", 0, 3}, token{"\u653f\u5e9c", 3, 5}, token{"\u516c\u5e03", 5, 7}, token{"\u584c", 7, 8}, token{"\u6865", 8, 9}, token{"\u539f\u56e0", 9, 11}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u5728", 1, 2}, token{"\u673a\u573a", 2, 4}, token{"\u5165\u53e3", 4, 6}, token{"\u5165\u53e3\u5904", 4, 7}},
|
||||
[]token{token{"\u90a2", 0, 1}, token{"\u6c38", 1, 2}, token{"\u81e3", 2, 3}, token{"\u6444\u5f71", 3, 5}, token{"\u62a5\u9053", 5, 7}},
|
||||
[]token{token{"BP", 0, 2}, token{"\u795e\u7ecf", 2, 4}, token{"\u7f51\u7edc", 4, 6}, token{"\u795e\u7ecf\u7f51", 2, 5}, token{"\u795e\u7ecf\u7f51\u7edc", 2, 6}, token{"\u5982\u4f55", 6, 8}, token{"\u8bad\u7ec3", 8, 10}, token{"\u624d\u80fd", 10, 12}, token{"\u5728", 12, 13}, token{"\u5206\u7c7b", 13, 15}, token{"\u65f6", 15, 16}, token{"\u589e\u52a0", 16, 18}, token{"\u533a\u5206", 18, 20}, token{"\u5206\u5ea6", 19, 21}, token{"\u533a\u5206\u5ea6", 18, 21}, token{"\uff1f", 21, 22}},
|
||||
[]token{token{"\u5357\u4eac", 0, 2}, token{"\u4eac\u5e02", 1, 3}, token{"\u5357\u4eac\u5e02", 0, 3}, token{"\u957f\u6c5f", 3, 5}, token{"\u5927\u6865", 5, 7}, token{"\u957f\u6c5f\u5927\u6865", 3, 7}},
|
||||
[]token{token{"\u5e94", 0, 1}, token{"\u4e00\u4e9b", 1, 3}, token{"\u4f7f\u7528", 3, 5}, token{"\u7528\u8005", 4, 6}, token{"\u4f7f\u7528\u8005", 3, 6}, token{"\u7684", 6, 7}, token{"\u5efa\u8bae", 7, 9}, token{"\uff0c", 9, 10}, token{"\u4e5f", 10, 11}, token{"\u4e3a\u4e86", 11, 13}, token{"\u4fbf\u4e8e", 13, 15}, token{"\u5229\u7528", 15, 17}, token{"NiuTrans", 17, 25}, token{"\u7528\u4e8e", 25, 27}, token{"SMT", 27, 30}, token{"\u7814\u7a76", 30, 32}},
|
||||
[]token{token{"\u957f\u6625", 0, 2}, token{"\u957f\u6625\u5e02", 0, 3}, token{"\u957f\u6625", 3, 5}, token{"\u836f\u5e97", 5, 7}},
|
||||
[]token{token{"\u9093\u9896\u8d85", 0, 3}, token{"\u751f\u524d", 3, 5}, token{"\u6700", 5, 6}, token{"\u559c\u6b22", 6, 8}, token{"\u7684", 8, 9}, token{"\u8863\u670d", 9, 11}},
|
||||
[]token{token{"\u9526\u6d9b", 1, 3}, token{"\u80e1\u9526\u6d9b", 0, 3}, token{"\u662f", 3, 4}, token{"\u70ed\u7231", 4, 6}, token{"\u4e16\u754c", 6, 8}, token{"\u548c\u5e73", 8, 10}, token{"\u7684", 10, 11}, token{"\u653f\u6cbb", 11, 13}, token{"\u653f\u6cbb\u5c40", 11, 14}, token{"\u5e38\u59d4", 14, 16}},
|
||||
[]token{token{"\u7a0b\u5e8f", 0, 2}, token{"\u7a0b\u5e8f\u5458", 0, 3}, token{"\u795d", 3, 4}, token{"\u6d77\u6797", 4, 6}, token{"\u548c", 6, 7}, token{"\u6731", 7, 8}, token{"\u4f1a", 8, 9}, token{"\u9707", 9, 10}, token{"\u662f", 10, 11}, token{"\u5728", 11, 12}, token{"\u5b59", 12, 13}, token{"\u5065", 13, 14}, token{"\u7684", 14, 15}, token{"\u5de6\u9762", 15, 17}, token{"\u548c", 17, 18}, token{"\u53f3\u9762", 18, 20}, token{",", 20, 21}, token{" ", 21, 22}, token{"\u8303", 22, 23}, token{"\u51ef", 23, 24}, token{"\u5728", 24, 25}, token{"\u6700", 25, 26}, token{"\u53f3\u9762", 26, 28}, token{".", 28, 29}, token{"\u518d", 29, 30}, token{"\u5f80", 30, 31}, token{"\u5de6", 31, 32}, token{"\u662f", 32, 33}, token{"\u674e", 33, 34}, token{"\u677e", 34, 35}, token{"\u6d2a", 35, 36}},
|
||||
[]token{token{"\u4e00\u6b21", 0, 2}, token{"\u4e00\u6b21\u6027", 0, 3}, token{"\u4ea4", 3, 4}, token{"\u591a\u5c11", 4, 6}, token{"\u94b1", 6, 7}},
|
||||
[]token{token{"\u4e24\u5757", 0, 2}, token{"\u4e94", 2, 3}, token{"\u4e00\u5957", 3, 5}, token{"\uff0c", 5, 6}, token{"\u4e09\u5757", 6, 8}, token{"\u516b", 8, 9}, token{"\u4e00\u65a4", 9, 11}, token{"\uff0c", 11, 12}, token{"\u56db\u5757", 12, 14}, token{"\u4e03", 14, 15}, token{"\u4e00\u672c", 15, 17}, token{"\uff0c", 17, 18}, token{"\u4e94\u5757", 18, 20}, token{"\u516d", 20, 21}, token{"\u4e00\u6761", 21, 23}},
|
||||
[]token{token{"\u5c0f", 0, 1}, token{"\u548c\u5c1a", 1, 3}, token{"\u7559", 3, 4}, token{"\u4e86", 4, 5}, token{"\u4e00\u4e2a", 5, 7}, token{"\u50cf", 7, 8}, token{"\u5927", 8, 9}, token{"\u548c\u5c1a", 9, 11}, token{"\u4e00\u6837", 11, 13}, token{"\u7684", 13, 14}, token{"\u548c\u5c1a", 14, 16}, token{"\u548c\u5c1a\u5934", 14, 17}},
|
||||
[]token{token{"\u6211", 0, 1}, token{"\u662f", 1, 2}, token{"\u4e2d\u534e", 2, 4}, token{"\u534e\u4eba", 3, 5}, token{"\u4eba\u6c11", 4, 6}, token{"\u5171\u548c", 6, 8}, token{"\u5171\u548c\u56fd", 6, 9}, token{"\u4e2d\u534e\u4eba\u6c11\u5171\u548c\u56fd", 2, 9}, token{"\u516c\u6c11", 9, 11}, token{";", 11, 12}, token{"\u6211", 12, 13}, token{"\u7238\u7238", 13, 15}, token{"\u662f", 15, 16}, token{"\u5171\u548c", 16, 18}, token{"\u5171\u548c\u515a", 16, 19}, token{"\u515a\u5458", 19, 21}, token{";", 21, 22}, token{" ", 22, 23}, token{"\u5730\u94c1", 23, 25}, token{"\u548c\u5e73", 25, 27}, token{"\u548c\u5e73\u95e8", 25, 28}, token{"\u7ad9", 28, 29}},
|
||||
[]token{token{"\u5f20\u6653\u6885", 0, 3}, token{"\u53bb", 3, 4}, token{"\u4eba\u6c11", 4, 6}, token{"\u533b\u9662", 6, 8}, token{"\u505a", 8, 9}, token{"\u4e86", 9, 10}, token{"\u4e2a", 10, 11}, token{"B\u8d85", 11, 13}, token{"\u7136\u540e", 13, 15}, token{"\u53bb", 15, 16}, token{"\u4e70", 16, 17}, token{"\u4e86", 17, 18}, token{"\u4ef6", 18, 19}, token{"T\u6064", 19, 21}},
|
||||
[]token{token{"AT&T", 0, 4}, token{"\u662f", 4, 5}, token{"\u4e00\u4ef6", 5, 7}, token{"\u4e0d\u9519", 7, 9}, token{"\u7684", 9, 10}, token{"\u516c\u53f8", 10, 12}, token{"\uff0c", 12, 13}, token{"\u7ed9", 13, 14}, token{"\u4f60", 14, 15}, token{"\u53d1", 15, 16}, token{"offer", 16, 21}, token{"\u4e86", 21, 22}, token{"\u5417", 22, 23}, token{"\uff1f", 23, 24}},
|
||||
[]token{token{"C++", 0, 3}, token{"\u548c", 3, 4}, token{"c#", 4, 6}, token{"\u662f", 6, 7}, token{"\u4ec0\u4e48", 7, 9}, token{"\u5173\u7cfb", 9, 11}, token{"\uff1f", 11, 12}, token{"11", 12, 14}, token{"+", 14, 15}, token{"122", 15, 18}, token{"=", 18, 19}, token{"133", 19, 22}, token{"\uff0c", 22, 23}, token{"\u662f", 23, 24}, token{"\u5417", 24, 25}, token{"\uff1f", 25, 26}, token{"PI", 26, 28}, token{"=", 28, 29}, token{"3", 29, 30}, token{".", 30, 31}, token{"14159", 31, 36}},
|
||||
[]token{token{"\u4f60", 0, 1}, token{"\u8ba4\u8bc6", 1, 3}, token{"\u90a3\u4e2a", 3, 5}, token{"\u548c", 5, 6}, token{"\u4e3b\u5e2d", 6, 8}, token{"\u63e1\u624b", 8, 10}, token{"\u7684", 10, 11}, token{"\u7684\u54e5", 11, 13}, token{"\u5417", 13, 14}, token{"\uff1f", 14, 15}, token{"\u4ed6", 15, 16}, token{"\u5f00", 16, 17}, token{"\u4e00\u8f86", 17, 19}, token{"\u9ed1\u8272", 19, 21}, token{"\u7684\u58eb", 21, 23}, token{"\u3002", 23, 24}},
|
||||
[]token{token{"\u67aa\u6746", 0, 2}, token{"\u6746\u5b50", 1, 3}, token{"\u67aa\u6746\u5b50", 0, 3}, token{"\u4e2d", 3, 4}, token{"\u51fa", 4, 5}, token{"\u653f\u6743", 5, 7}},
|
||||
[]token{token{"\u5f20", 0, 1}, token{"\u4e09", 1, 2}, token{"\u98ce", 2, 3}, token{"\u540c\u5b66", 3, 5}, token{"\u8d70\u4e0a", 5, 7}, token{"\u4e86", 7, 8}, token{"\u5f52\u8def", 9, 11}, token{"\u4e0d\u5f52\u8def", 8, 11}},
|
||||
[]token{token{"\u963fQ", 0, 2}, token{"\u8170\u95f4", 2, 4}, token{"\u6302", 4, 5}, token{"\u7740", 5, 6}, token{"BB\u673a", 6, 9}, token{"\u624b\u91cc", 9, 11}, token{"\u62ff", 11, 12}, token{"\u7740", 12, 13}, token{"\u5927\u54e5", 13, 15}, token{"\u5927\u54e5\u5927", 13, 16}, token{"\uff0c", 16, 17}, token{"\u8bf4", 17, 18}, token{"\uff1a", 18, 19}, token{"\u6211", 19, 20}, token{"\u4e00\u822c", 20, 22}, token{"\u5403\u996d", 22, 24}, token{"\u4e0d", 24, 25}, token{"AA\u5236", 25, 28}, token{"\u7684", 28, 29}, token{"\u3002", 29, 30}},
|
||||
[]token{token{"\u5728", 0, 1}, token{"1\u53f7\u5e97", 1, 4}, token{"\u80fd", 4, 5}, token{"\u4e70", 5, 6}, token{"\u5230", 6, 7}, token{"\u5c0fS", 7, 9}, token{"\u548c", 9, 10}, token{"\u5927S", 10, 12}, token{"\u516b\u5366", 12, 14}, token{"\u7684", 14, 15}, token{"\u4e66", 15, 16}, token{"\u3002", 16, 17}},
|
||||
}
|
||||
)
|
||||
|
||||
func TesttokenizeDefaultMode(t *testing.T) {
|
||||
for index, sentence := range test_contents {
|
||||
tokens := Tokenize(sentence, "default", true)
|
||||
if len(tokens) != len(result[index]) {
|
||||
t.Error(len(tokens))
|
||||
}
|
||||
for i, token := range tokens {
|
||||
if token != result[index][i] {
|
||||
t.Error(token)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TesttokenizeNoHMM(t *testing.T) {
|
||||
for index, sentence := range test_contents {
|
||||
tokens := Tokenize(sentence, "default", false)
|
||||
if len(tokens) != len(noHmmResult[index]) {
|
||||
t.Error(len(tokens))
|
||||
}
|
||||
for i, token := range tokens {
|
||||
if token != noHmmResult[index][i] {
|
||||
t.Error(token)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
110
tokenizers/jieba.go
Normal file
110
tokenizers/jieba.go
Normal file
@@ -0,0 +1,110 @@
|
||||
package tokenizers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/blevesearch/bleve/analysis"
|
||||
"github.com/blevesearch/bleve/registry"
|
||||
"github.com/wangbin/jiebago"
|
||||
"regexp"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
const Name = "jieba"
|
||||
|
||||
var IdeographRegexp = regexp.MustCompile(`\p{Han}+`)
|
||||
|
||||
type JiebaTokenizer struct {
|
||||
dictFileName string
|
||||
hmm, searchMode bool
|
||||
}
|
||||
|
||||
func NewJiebaTokenizer(dictFileName string, hmm, searchMode bool) (analysis.Tokenizer, error) {
|
||||
err := jiebago.SetDictionary(dictFileName)
|
||||
return &JiebaTokenizer{
|
||||
dictFileName: dictFileName,
|
||||
hmm: hmm,
|
||||
searchMode: searchMode,
|
||||
}, err
|
||||
}
|
||||
|
||||
func (jt *JiebaTokenizer) Tokenize(input []byte) analysis.TokenStream {
|
||||
rv := make(analysis.TokenStream, 0)
|
||||
runeStart := 0
|
||||
start := 0
|
||||
end := 0
|
||||
pos := 1
|
||||
var width int
|
||||
var gram string
|
||||
for word := range jiebago.Cut(string(input), false, jt.hmm) {
|
||||
if jt.searchMode {
|
||||
runes := []rune(word)
|
||||
width = len(runes)
|
||||
for _, step := range [2]int{2, 3} {
|
||||
if width > step {
|
||||
for i := 0; i < width-step+1; i++ {
|
||||
gram = string(runes[i : i+step])
|
||||
gramLen := len(gram)
|
||||
if value, ok := jiebago.Trie.Freq[gram]; ok && value > 0 {
|
||||
gramStart := start + len(string(runes[:i]))
|
||||
token := analysis.Token{
|
||||
Term: []byte(gram),
|
||||
Start: gramStart,
|
||||
End: gramStart + gramLen,
|
||||
Position: pos,
|
||||
Type: detectTokenType(gram),
|
||||
}
|
||||
rv = append(rv, &token)
|
||||
pos++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
end = start + len(word)
|
||||
token := analysis.Token{
|
||||
Term: []byte(word),
|
||||
Start: start,
|
||||
End: end,
|
||||
Position: pos,
|
||||
Type: detectTokenType(word),
|
||||
}
|
||||
rv = append(rv, &token)
|
||||
pos++
|
||||
runeStart += width
|
||||
start = end
|
||||
}
|
||||
return rv
|
||||
}
|
||||
|
||||
func JiebaTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (
|
||||
analysis.Tokenizer, error) {
|
||||
dictFileName, ok := config["file"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("must specify dictionary file path")
|
||||
}
|
||||
hmm, ok := config["hmm"].(bool)
|
||||
if !ok {
|
||||
hmm = true
|
||||
}
|
||||
searchMode, ok := config["search"].(bool)
|
||||
if !ok {
|
||||
searchMode = true
|
||||
}
|
||||
|
||||
return NewJiebaTokenizer(dictFileName, hmm, searchMode)
|
||||
}
|
||||
|
||||
func detectTokenType(term string) analysis.TokenType {
|
||||
if IdeographRegexp.MatchString(term) {
|
||||
return analysis.Ideographic
|
||||
}
|
||||
_, err := strconv.ParseFloat(term, 64)
|
||||
if err == nil {
|
||||
return analysis.Numeric
|
||||
}
|
||||
return analysis.AlphaNumeric
|
||||
}
|
||||
|
||||
func init() {
|
||||
registry.RegisterTokenizer(Name, JiebaTokenizerConstructor)
|
||||
}
|
||||
22515
tokenizers/jieba_test.go
Normal file
22515
tokenizers/jieba_test.go
Normal file
File diff suppressed because it is too large
Load Diff
29
trie.go
29
trie.go
@@ -1,7 +1,6 @@
|
||||
package jiebago
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
@@ -19,34 +18,6 @@ type trie struct {
|
||||
Freq map[string]float64
|
||||
}
|
||||
|
||||
func (t trie) MarshalBinary() ([]byte, error) {
|
||||
var b bytes.Buffer
|
||||
enc := gob.NewEncoder(&b)
|
||||
err := enc.Encode(t.Total)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = enc.Encode(t.Freq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.Bytes(), nil
|
||||
}
|
||||
|
||||
func (t *trie) UnmarshalBinary(data []byte) error {
|
||||
b := bytes.NewBuffer(data)
|
||||
dec := gob.NewDecoder(b)
|
||||
err := dec.Decode(&t.Total)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = dec.Decode(&t.Freq)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *trie) load(dictFileName string) error {
|
||||
dictFilePath, err := DictPath(dictFileName)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user