From 5c1e948fb8d5947d7ecd367218f2494bb7a65acb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=98=94=E9=9F=B3=E5=B9=BB=E7=A6=BB?= <florwa_natat@qq.com>
Date: Sun, 4 Jan 2026 00:00:01 +0800
Subject: [PATCH] =?UTF-8?q?optimize:=20addToken=20=E6=80=A7=E8=83=BD?=
 =?UTF-8?q?=E4=BC=98=E5=8C=96=20(#1)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 dictionary.go | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/dictionary.go b/dictionary.go
index cb1f72b..840a8cd 100755
--- a/dictionary.go
+++ b/dictionary.go
@@ -36,12 +36,9 @@ func (d *Dictionary) AddToken(token dictionary.Token) {
 func (d *Dictionary) addToken(token dictionary.Token) {
 	d.freqMap[token.Text()] = token.Frequency()
 	d.total += token.Frequency()
-	runes := []rune(token.Text())
-	n := len(runes)
-	for i := 0; i < n; i++ { //TODO: n-1?
-		frag := string(runes[:i+1])
-		if _, ok := d.freqMap[frag]; !ok {
-			d.freqMap[frag] = 0.0
+	for i := range token.Text() {
+		if _, ok := d.freqMap[token.Text()[:i]]; i > 0 && !ok {
+			d.freqMap[token.Text()[:i]] = 0.0
 		}
 	}
 }