use github.com/deckarep/golang-set instead of Trie, to reduce memory usage and improve performance, this is corresponding to jieba commit #4a93f21918a26083c039970edb9457c589c3a0ab

tokenize.go

@@ -24,7 +24,7 @@ func Tokenize(sentence string, mode string, HMM bool) []Token {
 				if width > step {
 					for i := 0; i < width-step+1; i++ {
 						gram := string(runes[i : i+step])
-						if _, ok := TT.Freq[gram]; ok {
+						if _, ok := trie.Freq[gram]; ok {
 							tokens = append(tokens, Token{gram, start + i, start + i + step})
 						}
 					}