feat(me): concurrency send

.github/workflows/ci.yml

@@ -18,6 +18,9 @@ jobs:
 
       - name: Get dependencies
         run: go mod tidy
+      
+      - name: Enable debug log
+        run: sed -i 's/false/true/g' config/global.go
 
       - name: Build
         run: go build -v ./...

README.md

@@ -47,7 +47,7 @@ Peers:
     PublicKey: 徯萃嵾爻燸攗窍褃冔蒔犡緇袿屿組待族砇嘀
     PresharedKey: 瀸敀爅崾嘊嵜紼樴稍毯攣矐訷蟷扛嬋庩崛昀
     EndPoint: 1.2.3.4:56789
-    AllowedIPs: ["192.168.233.2/32", "x192.168.233.3/32"] # allow trans to 192.168.233.3, but don not create route
+    AllowedIPs: ["192.168.233.2/32", "x192.168.233.3/32"] # accept packets from 192.168.233.3, but don not create route
     KeepAliveSeconds: 0
     QueryList: ["192.168.233.3"]
     MTU: 1400
@@ -60,8 +60,9 @@ Peers:
     PublicKey: 牢喨粷詸衭譛浾蘹櫠砙杹蟫瑳叩刋橋経挵蘀
     PresharedKey: 竅琚喫従痸告烈兇厕趭萨假蔛瀇譄施烸蝫瘀
     EndPoint: ""
-    AllowedIPs: ["192.168.233.3/32"]
+    AllowedIPs: ["192.168.233.3/32", "y192.168.66.1/32"] # add route to 192.168.66.1 into inner route table but do not add it to system one
     MTU: 752
+    DoublePacket: true
     KeepAliveSeconds: 0
     AllowTrans: false
 ```

config/cfg.go

@@ -2,9 +2,9 @@ package config
 
 import (
 	"bytes"
-	"log"
 	"os"
 
+	"github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
 )
 
@@ -15,7 +15,7 @@ type Config struct {
 	PrivateKey string `yaml:"PrivateKey"`
 	Network    string `yaml:"Network"` // Network udp, tcp or ws (WIP)
 	EndPoint   string `yaml:"EndPoint"`
-	MTU        int64  `yaml:"MTU"`
+	MTU        int64  `yaml:"MTU"` // MTU of nic (will minus packet header len)
 	SpeedLoop  uint16 `yaml:"SpeedLoop"`
 	Mask       uint64 `yaml:"Mask"` // Mask 是异或报文所用掩码, 必须保证各端统一
 	Peers      []Peer `yaml:"Peers"`
@@ -33,7 +33,8 @@ type Peer struct {
 	QuerySeconds     int64    `yaml:"QuerySeconds"`
 	AllowTrans       bool     `yaml:"AllowTrans"`
 	UseZstd          bool     `yaml:"UseZstd"`
-	MTU              int64    `yaml:"MTU"`
+	DoublePacket     bool     `yaml:"DoublePacket"`
+	MTU              int64    `yaml:"MTU"` // MTU of PDU passed to p2p
 	MTURandomRange   int64    `yaml:"MTURandomRange"`
 }
 
@@ -41,11 +42,11 @@ type Peer struct {
 func Parse(path string) (c Config) {
 	file, err := os.ReadFile(path)
 	if err != nil {
-		log.Fatal("open config file failed:", err)
+		logrus.Fatal("open config file failed:", err)
 	}
 	err = yaml.NewDecoder(bytes.NewReader(file)).Decode(&c)
 	if err != nil {
-		log.Fatal("invalid config file:", err)
+		logrus.Fatal("invalid config file:", err)
 	}
 	return
 }

config/global.go

@@ -0,0 +1,3 @@
+package config
+
+const ShowDebugLog = false

gold/head/flags.go

@@ -0,0 +1,34 @@
+package head
+
+import "encoding/binary"
+
+type PacketFlags uint16
+
+func (pf PacketFlags) IsValid() bool {
+	return pf&0x8000 == 0
+}
+
+func (pf PacketFlags) DontFrag() bool {
+	return pf&0x4000 == 0x4000
+}
+
+func (pf PacketFlags) NoFrag() bool {
+	return pf == 0x4000
+}
+
+func (pf PacketFlags) IsSingle() bool {
+	return pf == 0
+}
+
+func (pf PacketFlags) ZeroOffset() bool {
+	return pf&0x1fff == 0
+}
+
+func (pf PacketFlags) Offset() uint16 {
+	return uint16(pf << 3)
+}
+
+// Flags extract flags from raw data
+func Flags(data []byte) PacketFlags {
+	return PacketFlags(binary.LittleEndian.Uint16(data[10:12]))
+}

gold/head/packet.go

@@ -4,12 +4,13 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"errors"
-	"hash/crc64"
 	"net"
 
-	"github.com/fumiama/WireGold/helper"
 	blake2b "github.com/fumiama/blake2b-simd"
 	"github.com/sirupsen/logrus"
+
+	"github.com/fumiama/WireGold/config"
+	"github.com/fumiama/WireGold/helper"
 )
 
 const PacketHeadLen = 60
@@ -19,37 +20,6 @@ var (
 	ErrDataLenLT60    = errors.New("data len < 60")
 )
 
-type PacketFlags uint16
-
-func (pf PacketFlags) IsValid() bool {
-	return pf&0x8000 == 0
-}
-
-func (pf PacketFlags) DontFrag() bool {
-	return pf&0x4000 == 0x4000
-}
-
-func (pf PacketFlags) NoFrag() bool {
-	return pf == 0x4000
-}
-
-func (pf PacketFlags) IsSingle() bool {
-	return pf == 0
-}
-
-func (pf PacketFlags) ZeroOffset() bool {
-	return pf&0x1fff == 0
-}
-
-func (pf PacketFlags) Offset() uint16 {
-	return uint16(pf << 3)
-}
-
-// Flags extract flags from raw data
-func Flags(data []byte) PacketFlags {
-	return PacketFlags(binary.LittleEndian.Uint16(data[10:12]))
-}
-
 // Packet 是发送和接收的最小单位
 type Packet struct {
 	// idxdatsz len(Data)
@@ -75,7 +45,7 @@ type Packet struct {
 	// 生成时 Hash 全 0
 	// https://github.com/fumiama/blake2b-simd
 	Hash [32]byte
-	// crc64 包头字段的 checksum 值，可以认为在一定时间内唯一
+	// crc64 包头字段的 checksum 值，可以认为在一定时间内唯一 (现已更改算法为 md5 但名字未变)
 	crc64 uint64
 	// data 承载的数据
 	data []byte
@@ -89,7 +59,6 @@ type Packet struct {
 
 // NewPacket 生成一个新包
 func NewPacket(proto uint8, srcPort uint16, dst net.IP, dstPort uint16, data []byte) (p *Packet) {
-	// logrus.Debugln("[packet] new: [proto:", proto, ", srcport:", srcPort, ", dstport:", dstPort, ", dst:", dst, ", data:", data)
 	p = SelectPacket()
 	p.Proto = proto
 	p.TTL = 16
@@ -107,8 +76,8 @@ func (p *Packet) Unmarshal(data []byte) (complete bool, err error) {
 		err = ErrDataLenLT60
 		return
 	}
-	p.crc64 = binary.LittleEndian.Uint64(data[52:PacketHeadLen])
-	if crc64.Checksum(data[:52], crc64.MakeTable(crc64.ISO)) != p.crc64 {
+	p.crc64 = CRC64(data)
+	if CalcCRC64(data) != p.crc64 {
 		err = ErrBadCRCChecksum
 		return
 	}
@@ -147,7 +116,9 @@ func (p *Packet) Unmarshal(data []byte) (complete bool, err error) {
 
 	if p.rembytes > 0 {
 		p.rembytes -= copy(p.data[flags.Offset():], data[PacketHeadLen:])
-		logrus.Debugln("[packet] copied frag", hex.EncodeToString(p.Hash[:]), "rembytes:", p.rembytes)
+		if config.ShowDebugLog {
+			logrus.Debugln("[packet] copied frag", hex.EncodeToString(p.Hash[:]), "rembytes:", p.rembytes)
+		}
 	}
 
 	complete = p.rembytes == 0
@@ -155,14 +126,15 @@ func (p *Packet) Unmarshal(data []byte) (complete bool, err error) {
 	return
 }
 
+// DecreaseAndGetTTL TTL 自减后返回
+func (p *Packet) DecreaseAndGetTTL() uint8 {
+	p.TTL--
+	return p.TTL
+}
+
 // Marshal 将自身数据编码为 []byte
 // offset 必须为 8 的倍数，表示偏移的 8 位
 func (p *Packet) Marshal(src net.IP, teatype uint8, additional uint16, datasz uint32, offset uint16, dontfrag, hasmore bool) ([]byte, func()) {
-	p.TTL--
-	if p.TTL == 0 {
-		return nil, nil
-	}
-
 	if src != nil {
 		p.Src = src
 		p.idxdatsz = (uint32(teatype) << 27) | (uint32(additional&0x07ff) << 16) | datasz&0xffff
@@ -186,7 +158,8 @@ func (p *Packet) Marshal(src net.IP, teatype uint8, additional uint16, datasz ui
 		w.Write(p.Src.To4())
 		w.Write(p.Dst.To4())
 		w.Write(p.Hash[:])
-		w.WriteUInt64(crc64.Checksum(w.Bytes(), crc64.MakeTable(crc64.ISO)))
+		p.crc64 = CalcCRC64(w.Bytes())
+		w.WriteUInt64(p.crc64)
 		w.Write(p.Body())
 	})
 }
@@ -196,10 +169,13 @@ func (p *Packet) FillHash() {
 	h := blake2b.New256()
 	_, err := h.Write(p.Body())
 	if err != nil {
-		logrus.Error("[packet] err when fill hash:", err)
+		logrus.Errorln("[packet] err when fill hash:", err)
 		return
 	}
-	logrus.Debugln("[packet] sum calulated:", hex.EncodeToString(h.Sum(p.Hash[:0])))
+	hsh := h.Sum(p.Hash[:0])
+	if config.ShowDebugLog {
+		logrus.Debugln("[packet] sum calulated:", hex.EncodeToString(hsh))
+	}
 }
 
 // IsVaildHash 验证 packet 合法性
@@ -207,12 +183,15 @@ func (p *Packet) IsVaildHash() bool {
 	h := blake2b.New256()
 	_, err := h.Write(p.Body())
 	if err != nil {
-		logrus.Error("[packet] err when check hash:", err)
+		logrus.Errorln("[packet] err when check hash:", err)
 		return false
 	}
 	var sum [32]byte
-	logrus.Debugln("[packet] sum calulated:", hex.EncodeToString(h.Sum(sum[:0])))
-	logrus.Debugln("[packet] sum in packet:", hex.EncodeToString(p.Hash[:]))
+	_ = h.Sum(sum[:0])
+	if config.ShowDebugLog {
+		logrus.Debugln("[packet] sum calulated:", hex.EncodeToString(sum[:]))
+		logrus.Debugln("[packet] sum in packet:", hex.EncodeToString(p.Hash[:]))
+	}
 	return sum == p.Hash
 }
 
@@ -236,6 +215,10 @@ func (p *Packet) Put() {
 	PutPacket(p)
 }
 
+func (p *Packet) CRC64() uint64 {
+	return p.crc64
+}
+
 // Body returns data
 func (p *Packet) Body() []byte {
 	return p.data[p.a:p.b]
@@ -279,3 +262,10 @@ func (p *Packet) Copy() *Packet {
 	newp.buffered = false
 	return newp
 }
+
+func (p *Packet) CopyWithBody() *Packet {
+	newp := p.Copy()
+	newp.data = helper.MakeBytes(len(p.data))
+	copy(newp.data, p.data)
+	return newp
+}

gold/head/raw.go

@@ -0,0 +1,22 @@
+package head
+
+import (
+	"crypto/md5"
+	"encoding/binary"
+)
+
+// CRC64 extract packet header checksum
+func CRC64(data []byte) uint64 {
+	return binary.LittleEndian.Uint64(data[52:PacketHeadLen])
+}
+
+// CalcCRC64 calculate packet header checksum
+func CalcCRC64(data []byte) uint64 {
+	m := md5.Sum(data[:52])
+	return binary.LittleEndian.Uint64(m[:8])
+}
+
+// Hash extract 32 bytes blake2b hash from raw bytes
+func Hash(data []byte) []byte {
+	return data[20:52]
+}

gold/link/crypto.go

@@ -121,29 +121,37 @@ func decode(aead cipher.AEAD, additional uint16, b []byte) ([]byte, error) {
 }
 
 // xorenc 按 8 字节, 以初始 m.mask 循环异或编码 data
-func (m *Me) xorenc(data []byte) []byte {
+func (m *Me) xorenc(data []byte, seq uint32) []byte {
 	batchsz := len(data) / 8
 	remain := len(data) % 8
 	sum := m.mask
+	newdat := helper.MakeBytes(len(data) + 8)
+	binary.LittleEndian.PutUint32(newdat[:4], seq)
+	_, _ = rand.Read(newdat[4:8])
 	if remain > 0 {
 		var buf [8]byte
 		p := batchsz * 8
 		copy(buf[:], data[p:])
 		sum ^= binary.LittleEndian.Uint64(buf[:])
 		binary.LittleEndian.PutUint64(buf[:], sum)
-		copy(data[p:], buf[:])
+		copy(newdat[8+p:], buf[:])
 	}
 	for i := batchsz - 1; i >= 0; i-- {
 		a := i * 8
 		b := (i + 1) * 8
 		sum ^= binary.LittleEndian.Uint64(data[a:b])
-		binary.LittleEndian.PutUint64(data[a:b], sum)
+		binary.LittleEndian.PutUint64(newdat[a+8:b+8], sum)
 	}
-	return data
+	sum ^= binary.LittleEndian.Uint64(newdat[:8])
+	binary.LittleEndian.PutUint64(newdat[:8], sum)
+	return newdat
 }
 
 // xordec 按 8 字节, 以初始 m.mask 循环异或解码 data
-func (m *Me) xordec(data []byte) []byte {
+func (m *Me) xordec(data []byte) (uint32, []byte) {
+	if len(data) <= 8 {
+		return 0, nil
+	}
 	batchsz := len(data) / 8
 	remain := len(data) % 8
 	this := uint64(0)
@@ -173,5 +181,5 @@ func (m *Me) xordec(data []byte) []byte {
 	} else {
 		binary.LittleEndian.PutUint64(data[len(data)-8:], next^m.mask)
 	}
-	return data
+	return binary.LittleEndian.Uint32(data[:4]), data[8:]
 }

gold/link/crypto_test.go

@@ -27,8 +27,9 @@ func TestXOR(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		if !bytes.Equal(m.xordec(m.xorenc(r1.Bytes())), r2.Bytes()) {
-			t.Fatal("unexpected xor at", i)
+		seq, dec := m.xordec(m.xorenc(r1.Bytes(), uint32(i)))
+		if !bytes.Equal(dec, r2.Bytes()) || seq != uint32(i) {
+			t.Fatal("unexpected xor at", i, "seq", seq)
 		}
 	}
 }

gold/link/link.go

@@ -44,6 +44,8 @@ type Link struct {
 	allowtrans bool
 	// 是否对数据进行 zstd 压缩
 	usezstd bool
+	// 是否采用双倍发包对抗强丢包
+	doublepacket bool
 	// udp 数据包的最大大小
 	mtu uint16
 	// 随机放缩 mtu 范围 (只减不增)

gold/link/listen.go

@@ -13,6 +13,7 @@ import (
 	"github.com/klauspost/compress/zstd"
 	"github.com/sirupsen/logrus"
 
+	"github.com/fumiama/WireGold/config"
 	"github.com/fumiama/WireGold/gold/head"
 	"github.com/fumiama/WireGold/gold/p2p"
 	"github.com/fumiama/WireGold/helper"
@@ -20,6 +21,52 @@ import (
 
 const lstnbufgragsz = 65536
 
+type lstnq struct {
+	index int
+	addr  p2p.EndPoint
+	buf   []byte
+}
+
+type listenqueue chan lstnq
+
+func (q listenqueue) listen(m *Me, hasntfinished []sync.Mutex) {
+	recvtotlcnt := uint64(0)
+	recvloopcnt := uint16(0)
+	recvlooptime := time.Now().UnixMilli()
+	for lstn := range q {
+		recvtotlcnt += uint64(len(lstn.buf))
+		recvloopcnt++
+		if recvloopcnt%m.speedloop == 0 {
+			now := time.Now().UnixMilli()
+			logrus.Infof("[listen] queue recv avg speed: %.2f KB/s", float64(recvtotlcnt)/float64(now-recvlooptime))
+			recvtotlcnt = 0
+			recvlooptime = now
+		}
+		packet := m.wait(lstn.buf[:len(lstn.buf):lstnbufgragsz])
+		if packet == nil {
+			if lstn.index < 0 {
+				if config.ShowDebugLog {
+					logrus.Debugln("[listen] queue waiting")
+				}
+				helper.PutBytes(lstn.buf)
+				continue
+			}
+			if config.ShowDebugLog {
+				logrus.Debugln("[listen] queue waiting, unlock index", lstn.index)
+			}
+			hasntfinished[lstn.index].Unlock()
+			continue
+		}
+		if lstn.index >= 0 {
+			go m.dispatch(packet, lstn.addr, lstn.index, hasntfinished[lstn.index].Unlock)
+		} else {
+			go m.dispatch(packet, lstn.addr, lstn.index, func() {
+				helper.PutBytes(lstn.buf)
+			})
+		}
+	}
+}
+
 // 监听本机 endpoint
 func (m *Me) listen() (conn p2p.Conn, err error) {
 	conn, err = m.ep.Listen()
@@ -29,27 +76,39 @@ func (m *Me) listen() (conn p2p.Conn, err error) {
 	m.ep = conn.LocalAddr()
 	logrus.Infoln("[listen] at", m.ep)
 	go func() {
-		recvtotlcnt := uint64(0)
-		recvloopcnt := uint16(0)
-		recvlooptime := time.Now().UnixMilli()
-		n := runtime.NumCPU()
+		n := uint(runtime.NumCPU())
 		if n > 64 {
 			n = 64 // 只用最多 64 核
 		}
 		logrus.Infoln("[listen] use cpu num:", n)
-		listenbuff := make([]byte, lstnbufgragsz*n)
+		listenbuf := make([]byte, lstnbufgragsz*n)
 		hasntfinished := make([]sync.Mutex, n)
-		for i := 0; err == nil; i++ {
-			i %= n
+		q := make(listenqueue, n)
+		defer close(q)
+		go q.listen(m, hasntfinished)
+		for {
+			usenewbuf := false
+			i := uint(0)
 			for !hasntfinished[i].TryLock() {
 				i++
 				i %= n
-				if i == 0 { // looked up a full round
-					time.Sleep(time.Millisecond * 10)
+				if i == 0 { // looked up a full round, make a new buf
+					usenewbuf = true
+					if config.ShowDebugLog {
+						logrus.Debugln("[listen] use new buf")
+					}
+					break
 				}
 			}
-			logrus.Debugln("[listen] lock index", i)
-			lbf := listenbuff[i*lstnbufgragsz : (i+1)*lstnbufgragsz]
+			if config.ShowDebugLog && !usenewbuf {
+				logrus.Debugln("[listen] lock index", i)
+			}
+			var lbf []byte
+			if usenewbuf {
+				lbf = helper.MakeBytes(lstnbufgragsz)
+			} else {
+				lbf = listenbuf[i*lstnbufgragsz : (i+1)*lstnbufgragsz]
+			}
 			n, addr, err := conn.ReadFromPeer(lbf)
 			if m.connections == nil || errors.Is(err, net.ErrClosed) {
 				logrus.Warnln("[listen] quit listening")
@@ -62,27 +121,24 @@ func (m *Me) listen() (conn p2p.Conn, err error) {
 					logrus.Errorln("[listen] reconnect udp err:", err)
 					return
 				}
-				logrus.Debugln("[listen] unlock index", i)
-				hasntfinished[i].Unlock()
-				i--
+				if !usenewbuf {
+					if config.ShowDebugLog {
+						logrus.Debugln("[listen] unlock index", i)
+					}
+					hasntfinished[i].Unlock()
+					i--
+				}
 				continue
 			}
-			recvtotlcnt += uint64(n)
-			recvloopcnt++
-			if recvloopcnt%m.speedloop == 0 {
-				now := time.Now().UnixMilli()
-				logrus.Infof("[listen] recv avg speed: %.2f KB/s", float64(recvtotlcnt)/float64(now-recvlooptime))
-				recvtotlcnt = 0
-				recvlooptime = now
+			lq := lstnq{
+				index: -1,
+				addr:  addr,
+				buf:   lbf[:n],
 			}
-			packet := m.wait(lbf[:n:lstnbufgragsz])
-			if packet == nil {
-				logrus.Debugln("[listen] waiting, unlock index", i)
-				hasntfinished[i].Unlock()
-				i--
-				continue
+			if !usenewbuf {
+				lq.index = int(i)
 			}
-			go m.dispatch(packet, addr, i, hasntfinished[i].Unlock)
+			q <- lq
 		}
 	}()
 	return
@@ -90,8 +146,10 @@ func (m *Me) listen() (conn p2p.Conn, err error) {
 
 func (m *Me) dispatch(packet *head.Packet, addr p2p.EndPoint, index int, finish func()) {
 	defer finish()
-	defer logrus.Debugln("[listen] dispatched, unlock index", index)
-	logrus.Debugln("[listen] start dispatching index", index)
+	if config.ShowDebugLog {
+		defer logrus.Debugln("[listen] dispatched, unlock index", index)
+		logrus.Debugln("[listen] start dispatching index", index)
+	}
 	r := packet.Len() - packet.BodyLen()
 	if r > 0 {
 		logrus.Warnln("[listen] @", index, "packet from endpoint", addr, "len", packet.BodyLen(), "is smaller than it declared len", packet.Len(), ", drop it")
@@ -99,7 +157,9 @@ func (m *Me) dispatch(packet *head.Packet, addr p2p.EndPoint, index int, finish
 		return
 	}
 	p, ok := m.IsInPeer(packet.Src.String())
-	logrus.Debugln("[listen] @", index, "recv from endpoint", addr, "src", packet.Src, "dst", packet.Dst)
+	if config.ShowDebugLog {
+		logrus.Debugln("[listen] @", index, "recv from endpoint", addr, "src", packet.Src, "dst", packet.Dst)
+	}
 	if !ok {
 		logrus.Warnln("[listen] @", index, "packet from", packet.Src, "to", packet.Dst, "is refused")
 		packet.Put()
@@ -125,7 +185,9 @@ func (m *Me) dispatch(packet *head.Packet, addr p2p.EndPoint, index int, finish
 		var err error
 		data, err := p.Decode(packet.CipherIndex(), addt, packet.Body())
 		if err != nil {
-			logrus.Debugln("[listen] @", index, "drop invalid packet key idx:", packet.CipherIndex(), "addt:", addt, "err:", err)
+			if config.ShowDebugLog {
+				logrus.Debugln("[listen] @", index, "drop invalid packet key idx:", packet.CipherIndex(), "addt:", addt, "err:", err)
+			}
 			packet.Put()
 			return
 		}
@@ -137,14 +199,18 @@ func (m *Me) dispatch(packet *head.Packet, addr p2p.EndPoint, index int, finish
 			_, err = io.Copy(w, dec)
 			dec.Close()
 			if err != nil {
-				logrus.Debugln("[listen] @", index, "drop invalid zstd packet:", err)
+				if config.ShowDebugLog {
+					logrus.Debugln("[listen] @", index, "drop invalid zstd packet:", err)
+				}
 				packet.Put()
 				return
 			}
 			packet.SetBody(w.Bytes(), true)
 		}
 		if !packet.IsVaildHash() {
-			logrus.Debugln("[listen] @", index, "drop invalid hash packet")
+			if config.ShowDebugLog {
+				logrus.Debugln("[listen] @", index, "drop invalid hash packet")
+			}
 			packet.Put()
 			return
 		}
@@ -154,7 +220,9 @@ func (m *Me) dispatch(packet *head.Packet, addr p2p.EndPoint, index int, finish
 			case LINK_STATUS_DOWN:
 				n, err := p.WriteAndPut(head.NewPacket(head.ProtoHello, m.SrcPort(), p.peerip, m.DstPort(), nil), false)
 				if err == nil {
-					logrus.Debugln("[listen] @", index, "send", n, "bytes hello ack packet")
+					if config.ShowDebugLog {
+						logrus.Debugln("[listen] @", index, "send", n, "bytes hello ack packet")
+					}
 					p.status = LINK_STATUS_HALFUP
 				} else {
 					logrus.Errorln("[listen] @", index, "send hello ack packet error:", err)
@@ -174,17 +242,19 @@ func (m *Me) dispatch(packet *head.Packet, addr p2p.EndPoint, index int, finish
 			packet.Put()
 		case head.ProtoData:
 			if p.pipe != nil {
-				p.pipe <- packet
-				logrus.Debugln("[listen] @", index, "deliver to pipe of", p.peerip)
+				p.pipe <- packet.CopyWithBody()
+				if config.ShowDebugLog {
+					logrus.Debugln("[listen] @", index, "deliver to pipe of", p.peerip)
+				}
 			} else {
 				_, err := m.nic.Write(packet.Body())
 				if err != nil {
 					logrus.Errorln("[listen] @", index, "deliver", packet.BodyLen(), "bytes data to nic err:", err)
-				} else {
+				} else if config.ShowDebugLog {
 					logrus.Debugln("[listen] @", index, "deliver", packet.BodyLen(), "bytes data to nic")
 				}
-				packet.Put()
 			}
+			packet.Put()
 		default:
 			logrus.Warnln("[listen] @", index, "recv unknown proto:", packet.Proto)
 			packet.Put()
@@ -204,7 +274,9 @@ func (m *Me) dispatch(packet *head.Packet, addr p2p.EndPoint, index int, finish
 		}
 		n, err := lnk.WriteAndPut(packet, true)
 		if err == nil {
-			logrus.Debugln("[listen] @", index, "trans", n, "bytes packet to", packet.Dst.String()+":"+strconv.Itoa(int(packet.DstPort)))
+			if config.ShowDebugLog {
+				logrus.Debugln("[listen] @", index, "trans", n, "bytes packet to", packet.Dst.String()+":"+strconv.Itoa(int(packet.DstPort)))
+			}
 		} else {
 			logrus.Errorln("[listen] @", index, "trans packet to", packet.Dst.String()+":"+strconv.Itoa(int(packet.DstPort)), "err:", err)
 		}

gold/link/me.go

@@ -13,8 +13,10 @@ import (
 	"github.com/fumiama/water/waterutil"
 	"github.com/sirupsen/logrus"
 
+	"github.com/fumiama/WireGold/config"
 	"github.com/fumiama/WireGold/gold/head"
 	"github.com/fumiama/WireGold/gold/p2p"
+	"github.com/fumiama/WireGold/helper"
 	"github.com/fumiama/WireGold/lower"
 )
 
@@ -38,11 +40,11 @@ type Me struct {
 	// 本机监听的连接端点, 也用于向对端直接发送报文
 	conn p2p.Conn
 	// 本机网卡
-	nic lower.NICIO
+	nic *lower.NICIO
 	// 本机路由表
 	router *Router
 	// 本机未接收完全分片池
-	recving *ttl.Cache[[32]byte, *head.Packet]
+	recving *ttl.Cache[uint64, *head.Packet]
 	// 抗重放攻击记录池
 	recved *ttl.Cache[uint64, bool]
 	// 本机上层配置
@@ -59,11 +61,17 @@ type MyConfig struct {
 	Network                          string
 	NetworkConfigs                   []any
 	PrivateKey                       *[32]byte
-	NIC                              lower.NICIO
+	NICConfig                        *NICConfig
 	SrcPort, DstPort, MTU, SpeedLoop uint16
 	Mask                             uint64
 }
 
+type NICConfig struct {
+	IP     net.IP
+	SubNet *net.IPNet
+	CIDRs  []string
+}
+
 // NewMe 设置本机参数
 func NewMe(cfg *MyConfig) (m Me) {
 	m.privKey = *cfg.PrivateKey
@@ -83,12 +91,15 @@ func NewMe(cfg *MyConfig) (m Me) {
 	}
 	m.me = ip
 	m.subnet = *cidr
+	m.speedloop = cfg.SpeedLoop
+	if m.speedloop == 0 {
+		m.speedloop = 4096
+	}
 	m.conn, err = m.listen()
 	if err != nil {
 		panic(err)
 	}
 	m.connections = make(map[string]*Link)
-	m.nic = cfg.NIC
 	m.router = &Router{
 		list:  make([]*net.IPNet, 1, 16),
 		table: make(map[string]*Link, 16),
@@ -97,16 +108,18 @@ func NewMe(cfg *MyConfig) (m Me) {
 	m.router.SetDefault(nil)
 	m.srcport = cfg.SrcPort
 	m.dstport = cfg.DstPort
-	m.mtu = cfg.MTU & 0xfff8
-	m.speedloop = cfg.SpeedLoop
-	if m.speedloop == 0 {
-		m.speedloop = 4096
+	m.mtu = (cfg.MTU - head.PacketHeadLen) & 0xfff8
+	if cfg.NICConfig != nil {
+		m.nic = lower.NewNIC(
+			cfg.NICConfig.IP, cfg.NICConfig.SubNet,
+			strconv.FormatUint(uint64(m.MTU()), 10), cfg.NICConfig.CIDRs...,
+		)
 	}
 	m.mask = cfg.Mask
 	var buf [8]byte
 	binary.BigEndian.PutUint64(buf[:], m.mask)
 	logrus.Infoln("[me] xor mask", hex.EncodeToString(buf[:]))
-	m.recving = ttl.NewCache[[32]byte, *head.Packet](time.Second * 30)
+	m.recving = ttl.NewCache[uint64, *head.Packet](time.Second * 30)
 	m.recved = ttl.NewCache[uint64, bool](time.Second * 30)
 	return
 }
@@ -151,7 +164,9 @@ func (m *Me) Close() error {
 
 func (m *Me) Write(packet []byte) (n int, err error) {
 	n = m.sendAllSameDst(packet)
-	logrus.Debugln("[me] writer ate", len(packet), "bytes, remain", len(packet)-n, "bytes")
+	if config.ShowDebugLog {
+		logrus.Debugln("[me] writer ate", len(packet), "bytes, remain", len(packet)-n, "bytes")
+	}
 	return
 }
 
@@ -180,7 +195,9 @@ func (m *Me) sendAllSameDst(packet []byte) (n int) {
 			}
 			n += pktl
 			rem = packet[n:]
-			logrus.Debugln("[me] skip to send", len(packet), "bytes ipv6 packet")
+			if config.ShowDebugLog {
+				logrus.Debugln("[me] skip to send", len(packet), "bytes ipv6 packet")
+			}
 		}
 		if len(rem) == 0 || !waterutil.IsIPv4(rem) {
 			logrus.Warnln("[me] skip to send", len(packet), "bytes full packet")
@@ -193,12 +210,16 @@ func (m *Me) sendAllSameDst(packet []byte) (n int) {
 	for len(ptr) > 20 && p.issame(ptr) {
 		totl := waterutil.IPv4TotalLength(ptr)
 		if int(totl) > len(ptr) {
-			logrus.Debugln("[me] wrap got invalid totl, break")
+			if config.ShowDebugLog {
+				logrus.Debugln("[me] wrap got invalid totl, break")
+			}
 			break
 		}
 		i += int(totl)
 		ptr = rem[i:]
-		logrus.Debugln("[me] wrap", totl, "bytes packet to send together")
+		if config.ShowDebugLog {
+			logrus.Debugln("[me] wrap", totl, "bytes packet to send together")
+		}
 	}
 	if i == 0 {
 		return
@@ -207,9 +228,13 @@ func (m *Me) sendAllSameDst(packet []byte) (n int) {
 	packet = rem[:i]
 	rem = rem[i:]
 	dst := waterutil.IPv4Destination(packet)
-	logrus.Debugln("[me] sending", len(packet), "bytes packet from :"+strconv.Itoa(int(m.SrcPort())), "to", dst.String()+":"+strconv.Itoa(int(m.DstPort())), "remain:", len(rem), "bytes")
+	if config.ShowDebugLog {
+		logrus.Debugln("[me] sending", len(packet), "bytes packet from :"+strconv.Itoa(int(m.SrcPort())), "to", dst.String()+":"+strconv.Itoa(int(m.DstPort())), "remain:", len(rem), "bytes")
+	}
 	if m.me.Equal(dst) { // is to myself, write to nic (pipe not allow loopback)
-		logrus.Debugln("[me] loopback packet")
+		if config.ShowDebugLog {
+			logrus.Debugln("[me] loopback packet")
+		}
 		_, err := m.nic.Write(packet)
 		if err != nil {
 			logrus.Warnln("[me] write to loopback err:", err)
@@ -221,9 +246,14 @@ func (m *Me) sendAllSameDst(packet []byte) (n int) {
 		logrus.Warnln("[me] drop packet to", dst.String()+":"+strconv.Itoa(int(m.DstPort())), ": nil nexthop")
 		return
 	}
-	_, err := lnk.WriteAndPut(head.NewPacket(head.ProtoData, m.SrcPort(), lnk.peerip, m.DstPort(), packet), false)
-	if err != nil {
-		logrus.Warnln("[me] write to peer", lnk.peerip, "err:", err)
-	}
+	pcp := helper.MakeBytes(len(packet))
+	copy(pcp, packet)
+	go func(packet []byte) {
+		defer helper.PutBytes(packet)
+		_, err := lnk.WriteAndPut(head.NewPacket(head.ProtoData, m.SrcPort(), lnk.peerip, m.DstPort(), packet), false)
+		if err != nil {
+			logrus.Warnln("[me] write to peer", lnk.peerip, "err:", err)
+		}
+	}(pcp)
 	return
 }

gold/link/nat.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 
+	"github.com/fumiama/WireGold/config"
 	"github.com/fumiama/WireGold/gold/head"
 	"github.com/fumiama/WireGold/gold/p2p"
 	"github.com/fumiama/WireGold/helper"
@@ -63,7 +64,9 @@ func (l *Link) onNotify(packet []byte) {
 				continue
 			}
 		}
-		logrus.Debugln("[nat] notify drop invalid peer:", peer, "ep:", ep)
+		if config.ShowDebugLog {
+			logrus.Debugln("[nat] notify drop invalid peer:", peer, "ep:", ep)
+		}
 	}
 }
 

gold/link/peer.go

@@ -22,6 +22,7 @@ type PeerConfig struct {
 	MTURandomRange          uint16
 	AllowTrans, NoPipe      bool
 	UseZstd                 bool
+	DoublePacket            bool
 }
 
 // AddPeer 添加一个 peer
@@ -32,7 +33,7 @@ func (m *Me) AddPeer(cfg *PeerConfig) (l *Link) {
 	if ok {
 		return
 	}
-	if cfg.MTU == 0 || (m.mtu != 0 && cfg.MTU > m.mtu) {
+	if cfg.MTU == 0 {
 		panic("invalid mtu for peer " + cfg.PeerIP)
 	}
 	l = &Link{
@@ -41,6 +42,7 @@ func (m *Me) AddPeer(cfg *PeerConfig) (l *Link) {
 		rawep:          cfg.EndPoint,
 		allowtrans:     cfg.AllowTrans,
 		usezstd:        cfg.UseZstd,
+		doublepacket:   cfg.DoublePacket,
 		me:             m,
 		mtu:            cfg.MTU,
 		mturandomrange: cfg.MTURandomRange,
@@ -87,7 +89,8 @@ func (m *Me) AddPeer(cfg *PeerConfig) (l *Link) {
 				continue
 			}
 			noroute := ipnet[0] == 'x'
-			if noroute {
+			innerroute := ipnet[0] == 'y'
+			if noroute || innerroute {
 				ipnet = ipnet[1:]
 				if len(ipnet) == 0 {
 					continue

gold/link/recv.go

@@ -3,9 +3,10 @@ package link
 import (
 	"encoding/binary"
 	"encoding/hex"
+	"hash/crc64"
 	"strconv"
-	"unsafe"
 
+	"github.com/fumiama/WireGold/config"
 	"github.com/fumiama/WireGold/gold/head"
 	"github.com/sirupsen/logrus"
 )
@@ -25,56 +26,80 @@ func (m *Me) wait(data []byte) *head.Packet {
 		bound = len(data)
 		endl = "."
 	}
-	logrus.Debugln("[recv] data bytes", hex.EncodeToString(data[:bound]), endl)
-	data = m.xordec(data)
-	logrus.Debugln("[recv] data xored", hex.EncodeToString(data[:bound]), endl)
+	if config.ShowDebugLog {
+		logrus.Debugln("[recv] data bytes", hex.EncodeToString(data[:bound]), endl)
+	}
+	seq, data := m.xordec(data)
+	if config.ShowDebugLog {
+		logrus.Debugln("[recv] data xored", hex.EncodeToString(data[:bound]), endl)
+	}
 	flags := head.Flags(data)
 	if !flags.IsValid() {
-		logrus.Debugln("[recv] drop invalid flags packet:", hex.EncodeToString(data[11:12]), hex.EncodeToString(data[10:11]))
+		if config.ShowDebugLog {
+			logrus.Debugln("[recv] drop invalid flags packet:", hex.EncodeToString(data[11:12]), hex.EncodeToString(data[10:11]))
+		}
 		return nil
 	}
-	crc := binary.LittleEndian.Uint64(data[52:head.PacketHeadLen])
-	if m.recved.Get(crc) { // 是重放攻击
-		logrus.Warnln("[recv] ignore duplicated crc packet", strconv.FormatUint(crc, 16))
+	crc := head.CRC64(data)
+	crclog := crc
+	crc ^= (uint64(seq) << 16)
+	if config.ShowDebugLog {
+		logrus.Debugf("[recv] packet crc %016x, seq %08x, xored crc %016x", crclog, seq, crc)
+	}
+	if m.recved.Get(crc) {
+		if config.ShowDebugLog {
+			logrus.Debugln("[recv] ignore duplicated crc packet", strconv.FormatUint(crc, 16))
+		}
 		return nil
 	}
-	logrus.Debugln("[recv]", len(data), "bytes data with flag", hex.EncodeToString(data[11:12]), hex.EncodeToString(data[10:11]))
+	m.recved.Set(crc, true)
+	if config.ShowDebugLog {
+		logrus.Debugln("[recv]", strconv.FormatUint(crc, 16), len(data), "bytes data with flag", hex.EncodeToString(data[11:12]), hex.EncodeToString(data[10:11]))
+	}
 	if flags.IsSingle() || flags.NoFrag() {
 		h := head.SelectPacket()
 		_, err := h.Unmarshal(data)
 		if err != nil {
-			logrus.Errorln("[recv] unmarshal err:", err)
+			logrus.Errorln("[recv]", strconv.FormatUint(crc, 16), "unmarshal err:", err)
 			return nil
 		}
-		m.recved.Set(crc, true)
 		return h
 	}
 
-	hashd := data[20:52]
-	hsh := *(*[32]byte)(*(*unsafe.Pointer)(unsafe.Pointer(&hashd)))
+	crchash := crc64.New(crc64.MakeTable(crc64.ISO))
+	_, _ = crchash.Write(head.Hash(data))
+	var buf [4]byte
+	binary.LittleEndian.PutUint32(buf[:], seq)
+	_, _ = crchash.Write(buf[:])
+	hsh := crchash.Sum64()
 	h := m.recving.Get(hsh)
 	if h != nil {
-		logrus.Debugln("[recv] get another frag part of", hex.EncodeToString(hashd))
+		if config.ShowDebugLog {
+			logrus.Debugln("[recv]", strconv.FormatUint(crc, 16), "get another frag part of", strconv.FormatUint(hsh, 16))
+		}
 		ok, err := h.Unmarshal(data)
 		if err == nil {
 			if ok {
 				m.recving.Delete(hsh)
-				m.recved.Set(crc, true)
-				logrus.Debugln("[recv] all parts of", hex.EncodeToString(hashd), "has reached")
+				if config.ShowDebugLog {
+					logrus.Debugln("[recv]", strconv.FormatUint(crc, 16), "all parts of", strconv.FormatUint(hsh, 16), "has reached")
+				}
 				return h
 			}
 		} else {
 			h.Put()
-			logrus.Errorln("[recv] unmarshal err:", err)
+			logrus.Errorln("[recv]", strconv.FormatUint(crc, 16), "unmarshal err:", err)
 		}
 		return nil
 	}
-	logrus.Debugln("[recv] get new frag part of", hex.EncodeToString(hashd))
+	if config.ShowDebugLog {
+		logrus.Debugln("[recv]", strconv.FormatUint(crc, 16), "get new frag part of", strconv.FormatUint(hsh, 16))
+	}
 	h = head.SelectPacket()
 	_, err := h.Unmarshal(data)
 	if err != nil {
 		h.Put()
-		logrus.Errorln("[recv] unmarshal err:", err)
+		logrus.Errorln("[recv]", strconv.FormatUint(crc, 16), "unmarshal err:", err)
 		return nil
 	}
 	m.recving.Set(hsh, h)

gold/link/router.go

@@ -7,6 +7,8 @@ import (
 
 	"github.com/FloatTech/ttl"
 	"github.com/sirupsen/logrus"
+
+	"github.com/fumiama/WireGold/config"
 )
 
 type Router struct {
@@ -45,7 +47,9 @@ func (r *Router) SetDefault(l *Link) {
 func (r *Router) NextHop(ip string) (l *Link) {
 	l = r.cache.Get(ip)
 	if l != nil {
-		logrus.Debugln("[router] get cached nexthop to", ip, "link", l)
+		if config.ShowDebugLog {
+			logrus.Debugln("[router] get cached nexthop to", ip, "link", l)
+		}
 		return
 	}
 	ipb := net.ParseIP(ip)
@@ -62,7 +66,9 @@ func (r *Router) NextHop(ip string) (l *Link) {
 	for _, c := range r.list {
 		if c.Contains(ipb) {
 			l = r.table[c.String()]
-			logrus.Debugln("[router] get nexthop to", ipb, "-->", c, "link", l)
+			if config.ShowDebugLog {
+				logrus.Debugln("[router] get nexthop to", ipb, "-->", c, "link", l)
+			}
 			r.cache.Set(ip, l)
 			return l
 		}

gold/link/send.go

@@ -2,16 +2,20 @@ package link
 
 import (
 	"bytes"
+	crand "crypto/rand"
+	"encoding/binary"
 	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
 	"math/rand"
 
-	"github.com/fumiama/WireGold/gold/head"
-	"github.com/fumiama/WireGold/helper"
 	"github.com/klauspost/compress/zstd"
 	"github.com/sirupsen/logrus"
+
+	"github.com/fumiama/WireGold/config"
+	"github.com/fumiama/WireGold/gold/head"
+	"github.com/fumiama/WireGold/helper"
 )
 
 var (
@@ -24,11 +28,17 @@ func (l *Link) WriteAndPut(p *head.Packet, istransfer bool) (n int, err error) {
 	defer p.Put()
 	teatype := l.randkeyidx()
 	sndcnt := uint16(l.incgetsndcnt())
+	var buf [4]byte
+	_, _ = crand.Read(buf[:2])
+	binary.BigEndian.PutUint16(buf[2:4], sndcnt)
+	seq := binary.BigEndian.Uint32(buf[:])
 	mtu := l.mtu
 	if l.mturandomrange > 0 {
 		mtu -= uint16(rand.Intn(int(l.mturandomrange)))
 	}
-	logrus.Debugln("[send] mtu:", mtu, ", addt:", sndcnt&0x07ff, ", key index:", teatype)
+	if config.ShowDebugLog {
+		logrus.Debugln("[send] mtu:", mtu, ", addt:", sndcnt&0x07ff, ", key index:", teatype)
+	}
 	if !istransfer {
 		l.encrypt(p, sndcnt, teatype)
 	}
@@ -39,7 +49,7 @@ func (l *Link) WriteAndPut(p *head.Packet, istransfer bool) (n int, err error) {
 	}
 	remlen := p.BodyLen()
 	if remlen <= delta {
-		return l.write(p, teatype, sndcnt, uint32(remlen), 0, istransfer, false)
+		return l.write(p, teatype, sndcnt, uint32(remlen), 0, istransfer, false, seq)
 	}
 	if istransfer && p.Flags.DontFrag() && remlen > delta {
 		return 0, ErrDropBigDontFragPkt
@@ -50,9 +60,11 @@ func (l *Link) WriteAndPut(p *head.Packet, istransfer bool) (n int, err error) {
 	packet := p.Copy()
 	for remlen > delta {
 		remlen -= delta
-		logrus.Debugln("[send] split frag [", pos, "~", pos+delta, "], remain:", remlen)
+		if config.ShowDebugLog {
+			logrus.Debugln("[send] split frag [", pos, "~", pos+delta, "], remain:", remlen)
+		}
 		packet.CropBody(pos, pos+delta)
-		cnt, err := l.write(packet, teatype, sndcnt, totl, uint16(pos>>3), istransfer, true)
+		cnt, err := l.write(packet, teatype, sndcnt, totl, uint16(pos>>3), istransfer, true, seq)
 		n += cnt
 		if err != nil {
 			return n, err
@@ -62,10 +74,12 @@ func (l *Link) WriteAndPut(p *head.Packet, istransfer bool) (n int, err error) {
 	}
 	packet.Put()
 	if remlen > 0 {
-		logrus.Debugln("[send] last frag [", pos, "~", pos+remlen, "]")
+		if config.ShowDebugLog {
+			logrus.Debugln("[send] last frag [", pos, "~", pos+remlen, "]")
+		}
 		p.CropBody(pos, pos+remlen)
 		cnt := 0
-		cnt, err = l.write(p, teatype, sndcnt, totl, uint16(pos>>3), istransfer, false)
+		cnt, err = l.write(p, teatype, sndcnt, totl, uint16(pos>>3), istransfer, false, seq)
 		n += cnt
 	}
 	return n, err
@@ -73,7 +87,9 @@ func (l *Link) WriteAndPut(p *head.Packet, istransfer bool) (n int, err error) {
 
 func (l *Link) encrypt(p *head.Packet, sndcnt uint16, teatype uint8) {
 	p.FillHash()
-	logrus.Debugln("[send] data len before encrypt:", p.BodyLen())
+	if config.ShowDebugLog {
+		logrus.Debugln("[send] data len before encrypt:", p.BodyLen())
+	}
 	data := p.Body()
 	if l.usezstd {
 		w := helper.SelectWriter()
@@ -82,14 +98,29 @@ func (l *Link) encrypt(p *head.Packet, sndcnt uint16, teatype uint8) {
 		_, _ = io.Copy(enc, bytes.NewReader(data))
 		enc.Close()
 		data = w.Bytes()
-		logrus.Debugln("[send] data len after zstd:", len(data))
+		if config.ShowDebugLog {
+			logrus.Debugln("[send] data len after zstd:", len(data))
+		}
 	}
 	p.SetBody(l.Encode(teatype, sndcnt&0x07ff, data), true)
-	logrus.Debugln("[send] data len after xchacha20:", p.BodyLen(), "addt:", sndcnt)
+	if config.ShowDebugLog {
+		logrus.Debugln("[send] data len after xchacha20:", p.BodyLen(), "addt:", sndcnt)
+	}
+}
+
+// write 向 peer 发包
+func (l *Link) write(p *head.Packet, teatype uint8, additional uint16, datasz uint32, offset uint16, istransfer, hasmore bool, seq uint32) (int, error) {
+	if p.DecreaseAndGetTTL() <= 0 {
+		return 0, ErrTTL
+	}
+	if l.doublepacket {
+		_, _ = l.writeonce(p, teatype, additional, datasz, offset, istransfer, hasmore, seq)
+	}
+	return l.writeonce(p, teatype, additional, datasz, offset, istransfer, hasmore, seq)
 }
 
 // write 向 peer 发一个包
-func (l *Link) write(p *head.Packet, teatype uint8, additional uint16, datasz uint32, offset uint16, istransfer, hasmore bool) (int, error) {
+func (l *Link) writeonce(p *head.Packet, teatype uint8, additional uint16, datasz uint32, offset uint16, istransfer, hasmore bool, seq uint32) (int, error) {
 	peerep := l.endpoint
 	if peerep == nil {
 		return 0, errors.New("nil endpoint of " + p.Dst.String())
@@ -103,9 +134,6 @@ func (l *Link) write(p *head.Packet, teatype uint8, additional uint16, datasz ui
 	} else {
 		d, cl = p.Marshal(l.me.me, teatype, additional, datasz, offset, false, hasmore)
 	}
-	if d == nil {
-		return 0, ErrTTL
-	}
 	defer cl()
 
 	bound := 64
@@ -114,9 +142,14 @@ func (l *Link) write(p *head.Packet, teatype uint8, additional uint16, datasz ui
 		bound = len(d)
 		endl = "."
 	}
-	logrus.Debugln("[send] write", len(d), "bytes data from ep", l.me.conn.LocalAddr(), "to", peerep, "offset:", fmt.Sprintf("%04x", offset))
-	logrus.Debugln("[send] data bytes", hex.EncodeToString(d[:bound]), endl)
-	d = l.me.xorenc(d)
-	logrus.Debugln("[send] data xored", hex.EncodeToString(d[:bound]), endl)
+	if config.ShowDebugLog {
+		logrus.Debugln("[send] write", len(d), "bytes data from ep", l.me.conn.LocalAddr(), "to", peerep, "offset", fmt.Sprintf("%04x", offset), "crc", fmt.Sprintf("%016x", p.CRC64()))
+		logrus.Debugln("[send] data bytes", hex.EncodeToString(d[:bound]), endl)
+	}
+	d = l.me.xorenc(d, seq)
+	if config.ShowDebugLog {
+		logrus.Debugln("[send] data xored", hex.EncodeToString(d[:bound]), endl)
+	}
+	defer helper.PutBytes(d)
 	return l.me.conn.WriteToPeer(d, peerep)
 }

gold/p2p/ip/init.go

@@ -13,7 +13,7 @@ func NewEndpoint(endpoint string, configs ...any) (p2p.EndPoint, error) {
 	if err != nil {
 		return nil, err
 	}
-	ptcl := uint(0x04) // IPIP
+	ptcl := uint(0x6C) // IPComp https://datatracker.ietf.org/doc/html/rfc3173
 	if len(configs) > 0 {
 		ptcl = configs[0].(uint)
 	}

gold/p2p/tcp/init.go

@@ -12,6 +12,7 @@ import (
 type Config struct {
 	DialTimeout        time.Duration
 	PeersTimeout       time.Duration
+	KeepInterval       time.Duration
 	ReceiveChannelSize int
 }
 
@@ -34,6 +35,7 @@ func newEndpoint(endpoint string, configs ...any) (*EndPoint, error) {
 		addr:         net.TCPAddrFromAddrPort(addr),
 		dialtimeout:  cfg.DialTimeout,
 		peerstimeout: cfg.PeersTimeout,
+		keepinterval: cfg.KeepInterval,
 		recvchansize: cfg.ReceiveChannelSize,
 	}, nil
 }

gold/p2p/tcp/pdu.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"time"
 
+	"github.com/fumiama/WireGold/config"
 	"github.com/fumiama/WireGold/helper"
 	"github.com/sirupsen/logrus"
 )
@@ -20,16 +21,14 @@ type packetType uint8
 const (
 	packetTypeKeepAlive packetType = iota
 	packetTypeNormal
+	packetTypeSubKeepAlive
 	packetTypeTop
 )
 
-const magic = 0x12d3fde9
-
-var magicbuf [4]byte
-
-func init() {
-	binary.LittleEndian.PutUint32(magicbuf[:], magic)
-}
+var (
+	magicbuf = []byte("GET ")
+	magic    = binary.LittleEndian.Uint32(magicbuf)
+)
 
 type packet struct {
 	typ packetType
@@ -44,7 +43,7 @@ func (p *packet) pack() (net.Buffers, func()) {
 		w.WriteByte(byte(p.typ))
 		w.WriteUInt16(p.len)
 	})
-	return net.Buffers{magicbuf[:], d, p.dat}, cl
+	return net.Buffers{magicbuf, d, p.dat}, cl
 }
 
 func (p *packet) Read(_ []byte) (int, error) {
@@ -89,11 +88,11 @@ func (p *packet) WriteTo(w io.Writer) (n int64, err error) {
 	return io.Copy(w, &buf)
 }
 
-func isvalid(tcpconn *net.TCPConn) bool {
+func isvalid(tcpconn *net.TCPConn, timeout time.Duration) (issub, ok bool) {
 	pckt := packet{}
 
 	stopch := make(chan struct{})
-	t := time.AfterFunc(time.Second, func() {
+	t := time.AfterFunc(timeout, func() {
 		stopch <- struct{}{}
 	})
 
@@ -106,21 +105,29 @@ func isvalid(tcpconn *net.TCPConn) bool {
 
 	select {
 	case <-stopch:
-		logrus.Debugln("[tcp] validate recv from", tcpconn.RemoteAddr(), "timeout")
-		return false
+		if config.ShowDebugLog {
+			logrus.Debugln("[tcp] validate recv from", tcpconn.RemoteAddr(), "timeout")
+		}
+		return
 	case <-copych:
 		t.Stop()
 	}
 
 	if err != nil {
-		logrus.Debugln("[tcp] validate recv from", tcpconn.RemoteAddr(), "err:", err)
-		return false
+		if config.ShowDebugLog {
+			logrus.Debugln("[tcp] validate recv from", tcpconn.RemoteAddr(), "err:", err)
+		}
+		return
 	}
-	if pckt.typ != packetTypeKeepAlive {
-		logrus.Debugln("[tcp] validate got invalid typ", pckt.typ, "from", tcpconn.RemoteAddr())
-		return false
+	if pckt.typ != packetTypeKeepAlive && pckt.typ != packetTypeSubKeepAlive {
+		if config.ShowDebugLog {
+			logrus.Debugln("[tcp] validate got invalid typ", pckt.typ, "from", tcpconn.RemoteAddr())
+		}
+		return
 	}
 
-	logrus.Debugln("[tcp] passed validate recv from", tcpconn.RemoteAddr())
-	return true
+	if config.ShowDebugLog {
+		logrus.Debugln("[tcp] passed validate recv from", tcpconn.RemoteAddr())
+	}
+	return pckt.typ == packetTypeSubKeepAlive, true
 }

gold/p2p/tcp/tcp.go

@@ -10,15 +10,18 @@ import (
 	"time"
 
 	"github.com/FloatTech/ttl"
+	"github.com/sirupsen/logrus"
+
+	"github.com/fumiama/WireGold/config"
 	"github.com/fumiama/WireGold/gold/p2p"
 	"github.com/fumiama/WireGold/helper"
-	"github.com/sirupsen/logrus"
 )
 
 type EndPoint struct {
 	addr         *net.TCPAddr
 	dialtimeout  time.Duration
 	peerstimeout time.Duration
+	keepinterval time.Duration
 	recvchansize int
 }
 
@@ -62,17 +65,23 @@ func (ep *EndPoint) Listen() (p2p.Conn, error) {
 		addr: ep,
 		lstn: lstn,
 		peers: ttl.NewCacheOn(peerstimeout, [4]func(string, *net.TCPConn){
-			nil, nil, func(_ string, t *net.TCPConn) {
+			func(_ string, t *net.TCPConn) {
+				_ = t.SetLinger(0)
+				_ = t.SetNoDelay(true)
+			}, nil, func(_ string, t *net.TCPConn) {
 				err := t.CloseWrite()
-				if err != nil {
-					logrus.Debugln("[tcp] close write from", t.LocalAddr(), "to", t.RemoteAddr(), "err:", err)
-				} else {
-					logrus.Debugln("[tcp] close write from", t.LocalAddr(), "to", t.RemoteAddr())
+				if config.ShowDebugLog {
+					if err != nil {
+						logrus.Debugln("[tcp] close write from", t.LocalAddr(), "to", t.RemoteAddr(), "err:", err)
+					} else {
+						logrus.Debugln("[tcp] close write from", t.LocalAddr(), "to", t.RemoteAddr())
+					}
 				}
 			}, nil,
 		}),
 		recv: make(chan *connrecv, chansz),
 		cplk: &sync.Mutex{},
+		sblk: &sync.RWMutex{},
 	}
 	go conn.accept()
 	return conn, nil
@@ -84,13 +93,21 @@ type connrecv struct {
 	pckt packet
 }
 
+type subconn struct {
+	cplk sync.Mutex
+	conn *net.TCPConn
+}
+
 // Conn 伪装成无状态的有状态连接
 type Conn struct {
-	addr  *EndPoint
-	lstn  *net.TCPListener
-	peers *ttl.Cache[string, *net.TCPConn]
-	recv  chan *connrecv
-	cplk  *sync.Mutex
+	addr   *EndPoint
+	lstn   *net.TCPListener
+	peers  *ttl.Cache[string, *net.TCPConn]
+	recv   chan *connrecv
+	cplk   *sync.Mutex
+	sblk   *sync.RWMutex
+	subs   []*subconn
+	suberr bool
 }
 
 func (conn *Conn) accept() {
@@ -108,46 +125,98 @@ func (conn *Conn) accept() {
 			_ = conn.Close()
 			newc, err := conn.addr.Listen()
 			if err != nil {
-				logrus.Warn("[tcp] re-listen on", conn.addr, "err:", err)
+				logrus.Warnln("[tcp] re-listen on", conn.addr, "err:", err)
 				return
 			}
 			*conn = *newc.(*Conn)
-			logrus.Info("[tcp] re-listen on", conn.addr)
+			logrus.Infoln("[tcp] re-listen on", conn.addr)
 			continue
 		}
 		go conn.receive(tcpconn, false)
 	}
 }
 
+func delsubs(i int, subs []*subconn) []*subconn {
+	tcpconn := subs[i].conn
+	err := tcpconn.CloseWrite()
+	if config.ShowDebugLog {
+		if err != nil {
+			logrus.Debugln("[tcp] close sub write from", tcpconn.LocalAddr(), "to", tcpconn.RemoteAddr(), "err:", err)
+		} else {
+			logrus.Debugln("[tcp] close sub write from", tcpconn.LocalAddr(), "to", tcpconn.RemoteAddr())
+		}
+	}
+	switch i {
+	case 0:
+		subs = subs[1:]
+	case len(subs) - 1:
+		subs = subs[:len(subs)-1]
+	default:
+		subs = append(subs[:i], subs[i+1:]...)
+	}
+	return subs
+}
+
 func (conn *Conn) receive(tcpconn *net.TCPConn, hasvalidated bool) {
 	ep, _ := newEndpoint(tcpconn.RemoteAddr().String(), &Config{
 		DialTimeout:        conn.addr.dialtimeout,
 		PeersTimeout:       conn.addr.peerstimeout,
+		KeepInterval:       conn.addr.keepinterval,
 		ReceiveChannelSize: conn.addr.recvchansize,
 	})
 
-	if !hasvalidated {
-		if !isvalid(tcpconn) {
-			return
-		}
-		logrus.Debugln("[tcp] accept from", ep)
-		conn.peers.Set(ep.String(), tcpconn)
-	}
+	issub, ok := false, false
 
 	peerstimeout := conn.addr.peerstimeout
 	if peerstimeout < time.Second*30 {
 		peerstimeout = time.Second * 30
 	}
 	peerstimeout *= 2
-	defer conn.peers.Delete(ep.String())
+
+	if !hasvalidated {
+		issub, ok = isvalid(tcpconn, peerstimeout)
+		if !ok {
+			return
+		}
+		if config.ShowDebugLog {
+			logrus.Debugln("[tcp] accept from", ep, "issub:", issub)
+		}
+		if issub {
+			conn.sblk.Lock()
+			conn.subs = append(conn.subs, &subconn{conn: tcpconn})
+			conn.sblk.Unlock()
+		} else {
+			conn.peers.Set(ep.String(), tcpconn)
+		}
+	}
+
+	if issub {
+		defer func() {
+			conn.sblk.Lock()
+			for i, sub := range conn.subs {
+				if sub.conn == tcpconn {
+					conn.subs = delsubs(i, conn.subs)
+					break
+				}
+			}
+			conn.sblk.Unlock()
+		}()
+	} else {
+		defer conn.peers.Delete(ep.String())
+	}
+
+	go conn.keep(ep)
+
 	for {
 		r := &connrecv{addr: ep}
 		if conn.addr == nil || conn.lstn == nil || conn.peers == nil || conn.recv == nil {
 			return
 		}
-		tcpconn := conn.peers.Get(ep.String())
-		if tcpconn == nil {
-			return
+		if !issub {
+			tcpconn = conn.peers.Get(ep.String())
+			if tcpconn == nil {
+				return
+			}
 		}
 		r.conn = tcpconn
 
@@ -165,28 +234,83 @@ func (conn *Conn) receive(tcpconn *net.TCPConn, hasvalidated bool) {
 
 		select {
 		case <-stopch:
-			logrus.Debugln("[tcp] recv from", ep, "timeout")
+			if config.ShowDebugLog {
+				logrus.Debugln("[tcp] recv from", ep, "timeout")
+			}
 			_ = tcpconn.CloseRead()
 			return
 		case <-copych:
 			t.Stop()
 		}
 
-		if err != nil {
-			logrus.Debugln("[tcp] recv from", ep, "err:", err)
-			_ = tcpconn.CloseRead()
+		if conn.addr == nil || conn.lstn == nil || conn.peers == nil || conn.recv == nil {
 			return
 		}
+
+		if err != nil {
+			if config.ShowDebugLog {
+				logrus.Debugln("[tcp] recv from", ep, "err:", err)
+			}
+			if errors.Is(err, net.ErrClosed) || errors.Is(err, io.ErrClosedPipe) {
+				_ = tcpconn.CloseRead()
+				return
+			}
+			continue
+		}
 		if r.pckt.typ >= packetTypeTop {
-			logrus.Debugln("[tcp] close reading invalid conn from", ep, "typ", r.pckt.typ, "len", r.pckt.len)
+			if config.ShowDebugLog {
+				logrus.Debugln("[tcp] close reading invalid conn from", ep, "typ", r.pckt.typ, "len", r.pckt.len)
+			}
 			_ = tcpconn.CloseRead()
 			return
 		}
-		logrus.Debugln("[tcp] dispatch packet from", ep, "typ", r.pckt.typ, "len", r.pckt.len)
+		if config.ShowDebugLog {
+			logrus.Debugln("[tcp] dispatch packet from", ep, "typ", r.pckt.typ, "len", r.pckt.len)
+		}
 		conn.recv <- r
 	}
 }
 
+func (conn *Conn) keep(ep *EndPoint) {
+	keepinterval := ep.keepinterval
+	if keepinterval < time.Second*4 {
+		keepinterval = time.Second * 4
+	}
+	t := time.NewTicker(keepinterval)
+	defer t.Stop()
+	for range t.C {
+		if conn.addr == nil {
+			return
+		}
+		tcpconn := conn.peers.Get(ep.String())
+		if tcpconn != nil {
+			_, err := io.Copy(tcpconn, &packet{typ: packetTypeKeepAlive})
+			if conn.addr == nil {
+				return
+			}
+			if err != nil {
+				logrus.Warnln("[tcp] keep main conn alive to", ep, "err:", err)
+				conn.peers.Delete(ep.String())
+			} else if config.ShowDebugLog {
+				logrus.Debugln("[tcp] keep main conn alive to", ep)
+			}
+		}
+		conn.sblk.RLock()
+		for i, sub := range conn.subs {
+			_, err := io.Copy(sub.conn, &packet{typ: packetTypeSubKeepAlive})
+			if conn.addr == nil {
+				return
+			}
+			if err != nil {
+				logrus.Warnln("[tcp] keep sub conn alive to", sub.conn.RemoteAddr(), "err:", err)
+				conn.subs = delsubs(i, conn.subs) // del 1 link at once
+				break
+			}
+		}
+		conn.sblk.RUnlock()
+	}
+}
+
 func (conn *Conn) Close() error {
 	if conn.lstn != nil {
 		_ = conn.lstn.Close()
@@ -229,26 +353,36 @@ func (conn *Conn) ReadFromPeer(b []byte) (int, p2p.EndPoint, error) {
 	return n, p.addr, nil
 }
 
-func (conn *Conn) WriteToPeer(b []byte, ep p2p.EndPoint) (n int, err error) {
-	tcpep, ok := ep.(*EndPoint)
-	if !ok {
-		return 0, p2p.ErrEndpointTypeMistatch
-	}
-	blen := len(b)
-	if blen >= 65536 {
-		return 0, errors.New("data size " + strconv.Itoa(blen) + " is too large")
-	}
+// writeToPeer after acquiring lock
+func (conn *Conn) writeToPeer(b []byte, tcpep *EndPoint, issub bool) (n int, err error) {
 	retried := false
-	conn.cplk.Lock()
-	defer conn.cplk.Unlock()
-	tcpconn := conn.peers.Get(tcpep.String())
+	ok := false
+	var (
+		tcpconn *net.TCPConn
+		subc    *subconn
+	)
 RECONNECT:
+	if issub {
+		conn.sblk.RLock()
+		for _, sub := range conn.subs {
+			if sub.cplk.TryLock() {
+				tcpconn = sub.conn
+				subc = sub
+				break
+			}
+		}
+		conn.sblk.RUnlock()
+	} else {
+		tcpconn = conn.peers.Get(tcpep.String())
+	}
 	if tcpconn == nil {
 		dialtimeout := tcpep.dialtimeout
 		if dialtimeout < time.Second {
 			dialtimeout = time.Second
 		}
-		logrus.Debugln("[tcp] dial to", tcpep.addr, "timeout", dialtimeout)
+		if config.ShowDebugLog {
+			logrus.Debugln("[tcp] dial to", tcpep.addr, "timeout", dialtimeout, "issub", issub)
+		}
 		var cn net.Conn
 		// must use another port to send because there's no exsiting conn
 		cn, err = net.DialTimeout(tcpep.Network(), tcpep.addr.String(), dialtimeout)
@@ -259,31 +393,83 @@ RECONNECT:
 		if !ok {
 			return 0, errors.New("expect *net.TCPConn but got " + reflect.ValueOf(cn).Type().String())
 		}
-		_, err = io.Copy(tcpconn, &packet{
-			typ: packetTypeKeepAlive,
-		})
+		pkt := &packet{}
+		if issub {
+			pkt.typ = packetTypeSubKeepAlive
+		} else {
+			pkt.typ = packetTypeKeepAlive
+		}
+		_, err = io.Copy(tcpconn, pkt)
 		if err != nil {
-			logrus.Debugln("[tcp] dial to", tcpep.addr, "success, but write err:", err)
+			if config.ShowDebugLog {
+				logrus.Debugln("[tcp] dial to", tcpep.addr, "issub", issub, "success, but write err:", err)
+			}
 			return 0, err
 		}
-		logrus.Debugln("[tcp] dial to", tcpep.addr, "success, local:", tcpconn.LocalAddr())
-		conn.peers.Set(tcpep.String(), tcpconn)
+		if config.ShowDebugLog {
+			logrus.Debugln("[tcp] dial to", tcpep.addr, "success, local:", tcpconn.LocalAddr(), "issub", issub)
+		}
+		if !issub {
+			conn.peers.Set(tcpep.String(), tcpconn)
+		} else {
+			conn.sblk.Lock()
+			conn.subs = append(conn.subs, &subconn{conn: tcpconn})
+			conn.sblk.Unlock()
+		}
 		go conn.receive(tcpconn, true)
-	} else {
+	} else if config.ShowDebugLog {
 		logrus.Debugln("[tcp] reuse tcpconn from", tcpconn.LocalAddr(), "to", tcpconn.RemoteAddr())
 	}
 	cnt, err := io.Copy(tcpconn, &packet{
 		typ: packetTypeNormal,
-		len: uint16(blen),
+		len: uint16(len(b)),
 		dat: b,
 	})
 	if err != nil {
-		conn.peers.Delete(tcpep.String())
+		if subc == nil {
+			conn.peers.Delete(tcpep.String())
+		} else {
+			conn.sblk.Lock()
+			for i, sub := range conn.subs {
+				if sub == subc {
+					conn.subs = delsubs(i, conn.subs)
+					break
+				}
+			}
+			conn.sblk.Unlock()
+		}
 		if !retried {
+			logrus.Warnln("[tcp] reconnect due to write to", tcpconn.RemoteAddr(), "err:", err)
 			retried = true
 			tcpconn = nil
 			goto RECONNECT
 		}
 	}
+	if subc != nil {
+		subc.cplk.Unlock()
+	}
 	return int(cnt) - 3, err
 }
+
+func (conn *Conn) WriteToPeer(b []byte, ep p2p.EndPoint) (n int, err error) {
+	tcpep, ok := ep.(*EndPoint)
+	if !ok {
+		return 0, p2p.ErrEndpointTypeMistatch
+	}
+	if len(b) >= 65536 {
+		return 0, errors.New("data size " + strconv.Itoa(len(b)) + " is too large")
+	}
+	if (!conn.suberr || len(conn.subs) > 0) && !conn.cplk.TryLock() {
+		if config.ShowDebugLog {
+			logrus.Debug("[tcp] try sub write to", tcpep)
+		}
+		n, err = conn.writeToPeer(b, tcpep, true) // try sub write
+		if err == nil {
+			return
+		}
+		conn.suberr = true // fast fail
+		conn.cplk.Lock()   // add to main queue
+	}
+	defer conn.cplk.Unlock()
+	return conn.writeToPeer(b, tcpep, false)
+}

gold/p2p/udplite/lite.go

@@ -75,11 +75,11 @@ func listenUDPLite(network string, laddr *net.UDPAddr) (*net.UDPConn, error) {
 	}
 	var errsys error
 	err = rc.Control(func(fd uintptr) {
-		errsys = syscall.SetsockoptInt(int(fd), SOL_UDPLITE, UDPLITE_SEND_CSCOV, head.PacketHeadLen)
+		errsys = syscall.SetsockoptInt(int(fd), SOL_UDPLITE, UDPLITE_SEND_CSCOV, head.PacketHeadLen+8) // for xor rand
 		if errsys != nil {
 			return
 		}
-		errsys = syscall.SetsockoptInt(int(fd), SOL_UDPLITE, UDPLITE_RECV_CSCOV, head.PacketHeadLen)
+		errsys = syscall.SetsockoptInt(int(fd), SOL_UDPLITE, UDPLITE_RECV_CSCOV, head.PacketHeadLen+8) // for xor rand
 	})
 	if err != nil {
 		_ = conn.Close()

helper/writer.go

@@ -131,7 +131,7 @@ func (w *Writer) Skip(n int) (int, error) {
 		}
 		return 0, io.EOF
 	}
-	n = min(n, len(b.buf[b.off:]))
+	n = minnum(n, len(b.buf[b.off:]))
 	b.off += n
 	if n > 0 {
 		b.lastRead = opRead
@@ -168,8 +168,8 @@ const (
 	opReadRune4 readOp = 4  // Read rune of size 4.
 )
 
-// min 返回两数最小值，该函数将被内联
-func min[T int | int8 | uint8 | int16 | uint16 | int32 | uint32 | int64 | uint64](a, b T) T {
+// minnum 返回两数最小值，该函数将被内联
+func minnum[T int | int8 | uint8 | int16 | uint16 | int32 | uint32 | int64 | uint64](a, b T) T {
 	if a > b {
 		return b
 	}

lower/nic.go

@@ -1,7 +1,6 @@
 package lower
 
 import (
-	"io"
 	"net"
 	"os"
 	"os/exec"
@@ -11,14 +10,8 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-type NICIO interface {
-	io.ReadWriteCloser
-	Up()
-	Down()
-}
-
-// NIC 虚拟网卡
-type NIC struct {
+// NICIO 虚拟网卡
+type NICIO struct {
 	ifce     *water.Interface
 	ip       net.IP
 	subnet   *net.IPNet
@@ -31,17 +24,17 @@ type NIC struct {
 // 网卡地址为 ip, 所属子网为 subnet
 // 以本网卡为下一跳的所有子网为 cidrs
 // cidrs 不包括本网卡 subnet
-func NewNIC(ip net.IP, subnet *net.IPNet, mtu string, cidrs ...string) NICIO {
+func NewNIC(ip net.IP, subnet *net.IPNet, mtu string, cidrs ...string) *NICIO {
 	ifce, err := water.New(water.Config{DeviceType: water.TUN})
 	if err != nil {
-		logrus.Error(err)
+		logrus.Errorln(err)
 		os.Exit(1)
 	}
 	subn, bitsn := subnet.Mask.Size()
 	if bitsn != 32 {
 		panic("mask len " + strconv.Itoa(bitsn) + " is not supported")
 	}
-	n := &NIC{
+	n := &NICIO{
 		ifce:     ifce,
 		ip:       ip,
 		subnet:   subnet,
@@ -53,16 +46,16 @@ func NewNIC(ip net.IP, subnet *net.IPNet, mtu string, cidrs ...string) NICIO {
 }
 
 // Read 匹配 PacketsIO Interface
-func (nc *NIC) Read(buf []byte) (int, error) {
+func (nc *NICIO) Read(buf []byte) (int, error) {
 	return nc.ifce.Read(buf)
 }
 
-func (nc *NIC) Write(packet []byte) (int, error) {
+func (nc *NICIO) Write(packet []byte) (int, error) {
 	return nc.ifce.Write(packet)
 }
 
 // Close 关闭网卡
-func (n *NIC) Close() error {
+func (n *NICIO) Close() error {
 	return n.ifce.Close()
 }
 

lower/tun_darwin.go

@@ -5,7 +5,7 @@ package lower
 
 import "net"
 
-func (n *NIC) Up() {
+func (n *NICIO) Up() {
 	execute("ifconfig", n.ifce.Name(), "mtu", n.mtu) // max: 9159
 	execute(
 		"ifconfig", n.ifce.Name(),
@@ -19,7 +19,7 @@ func (n *NIC) Up() {
 	}
 }
 
-func (n *NIC) Down() {
+func (n *NICIO) Down() {
 	execute("route", "delete", n.subnet.String(), "-interface", n.ifce.Name())
 	for _, c := range n.cidrs {
 		execute("route", "delete", c, "-interface", n.ifce.Name())

lower/tun_linux.go

@@ -3,7 +3,7 @@
 
 package lower
 
-func (n *NIC) Up() {
+func (n *NICIO) Up() {
 	execute("/sbin/ip", "link", "set", "dev", n.ifce.Name(), "mtu", n.mtu)
 	execute("/sbin/ip", "addr", "add", n.rawipnet, "dev", n.ifce.Name())
 	execute("/sbin/ip", "link", "set", "dev", n.ifce.Name(), "up")
@@ -12,7 +12,7 @@ func (n *NIC) Up() {
 	}
 }
 
-func (n *NIC) Down() {
+func (n *NICIO) Down() {
 	for _, c := range n.cidrs {
 		execute("/sbin/ip", "route", "del", c, "dev", n.ifce.Name())
 	}

lower/tun_stub.go

@@ -3,10 +3,10 @@
 
 package lower
 
-func (n *NIC) Up() {
+func (n *NICIO) Up() {
 	panic("not support lower on this os now")
 }
 
-func (n *NIC) Down() {
+func (n *NICIO) Down() {
 	panic("not support lower on this os now")
 }

lower/tun_windows.go

@@ -5,7 +5,7 @@ package lower
 
 import "net"
 
-func (n *NIC) Up() {
+func (n *NICIO) Up() {
 	execute("cmd", "/c", "netsh interface ip set address name=\""+n.ifce.Name()+"\" source=static addr=\""+n.ip.String()+"\" mask=\""+(net.IP)(n.subnet.Mask).String()+"\" gateway=none")
 	execute("cmd", "/c", "netsh interface ipv4 set subinterface \""+n.ifce.Name()+"\" mtu="+n.mtu)
 	for _, c := range n.cidrs {
@@ -17,7 +17,7 @@ func (n *NIC) Up() {
 	}
 }
 
-func (n *NIC) Down() {
+func (n *NICIO) Down() {
 	// execute("netsh", "interface", "set", "interface", n.ifce.Name(), "disabled")
 	for _, c := range n.cidrs {
 		ip, _, err := net.ParseCIDR(c)

main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/fumiama/WireGold/config"
+	"github.com/fumiama/WireGold/gold/head"
 	"github.com/fumiama/WireGold/helper"
 	"github.com/fumiama/WireGold/upper"
 	"github.com/fumiama/WireGold/upper/services/wg"
@@ -144,8 +145,8 @@ func main() {
 	if c.EndPoint == "" {
 		displayHelp("nil endpoint")
 	}
-	if c.MTU == 0 {
-		displayHelp("nil mtu")
+	if c.MTU <= head.PacketHeadLen {
+		displayHelp("invalid mtu")
 	}
 	w, err := wg.NewWireGold(&c)
 	if err != nil {

upper/services/tunnel/tunnel.go

@@ -13,6 +13,7 @@ import (
 	_ "github.com/fumiama/WireGold/gold/p2p/udp"     // support udp connection
 	_ "github.com/fumiama/WireGold/gold/p2p/udplite" // support udplite connection
 
+	"github.com/fumiama/WireGold/config"
 	"github.com/fumiama/WireGold/gold/head"
 	"github.com/fumiama/WireGold/gold/link"
 )
@@ -107,14 +108,20 @@ func (s *Tunnel) handleWrite() {
 			end = len(b)
 			endl = "."
 		}
-		logrus.Debugln("[tunnel] write send", hex.EncodeToString(b[:end]), endl)
+		if config.ShowDebugLog {
+			logrus.Debugln("[tunnel] write send", hex.EncodeToString(b[:end]), endl)
+		}
 		if b == nil {
 			logrus.Errorln("[tunnel] write recv nil")
 			break
 		}
-		logrus.Debugln("[tunnel] writing", len(b), "bytes...")
+		if config.ShowDebugLog {
+			logrus.Debugln("[tunnel] writing", len(b), "bytes...")
+		}
 		for len(b) > int(s.mtu)-4 {
-			logrus.Debugln("[tunnel] seq", seq, "split buffer")
+			if config.ShowDebugLog {
+				logrus.Debugln("[tunnel] seq", seq, "split buffer")
+			}
 			binary.LittleEndian.PutUint32(buf[:4], seq)
 			seq++
 			copy(buf[4:], b[:s.mtu-4])
@@ -125,7 +132,9 @@ func (s *Tunnel) handleWrite() {
 				logrus.Errorln("[tunnel] seq", seq-1, "write err:", err)
 				return
 			}
-			logrus.Debugln("[tunnel] seq", seq-1, "write succeeded")
+			if config.ShowDebugLog {
+				logrus.Debugln("[tunnel] seq", seq-1, "write succeeded")
+			}
 			b = b[s.mtu-4:]
 		}
 		binary.LittleEndian.PutUint32(buf[:4], seq)
@@ -138,7 +147,9 @@ func (s *Tunnel) handleWrite() {
 			logrus.Errorln("[tunnel] seq", seq-1, "write err:", err)
 			break
 		}
-		logrus.Debugln("[tunnel] seq", seq-1, "write succeeded")
+		if config.ShowDebugLog {
+			logrus.Debugln("[tunnel] seq", seq-1, "write succeeded")
+		}
 	}
 }
 
@@ -147,7 +158,9 @@ func (s *Tunnel) handleRead() {
 	seqmap := make(map[uint32]*head.Packet)
 	for {
 		if p, ok := seqmap[seq]; ok {
-			logrus.Debugln("[tunnel] dispatch cached seq", seq)
+			if config.ShowDebugLog {
+				logrus.Debugln("[tunnel] dispatch cached seq", seq)
+			}
 			delete(seqmap, seq)
 			seq++
 			s.out <- p
@@ -164,15 +177,21 @@ func (s *Tunnel) handleRead() {
 			end = p.BodyLen()
 			endl = "."
 		}
-		logrus.Debugln("[tunnel] read recv", hex.EncodeToString(p.Body()[:end]), endl)
+		if config.ShowDebugLog {
+			logrus.Debugln("[tunnel] read recv", hex.EncodeToString(p.Body()[:end]), endl)
+		}
 		recvseq := binary.LittleEndian.Uint32(p.Body()[:4])
 		if recvseq == seq {
-			logrus.Debugln("[tunnel] dispatch seq", seq)
+			if config.ShowDebugLog {
+				logrus.Debugln("[tunnel] dispatch seq", seq)
+			}
 			seq++
 			s.out <- p
 			continue
 		}
 		seqmap[recvseq] = p
-		logrus.Debugln("[tunnel] cache seq", recvseq)
+		if config.ShowDebugLog {
+			logrus.Debugln("[tunnel] cache seq", recvseq)
+		}
 	}
 }

upper/services/tunnel/tunnel_test.go

@@ -81,6 +81,7 @@ func testTunnel(t *testing.T, nw string, isplain bool, pshk *[32]byte, mtu uint1
 		MTU:            mtu,
 		MTURandomRange: mtu / 2,
 		UseZstd:        true,
+		DoublePacket:   true,
 	})
 	p.AddPeer(&link.PeerConfig{
 		PeerIP:         "192.168.1.2",
@@ -106,17 +107,17 @@ func testTunnel(t *testing.T, nw string, isplain bool, pshk *[32]byte, mtu uint1
 	time.Sleep(time.Second) // wait link up
 
 	sendb := ([]byte)("1234")
-	tunnme.Write(sendb)
+	go tunnme.Write(sendb)
 	buf := make([]byte, 4)
 	tunnpeer.Read(buf)
 	if string(sendb) != string(buf) {
-		t.Log("error: recv", buf)
+		logrus.Errorln("error: recv", buf, "expect", sendb)
 		t.Fail()
 	}
 
 	sendb = make([]byte, 4096)
 	rand.Read(sendb)
-	tunnme.Write(sendb)
+	go tunnme.Write(sendb)
 	buf = make([]byte, 4096)
 	_, err = io.ReadFull(&tunnpeer, buf)
 	if err != nil {
@@ -126,20 +127,31 @@ func testTunnel(t *testing.T, nw string, isplain bool, pshk *[32]byte, mtu uint1
 		t.Fatal("error: recv 4096 bytes data")
 	}
 
-	sendb = make([]byte, 65535)
+	sendbufs := make(chan []byte, 32)
+
+	go func() {
+		time.Sleep(time.Second)
+		for i := 0; i < 32; i++ {
+			sendb := make([]byte, 65535)
+			rand.Read(sendb)
+			n, _ := tunnme.Write(sendb)
+			sendbufs <- sendb
+			logrus.Infoln("loop", i, "write", n, "bytes")
+		}
+		close(sendbufs)
+	}()
 	buf = make([]byte, 65535)
-	for i := 0; i < 32; i++ {
-		rand.Read(sendb)
-		n, _ := tunnme.Write(sendb)
-		t.Log("loop", i, "write", n, "bytes")
-		n, err = io.ReadFull(&tunnpeer, buf)
+	i := 0
+	for sendb := range sendbufs {
+		n, err := io.ReadFull(&tunnpeer, buf)
 		if err != nil {
 			t.Fatal(err)
 		}
-		t.Log("loop", i, "read", n, "bytes")
+		logrus.Infoln("loop", i, "read", n, "bytes")
 		if string(sendb) != string(buf) {
 			t.Fatal("loop", i, "error: recv 65535 bytes data")
 		}
+		i++
 	}
 
 	rand.Read(sendb)

upper/services/wg/wg.go

@@ -17,7 +17,6 @@ import (
 	"github.com/fumiama/WireGold/config"
 	"github.com/fumiama/WireGold/gold/link"
 	"github.com/fumiama/WireGold/helper"
-	"github.com/fumiama/WireGold/lower"
 )
 
 const suffix32 = "㴄"
@@ -60,7 +59,7 @@ func (wg *WG) Start(srcport, destport uint16) {
 func (wg *WG) Run(srcport, destport uint16) {
 	wg.init(srcport, destport)
 	_, _ = wg.me.ListenNIC()
-	logrus.Info("[wg] stopped")
+	logrus.Infoln("[wg] stopped")
 }
 
 func (wg *WG) Stop() {
@@ -80,7 +79,7 @@ func (wg *WG) init(srcport, dstport uint16) {
 	}
 	for _, p := range wg.c.Peers {
 		for _, ip := range p.AllowedIPs {
-			if len(ip) == 0 || ip[0] == 'x' {
+			if len(ip) == 0 || ip[0] == 'x' || ip[0] == 'y' {
 				continue
 			}
 			ipnet, _, err := net.ParseCIDR(ip)
@@ -104,12 +103,16 @@ func (wg *WG) init(srcport, dstport uint16) {
 		MyEndpoint:   wg.c.EndPoint,
 		Network:      wg.c.Network,
 		PrivateKey:   &wg.key,
-		NIC:          lower.NewNIC(myip, mysubnet, strconv.FormatInt(wg.c.MTU, 10), cidrs...),
-		SrcPort:      srcport,
-		DstPort:      dstport,
-		MTU:          uint16(wg.c.MTU),
-		SpeedLoop:    wg.c.SpeedLoop,
-		Mask:         wg.c.Mask,
+		NICConfig: &link.NICConfig{
+			IP:     myip,
+			SubNet: mysubnet,
+			CIDRs:  cidrs,
+		},
+		SrcPort:   srcport,
+		DstPort:   dstport,
+		MTU:       uint16(wg.c.MTU),
+		SpeedLoop: wg.c.SpeedLoop,
+		Mask:      wg.c.Mask,
 	})
 
 	for _, peer := range wg.c.Peers {
@@ -154,6 +157,7 @@ func (wg *WG) init(srcport, dstport uint16) {
 			AllowTrans:     peer.AllowTrans,
 			NoPipe:         true,
 			UseZstd:        peer.UseZstd,
+			DoublePacket:   peer.DoublePacket,
 		})
 	}
 }