fix(nat): hello packet dead loop

.github/workflows/ci.yml

@@ -11,19 +11,22 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@master
         with:
-          go-version: ^1.20
+          go-version: ^1.23
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@master
 
       - name: Get dependencies
         run: go mod tidy
+      
+      - name: Enable debug log
+        run: sed -i 's/false/true/g' config/global.go
 
       - name: Build
-        run: go build -v ./...
+        run: go build -ldflags=-checklinkname=0 -v ./...
 
       - name: Test
-        run: go test $(go list ./...)
+        run: sudo go test -ldflags=-checklinkname=0 $(go list ./...) # ip test needs sudo
 
   lint:
     name: Lint
@@ -32,7 +35,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@master
         with:
-          go-version: ^1.20
+          go-version: ^1.23
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@master

.github/workflows/release.yml

@@ -14,16 +14,16 @@ jobs:
     steps:
 
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@master
         with:
-          go-version: ^1.20
+          go-version: ^1.23
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@master
 
       - name: Cache Go
         id: cache
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           # A list of files, directories, and wildcard patterns to cache and restore
           path: ~/go/pkg/mod
@@ -33,25 +33,25 @@ jobs:
         run: go mod tidy
 
       - name: Build linux-x64
-        run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o artifacts/wg-linux-x64 -trimpath
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -checklinkname=0" -o artifacts/wg-linux-x64 -trimpath
       - name: Build linux-x86
-        run: CGO_ENABLED=0 GOOS=linux GOARCH=386 go build -ldflags="-s -w" -o artifacts/wg-linux-x86 -trimpath
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=386 go build -ldflags="-s -w -checklinkname=0" -o artifacts/wg-linux-x86 -trimpath
       - name: Build windows-x64
-        run: CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags="-s -w" -o artifacts/wg-windows-x64.exe -trimpath
+        run: CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags="-s -w -checklinkname=0" -o artifacts/wg-windows-x64.exe -trimpath
       - name: Build windows-x86
-        run: CGO_ENABLED=0 GOOS=windows GOARCH=386 go build -ldflags="-s -w" -o artifacts/wg-windows-x86.exe -trimpath
+        run: CGO_ENABLED=0 GOOS=windows GOARCH=386 go build -ldflags="-s -w -checklinkname=0" -o artifacts/wg-windows-x86.exe -trimpath
       - name: Build arm64
-        run: CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GOARM=7 go build -ldflags="-s -w" -o artifacts/wg-linux-arm64 -trimpath
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GOARM=7 go build -ldflags="-s -w -checklinkname=0" -o artifacts/wg-linux-arm64 -trimpath
       - name: Build armv6
-        run: CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -ldflags="-s -w" -o artifacts/wg-linux-armv6 -trimpath
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -ldflags="-s -w -checklinkname=0" -o artifacts/wg-linux-armv6 -trimpath
       - name: Build mips
-        run: CGO_ENABLED=0 GOOS=linux GOARCH=mips go build -ldflags="-s -w" -o artifacts/wg-linux-mips -trimpath
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=mips go build -ldflags="-s -w -checklinkname=0" -o artifacts/wg-linux-mips -trimpath
       - name: Build mips-softfloat
-        run: CGO_ENABLED=0 GOOS=linux GOARCH=mips GOMIPS=softfloat go build -ldflags="-s -w" -o artifacts/wg-linux-mips-softfloat -trimpath
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=mips GOMIPS=softfloat go build -ldflags="-s -w -checklinkname=0" -o artifacts/wg-linux-mips-softfloat -trimpath
       - name: Build mipsel
-        run: CGO_ENABLED=0 GOOS=linux GOARCH=mipsle go build -ldflags="-s -w" -o artifacts/wg-linux-mipsel -trimpath
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=mipsle go build -ldflags="-s -w -checklinkname=0" -o artifacts/wg-linux-mipsel -trimpath
       - name: Build mipsel-softfloat
-        run: CGO_ENABLED=0 GOOS=linux GOARCH=mipsle GOMIPS=softfloat go build -ldflags="-s -w" -o artifacts/wg-linux-mipsel-softfloat -trimpath
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=mipsle GOMIPS=softfloat go build -ldflags="-s -w -checklinkname=0" -o artifacts/wg-linux-mipsel-softfloat -trimpath
 
       - name: Upload binaries to release
         uses: svenstaro/upload-release-action@v2

.golangci.yml

@@ -24,7 +24,7 @@ linters:
     - dogsled
     - dupl
     - errcheck
-    - exportloopref
+    #- exportloopref
     - exhaustive
     #- funlen
     #- goconst

README.md

@@ -41,14 +41,14 @@ EndPoint: 0.0.0.0:56789
 MTU: 1504
 SpeedLoop: 4096
 Mask: 0x1234567890abcdef
+Base14: true
 Peers:
   -
     IP: "192.168.233.2"
-    SubNet: 192.168.233.0/24
     PublicKey: 徯萃嵾爻燸攗窍褃冔蒔犡緇袿屿組待族砇嘀
     PresharedKey: 瀸敀爅崾嘊嵜紼樴稍毯攣矐訷蟷扛嬋庩崛昀
     EndPoint: 1.2.3.4:56789
-    AllowedIPs: ["192.168.233.2/32", "x192.168.233.3/32"] # allow trans to 192.168.233.3, but don not create route
+    AllowedIPs: ["192.168.233.2/32", "x192.168.233.3/32"] # accept packets from 192.168.233.3, but don not create route
     KeepAliveSeconds: 0
     QueryList: ["192.168.233.3"]
     MTU: 1400
@@ -58,12 +58,140 @@ Peers:
     AllowTrans: true
   -
     IP: "192.168.233.3"
-    SubNet: 192.168.233.0/24
     PublicKey: 牢喨粷詸衭譛浾蘹櫠砙杹蟫瑳叩刋橋経挵蘀
     PresharedKey: 竅琚喫従痸告烈兇厕趭萨假蔛瀇譄施烸蝫瘀
     EndPoint: ""
-    AllowedIPs: ["192.168.233.3/32"]
+    AllowedIPs: ["192.168.233.3/32", "y192.168.66.1/32"] # add route to 192.168.66.1 into inner route table but do not add it to system one
     MTU: 752
+    DoublePacket: true
     KeepAliveSeconds: 0
     AllowTrans: false
 ```
+
+## Benckmark on localhost
+> This benckmark is tested on Mac Book Air M1 within battery mode.
+
+### UDP MTU 4096
+```bash
+goos: darwin
+goarch: arm64
+pkg: github.com/fumiama/WireGold/upper/services/tunnel
+cpu: Apple M1
+BenchmarkTunnelUDP/1024-plain-nob14-8               4753            250629 ns/op           4.09 MB/s     3570503 B/op        117 allocs/op
+BenchmarkTunnelUDP/1024-normal-nob14-8              4473            261136 ns/op           3.92 MB/s     3570565 B/op        118 allocs/op
+BenchmarkTunnelUDP/1024-plain-b14-8                 4250            275495 ns/op           3.72 MB/s     3575369 B/op        121 allocs/op
+BenchmarkTunnelUDP/1024-normal-b14-8                4204            278062 ns/op           3.68 MB/s     3575844 B/op        122 allocs/op
+BenchmarkTunnelUDP/1024-preshared-nob14-8           4196            282707 ns/op           3.62 MB/s     3570427 B/op        118 allocs/op
+BenchmarkTunnelUDP/1024-preshared-b14-8             4130            290609 ns/op           3.52 MB/s     3575814 B/op        121 allocs/op
+BenchmarkTunnelUDP/2048-plain-nob14-8               4162            280754 ns/op           7.29 MB/s     3578825 B/op        117 allocs/op
+BenchmarkTunnelUDP/2048-normal-nob14-8              4088            292733 ns/op           7.00 MB/s     3578858 B/op        118 allocs/op
+BenchmarkTunnelUDP/2048-plain-b14-8                 4023            290264 ns/op           7.06 MB/s     3589002 B/op        121 allocs/op
+BenchmarkTunnelUDP/2048-normal-b14-8                3904            298841 ns/op           6.85 MB/s     3589116 B/op        122 allocs/op
+BenchmarkTunnelUDP/2048-preshared-nob14-8           3861            285661 ns/op           7.17 MB/s     3578690 B/op        117 allocs/op
+BenchmarkTunnelUDP/2048-preshared-b14-8             4344            273054 ns/op           7.50 MB/s     3589101 B/op        122 allocs/op
+BenchmarkTunnelUDP/3072-plain-nob14-8               4293            273633 ns/op          11.23 MB/s     3582928 B/op        121 allocs/op
+BenchmarkTunnelUDP/3072-normal-nob14-8              4086            287291 ns/op          10.69 MB/s     3583011 B/op        122 allocs/op
+BenchmarkTunnelUDP/3072-plain-b14-8                 4144            295141 ns/op          10.41 MB/s     3600854 B/op        128 allocs/op
+BenchmarkTunnelUDP/3072-normal-b14-8                3759            310645 ns/op           9.89 MB/s     3601217 B/op        130 allocs/op
+BenchmarkTunnelUDP/3072-preshared-nob14-8           3805            305162 ns/op          10.07 MB/s     3583013 B/op        122 allocs/op
+BenchmarkTunnelUDP/3072-preshared-b14-8             3574            320329 ns/op           9.59 MB/s     3601409 B/op        130 allocs/op
+BenchmarkTunnelUDP/4096-plain-nob14-8               2448            482246 ns/op           8.49 MB/s     7157679 B/op        241 allocs/op
+BenchmarkTunnelUDP/4096-normal-nob14-8              2328            508454 ns/op           8.06 MB/s     7158430 B/op        244 allocs/op
+BenchmarkTunnelUDP/4096-plain-b14-8                 2290            527956 ns/op           7.76 MB/s     7181296 B/op        254 allocs/op
+BenchmarkTunnelUDP/4096-normal-b14-8                2276            528681 ns/op           7.75 MB/s     7181960 B/op        256 allocs/op
+BenchmarkTunnelUDP/4096-preshared-nob14-8           2284            515491 ns/op           7.95 MB/s     7158606 B/op        243 allocs/op
+BenchmarkTunnelUDP/4096-preshared-b14-8             2026            560506 ns/op           7.31 MB/s     7181980 B/op        256 allocs/op
+```
+### UDP MTU 1024
+```bash
+goos: darwin
+goarch: arm64
+pkg: github.com/fumiama/WireGold/upper/services/tunnel
+cpu: Apple M1
+BenchmarkTunnelUDPSmallMTU/1024-plain-nob14-8       3766            326631 ns/op           3.14 MB/s     3568299 B/op        126 allocs/op
+BenchmarkTunnelUDPSmallMTU/1024-normal-nob14-8      3728            319552 ns/op           3.20 MB/s     3568454 B/op        128 allocs/op
+BenchmarkTunnelUDPSmallMTU/1024-plain-b14-8         3625            323816 ns/op           3.16 MB/s     3575638 B/op        137 allocs/op
+BenchmarkTunnelUDPSmallMTU/1024-normal-b14-8        3446            333510 ns/op           3.07 MB/s     3575925 B/op        138 allocs/op
+BenchmarkTunnelUDPSmallMTU/1024-preshared-nob14-8   3537            335220 ns/op           3.05 MB/s     3568481 B/op        128 allocs/op
+BenchmarkTunnelUDPSmallMTU/1024-preshared-b14-8     3486            337967 ns/op           3.03 MB/s     3575890 B/op        138 allocs/op
+BenchmarkTunnelUDPSmallMTU/2048-plain-nob14-8       3303            349278 ns/op           5.86 MB/s     3592804 B/op        140 allocs/op
+BenchmarkTunnelUDPSmallMTU/2048-normal-nob14-8      3331            376226 ns/op           5.44 MB/s     3593065 B/op        142 allocs/op
+BenchmarkTunnelUDPSmallMTU/2048-plain-b14-8         3116            421630 ns/op           4.86 MB/s     3605117 B/op        157 allocs/op
+BenchmarkTunnelUDPSmallMTU/2048-normal-b14-8        2634            381676 ns/op           5.37 MB/s     3606455 B/op        158 allocs/op
+BenchmarkTunnelUDPSmallMTU/2048-preshared-nob14-8   3138            391760 ns/op           5.23 MB/s     3591788 B/op        142 allocs/op
+BenchmarkTunnelUDPSmallMTU/2048-preshared-b14-8     2959            391663 ns/op           5.23 MB/s     3605364 B/op        158 allocs/op
+BenchmarkTunnelUDPSmallMTU/3072-plain-nob14-8       3046            421705 ns/op           7.28 MB/s     3620443 B/op        156 allocs/op
+BenchmarkTunnelUDPSmallMTU/3072-normal-nob14-8      3001            413043 ns/op           7.44 MB/s     3631990 B/op        157 allocs/op
+BenchmarkTunnelUDPSmallMTU/3072-plain-b14-8         2503            406906 ns/op           7.55 MB/s     3640574 B/op        177 allocs/op
+BenchmarkTunnelUDPSmallMTU/3072-normal-b14-8        2776            416946 ns/op           7.37 MB/s     3643066 B/op        179 allocs/op
+BenchmarkTunnelUDPSmallMTU/3072-preshared-nob14-8   2947            422378 ns/op           7.27 MB/s     3626004 B/op        157 allocs/op
+BenchmarkTunnelUDPSmallMTU/3072-preshared-b14-8     2547            459951 ns/op           6.68 MB/s     3648033 B/op        179 allocs/op
+BenchmarkTunnelUDPSmallMTU/4096-plain-nob14-8       1776            628904 ns/op           6.51 MB/s     7232490 B/op        285 allocs/op
+BenchmarkTunnelUDPSmallMTU/4096-normal-nob14-8      1782            643967 ns/op           6.36 MB/s     7238574 B/op        288 allocs/op
+BenchmarkTunnelUDPSmallMTU/4096-plain-b14-8         1549            674359 ns/op           6.07 MB/s     7262233 B/op        317 allocs/op
+BenchmarkTunnelUDPSmallMTU/4096-normal-b14-8        1826            690961 ns/op           5.93 MB/s     7260027 B/op        319 allocs/op
+BenchmarkTunnelUDPSmallMTU/4096-preshared-nob14-8   1868            649732 ns/op           6.30 MB/s     7242787 B/op        288 allocs/op
+BenchmarkTunnelUDPSmallMTU/4096-preshared-b14-8     1654            682244 ns/op           6.00 MB/s     7255985 B/op        318 allocs/op
+```
+### TCP MTU 4096
+```bash
+goos: darwin
+goarch: arm64
+pkg: github.com/fumiama/WireGold/upper/services/tunnel
+cpu: Apple M1
+BenchmarkTunnelTCP/1024-plain-nob14-8               2323            459188 ns/op           2.23 MB/s     3576540 B/op        166 allocs/op
+BenchmarkTunnelTCP/1024-normal-nob14-8              2472            438347 ns/op           2.34 MB/s     3576692 B/op        168 allocs/op
+BenchmarkTunnelTCP/1024-plain-b14-8                 2728            418395 ns/op           2.45 MB/s     3583603 B/op        171 allocs/op
+BenchmarkTunnelTCP/1024-normal-b14-8                2668            463060 ns/op           2.21 MB/s     3584519 B/op        172 allocs/op
+BenchmarkTunnelTCP/1024-preshared-nob14-8           2660            454945 ns/op           2.25 MB/s     3576708 B/op        168 allocs/op
+BenchmarkTunnelTCP/1024-preshared-b14-8             2690            437373 ns/op           2.34 MB/s     3584515 B/op        172 allocs/op
+BenchmarkTunnelTCP/2048-plain-nob14-8               2580            455416 ns/op           4.50 MB/s     3590368 B/op        168 allocs/op
+BenchmarkTunnelTCP/2048-normal-nob14-8              2294            458178 ns/op           4.47 MB/s     3590512 B/op        171 allocs/op
+BenchmarkTunnelTCP/2048-plain-b14-8                 2414            462412 ns/op           4.43 MB/s     3605344 B/op        174 allocs/op
+BenchmarkTunnelTCP/2048-normal-b14-8                2511            508527 ns/op           4.03 MB/s     3605658 B/op        177 allocs/op
+BenchmarkTunnelTCP/2048-preshared-nob14-8           2433            482086 ns/op           4.25 MB/s     3590571 B/op        171 allocs/op
+BenchmarkTunnelTCP/2048-preshared-b14-8             2361            494409 ns/op           4.14 MB/s     3605739 B/op        177 allocs/op
+BenchmarkTunnelTCP/3072-plain-nob14-8               2487            498395 ns/op           6.16 MB/s     3600311 B/op        199 allocs/op
+BenchmarkTunnelTCP/3072-normal-nob14-8              2170            542424 ns/op           5.66 MB/s     3600596 B/op        202 allocs/op
+BenchmarkTunnelTCP/3072-plain-b14-8                 2259            524854 ns/op           5.85 MB/s     3621274 B/op        205 allocs/op
+BenchmarkTunnelTCP/3072-normal-b14-8                2307            537656 ns/op           5.71 MB/s     3621514 B/op        209 allocs/op
+BenchmarkTunnelTCP/3072-preshared-nob14-8           1855            545493 ns/op           5.63 MB/s     3600418 B/op        201 allocs/op
+BenchmarkTunnelTCP/3072-preshared-b14-8             2198            535328 ns/op           5.74 MB/s     3621536 B/op        208 allocs/op
+BenchmarkTunnelTCP/4096-plain-nob14-8               2043            587272 ns/op           6.97 MB/s     7181814 B/op        391 allocs/op
+BenchmarkTunnelTCP/4096-normal-nob14-8              1832            609909 ns/op           6.72 MB/s     7182940 B/op        394 allocs/op
+BenchmarkTunnelTCP/4096-plain-b14-8                 2044            572149 ns/op           7.16 MB/s     7209279 B/op        405 allocs/op
+BenchmarkTunnelTCP/4096-normal-b14-8                2019            655180 ns/op           6.25 MB/s     7210261 B/op        409 allocs/op
+BenchmarkTunnelTCP/4096-preshared-nob14-8           1652            636402 ns/op           6.44 MB/s     7182914 B/op        394 allocs/op
+BenchmarkTunnelTCP/4096-preshared-b14-8             1885            624237 ns/op           6.56 MB/s     7210327 B/op        408 allocs/op
+```
+### TCP MTU 1024
+```bash
+goos: darwin
+goarch: arm64
+pkg: github.com/fumiama/WireGold/upper/services/tunnel
+cpu: Apple M1
+BenchmarkTunnelTCPSmallMTU/1024-plain-nob14-8       2061            582289 ns/op           1.76 MB/s     3577539 B/op        234 allocs/op
+BenchmarkTunnelTCPSmallMTU/1024-normal-nob14-8      2172            561002 ns/op           1.83 MB/s     3577725 B/op        237 allocs/op
+BenchmarkTunnelTCPSmallMTU/1024-plain-b14-8         2002            625224 ns/op           1.64 MB/s     3584694 B/op        244 allocs/op
+BenchmarkTunnelTCPSmallMTU/1024-normal-b14-8        1957            590091 ns/op           1.74 MB/s     3585060 B/op        247 allocs/op
+BenchmarkTunnelTCPSmallMTU/1024-preshared-nob14-8   2127            552614 ns/op           1.85 MB/s     3577669 B/op        236 allocs/op
+BenchmarkTunnelTCPSmallMTU/1024-preshared-b14-8     2084            602128 ns/op           1.70 MB/s     3585057 B/op        247 allocs/op
+BenchmarkTunnelTCPSmallMTU/2048-plain-nob14-8       1899            595303 ns/op           3.44 MB/s     3596277 B/op        320 allocs/op
+BenchmarkTunnelTCPSmallMTU/2048-normal-nob14-8      1656            604450 ns/op           3.39 MB/s     3596115 B/op        323 allocs/op
+BenchmarkTunnelTCPSmallMTU/2048-plain-b14-8         1729            624733 ns/op           3.28 MB/s     3610414 B/op        339 allocs/op
+BenchmarkTunnelTCPSmallMTU/2048-normal-b14-8        1568            653317 ns/op           3.13 MB/s     3611234 B/op        342 allocs/op
+BenchmarkTunnelTCPSmallMTU/2048-preshared-nob14-8   1858            664597 ns/op           3.08 MB/s     3595764 B/op        322 allocs/op
+BenchmarkTunnelTCPSmallMTU/2048-preshared-b14-8     1404            767077 ns/op           2.67 MB/s     3609789 B/op        339 allocs/op
+BenchmarkTunnelTCPSmallMTU/3072-plain-nob14-8       1761            846583 ns/op           3.63 MB/s     3614569 B/op        410 allocs/op
+BenchmarkTunnelTCPSmallMTU/3072-normal-nob14-8      1887            743407 ns/op           4.13 MB/s     3612869 B/op        411 allocs/op
+BenchmarkTunnelTCPSmallMTU/3072-plain-b14-8         1582            679431 ns/op           4.52 MB/s     3639650 B/op        435 allocs/op
+BenchmarkTunnelTCPSmallMTU/3072-normal-b14-8        1688            720574 ns/op           4.26 MB/s     3634744 B/op        435 allocs/op
+BenchmarkTunnelTCPSmallMTU/3072-preshared-nob14-8   1762            731901 ns/op           4.20 MB/s     3616570 B/op        414 allocs/op
+BenchmarkTunnelTCPSmallMTU/3072-preshared-b14-8     1656            716281 ns/op           4.29 MB/s     3636078 B/op        434 allocs/op
+BenchmarkTunnelTCPSmallMTU/4096-plain-nob14-8       1482            847378 ns/op           4.83 MB/s     7214173 B/op        666 allocs/op
+BenchmarkTunnelTCPSmallMTU/4096-normal-nob14-8      1354            818199 ns/op           5.01 MB/s     7219760 B/op        665 allocs/op
+BenchmarkTunnelTCPSmallMTU/4096-plain-b14-8         1557            784260 ns/op           5.22 MB/s     7243407 B/op        697 allocs/op
+BenchmarkTunnelTCPSmallMTU/4096-normal-b14-8        1316            811760 ns/op           5.05 MB/s     7241275 B/op        699 allocs/op
+BenchmarkTunnelTCPSmallMTU/4096-preshared-nob14-8   1299            806369 ns/op           5.08 MB/s     7216648 B/op        666 allocs/op
+BenchmarkTunnelTCPSmallMTU/4096-preshared-b14-8     1278            858201 ns/op           4.77 MB/s     7242324 B/op        703 allocs/op
+```

config/cfg.go

@@ -2,37 +2,29 @@ package config
 
 import (
 	"bytes"
-	"log"
 	"os"
 
+	"github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
 )
 
-// EndPoint 一个终结点的信息
-type EndPoint struct {
-	Host             string `yaml:"Host"`
-	Port             int64  `yaml:"Port"`
-	Poly             uint64 `yaml:"Poly"`             // Poly 是 port 随机切换算法的生成多项式, 0 为禁用
-	ReconnectSeconds int64  `yaml:"ReconnectSeconds"` // ReconnectSeconds 断开重连间隔, 每次到时即向对端通报并切换到新的端口, 0 为禁用
-	FECMethod        string `yaml:"FECMethod"`        // FECMethod 可选 1/2 2/3
-}
-
 // Config WireGold 配置文件
 type Config struct {
 	IP         string `yaml:"IP"`
 	SubNet     string `yaml:"SubNet"`
 	PrivateKey string `yaml:"PrivateKey"`
+	Network    string `yaml:"Network"` // Network udp, tcp or ws (WIP)
 	EndPoint   string `yaml:"EndPoint"`
-	MTU        int64  `yaml:"MTU"`
+	MTU        int64  `yaml:"MTU"` // MTU of nic (will minus packet header len)
 	SpeedLoop  uint16 `yaml:"SpeedLoop"`
-	Mask       uint64 `yaml:"Mask"` // Mask 是异或报文所用掩码, 必须保证各端统一
+	Mask       uint64 `yaml:"Mask"`   // Mask 是异或报文所用掩码, 必须保证各端统一
+	Base14     bool   `yaml:"Base14"` // Base14 是否将最终报文进行 base16384 编码后再发送
 	Peers      []Peer `yaml:"Peers"`
 }
 
 // Peer 对端信息
 type Peer struct {
 	IP               string   `yaml:"IP"`
-	SubNet           string   `yaml:"SubNet"`
 	PublicKey        string   `yaml:"PublicKey"`
 	PresharedKey     string   `yaml:"PresharedKey"`
 	EndPoint         string   `yaml:"EndPoint"`
@@ -42,7 +34,8 @@ type Peer struct {
 	QuerySeconds     int64    `yaml:"QuerySeconds"`
 	AllowTrans       bool     `yaml:"AllowTrans"`
 	UseZstd          bool     `yaml:"UseZstd"`
-	MTU              int64    `yaml:"MTU"`
+	DoublePacket     bool     `yaml:"DoublePacket"`
+	MTU              int64    `yaml:"MTU"` // MTU of PDU passed to p2p
 	MTURandomRange   int64    `yaml:"MTURandomRange"`
 }
 
@@ -50,11 +43,11 @@ type Peer struct {
 func Parse(path string) (c Config) {
 	file, err := os.ReadFile(path)
 	if err != nil {
-		log.Fatal("open config file failed:", err)
+		logrus.Fatal("open config file failed:", err)
 	}
 	err = yaml.NewDecoder(bytes.NewReader(file)).Decode(&c)
 	if err != nil {
-		log.Fatal("invalid config file:", err)
+		logrus.Fatal("invalid config file:", err)
 	}
 	return
 }

config/global.go

@@ -0,0 +1,3 @@
+package config
+
+const ShowDebugLog = true

go.mod

@@ -3,7 +3,8 @@ module github.com/fumiama/WireGold
 go 1.20
 
 require (
-	github.com/FloatTech/ttl v0.0.0-20230307105452-d6f7b2b647d1
+	github.com/FloatTech/ttl v0.0.0-20240716161252-965925764562
+	github.com/RomiChan/syncx v0.0.0-20240418144900-b7402ffdebc7
 	github.com/fumiama/blake2b-simd v0.0.0-20220412110131-4481822068bb
 	github.com/fumiama/go-base16384 v1.7.0
 	github.com/fumiama/go-x25519 v1.0.0

go.sum

@@ -1,5 +1,7 @@
-github.com/FloatTech/ttl v0.0.0-20230307105452-d6f7b2b647d1 h1:g4pTnDJUW4VbJ9NvoRfUvdjDrHz/6QhfN/LoIIpICbo=
+github.com/FloatTech/ttl v0.0.0-20240716161252-965925764562 h1:snfw7FNFym1eNnLrQ/VCf80LiQo9C7jHgrunZDwiRcY=
-github.com/FloatTech/ttl v0.0.0-20230307105452-d6f7b2b647d1/go.mod h1:fHZFWGquNXuHttu9dUYoKuNbm3dzLETnIOnm1muSfDs=
+github.com/FloatTech/ttl v0.0.0-20240716161252-965925764562/go.mod h1:fHZFWGquNXuHttu9dUYoKuNbm3dzLETnIOnm1muSfDs=
+github.com/RomiChan/syncx v0.0.0-20240418144900-b7402ffdebc7 h1:S/ferNiehVjNaBMNNBxUjLtVmP/YWD6Yh79RfPv4ehU=
+github.com/RomiChan/syncx v0.0.0-20240418144900-b7402ffdebc7/go.mod h1:vD7Ra3Q9onRtojoY5sMCLQ7JBgjUsrXDnDKyFxqpf9w=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

gold/head/flags.go

@@ -0,0 +1,34 @@
+package head
+
+import "encoding/binary"
+
+type PacketFlags uint16
+
+func (pf PacketFlags) IsValid() bool {
+	return pf&0x8000 == 0
+}
+
+func (pf PacketFlags) DontFrag() bool {
+	return pf&0x4000 == 0x4000
+}
+
+func (pf PacketFlags) NoFrag() bool {
+	return pf == 0x4000
+}
+
+func (pf PacketFlags) IsSingle() bool {
+	return pf == 0
+}
+
+func (pf PacketFlags) ZeroOffset() bool {
+	return pf&0x1fff == 0
+}
+
+func (pf PacketFlags) Offset() uint16 {
+	return uint16(pf << 3)
+}
+
+// Flags extract flags from raw data
+func Flags(data []byte) PacketFlags {
+	return PacketFlags(binary.LittleEndian.Uint16(data[10:12]))
+}

gold/head/hello.go

@@ -0,0 +1,8 @@
+package head
+
+type Hello uint8
+
+const (
+	HelloPing Hello = iota
+	HelloPong
+)

gold/head/nat.go

@@ -1,7 +1,7 @@
 package head
 
-// Notify 是 map[peerip]endpoint
+// Notify 是 map[peerip]{network, endpoint}
-type Notify = map[string]string
+type Notify = map[string][2]string
 
 // Query 是 peerips 组成的数组
 type Query = []string

gold/head/packet.go

@@ -4,44 +4,21 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"errors"
-	"hash/crc64"
 	"net"
 
-	"github.com/fumiama/WireGold/helper"
 	blake2b "github.com/fumiama/blake2b-simd"
 	"github.com/sirupsen/logrus"
+
+	"github.com/fumiama/WireGold/config"
+	"github.com/fumiama/WireGold/helper"
 )
 
-type PacketFlags uint16
+const PacketHeadLen = 60
 
-func (pf PacketFlags) IsValid() bool {
+var (
-	return pf&0x8000 == 0
+	ErrBadCRCChecksum = errors.New("bad crc checksum")
-}
+	ErrDataLenLT60    = errors.New("data len < 60")
-
+)
-func (pf PacketFlags) DontFrag() bool {
-	return pf&0x4000 == 0x4000
-}
-
-func (pf PacketFlags) NoFrag() bool {
-	return pf == 0x4000
-}
-
-func (pf PacketFlags) IsSingle() bool {
-	return pf == 0
-}
-
-func (pf PacketFlags) ZeroOffset() bool {
-	return pf&0x1fff == 0
-}
-
-func (pf PacketFlags) Offset() uint16 {
-	return uint16(pf << 3)
-}
-
-// Flags extract flags from raw data
-func Flags(data []byte) PacketFlags {
-	return PacketFlags(binary.LittleEndian.Uint16(data[10:12]))
-}
 
 // Packet 是发送和接收的最小单位
 type Packet struct {
@@ -68,49 +45,56 @@ type Packet struct {
 	// 生成时 Hash 全 0
 	// https://github.com/fumiama/blake2b-simd
 	Hash [32]byte
-	// crc64 包头字段的 checksum 值，可以认为在一定时间内唯一
+	// crc64 包头字段的 checksum 值，可以认为在一定时间内唯一 (现已更改算法为 md5 但名字未变)
 	crc64 uint64
-	// Data 承载的数据
+	// data 承载的数据
-	Data []byte
+	data []byte
+	// Data 当前的偏移
+	a, b int
 	// 记录还有多少字节未到达
 	rembytes int
+	// 是否经由 helper.MakeBytes 创建 Data
+	buffered bool
 }
 
 // NewPacket 生成一个新包
 func NewPacket(proto uint8, srcPort uint16, dst net.IP, dstPort uint16, data []byte) (p *Packet) {
-	// logrus.Debugln("[packet] new: [proto:", proto, ", srcport:", srcPort, ", dstport:", dstPort, ", dst:", dst, ", data:", data)
 	p = SelectPacket()
 	p.Proto = proto
 	p.TTL = 16
 	p.SrcPort = srcPort
 	p.DstPort = dstPort
 	p.Dst = dst
-	p.Data = data
+	p.data = data
+	p.b = len(data)
 	return
 }
 
 // Unmarshal 将 data 的数据解码到自身
 func (p *Packet) Unmarshal(data []byte) (complete bool, err error) {
 	if len(data) < 60 {
-		err = errors.New("data len < 60")
+		err = ErrDataLenLT60
 		return
 	}
-	p.crc64 = binary.LittleEndian.Uint64(data[52:60])
+	p.crc64 = CRC64(data)
-	if crc64.Checksum(data[:52], crc64.MakeTable(crc64.ISO)) != p.crc64 {
+	if CalcCRC64(data) != p.crc64 {
-		err = errors.New("bad crc checksum")
+		err = ErrBadCRCChecksum
 		return
 	}
 
-	sz := p.idxdatsz & 0x0000ffff
+	sz := p.Len()
-	if sz == 0 && len(p.Data) == 0 {
+	if sz == 0 && len(p.data) == 0 {
 		p.idxdatsz = binary.LittleEndian.Uint32(data[:4])
-		sz = p.idxdatsz & 0x0000ffff
+		sz = p.Len()
-		if int(sz)+52 == len(data) {
+		if sz+52 == len(data) {
-			p.Data = data[52:]
+			p.data = data[52:]
+			p.b = len(p.data)
 			p.rembytes = 0
 		} else {
-			p.Data = make([]byte, sz)
+			p.data = helper.MakeBytes(sz)
-			p.rembytes = int(sz)
+			p.buffered = true
+			p.b = sz
+			p.rembytes = sz
 		}
 		pt := binary.LittleEndian.Uint16(data[4:6])
 		p.Proto = uint8(pt)
@@ -131,8 +115,10 @@ func (p *Packet) Unmarshal(data []byte) (complete bool, err error) {
 	}
 
 	if p.rembytes > 0 {
-		p.rembytes -= copy(p.Data[flags.Offset():], data[60:])
+		p.rembytes -= copy(p.data[flags.Offset():], data[PacketHeadLen:])
-		logrus.Debugln("[packet] copied frag", hex.EncodeToString(p.Hash[:]), "rembytes:", p.rembytes)
+		if config.ShowDebugLog {
+			logrus.Debugln("[packet] copied frag", hex.EncodeToString(p.Hash[:]), "rembytes:", p.rembytes)
+		}
 	}
 
 	complete = p.rembytes == 0
@@ -140,26 +126,28 @@ func (p *Packet) Unmarshal(data []byte) (complete bool, err error) {
 	return
 }
 
+// DecreaseAndGetTTL TTL 自减后返回
+func (p *Packet) DecreaseAndGetTTL() uint8 {
+	p.TTL--
+	return p.TTL
+}
+
 // Marshal 将自身数据编码为 []byte
 // offset 必须为 8 的倍数，表示偏移的 8 位
 func (p *Packet) Marshal(src net.IP, teatype uint8, additional uint16, datasz uint32, offset uint16, dontfrag, hasmore bool) ([]byte, func()) {
-	p.TTL--
+	if src != nil {
-	if p.TTL == 0 {
+		p.Src = src
-		return nil, nil
+		p.idxdatsz = (uint32(teatype) << 27) | (uint32(additional&0x07ff) << 16) | datasz&0xffff
 	}
 
-	if src != nil {
+	offset &= 0x1fff
-		p.idxdatsz = (uint32(teatype) << 27) | (uint32(additional&0x07ff) << 16) | datasz&0xffff
+	if dontfrag {
-		p.Src = src
+		offset |= 0x4000
-		offset &= 0x1fff
-		if dontfrag {
-			offset |= 0x4000
-		}
-		if hasmore {
-			offset |= 0x2000
-		}
-		p.Flags = PacketFlags(offset)
 	}
+	if hasmore {
+		offset |= 0x2000
+	}
+	p.Flags = PacketFlags(offset)
 
 	return helper.OpenWriterF(func(w *helper.Writer) {
 		w.WriteUInt32(p.idxdatsz)
@@ -170,33 +158,40 @@ func (p *Packet) Marshal(src net.IP, teatype uint8, additional uint16, datasz ui
 		w.Write(p.Src.To4())
 		w.Write(p.Dst.To4())
 		w.Write(p.Hash[:])
-		w.WriteUInt64(crc64.Checksum(w.Bytes(), crc64.MakeTable(crc64.ISO)))
+		p.crc64 = CalcCRC64(w.Bytes())
-		w.Write(p.Data)
+		w.WriteUInt64(p.crc64)
+		w.Write(p.Body())
 	})
 }
 
 // FillHash 生成 p.Data 的 Hash
 func (p *Packet) FillHash() {
 	h := blake2b.New256()
-	_, err := h.Write(p.Data)
+	_, err := h.Write(p.Body())
 	if err != nil {
-		logrus.Error("[packet] err when fill hash:", err)
+		logrus.Errorln("[packet] err when fill hash:", err)
 		return
 	}
-	logrus.Debugln("[packet] sum calulated:", hex.EncodeToString(h.Sum(p.Hash[:0])))
+	hsh := h.Sum(p.Hash[:0])
+	if config.ShowDebugLog {
+		logrus.Debugln("[packet] sum calulated:", hex.EncodeToString(hsh))
+	}
 }
 
 // IsVaildHash 验证 packet 合法性
 func (p *Packet) IsVaildHash() bool {
 	h := blake2b.New256()
-	_, err := h.Write(p.Data)
+	_, err := h.Write(p.Body())
 	if err != nil {
-		logrus.Error("[packet] err when check hash:", err)
+		logrus.Errorln("[packet] err when check hash:", err)
 		return false
 	}
 	var sum [32]byte
-	logrus.Debugln("[packet] sum calulated:", hex.EncodeToString(h.Sum(sum[:0])))
+	_ = h.Sum(sum[:0])
-	logrus.Debugln("[packet] sum in packet:", hex.EncodeToString(p.Hash[:]))
+	if config.ShowDebugLog {
+		logrus.Debugln("[packet] sum calulated:", hex.EncodeToString(sum[:]))
+		logrus.Debugln("[packet] sum in packet:", hex.EncodeToString(p.Hash[:]))
+	}
 	return sum == p.Hash
 }
 
@@ -219,3 +214,58 @@ func (p *Packet) Len() int {
 func (p *Packet) Put() {
 	PutPacket(p)
 }
+
+func (p *Packet) CRC64() uint64 {
+	return p.crc64
+}
+
+// Body returns data
+func (p *Packet) Body() []byte {
+	return p.data[p.a:p.b]
+}
+
+func (p *Packet) BodyLen() int {
+	return p.b - p.a
+}
+
+func (p *Packet) SetBody(b []byte, buffered bool) {
+	p.a = 0
+	p.b = len(b)
+	if len(b) <= cap(p.data) {
+		p.data = p.data[:len(b)]
+		copy(p.data, b)
+		if buffered {
+			helper.PutBytes(b)
+		}
+		return
+	}
+	if p.buffered {
+		helper.PutBytes(p.data)
+	}
+	p.data = b
+	p.buffered = buffered
+}
+
+func (p *Packet) CropBody(a, b int) {
+	if b > len(p.data) {
+		b = len(p.data)
+	}
+	if a < 0 || b < 0 || a > b {
+		return
+	}
+	p.a, p.b = a, b
+}
+
+func (p *Packet) Copy() *Packet {
+	newp := SelectPacket()
+	*newp = *p
+	newp.buffered = false
+	return newp
+}
+
+func (p *Packet) CopyWithBody() *Packet {
+	newp := p.Copy()
+	newp.data = helper.MakeBytes(len(p.data))
+	copy(newp.data, p.data)
+	return newp
+}

gold/head/pool.go

@@ -1,6 +1,10 @@
 package head
 
-import "sync"
+import (
+	"sync"
+
+	"github.com/fumiama/WireGold/helper"
+)
 
 var packetPool = sync.Pool{
 	New: func() interface{} {
@@ -16,6 +20,12 @@ func SelectPacket() *Packet {
 // PutPacket 将 Packet 放回池中
 func PutPacket(p *Packet) {
 	p.idxdatsz = 0
-	p.Data = nil
+	if p.buffered {
+		helper.PutBytes(p.data)
+		p.buffered = false
+	}
+	p.a, p.b = 0, 0
+	p.data = nil
+	p.rembytes = 0
 	packetPool.Put(p)
 }

gold/head/raw.go

@@ -0,0 +1,22 @@
+package head
+
+import (
+	"crypto/md5"
+	"encoding/binary"
+)
+
+// CRC64 extract packet header checksum
+func CRC64(data []byte) uint64 {
+	return binary.LittleEndian.Uint64(data[52:PacketHeadLen])
+}
+
+// CalcCRC64 calculate packet header checksum
+func CalcCRC64(data []byte) uint64 {
+	m := md5.Sum(data[:52])
+	return binary.LittleEndian.Uint64(m[:8])
+}
+
+// Hash extract 32 bytes blake2b hash from raw bytes
+func Hash(data []byte) []byte {
+	return data[20:52]
+}

gold/link/crypto.go

@@ -8,6 +8,7 @@ import (
 	"math/bits"
 	mrand "math/rand"
 
+	"github.com/fumiama/WireGold/helper"
 	"github.com/sirupsen/logrus"
 )
 
@@ -51,13 +52,13 @@ func expandkeyunit(v1, v2 byte) (v uint16) {
 	return
 }
 
-// Encode 使用 xchacha20poly1305 和密钥序列加密
+// Encode by aead and put b into pool
 func (l *Link) Encode(teatype uint8, additional uint16, b []byte) (eb []byte) {
 	if len(b) == 0 || teatype >= 32 {
 		return
 	}
 	if l.keys[0] == nil {
-		eb = make([]byte, len(b))
+		eb = helper.MakeBytes(len(b))
 		copy(eb, b)
 		return
 	}
@@ -70,13 +71,14 @@ func (l *Link) Encode(teatype uint8, additional uint16, b []byte) (eb []byte) {
 	return
 }
 
-// Decode 使用 xchacha20poly1305 和密钥序列解密
+// Decode by aead and put b into pool
 func (l *Link) Decode(teatype uint8, additional uint16, b []byte) (db []byte, err error) {
 	if len(b) == 0 || teatype >= 32 {
 		return
 	}
 	if l.keys[0] == nil {
-		db = b
+		db = helper.MakeBytes(len(b))
+		copy(db, b)
 		return
 	}
 	aead := l.keys[teatype]
@@ -86,11 +88,10 @@ func (l *Link) Decode(teatype uint8, additional uint16, b []byte) (db []byte, er
 	return decode(aead, additional, b)
 }
 
-// encode 使用 xchacha20poly1305 加密
 func encode(aead cipher.AEAD, additional uint16, b []byte) []byte {
 	nsz := aead.NonceSize()
 	// Accocate capacity for all the stuffs.
-	buf := make([]byte, 2+nsz+len(b)+aead.Overhead())
+	buf := helper.MakeBytes(2 + nsz + len(b) + aead.Overhead())
 	binary.LittleEndian.PutUint16(buf[:2], additional)
 	nonce := buf[2 : 2+nsz]
 	// Select a random nonce
@@ -103,7 +104,6 @@ func encode(aead cipher.AEAD, additional uint16, b []byte) []byte {
 	return nonce[:nsz+len(eb)]
 }
 
-// decode 使用 xchacha20poly1305 解密
 func decode(aead cipher.AEAD, additional uint16, b []byte) ([]byte, error) {
 	nsz := aead.NonceSize()
 	if len(b) < nsz {
@@ -117,61 +117,51 @@ func decode(aead cipher.AEAD, additional uint16, b []byte) ([]byte, error) {
 	// Decrypt the message and check it wasn't tampered with.
 	var buf [2]byte
 	binary.LittleEndian.PutUint16(buf[:], additional)
-	return aead.Open(nil, nonce, ciphertext, buf[:])
+	return aead.Open(helper.SelectWriter().Bytes(), nonce, ciphertext, buf[:])
 }
 
 // xorenc 按 8 字节, 以初始 m.mask 循环异或编码 data
-func (m *Me) xorenc(data []byte) []byte {
+func (m *Me) xorenc(data []byte, seq uint32) []byte {
 	batchsz := len(data) / 8
 	remain := len(data) % 8
 	sum := m.mask
-	if remain > 0 {
+	newdat := helper.MakeBytes(8 + batchsz*8 + 8) // seqrand dat tail
-		var buf [8]byte
+	binary.LittleEndian.PutUint32(newdat[:4], seq)
-		p := batchsz * 8
+	_, _ = rand.Read(newdat[4:8])                 // seqrand
-		copy(buf[:], data[p:])
+	sum ^= binary.LittleEndian.Uint64(newdat[:8]) // init from seqrand
-		sum ^= binary.LittleEndian.Uint64(buf[:])
+	binary.LittleEndian.PutUint64(newdat[:8], sum)
-		binary.LittleEndian.PutUint64(buf[:], sum)
+	for i := 0; i < batchsz; i++ { // range on batch data
-		copy(data[p:], buf[:])
-	}
-	for i := batchsz - 1; i >= 0; i-- {
 		a := i * 8
 		b := (i + 1) * 8
 		sum ^= binary.LittleEndian.Uint64(data[a:b])
-		binary.LittleEndian.PutUint64(data[a:b], sum)
+		binary.LittleEndian.PutUint64(newdat[a+8:b+8], sum)
 	}
-	return data
+	p := batchsz * 8
+	copy(newdat[8+p:], data[p:])
+	newdat[len(newdat)-1] = byte(remain)
+	sum ^= binary.LittleEndian.Uint64(newdat[8+p:])
+	binary.LittleEndian.PutUint64(newdat[8+p:], sum)
+	return newdat
 }
 
 // xordec 按 8 字节, 以初始 m.mask 循环异或解码 data
-func (m *Me) xordec(data []byte) []byte {
+func (m *Me) xordec(data []byte) (uint32, []byte) {
-	batchsz := len(data) / 8
+	if len(data) < 16 || len(data)%8 != 0 {
-	remain := len(data) % 8
+		return 0, nil
-	this := uint64(0)
-	next := uint64(0)
-	if len(data) >= 8 {
-		next = binary.LittleEndian.Uint64(data[:8])
 	}
-	for i := 0; i < batchsz-1; i++ {
+	batchsz := len(data) / 8
+	sum := m.mask
+	for i := 0; i < batchsz; i++ {
 		a := i * 8
 		b := (i + 1) * 8
-		this = next
+		x := binary.LittleEndian.Uint64(data[a:b])
-		next = binary.LittleEndian.Uint64(data[a+8 : b+8])
+		sum ^= x
-		binary.LittleEndian.PutUint64(data[a:b], this^next)
+		binary.LittleEndian.PutUint64(data[a:b], sum)
+		sum = x
 	}
-	if remain > 0 {
+	remain := data[len(data)-1]
-		var buf [8]byte
+	if remain >= 8 {
-		a := (batchsz - 1) * 8
+		return 0, nil
-		b := batchsz * 8
-		copy(buf[:], data[b:])
-		this = next
-		next = binary.LittleEndian.Uint64(buf[:]) | (m.mask & (uint64(0xffffffff_ffffffff) << (uint64(remain) * 8)))
-		if batchsz > 0 {
-			binary.LittleEndian.PutUint64(data[a:b], this^next)
-		}
-		binary.LittleEndian.PutUint64(buf[:], next^m.mask)
-		copy(data[b:], buf[:])
-	} else {
-		binary.LittleEndian.PutUint64(data[len(data)-8:], next^m.mask)
 	}
-	return data
+	return binary.LittleEndian.Uint32(data[:4]), data[8 : len(data)-8+int(remain)]
 }

gold/link/crypto_test.go

@@ -17,7 +17,7 @@ func TestXOR(t *testing.T) {
 	}
 	buf := make([]byte, 4096)
 	buf2 := make([]byte, 4096)
-	for i := 1; i < 4096; i++ {
+	for i := 0; i < 4096; i++ {
 		data := buf[:i]
 		orgdata := buf2[:i]
 		r1 := bytes.NewBuffer(data[:0])
@@ -27,8 +27,12 @@ func TestXOR(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		if !bytes.Equal(m.xordec(m.xorenc(r1.Bytes())), r2.Bytes()) {
+		seq, dec := m.xordec(m.xorenc(r1.Bytes(), uint32(i)))
-			t.Fatal("unexpected xor at", i)
+		if !bytes.Equal(dec, r2.Bytes()) {
+			t.Fatal("unexpected xor at", i, "except", hex.EncodeToString(r2.Bytes()), "got", hex.EncodeToString(dec))
+		}
+		if seq != uint32(i) {
+			t.Fatal("unexpected xor at", i, "seq", seq)
 		}
 	}
 }

gold/link/link.go

@@ -5,12 +5,18 @@ import (
 	"errors"
 	"net"
 	"sync/atomic"
+	"time"
 
 	"github.com/fumiama/WireGold/gold/head"
+	"github.com/fumiama/WireGold/gold/p2p"
 	"github.com/fumiama/WireGold/helper"
 	base14 "github.com/fumiama/go-base16384"
 )
 
+var (
+	ErrPerrNotExist = errors.New("peer not exist")
+)
+
 // Link 是本机到 peer 的连接抽象
 type Link struct {
 	// peer 的公钥
@@ -24,43 +30,41 @@ type Link struct {
 	// peer 的虚拟 ip
 	peerip net.IP
 	// peer 的公网 endpoint
-	endpoint *net.UDPAddr
+	endpoint p2p.EndPoint
+	// peer 在设置的原始值
+	rawep string
 	// 本机允许接收/发送的 ip 网段
 	allowedips []*net.IPNet
 	// 连接所用对称加密密钥集
 	keys [32]cipher.AEAD
 	// 本机信息
 	me *Me
-	// 连接的状态，详见下方 const
+	// 最后一次收到报文的时间
-	status int8
+	lastalive *time.Time
 	// 是否允许转发
 	allowtrans bool
 	// 是否对数据进行 zstd 压缩
 	usezstd bool
+	// 是否采用双倍发包对抗强丢包
+	doublepacket bool
 	// udp 数据包的最大大小
 	mtu uint16
 	// 随机放缩 mtu 范围 (只减不增)
 	mturandomrange uint16
 }
 
-const (
-	LINK_STATUS_DOWN = iota
-	LINK_STATUS_HALFUP
-	LINK_STATUS_UP
-)
-
 // Connect 初始化与 peer 的连接
 func (m *Me) Connect(peer string) (*Link, error) {
 	p, ok := m.IsInPeer(net.ParseIP(peer).String())
 	if ok {
 		return p, nil
 	}
-	return nil, errors.New("peer not exist")
+	return nil, ErrPerrNotExist
 }
 
 // Close 关闭到 peer 的连接
 func (l *Link) Close() {
-	l.status = LINK_STATUS_DOWN
+	l.Destroy()
 }
 
 // Destroy 从 connections 移除 peer

gold/link/listen.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"io"
 	"net"
-	"net/netip"
 	"runtime"
 	"strconv"
 	"sync"
@@ -16,97 +15,175 @@ import (
 	"github.com/klauspost/compress/zstd"
 	"github.com/sirupsen/logrus"
 
+	"github.com/fumiama/WireGold/config"
 	"github.com/fumiama/WireGold/gold/head"
+	"github.com/fumiama/WireGold/gold/p2p"
+	"github.com/fumiama/WireGold/helper"
 )
 
-// 监听本机 UDP endpoint
+const lstnbufgragsz = 65536
-func (m *Me) listenudp() (conn *net.UDPConn, err error) {
+
-	conn, err = net.ListenUDP("udp", net.UDPAddrFromAddrPort(netip.MustParseAddrPort(m.udpep.String())))
+type lstnq struct {
+	index int
+	addr  p2p.EndPoint
+	buf   []byte
+}
+
+type listenqueue chan lstnq
+
+func (q listenqueue) listen(m *Me, hasntfinished []sync.Mutex) {
+	recvtotlcnt := uint64(0)
+	recvloopcnt := uint16(0)
+	recvlooptime := time.Now().UnixMilli()
+	for lstn := range q {
+		recvtotlcnt += uint64(len(lstn.buf))
+		recvloopcnt++
+		if recvloopcnt%m.speedloop == 0 {
+			now := time.Now().UnixMilli()
+			logrus.Infof("[listen] queue recv avg speed: %.2f KB/s", float64(recvtotlcnt)/float64(now-recvlooptime))
+			recvtotlcnt = 0
+			recvlooptime = now
+		}
+		packet := m.wait(lstn.buf[:len(lstn.buf):lstnbufgragsz])
+		if packet == nil {
+			if lstn.index < 0 {
+				if config.ShowDebugLog {
+					logrus.Debugln("[listen] queue waiting")
+				}
+				helper.PutBytes(lstn.buf)
+				continue
+			}
+			if config.ShowDebugLog {
+				logrus.Debugln("[listen] queue waiting, unlock index", lstn.index)
+			}
+			hasntfinished[lstn.index].Unlock()
+			continue
+		}
+		if lstn.index >= 0 {
+			go m.dispatch(packet, lstn.addr, lstn.index, hasntfinished[lstn.index].Unlock)
+		} else {
+			go m.dispatch(packet, lstn.addr, lstn.index, func() {
+				helper.PutBytes(lstn.buf)
+			})
+		}
+	}
+}
+
+// 监听本机 endpoint
+func (m *Me) listen() (conn p2p.Conn, err error) {
+	conn, err = m.ep.Listen()
 	if err != nil {
 		return
 	}
-	m.udpep = conn.LocalAddr()
+	m.ep = conn.LocalAddr()
-	logrus.Infoln("[listen] at", m.udpep)
+	logrus.Infoln("[listen] at", m.ep)
 	go func() {
-		recvtotlcnt := uint64(0)
+		n := uint(runtime.NumCPU())
-		recvloopcnt := uint16(0)
-		recvlooptime := time.Now().UnixMilli()
-		n := runtime.NumCPU()
 		if n > 64 {
 			n = 64 // 只用最多 64 核
 		}
 		logrus.Infoln("[listen] use cpu num:", n)
-		listenbuff := make([]byte, 65536*n)
+		listenbuf := make([]byte, lstnbufgragsz*n)
 		hasntfinished := make([]sync.Mutex, n)
-		for i := 0; err == nil; i++ {
+		q := make(listenqueue, n)
-			i %= n
+		defer close(q)
+		go q.listen(m, hasntfinished)
+		for {
+			usenewbuf := false
+			i := uint(0)
 			for !hasntfinished[i].TryLock() {
 				i++
 				i %= n
-				if i == 0 { // looked up a full round
+				if i == 0 { // looked up a full round, make a new buf
-					time.Sleep(time.Millisecond * 10)
+					usenewbuf = true
+					if config.ShowDebugLog {
+						logrus.Debugln("[listen] use new buf")
+					}
+					break
 				}
 			}
-			logrus.Debugln("[listen] lock index", i)
+			if config.ShowDebugLog && !usenewbuf {
-			lbf := listenbuff[i*65536 : (i+1)*65536]
+				logrus.Debugln("[listen] lock index", i)
-			n, addr, err := conn.ReadFromUDP(lbf)
+			}
-			if m.loop == nil || errors.Is(err, net.ErrClosed) {
+			var lbf []byte
+			if usenewbuf {
+				lbf = helper.MakeBytes(lstnbufgragsz)
+			} else {
+				lbf = listenbuf[i*lstnbufgragsz : (i+1)*lstnbufgragsz]
+			}
+			n, addr, err := conn.ReadFromPeer(lbf)
+			if m.connections == nil || errors.Is(err, net.ErrClosed) {
 				logrus.Warnln("[listen] quit listening")
 				return
 			}
 			if err != nil {
-				logrus.Warnln("[listen] read from udp err, reconnect:", err)
+				logrus.Warnln("[listen] read from conn err, reconnect:", err)
-				conn, err = net.ListenUDP("udp", net.UDPAddrFromAddrPort(netip.MustParseAddrPort(m.udpep.String())))
+				conn, err = m.ep.Listen()
 				if err != nil {
 					logrus.Errorln("[listen] reconnect udp err:", err)
 					return
 				}
-				logrus.Debugln("[listen] unlock index", i)
+				if !usenewbuf {
-				hasntfinished[i].Unlock()
+					if config.ShowDebugLog {
-				i--
+						logrus.Debugln("[listen] unlock index", i)
+					}
+					hasntfinished[i].Unlock()
+					i--
+				}
 				continue
 			}
-			recvtotlcnt += uint64(n)
+			if n <= 0 {
-			recvloopcnt++
+				if config.ShowDebugLog {
-			if recvloopcnt%m.speedloop == 0 {
+					logrus.Debugln("[listen] unexpected read n =", n)
-				now := time.Now().UnixMilli()
+				}
-				logrus.Infof("[listen] recv avg speed: %.2f KB/s", float64(recvtotlcnt)/float64(now-recvlooptime))
-				recvtotlcnt = 0
-				recvlooptime = now
-			}
-			packet := m.wait(lbf[:n])
-			if packet == nil {
-				logrus.Debugln("[listen] unlock index", i)
-				hasntfinished[i].Unlock()
-				i--
 				continue
 			}
-			go m.listenthread(packet, addr, i, hasntfinished[i].Unlock)
+			lq := lstnq{
+				index: -1,
+				addr:  addr,
+				buf:   lbf[:n],
+			}
+			if !usenewbuf {
+				lq.index = int(i)
+			}
+			q <- lq
 		}
 	}()
 	return
 }
 
-func (m *Me) listenthread(packet *head.Packet, addr *net.UDPAddr, index int, finish func()) {
+func (m *Me) dispatch(packet *head.Packet, addr p2p.EndPoint, index int, finish func()) {
 	defer finish()
-	defer logrus.Debugln("[listen] unlock index", index)
+	if config.ShowDebugLog {
-	r := packet.Len() - len(packet.Data)
+		defer logrus.Debugln("[listen] dispatched, unlock index", index)
+		logrus.Debugln("[listen] start dispatching index", index)
+	}
+	r := packet.Len() - packet.BodyLen()
 	if r > 0 {
-		logrus.Warnln("[listen] @", index, "packet from endpoint", addr, "is smaller than it declared: drop it")
+		logrus.Warnln("[listen] @", index, "packet from endpoint", addr, "len", packet.BodyLen(), "is smaller than it declared len", packet.Len(), ", drop it")
 		packet.Put()
 		return
 	}
 	p, ok := m.IsInPeer(packet.Src.String())
-	logrus.Debugln("[listen] @", index, "recv from endpoint", addr, "src", packet.Src, "dst", packet.Dst)
+	if config.ShowDebugLog {
+		logrus.Debugln("[listen] @", index, "recv from endpoint", addr, "src", packet.Src, "dst", packet.Dst)
+	}
 	if !ok {
 		logrus.Warnln("[listen] @", index, "packet from", packet.Src, "to", packet.Dst, "is refused")
 		packet.Put()
 		return
 	}
-	if p.endpoint == nil || p.endpoint.String() != addr.String() {
+	if helper.IsNilInterface(p.endpoint) || !p.endpoint.Euqal(addr) {
-		logrus.Infoln("[listen] @", index, "set endpoint of peer", p.peerip, "to", addr.String())
+		if m.ep.Network() == "tcp" && !addr.Euqal(p.endpoint) {
-		atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&p.endpoint)), unsafe.Pointer(addr))
+			logrus.Infoln("[listen] @", index, "set endpoint of peer", p.peerip, "to", addr.String())
+			p.endpoint = addr
+		} else { // others are all no status link
+			logrus.Infoln("[listen] @", index, "set endpoint of peer", p.peerip, "to", addr.String())
+			p.endpoint = addr
+		}
 	}
+	now := time.Now()
+	atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&p.lastalive)), unsafe.Pointer(&now))
 	switch {
 	case p.IsToMe(packet.Dst):
 		if !p.Accept(packet.Src) {
@@ -116,65 +193,76 @@ func (m *Me) listenthread(packet *head.Packet, addr *net.UDPAddr, index int, fin
 		}
 		addt := packet.AdditionalData()
 		var err error
-		packet.Data, err = p.Decode(packet.CipherIndex(), addt, packet.Data)
+		data, err := p.Decode(packet.CipherIndex(), addt, packet.Body())
 		if err != nil {
-			logrus.Debugln("[listen] @", index, "drop invalid packet", ", key idx:", packet.CipherIndex(), "addt:", addt, "err:", err)
+			if config.ShowDebugLog {
+				logrus.Debugln("[listen] @", index, "drop invalid packet key idx:", packet.CipherIndex(), "addt:", addt, "err:", err)
+			}
 			packet.Put()
 			return
 		}
+		packet.SetBody(data, true)
 		if p.usezstd {
-			dec, _ := zstd.NewReader(bytes.NewReader(packet.Data))
+			dec, _ := zstd.NewReader(bytes.NewReader(packet.Body()))
 			var err error
-			packet.Data, err = io.ReadAll(dec)
+			w := helper.SelectWriter()
+			_, err = io.Copy(w, dec)
 			dec.Close()
 			if err != nil {
-				logrus.Debugln("[listen] @", index, "drop invalid zstd packet:", err)
+				if config.ShowDebugLog {
+					logrus.Debugln("[listen] @", index, "drop invalid zstd packet:", err)
+				}
 				packet.Put()
 				return
 			}
+			packet.SetBody(w.Bytes(), true)
 		}
 		if !packet.IsVaildHash() {
-			logrus.Debugln("[listen] @", index, "drop invalid hash packet")
+			if config.ShowDebugLog {
+				logrus.Debugln("[listen] @", index, "drop invalid hash packet")
+			}
 			packet.Put()
 			return
 		}
 		switch packet.Proto {
 		case head.ProtoHello:
-			switch p.status {
+			switch {
-			case LINK_STATUS_DOWN:
+			case len(packet.Body()) == 0:
-				n, err := p.WriteAndPut(head.NewPacket(head.ProtoHello, m.SrcPort(), p.peerip, m.DstPort(), nil), false)
+				logrus.Warnln("[listen] @", index, "recv old hello packet, do nothing")
+			case packet.Body()[0] == byte(head.HelloPing):
+				n, err := p.WriteAndPut(head.NewPacket(head.ProtoHello, m.SrcPort(), p.peerip, m.DstPort(), []byte{byte(head.HelloPong)}), false)
 				if err == nil {
-					logrus.Debugln("[listen] @", index, "send", n, "bytes hello ack packet")
+					logrus.Infoln("[listen] @", index, "recv hello, send", n, "bytes hello ack packet")
-					p.status = LINK_STATUS_HALFUP
 				} else {
 					logrus.Errorln("[listen] @", index, "send hello ack packet error:", err)
 				}
-			case LINK_STATUS_HALFUP:
+			default:
-				p.status = LINK_STATUS_UP
+				logrus.Infoln("[listen] @", index, "recv hello ack packet, do nothing")
-			case LINK_STATUS_UP:
 			}
 			packet.Put()
 		case head.ProtoNotify:
 			logrus.Infoln("[listen] @", index, "recv notify from", packet.Src)
-			go p.onNotify(packet.Data)
+			go p.onNotify(packet.Body())
 			packet.Put()
 		case head.ProtoQuery:
 			logrus.Infoln("[listen] @", index, "recv query from", packet.Src)
-			go p.onQuery(packet.Data)
+			go p.onQuery(packet.Body())
 			packet.Put()
 		case head.ProtoData:
 			if p.pipe != nil {
-				p.pipe <- packet
+				p.pipe <- packet.CopyWithBody()
-				logrus.Debugln("[listen] @", index, "deliver to pipe of", p.peerip)
+				if config.ShowDebugLog {
-			} else {
+					logrus.Debugln("[listen] @", index, "deliver to pipe of", p.peerip)
-				_, err := m.nic.Write(packet.Data)
+				}
-				if err != nil {
+			} else {
-					logrus.Errorln("[listen] @", index, "deliver", len(packet.Data), "bytes data to nic err:", err)
+				_, err := m.nic.Write(packet.Body())
-				} else {
+				if err != nil {
-					logrus.Debugln("[listen] @", index, "deliver", len(packet.Data), "bytes data to nic")
+					logrus.Errorln("[listen] @", index, "deliver", packet.BodyLen(), "bytes data to nic err:", err)
+				} else if config.ShowDebugLog {
+					logrus.Debugln("[listen] @", index, "deliver", packet.BodyLen(), "bytes data to nic")
 				}
-				packet.Put()
 			}
+			packet.Put()
 		default:
 			logrus.Warnln("[listen] @", index, "recv unknown proto:", packet.Proto)
 			packet.Put()
@@ -194,7 +282,9 @@ func (m *Me) listenthread(packet *head.Packet, addr *net.UDPAddr, index int, fin
 		}
 		n, err := lnk.WriteAndPut(packet, true)
 		if err == nil {
-			logrus.Debugln("[listen] @", index, "trans", n, "bytes packet to", packet.Dst.String()+":"+strconv.Itoa(int(packet.DstPort)))
+			if config.ShowDebugLog {
+				logrus.Debugln("[listen] @", index, "trans", n, "bytes packet to", packet.Dst.String()+":"+strconv.Itoa(int(packet.DstPort)))
+			}
 		} else {
 			logrus.Errorln("[listen] @", index, "trans packet to", packet.Dst.String()+":"+strconv.Itoa(int(packet.DstPort)), "err:", err)
 		}

gold/link/me.go

@@ -10,14 +10,20 @@ import (
 	"time"
 
 	"github.com/FloatTech/ttl"
-	"github.com/fumiama/WireGold/gold/head"
-	"github.com/fumiama/WireGold/lower"
 	"github.com/fumiama/water/waterutil"
 	"github.com/sirupsen/logrus"
+
+	"github.com/fumiama/WireGold/config"
+	"github.com/fumiama/WireGold/gold/head"
+	"github.com/fumiama/WireGold/gold/p2p"
+	"github.com/fumiama/WireGold/helper"
+	"github.com/fumiama/WireGold/lower"
 )
 
 // Me 是本机的抽象
 type Me struct {
+	// 用于自我重连
+	cfg *MyConfig
 	// 本机私钥
 	// 利用 Curve25519 生成
 	// https://pkg.go.dev/golang.org/x/crypto/curve25519
@@ -27,44 +33,61 @@ type Me struct {
 	me net.IP
 	// 本机子网
 	subnet net.IPNet
-	// 本机 UDP endpoint
+	// 本机 endpoint
-	udpep net.Addr
+	ep p2p.EndPoint
-	// 本机环回 link
-	loop *Link
 	// 本机活跃的所有连接
 	connections map[string]*Link
 	// 读写同步锁
 	connmapmu sync.RWMutex
-	// 本机监听的 udp 连接, 用于向对端直接发送报文
+	// 本机监听的连接端点, 也用于向对端直接发送报文
-	udpconn *net.UDPConn
+	conn p2p.Conn
 	// 本机网卡
-	nic lower.NICIO
+	nic *lower.NICIO
 	// 本机路由表
 	router *Router
 	// 本机未接收完全分片池
-	recving *ttl.Cache[[32]byte, *head.Packet]
+	recving *ttl.Cache[uint64, *head.Packet]
 	// 抗重放攻击记录池
 	recved *ttl.Cache[uint64, bool]
 	// 本机上层配置
 	srcport, dstport, mtu, speedloop uint16
 	// 报头掩码
 	mask uint64
+	// 是否进行 base16384 编码
+	base14 bool
+	// 本机网络端点初始化配置
+	networkconfigs []any
 }
 
 type MyConfig struct {
 	MyIPwithMask                     string
 	MyEndpoint                       string
+	Network                          string
+	NetworkConfigs                   []any
 	PrivateKey                       *[32]byte
-	NIC                              lower.NICIO
+	NICConfig                        *NICConfig
 	SrcPort, DstPort, MTU, SpeedLoop uint16
 	Mask                             uint64
+	Base14                           bool
+}
+
+type NICConfig struct {
+	IP     net.IP
+	SubNet *net.IPNet
+	CIDRs  []string
 }
 
 // NewMe 设置本机参数
 func NewMe(cfg *MyConfig) (m Me) {
+	m.cfg = cfg
 	m.privKey = *cfg.PrivateKey
 	var err error
-	m.udpep, err = net.ResolveUDPAddr("udp", cfg.MyEndpoint)
+	nw := cfg.Network
+	if nw == "" {
+		nw = "udp"
+	}
+	m.networkconfigs = cfg.NetworkConfigs
+	m.ep, err = p2p.NewEndPoint(nw, cfg.MyEndpoint, m.networkconfigs...)
 	if err != nil {
 		panic(err)
 	}
@@ -74,45 +97,67 @@ func NewMe(cfg *MyConfig) (m Me) {
 	}
 	m.me = ip
 	m.subnet = *cidr
-	m.udpconn, err = m.listenudp()
+	m.speedloop = cfg.SpeedLoop
+	if m.speedloop == 0 {
+		m.speedloop = 4096
+	}
+	m.conn, err = m.listen()
 	if err != nil {
 		panic(err)
 	}
 	m.connections = make(map[string]*Link)
-	m.nic = cfg.NIC
 	m.router = &Router{
 		list:  make([]*net.IPNet, 1, 16),
 		table: make(map[string]*Link, 16),
 		cache: ttl.NewCache[string, *Link](time.Minute),
 	}
 	m.router.SetDefault(nil)
-	_, localp, err := net.SplitHostPort(m.EndPoint().String())
-	if err != nil {
-		panic(err)
-	}
-	m.loop = m.AddPeer(&PeerConfig{
-		PeerIP:     m.me.String(),
-		EndPoint:   "127.0.0.1:" + localp,
-		AllowedIPs: []string{cfg.MyIPwithMask},
-		NoPipe:     cfg.NIC != nil,
-		MTU:        cfg.MTU,
-	})
 	m.srcport = cfg.SrcPort
 	m.dstport = cfg.DstPort
-	m.mtu = cfg.MTU & 0xfff8
+	m.mtu = (cfg.MTU - head.PacketHeadLen) & 0xfff8
-	m.speedloop = cfg.SpeedLoop
+	if cfg.NICConfig != nil {
-	if m.speedloop == 0 {
+		m.nic = lower.NewNIC(
-		m.speedloop = 4096
+			cfg.NICConfig.IP, cfg.NICConfig.SubNet,
+			strconv.FormatUint(uint64(m.MTU()), 10), cfg.NICConfig.CIDRs...,
+		)
 	}
 	m.mask = cfg.Mask
+	m.base14 = cfg.Base14
 	var buf [8]byte
 	binary.BigEndian.PutUint64(buf[:], m.mask)
 	logrus.Infoln("[me] xor mask", hex.EncodeToString(buf[:]))
-	m.recving = ttl.NewCache[[32]byte, *head.Packet](time.Second * 30)
+	m.recving = ttl.NewCache[uint64, *head.Packet](time.Second * 30)
 	m.recved = ttl.NewCache[uint64, bool](time.Second * 30)
 	return
 }
 
+// Restart 重新连接
+func (m *Me) Restart() error {
+	oldconn := m.conn
+	m.conn = nil
+	if helper.IsNonNilInterface(oldconn) {
+		_ = oldconn.Close()
+	}
+	var err error
+	nw := m.cfg.Network
+	if nw == "" {
+		nw = "udp"
+	}
+	m.networkconfigs = m.cfg.NetworkConfigs
+	m.ep, err = p2p.NewEndPoint(nw, m.cfg.MyEndpoint, m.networkconfigs...)
+	if err != nil {
+		return err
+	}
+	ip, cidr, err := net.ParseCIDR(m.cfg.MyIPwithMask)
+	if err != nil {
+		return err
+	}
+	m.me = ip
+	m.subnet = *cidr
+	m.conn, err = m.listen()
+	return err
+}
+
 func (m *Me) SrcPort() uint16 {
 	return m.srcport
 }
@@ -125,16 +170,15 @@ func (m *Me) MTU() uint16 {
 	return m.mtu
 }
 
-func (m *Me) EndPoint() net.Addr {
+func (m *Me) EndPoint() p2p.EndPoint {
-	return m.udpep
+	return m.ep
 }
 
 func (m *Me) Close() error {
-	m.loop = nil
 	m.connections = nil
-	if m.udpconn != nil {
+	if helper.IsNonNilInterface(m.conn) {
-		_ = m.udpconn.Close()
+		_ = m.conn.Close()
-		m.udpconn = nil
+		m.conn = nil
 	}
 	m.router = nil
 	if m.recving != nil {
@@ -154,7 +198,9 @@ func (m *Me) Close() error {
 
 func (m *Me) Write(packet []byte) (n int, err error) {
 	n = m.sendAllSameDst(packet)
-	logrus.Debugln("[me] writer ate", len(packet), "bytes, remain", len(packet)-n, "bytes")
+	if config.ShowDebugLog {
+		logrus.Debugln("[me] writer ate", len(packet), "bytes, remain", len(packet)-n, "bytes")
+	}
 	return
 }
 
@@ -183,7 +229,9 @@ func (m *Me) sendAllSameDst(packet []byte) (n int) {
 			}
 			n += pktl
 			rem = packet[n:]
-			logrus.Debugln("[me] skip to send", len(packet), "bytes ipv6 packet")
+			if config.ShowDebugLog {
+				logrus.Debugln("[me] skip to send", len(packet), "bytes ipv6 packet")
+			}
 		}
 		if len(rem) == 0 || !waterutil.IsIPv4(rem) {
 			logrus.Warnln("[me] skip to send", len(packet), "bytes full packet")
@@ -196,12 +244,16 @@ func (m *Me) sendAllSameDst(packet []byte) (n int) {
 	for len(ptr) > 20 && p.issame(ptr) {
 		totl := waterutil.IPv4TotalLength(ptr)
 		if int(totl) > len(ptr) {
-			logrus.Debugln("[me] wrap got invalid totl, break")
+			if config.ShowDebugLog {
+				logrus.Debugln("[me] wrap got invalid totl, break")
+			}
 			break
 		}
 		i += int(totl)
 		ptr = rem[i:]
-		logrus.Debugln("[me] wrap", totl, "bytes packet to send together")
+		if config.ShowDebugLog {
+			logrus.Debugln("[me] wrap", totl, "bytes packet to send together")
+		}
 	}
 	if i == 0 {
 		return
@@ -210,15 +262,32 @@ func (m *Me) sendAllSameDst(packet []byte) (n int) {
 	packet = rem[:i]
 	rem = rem[i:]
 	dst := waterutil.IPv4Destination(packet)
-	logrus.Debugln("[me] sending", len(packet), "bytes packet from :"+strconv.Itoa(int(m.SrcPort())), "to", dst.String()+":"+strconv.Itoa(int(m.DstPort())), "remain:", len(rem), "bytes")
+	if config.ShowDebugLog {
+		logrus.Debugln("[me] sending", len(packet), "bytes packet from :"+strconv.Itoa(int(m.SrcPort())), "to", dst.String()+":"+strconv.Itoa(int(m.DstPort())), "remain:", len(rem), "bytes")
+	}
+	if m.me.Equal(dst) { // is to myself, write to nic (pipe not allow loopback)
+		if config.ShowDebugLog {
+			logrus.Debugln("[me] loopback packet")
+		}
+		_, err := m.nic.Write(packet)
+		if err != nil {
+			logrus.Warnln("[me] write to loopback err:", err)
+		}
+		return
+	}
 	lnk := m.router.NextHop(dst.String())
 	if lnk == nil {
 		logrus.Warnln("[me] drop packet to", dst.String()+":"+strconv.Itoa(int(m.DstPort())), ": nil nexthop")
 		return
 	}
-	_, err := lnk.WriteAndPut(head.NewPacket(head.ProtoData, m.SrcPort(), lnk.peerip, m.DstPort(), packet), false)
+	pcp := helper.MakeBytes(len(packet))
-	if err != nil {
+	copy(pcp, packet)
-		logrus.Warnln("[me] write to peer", lnk.peerip, "err:", err)
+	go func(packet []byte) {
-	}
+		defer helper.PutBytes(packet)
+		_, err := lnk.WriteAndPut(head.NewPacket(head.ProtoData, m.SrcPort(), lnk.peerip, m.DstPort(), packet), false)
+		if err != nil {
+			logrus.Warnln("[me] write to peer", lnk.peerip, "err:", err)
+		}
+	}(pcp)
 	return
 }

gold/link/nat.go

@@ -2,12 +2,15 @@ package link
 
 import (
 	"encoding/json"
-	"net"
+	"sync/atomic"
 	"time"
+	"unsafe"
 
 	"github.com/sirupsen/logrus"
 
+	"github.com/fumiama/WireGold/config"
 	"github.com/fumiama/WireGold/gold/head"
+	"github.com/fumiama/WireGold/gold/p2p"
 	"github.com/fumiama/WireGold/helper"
 )
 
@@ -19,11 +22,24 @@ func (l *Link) keepAlive(dur int64) {
 		logrus.Infoln("[nat] start to keep alive")
 		t := time.NewTicker(time.Second * time.Duration(dur))
 		for range t.C {
-			n, err := l.WriteAndPut(head.NewPacket(head.ProtoHello, l.me.srcport, l.peerip, l.me.dstport, nil), false)
+			if l.me.connections == nil {
+				return
+			}
+			la := (*time.Time)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&l.lastalive))))
+			if la != nil && time.Since(*la) > 10*time.Second*time.Duration(dur) { // 可能已经被阻断， 断开重连
+				logrus.Warnln("[nat] no response after 10 keep alive tries, re-connecting...")
+				err := l.me.Restart()
+				if err != nil {
+					logrus.Errorln("[nat] re-connect me err:", err)
+				} else {
+					logrus.Infoln("[nat] re-connect me succeeded")
+				}
+			}
+			n, err := l.WriteAndPut(head.NewPacket(head.ProtoHello, l.me.srcport, l.peerip, l.me.dstport, []byte{byte(head.HelloPing)}), false)
 			if err == nil {
 				logrus.Infoln("[nat] send", n, "bytes keep alive packet")
 			} else {
-				logrus.Errorln("[nat] send keep alive packet error:", err)
+				logrus.Warnln("[nat] send keep alive packet error:", err)
 			}
 		}
 	}
@@ -44,18 +60,25 @@ func (l *Link) onNotify(packet []byte) {
 	// ---- 遍历 Notify，注册对方的 endpoint 到
 	// ---- connections，注意使用读写锁connmapmu
 	for peer, ep := range notify {
-		addr, err := net.ResolveUDPAddr("udp", ep)
+		nw, epstr := ep[0], ep[1]
+		if nw != l.me.ep.Network() {
+			logrus.Warnln("[nat] ignore different network notify", nw, "addr", epstr)
+			continue
+		}
+		addr, err := p2p.NewEndPoint(nw, epstr, l.me.networkconfigs...)
 		if err == nil {
 			p, ok := l.me.IsInPeer(peer)
 			if ok {
-				if p.endpoint.String() != ep {
+				if helper.IsNilInterface(p.endpoint) || !p.endpoint.Euqal(addr) {
 					p.endpoint = addr
 					logrus.Infoln("[nat] notify set ep of peer", peer, "to", ep)
 				}
 				continue
 			}
 		}
-		logrus.Debugln("[nat] notify drop invalid peer:", peer, "ep:", ep)
+		if config.ShowDebugLog {
+			logrus.Debugln("[nat] notify drop invalid peer:", peer, "ep:", ep)
+		}
 	}
 }
 
@@ -73,14 +96,35 @@ func (l *Link) onQuery(packet []byte) {
 		return
 	}
 
+	if l == nil || l.me == nil {
+		logrus.Errorln("[nat] nil link/me")
+		return
+	}
+
 	// 2. notify分发
 	// ---- 封装 Notify 到 新的 packet
 	// ---- 调用 l.Send 发送到对方
 	notify := make(head.Notify, len(peers))
 	for _, p := range peers {
 		lnk, ok := l.me.IsInPeer(p)
-		if ok {
+		eps := ""
-			notify[p] = lnk.endpoint.String()
+		if l.me.ep.Network() == "udp" { // udp has real p2p
+			if helper.IsNilInterface(lnk.endpoint) {
+				continue
+			}
+			eps = lnk.endpoint.String()
+		}
+		if eps == "" {
+			eps = l.rawep // use registered ep only
+		}
+		if eps == "" {
+			continue
+		}
+		if ok && helper.IsNonNilInterface(lnk.endpoint) {
+			notify[p] = [2]string{
+				lnk.endpoint.Network(),
+				eps,
+			}
 		}
 	}
 	if len(notify) > 0 {

gold/link/peer.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/fumiama/WireGold/gold/head"
+	"github.com/fumiama/WireGold/gold/p2p"
 	curve "github.com/fumiama/go-x25519"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/crypto/chacha20poly1305"
@@ -21,6 +22,7 @@ type PeerConfig struct {
 	MTURandomRange          uint16
 	AllowTrans, NoPipe      bool
 	UseZstd                 bool
+	DoublePacket            bool
 }
 
 // AddPeer 添加一个 peer
@@ -31,14 +33,16 @@ func (m *Me) AddPeer(cfg *PeerConfig) (l *Link) {
 	if ok {
 		return
 	}
-	if cfg.MTU == 0 || (m.mtu != 0 && cfg.MTU > m.mtu) {
+	if cfg.MTU == 0 {
 		panic("invalid mtu for peer " + cfg.PeerIP)
 	}
 	l = &Link{
 		pubk:           cfg.PubicKey,
 		peerip:         net.ParseIP(cfg.PeerIP),
+		rawep:          cfg.EndPoint,
 		allowtrans:     cfg.AllowTrans,
 		usezstd:        cfg.UseZstd,
+		doublepacket:   cfg.DoublePacket,
 		me:             m,
 		mtu:            cfg.MTU,
 		mturandomrange: cfg.MTURandomRange,
@@ -72,7 +76,7 @@ func (m *Me) AddPeer(cfg *PeerConfig) (l *Link) {
 		}
 	}
 	if cfg.EndPoint != "" {
-		e, err := net.ResolveUDPAddr("udp", cfg.EndPoint)
+		e, err := p2p.NewEndPoint(m.ep.Network(), cfg.EndPoint, m.networkconfigs...)
 		if err != nil {
 			panic(err)
 		}
@@ -85,7 +89,8 @@ func (m *Me) AddPeer(cfg *PeerConfig) (l *Link) {
 				continue
 			}
 			noroute := ipnet[0] == 'x'
-			if noroute {
+			innerroute := ipnet[0] == 'y'
+			if noroute || innerroute {
 				ipnet = ipnet[1:]
 				if len(ipnet) == 0 {
 					continue

gold/link/recv.go

@@ -3,10 +3,12 @@ package link
 import (
 	"encoding/binary"
 	"encoding/hex"
+	"hash/crc64"
 	"strconv"
-	"unsafe"
 
+	"github.com/fumiama/WireGold/config"
 	"github.com/fumiama/WireGold/gold/head"
+	base14 "github.com/fumiama/go-base16384"
 	"github.com/sirupsen/logrus"
 )
 
@@ -16,7 +18,10 @@ func (l *Link) Read() *head.Packet {
 }
 
 func (m *Me) wait(data []byte) *head.Packet {
-	if len(data) < 60 { // not a valid packet
+	if len(data) < head.PacketHeadLen { // not a valid packet
+		if config.ShowDebugLog {
+			logrus.Debugln("[recv] invalid data len", len(data))
+		}
 		return nil
 	}
 	bound := 64
@@ -25,56 +30,106 @@ func (m *Me) wait(data []byte) *head.Packet {
 		bound = len(data)
 		endl = "."
 	}
-	logrus.Debugln("[recv] data bytes", hex.EncodeToString(data[:bound]), endl)
+	if config.ShowDebugLog {
-	data = m.xordec(data)
+		logrus.Debugln("[recv] data bytes, len", len(data), "val", hex.EncodeToString(data[:bound]), endl)
-	logrus.Debugln("[recv] data xored", hex.EncodeToString(data[:bound]), endl)
+	}
+	if m.base14 {
+		data = base14.Decode(data)
+		if len(data) < bound {
+			bound = len(data)
+			endl = "."
+		}
+		if config.ShowDebugLog {
+			logrus.Debugln("[recv] data b14ed, len", len(data), "val", hex.EncodeToString(data[:bound]), endl)
+		}
+		if len(data) < head.PacketHeadLen { // not a valid packet
+			if config.ShowDebugLog {
+				logrus.Debugln("[recv] invalid data len", len(data))
+			}
+			return nil
+		}
+	}
+	seq, data := m.xordec(data)
+	if len(data) < bound {
+		bound = len(data)
+		endl = "."
+	}
+	if config.ShowDebugLog {
+		logrus.Debugln("[recv] data xored, len", len(data), "val", hex.EncodeToString(data[:bound]), endl)
+	}
+	if len(data) < head.PacketHeadLen { // not a valid packet
+		if config.ShowDebugLog {
+			logrus.Debugln("[recv] invalid data len", len(data))
+		}
+		return nil
+	}
 	flags := head.Flags(data)
 	if !flags.IsValid() {
-		logrus.Debugln("[recv] drop invalid flags packet:", hex.EncodeToString(data[11:12]), hex.EncodeToString(data[10:11]))
+		if config.ShowDebugLog {
+			logrus.Debugln("[recv] drop invalid flags packet:", hex.EncodeToString(data[11:12]), hex.EncodeToString(data[10:11]))
+		}
 		return nil
 	}
-	crc := binary.LittleEndian.Uint64(data[52:60])
+	crc := head.CRC64(data)
-	if m.recved.Get(crc) { // 是重放攻击
+	crclog := crc
-		logrus.Warnln("[recv] ignore duplicated crc packet", strconv.FormatUint(crc, 16))
+	crc ^= (uint64(seq) << 16)
+	if config.ShowDebugLog {
+		logrus.Debugf("[recv] packet crc %016x, seq %08x, xored crc %016x", crclog, seq, crc)
+	}
+	if m.recved.Get(crc) {
+		if config.ShowDebugLog {
+			logrus.Debugln("[recv] ignore duplicated crc packet", strconv.FormatUint(crc, 16))
+		}
 		return nil
 	}
-	logrus.Debugln("[recv]", len(data), "bytes data with flag", hex.EncodeToString(data[11:12]), hex.EncodeToString(data[10:11]))
+	m.recved.Set(crc, true)
+	if config.ShowDebugLog {
+		logrus.Debugln("[recv]", strconv.FormatUint(crc, 16), len(data), "bytes data with flag", hex.EncodeToString(data[11:12]), hex.EncodeToString(data[10:11]))
+	}
 	if flags.IsSingle() || flags.NoFrag() {
 		h := head.SelectPacket()
 		_, err := h.Unmarshal(data)
 		if err != nil {
-			logrus.Errorln("[recv] unmarshal err:", err)
+			logrus.Errorln("[recv]", strconv.FormatUint(crc, 16), "unmarshal err:", err)
 			return nil
 		}
-		m.recved.Set(crc, true)
 		return h
 	}
 
-	hashd := data[20:52]
+	crchash := crc64.New(crc64.MakeTable(crc64.ISO))
-	hsh := *(*[32]byte)(*(*unsafe.Pointer)(unsafe.Pointer(&hashd)))
+	_, _ = crchash.Write(head.Hash(data))
+	var buf [4]byte
+	binary.LittleEndian.PutUint32(buf[:], seq)
+	_, _ = crchash.Write(buf[:])
+	hsh := crchash.Sum64()
 	h := m.recving.Get(hsh)
 	if h != nil {
-		logrus.Debugln("[recv] get another frag part of", hex.EncodeToString(hashd))
+		if config.ShowDebugLog {
+			logrus.Debugln("[recv]", strconv.FormatUint(crc, 16), "get another frag part of", strconv.FormatUint(hsh, 16))
+		}
 		ok, err := h.Unmarshal(data)
 		if err == nil {
 			if ok {
 				m.recving.Delete(hsh)
-				m.recved.Set(crc, true)
+				if config.ShowDebugLog {
-				logrus.Debugln("[recv] all parts of", hex.EncodeToString(hashd), "has reached")
+					logrus.Debugln("[recv]", strconv.FormatUint(crc, 16), "all parts of", strconv.FormatUint(hsh, 16), "has reached")
+				}
 				return h
 			}
 		} else {
 			h.Put()
-			logrus.Errorln("[recv] unmarshal err:", err)
+			logrus.Errorln("[recv]", strconv.FormatUint(crc, 16), "unmarshal err:", err)
 		}
 		return nil
 	}
-	logrus.Debugln("[recv] get new frag part of", hex.EncodeToString(hashd))
+	if config.ShowDebugLog {
+		logrus.Debugln("[recv]", strconv.FormatUint(crc, 16), "get new frag part of", strconv.FormatUint(hsh, 16))
+	}
 	h = head.SelectPacket()
 	_, err := h.Unmarshal(data)
 	if err != nil {
 		h.Put()
-		logrus.Errorln("[recv] unmarshal err:", err)
+		logrus.Errorln("[recv]", strconv.FormatUint(crc, 16), "unmarshal err:", err)
 		return nil
 	}
 	m.recving.Set(hsh, h)

gold/link/router.go

@@ -7,6 +7,8 @@ import (
 
 	"github.com/FloatTech/ttl"
 	"github.com/sirupsen/logrus"
+
+	"github.com/fumiama/WireGold/config"
 )
 
 type Router struct {
@@ -45,7 +47,9 @@ func (r *Router) SetDefault(l *Link) {
 func (r *Router) NextHop(ip string) (l *Link) {
 	l = r.cache.Get(ip)
 	if l != nil {
-		logrus.Debugln("[router] get cached nexthop to", ip, "link", l)
+		if config.ShowDebugLog {
+			logrus.Debugln("[router] get cached nexthop to", ip, "link", l)
+		}
 		return
 	}
 	ipb := net.ParseIP(ip)
@@ -62,7 +66,9 @@ func (r *Router) NextHop(ip string) (l *Link) {
 	for _, c := range r.list {
 		if c.Contains(ipb) {
 			l = r.table[c.String()]
-			logrus.Debugln("[router] get nexthop to", ipb, "-->", c, "link", l)
+			if config.ShowDebugLog {
+				logrus.Debugln("[router] get nexthop to", ipb, "-->", c, "link", l)
+			}
 			r.cache.Set(ip, l)
 			return l
 		}

gold/link/send.go

@@ -2,16 +2,26 @@ package link
 
 import (
 	"bytes"
+	crand "crypto/rand"
+	"encoding/binary"
 	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
 	"math/rand"
 
-	"github.com/fumiama/WireGold/gold/head"
-	"github.com/fumiama/WireGold/helper"
 	"github.com/klauspost/compress/zstd"
 	"github.com/sirupsen/logrus"
+
+	"github.com/fumiama/WireGold/config"
+	"github.com/fumiama/WireGold/gold/head"
+	"github.com/fumiama/WireGold/helper"
+	base14 "github.com/fumiama/go-base16384"
+)
+
+var (
+	ErrDropBigDontFragPkt = errors.New("drop big don't fragmnet packet")
+	ErrTTL                = errors.New("ttl exceeded")
 )
 
 // WriteAndPut 向 peer 发包并将包放回缓存池
@@ -19,47 +29,58 @@ func (l *Link) WriteAndPut(p *head.Packet, istransfer bool) (n int, err error) {
 	defer p.Put()
 	teatype := l.randkeyidx()
 	sndcnt := uint16(l.incgetsndcnt())
+	var buf [4]byte
+	_, _ = crand.Read(buf[:2])
+	binary.BigEndian.PutUint16(buf[2:4], sndcnt)
+	seq := binary.BigEndian.Uint32(buf[:])
 	mtu := l.mtu
 	if l.mturandomrange > 0 {
 		mtu -= uint16(rand.Intn(int(l.mturandomrange)))
 	}
-	logrus.Debugln("[send] mtu:", mtu, ", addt:", sndcnt&0x07ff, ", key index:", teatype)
+	if config.ShowDebugLog {
+		logrus.Debugln("[send] mtu:", mtu, ", addt:", sndcnt&0x07ff, ", key index:", teatype)
+	}
 	if !istransfer {
 		l.encrypt(p, sndcnt, teatype)
 	}
-	delta := (int(mtu) - 60) & 0x0000fff8
+	delta := (int(mtu) - head.PacketHeadLen) & 0x0000fff8
 	if delta <= 0 {
 		logrus.Warnln("[send] reset invalid data frag len", delta, "to 8")
 		delta = 8
 	}
-	if len(p.Data) <= delta {
+	remlen := p.BodyLen()
-		return l.write(p, teatype, sndcnt, uint32(len(p.Data)), 0, istransfer, false)
+	if remlen <= delta {
+		return l.write(p, teatype, sndcnt, uint32(remlen), 0, istransfer, false, seq)
 	}
-	if istransfer && p.Flags.DontFrag() && len(p.Data) > delta {
+	if istransfer && p.Flags.DontFrag() && remlen > delta {
-		return 0, errors.New("drop don't fragmnet big trans packet")
+		return 0, ErrDropBigDontFragPkt
 	}
-	data := p.Data
 	ttl := p.TTL
-	totl := uint32(len(data))
+	totl := uint32(remlen)
 	pos := 0
-	packet := head.SelectPacket()
+	packet := p.Copy()
-	*packet = *p
+	for remlen > delta {
-	for ; int(totl)-pos > delta; pos += delta {
+		remlen -= delta
-		logrus.Debugln("[send] split frag [", pos, "~", pos+delta, "], remain:", int(totl)-pos-delta)
+		if config.ShowDebugLog {
-		packet.Data = data[:delta]
+			logrus.Debugln("[send] split frag [", pos, "~", pos+delta, "], remain:", remlen)
-		cnt, err := l.write(packet, teatype, sndcnt, totl, uint16(pos>>3), istransfer, true)
+		}
+		packet.CropBody(pos, pos+delta)
+		cnt, err := l.write(packet, teatype, sndcnt, totl, uint16(pos>>3), istransfer, true, seq)
 		n += cnt
 		if err != nil {
 			return n, err
 		}
-		data = data[delta:]
 		packet.TTL = ttl
+		pos += delta
 	}
 	packet.Put()
-	if len(data) > 0 {
+	if remlen > 0 {
-		p.Data = data
+		if config.ShowDebugLog {
+			logrus.Debugln("[send] last frag [", pos, "~", pos+remlen, "]")
+		}
+		p.CropBody(pos, pos+remlen)
 		cnt := 0
-		cnt, err = l.write(p, teatype, sndcnt, totl, uint16(pos>>3), istransfer, false)
+		cnt, err = l.write(p, teatype, sndcnt, totl, uint16(pos>>3), istransfer, false, seq)
 		n += cnt
 	}
 	return n, err
@@ -67,47 +88,77 @@ func (l *Link) WriteAndPut(p *head.Packet, istransfer bool) (n int, err error) {
 
 func (l *Link) encrypt(p *head.Packet, sndcnt uint16, teatype uint8) {
 	p.FillHash()
-	logrus.Debugln("[send] data len before encrypt:", len(p.Data))
+	if config.ShowDebugLog {
+		logrus.Debugln("[send] data len before encrypt:", p.BodyLen())
+	}
+	data := p.Body()
 	if l.usezstd {
 		w := helper.SelectWriter()
 		defer helper.PutWriter(w)
 		enc, _ := zstd.NewWriter(w, zstd.WithEncoderLevel(zstd.SpeedFastest))
-		_, _ = io.Copy(enc, bytes.NewReader(p.Data))
+		_, _ = io.Copy(enc, bytes.NewReader(data))
 		enc.Close()
-		p.Data = w.Bytes()
+		data = w.Bytes()
-		logrus.Debugln("[send] data len after zstd:", len(p.Data))
+		if config.ShowDebugLog {
+			logrus.Debugln("[send] data len after zstd:", len(data))
+		}
 	}
-	p.Data = l.Encode(teatype, sndcnt&0x07ff, p.Data)
+	p.SetBody(l.Encode(teatype, sndcnt&0x07ff, data), true)
-	logrus.Debugln("[send] data len after xchacha20:", len(p.Data), "addt:", sndcnt)
+	if config.ShowDebugLog {
+		logrus.Debugln("[send] data len after xchacha20:", p.BodyLen(), "addt:", sndcnt)
+	}
+}
+
+// write 向 peer 发包
+func (l *Link) write(p *head.Packet, teatype uint8, additional uint16, datasz uint32, offset uint16, istransfer, hasmore bool, seq uint32) (int, error) {
+	if p.DecreaseAndGetTTL() <= 0 {
+		return 0, ErrTTL
+	}
+	if l.doublepacket {
+		_, _ = l.writeonce(p, teatype, additional, datasz, offset, istransfer, hasmore, seq)
+	}
+	return l.writeonce(p, teatype, additional, datasz, offset, istransfer, hasmore, seq)
 }
 
 // write 向 peer 发一个包
-func (l *Link) write(p *head.Packet, teatype uint8, additional uint16, datasz uint32, offset uint16, istransfer, hasmore bool) (n int, err error) {
+func (l *Link) writeonce(p *head.Packet, teatype uint8, additional uint16, datasz uint32, offset uint16, istransfer, hasmore bool, seq uint32) (int, error) {
+	peerep := l.endpoint
+	if helper.IsNilInterface(peerep) {
+		return 0, errors.New("nil endpoint of " + p.Dst.String())
+	}
+
 	var d []byte
 	var cl func()
+	// TODO: now all packet allow frag, adapt to DF
 	if istransfer {
-		d, cl = p.Marshal(nil, teatype, additional, 0, 0, false, false)
+		d, cl = p.Marshal(nil, 0, 0, 0, offset, false, hasmore)
 	} else {
 		d, cl = p.Marshal(l.me.me, teatype, additional, datasz, offset, false, hasmore)
 	}
-	if d == nil {
+	defer cl()
-		return 0, errors.New("[send] ttl exceeded")
+
-	}
-	peerep := l.endpoint
-	if peerep == nil {
-		return 0, errors.New("[send] nil endpoint of " + p.Dst.String())
-	}
 	bound := 64
 	endl := "..."
 	if len(d) < bound {
 		bound = len(d)
 		endl = "."
 	}
-	logrus.Debugln("[send] write", len(d), "bytes data from ep", l.me.udpconn.LocalAddr(), "to", peerep, "offset:", fmt.Sprintf("%04x", offset))
+	conn := l.me.conn
-	logrus.Debugln("[send] data bytes", hex.EncodeToString(d[:bound]), endl)
+	if conn == nil {
-	d = l.me.xorenc(d)
+		return 0, io.ErrClosedPipe
-	logrus.Debugln("[send] data xored", hex.EncodeToString(d[:bound]), endl)
+	}
-	n, err = l.me.udpconn.WriteToUDP(d, peerep)
+	if config.ShowDebugLog {
-	cl()
+
-	return
+		logrus.Debugln("[send] write", len(d), "bytes data from ep", conn.LocalAddr(), "to", peerep, "offset", fmt.Sprintf("%04x", offset), "crc", fmt.Sprintf("%016x", p.CRC64()))
+		logrus.Debugln("[send] data bytes", hex.EncodeToString(d[:bound]), endl)
+	}
+	d = l.me.xorenc(d, seq)
+	if l.me.base14 {
+		d = base14.Encode(d)
+	}
+	if config.ShowDebugLog {
+		logrus.Debugln("[send] data xored", hex.EncodeToString(d[:bound]), endl)
+	}
+	defer helper.PutBytes(d)
+	return conn.WriteToPeer(d, peerep)
 }

gold/p2p/define.go

@@ -0,0 +1,44 @@
+package p2p
+
+import (
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/RomiChan/syncx"
+)
+
+var (
+	ErrEndpointTypeMistatch = errors.New("endpoint type mismatch")
+)
+
+type Initializer func(endpoint string, configs ...any) (EndPoint, error)
+
+var factory syncx.Map[string, Initializer]
+
+func Register(network string, initializer Initializer) (actual Initializer, hasexist bool) {
+	return factory.LoadOrStore(network, initializer)
+}
+
+type EndPoint interface {
+	fmt.Stringer
+	Network() string
+	Euqal(EndPoint) bool
+	Listen() (Conn, error)
+}
+
+func NewEndPoint(network, endpoint string, configs ...any) (EndPoint, error) {
+	initializer, ok := factory.Load(network)
+	if !ok {
+		return nil, errors.New("network " + network + " not found")
+	}
+	return initializer(endpoint, configs...)
+}
+
+type Conn interface {
+	io.Closer
+	fmt.Stringer // basically, the local address string
+	LocalAddr() EndPoint
+	ReadFromPeer([]byte) (int, EndPoint, error)
+	WriteToPeer([]byte, EndPoint) (int, error)
+}

gold/p2p/ip/init.go

@@ -0,0 +1,35 @@
+package ip
+
+import (
+	"net"
+	"net/netip"
+
+	"github.com/fumiama/WireGold/gold/p2p"
+	"github.com/fumiama/WireGold/helper"
+)
+
+func NewEndpoint(endpoint string, configs ...any) (p2p.EndPoint, error) {
+	addr, err := netip.ParseAddr(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	ptcl := uint(0x6C) // IPComp https://datatracker.ietf.org/doc/html/rfc3173
+	if len(configs) > 0 {
+		ptcl = configs[0].(uint)
+	}
+	return &EndPoint{
+		addr: &net.IPAddr{
+			IP:   addr.AsSlice(),
+			Zone: addr.Zone(),
+		},
+		ptcl: ptcl,
+	}, nil
+}
+
+func init() {
+	name := helper.FolderName()
+	_, hasexist := p2p.Register(name, NewEndpoint)
+	if hasexist {
+		panic("network " + name + " has been registered")
+	}
+}

gold/p2p/ip/ip.go

@@ -0,0 +1,79 @@
+package ip
+
+import (
+	"net"
+	"strconv"
+
+	"github.com/fumiama/WireGold/gold/p2p"
+)
+
+type EndPoint struct {
+	addr *net.IPAddr
+	ptcl uint
+}
+
+func (ep *EndPoint) String() string {
+	return ep.addr.String()
+}
+
+func (ep *EndPoint) Network() string {
+	return ep.addr.Network()
+}
+
+func (ep *EndPoint) Euqal(ep2 p2p.EndPoint) bool {
+	if ep == nil || ep2 == nil {
+		return ep == nil && ep2 == nil
+	}
+	ipep2, ok := ep2.(*EndPoint)
+	if !ok {
+		return false
+	}
+	ipep1 := ep
+	return ipep1.addr.IP.Equal(ipep2.addr.IP) &&
+		ipep1.addr.Zone == ipep2.addr.Zone
+}
+
+func (ep *EndPoint) Listen() (p2p.Conn, error) {
+	conn, err := net.ListenIP(
+		"ip:"+strconv.Itoa(int(ep.ptcl)),
+		ep.addr,
+	)
+	return &Conn{
+		ep:   ep,
+		conn: conn,
+	}, err
+}
+
+type Conn struct {
+	ep   *EndPoint
+	conn *net.IPConn
+}
+
+func (conn *Conn) Close() error {
+	return conn.conn.Close()
+}
+
+func (conn *Conn) String() string {
+	return conn.conn.LocalAddr().String()
+}
+
+func (conn *Conn) LocalAddr() p2p.EndPoint {
+	ep, _ := NewEndpoint(conn.conn.LocalAddr().String())
+	return ep
+}
+
+func (conn *Conn) ReadFromPeer(b []byte) (int, p2p.EndPoint, error) {
+	n, addr, err := conn.conn.ReadFromIP(b)
+	return n, &EndPoint{
+		addr: addr,
+		ptcl: conn.ep.ptcl,
+	}, err
+}
+
+func (conn *Conn) WriteToPeer(b []byte, ep p2p.EndPoint) (int, error) {
+	ipep, ok := ep.(*EndPoint)
+	if !ok {
+		return 0, p2p.ErrEndpointTypeMistatch
+	}
+	return conn.conn.WriteToIP(b, ipep.addr)
+}

gold/p2p/tcp/init.go

@@ -0,0 +1,49 @@
+package tcp
+
+import (
+	"net"
+	"net/netip"
+	"time"
+
+	"github.com/fumiama/WireGold/gold/p2p"
+	"github.com/fumiama/WireGold/helper"
+)
+
+type Config struct {
+	DialTimeout        time.Duration
+	PeersTimeout       time.Duration
+	KeepInterval       time.Duration
+	ReceiveChannelSize int
+}
+
+func NewEndpoint(endpoint string, configs ...any) (p2p.EndPoint, error) {
+	return newEndpoint(endpoint, configs...)
+}
+
+func newEndpoint(endpoint string, configs ...any) (*EndPoint, error) {
+	var cfg *Config
+	if len(configs) == 0 || configs[0] == nil {
+		cfg = &Config{}
+	} else {
+		cfg = configs[0].(*Config)
+	}
+	addr, err := netip.ParseAddrPort(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	return &EndPoint{
+		addr:         net.TCPAddrFromAddrPort(addr),
+		dialtimeout:  cfg.DialTimeout,
+		peerstimeout: cfg.PeersTimeout,
+		keepinterval: cfg.KeepInterval,
+		recvchansize: cfg.ReceiveChannelSize,
+	}, nil
+}
+
+func init() {
+	name := helper.FolderName()
+	_, hasexist := p2p.Register(name, NewEndpoint)
+	if hasexist {
+		panic("network " + name + " has been registered")
+	}
+}

gold/p2p/tcp/pdu.go

@@ -0,0 +1,136 @@
+package tcp
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+	"net"
+	"time"
+
+	"github.com/fumiama/WireGold/config"
+	"github.com/fumiama/WireGold/helper"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	ErrInvalidMagic = errors.New("invalid magic")
+)
+
+type packetType uint8
+
+const (
+	packetTypeKeepAlive packetType = iota
+	packetTypeNormal
+	packetTypeSubKeepAlive
+	packetTypeTop
+)
+
+var (
+	magicbuf = []byte("GET ")
+	magic    = binary.LittleEndian.Uint32(magicbuf)
+)
+
+type packet struct {
+	typ packetType
+	len uint16
+	dat []byte
+	io.ReaderFrom
+	io.WriterTo
+}
+
+func (p *packet) pack() (net.Buffers, func()) {
+	d, cl := helper.OpenWriterF(func(w *helper.Writer) {
+		w.WriteByte(byte(p.typ))
+		w.WriteUInt16(p.len)
+	})
+	return net.Buffers{magicbuf, d, p.dat}, cl
+}
+
+func (p *packet) Read(_ []byte) (int, error) {
+	panic("stub")
+}
+
+func (p *packet) Write(_ []byte) (int, error) {
+	panic("stub")
+}
+
+func (p *packet) ReadFrom(r io.Reader) (n int64, err error) {
+	var buf [4]byte
+	cnt, err := io.ReadFull(r, buf[:])
+	n = int64(cnt)
+	if err != nil {
+		return
+	}
+	if binary.LittleEndian.Uint32(buf[:]) != magic {
+		err = ErrInvalidMagic
+		if config.ShowDebugLog {
+			logrus.Debugf("[tcp] expect magic %08x but got %08x", magic, binary.LittleEndian.Uint32(buf[:]))
+		}
+		return
+	}
+	cnt, err = io.ReadFull(r, buf[:3])
+	n += int64(cnt)
+	if err != nil {
+		return
+	}
+	p.typ = packetType(buf[0])
+	p.len = binary.LittleEndian.Uint16(buf[1:3])
+	w := helper.SelectWriter()
+	copied, err := io.CopyN(w, r, int64(p.len))
+	n += copied
+	if err != nil {
+		return
+	}
+	p.dat = w.Bytes()
+	return
+}
+
+func (p *packet) WriteTo(w io.Writer) (n int64, err error) {
+	buf, cl := p.pack()
+	defer cl()
+	return io.Copy(w, &buf)
+}
+
+func isvalid(tcpconn *net.TCPConn, timeout time.Duration) (issub, ok bool) {
+	pckt := packet{}
+
+	stopch := make(chan struct{})
+	t := time.AfterFunc(timeout, func() {
+		stopch <- struct{}{}
+	})
+
+	var err error
+	copych := make(chan struct{})
+	go func() {
+		_, err = io.Copy(&pckt, tcpconn)
+		copych <- struct{}{}
+	}()
+
+	select {
+	case <-stopch:
+		if config.ShowDebugLog {
+			logrus.Debugln("[tcp] validate recv from", tcpconn.RemoteAddr(), "timeout")
+		}
+		return
+	case <-copych:
+		t.Stop()
+	}
+
+	if err != nil {
+		if config.ShowDebugLog {
+			logrus.Debugln("[tcp] validate recv from", tcpconn.RemoteAddr(), "err:", err)
+		}
+		return
+	}
+	if pckt.typ != packetTypeKeepAlive && pckt.typ != packetTypeSubKeepAlive {
+		if config.ShowDebugLog {
+			logrus.Debugln("[tcp] validate got invalid typ", pckt.typ, "from", tcpconn.RemoteAddr())
+		}
+		return
+	}
+
+	if config.ShowDebugLog {
+		logrus.Debugln("[tcp] passed validate recv from", tcpconn.RemoteAddr())
+	}
+	return pckt.typ == packetTypeSubKeepAlive, true
+}

gold/p2p/tcp/tcp.go

@@ -0,0 +1,510 @@
+package tcp
+
+import (
+	"errors"
+	"io"
+	"net"
+	"reflect"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/FloatTech/ttl"
+	"github.com/sirupsen/logrus"
+
+	"github.com/fumiama/WireGold/config"
+	"github.com/fumiama/WireGold/gold/p2p"
+	"github.com/fumiama/WireGold/helper"
+)
+
+type EndPoint struct {
+	addr         *net.TCPAddr
+	dialtimeout  time.Duration
+	peerstimeout time.Duration
+	keepinterval time.Duration
+	recvchansize int
+}
+
+func (ep *EndPoint) String() string {
+	return ep.addr.String()
+}
+
+func (ep *EndPoint) Network() string {
+	return ep.addr.Network()
+}
+
+func (ep *EndPoint) Euqal(ep2 p2p.EndPoint) bool {
+	if ep == nil || ep2 == nil {
+		return ep == nil && ep2 == nil
+	}
+	tcpep2, ok := ep2.(*EndPoint)
+	if !ok {
+		return false
+	}
+	tcpep1 := ep
+	return tcpep1.addr.IP.Equal(tcpep2.addr.IP) &&
+		tcpep1.addr.Port == tcpep2.addr.Port &&
+		tcpep1.addr.Zone == tcpep2.addr.Zone
+}
+
+func (ep *EndPoint) Listen() (p2p.Conn, error) {
+	lstn, err := net.ListenTCP(ep.addr.Network(), ep.addr)
+	if err != nil {
+		return nil, err
+	}
+	ep.addr = lstn.Addr().(*net.TCPAddr)
+	peerstimeout := ep.peerstimeout
+	if peerstimeout < time.Second*30 {
+		peerstimeout = time.Second * 30
+	}
+	chansz := ep.recvchansize
+	if chansz < 32 {
+		chansz = 32
+	}
+	conn := &Conn{
+		addr: ep,
+		lstn: lstn,
+		peers: ttl.NewCacheOn(peerstimeout, [4]func(string, *net.TCPConn){
+			func(_ string, t *net.TCPConn) {
+				_ = t.SetLinger(0)
+				_ = t.SetNoDelay(true)
+			}, nil, func(_ string, t *net.TCPConn) {
+				err := t.CloseWrite()
+				if config.ShowDebugLog {
+					if err != nil {
+						logrus.Debugln("[tcp] close write from", t.LocalAddr(), "to", t.RemoteAddr(), "err:", err)
+					} else {
+						logrus.Debugln("[tcp] close write from", t.LocalAddr(), "to", t.RemoteAddr())
+					}
+				}
+			}, nil,
+		}),
+		recv: make(chan *connrecv, chansz),
+		cplk: &sync.Mutex{},
+		sblk: &sync.RWMutex{},
+	}
+	go conn.accept()
+	return conn, nil
+}
+
+type connrecv struct {
+	addr *EndPoint // cast from tcpconn.RemoteAddr()
+	conn *net.TCPConn
+	pckt packet
+}
+
+type subconn struct {
+	cplk sync.Mutex
+	last time.Time // last active time
+	conn *net.TCPConn
+}
+
+// Conn 伪装成无状态的有状态连接
+type Conn struct {
+	addr   *EndPoint
+	lstn   *net.TCPListener
+	peers  *ttl.Cache[string, *net.TCPConn]
+	recv   chan *connrecv
+	cplk   *sync.Mutex
+	sblk   *sync.RWMutex
+	subs   []*subconn
+	suberr bool
+}
+
+func (conn *Conn) accept() {
+	for {
+		tcpconn, err := conn.lstn.AcceptTCP()
+		if err != nil {
+			if errors.Is(err, net.ErrClosed) { // normal close
+				logrus.Infoln("[tcp] accept of", conn.addr, "got closed")
+				return
+			}
+			if conn.addr == nil || conn.lstn == nil || conn.peers == nil || conn.recv == nil {
+				return
+			}
+			logrus.Warnln("[tcp] accept on", conn.addr, "err:", err)
+			_ = conn.Close()
+			newc, err := conn.addr.Listen()
+			if err != nil {
+				logrus.Warnln("[tcp] re-listen on", conn.addr, "err:", err)
+				return
+			}
+			*conn = *newc.(*Conn)
+			logrus.Infoln("[tcp] re-listen on", conn.addr)
+			continue
+		}
+		go conn.receive(tcpconn, false)
+	}
+}
+
+func delsubs(i int, subs []*subconn) []*subconn {
+	tcpconn := subs[i].conn
+	err := tcpconn.CloseWrite()
+	if config.ShowDebugLog {
+		if err != nil {
+			logrus.Debugln("[tcp] close sub write from", tcpconn.LocalAddr(), "to", tcpconn.RemoteAddr(), "err:", err)
+		} else {
+			logrus.Debugln("[tcp] close sub write from", tcpconn.LocalAddr(), "to", tcpconn.RemoteAddr())
+		}
+	}
+	switch i {
+	case 0:
+		subs = subs[1:]
+	case len(subs) - 1:
+		subs = subs[:len(subs)-1]
+	default:
+		subs = append(subs[:i], subs[i+1:]...)
+	}
+	return subs
+}
+
+func (conn *Conn) receive(tcpconn *net.TCPConn, hasvalidated bool) {
+	if conn.peers == nil {
+		return
+	}
+
+	ep, _ := newEndpoint(tcpconn.RemoteAddr().String(), &Config{
+		DialTimeout:        conn.addr.dialtimeout,
+		PeersTimeout:       conn.addr.peerstimeout,
+		KeepInterval:       conn.addr.keepinterval,
+		ReceiveChannelSize: conn.addr.recvchansize,
+	})
+
+	issub, ok := false, false
+
+	peerstimeout := conn.addr.peerstimeout
+	if peerstimeout < time.Second*30 {
+		peerstimeout = time.Second * 30
+	}
+	peerstimeout *= 2
+
+	if !hasvalidated {
+		issub, ok = isvalid(tcpconn, peerstimeout)
+		if !ok {
+			return
+		}
+		if config.ShowDebugLog {
+			logrus.Debugln("[tcp] accept from", ep, "issub:", issub)
+		}
+		if issub {
+			conn.sblk.Lock()
+			conn.subs = append(conn.subs, &subconn{conn: tcpconn, last: time.Now()})
+			conn.sblk.Unlock()
+		} else {
+			if conn.peers == nil {
+				return
+			}
+			conn.peers.Set(ep.String(), tcpconn)
+		}
+	}
+
+	if issub {
+		defer func() {
+			conn.sblk.Lock()
+			subs := conn.subs
+			for i, sub := range subs {
+				if sub.conn == tcpconn {
+					conn.subs = delsubs(i, conn.subs)
+					break
+				}
+			}
+			conn.sblk.Unlock()
+		}()
+	} else {
+		if conn.peers == nil {
+			return
+		}
+		defer conn.peers.Delete(ep.String())
+	}
+
+	go conn.keep(ep)
+
+	for {
+		r := &connrecv{addr: ep}
+		if conn.addr == nil || conn.lstn == nil || conn.peers == nil || conn.recv == nil {
+			return
+		}
+		if !issub {
+			tcpconn = conn.peers.Get(ep.String())
+			if tcpconn == nil {
+				return
+			}
+		}
+		r.conn = tcpconn
+
+		t := time.NewTimer(peerstimeout)
+
+		var err error
+		copych := make(chan struct{})
+		go func() {
+			_, err = io.Copy(&r.pckt, tcpconn)
+			copych <- struct{}{}
+		}()
+
+		select {
+		case <-t.C:
+			if config.ShowDebugLog {
+				logrus.Debugln("[tcp] recv from", ep, "timeout")
+			}
+			_ = tcpconn.CloseRead()
+			return
+		case <-copych:
+			t.Stop()
+		}
+
+		if conn.addr == nil || conn.lstn == nil || conn.peers == nil || conn.recv == nil {
+			return
+		}
+
+		if err != nil {
+			if config.ShowDebugLog {
+				logrus.Debugln("[tcp] recv from", ep, "err:", err)
+			}
+			if errors.Is(err, net.ErrClosed) ||
+				errors.Is(err, io.ErrClosedPipe) ||
+				errors.Is(err, io.EOF) ||
+				errors.Is(err, ErrInvalidMagic) {
+				_ = tcpconn.CloseRead()
+				return
+			}
+			continue
+		}
+		if r.pckt.typ >= packetTypeTop {
+			if config.ShowDebugLog {
+				logrus.Debugln("[tcp] close reading invalid conn from", ep, "typ", r.pckt.typ, "len", r.pckt.len)
+			}
+			_ = tcpconn.CloseRead()
+			return
+		}
+		if config.ShowDebugLog {
+			logrus.Debugln("[tcp] dispatch packet from", ep, "typ", r.pckt.typ, "len", r.pckt.len)
+		}
+		conn.recv <- r
+	}
+}
+
+func (conn *Conn) keep(ep *EndPoint) {
+	keepinterval := ep.keepinterval
+	if keepinterval < time.Second*10 {
+		keepinterval = time.Second * 10
+	}
+	t := time.NewTicker(keepinterval)
+	defer t.Stop()
+	for range t.C {
+		if conn.addr == nil || conn.peers == nil {
+			return
+		}
+		tcpconn := conn.peers.Get(ep.String())
+		if tcpconn != nil {
+			_, err := io.Copy(tcpconn, &packet{typ: packetTypeKeepAlive})
+			if conn.addr == nil {
+				return
+			}
+			if err != nil {
+				logrus.Warnln("[tcp] keep main conn alive from", conn, "to", ep, "err:", err)
+				conn.peers.Delete(ep.String())
+			} else if config.ShowDebugLog {
+				logrus.Debugln("[tcp] keep main conn alive from", conn, "to", ep)
+			}
+		}
+		conn.sblk.RLock()
+		subs := conn.subs
+		for i, sub := range subs {
+			if time.Since(sub.last) < keepinterval {
+				if config.ShowDebugLog {
+					logrus.Debugln("[tcp] skip to keep busy sub conn from", conn, "to", ep)
+				}
+				continue
+			}
+			_, err := io.Copy(sub.conn, &packet{typ: packetTypeSubKeepAlive})
+			if conn.addr == nil {
+				return
+			}
+			if err != nil {
+				logrus.Warnln("[tcp] keep sub conn alive from", conn, "to", sub.conn.RemoteAddr(), "err:", err)
+				conn.subs = delsubs(i, conn.subs) // del 1 link at once
+				break
+			}
+			if config.ShowDebugLog {
+				logrus.Debugln("[tcp] keep sub conn alive from", conn, "to", ep)
+			}
+		}
+		conn.sblk.RUnlock()
+	}
+}
+
+func (conn *Conn) Close() error {
+	lstn := conn.lstn
+	peers := conn.peers
+	recv := conn.recv
+	conn.addr = nil
+	conn.lstn = nil
+	conn.peers = nil
+	conn.recv = nil
+
+	if lstn != nil {
+		_ = lstn.Close()
+	}
+	if peers != nil {
+		peers.Destroy()
+	}
+	if recv != nil {
+		close(recv)
+	}
+
+	return nil
+}
+
+func (conn *Conn) String() string {
+	return conn.addr.String()
+}
+
+func (conn *Conn) LocalAddr() p2p.EndPoint {
+	return conn.addr
+}
+
+func (conn *Conn) ReadFromPeer(b []byte) (int, p2p.EndPoint, error) {
+	var p *connrecv
+	for {
+		p = <-conn.recv
+		if p == nil {
+			return 0, nil, net.ErrClosed
+		}
+		if conn.peers == nil {
+			return 0, nil, net.ErrClosed
+		}
+		conn.peers.Set(p.addr.String(), p.conn)
+		if p.pckt.typ == packetTypeNormal {
+			break
+		}
+		defer helper.PutBytes(p.pckt.dat)
+	}
+	n := copy(b, p.pckt.dat)
+	return n, p.addr, nil
+}
+
+// writeToPeer after acquiring lock
+func (conn *Conn) writeToPeer(b []byte, tcpep *EndPoint, issub bool) (n int, err error) {
+	retried := false
+	ok := false
+	var (
+		tcpconn *net.TCPConn
+		subc    *subconn
+	)
+RECONNECT:
+	if issub {
+		conn.sblk.RLock()
+		for _, sub := range conn.subs {
+			if sub.cplk.TryLock() {
+				tcpconn = sub.conn
+				subc = sub
+				break
+			}
+		}
+		conn.sblk.RUnlock()
+	} else {
+		tcpconn = conn.peers.Get(tcpep.String())
+	}
+	if tcpconn == nil {
+		dialtimeout := tcpep.dialtimeout
+		if dialtimeout < time.Second {
+			dialtimeout = time.Second
+		}
+		if config.ShowDebugLog {
+			logrus.Debugln("[tcp] dial to", tcpep.addr, "timeout", dialtimeout, "issub", issub)
+		}
+		var cn net.Conn
+		// must use another port to send because there's no exsiting conn
+		cn, err = net.DialTimeout(tcpep.Network(), tcpep.addr.String(), dialtimeout)
+		if err != nil {
+			return
+		}
+		tcpconn, ok = cn.(*net.TCPConn)
+		if !ok {
+			return 0, errors.New("expect *net.TCPConn but got " + reflect.ValueOf(cn).Type().String())
+		}
+		pkt := &packet{}
+		if issub {
+			pkt.typ = packetTypeSubKeepAlive
+		} else {
+			pkt.typ = packetTypeKeepAlive
+		}
+		_, err = io.Copy(tcpconn, pkt)
+		if err != nil {
+			if config.ShowDebugLog {
+				logrus.Debugln("[tcp] dial to", tcpep.addr, "issub", issub, "success, but write err:", err)
+			}
+			return 0, err
+		}
+		if config.ShowDebugLog {
+			logrus.Debugln("[tcp] dial to", tcpep.addr, "success, local:", tcpconn.LocalAddr(), "issub", issub)
+		}
+		if !issub {
+			conn.peers.Set(tcpep.String(), tcpconn)
+		} else {
+			conn.sblk.Lock()
+			conn.subs = append(conn.subs, &subconn{conn: tcpconn, last: time.Now()})
+			conn.sblk.Unlock()
+		}
+		go conn.receive(tcpconn, true)
+	} else if config.ShowDebugLog {
+		logrus.Debugln("[tcp] reuse tcpconn from", tcpconn.LocalAddr(), "to", tcpconn.RemoteAddr())
+	}
+	cnt, err := io.Copy(tcpconn, &packet{
+		typ: packetTypeNormal,
+		len: uint16(len(b)),
+		dat: b,
+	})
+	if err != nil {
+		if subc == nil {
+			conn.peers.Delete(tcpep.String())
+		} else {
+			conn.sblk.Lock()
+			subs := conn.subs
+			for i, sub := range subs {
+				if sub == subc {
+					conn.subs = delsubs(i, conn.subs)
+					break
+				}
+			}
+			conn.sblk.Unlock()
+		}
+		if !retried {
+			logrus.Warnln("[tcp] reconnect due to write to", tcpconn.RemoteAddr(), "err:", err)
+			retried = true
+			tcpconn = nil
+			goto RECONNECT
+		}
+	}
+	if subc != nil {
+		subc.last = time.Now()
+		subc.cplk.Unlock()
+	}
+	return int(cnt) - 3, err
+}
+
+func (conn *Conn) WriteToPeer(b []byte, ep p2p.EndPoint) (n int, err error) {
+	tcpep, ok := ep.(*EndPoint)
+	if !ok {
+		return 0, p2p.ErrEndpointTypeMistatch
+	}
+	if len(b) >= 65536 {
+		return 0, errors.New("data size " + strconv.Itoa(len(b)) + " is too large")
+	}
+	locked := conn.cplk.TryLock()
+	if !locked {
+		if !conn.suberr || len(conn.subs) > 0 {
+			if config.ShowDebugLog {
+				logrus.Debug("[tcp] try sub write to", tcpep)
+			}
+			n, err = conn.writeToPeer(b, tcpep, true) // try sub write
+			if err == nil {
+				return
+			}
+			conn.suberr = true // fast fail
+		}
+		conn.cplk.Lock() // add to main queue
+	}
+	defer conn.cplk.Unlock()
+	return conn.writeToPeer(b, tcpep, false)
+}

gold/p2p/udp/init.go

@@ -0,0 +1,25 @@
+package udp
+
+import (
+	"net"
+	"net/netip"
+
+	"github.com/fumiama/WireGold/gold/p2p"
+	"github.com/fumiama/WireGold/helper"
+)
+
+func NewEndpoint(endpoint string, _ ...any) (p2p.EndPoint, error) {
+	addr, err := netip.ParseAddrPort(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	return (*EndPoint)(net.UDPAddrFromAddrPort(addr)), nil
+}
+
+func init() {
+	name := helper.FolderName()
+	_, hasexist := p2p.Register(name, NewEndpoint)
+	if hasexist {
+		panic("network " + name + " has been registered")
+	}
+}

gold/p2p/udp/udp.go

@@ -0,0 +1,62 @@
+package udp
+
+import (
+	"net"
+
+	"github.com/fumiama/WireGold/gold/p2p"
+)
+
+type EndPoint net.UDPAddr
+
+func (ep *EndPoint) String() string {
+	return (*net.UDPAddr)(ep).String()
+}
+
+func (ep *EndPoint) Network() string {
+	return (*net.UDPAddr)(ep).Network()
+}
+
+func (ep *EndPoint) Euqal(ep2 p2p.EndPoint) bool {
+	if ep == nil || ep2 == nil {
+		return ep == nil && ep2 == nil
+	}
+	udpep2, ok := ep2.(*EndPoint)
+	if !ok {
+		return false
+	}
+	udpep1 := ep
+	return udpep1.IP.Equal(udpep2.IP) && udpep1.Port == udpep2.Port && udpep1.Zone == udpep2.Zone
+}
+
+func (ep *EndPoint) Listen() (p2p.Conn, error) {
+	conn, err := net.ListenUDP((*net.UDPAddr)(ep).Network(), (*net.UDPAddr)(ep))
+	return (*Conn)(conn), err
+}
+
+type Conn net.UDPConn
+
+func (conn *Conn) Close() error {
+	return (*net.UDPConn)(conn).Close()
+}
+
+func (conn *Conn) String() string {
+	return (*net.UDPConn)(conn).LocalAddr().String()
+}
+
+func (conn *Conn) LocalAddr() p2p.EndPoint {
+	ep, _ := NewEndpoint((*net.UDPConn)(conn).LocalAddr().String())
+	return ep
+}
+
+func (conn *Conn) ReadFromPeer(b []byte) (int, p2p.EndPoint, error) {
+	n, addr, err := (*net.UDPConn)(conn).ReadFromUDP(b)
+	return n, (*EndPoint)(addr), err
+}
+
+func (conn *Conn) WriteToPeer(b []byte, ep p2p.EndPoint) (int, error) {
+	udpep, ok := ep.(*EndPoint)
+	if !ok {
+		return 0, p2p.ErrEndpointTypeMistatch
+	}
+	return (*net.UDPConn)(conn).WriteTo(b, (*net.UDPAddr)(udpep))
+}

gold/p2p/udplite/init.go

@@ -0,0 +1,27 @@
+//go:build !darwin && !windows
+
+package udplite
+
+import (
+	"net"
+	"net/netip"
+
+	"github.com/fumiama/WireGold/gold/p2p"
+	"github.com/fumiama/WireGold/helper"
+)
+
+func NewEndpoint(endpoint string, _ ...any) (p2p.EndPoint, error) {
+	addr, err := netip.ParseAddrPort(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	return (*EndPoint)(net.UDPAddrFromAddrPort(addr)), nil
+}
+
+func init() {
+	name := helper.FolderName()
+	_, hasexist := p2p.Register(name, NewEndpoint)
+	if hasexist {
+		panic("network " + name + " has been registered")
+	}
+}

gold/p2p/udplite/lite.go

@@ -0,0 +1,93 @@
+//go:build !darwin && !windows
+
+package udplite
+
+import (
+	"context"
+	"net"
+	"syscall"
+	"unsafe"
+
+	"github.com/fumiama/WireGold/gold/head"
+)
+
+// https://www.kernel.org/doc/Documentation/networking/udplite.txt
+const (
+	IPPROTO_UDPLITE    = 136
+	SOL_UDPLITE        = 136
+	UDPLITE_SEND_CSCOV = 10
+	UDPLITE_RECV_CSCOV = 11
+)
+
+type sysListener struct {
+	net.ListenConfig
+	network, address string
+}
+
+type sockaddr interface {
+	net.Addr
+}
+
+//go:linkname toLocal net.(*UDPAddr).toLocal
+func toLocal(a *net.UDPAddr, net string) sockaddr
+
+//go:linkname internetSocket net.internetSocket
+func internetSocket(ctx context.Context, net string, laddr, raddr sockaddr, sotype, proto int, mode string, ctrlCtxFn func(context.Context, string, string, syscall.RawConn) error) (fd unsafe.Pointer, err error)
+
+//go:linkname newUDPConn net.newUDPConn
+func newUDPConn(fd unsafe.Pointer) *net.UDPConn
+
+var sockaddrinterfaceinstance = toLocal(&net.UDPAddr{}, "")
+
+func (sl *sysListener) listenUDP(ctx context.Context, laddr *net.UDPAddr) (*net.UDPConn, error) {
+	var ctrlCtxFn func(cxt context.Context, network, address string, c syscall.RawConn) error
+	if sl.ListenConfig.Control != nil {
+		ctrlCtxFn = func(cxt context.Context, network, address string, c syscall.RawConn) error {
+			return sl.ListenConfig.Control(network, address, c)
+		}
+	}
+	sockladdr := sockaddrinterfaceinstance
+	*(**net.UDPAddr)(unsafe.Add(unsafe.Pointer(&sockladdr), unsafe.Sizeof(uintptr(0)))) = laddr
+	fd, err := internetSocket(ctx, sl.network, sockladdr, nil, syscall.SOCK_DGRAM, IPPROTO_UDPLITE, "listen", ctrlCtxFn)
+	if err != nil {
+		return nil, err
+	}
+	return newUDPConn(fd), nil
+}
+
+func listenUDPLite(network string, laddr *net.UDPAddr) (*net.UDPConn, error) {
+	if laddr == nil {
+		laddr = &net.UDPAddr{}
+	}
+	sl := &sysListener{network: network, address: laddr.String()}
+	conn, err := sl.listenUDP(context.Background(), laddr)
+	if err != nil {
+		var laddrgeneral net.Addr
+		if laddr != nil {
+			laddrgeneral = laddr
+		}
+		return nil, &net.OpError{Op: "listen", Net: network, Source: nil, Addr: laddrgeneral, Err: err}
+	}
+	rc, err := conn.SyscallConn()
+	if err != nil {
+		_ = conn.Close()
+		return nil, err
+	}
+	var errsys error
+	err = rc.Control(func(fd uintptr) {
+		errsys = syscall.SetsockoptInt(int(fd), SOL_UDPLITE, UDPLITE_SEND_CSCOV, head.PacketHeadLen+8) // for xor rand
+		if errsys != nil {
+			return
+		}
+		errsys = syscall.SetsockoptInt(int(fd), SOL_UDPLITE, UDPLITE_RECV_CSCOV, head.PacketHeadLen+8) // for xor rand
+	})
+	if err != nil {
+		_ = conn.Close()
+		return nil, err
+	}
+	if errsys != nil {
+		_ = conn.Close()
+		return nil, errsys
+	}
+	return conn, nil
+}

gold/p2p/udplite/stub.go

@@ -0,0 +1 @@
+package udplite

gold/p2p/udplite/udp.go

@@ -0,0 +1,64 @@
+//go:build !darwin && !windows
+
+package udplite
+
+import (
+	"net"
+
+	"github.com/fumiama/WireGold/gold/p2p"
+)
+
+type EndPoint net.UDPAddr
+
+func (ep *EndPoint) String() string {
+	return (*net.UDPAddr)(ep).String()
+}
+
+func (ep *EndPoint) Network() string {
+	return "udplite"
+}
+
+func (ep *EndPoint) Euqal(ep2 p2p.EndPoint) bool {
+	if ep == nil || ep2 == nil {
+		return ep == nil && ep2 == nil
+	}
+	udpep2, ok := ep2.(*EndPoint)
+	if !ok {
+		return false
+	}
+	udpep1 := ep
+	return udpep1.IP.Equal(udpep2.IP) && udpep1.Port == udpep2.Port && udpep1.Zone == udpep2.Zone
+}
+
+func (ep *EndPoint) Listen() (p2p.Conn, error) {
+	conn, err := listenUDPLite((*net.UDPAddr)(ep).Network(), (*net.UDPAddr)(ep))
+	return (*Conn)(conn), err
+}
+
+type Conn net.UDPConn
+
+func (conn *Conn) Close() error {
+	return (*net.UDPConn)(conn).Close()
+}
+
+func (conn *Conn) String() string {
+	return (*net.UDPConn)(conn).LocalAddr().String()
+}
+
+func (conn *Conn) LocalAddr() p2p.EndPoint {
+	ep, _ := NewEndpoint((*net.UDPConn)(conn).LocalAddr().String())
+	return ep
+}
+
+func (conn *Conn) ReadFromPeer(b []byte) (int, p2p.EndPoint, error) {
+	n, addr, err := (*net.UDPConn)(conn).ReadFromUDP(b)
+	return n, (*EndPoint)(addr), err
+}
+
+func (conn *Conn) WriteToPeer(b []byte, ep p2p.EndPoint) (int, error) {
+	udpep, ok := ep.(*EndPoint)
+	if !ok {
+		return 0, p2p.ErrEndpointTypeMistatch
+	}
+	return (*net.UDPConn)(conn).WriteTo(b, (*net.UDPAddr)(udpep))
+}

helper/data.go

@@ -1,6 +1,7 @@
 package helper
 
 import (
+	"reflect"
 	"unsafe"
 )
 
@@ -30,3 +31,11 @@ func StringToBytes(s string) (b []byte) {
 	bh.cap = sh.len
 	return b
 }
+
+func IsNilInterface(x any) bool {
+	return x == nil || reflect.ValueOf(x).IsZero()
+}
+
+func IsNonNilInterface(x any) bool {
+	return !IsNilInterface(x)
+}

helper/file.go

@@ -1,6 +1,10 @@
 package helper
 
-import "os"
+import (
+	"os"
+	"runtime"
+	"strings"
+)
 
 // IsExist 文件/路径存在
 func IsExist(path string) bool {
@@ -13,3 +17,21 @@ func IsNotExist(path string) bool {
 	_, err := os.Stat(path)
 	return err != nil && os.IsNotExist(err)
 }
+
+// FolderName 本文件所在最下级文件夹名
+func FolderName() string {
+	_, file, _, ok := runtime.Caller(1)
+	if !ok {
+		return "<unk>"
+	}
+	i := strings.LastIndex(file, "/")
+	if i <= 0 {
+		return file
+	}
+	file = file[:i]
+	i = strings.LastIndex(file, "/")
+	if i <= 0 || i+1 >= len(file) {
+		return file
+	}
+	return file[i+1:]
+}

helper/pool.go

@@ -13,6 +13,19 @@ var bufferPool = sync.Pool{
 	},
 }
 
+func MakeBytes(sz int) []byte {
+	w := SelectWriter()
+	b := w.Bytes()
+	if cap(b) >= sz {
+		return b[:sz]
+	}
+	return make([]byte, sz)
+}
+
+func PutBytes(b []byte) {
+	PutWriter((*Writer)(bytes.NewBuffer(b)))
+}
+
 // SelectWriter 从池中取出一个 Writer
 func SelectWriter() *Writer {
 	// 因为 bufferPool 定义有 New 函数

helper/writer.go

@@ -131,7 +131,7 @@ func (w *Writer) Skip(n int) (int, error) {
 		}
 		return 0, io.EOF
 	}
-	n = min(n, len(b.buf[b.off:]))
+	n = minnum(n, len(b.buf[b.off:]))
 	b.off += n
 	if n > 0 {
 		b.lastRead = opRead
@@ -168,8 +168,8 @@ const (
 	opReadRune4 readOp = 4  // Read rune of size 4.
 )
 
-// min 返回两数最小值，该函数将被内联
+// minnum 返回两数最小值，该函数将被内联
-func min[T int | int8 | uint8 | int16 | uint16 | int32 | uint32 | int64 | uint64](a, b T) T {
+func minnum[T int | int8 | uint8 | int16 | uint16 | int32 | uint32 | int64 | uint64](a, b T) T {
 	if a > b {
 		return b
 	}

lower/nic.go

@@ -1,60 +1,61 @@
 package lower
 
 import (
-	"io"
+	"net"
 	"os"
 	"os/exec"
+	"strconv"
 
 	"github.com/fumiama/water"
 	"github.com/sirupsen/logrus"
 )
 
-type NICIO interface {
+// NICIO 虚拟网卡
-	io.ReadWriteCloser
+type NICIO struct {
-	Up()
+	ifce     *water.Interface
-	Down()
+	ip       net.IP
-}
+	subnet   *net.IPNet
-
+	rawipnet string
-// NIC 虚拟网卡
+	mtu      string
-type NIC struct {
+	cidrs    []string
-	ifce   *water.Interface
-	ip     string
-	subnet string
-	mtu    string
-	cidrs  []string
 }
 
 // NewNIC 新建 TUN 网络接口卡
 // 网卡地址为 ip, 所属子网为 subnet
 // 以本网卡为下一跳的所有子网为 cidrs
 // cidrs 不包括本网卡 subnet
-func NewNIC(ip, subnet, mtu string, cidrs ...string) NICIO {
+func NewNIC(ip net.IP, subnet *net.IPNet, mtu string, cidrs ...string) *NICIO {
 	ifce, err := water.New(water.Config{DeviceType: water.TUN})
 	if err != nil {
-		logrus.Error(err)
+		logrus.Errorln(err)
 		os.Exit(1)
 	}
-	n := &NIC{
+	subn, bitsn := subnet.Mask.Size()
-		ifce:   ifce,
+	if bitsn != 32 {
-		ip:     ip,
+		panic("mask len " + strconv.Itoa(bitsn) + " is not supported")
-		subnet: subnet,
+	}
-		mtu:    mtu,
+	n := &NICIO{
-		cidrs:  cidrs,
+		ifce:     ifce,
+		ip:       ip,
+		subnet:   subnet,
+		rawipnet: ip.String() + "/" + strconv.Itoa(subn),
+		mtu:      mtu,
+		cidrs:    cidrs,
 	}
 	return n
 }
 
 // Read 匹配 PacketsIO Interface
-func (nc *NIC) Read(buf []byte) (int, error) {
+func (nc *NICIO) Read(buf []byte) (int, error) {
 	return nc.ifce.Read(buf)
 }
 
-func (nc *NIC) Write(packet []byte) (int, error) {
+func (nc *NICIO) Write(packet []byte) (int, error) {
 	return nc.ifce.Write(packet)
 }
 
 // Close 关闭网卡
-func (n *NIC) Close() error {
+func (n *NICIO) Close() error {
 	return n.ifce.Close()
 }
 

lower/tun_darwin.go

@@ -3,17 +3,24 @@
 
 package lower
 
-func (n *NIC) Up() {
+import "net"
+
+func (n *NICIO) Up() {
 	execute("ifconfig", n.ifce.Name(), "mtu", n.mtu) // max: 9159
-	execute("ifconfig", n.ifce.Name(), "inet", n.ip, n.ip, "up")
+	execute(
-	execute("route", "add", n.subnet, "-interface", n.ifce.Name())
+		"ifconfig", n.ifce.Name(),
+		"inet", n.ip.String(), n.ip.String(),
+		"netmask", (net.IP)(n.subnet.Mask).String(),
+		"up",
+	)
+	execute("route", "add", n.subnet.String(), "-interface", n.ifce.Name())
 	for _, c := range n.cidrs {
 		execute("route", "add", c, "-interface", n.ifce.Name())
 	}
 }
 
-func (n *NIC) Down() {
+func (n *NICIO) Down() {
-	execute("route", "delete", n.subnet, "-interface", n.ifce.Name())
+	execute("route", "delete", n.subnet.String(), "-interface", n.ifce.Name())
 	for _, c := range n.cidrs {
 		execute("route", "delete", c, "-interface", n.ifce.Name())
 	}

lower/tun_linux.go

@@ -3,18 +3,16 @@
 
 package lower
 
-func (n *NIC) Up() {
+func (n *NICIO) Up() {
 	execute("/sbin/ip", "link", "set", "dev", n.ifce.Name(), "mtu", n.mtu)
-	execute("/sbin/ip", "addr", "add", n.ip, "dev", n.ifce.Name())
+	execute("/sbin/ip", "addr", "add", n.rawipnet, "dev", n.ifce.Name())
 	execute("/sbin/ip", "link", "set", "dev", n.ifce.Name(), "up")
-	execute("/sbin/ip", "route", "add", n.subnet, "dev", n.ifce.Name())
 	for _, c := range n.cidrs {
 		execute("/sbin/ip", "route", "add", c, "dev", n.ifce.Name())
 	}
 }
 
-func (n *NIC) Down() {
+func (n *NICIO) Down() {
-	execute("/sbin/ip", "route", "del", n.subnet, "dev", n.ifce.Name())
 	for _, c := range n.cidrs {
 		execute("/sbin/ip", "route", "del", c, "dev", n.ifce.Name())
 	}

lower/tun_stub.go

@@ -3,10 +3,10 @@
 
 package lower
 
-func (n *NIC) Up() {
+func (n *NICIO) Up() {
 	panic("not support lower on this os now")
 }
 
-func (n *NIC) Down() {
+func (n *NICIO) Down() {
 	panic("not support lower on this os now")
 }

lower/tun_windows.go

@@ -5,24 +5,19 @@ package lower
 
 import "net"
 
-func (n *NIC) Up() {
+func (n *NICIO) Up() {
-	// execute("netsh", "interface", "set", "interface", n.ifce.Name(), "enabled")
+	execute("cmd", "/c", "netsh interface ip set address name=\""+n.ifce.Name()+"\" source=static addr=\""+n.ip.String()+"\" mask=\""+(net.IP)(n.subnet.Mask).String()+"\" gateway=none")
-	_, ipn, err := net.ParseCIDR(n.subnet)
-	if err != nil {
-		panic(err)
-	}
-	execute("cmd", "/c", "netsh interface ip set address name=\""+n.ifce.Name()+"\" source=static addr=\""+n.ip+"\" mask=\""+(net.IP)(ipn.Mask).String()+"\" gateway=none")
 	execute("cmd", "/c", "netsh interface ipv4 set subinterface \""+n.ifce.Name()+"\" mtu="+n.mtu)
 	for _, c := range n.cidrs {
 		ip, cidr, err := net.ParseCIDR(c)
 		if err != nil {
 			panic(err)
 		}
-		execute("cmd", "/c", "route ADD "+ip.String()+" MASK "+(net.IP)(cidr.Mask).String()+" "+n.ip)
+		execute("cmd", "/c", "route ADD "+ip.String()+" MASK "+(net.IP)(cidr.Mask).String()+" "+n.ip.String())
 	}
 }
 
-func (n *NIC) Down() {
+func (n *NICIO) Down() {
 	// execute("netsh", "interface", "set", "interface", n.ifce.Name(), "disabled")
 	for _, c := range n.cidrs {
 		ip, _, err := net.ParseCIDR(c)

main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/fumiama/WireGold/config"
+	"github.com/fumiama/WireGold/gold/head"
 	"github.com/fumiama/WireGold/helper"
 	"github.com/fumiama/WireGold/upper"
 	"github.com/fumiama/WireGold/upper/services/wg"
@@ -144,8 +145,8 @@ func main() {
 	if c.EndPoint == "" {
 		displayHelp("nil endpoint")
 	}
-	if c.MTU == 0 {
+	if c.MTU <= head.PacketHeadLen {
-		displayHelp("nil mtu")
+		displayHelp("invalid mtu")
 	}
 	w, err := wg.NewWireGold(&c)
 	if err != nil {

upper/services/tunnel/tunnel.go

@@ -8,6 +8,12 @@ import (
 
 	"github.com/sirupsen/logrus"
 
+	_ "github.com/fumiama/WireGold/gold/p2p/ip"      // support ip connection
+	_ "github.com/fumiama/WireGold/gold/p2p/tcp"     // support tcp connection
+	_ "github.com/fumiama/WireGold/gold/p2p/udp"     // support udp connection
+	_ "github.com/fumiama/WireGold/gold/p2p/udplite" // support udplite connection
+
+	"github.com/fumiama/WireGold/config"
 	"github.com/fumiama/WireGold/gold/head"
 	"github.com/fumiama/WireGold/gold/link"
 )
@@ -36,7 +42,7 @@ func Create(me *link.Me, peer string) (s Tunnel, err error) {
 }
 
 func (s *Tunnel) Start(srcport, destport, mtu uint16) {
-	logrus.Infoln("[tunnel] start from", srcport, "to", destport)
+	logrus.Infoln("[tunnel] start port through", srcport, "->", destport, "mtu", mtu)
 	s.src = srcport
 	s.dest = destport
 	s.mtu = mtu
@@ -68,11 +74,11 @@ func (s *Tunnel) Read(p []byte) (int, error) {
 			return 0, io.EOF
 		}
 		defer pkt.Put()
-		if len(pkt.Data) < 4 {
+		if pkt.BodyLen() < 4 {
-			logrus.Warnln("[tunnel] unexpected packet data len", len(pkt.Data), "content", pkt.Data)
+			logrus.Warnln("[tunnel] unexpected packet data len", pkt.BodyLen(), "content", hex.EncodeToString(pkt.Body()))
 			return 0, io.EOF
 		}
-		d = pkt.Data[4:]
+		d = pkt.Body()[4:]
 	}
 	if d != nil {
 		if len(p) >= len(d) {
@@ -102,14 +108,20 @@ func (s *Tunnel) handleWrite() {
 			end = len(b)
 			endl = "."
 		}
-		logrus.Debugln("[tunnel] write send", hex.EncodeToString(b[:end]), endl)
+		if config.ShowDebugLog {
+			logrus.Debugln("[tunnel] write send", hex.EncodeToString(b[:end]), endl)
+		}
 		if b == nil {
 			logrus.Errorln("[tunnel] write recv nil")
 			break
 		}
-		logrus.Debugln("[tunnel] writing", len(b), "bytes...")
+		if config.ShowDebugLog {
+			logrus.Debugln("[tunnel] writing", len(b), "bytes...")
+		}
 		for len(b) > int(s.mtu)-4 {
-			logrus.Infoln("[tunnel] seq", seq, "split buffer")
+			if config.ShowDebugLog {
+				logrus.Debugln("[tunnel] seq", seq, "split buffer")
+			}
 			binary.LittleEndian.PutUint32(buf[:4], seq)
 			seq++
 			copy(buf[4:], b[:s.mtu-4])
@@ -120,7 +132,9 @@ func (s *Tunnel) handleWrite() {
 				logrus.Errorln("[tunnel] seq", seq-1, "write err:", err)
 				return
 			}
-			logrus.Debugln("[tunnel] seq", seq-1, "write succeeded")
+			if config.ShowDebugLog {
+				logrus.Debugln("[tunnel] seq", seq-1, "write succeeded")
+			}
 			b = b[s.mtu-4:]
 		}
 		binary.LittleEndian.PutUint32(buf[:4], seq)
@@ -133,7 +147,9 @@ func (s *Tunnel) handleWrite() {
 			logrus.Errorln("[tunnel] seq", seq-1, "write err:", err)
 			break
 		}
-		logrus.Debugln("[tunnel] seq", seq-1, "write succeeded")
+		if config.ShowDebugLog {
+			logrus.Debugln("[tunnel] seq", seq-1, "write succeeded")
+		}
 	}
 }
 
@@ -142,7 +158,9 @@ func (s *Tunnel) handleRead() {
 	seqmap := make(map[uint32]*head.Packet)
 	for {
 		if p, ok := seqmap[seq]; ok {
-			logrus.Debugln("[tunnel] dispatch cached seq", seq)
+			if config.ShowDebugLog {
+				logrus.Debugln("[tunnel] dispatch cached seq", seq)
+			}
 			delete(seqmap, seq)
 			seq++
 			s.out <- p
@@ -155,19 +173,25 @@ func (s *Tunnel) handleRead() {
 		}
 		end := 64
 		endl := "..."
-		if len(p.Data) < 64 {
+		if p.BodyLen() < 64 {
-			end = len(p.Data)
+			end = p.BodyLen()
 			endl = "."
 		}
-		logrus.Debugln("[tunnel] read recv", hex.EncodeToString(p.Data[:end]), endl)
+		if config.ShowDebugLog {
-		recvseq := binary.LittleEndian.Uint32(p.Data[:4])
+			logrus.Debugln("[tunnel] read recv", hex.EncodeToString(p.Body()[:end]), endl)
+		}
+		recvseq := binary.LittleEndian.Uint32(p.Body()[:4])
 		if recvseq == seq {
-			logrus.Debugln("[tunnel] dispatch seq", seq)
+			if config.ShowDebugLog {
+				logrus.Debugln("[tunnel] dispatch seq", seq)
+			}
 			seq++
 			s.out <- p
 			continue
 		}
 		seqmap[recvseq] = p
-		logrus.Debugln("[tunnel] cache seq", recvseq)
+		if config.ShowDebugLog {
+			logrus.Debugln("[tunnel] cache seq", recvseq)
+		}
 	}
 }

upper/services/tunnel/tunnel_test.go

@@ -4,7 +4,9 @@ import (
 	"bytes"
 	"crypto/rand"
 	"encoding/hex"
+	"fmt"
 	"io"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
@@ -16,7 +18,84 @@ import (
 	"github.com/fumiama/WireGold/helper"
 )
 
-func testTunnel(t *testing.T, isplain bool, pshk *[32]byte) {
+func TestTunnelUDP(t *testing.T) {
+	testTunnelNetwork(t, "udp", 4096)
+}
+
+func TestTunnelUDPSmallMTU(t *testing.T) {
+	testTunnelNetwork(t, "udp", 1024)
+}
+
+func TestTunnelUDPLite(t *testing.T) {
+	if runtime.GOOS == "darwin" {
+		return
+	}
+	testTunnelNetwork(t, "udplite", 4096)
+}
+
+func TestTunnelUDPLiteSmallMTU(t *testing.T) {
+	if runtime.GOOS == "darwin" {
+		return
+	}
+	testTunnelNetwork(t, "udplite", 1024)
+}
+
+func TestTunnelTCP(t *testing.T) {
+	testTunnelNetwork(t, "tcp", 4096)
+}
+
+func TestTunnelTCPSmallMTU(t *testing.T) {
+	testTunnelNetwork(t, "tcp", 1024)
+}
+
+func TestTunnelIP(t *testing.T) {
+	testTunnelNetwork(t, "ip", 4096)
+}
+
+func TestTunnelIPSmallMTU(t *testing.T) {
+	testTunnelNetwork(t, "ip", 1024)
+}
+
+func BenchmarkTunnelUDP(b *testing.B) {
+	benchmarkTunnelNetwork(b, "udp", 4096)
+}
+
+func BenchmarkTunnelUDPSmallMTU(b *testing.B) {
+	benchmarkTunnelNetwork(b, "udp", 1024)
+}
+
+func BenchmarkTunnelUDPLite(b *testing.B) {
+	if runtime.GOOS == "darwin" {
+		return
+	}
+	benchmarkTunnelNetwork(b, "udplite", 4096)
+}
+
+func BenchmarkTunnelUDPLiteSmallMTU(b *testing.B) {
+	if runtime.GOOS == "darwin" {
+		return
+	}
+	benchmarkTunnelNetwork(b, "udplite", 1024)
+}
+
+func BenchmarkTunnelTCP(b *testing.B) {
+	benchmarkTunnelNetwork(b, "tcp", 4096)
+}
+
+func BenchmarkTunnelTCPSmallMTU(b *testing.B) {
+	benchmarkTunnelNetwork(b, "tcp", 1024)
+}
+
+func BenchmarkTunnelIP(b *testing.B) {
+	benchmarkTunnelNetwork(b, "ip", 4096)
+}
+
+func BenchmarkTunnelIPSmallMTU(b *testing.B) {
+	benchmarkTunnelNetwork(b, "ip", 1024)
+}
+
+func testTunnel(t *testing.T, nw string, isplain, isbase14 bool, pshk *[32]byte, mtu uint16) {
+	fmt.Println("start", nw, "testing")
 	selfpk, err := curve.New(nil)
 	if err != nil {
 		panic(err)
@@ -30,23 +109,39 @@ func testTunnel(t *testing.T, isplain bool, pshk *[32]byte) {
 	t.Log("peer priv key:", hex.EncodeToString(peerpk.Private()[:]))
 	t.Log("peer publ key:", hex.EncodeToString(peerpk.Public()[:]))
 
+	epm := "127.0.0.1"
+	if nw != "ip" {
+		epm += ":0"
+	}
+	// under macos you need to run
+	//
+	// sudo ifconfig lo0 alias 127.0.0.2
+	epp := "127.0.0.2"
+	if nw != "ip" {
+		epp += ":0"
+	}
+
 	m := link.NewMe(&link.MyConfig{
 		MyIPwithMask: "192.168.1.2/32",
-		MyEndpoint:   "127.0.0.1:0",
+		MyEndpoint:   epm,
+		Network:      nw,
 		PrivateKey:   selfpk.Private(),
 		SrcPort:      1,
 		DstPort:      1,
-		MTU:          4096,
+		MTU:          mtu,
+		Base14:       isbase14,
 	})
 	defer m.Close()
 
 	p := link.NewMe(&link.MyConfig{
 		MyIPwithMask: "192.168.1.3/32",
-		MyEndpoint:   "127.0.0.1:0",
+		MyEndpoint:   epp,
+		Network:      nw,
 		PrivateKey:   peerpk.Private(),
 		SrcPort:      1,
 		DstPort:      1,
-		MTU:          4096,
+		MTU:          mtu,
+		Base14:       isbase14,
 	})
 	defer p.Close()
 
@@ -63,9 +158,10 @@ func testTunnel(t *testing.T, isplain bool, pshk *[32]byte) {
 		AllowedIPs:     []string{"192.168.1.3/32"},
 		PubicKey:       ppp,
 		PresharedKey:   pshk,
-		MTU:            4096,
+		MTU:            mtu,
-		MTURandomRange: 1024,
+		MTURandomRange: mtu / 2,
 		UseZstd:        true,
+		DoublePacket:   true,
 	})
 	p.AddPeer(&link.PeerConfig{
 		PeerIP:         "192.168.1.2",
@@ -73,8 +169,8 @@ func testTunnel(t *testing.T, isplain bool, pshk *[32]byte) {
 		AllowedIPs:     []string{"192.168.1.2/32"},
 		PubicKey:       spp,
 		PresharedKey:   pshk,
-		MTU:            4096,
+		MTU:            mtu,
-		MTURandomRange: 1024,
+		MTURandomRange: mtu / 2,
 		UseZstd:        true,
 	})
 	tunnme, err := Create(&m, "192.168.1.3")
@@ -91,17 +187,17 @@ func testTunnel(t *testing.T, isplain bool, pshk *[32]byte) {
 	time.Sleep(time.Second) // wait link up
 
 	sendb := ([]byte)("1234")
-	tunnme.Write(sendb)
+	go tunnme.Write(sendb)
 	buf := make([]byte, 4)
 	tunnpeer.Read(buf)
 	if string(sendb) != string(buf) {
-		t.Log("error: recv", buf)
+		logrus.Errorln("error: recv", buf, "expect", sendb)
 		t.Fail()
 	}
 
 	sendb = make([]byte, 4096)
 	rand.Read(sendb)
-	tunnme.Write(sendb)
+	go tunnme.Write(sendb)
 	buf = make([]byte, 4096)
 	_, err = io.ReadFull(&tunnpeer, buf)
 	if err != nil {
@@ -111,20 +207,31 @@ func testTunnel(t *testing.T, isplain bool, pshk *[32]byte) {
 		t.Fatal("error: recv 4096 bytes data")
 	}
 
-	sendb = make([]byte, 65535)
+	sendbufs := make(chan []byte, 32)
+
+	go func() {
+		time.Sleep(time.Second)
+		for i := 0; i < 32; i++ {
+			sendb := make([]byte, 65535)
+			rand.Read(sendb)
+			n, _ := tunnme.Write(sendb)
+			sendbufs <- sendb
+			logrus.Debugln("loop", i, "write", n, "bytes")
+		}
+		close(sendbufs)
+	}()
 	buf = make([]byte, 65535)
-	for i := 0; i < 32; i++ {
+	i := 0
-		rand.Read(sendb)
+	for sendb := range sendbufs {
-		n, _ := tunnme.Write(sendb)
+		n, err := io.ReadFull(&tunnpeer, buf)
-		t.Log("loop", i, "write", n, "bytes")
-		n, err = io.ReadFull(&tunnpeer, buf)
 		if err != nil {
 			t.Fatal(err)
 		}
-		t.Log("loop", i, "read", n, "bytes")
+		logrus.Debugln("loop", i, "read", n, "bytes")
 		if string(sendb) != string(buf) {
 			t.Fatal("loop", i, "error: recv 65535 bytes data")
 		}
+		i++
 	}
 
 	rand.Read(sendb)
@@ -146,20 +253,164 @@ func testTunnel(t *testing.T, isplain bool, pshk *[32]byte) {
 	}
 }
 
-func TestTunnel(t *testing.T) {
+func benchmarkTunnel(b *testing.B, sz int, nw string, isplain, isbase14 bool, pshk *[32]byte, mtu uint16) {
+	selfpk, err := curve.New(nil)
+	if err != nil {
+		panic(err)
+	}
+	peerpk, err := curve.New(nil)
+	if err != nil {
+		panic(err)
+	}
+
+	epm := "127.0.0.1"
+	if nw != "ip" {
+		epm += ":0"
+	}
+	// under macos you need to run
+	//
+	// sudo ifconfig lo0 alias 127.0.0.2
+	epp := "127.0.0.2"
+	if nw != "ip" {
+		epp += ":0"
+	}
+
+	m := link.NewMe(&link.MyConfig{
+		MyIPwithMask: "192.168.1.2/32",
+		MyEndpoint:   epm,
+		Network:      nw,
+		PrivateKey:   selfpk.Private(),
+		SrcPort:      1,
+		DstPort:      1,
+		MTU:          mtu,
+		Base14:       isbase14,
+	})
+	defer m.Close()
+
+	p := link.NewMe(&link.MyConfig{
+		MyIPwithMask: "192.168.1.3/32",
+		MyEndpoint:   epp,
+		Network:      nw,
+		PrivateKey:   peerpk.Private(),
+		SrcPort:      1,
+		DstPort:      1,
+		MTU:          mtu,
+		Base14:       isbase14,
+	})
+	defer p.Close()
+
+	ppp := peerpk.Public()
+	spp := selfpk.Public()
+	if isplain {
+		ppp = nil
+		spp = nil
+	}
+
+	m.AddPeer(&link.PeerConfig{
+		PeerIP:         "192.168.1.3",
+		EndPoint:       p.EndPoint().String(),
+		AllowedIPs:     []string{"192.168.1.3/32"},
+		PubicKey:       ppp,
+		PresharedKey:   pshk,
+		MTU:            mtu,
+		MTURandomRange: mtu / 2,
+		UseZstd:        true,
+		DoublePacket:   true,
+	})
+	p.AddPeer(&link.PeerConfig{
+		PeerIP:         "192.168.1.2",
+		EndPoint:       m.EndPoint().String(),
+		AllowedIPs:     []string{"192.168.1.2/32"},
+		PubicKey:       spp,
+		PresharedKey:   pshk,
+		MTU:            mtu,
+		MTURandomRange: mtu / 2,
+		UseZstd:        true,
+	})
+	tunnme, err := Create(&m, "192.168.1.3")
+	if err != nil {
+		b.Fatal(err)
+	}
+	tunnme.Start(1, 1, 4096)
+	tunnpeer, err := Create(&p, "192.168.1.2")
+	if err != nil {
+		b.Fatal(err)
+	}
+	tunnpeer.Start(1, 1, 4096)
+
+	time.Sleep(time.Second) // wait link up
+
+	b.SetBytes(int64(sz))
+	b.ResetTimer()
+	sendb := make([]byte, sz)
+	for i := 0; i < b.N; i++ {
+		rand.Read(sendb)
+		go tunnme.Write(sendb)
+		buf := make([]byte, sz)
+		_, err = io.ReadFull(&tunnpeer, buf)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+	b.StopTimer()
+	tunnme.Stop()
+	tunnpeer.Stop()
+}
+
+func testTunnelNetwork(t *testing.T, nw string, mtu uint16) {
 	logrus.SetLevel(logrus.DebugLevel)
 	logrus.SetFormatter(&logFormat{enableColor: false})
 
-	testTunnel(t, true, nil) // test plain text
+	// test without base14
+	testTunnel(t, nw, true, false, nil, mtu)  // test plain text
+	testTunnel(t, nw, false, false, nil, mtu) // test normal
 
-	testTunnel(t, false, nil) // test normal
+	// test with base14
+	testTunnel(t, nw, true, true, nil, mtu)  // test plain text
+	testTunnel(t, nw, false, true, nil, mtu) // test normal
 
 	var buf [32]byte
 	_, err := rand.Read(buf[:])
 	if err != nil {
 		panic(err)
 	}
-	testTunnel(t, false, &buf) // test preshared
+	// test without base14
+	testTunnel(t, nw, false, false, &buf, mtu) // test preshared
+
+	// test with base14
+	testTunnel(t, nw, false, true, &buf, mtu) // test preshared
+}
+
+func benchmarkTunnelNetwork(b *testing.B, nw string, mtu uint16) {
+	logrus.SetLevel(logrus.ErrorLevel)
+	logrus.SetFormatter(&logFormat{enableColor: false})
+
+	for i := 1; i <= 4; i++ {
+		sz := 1024 * i
+		b.Run(fmt.Sprintf("%d-plain-nob14", sz), func(b *testing.B) {
+			benchmarkTunnel(b, sz, nw, true, false, nil, mtu) // test plain text
+		})
+		b.Run(fmt.Sprintf("%d-normal-nob14", sz), func(b *testing.B) {
+			benchmarkTunnel(b, sz, nw, false, false, nil, mtu) // test normal
+		})
+		b.Run(fmt.Sprintf("%d-plain-b14", sz), func(b *testing.B) {
+			benchmarkTunnel(b, sz, nw, true, true, nil, mtu) // test plain text
+		})
+		b.Run(fmt.Sprintf("%d-normal-b14", sz), func(b *testing.B) {
+			benchmarkTunnel(b, sz, nw, false, true, nil, mtu) // test normal
+		})
+		var buf [32]byte
+		_, err := rand.Read(buf[:])
+		if err != nil {
+			panic(err)
+		}
+		b.Run(fmt.Sprintf("%d-preshared-nob14", sz), func(b *testing.B) {
+			benchmarkTunnel(b, sz, nw, false, false, &buf, mtu) // test preshared
+		})
+		b.Run(fmt.Sprintf("%d-preshared-b14", sz), func(b *testing.B) {
+			benchmarkTunnel(b, sz, nw, false, true, &buf, mtu) // test preshared
+		})
+	}
 }
 
 // logFormat specialize for go-cqhttp
@@ -169,8 +420,7 @@ type logFormat struct {
 
 // Format implements logrus.Formatter
 func (f logFormat) Format(entry *logrus.Entry) ([]byte, error) {
-	buf := helper.SelectWriter()
+	buf := helper.SelectWriter() // this writer will not be put back
-	defer helper.PutWriter(buf)
 
 	buf.WriteByte('[')
 	if f.enableColor {
@@ -184,9 +434,7 @@ func (f logFormat) Format(entry *logrus.Entry) ([]byte, error) {
 	buf.WriteString(entry.Message)
 	buf.WriteString("\n")
 
-	ret := make([]byte, len(buf.Bytes()))
+	return buf.Bytes(), nil
-	copy(ret, buf.Bytes()) // copy buffer
-	return ret, nil
 }
 
 const (

upper/services/wg/wg.go

@@ -9,10 +9,14 @@ import (
 	curve "github.com/fumiama/go-x25519"
 	"github.com/sirupsen/logrus"
 
+	_ "github.com/fumiama/WireGold/gold/p2p/ip"      // support ip connection
+	_ "github.com/fumiama/WireGold/gold/p2p/tcp"     // support tcp connection
+	_ "github.com/fumiama/WireGold/gold/p2p/udp"     // support udp connection
+	_ "github.com/fumiama/WireGold/gold/p2p/udplite" // support udplite connection
+
 	"github.com/fumiama/WireGold/config"
 	"github.com/fumiama/WireGold/gold/link"
 	"github.com/fumiama/WireGold/helper"
-	"github.com/fumiama/WireGold/lower"
 )
 
 const suffix32 = "㴄"
@@ -34,7 +38,7 @@ func NewWireGold(c *config.Config) (wg WG, err error) {
 	}
 	n := copy(wg.key[:], base14.Decode(k))
 	if n != 32 {
-		err = errors.New("private key length is not 32")
+		err = errors.New("private key length != 32, got " + strconv.Itoa(n))
 		return
 	}
 
@@ -55,7 +59,7 @@ func (wg *WG) Start(srcport, destport uint16) {
 func (wg *WG) Run(srcport, destport uint16) {
 	wg.init(srcport, destport)
 	_, _ = wg.me.ListenNIC()
-	logrus.Info("[wg] stopped")
+	logrus.Infoln("[wg] stopped")
 }
 
 func (wg *WG) Stop() {
@@ -69,9 +73,13 @@ func (wg *WG) init(srcport, dstport uint16) {
 	if err != nil {
 		panic(err)
 	}
+	myip := net.ParseIP(wg.c.IP)
+	if myip == nil {
+		panic("invalid ip " + wg.c.IP)
+	}
 	for _, p := range wg.c.Peers {
 		for _, ip := range p.AllowedIPs {
-			if len(ip) == 0 || ip[0] == 'x' {
+			if len(ip) == 0 || ip[0] == 'x' || ip[0] == 'y' {
 				continue
 			}
 			ipnet, _, err := net.ParseCIDR(ip)
@@ -91,15 +99,21 @@ func (wg *WG) init(srcport, dstport uint16) {
 	}
 
 	wg.me = link.NewMe(&link.MyConfig{
-		MyIPwithMask: wg.c.IP + "/32",
+		MyIPwithMask: myip.String() + "/32",
 		MyEndpoint:   wg.c.EndPoint,
+		Network:      wg.c.Network,
 		PrivateKey:   &wg.key,
-		NIC:          lower.NewNIC(wg.c.IP, wg.c.SubNet, strconv.FormatInt(wg.c.MTU, 10), cidrs...),
+		NICConfig: &link.NICConfig{
-		SrcPort:      srcport,
+			IP:     myip,
-		DstPort:      dstport,
+			SubNet: mysubnet,
-		MTU:          uint16(wg.c.MTU),
+			CIDRs:  cidrs,
-		SpeedLoop:    wg.c.SpeedLoop,
+		},
-		Mask:         wg.c.Mask,
+		SrcPort:   srcport,
+		DstPort:   dstport,
+		MTU:       uint16(wg.c.MTU),
+		SpeedLoop: wg.c.SpeedLoop,
+		Mask:      wg.c.Mask,
+		Base14:    wg.c.Base14,
 	})
 
 	for _, peer := range wg.c.Peers {
@@ -144,6 +158,7 @@ func (wg *WG) init(srcport, dstport uint16) {
 			AllowTrans:     peer.AllowTrans,
 			NoPipe:         true,
 			UseZstd:        peer.UseZstd,
+			DoublePacket:   peer.DoublePacket,
 		})
 	}
 }