feat: impl. new protol design & new head

internal/algo/crypto.go

@@ -0,0 +1,115 @@
+package algo
+
+import (
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/binary"
+	"errors"
+
+	"github.com/fumiama/orbyte/pbuf"
+)
+
+var (
+	ErrCipherTextTooShort = errors.New("ciphertext too short")
+)
+
+func EncodeAEAD(aead cipher.AEAD, additional uint16, b []byte) pbuf.Bytes {
+	nsz := aead.NonceSize()
+	// Accocate capacity for all the stuffs.
+	buf := pbuf.NewBytes(2 + nsz + len(b) + aead.Overhead())
+	n := 0
+	buf.V(func(buf []byte) {
+		binary.LittleEndian.PutUint16(buf[:2], additional)
+		nonce := buf[2 : 2+nsz]
+		// Select a random nonce
+		_, err := rand.Read(nonce)
+		if err != nil {
+			panic(err)
+		}
+		// Encrypt the message and append the ciphertext to the nonce.
+		eb := aead.Seal(nonce[nsz:nsz], nonce, b, buf[:2])
+		n = len(eb)
+	})
+	return buf.Slice(2, 2+nsz+n)
+}
+
+func DecodeAEAD(aead cipher.AEAD, additional uint16, b []byte) (data pbuf.Bytes, err error) {
+	nsz := aead.NonceSize()
+	if len(b) < nsz {
+		return pbuf.Bytes{}, ErrCipherTextTooShort
+	}
+	// Split nonce and ciphertext.
+	nonce, ciphertext := b[:nsz], b[nsz:]
+	if len(ciphertext) == 0 {
+		return pbuf.Bytes{}, nil
+	}
+	// Decrypt the message and check it wasn't tampered with.
+	var buf [2]byte
+	binary.LittleEndian.PutUint16(buf[:], additional)
+	data = pbuf.NewBytes(len(ciphertext))
+	n := 0
+	data.V(func(b []byte) {
+		var d []byte
+		d, err = aead.Open(b[:0], nonce, ciphertext, buf[:])
+		n = len(d)
+	})
+	if err != nil {
+		return
+	}
+	return data.SliceTo(n), nil
+}
+
+func EncodeXORLen(datalen int) int {
+	batchsz := datalen / 8
+	return 8 + batchsz*8 + 8 // seqrand dat tail
+}
+
+// EncodeXOR 按 8 字节, 以初始 mask 循环异或编码 data
+func EncodeXOR(data []byte, mask uint64, seq uint32) pbuf.Bytes {
+	batchsz := len(data) / 8
+	remain := len(data) % 8
+	sum := mask
+	newdat := pbuf.NewBytes(EncodeXORLen(len(data)))
+	newdat.V(func(buf []byte) {
+		binary.LittleEndian.PutUint32(buf[:4], seq)
+		_, _ = rand.Read(buf[4:8])                 // seqrand
+		sum ^= binary.LittleEndian.Uint64(buf[:8]) // init from seqrand
+		binary.LittleEndian.PutUint64(buf[:8], sum)
+		for i := 0; i < batchsz; i++ { // range on batch data
+			a := i * 8
+			b := (i + 1) * 8
+			sum ^= binary.LittleEndian.Uint64(data[a:b])
+			binary.LittleEndian.PutUint64(buf[a+8:b+8], sum)
+		}
+		p := batchsz * 8
+		copy(buf[8+p:], data[p:])
+		buf[newdat.Len()-1] = byte(remain)
+		sum ^= binary.LittleEndian.Uint64(buf[8+p:])
+		binary.LittleEndian.PutUint64(buf[8+p:], sum)
+	})
+	return newdat
+}
+
+// DecodeXOR 按 8 字节, 以初始 mask 循环异或解码 data,
+// 解码结果完全覆盖 data.
+func DecodeXOR(data []byte, mask uint64) (uint32, []byte) {
+	if len(data) < 16 || len(data)%8 != 0 {
+		return 0, nil
+	}
+	batchsz := len(data) / 8
+	sum := mask
+	for i := 0; i < batchsz; i++ {
+		a := i * 8
+		b := (i + 1) * 8
+		x := binary.LittleEndian.Uint64(data[a:b])
+		sum ^= x
+		binary.LittleEndian.PutUint64(data[a:b], sum)
+		sum = x
+	}
+	remain := data[len(data)-1]
+	if remain >= 8 {
+		return 0, nil
+	}
+	return binary.LittleEndian.Uint32(data[:4]),
+		data[8 : len(data)-8+int(remain)]
+}

internal/algo/crypto_test.go

@@ -0,0 +1,90 @@
+package algo
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/binary"
+	"encoding/hex"
+	"io"
+	"testing"
+
+	"golang.org/x/crypto/chacha20poly1305"
+)
+
+func TestXOR(t *testing.T) {
+	mask := uint64(0x12345678_90abcdef)
+	buf := make([]byte, 4096)
+	buf2 := make([]byte, 4096)
+	for i := 0; i < 4096; i++ {
+		data := buf[:i]
+		orgdata := buf2[:i]
+		r1 := bytes.NewBuffer(data[:0])
+		r2 := bytes.NewBuffer(orgdata[:0])
+		w := io.MultiWriter(r1, r2)
+		_, err := io.CopyN(w, rand.Reader, int64(i))
+		if err != nil {
+			t.Fatal(err)
+		}
+		seq, dec := DecodeXOR(EncodeXOR(r1.Bytes(), mask, uint32(i)).Trans(), mask)
+		if !bytes.Equal(dec, r2.Bytes()) {
+			t.Fatal("unexpected xor at", i, "except", hex.EncodeToString(r2.Bytes()), "got", hex.EncodeToString(dec))
+		}
+		if seq != uint32(i) {
+			t.Fatal("unexpected xor at", i, "seq", seq)
+		}
+	}
+}
+
+func TestXChacha20(t *testing.T) {
+	k := make([]byte, 32)
+	_, err := rand.Read(k)
+	if err != nil {
+		t.Fatal(err)
+	}
+	aead, err := chacha20poly1305.NewX(k)
+	if err != nil {
+		t.Fatal(err)
+	}
+	data := make([]byte, 4096)
+	_, err = rand.Read(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < 4096; i++ {
+		db, err := DecodeAEAD(aead, uint16(i), EncodeAEAD(aead, uint16(i), data[:i]).Trans())
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(db.Trans(), data[:i]) {
+			t.Fatal("unexpected preshared at idx(len)", i, "addt", uint16(i))
+		}
+	}
+}
+
+func TestExpandKeyUnit(t *testing.T) {
+	k1 := byte(0b10001010)
+	k2 := byte(0b10111010)     // rev 01011101
+	v := expandkeyunit(k1, k2) // x1x0x0x0x1x0x1x0 | 0x1x0x1x1x1x0x1x = 0110001011100110
+	if v != 0b0110001011100110 {
+		buf := [2]byte{}
+		binary.BigEndian.PutUint16(buf[:], v)
+		t.Fatal(hex.EncodeToString(buf[:]))
+	}
+}
+
+func TestMixKeys(t *testing.T) {
+	k1, _ := hex.DecodeString("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
+	k2, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000000")
+	k := MixKeys(k1, k2)
+	kexp, _ := hex.DecodeString("55555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555")
+	if !bytes.Equal(k, kexp) {
+		t.Fatal(hex.EncodeToString(k))
+	}
+	k1, _ = hex.DecodeString("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef")
+	k2, _ = hex.DecodeString("deadbeef1239876540deadbeef1239876540deadbeef1239876540abcdef4567")
+	k = MixKeys(k1, k2)
+	kexp, _ = hex.DecodeString("2ca9188d3ebb4a9f22e34d4479d857fca48390253ebbe23f22cbcf6e59507ddc06a9b08794316abfa26b67cedb7a5d542c8912adb493c0352aebe76e73dadf7e")
+	if !bytes.Equal(k, kexp) {
+		t.Fatal(hex.EncodeToString(k))
+	}
+}

internal/algo/hash.go

@@ -0,0 +1,47 @@
+package algo
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/binary"
+	"encoding/hex"
+
+	"github.com/fumiama/WireGold/config"
+	"github.com/fumiama/blake2b-simd"
+	"github.com/sirupsen/logrus"
+)
+
+// Blake2bHash8 生成 data 的 blake2b hash, 返回前八位
+func Blake2bHash8(precrc64 uint64, data []byte) uint64 {
+	var tgt [32]byte
+	h := blake2b.New256()
+	binary.LittleEndian.PutUint64(tgt[:8], precrc64)
+	_, _ = h.Write(tgt[:8])
+	_, _ = h.Write(data)
+	b := h.Sum(tgt[:0])[:8]
+	if config.ShowDebugLog {
+		logrus.Debugln("[algo] blk2b hash:", hex.EncodeToString(b))
+	}
+	return binary.LittleEndian.Uint64(b)
+}
+
+// IsVaildBlake2bHash8 在收齐全部分片并解密后验证 packet 合法性
+func IsVaildBlake2bHash8(precrc64 uint64, hash8data []byte) bool {
+	var tgt [32]byte
+	h := blake2b.New256()
+	binary.LittleEndian.PutUint64(tgt[:8], precrc64)
+	_, _ = h.Write(tgt[:8])
+	_, _ = h.Write(hash8data[8:])
+	b := h.Sum(tgt[:0])[:8]
+	if config.ShowDebugLog {
+		logrus.Debugln("[algo] blk2b sum calulated:", hex.EncodeToString(b))
+		logrus.Debugln("[algo] blk2b sum in packet:", hex.EncodeToString(hash8data[:8]))
+	}
+	return bytes.Equal(b, hash8data[:8])
+}
+
+// MD5Hash8 calculate packet header checksum
+func MD5Hash8(data []byte) uint64 {
+	m := md5.Sum(data)
+	return binary.LittleEndian.Uint64(m[:8])
+}

internal/algo/key.go

@@ -0,0 +1,40 @@
+package algo
+
+import (
+	"encoding/binary"
+	"math/bits"
+	"math/rand"
+)
+
+func RandKeyIndex() uint8 {
+	return uint8(rand.Intn(32))
+}
+
+func MixKeys(k1, k2 []byte) []byte {
+	if len(k1) != 32 || len(k2) != 32 {
+		panic("unexpected key len")
+	}
+	k := make([]byte, 64)
+	for i := range k1 {
+		k1i, k2i := i, 31-i
+		k1v, k2v := k1[k1i], k2[k2i]
+		binary.LittleEndian.PutUint16(
+			k[i*2:(i+1)*2],
+			expandkeyunit(k1v, k2v),
+		)
+	}
+	return k
+}
+
+func expandkeyunit(v1, v2 byte) (v uint16) {
+	v1s, v2s := uint16(v1), uint16(bits.Reverse8(v2))
+	for i := 0; i < 8; i++ {
+		v |= v1s & (1 << (i * 2))
+		v1s <<= 1
+	}
+	for i := 0; i < 8; i++ {
+		v2s <<= 1
+		v |= v2s & (2 << (i * 2))
+	}
+	return
+}

internal/algo/zstd.go

@@ -0,0 +1,41 @@
+package algo
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/fumiama/WireGold/internal/bin"
+	"github.com/fumiama/orbyte/pbuf"
+	"github.com/klauspost/compress/zstd"
+)
+
+func EncodeZstd(data []byte) pbuf.Bytes {
+	return bin.SelectWriter().P(func(w *pbuf.Buffer) {
+		enc, err := zstd.NewWriter(w, zstd.WithEncoderLevel(zstd.SpeedFastest))
+		if err != nil {
+			panic(err)
+		}
+		_, err = io.Copy(enc, bytes.NewReader(data))
+		if err != nil {
+			panic(err)
+		}
+		err = enc.Close()
+		if err != nil {
+			panic(err)
+		}
+	}).ToBytes()
+}
+
+func DecodeZstd(data []byte) (b pbuf.Bytes, err error) {
+	dec, err := zstd.NewReader(bytes.NewReader(data))
+	if err != nil {
+		return pbuf.Bytes{}, err
+	}
+
+	b = bin.SelectWriter().P(func(w *pbuf.Buffer) {
+		_, err = io.Copy(w, dec)
+		dec.Close()
+	}).ToBytes()
+
+	return
+}

internal/bin/data.go

@@ -0,0 +1,45 @@
+package bin
+
+import (
+	"encoding/binary"
+	"reflect"
+	"unsafe"
+)
+
+// IsLittleEndian judge by binary packet.
+var IsLittleEndian = reflect.ValueOf(&binary.NativeEndian).Elem().Field(0).Type().String() == "binary.littleEndian"
+
+// slice is the runtime representation of a slice.
+// It cannot be used safely or portably and its representation may
+// change in a later release.
+//
+// Unlike reflect.SliceHeader, its Data field is sufficient to guarantee the
+// data it references will not be garbage collected.
+type slice struct {
+	data unsafe.Pointer
+	len  int
+	cap  int
+}
+
+// BytesToString 没有内存开销的转换
+func BytesToString(b []byte) string {
+	return *(*string)(unsafe.Pointer(&b))
+}
+
+// StringToBytes 没有内存开销的转换
+func StringToBytes(s string) (b []byte) {
+	bh := (*slice)(unsafe.Pointer(&b))
+	sh := (*slice)(unsafe.Pointer(&s)) // 不要访问 sh.cap
+	bh.data = sh.data
+	bh.len = sh.len
+	bh.cap = sh.len
+	return b
+}
+
+func IsNilInterface(x any) bool {
+	return x == nil || reflect.ValueOf(x).IsZero()
+}
+
+func IsNonNilInterface(x any) bool {
+	return !IsNilInterface(x)
+}

internal/bin/pool.go

@@ -0,0 +1,10 @@
+package bin
+
+import (
+	"github.com/fumiama/orbyte/pbuf"
+)
+
+// SelectWriter 从池中取出一个 Writer
+func SelectWriter() *Writer {
+	return (*Writer)(pbuf.NewBuffer(nil))
+}

internal/bin/writer.go

@@ -0,0 +1,66 @@
+package bin
+
+// https://github.com/Mrs4s/MiraiGo/blob/master/binary/writer.go
+
+import (
+	"encoding/binary"
+
+	"github.com/fumiama/orbyte/pbuf"
+)
+
+// Writer 写入
+type Writer pbuf.OBuffer
+
+func NewWriterF(f func(writer *Writer)) pbuf.Bytes {
+	w := SelectWriter()
+	f(w)
+	return w.ToBytes()
+}
+
+func (w *Writer) P(f func(*pbuf.Buffer)) *Writer {
+	(*pbuf.OBuffer)(w).P(f)
+	return w
+}
+
+func (w *Writer) Write(b []byte) (n int, err error) {
+	w.P(func(buf *pbuf.Buffer) {
+		n, err = buf.Write(b)
+	})
+	return
+}
+
+func (w *Writer) WriteByte(b byte) (err error) {
+	w.P(func(buf *pbuf.Buffer) {
+		err = buf.WriteByte(b)
+	})
+	return
+}
+
+func (w *Writer) WriteString(s string) (n int, err error) {
+	w.P(func(buf *pbuf.Buffer) {
+		n, err = buf.WriteString(s)
+	})
+	return
+}
+
+func (w *Writer) WriteUInt16(v uint16) {
+	b := make([]byte, 2)
+	binary.LittleEndian.PutUint16(b, v)
+	w.Write(b)
+}
+
+func (w *Writer) WriteUInt32(v uint32) {
+	b := make([]byte, 4)
+	binary.LittleEndian.PutUint32(b, v)
+	w.Write(b)
+}
+
+func (w *Writer) WriteUInt64(v uint64) {
+	b := make([]byte, 8)
+	binary.LittleEndian.PutUint64(b, v)
+	w.Write(b)
+}
+
+func (w *Writer) ToBytes() pbuf.Bytes {
+	return pbuf.BufferItemToBytes((*pbuf.OBuffer)(w))
+}

internal/file/file.go

@@ -0,0 +1,37 @@
+package file
+
+import (
+	"os"
+	"runtime"
+	"strings"
+)
+
+// IsExist 文件/路径存在
+func IsExist(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil || os.IsExist(err)
+}
+
+// IsNotExist 文件/路径不存在
+func IsNotExist(path string) bool {
+	_, err := os.Stat(path)
+	return err != nil && os.IsNotExist(err)
+}
+
+// FolderName 本文件所在最下级文件夹名
+func FolderName() string {
+	_, file, _, ok := runtime.Caller(1)
+	if !ok {
+		return "<unk>"
+	}
+	i := strings.LastIndex(file, "/")
+	if i <= 0 {
+		return file
+	}
+	file = file[:i]
+	i = strings.LastIndex(file, "/")
+	if i <= 0 || i+1 >= len(file) {
+		return file
+	}
+	return file[i+1:]
+}

internal/file/log.go

@@ -0,0 +1,52 @@
+package file
+
+import (
+	"encoding/hex"
+	"runtime"
+	"strings"
+)
+
+func Header() string {
+	file, fn := fileFuncName(2)
+	sb := strings.Builder{}
+	sb.WriteString("[")
+	sb.WriteString(file)
+	sb.WriteString("] ")
+	sb.WriteString(fn)
+	return sb.String()
+}
+
+func fileFuncName(skip int) (string, string) {
+	pc, file, _, ok := runtime.Caller(skip)
+	if !ok {
+		return "unknown", "unknown"
+	}
+	fn := runtime.FuncForPC(pc).Name()
+	i := strings.LastIndex(fn, "/")
+	fn = fn[i+1:]
+	i = strings.LastIndex(file, "/")
+	if i < 0 {
+		i = strings.LastIndex(file, "\\")
+		if i < 0 {
+			return file, fn
+		}
+	}
+	nm := file[i+1:]
+	if len(nm) == 0 {
+		return file, fn
+	}
+	i = strings.LastIndex(nm, ".")
+	if i <= 0 {
+		return nm, fn
+	}
+	return nm[:i], fn
+}
+
+func ToLimitHexString(data []byte, bound int) string {
+	endl := "..."
+	if len(data) < bound {
+		bound = len(data)
+		endl = "."
+	}
+	return hex.EncodeToString(data[:bound]) + endl
+}