feat(trs): new implementation

cfg.go

@@ -4,21 +4,25 @@ import (
 	"archive/zip"
 	"bytes"
 	_ "embed"
-	"fmt"
-	"io"
-	"net/http"
 	"os"
-	"runtime"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
 
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
 )
 
+type config struct {
+	BaseURL string    `yaml:"BaseURL"`
+	Targets []targets `yaml:"Targets"`
+}
+
+type targets struct {
+	Refer  string   `yaml:"Refer"`
+	Folder string   `yaml:"Folder"`
+	Copy   []string `yaml:"Copy"`
+	OS     string   `yaml:"OS"`
+	Arch   string   `yaml:"Arch"`
+}
+
 //go:embed cfg.zip
 var cfgdata []byte
 
@@ -51,113 +55,3 @@ func readconfig(path string, usecust bool) (c config, err error) {
 	}
 	return
 }
-
-type config struct {
-	BaseURL string    `yaml:"BaseURL"`
-	Targets []targets `yaml:"Targets"`
-}
-
-type targets struct {
-	Refer  string   `yaml:"Refer"`
-	Folder string   `yaml:"Folder"`
-	Copy   []string `yaml:"Copy"`
-	OS     string   `yaml:"OS"`
-	Arch   string   `yaml:"Arch"`
-}
-
-func (c *config) download(path, prefix string, usecust, force bool) error {
-	for i, t := range c.Targets {
-		if t.Refer != "" {
-			refp := path[:strings.LastIndex(path, "/")+1] + t.Refer
-			logrus.Infof("#%s%d refer to target '%s'.", prefix, i+1, refp)
-			refcfg, err := readconfig(refp, usecust)
-			if err != nil {
-				return err
-			}
-			err = refcfg.download(refp, prefix+strconv.Itoa(i+1)+".", usecust, force)
-			if err != nil {
-				return err
-			}
-			continue
-		}
-		if t.OS != "" && t.OS != runtime.GOOS {
-			logrus.Warnf("#%s%d target required OS: %s but you are %s, skip.", prefix, i+1, t.OS, runtime.GOOS)
-			continue
-		}
-		if t.Arch != "" && t.Arch != runtime.GOARCH {
-			logrus.Warnf("#%s%d target required Arch: %s but you are %s, skip.", prefix, i+1, t.Arch, runtime.GOARCH)
-			continue
-		}
-		err := os.MkdirAll(t.Folder, 0755)
-		if err != nil {
-			return errors.Wrap(err, fmt.Sprintf("#%s%d make target folder '%s'", prefix, i+1, t.Folder))
-		}
-		logrus.Infof("#%s%d open target folder '%s'.", prefix, i+1, t.Folder)
-		if len(t.Copy) == 0 {
-			logrus.Warningf("#%s%d empty copy target.", prefix, i+1)
-			continue
-		}
-		wg := sync.WaitGroup{}
-		wg.Add(len(t.Copy))
-		logrus.Infof("#%s%d download copy: '%v'.", prefix, i+1, t.Copy)
-		for j, cp := range t.Copy {
-			go func(i int, cp, prefix string) {
-				defer wg.Done()
-				if strings.Contains(cp, "/") { // have innner folder
-					infldr := t.Folder + "/" + cp[:strings.LastIndex(cp, "/")]
-					err := os.MkdirAll(infldr, 0755)
-					if err != nil {
-						logrus.Errorf("#%s%d make target inner folder '%s' err: %v", prefix, i+1, t.Folder, err)
-						return
-					}
-					logrus.Infof("#%s%d make target inner folder '%s'.", prefix, i+1, t.Folder)
-				}
-				sleep := time.Millisecond * 100 * time.Duration(i)
-				if sleep > time.Millisecond {
-					time.Sleep(sleep)
-				}
-				fname := t.Folder + "/" + cp
-				if !force {
-					if _, err := os.Stat(fname); err == nil || os.IsExist(err) {
-						logrus.Warnf("#%s%d skip exist file %s", prefix, i+1, fname)
-						return
-					}
-				}
-				req, err := http.NewRequest("GET", c.BaseURL+"/"+cp, nil)
-				if err != nil {
-					logrus.Errorf("#%s%d new request to %s err: %v", prefix, i+1, cp, err)
-					return
-				}
-				logrus.Infof("#%s%d get: %s", prefix, i+1, req.URL)
-				req.Header.Add("user-agent", ua)
-				resp, err := cli.Do(req)
-				if err != nil {
-					logrus.Errorf("#%s%d get %s err: %v", prefix, i+1, req.URL, err)
-					return
-				}
-				defer resp.Body.Close()
-				if resp.StatusCode != http.StatusOK {
-					err := errors.New(fmt.Sprintf("HTTP %d %s", resp.StatusCode, resp.Status))
-					logrus.Errorf("#%s%d get %s err: %v", prefix, i+1, req.URL, err)
-					return
-				}
-				f, err := os.Create(fname)
-				if err != nil {
-					logrus.Errorf("#%s%d create file %s err: %v", prefix, i+1, fname, err)
-					return
-				}
-				defer f.Close()
-				logrus.Infof("#%s%d writing file %s", prefix, i+1, fname)
-				pm := newmeter(fmt.Sprintf("#%s%d", prefix, i+1), fname, int(resp.ContentLength))
-				_, err = io.Copy(io.MultiWriter(f, &pm), resp.Body)
-				if err != nil {
-					logrus.Errorf("#%s%d download file %s err: %v", prefix, i+1, fname, err)
-					return
-				}
-				logrus.Infof("#%s%d finished download %s", prefix, i+1, fname)
-			}(j, cp, fmt.Sprintf("%s%d.", prefix, i+1))
-		}
-		wg.Wait()
-	}
-	return nil
-}

dns.go

@@ -1,163 +0,0 @@
-package main
-
-// https://github.com/fumiama/comandy
-
-import (
-	"context"
-	"crypto/tls"
-	"errors"
-	"net"
-	"sync"
-	"time"
-
-	"github.com/fumiama/terasu"
-)
-
-var (
-	errNoDNSAvailable = errors.New("no dns available")
-)
-
-var dnsdialer = net.Dialer{
-	Timeout: time.Second * 8,
-}
-
-type dnsstat struct {
-	A string
-	E bool
-}
-
-type dnsservers struct {
-	sync.RWMutex
-	m map[string][]*dnsstat
-}
-
-// hasrecord no lock, use under lock
-func hasrecord(lst []*dnsstat, a string) bool {
-	for _, addr := range lst {
-		if addr.A == a {
-			return true
-		}
-	}
-	return false
-}
-
-func (ds *dnsservers) add(m map[string][]string) {
-	ds.Lock()
-	defer ds.Unlock()
-	addList := map[string][]*dnsstat{}
-	for host, addrs := range m {
-		for _, addr := range addrs {
-			if !hasrecord(ds.m[host], addr) && !hasrecord(addList[host], addr) {
-				addList[host] = append(addList[host], &dnsstat{addr, true})
-			}
-		}
-	}
-	for host, addrs := range addList {
-		ds.m[host] = append(ds.m[host], addrs...)
-	}
-}
-
-func (ds *dnsservers) dial(ctx context.Context) (tlsConn *tls.Conn, err error) {
-	err = errNoDNSAvailable
-
-	ds.RLock()
-	defer ds.RUnlock()
-
-	if dnsdialer.Timeout != 0 {
-		var cancel context.CancelFunc
-		ctx, cancel = context.WithTimeout(ctx, dnsdialer.Timeout)
-		defer cancel()
-	}
-
-	if !dnsdialer.Deadline.IsZero() {
-		var cancel context.CancelFunc
-		ctx, cancel = context.WithDeadline(ctx, dnsdialer.Deadline)
-		defer cancel()
-	}
-
-	var conn net.Conn
-	for host, addrs := range ds.m {
-		for _, addr := range addrs {
-			if !addr.E {
-				continue
-			}
-			conn, err = dnsdialer.DialContext(ctx, "tcp", addr.A)
-			if err != nil {
-				addr.E = false // no need to acquire write lock
-				continue
-			}
-			tlsConn = tls.Client(conn, &tls.Config{ServerName: host})
-			if usetrs {
-				err = terasu.Use(tlsConn).HandshakeContext(ctx)
-			} else {
-				err = tlsConn.HandshakeContext(ctx)
-			}
-			if err == nil {
-				return
-			}
-			_ = tlsConn.Close()
-			addr.E = false // no need to acquire write lock
-		}
-	}
-	return
-}
-
-var dotv6servers = dnsservers{
-	m: map[string][]*dnsstat{
-		"dot.sb": {
-			{"[2a09::]:853", true},
-			{"[2a11::]:853", true},
-		},
-		"dns.google": {
-			{"[2001:4860:4860::8888]:853", true},
-			{"[2001:4860:4860::8844]:853", true},
-		},
-		"cloudflare-dns.com": {
-			{"[2606:4700:4700::1111]:853", true},
-			{"[2606:4700:4700::1001]:853", true},
-		},
-		"dns.umbrella.com": {
-			{"[2620:0:ccc::2]:853", true},
-			{"[2620:0:ccd::2]:853", true},
-		},
-		"dns10.quad9.net": {
-			{"[2620:fe::10]:853", true},
-			{"[2620:fe::fe:10]:853", true},
-		},
-	},
-}
-
-var dotv4servers = dnsservers{
-	m: map[string][]*dnsstat{
-		"dot.sb": {
-			{"185.222.222.222:853", true},
-			{"45.11.45.11:853", true},
-		},
-		"dns.google": {
-			{"8.8.8.8:853", true},
-			{"8.8.4.4:853", true},
-		},
-		"cloudflare-dns.com": {
-			{"1.1.1.1:853", true},
-			{"1.0.0.1:853", true},
-		},
-		"dns.umbrella.com": {
-			{"208.67.222.222:853", true},
-			{"208.67.220.220:853", true},
-		},
-		"dns10.quad9.net": {
-			{"9.9.9.10:853", true},
-			{"149.112.112.10:853", true},
-		},
-	},
-}
-
-var resolver = &net.Resolver{
-	PreferGo: true,
-	Dial: func(ctx context.Context, _, _ string) (net.Conn, error) {
-		if canUseIPv6.Get() {
-			return dotv6servers.dial(ctx)
-		}
-		return dotv4servers.dial(ctx)
-	},
-}

go.mod

@@ -9,11 +9,13 @@ require (
 )
 
 require (
-	github.com/FloatTech/ttl v0.0.0-20230307105452-d6f7b2b647d1
+	github.com/fumiama/terasu v0.0.0-20240418151245-719e0c16831b
-	github.com/RomiChan/syncx v0.0.0-20221202055724-5f842c53020e
-	github.com/fumiama/terasu v0.0.0-20240416061047-62d3c9f6be80
-	golang.org/x/net v0.24.0
 	golang.org/x/sys v0.19.0
 )
 
-require golang.org/x/text v0.14.0 // indirect
+require (
+	github.com/FloatTech/ttl v0.0.0-20230307105452-d6f7b2b647d1 // indirect
+	github.com/RomiChan/syncx v0.0.0-20240418144900-b7402ffdebc7 // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+)

go.sum

@@ -1,12 +1,12 @@
 github.com/FloatTech/ttl v0.0.0-20230307105452-d6f7b2b647d1 h1:g4pTnDJUW4VbJ9NvoRfUvdjDrHz/6QhfN/LoIIpICbo=
 github.com/FloatTech/ttl v0.0.0-20230307105452-d6f7b2b647d1/go.mod h1:fHZFWGquNXuHttu9dUYoKuNbm3dzLETnIOnm1muSfDs=
-github.com/RomiChan/syncx v0.0.0-20221202055724-5f842c53020e h1:wR3MXQ3VbUlPKOOUwLOYgh/QaJThBTYtsl673O3lqSA=
+github.com/RomiChan/syncx v0.0.0-20240418144900-b7402ffdebc7 h1:S/ferNiehVjNaBMNNBxUjLtVmP/YWD6Yh79RfPv4ehU=
-github.com/RomiChan/syncx v0.0.0-20221202055724-5f842c53020e/go.mod h1:vD7Ra3Q9onRtojoY5sMCLQ7JBgjUsrXDnDKyFxqpf9w=
+github.com/RomiChan/syncx v0.0.0-20240418144900-b7402ffdebc7/go.mod h1:vD7Ra3Q9onRtojoY5sMCLQ7JBgjUsrXDnDKyFxqpf9w=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/fumiama/terasu v0.0.0-20240416061047-62d3c9f6be80 h1:O1JJZzcd5ggUw/9X8V9KxBZ9JZGWFmX/r1q2TPg+pZQ=
+github.com/fumiama/terasu v0.0.0-20240418151245-719e0c16831b h1:j6DMJg+jd4HPmhQtVwtiHBM1y9XskJgWhskUvWuhFuY=
-github.com/fumiama/terasu v0.0.0-20240416061047-62d3c9f6be80/go.mod h1:BFl0X1+rGJf8bLHl/kO+v05ryHrj/R4kyCrK89NvegA=
+github.com/fumiama/terasu v0.0.0-20240418151245-719e0c16831b/go.mod h1:afchyfKAb7J/zvaENtYzjIEPVbwiEjJaow05zzT4usM=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

http.go

@@ -1,81 +0,0 @@
-package main
-
-// https://github.com/fumiama/comandy
-
-import (
-	"context"
-	"crypto/tls"
-	"errors"
-	"net"
-	"net/http"
-	"time"
-
-	"github.com/FloatTech/ttl"
-	"github.com/fumiama/terasu"
-	"golang.org/x/net/http2"
-)
-
-var (
-	errEmptyHostAddress = errors.New("empty host addr")
-)
-
-var httpdialer = net.Dialer{
-	Timeout: time.Minute,
-}
-
-var lookupTable = ttl.NewCache[string, []string](time.Hour)
-
-var cli = http.Client{
-	Transport: &http2.Transport{
-		DialTLSContext: func(ctx context.Context, network, addr string, cfg *tls.Config) (net.Conn, error) {
-			if httpdialer.Timeout != 0 {
-				var cancel context.CancelFunc
-				ctx, cancel = context.WithTimeout(ctx, httpdialer.Timeout)
-				defer cancel()
-			}
-
-			if !httpdialer.Deadline.IsZero() {
-				var cancel context.CancelFunc
-				ctx, cancel = context.WithDeadline(ctx, httpdialer.Deadline)
-				defer cancel()
-			}
-
-			host, port, err := net.SplitHostPort(addr)
-			if err != nil {
-				return nil, err
-			}
-			addrs := lookupTable.Get(host)
-			if len(addrs) == 0 {
-				addrs, err = resolver.LookupHost(ctx, host)
-				if err != nil {
-					addrs, err = net.DefaultResolver.LookupHost(ctx, host)
-					if err != nil {
-						return nil, err
-					}
-				}
-				lookupTable.Set(host, addrs)
-			}
-			if len(addr) == 0 {
-				return nil, errEmptyHostAddress
-			}
-			var tlsConn *tls.Conn
-			for _, a := range addrs {
-				conn, err := httpdialer.DialContext(ctx, network, net.JoinHostPort(a, port))
-				if err != nil {
-					continue
-				}
-				tlsConn = tls.Client(conn, cfg)
-				if usetrs {
-					err = terasu.Use(tlsConn).HandshakeContext(ctx)
-				} else {
-					err = tlsConn.HandshakeContext(ctx)
-				}
-				if err == nil {
-					break
-				}
-				_ = tlsConn.Close()
-			}
-			return tlsConn, err
-		},
-	},
-}

ipv6.go

@@ -1,26 +0,0 @@
-package main
-
-// https://github.com/fumiama/comandy
-
-import (
-	"context"
-	"net/http"
-	"time"
-
-	"github.com/RomiChan/syncx"
-)
-
-var canUseIPv6 = syncx.Lazy[bool]{Init: func() bool {
-	ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
-	defer cancel()
-	req, err := http.NewRequestWithContext(ctx, "GET", "http://v6.ipv6-test.com/json/widgetdata.php?callback=?", nil)
-	if err != nil {
-		return false
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return false
-	}
-	_ = resp.Body.Close()
-	return true
-}}

main.go

@@ -7,6 +7,8 @@ import (
 	"runtime"
 	"time"
 
+	"github.com/fumiama/terasu/dns"
+	"github.com/fumiama/terasu/ip"
 	"github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
 
@@ -15,14 +17,12 @@ import (
 
 //go:generate ./pckcfg.sh assets packs tools
 
-var usetrs = true
-
 const ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 Edg/123.0.0.0"
 
 func main() {
 	logrus.Infoln("RVC Models Downloader start at", time.Now().Local().Format(time.DateTime+" (MST)"))
 	logrus.Infof("operating system: %s, architecture: %s", runtime.GOOS, runtime.GOARCH)
-	logrus.Infoln("can use ipv6:", canUseIPv6.Get())
+	logrus.Infoln("can use ipv6:", ip.IsIPv6Available.Get())
 	ntrs := flag.Bool("notrs", false, "use standard TLS client")
 	dnsf := flag.String("dns", "", "custom dns.yaml")
 	cust := flag.Bool("c", false, "use custom yaml instruction")
@@ -37,9 +37,6 @@ func main() {
 		fmt.Println(cmdlst.String())
 		return
 	}
-	if *ntrs {
-		usetrs = false
-	}
 	if *dnsf != "" {
 		f, err := os.Open(*dnsf)
 		if err != nil {
@@ -53,10 +50,10 @@ func main() {
 			return
 		}
 		_ = f.Close()
-		if canUseIPv6.Get() {
+		if ip.IsIPv6Available.Get() {
-			dotv6servers.add(m)
+			dns.IPv6Servers.Add(m)
 		} else {
-			dotv4servers.add(m)
+			dns.IPv4Servers.Add(m)
 		}
 		fmt.Println("custom dns file added")
 	}
@@ -65,7 +62,7 @@ func main() {
 		logrus.Errorln(err)
 		return
 	}
-	err = usercfg.download(args[0], "", *cust, *force)
+	err = usercfg.download(args[0], "", *cust, !*ntrs, *force)
 	if err != nil {
 		logrus.Errorln(err)
 		return

net.go

@@ -0,0 +1,119 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/fumiama/terasu/http2"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+func (c *config) download(path, prefix string, usecust, usetrs, force bool) error {
+	for i, t := range c.Targets {
+		if t.Refer != "" {
+			refp := path[:strings.LastIndex(path, "/")+1] + t.Refer
+			logrus.Infof("#%s%d refer to target '%s'.", prefix, i+1, refp)
+			refcfg, err := readconfig(refp, usecust)
+			if err != nil {
+				return err
+			}
+			err = refcfg.download(refp, prefix+strconv.Itoa(i+1)+".", usecust, usetrs, force)
+			if err != nil {
+				return err
+			}
+			continue
+		}
+		if t.OS != "" && t.OS != runtime.GOOS {
+			logrus.Warnf("#%s%d target required OS: %s but you are %s, skip.", prefix, i+1, t.OS, runtime.GOOS)
+			continue
+		}
+		if t.Arch != "" && t.Arch != runtime.GOARCH {
+			logrus.Warnf("#%s%d target required Arch: %s but you are %s, skip.", prefix, i+1, t.Arch, runtime.GOARCH)
+			continue
+		}
+		err := os.MkdirAll(t.Folder, 0755)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("#%s%d make target folder '%s'", prefix, i+1, t.Folder))
+		}
+		logrus.Infof("#%s%d open target folder '%s'.", prefix, i+1, t.Folder)
+		if len(t.Copy) == 0 {
+			logrus.Warningf("#%s%d empty copy target.", prefix, i+1)
+			continue
+		}
+		wg := sync.WaitGroup{}
+		wg.Add(len(t.Copy))
+		logrus.Infof("#%s%d download copy: '%v'.", prefix, i+1, t.Copy)
+		for j, cp := range t.Copy {
+			go func(i int, cp, prefix string) {
+				defer wg.Done()
+				if strings.Contains(cp, "/") { // have innner folder
+					infldr := t.Folder + "/" + cp[:strings.LastIndex(cp, "/")]
+					err := os.MkdirAll(infldr, 0755)
+					if err != nil {
+						logrus.Errorf("#%s%d make target inner folder '%s' err: %v", prefix, i+1, t.Folder, err)
+						return
+					}
+					logrus.Infof("#%s%d make target inner folder '%s'.", prefix, i+1, t.Folder)
+				}
+				sleep := time.Millisecond * 100 * time.Duration(i)
+				if sleep > time.Millisecond {
+					time.Sleep(sleep)
+				}
+				fname := t.Folder + "/" + cp
+				if !force {
+					if _, err := os.Stat(fname); err == nil || os.IsExist(err) {
+						logrus.Warnf("#%s%d skip exist file %s", prefix, i+1, fname)
+						return
+					}
+				}
+				req, err := http.NewRequest("GET", c.BaseURL+"/"+cp, nil)
+				if err != nil {
+					logrus.Errorf("#%s%d new request to %s err: %v", prefix, i+1, cp, err)
+					return
+				}
+				logrus.Infof("#%s%d get: %s", prefix, i+1, req.URL)
+				req.Header.Add("user-agent", ua)
+				var resp *http.Response
+				if usetrs {
+					resp, err = http2.DefaultClient.Do(req)
+				} else {
+					resp, err = http.DefaultClient.Do(req)
+				}
+				if err != nil {
+					logrus.Errorf("#%s%d get %s err: %v", prefix, i+1, req.URL, err)
+					return
+				}
+				defer resp.Body.Close()
+				if resp.StatusCode != http.StatusOK {
+					err := errors.New(fmt.Sprintf("HTTP %d %s", resp.StatusCode, resp.Status))
+					logrus.Errorf("#%s%d get %s err: %v", prefix, i+1, req.URL, err)
+					return
+				}
+				f, err := os.Create(fname)
+				if err != nil {
+					logrus.Errorf("#%s%d create file %s err: %v", prefix, i+1, fname, err)
+					return
+				}
+				defer f.Close()
+				logrus.Infof("#%s%d writing file %s", prefix, i+1, fname)
+				pm := newmeter(fmt.Sprintf("#%s%d", prefix, i+1), fname, int(resp.ContentLength))
+				_, err = io.Copy(io.MultiWriter(f, &pm), resp.Body)
+				if err != nil {
+					logrus.Errorf("#%s%d download file %s err: %v", prefix, i+1, fname, err)
+					return
+				}
+				logrus.Infof("#%s%d finished download %s", prefix, i+1, fname)
+			}(j, cp, fmt.Sprintf("%s%d.", prefix, i+1))
+		}
+		wg.Wait()
+	}
+	return nil
+}