mirror of
https://github.com/fumiama/RVC-Models-Downloader.git
synced 2024-09-29 14:26:25 +09:00
feat(trs): new implementation
This commit is contained in:
源文雨
parent
c494c77c1c
commit
793c9f5d5d
132
cfg.go
132
cfg.go
@ -4,21 +4,25 @@ import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type config struct {
|
||||
BaseURL string `yaml:"BaseURL"`
|
||||
Targets []targets `yaml:"Targets"`
|
||||
}
|
||||
|
||||
type targets struct {
|
||||
Refer string `yaml:"Refer"`
|
||||
Folder string `yaml:"Folder"`
|
||||
Copy []string `yaml:"Copy"`
|
||||
OS string `yaml:"OS"`
|
||||
Arch string `yaml:"Arch"`
|
||||
}
|
||||
|
||||
//go:embed cfg.zip
|
||||
var cfgdata []byte
|
||||
|
||||
@ -51,113 +55,3 @@ func readconfig(path string, usecust bool) (c config, err error) {
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type config struct {
|
||||
BaseURL string `yaml:"BaseURL"`
|
||||
Targets []targets `yaml:"Targets"`
|
||||
}
|
||||
|
||||
type targets struct {
|
||||
Refer string `yaml:"Refer"`
|
||||
Folder string `yaml:"Folder"`
|
||||
Copy []string `yaml:"Copy"`
|
||||
OS string `yaml:"OS"`
|
||||
Arch string `yaml:"Arch"`
|
||||
}
|
||||
|
||||
func (c *config) download(path, prefix string, usecust, force bool) error {
|
||||
for i, t := range c.Targets {
|
||||
if t.Refer != "" {
|
||||
refp := path[:strings.LastIndex(path, "/")+1] + t.Refer
|
||||
logrus.Infof("#%s%d refer to target '%s'.", prefix, i+1, refp)
|
||||
refcfg, err := readconfig(refp, usecust)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = refcfg.download(refp, prefix+strconv.Itoa(i+1)+".", usecust, force)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if t.OS != "" && t.OS != runtime.GOOS {
|
||||
logrus.Warnf("#%s%d target required OS: %s but you are %s, skip.", prefix, i+1, t.OS, runtime.GOOS)
|
||||
continue
|
||||
}
|
||||
if t.Arch != "" && t.Arch != runtime.GOARCH {
|
||||
logrus.Warnf("#%s%d target required Arch: %s but you are %s, skip.", prefix, i+1, t.Arch, runtime.GOARCH)
|
||||
continue
|
||||
}
|
||||
err := os.MkdirAll(t.Folder, 0755)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, fmt.Sprintf("#%s%d make target folder '%s'", prefix, i+1, t.Folder))
|
||||
}
|
||||
logrus.Infof("#%s%d open target folder '%s'.", prefix, i+1, t.Folder)
|
||||
if len(t.Copy) == 0 {
|
||||
logrus.Warningf("#%s%d empty copy target.", prefix, i+1)
|
||||
continue
|
||||
}
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(len(t.Copy))
|
||||
logrus.Infof("#%s%d download copy: '%v'.", prefix, i+1, t.Copy)
|
||||
for j, cp := range t.Copy {
|
||||
go func(i int, cp, prefix string) {
|
||||
defer wg.Done()
|
||||
if strings.Contains(cp, "/") { // have innner folder
|
||||
infldr := t.Folder + "/" + cp[:strings.LastIndex(cp, "/")]
|
||||
err := os.MkdirAll(infldr, 0755)
|
||||
if err != nil {
|
||||
logrus.Errorf("#%s%d make target inner folder '%s' err: %v", prefix, i+1, t.Folder, err)
|
||||
return
|
||||
}
|
||||
logrus.Infof("#%s%d make target inner folder '%s'.", prefix, i+1, t.Folder)
|
||||
}
|
||||
sleep := time.Millisecond * 100 * time.Duration(i)
|
||||
if sleep > time.Millisecond {
|
||||
time.Sleep(sleep)
|
||||
}
|
||||
fname := t.Folder + "/" + cp
|
||||
if !force {
|
||||
if _, err := os.Stat(fname); err == nil || os.IsExist(err) {
|
||||
logrus.Warnf("#%s%d skip exist file %s", prefix, i+1, fname)
|
||||
return
|
||||
}
|
||||
}
|
||||
req, err := http.NewRequest("GET", c.BaseURL+"/"+cp, nil)
|
||||
if err != nil {
|
||||
logrus.Errorf("#%s%d new request to %s err: %v", prefix, i+1, cp, err)
|
||||
return
|
||||
}
|
||||
logrus.Infof("#%s%d get: %s", prefix, i+1, req.URL)
|
||||
req.Header.Add("user-agent", ua)
|
||||
resp, err := cli.Do(req)
|
||||
if err != nil {
|
||||
logrus.Errorf("#%s%d get %s err: %v", prefix, i+1, req.URL, err)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
err := errors.New(fmt.Sprintf("HTTP %d %s", resp.StatusCode, resp.Status))
|
||||
logrus.Errorf("#%s%d get %s err: %v", prefix, i+1, req.URL, err)
|
||||
return
|
||||
}
|
||||
f, err := os.Create(fname)
|
||||
if err != nil {
|
||||
logrus.Errorf("#%s%d create file %s err: %v", prefix, i+1, fname, err)
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
logrus.Infof("#%s%d writing file %s", prefix, i+1, fname)
|
||||
pm := newmeter(fmt.Sprintf("#%s%d", prefix, i+1), fname, int(resp.ContentLength))
|
||||
_, err = io.Copy(io.MultiWriter(f, &pm), resp.Body)
|
||||
if err != nil {
|
||||
logrus.Errorf("#%s%d download file %s err: %v", prefix, i+1, fname, err)
|
||||
return
|
||||
}
|
||||
logrus.Infof("#%s%d finished download %s", prefix, i+1, fname)
|
||||
}(j, cp, fmt.Sprintf("%s%d.", prefix, i+1))
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
163
dns.go
163
dns.go
@ -1,163 +0,0 @@
|
||||
package main
|
||||
|
||||
// https://github.com/fumiama/comandy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/fumiama/terasu"
|
||||
)
|
||||
|
||||
var (
|
||||
errNoDNSAvailable = errors.New("no dns available")
|
||||
)
|
||||
|
||||
var dnsdialer = net.Dialer{
|
||||
Timeout: time.Second * 8,
|
||||
}
|
||||
|
||||
type dnsstat struct {
|
||||
A string
|
||||
E bool
|
||||
}
|
||||
|
||||
type dnsservers struct {
|
||||
sync.RWMutex
|
||||
m map[string][]*dnsstat
|
||||
}
|
||||
|
||||
// hasrecord no lock, use under lock
|
||||
func hasrecord(lst []*dnsstat, a string) bool {
|
||||
for _, addr := range lst {
|
||||
if addr.A == a {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (ds *dnsservers) add(m map[string][]string) {
|
||||
ds.Lock()
|
||||
defer ds.Unlock()
|
||||
addList := map[string][]*dnsstat{}
|
||||
for host, addrs := range m {
|
||||
for _, addr := range addrs {
|
||||
if !hasrecord(ds.m[host], addr) && !hasrecord(addList[host], addr) {
|
||||
addList[host] = append(addList[host], &dnsstat{addr, true})
|
||||
}
|
||||
}
|
||||
}
|
||||
for host, addrs := range addList {
|
||||
ds.m[host] = append(ds.m[host], addrs...)
|
||||
}
|
||||
}
|
||||
|
||||
func (ds *dnsservers) dial(ctx context.Context) (tlsConn *tls.Conn, err error) {
|
||||
err = errNoDNSAvailable
|
||||
|
||||
ds.RLock()
|
||||
defer ds.RUnlock()
|
||||
|
||||
if dnsdialer.Timeout != 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, dnsdialer.Timeout)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
if !dnsdialer.Deadline.IsZero() {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithDeadline(ctx, dnsdialer.Deadline)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
var conn net.Conn
|
||||
for host, addrs := range ds.m {
|
||||
for _, addr := range addrs {
|
||||
if !addr.E {
|
||||
continue
|
||||
}
|
||||
conn, err = dnsdialer.DialContext(ctx, "tcp", addr.A)
|
||||
if err != nil {
|
||||
addr.E = false // no need to acquire write lock
|
||||
continue
|
||||
}
|
||||
tlsConn = tls.Client(conn, &tls.Config{ServerName: host})
|
||||
if usetrs {
|
||||
err = terasu.Use(tlsConn).HandshakeContext(ctx)
|
||||
} else {
|
||||
err = tlsConn.HandshakeContext(ctx)
|
||||
}
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
_ = tlsConn.Close()
|
||||
addr.E = false // no need to acquire write lock
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
var dotv6servers = dnsservers{
|
||||
m: map[string][]*dnsstat{
|
||||
"dot.sb": {
|
||||
{"[2a09::]:853", true},
|
||||
{"[2a11::]:853", true},
|
||||
},
|
||||
"dns.google": {
|
||||
{"[2001:4860:4860::8888]:853", true},
|
||||
{"[2001:4860:4860::8844]:853", true},
|
||||
},
|
||||
"cloudflare-dns.com": {
|
||||
{"[2606:4700:4700::1111]:853", true},
|
||||
{"[2606:4700:4700::1001]:853", true},
|
||||
},
|
||||
"dns.umbrella.com": {
|
||||
{"[2620:0:ccc::2]:853", true},
|
||||
{"[2620:0:ccd::2]:853", true},
|
||||
},
|
||||
"dns10.quad9.net": {
|
||||
{"[2620:fe::10]:853", true},
|
||||
{"[2620:fe::fe:10]:853", true},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var dotv4servers = dnsservers{
|
||||
m: map[string][]*dnsstat{
|
||||
"dot.sb": {
|
||||
{"185.222.222.222:853", true},
|
||||
{"45.11.45.11:853", true},
|
||||
},
|
||||
"dns.google": {
|
||||
{"8.8.8.8:853", true},
|
||||
{"8.8.4.4:853", true},
|
||||
},
|
||||
"cloudflare-dns.com": {
|
||||
{"1.1.1.1:853", true},
|
||||
{"1.0.0.1:853", true},
|
||||
},
|
||||
"dns.umbrella.com": {
|
||||
{"208.67.222.222:853", true},
|
||||
{"208.67.220.220:853", true},
|
||||
},
|
||||
"dns10.quad9.net": {
|
||||
{"9.9.9.10:853", true},
|
||||
{"149.112.112.10:853", true},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var resolver = &net.Resolver{
|
||||
PreferGo: true,
|
||||
Dial: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
if canUseIPv6.Get() {
|
||||
return dotv6servers.dial(ctx)
|
||||
}
|
||||
return dotv4servers.dial(ctx)
|
||||
},
|
||||
}
|
||||
12
go.mod
12
go.mod
@ -9,11 +9,13 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/FloatTech/ttl v0.0.0-20230307105452-d6f7b2b647d1
|
||||
github.com/RomiChan/syncx v0.0.0-20221202055724-5f842c53020e
|
||||
github.com/fumiama/terasu v0.0.0-20240416061047-62d3c9f6be80
|
||||
golang.org/x/net v0.24.0
|
||||
github.com/fumiama/terasu v0.0.0-20240418151245-719e0c16831b
|
||||
golang.org/x/sys v0.19.0
|
||||
)
|
||||
|
||||
require golang.org/x/text v0.14.0 // indirect
|
||||
require (
|
||||
github.com/FloatTech/ttl v0.0.0-20230307105452-d6f7b2b647d1 // indirect
|
||||
github.com/RomiChan/syncx v0.0.0-20240418144900-b7402ffdebc7 // indirect
|
||||
golang.org/x/net v0.24.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
)
|
||||
|
||||
8
go.sum
8
go.sum
@ -1,12 +1,12 @@
|
||||
github.com/FloatTech/ttl v0.0.0-20230307105452-d6f7b2b647d1 h1:g4pTnDJUW4VbJ9NvoRfUvdjDrHz/6QhfN/LoIIpICbo=
|
||||
github.com/FloatTech/ttl v0.0.0-20230307105452-d6f7b2b647d1/go.mod h1:fHZFWGquNXuHttu9dUYoKuNbm3dzLETnIOnm1muSfDs=
|
||||
github.com/RomiChan/syncx v0.0.0-20221202055724-5f842c53020e h1:wR3MXQ3VbUlPKOOUwLOYgh/QaJThBTYtsl673O3lqSA=
|
||||
github.com/RomiChan/syncx v0.0.0-20221202055724-5f842c53020e/go.mod h1:vD7Ra3Q9onRtojoY5sMCLQ7JBgjUsrXDnDKyFxqpf9w=
|
||||
github.com/RomiChan/syncx v0.0.0-20240418144900-b7402ffdebc7 h1:S/ferNiehVjNaBMNNBxUjLtVmP/YWD6Yh79RfPv4ehU=
|
||||
github.com/RomiChan/syncx v0.0.0-20240418144900-b7402ffdebc7/go.mod h1:vD7Ra3Q9onRtojoY5sMCLQ7JBgjUsrXDnDKyFxqpf9w=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/fumiama/terasu v0.0.0-20240416061047-62d3c9f6be80 h1:O1JJZzcd5ggUw/9X8V9KxBZ9JZGWFmX/r1q2TPg+pZQ=
|
||||
github.com/fumiama/terasu v0.0.0-20240416061047-62d3c9f6be80/go.mod h1:BFl0X1+rGJf8bLHl/kO+v05ryHrj/R4kyCrK89NvegA=
|
||||
github.com/fumiama/terasu v0.0.0-20240418151245-719e0c16831b h1:j6DMJg+jd4HPmhQtVwtiHBM1y9XskJgWhskUvWuhFuY=
|
||||
github.com/fumiama/terasu v0.0.0-20240418151245-719e0c16831b/go.mod h1:afchyfKAb7J/zvaENtYzjIEPVbwiEjJaow05zzT4usM=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
|
||||
81
http.go
81
http.go
@ -1,81 +0,0 @@
|
||||
package main
|
||||
|
||||
// https://github.com/fumiama/comandy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/FloatTech/ttl"
|
||||
"github.com/fumiama/terasu"
|
||||
"golang.org/x/net/http2"
|
||||
)
|
||||
|
||||
var (
|
||||
errEmptyHostAddress = errors.New("empty host addr")
|
||||
)
|
||||
|
||||
var httpdialer = net.Dialer{
|
||||
Timeout: time.Minute,
|
||||
}
|
||||
|
||||
var lookupTable = ttl.NewCache[string, []string](time.Hour)
|
||||
|
||||
var cli = http.Client{
|
||||
Transport: &http2.Transport{
|
||||
DialTLSContext: func(ctx context.Context, network, addr string, cfg *tls.Config) (net.Conn, error) {
|
||||
if httpdialer.Timeout != 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, httpdialer.Timeout)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
if !httpdialer.Deadline.IsZero() {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithDeadline(ctx, httpdialer.Deadline)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
addrs := lookupTable.Get(host)
|
||||
if len(addrs) == 0 {
|
||||
addrs, err = resolver.LookupHost(ctx, host)
|
||||
if err != nil {
|
||||
addrs, err = net.DefaultResolver.LookupHost(ctx, host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
lookupTable.Set(host, addrs)
|
||||
}
|
||||
if len(addr) == 0 {
|
||||
return nil, errEmptyHostAddress
|
||||
}
|
||||
var tlsConn *tls.Conn
|
||||
for _, a := range addrs {
|
||||
conn, err := httpdialer.DialContext(ctx, network, net.JoinHostPort(a, port))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
tlsConn = tls.Client(conn, cfg)
|
||||
if usetrs {
|
||||
err = terasu.Use(tlsConn).HandshakeContext(ctx)
|
||||
} else {
|
||||
err = tlsConn.HandshakeContext(ctx)
|
||||
}
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
_ = tlsConn.Close()
|
||||
}
|
||||
return tlsConn, err
|
||||
},
|
||||
},
|
||||
}
|
||||
26
ipv6.go
26
ipv6.go
@ -1,26 +0,0 @@
|
||||
package main
|
||||
|
||||
// https://github.com/fumiama/comandy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/RomiChan/syncx"
|
||||
)
|
||||
|
||||
var canUseIPv6 = syncx.Lazy[bool]{Init: func() bool {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "http://v6.ipv6-test.com/json/widgetdata.php?callback=?", nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
return true
|
||||
}}
|
||||
17
main.go
17
main.go
@ -7,6 +7,8 @@ import (
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
"github.com/fumiama/terasu/dns"
|
||||
"github.com/fumiama/terasu/ip"
|
||||
"github.com/sirupsen/logrus"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
@ -15,14 +17,12 @@ import (
|
||||
|
||||
//go:generate ./pckcfg.sh assets packs tools
|
||||
|
||||
var usetrs = true
|
||||
|
||||
const ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 Edg/123.0.0.0"
|
||||
|
||||
func main() {
|
||||
logrus.Infoln("RVC Models Downloader start at", time.Now().Local().Format(time.DateTime+" (MST)"))
|
||||
logrus.Infof("operating system: %s, architecture: %s", runtime.GOOS, runtime.GOARCH)
|
||||
logrus.Infoln("can use ipv6:", canUseIPv6.Get())
|
||||
logrus.Infoln("can use ipv6:", ip.IsIPv6Available.Get())
|
||||
ntrs := flag.Bool("notrs", false, "use standard TLS client")
|
||||
dnsf := flag.String("dns", "", "custom dns.yaml")
|
||||
cust := flag.Bool("c", false, "use custom yaml instruction")
|
||||
@ -37,9 +37,6 @@ func main() {
|
||||
fmt.Println(cmdlst.String())
|
||||
return
|
||||
}
|
||||
if *ntrs {
|
||||
usetrs = false
|
||||
}
|
||||
if *dnsf != "" {
|
||||
f, err := os.Open(*dnsf)
|
||||
if err != nil {
|
||||
@ -53,10 +50,10 @@ func main() {
|
||||
return
|
||||
}
|
||||
_ = f.Close()
|
||||
if canUseIPv6.Get() {
|
||||
dotv6servers.add(m)
|
||||
if ip.IsIPv6Available.Get() {
|
||||
dns.IPv6Servers.Add(m)
|
||||
} else {
|
||||
dotv4servers.add(m)
|
||||
dns.IPv4Servers.Add(m)
|
||||
}
|
||||
fmt.Println("custom dns file added")
|
||||
}
|
||||
@ -65,7 +62,7 @@ func main() {
|
||||
logrus.Errorln(err)
|
||||
return
|
||||
}
|
||||
err = usercfg.download(args[0], "", *cust, *force)
|
||||
err = usercfg.download(args[0], "", *cust, !*ntrs, *force)
|
||||
if err != nil {
|
||||
logrus.Errorln(err)
|
||||
return
|
||||
|
||||
119
net.go
Normal file
119
net.go
Normal file
@ -0,0 +1,119 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/fumiama/terasu/http2"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func (c *config) download(path, prefix string, usecust, usetrs, force bool) error {
|
||||
for i, t := range c.Targets {
|
||||
if t.Refer != "" {
|
||||
refp := path[:strings.LastIndex(path, "/")+1] + t.Refer
|
||||
logrus.Infof("#%s%d refer to target '%s'.", prefix, i+1, refp)
|
||||
refcfg, err := readconfig(refp, usecust)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = refcfg.download(refp, prefix+strconv.Itoa(i+1)+".", usecust, usetrs, force)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if t.OS != "" && t.OS != runtime.GOOS {
|
||||
logrus.Warnf("#%s%d target required OS: %s but you are %s, skip.", prefix, i+1, t.OS, runtime.GOOS)
|
||||
continue
|
||||
}
|
||||
if t.Arch != "" && t.Arch != runtime.GOARCH {
|
||||
logrus.Warnf("#%s%d target required Arch: %s but you are %s, skip.", prefix, i+1, t.Arch, runtime.GOARCH)
|
||||
continue
|
||||
}
|
||||
err := os.MkdirAll(t.Folder, 0755)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, fmt.Sprintf("#%s%d make target folder '%s'", prefix, i+1, t.Folder))
|
||||
}
|
||||
logrus.Infof("#%s%d open target folder '%s'.", prefix, i+1, t.Folder)
|
||||
if len(t.Copy) == 0 {
|
||||
logrus.Warningf("#%s%d empty copy target.", prefix, i+1)
|
||||
continue
|
||||
}
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(len(t.Copy))
|
||||
logrus.Infof("#%s%d download copy: '%v'.", prefix, i+1, t.Copy)
|
||||
for j, cp := range t.Copy {
|
||||
go func(i int, cp, prefix string) {
|
||||
defer wg.Done()
|
||||
if strings.Contains(cp, "/") { // have innner folder
|
||||
infldr := t.Folder + "/" + cp[:strings.LastIndex(cp, "/")]
|
||||
err := os.MkdirAll(infldr, 0755)
|
||||
if err != nil {
|
||||
logrus.Errorf("#%s%d make target inner folder '%s' err: %v", prefix, i+1, t.Folder, err)
|
||||
return
|
||||
}
|
||||
logrus.Infof("#%s%d make target inner folder '%s'.", prefix, i+1, t.Folder)
|
||||
}
|
||||
sleep := time.Millisecond * 100 * time.Duration(i)
|
||||
if sleep > time.Millisecond {
|
||||
time.Sleep(sleep)
|
||||
}
|
||||
fname := t.Folder + "/" + cp
|
||||
if !force {
|
||||
if _, err := os.Stat(fname); err == nil || os.IsExist(err) {
|
||||
logrus.Warnf("#%s%d skip exist file %s", prefix, i+1, fname)
|
||||
return
|
||||
}
|
||||
}
|
||||
req, err := http.NewRequest("GET", c.BaseURL+"/"+cp, nil)
|
||||
if err != nil {
|
||||
logrus.Errorf("#%s%d new request to %s err: %v", prefix, i+1, cp, err)
|
||||
return
|
||||
}
|
||||
logrus.Infof("#%s%d get: %s", prefix, i+1, req.URL)
|
||||
req.Header.Add("user-agent", ua)
|
||||
var resp *http.Response
|
||||
if usetrs {
|
||||
resp, err = http2.DefaultClient.Do(req)
|
||||
} else {
|
||||
resp, err = http.DefaultClient.Do(req)
|
||||
}
|
||||
if err != nil {
|
||||
logrus.Errorf("#%s%d get %s err: %v", prefix, i+1, req.URL, err)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
err := errors.New(fmt.Sprintf("HTTP %d %s", resp.StatusCode, resp.Status))
|
||||
logrus.Errorf("#%s%d get %s err: %v", prefix, i+1, req.URL, err)
|
||||
return
|
||||
}
|
||||
f, err := os.Create(fname)
|
||||
if err != nil {
|
||||
logrus.Errorf("#%s%d create file %s err: %v", prefix, i+1, fname, err)
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
logrus.Infof("#%s%d writing file %s", prefix, i+1, fname)
|
||||
pm := newmeter(fmt.Sprintf("#%s%d", prefix, i+1), fname, int(resp.ContentLength))
|
||||
_, err = io.Copy(io.MultiWriter(f, &pm), resp.Body)
|
||||
if err != nil {
|
||||
logrus.Errorf("#%s%d download file %s err: %v", prefix, i+1, fname, err)
|
||||
return
|
||||
}
|
||||
logrus.Infof("#%s%d finished download %s", prefix, i+1, fname)
|
||||
}(j, cp, fmt.Sprintf("%s%d.", prefix, i+1))
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user