diff --git a/data.go b/data.go index fc039ee..03f04bf 100644 --- a/data.go +++ b/data.go @@ -1,34 +1,32 @@ package imago import ( - "bytes" - "encoding/binary" - "fmt" "unsafe" ) -// Str2bytes Fast convert -func Str2bytes(s string) []byte { - x := (*[2]uintptr)(unsafe.Pointer(&s)) - h := [3]uintptr{x[0], x[1], x[1]} - return *(*[]byte)(unsafe.Pointer(&h)) +// slice is the runtime representation of a slice. +// It cannot be used safely or portably and its representation may +// change in a later release. +// +// Unlike reflect.SliceHeader, its Data field is sufficient to guarantee the +// data it references will not be garbage collected. +type slice struct { + data unsafe.Pointer + len int + cap int } -// Bytes2str Fast convert -func Bytes2str(b []byte) string { +// BytesToString 没有内存开销的转换 +func BytesToString(b []byte) string { return *(*string)(unsafe.Pointer(&b)) } -// bytes82uint64 字节数(大端)组转成int(无符号的) -func bytes82uint64(b []byte) (uint64, error) { - if len(b) == 9 { - b = b[:7] - } - if len(b) == 8 { - bytesBuffer := bytes.NewBuffer(b) - var tmp uint64 - err := binary.Read(bytesBuffer, binary.BigEndian, &tmp) - return tmp, err - } - return 0, fmt.Errorf("%s", "bytes lenth is invaild!") +// StringToBytes 没有内存开销的转换 +func StringToBytes(s string) (b []byte) { + bh := (*slice)(unsafe.Pointer(&b)) + sh := (*slice)(unsafe.Pointer(&s)) // 不要访问 sh.cap + bh.data = sh.data + bh.len = sh.len + bh.cap = sh.len + return b } diff --git a/go.mod b/go.mod index b0ef1d9..6adc993 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.16 require ( github.com/corona10/goimagehash v1.0.3 - github.com/fumiama/go-base16384 v1.1.0 - github.com/kolesa-team/go-webp v1.0.0 + github.com/fumiama/go-base16384 v1.2.1 + github.com/fumiama/simple-storage v0.0.0-20220106063715-c7863e9762dd + github.com/kolesa-team/go-webp v1.0.1 github.com/sirupsen/logrus v1.8.1 - github.com/t-tomalak/logrus-easy-formatter v0.0.0-20190827215021-c074f06c5816 ) diff --git a/go.sum b/go.sum index 7c8bc0e..83ff1b5 100644 --- a/go.sum +++ b/go.sum @@ -1,29 +1,80 @@ +github.com/Mrs4s/MiraiGo v0.0.0-20211120033824-43b23f4e6fcb h1:Rkj28fqIwGx/EgBzRYtpmJRfH6wqVn7cNdc7aJ0QE4M= +github.com/Mrs4s/MiraiGo v0.0.0-20211120033824-43b23f4e6fcb/go.mod h1:imVKbfKqqeit+C/eaWGb4MKQ3z3gN6pRpBU5RMtp5so= github.com/corona10/goimagehash v1.0.3 h1:NZM518aKLmoNluluhfHGxT3LGOnrojrxhGn63DR/CZA= github.com/corona10/goimagehash v1.0.3/go.mod h1:VkvE0mLn84L4aF8vCb6mafVajEb6QYMHl2ZJLn0mOGI= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fumiama/go-base16384 v1.1.0 h1:4npregEpgJ4HtHYNMcw7eabSSO8GIqfG3xGZWtcYcIA= -github.com/fumiama/go-base16384 v1.1.0/go.mod h1:YCaxEQyBX717X2lMlJvdccNhA83zhKRR/ipPOIIIQj8= -github.com/kolesa-team/go-webp v1.0.0 h1:HFE7NhW1jkM59HgUwRdcSJ+VVZz7F1RZk6VFY27YZpQ= -github.com/kolesa-team/go-webp v1.0.0/go.mod h1:P0sDD6OLl4jBVcwCcYRuu1C+kUrR4V19LYWs+yV3UcY= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fumiama/go-base16384 v1.2.1 h1:6OGprW8g/95m2ocmryHi8mipZ7bx9StFMZDKEqLvMiA= +github.com/fumiama/go-base16384 v1.2.1/go.mod h1:1HTC0QFL7BjS0DuO5Qm+fBYKQkHqmAapLbRpCxrhPXQ= +github.com/fumiama/go-hide-param v0.1.4/go.mod h1:vJkQlJIEI56nIyp7tCQu1/2QOyKtZpudsnJkGk9U1aY= +github.com/fumiama/gofastTEA v0.0.6 h1:Yni3MXDbJVa/c4CecgdZDgCJK+fLdvGph+OBqY2mtiI= +github.com/fumiama/gofastTEA v0.0.6/go.mod h1:+sBZ05nCA2skZkursHNvyr8kULlEetrYTM2y5kA4rQc= +github.com/fumiama/simple-storage v0.0.0-20220106063715-c7863e9762dd h1:29uEN7QH58h05zAK/xYoWqY/8qQWATcQ11qzgcLqNE8= +github.com/fumiama/simple-storage v0.0.0-20220106063715-c7863e9762dd/go.mod h1:ZL+V3VULLrhLhjio/qoG8w1C7E0dr2AnsmkI35IUoY0= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/kolesa-team/go-webp v1.0.1 h1:Btojkbzr6tt10zJ40xlbSfJeHFiNn0aR7H03QUqmMoI= +github.com/kolesa-team/go-webp v1.0.1/go.mod h1:oMvdivD6K+Q5qIIkVC2w4k2ZUnI1H+MyP7inwgWq9aA= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/t-tomalak/logrus-easy-formatter v0.0.0-20190827215021-c074f06c5816 h1:J6v8awz+me+xeb/cUTotKgceAYouhIB3pjzgRd6IlGk= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/t-tomalak/logrus-easy-formatter v0.0.0-20190827215021-c074f06c5816/go.mod h1:tzym/CEb5jnFI+Q0k4Qq3+LvRF4gO3E2pxS8fHP8jcA= +github.com/tidwall/gjson v1.8.0/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk= +github.com/tidwall/gjson v1.11.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/wdvxdr1123/ZeroBot v1.3.2 h1:EFZNb3awNbwxRtmDkWv3PH6Z9rUV6ZLFa3hBmRMRRCA= +github.com/wdvxdr1123/ZeroBot v1.3.2/go.mod h1:i2DIqQjtjE+3gvVi9r9sc+QpNaUuyTXx/HNXXayIpwI= golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d h1:RNPAfi2nHY7C2srAV8A49jpsYr0ADedCk1wq6fTMTvs= golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/imgdiff.go b/imgdiff.go index 46679f4..3126ded 100644 --- a/imgdiff.go +++ b/imgdiff.go @@ -11,16 +11,11 @@ import ( var lastchar = "㴁" func decodeDHash(imgname string) *goimagehash.ImageHash { - b, err := base14.UTF82utf16be(Str2bytes(imgname + lastchar)) + b, err := base14.UTF82utf16be(StringToBytes(imgname + lastchar)) if err == nil { dhb := base14.Decode(b) - if dhb != nil { - dh, err1 := bytes82uint64(dhb) - base14.Free(dhb) - if err1 == nil { - return goimagehash.NewImageHash(dh, goimagehash.DHash) - } - } + dh := binary.BigEndian.Uint64(dhb) + return goimagehash.NewImageHash(dh, goimagehash.DHash) } return nil } @@ -36,12 +31,11 @@ func HammDistance(img1 string, img2 string) (int, error) { func GetDHashStr(img image.Image) (string, error) { dh, err := goimagehash.DifferenceHash(img) if err == nil { - data := make([]byte, 8) - binary.BigEndian.PutUint64(data, dh.GetHash()) - e := base14.Encode(data) + var data [8]byte + binary.BigEndian.PutUint64(data[:], dh.GetHash()) + e := base14.Encode(data[:]) b, _ := base14.UTF16be2utf8(e) - base14.Free(e) - return string(b)[:15], nil + return BytesToString(b)[:15], nil } return "", err } diff --git a/storage.go b/storage.go index d50154a..ca69e83 100644 --- a/storage.go +++ b/storage.go @@ -2,45 +2,33 @@ package imago import ( - "bytes" - "image" "io" "math/rand" - "net/url" - "os" - "strings" "sync" - "github.com/kolesa-team/go-webp/decoder" - "github.com/kolesa-team/go-webp/encoder" - "github.com/kolesa-team/go-webp/webp" log "github.com/sirupsen/logrus" - easy "github.com/t-tomalak/logrus-easy-formatter" ) -var ( - images = make(map[string][]string) - mutex sync.Mutex -) - -func init() { - log.SetFormatter(&easy.Formatter{ - TimestampFormat: "2006-01-02 15:04:05", - LogFormat: "[imago][%time%][%lvl%]: %msg%\n", - }) - log.SetLevel(log.DebugLevel) +type StorageInstance interface { + GetImgBytes(imgdir, name string) ([]byte, error) + SaveImgBytes(b []byte, imgdir string, force bool, samediff int) (string, string) + SaveImg(r io.Reader, imgdir string, samediff int) (string, string) + ScanImgs(imgdir string) } -// Setloglevel -func Setloglevel(level log.Level) { - log.SetLevel(level) +type storage struct { + images map[string][]string + mutex sync.RWMutex + StorageInstance } -// Imgexsits Return whether the name is in map -func Imgexsits(name string) bool { +// IsImgExsits Return whether the name is in map +func (sr *storage) IsImgExsits(name string) bool { index := name[:3] tail := name[3:] - tails, ok := images[index] + sr.mutex.RLock() + tails, ok := sr.images[index] + sr.mutex.RUnlock() if ok { found := false for _, t := range tails { @@ -54,106 +42,19 @@ func Imgexsits(name string) bool { return false } -// Addimage manually add an image name into map -func Addimage(name string) { +// AddImage manually add an image name into map +func (sr *storage) AddImage(name string) { index := name[:3] tail := name[3:] - mutex.Lock() - defer mutex.Unlock() - if images[index] == nil { - images[index] = make([]string, 0) + sr.mutex.Lock() + defer sr.mutex.Unlock() + if sr.images[index] == nil { + sr.images[index] = make([]string, 0) log.Debugf("[addimage] create index %v.", index) } - images[index] = append(images[index], tail) + sr.images[index] = append(sr.images[index], tail) log.Debugf("[addimage] index %v append file %v.", index, tail) - images["sum"] = append(images["sum"], name) -} - -// Saveimgbytes Save image into imgdir with name like 编码后哈希.webp Return value: status, dhash -func Saveimgbytes(b []byte, imgdir string, force bool, samediff int) (string, string) { - r := bytes.NewReader(b) - img, _, err := image.Decode(r) - iswebp := false - if err != nil { - r.Seek(0, io.SeekStart) - img, err = webp.Decode(r, &decoder.Options{}) - if err == nil { - iswebp = true - } else { - log.Errorf("[saveimg] decode image error: %v", err) - return "\"stat\": \"notanimg\"", "" - } - } - dh, err := GetDHashStr(img) - if err != nil { - log.Errorf("[saveimg] get dhash error: %v", err) - return "\"stat\": \"dherr\"", "" - } - if force { - if Imgexsits(dh) { - log.Debugf("[saveimg] force find similar image %s.", dh) - return "\"stat\":\"exist\", \"img\": \"" + url.QueryEscape(dh) + "\"", dh - } - } else { - for _, name := range images["sum"] { - diff, err := HammDistance(dh, name) - if err == nil && diff <= samediff { // 认为是一张图片 - log.Debugf("[saveimg] old %s.", name) - return "\"stat\":\"exist\", \"img\": \"" + url.QueryEscape(name) + "\"", name - } - } - } - f, err := os.Create(imgdir + dh + ".webp") - if err != nil { - log.Errorf("[saveimg] create webp file error: %v", err) - return "\"stat\": \"ioerr\"", "" - } - defer f.Close() - if !iswebp { - options, err := encoder.NewLossyEncoderOptions(encoder.PresetDefault, 75) - if err != nil || webp.Encode(f, img, options) != nil { - log.Errorf("[saveimg] encode webp error: %v", err) - return "\"stat\": \"encerr\"", "" - } - } else { - r.Seek(0, io.SeekStart) - c, err := io.Copy(f, r) - if err != nil { - log.Errorf("[saveimg] copy file error: %v", err) - return "\"stat\": \"ioerr\"", "" - } - log.Debugf("[saveimg] save %d bytes.", c) - } - log.Debugf("[saveimg] new %s.", dh) - Addimage(dh) - return "\"stat\":\"success\", \"img\": \"" + url.QueryEscape(dh) + "\"", dh -} - -// Saveimg Save image into imgdir with name like 编码后哈希.webp Return value: status, dhash -func Saveimg(r io.Reader, imgdir string, samediff int) (string, string) { - imgbuff := make([]byte, 1024*1024) // 1m - r.Read(imgbuff) - return Saveimgbytes(imgbuff, imgdir, false, samediff) -} - -// Scanimgs Scan all images like 编码后哈希.webp -func Scanimgs(imgdir string) error { - entry, err := os.ReadDir(imgdir) - if err != nil { - return err - } - for _, i := range entry { - if !i.IsDir() { - name := i.Name() - if strings.HasSuffix(name, ".webp") { - name = name[:len(name)-5] - if len([]rune(name)) == 5 { - Addimage(name) - } - } - } - } - return nil + sr.images["sum"] = append(sr.images["sum"], name) } func namein(name string, list []string) bool { @@ -168,8 +69,10 @@ func namein(name string, list []string) bool { } // Pick Pick a random image -func Pick(exclude []string) string { - sum := images["sum"] +func (sr *storage) Pick(exclude []string) string { + sr.mutex.RLock() + sum := sr.images["sum"] + sr.mutex.RUnlock() le := len(exclude) ls := len(sum) if le >= ls { diff --git a/storage_native.go b/storage_native.go new file mode 100644 index 0000000..0890fa6 --- /dev/null +++ b/storage_native.go @@ -0,0 +1,122 @@ +package imago + +import ( + "bytes" + "image" + "io" + "net/url" + "os" + "strings" + + "github.com/kolesa-team/go-webp/decoder" + "github.com/kolesa-team/go-webp/encoder" + "github.com/kolesa-team/go-webp/webp" + log "github.com/sirupsen/logrus" +) + +type NativeStorage struct { + storage +} + +func NewNativeStorage() (ns *NativeStorage) { + ns = &NativeStorage{} + ns.images = make(map[string][]string) + return +} + +// GetImgBytes Get image from apiurl with name like 编码后哈希.webp Return value: imgbytes, error +func (*NativeStorage) GetImgBytes(imgdir, name string) ([]byte, error) { + return os.ReadFile(imgdir + "/" + name) +} + +// SaveImgBytes Save image into imgdir with name like 编码后哈希.webp Return value: status, dhash +func (ns *NativeStorage) SaveImgBytes(b []byte, imgdir string, force bool, samediff int) (string, string) { + r := bytes.NewReader(b) + img, _, err := image.Decode(r) + iswebp := false + if err != nil { + r.Seek(0, io.SeekStart) + img, err = webp.Decode(r, &decoder.Options{}) + if err == nil { + iswebp = true + } else { + log.Errorf("[saveimg] decode image error: %v", err) + return "\"stat\": \"notanimg\"", "" + } + } + dh, err := GetDHashStr(img) + if err != nil { + log.Errorf("[saveimg] get dhash error: %v", err) + return "\"stat\": \"dherr\"", "" + } + if force { + if ns.IsImgExsits(dh) { + log.Debugf("[saveimg] force find similar image %s.", dh) + return "\"stat\":\"exist\", \"img\": \"" + url.QueryEscape(dh) + "\"", dh + } + } else { + ns.mutex.RLock() + s := ns.images["sum"] + ns.mutex.RUnlock() + for _, name := range s { + diff, err := HammDistance(dh, name) + if err == nil && diff <= samediff { // 认为是一张图片 + log.Debugf("[saveimg] old %s.", name) + return "\"stat\":\"exist\", \"img\": \"" + url.QueryEscape(name) + "\"", name + } + } + } + f, err := os.Create(imgdir + "/" + dh + ".webp") + if err != nil { + log.Errorf("[saveimg] create webp file error: %v", err) + return "\"stat\": \"ioerr\"", "" + } + defer f.Close() + if !iswebp { + options, err := encoder.NewLossyEncoderOptions(encoder.PresetDefault, 75) + if err != nil || webp.Encode(f, img, options) != nil { + log.Errorf("[saveimg] encode webp error: %v", err) + return "\"stat\": \"encerr\"", "" + } + } else { + r.Seek(0, io.SeekStart) + c, err := io.Copy(f, r) + if err != nil { + log.Errorf("[saveimg] copy file error: %v", err) + return "\"stat\": \"ioerr\"", "" + } + log.Debugf("[saveimg] save %d bytes.", c) + } + log.Debugf("[saveimg] new %s.", dh) + ns.AddImage(dh) + return "\"stat\":\"success\", \"img\": \"" + url.QueryEscape(dh) + "\"", dh +} + +// SaveImg Save image into imgdir with name like 编码后哈希.webp Return value: status, dhash +func (n *NativeStorage) SaveImg(r io.Reader, imgdir string, samediff int) (string, string) { + imgbuff, err := io.ReadAll(r) + if err != nil { + return "\"stat\": \"ioerr\"", "" + } + return n.SaveImgBytes(imgbuff, imgdir, false, samediff) +} + +// ScanImgs Scan all images like 编码后哈希.webp +func (ns *NativeStorage) ScanImgs(imgdir string) error { + entry, err := os.ReadDir(imgdir) + if err != nil { + return err + } + for _, i := range entry { + if !i.IsDir() { + name := i.Name() + if strings.HasSuffix(name, ".webp") { + name = name[:len(name)-5] + if len([]rune(name)) == 5 { + ns.AddImage(name) + } + } + } + } + return nil +} diff --git a/storage_remote.go b/storage_remote.go new file mode 100644 index 0000000..2aea801 --- /dev/null +++ b/storage_remote.go @@ -0,0 +1,125 @@ +package imago + +import ( + "bytes" + "image" + "io" + "net/url" + "strings" + + "github.com/fumiama/simple-storage/client" + "github.com/kolesa-team/go-webp/decoder" + "github.com/kolesa-team/go-webp/encoder" + "github.com/kolesa-team/go-webp/webp" + log "github.com/sirupsen/logrus" +) + +type RemoteStorage struct { + storage + cli *client.Client +} + +func NewRemoteStorage(apiurl string, key string) (r *RemoteStorage) { + r = &RemoteStorage{cli: client.NewClient(apiurl, key)} + r.images = make(map[string][]string) + return +} + +// GetImgBytes Get image from apiurl with name like 编码后哈希.webp Return value: imgbytes, error +func (remo *RemoteStorage) GetImgBytes(imgdir, name string) (b []byte, e error) { + b, _, e = remo.cli.GetFile(imgdir, name) + return +} + +// SaveImgBytes Save image into apiurl with name like 编码后哈希.webp Return value: status, dhash +func (remo *RemoteStorage) SaveImgBytes(b []byte, imgdir string, force bool, samediff int) (string, string) { + return remo.saveImg(bytes.NewReader(b), imgdir, force, samediff) +} + +// SaveImgBytes Save image into apiurl with name like 编码后哈希.webp Return value: status, dhash +func (remo *RemoteStorage) saveImg(r io.ReadSeeker, imgdir string, force bool, samediff int) (string, string) { + img, _, err := image.Decode(r) + iswebp := false + if err != nil { + r.Seek(0, io.SeekStart) + img, err = webp.Decode(r, &decoder.Options{}) + if err == nil { + iswebp = true + } else { + log.Errorf("[saveimg] decode image error: %v", err) + return "\"stat\": \"notanimg\"", "" + } + } + dh, err := GetDHashStr(img) + if err != nil { + log.Errorf("[saveimg] get dhash error: %v", err) + return "\"stat\": \"dherr\"", "" + } + if force { + if remo.IsImgExsits(dh) { + log.Debugf("[saveimg] force find similar image %s.", dh) + return "\"stat\":\"exist\", \"img\": \"" + url.QueryEscape(dh) + "\"", dh + } + } else { + remo.mutex.RLock() + s := remo.images["sum"] + remo.mutex.RUnlock() + for _, name := range s { + diff, err := HammDistance(dh, name) + if err == nil && diff <= samediff { // 认为是一张图片 + log.Debugf("[saveimg] old %s.", name) + return "\"stat\":\"exist\", \"img\": \"" + url.QueryEscape(name) + "\"", name + } + } + } + f := new(bytes.Buffer) + if err != nil { + log.Errorf("[saveimg] create webp file error: %v", err) + return "\"stat\": \"ioerr\"", "" + } + if !iswebp { + options, err := encoder.NewLossyEncoderOptions(encoder.PresetDefault, 75) + if err != nil || webp.Encode(f, img, options) != nil { + log.Errorf("[saveimg] encode webp error: %v", err) + return "\"stat\": \"encerr\"", "" + } + } else { + r.Seek(0, io.SeekStart) + c, err := io.Copy(f, r) + if err != nil { + log.Errorf("[saveimg] copy file error: %v", err) + return "\"stat\": \"ioerr\"", "" + } + log.Debugf("[saveimg] save %d bytes.", c) + } + log.Debugf("[saveimg] new %s.", dh) + remo.AddImage(dh) + _ = remo.cli.SetFile(imgdir, dh+".webp", f.Bytes()) + return "\"stat\":\"success\", \"img\": \"" + url.QueryEscape(dh) + "\"", dh +} + +// SaveImg Save image into apiurl with name like 编码后哈希.webp Return value: status, dhash +func (remo *RemoteStorage) SaveImg(r io.Reader, imgdir string, samediff int) (string, string) { + imgbuff, err := io.ReadAll(r) + if err != nil { + return "\"stat\": \"ioerr\"", "" + } + return remo.saveImg(bytes.NewReader(imgbuff), imgdir, false, samediff) +} + +// ScanImgs Scan all images like 编码后哈希.webp from apiurl +func (remo *RemoteStorage) ScanImgs(imgdir string) error { + m, err := remo.cli.ListFiles(imgdir) + if err != nil { + return err + } + for name := range m { + if strings.HasSuffix(name, ".webp") { + name = name[:len(name)-5] + if len([]rune(name)) == 5 { + remo.AddImage(name) + } + } + } + return nil +}