diff --git a/README.md b/README.md index 1e28bbc..562b596 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,21 @@ # imago -Image saving & comparing tool for go. +Image saving & comparing tool for go based on webp. + +## Functions +### func Str2bytes(s string) []byte +Fast convert +### func Bytes2str(b []byte) string +Fast convert +### func GetDHashStr(img image.Image) (string, error) +Get image dhash encoded by [go-base16384](https://github.com/fumiama/go-base16384) +### func HammDistance(img1 string, img2 string) (int, error) +Get hamming distance between two dhash strings +### func Scanimgs(imgdir string) error +Scan all images like 编码后哈希.webp +### func Pick(exclude []string) string +Pick a random image +### func Saveimgbytes(b []byte, imgdir string, uid string, force bool) string +### func Saveimg(r io.Reader, imgdir string, uid string) string +Save image into imgdir with name like 编码后哈希.webp +### func Addimage(name string) +manually add an image name into map \ No newline at end of file diff --git a/data.go b/data.go new file mode 100644 index 0000000..fc039ee --- /dev/null +++ b/data.go @@ -0,0 +1,34 @@ +package imago + +import ( + "bytes" + "encoding/binary" + "fmt" + "unsafe" +) + +// Str2bytes Fast convert +func Str2bytes(s string) []byte { + x := (*[2]uintptr)(unsafe.Pointer(&s)) + h := [3]uintptr{x[0], x[1], x[1]} + return *(*[]byte)(unsafe.Pointer(&h)) +} + +// Bytes2str Fast convert +func Bytes2str(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +} + +// bytes82uint64 字节数(大端)组转成int(无符号的) +func bytes82uint64(b []byte) (uint64, error) { + if len(b) == 9 { + b = b[:7] + } + if len(b) == 8 { + bytesBuffer := bytes.NewBuffer(b) + var tmp uint64 + err := binary.Read(bytesBuffer, binary.BigEndian, &tmp) + return tmp, err + } + return 0, fmt.Errorf("%s", "bytes lenth is invaild!") +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d9cfb18 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module github.com/fumiama/imago + +go 1.16 + +require ( + github.com/corona10/goimagehash v1.0.3 + github.com/fumiama/go-base16384 v1.1.0 + github.com/kolesa-team/go-webp v1.0.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..088ec56 --- /dev/null +++ b/go.sum @@ -0,0 +1,13 @@ +github.com/corona10/goimagehash v1.0.3 h1:NZM518aKLmoNluluhfHGxT3LGOnrojrxhGn63DR/CZA= +github.com/corona10/goimagehash v1.0.3/go.mod h1:VkvE0mLn84L4aF8vCb6mafVajEb6QYMHl2ZJLn0mOGI= +github.com/fumiama/go-base16384 v1.1.0 h1:4npregEpgJ4HtHYNMcw7eabSSO8GIqfG3xGZWtcYcIA= +github.com/fumiama/go-base16384 v1.1.0/go.mod h1:YCaxEQyBX717X2lMlJvdccNhA83zhKRR/ipPOIIIQj8= +github.com/kolesa-team/go-webp v1.0.0 h1:HFE7NhW1jkM59HgUwRdcSJ+VVZz7F1RZk6VFY27YZpQ= +github.com/kolesa-team/go-webp v1.0.0/go.mod h1:P0sDD6OLl4jBVcwCcYRuu1C+kUrR4V19LYWs+yV3UcY= +github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= +github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d h1:RNPAfi2nHY7C2srAV8A49jpsYr0ADedCk1wq6fTMTvs= +golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/imgdiff.go b/imgdiff.go new file mode 100644 index 0000000..dcc3774 --- /dev/null +++ b/imgdiff.go @@ -0,0 +1,48 @@ +// Package imago 图片处理相关 +package imago + +import ( + "encoding/binary" + "image" + + "github.com/corona10/goimagehash" + base14 "github.com/fumiama/go-base16384" +) + +var lastchar = "㴁" + +func decodeDHash(imgname string) *goimagehash.ImageHash { + b, err := base14.UTF82utf16be(Str2bytes(imgname + lastchar)) + if err == nil { + dhb := base14.Decode(b) + if dhb != nil { + dh, err1 := bytes82uint64(dhb) + base14.Free(dhb) + if err1 == nil { + return goimagehash.NewImageHash(dh, goimagehash.DHash) + } + } + } + return nil +} + +// HammDistance Get hamming distance between two dhash strings +func HammDistance(img1 string, img2 string) (int, error) { + b1 := decodeDHash(img1) + b2 := decodeDHash(img2) + return b1.Distance(b2) +} + +// GetDHashStr Get image dhash encoded by go-base16384 +func GetDHashStr(img image.Image) (string, error) { + dh, err := goimagehash.DifferenceHash(img) + if err == nil { + data := make([]byte, 8) + binary.BigEndian.PutUint64(data, dh.GetHash()) + e := base14.Encode(data) + b, _ := base14.UTF16be2utf8(e) + base14.Free(e) + return string(b)[:15], nil + } + return "", err +} diff --git a/storage.go b/storage.go new file mode 100644 index 0000000..63c0c0a --- /dev/null +++ b/storage.go @@ -0,0 +1,169 @@ +package imago + +import ( + "bytes" + "fmt" + "image" + "io" + "math/rand" + "net/url" + "os" + "strings" + "sync" + + "github.com/kolesa-team/go-webp/decoder" + "github.com/kolesa-team/go-webp/encoder" + "github.com/kolesa-team/go-webp/webp" +) + +var ( + images = make(map[string][]string) + mutex sync.Mutex +) + +func Imgexsits(name string) bool { + index := name[:3] + tail := name[3:] + tails, ok := images[index] + if ok { + found := false + for _, t := range tails { + if tail == t { + found = true + break + } + } + return found + } + return false +} + +// Addimage manually add an image name into map +func Addimage(name string) { + index := name[:3] + tail := name[3:] + mutex.Lock() + defer mutex.Unlock() + if images[index] == nil { + images[index] = make([]string, 0) + fmt.Println("[addimage] create index", index, ".") + } + images[index] = append(images[index], tail) + fmt.Println("[addimage] index", index, "append file", tail, ".") + images["sum"] = append(images["sum"], name) +} + +// Saveimgbytes Save image into imgdir with name like 编码后哈希.webp +func Saveimgbytes(b []byte, imgdir string, uid string, force bool) string { + r := bytes.NewReader(b) + img, _, err := image.Decode(r) + iswebp := false + if err != nil { + r.Seek(0, io.SeekStart) + img, err = webp.Decode(r, &decoder.Options{}) + if err == nil { + iswebp = true + } else { + fmt.Printf("[saveimg] decode image error: %v\n", err) + return "\"stat\": \"notanimg\"" + } + } + dh, err := GetDHashStr(img) + if err != nil { + return "\"stat\": \"dherr\"" + } + if force && Imgexsits(dh) { + return "\"stat\":\"exist\", \"img\": \"" + url.QueryEscape(dh) + "\"" + } else { + for _, name := range images["sum"] { + diff, err := HammDistance(dh, name) + if err == nil && diff < 10 { // 认为是一张图片 + fmt.Printf("[saveimg] old %s.\n", name) + return "\"stat\":\"exist\", \"img\": \"" + url.QueryEscape(name) + "\"" + } + } + } + f, err := os.Create(imgdir + dh + ".webp") + if err != nil { + return "\"stat\": \"ioerr\"" + } + defer f.Close() + if !iswebp { + options, err := encoder.NewLossyEncoderOptions(encoder.PresetDefault, 75) + if err != nil || webp.Encode(f, img, options) != nil { + return "\"stat\": \"encerr\"" + } + } else { + r.Seek(0, io.SeekStart) + c, err := io.Copy(f, r) + if err != nil { + return "\"stat\": \"ioerr\"" + } + fmt.Printf("[saveimg] save %d bytes.\n", c) + } + fmt.Printf("[saveimg] new %s.\n", dh) + return "\"stat\":\"success\", \"img\": \"" + url.QueryEscape(dh) + "\"" +} + +// Saveimg Save image into imgdir with name like 编码后哈希.webp +func Saveimg(r io.Reader, imgdir string, uid string) string { + imgbuff := make([]byte, 1024*1024) // 1m + r.Read(imgbuff) + return Saveimgbytes(imgbuff, imgdir, uid, false) +} + +// Scanimgs Scan all images like 编码后哈希.webp +func Scanimgs(imgdir string) error { + entry, err := os.ReadDir(imgdir) + if err != nil { + return err + } + for _, i := range entry { + if !i.IsDir() { + name := i.Name() + if strings.HasSuffix(name, ".webp") { + name = name[:len(name)-5] + if len([]rune(name)) == 5 { + Addimage(name) + } + } + } + } + return nil +} + +func namein(name string, list []string) bool { + in := false + for _, item := range list { + if name == item { + in = true + break + } + } + return in +} + +// Pick Pick a random image +func Pick(exclude []string) string { + sum := images["sum"] + le := len(exclude) + ls := len(sum) + if le >= ls { + return "" + } else if le == 0 { + return sum[rand.Intn(len(sum))] + } else if ls/le > 10 { + name := sum[rand.Intn(len(sum))] + for namein(name, exclude) { + name = sum[rand.Intn(len(sum))] + } + return name + } else { + for _, n := range sum { + if !namein(n, exclude) { + return n + } + } + return "" + } +}