init

.github/workflows/release.yml

@@ -0,0 +1,63 @@
+name: release
+on:
+  push:
+    tags:
+      - v*
+
+env:
+  GITHUB_TOKEN: ${{ github.token }}
+
+jobs:
+  my-job:
+    name: Build DupImage on Push Tag 🚀
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Cache Go
+        id: cache
+        uses: actions/cache@v2
+        with:
+          # A list of files, directories, and wildcard patterns to cache and restore
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-build-${{ hashFiles('**/go.sum') }}
+
+      - name: Tidy Go modules
+        run: go mod tidy
+
+      - name: Build linux-x64
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o artifacts/di-linux-x64 -trimpath
+      - name: Build linux-x86
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=386 go build -ldflags="-s -w" -o artifacts/di-linux-x86 -trimpath
+      - name: Build windows-x64
+        run: CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags="-s -w" -o artifacts/di-windows-x64.exe -trimpath
+      - name: Build windows-x86
+        run: CGO_ENABLED=0 GOOS=windows GOARCH=386 go build -ldflags="-s -w" -o artifacts/di-windows-x86.exe -trimpath
+      - name: Build arm64
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GOARM=7 go build -ldflags="-s -w" -o artifacts/di-linux-arm64 -trimpath
+      - name: Build armv6
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -ldflags="-s -w" -o artifacts/di-linux-armv6 -trimpath
+      - name: Build mips
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=mips go build -ldflags="-s -w" -o artifacts/di-linux-mips -trimpath
+      - name: Build mips-softfloat
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=mips GOMIPS=softfloat go build -ldflags="-s -w" -o artifacts/di-linux-mips-softfloat -trimpath
+      - name: Build mipsel
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=mipsle go build -ldflags="-s -w" -o artifacts/di-linux-mipsel -trimpath
+      - name: Build mipsel-softfloat
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=mipsle GOMIPS=softfloat go build -ldflags="-s -w" -o artifacts/di-linux-mipsel-softfloat -trimpath
+
+      - name: Upload binaries to release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/di-*
+          tag: ${{ github.ref }}
+          overwrite: true
+          file_glob: true

.gitignore

@@ -13,3 +13,5 @@
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
+
+/test

README.md

@@ -1,2 +1,11 @@
 # dupimage
 Detect duplicated images and gather each group into a unique subfolder.
+
+## usage
+```powershell
+  -a    action sort
+  -d string
+        work directory (default "./")
+  -t uint
+        duplicate throttle, max is 64 (default 5)
+```

go.mod

@@ -0,0 +1,10 @@
+module github.com/fumiama/dupimage
+
+go 1.18
+
+require (
+	github.com/corona10/goimagehash v1.1.0
+	golang.org/x/image v0.0.0-20220722155232-062f8c9fd539
+)
+
+require github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect

go.sum

@@ -0,0 +1,8 @@
+github.com/corona10/goimagehash v1.1.0 h1:teNMX/1e+Wn/AYSbLHX8mj+mF9r60R1kBeqE9MkoYwI=
+github.com/corona10/goimagehash v1.1.0/go.mod h1:VkvE0mLn84L4aF8vCb6mafVajEb6QYMHl2ZJLn0mOGI=
+github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
+github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
+golang.org/x/image v0.0.0-20220722155232-062f8c9fd539 h1:/eM0PCrQI2xd471rI+snWuu251/+/jpBpZqir2mPdnU=
+golang.org/x/image v0.0.0-20220722155232-062f8c9fd539/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

main.go

@@ -0,0 +1,140 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"image"
+	"os"
+	"runtime"
+	"strconv"
+	"sync"
+
+	_ "image/gif"
+	_ "image/jpeg"
+	_ "image/png"
+
+	_ "golang.org/x/image/webp"
+
+	"github.com/corona10/goimagehash"
+)
+
+type imagecheck struct {
+	name string
+	dh   *goimagehash.ImageHash
+}
+
+func (ic *imagecheck) String() string {
+	return ic.name
+}
+
+func main() {
+	tht := flag.Uint("t", 5, "duplicate throttle, max is 64")
+	dir := flag.String("d", "./", "work directory")
+	a := flag.Bool("a", false, "action sort")
+	flag.Parse()
+	throttle := *tht
+	if throttle > 64 {
+		panic("invalid throttle")
+	}
+	err := os.Chdir(*dir)
+	if err != nil {
+		panic(err)
+	}
+	imgs, err := os.ReadDir("./")
+	if err != nil {
+		panic(err)
+	}
+	action := *a
+	chklst := make([]imagecheck, 0, len(imgs))
+	fmt.Println("read", len(imgs), "files...")
+	wg := sync.WaitGroup{}
+	mu := sync.Mutex{}
+	part := len(imgs) / runtime.NumCPU()
+	wg.Add(runtime.NumCPU())
+	for i := 0; i < runtime.NumCPU(); i++ {
+		from := i * part
+		to := (i + 1) * part
+		if to > len(imgs) {
+			to = len(imgs)
+		}
+		go func(from, to int) {
+			for i := from; i < to; i++ {
+				img := imgs[i]
+				if !img.IsDir() {
+					f, err := os.Open(img.Name())
+					if err != nil {
+						panic(err)
+					}
+					im, _, err := image.Decode(f)
+					if err != nil {
+						panic(err)
+					}
+					dh, err := goimagehash.DifferenceHash(im)
+					if err != nil {
+						panic(err)
+					}
+					mu.Lock()
+					chklst = append(chklst, imagecheck{
+						name: img.Name(),
+						dh:   dh,
+					})
+					fmt.Print("scan: ", len(chklst), " / ", len(imgs), "\r")
+					mu.Unlock()
+					_ = f.Close()
+				}
+			}
+			wg.Done()
+		}(from, to)
+	}
+	wg.Wait()
+	fmt.Println("read file success, comparing...")
+	dups := make([][]*imagecheck, len(chklst))
+	wg.Add(len(chklst))
+	for i := 0; i < len(chklst); i++ {
+		go func(i int) {
+			for j := len(chklst) - 1; j > i; j-- {
+				dis, err := chklst[i].dh.Distance(chklst[j].dh)
+				if err != nil {
+					panic(err)
+				}
+				if uint(dis) < throttle {
+					mu.Lock()
+					dups[i] = append(dups[i], &chklst[j])
+					mu.Unlock()
+				}
+			}
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+	fmt.Println("compare file success")
+	hasfound := false
+	for i, lst := range dups {
+		if len(lst) > 0 {
+			dups[i] = append(dups[i], &chklst[i])
+			hasfound = true
+		}
+	}
+	if hasfound {
+		j := 0
+		for _, lst := range dups {
+			if len(lst) > 0 {
+				j++
+				fmt.Println("[", j, "] duplicate: ", lst)
+				if action {
+					newdir := strconv.Itoa(j)
+					err = os.MkdirAll(newdir, 0755)
+					if err != nil {
+						panic(err)
+					}
+					for _, i := range lst {
+						err = os.Rename(i.name, newdir+"/"+i.name)
+						if err != nil {
+							panic(err)
+						}
+					}
+				}
+			}
+		}
+	}
+}