diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..c6f358b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,63 @@ +name: release +on: + push: + tags: + - v* + +env: + GITHUB_TOKEN: ${{ github.token }} + +jobs: + my-job: + name: Build DupImage on Push Tag 🚀 + runs-on: ubuntu-latest + steps: + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.18 + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Cache Go + id: cache + uses: actions/cache@v2 + with: + # A list of files, directories, and wildcard patterns to cache and restore + path: ~/go/pkg/mod + key: ${{ runner.os }}-build-${{ hashFiles('**/go.sum') }} + + - name: Tidy Go modules + run: go mod tidy + + - name: Build linux-x64 + run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o artifacts/di-linux-x64 -trimpath + - name: Build linux-x86 + run: CGO_ENABLED=0 GOOS=linux GOARCH=386 go build -ldflags="-s -w" -o artifacts/di-linux-x86 -trimpath + - name: Build windows-x64 + run: CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags="-s -w" -o artifacts/di-windows-x64.exe -trimpath + - name: Build windows-x86 + run: CGO_ENABLED=0 GOOS=windows GOARCH=386 go build -ldflags="-s -w" -o artifacts/di-windows-x86.exe -trimpath + - name: Build arm64 + run: CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GOARM=7 go build -ldflags="-s -w" -o artifacts/di-linux-arm64 -trimpath + - name: Build armv6 + run: CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -ldflags="-s -w" -o artifacts/di-linux-armv6 -trimpath + - name: Build mips + run: CGO_ENABLED=0 GOOS=linux GOARCH=mips go build -ldflags="-s -w" -o artifacts/di-linux-mips -trimpath + - name: Build mips-softfloat + run: CGO_ENABLED=0 GOOS=linux GOARCH=mips GOMIPS=softfloat go build -ldflags="-s -w" -o artifacts/di-linux-mips-softfloat -trimpath + - name: Build mipsel + run: CGO_ENABLED=0 GOOS=linux GOARCH=mipsle go build -ldflags="-s -w" -o artifacts/di-linux-mipsel -trimpath + - name: Build mipsel-softfloat + run: CGO_ENABLED=0 GOOS=linux GOARCH=mipsle GOMIPS=softfloat go build -ldflags="-s -w" -o artifacts/di-linux-mipsel-softfloat -trimpath + + - name: Upload binaries to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/di-* + tag: ${{ github.ref }} + overwrite: true + file_glob: true \ No newline at end of file diff --git a/.gitignore b/.gitignore index 66fd13c..e26f1a4 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ # Dependency directories (remove the comment below to include it) # vendor/ + +/test \ No newline at end of file diff --git a/README.md b/README.md index 2540107..dea3e8f 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,11 @@ # dupimage Detect duplicated images and gather each group into a unique subfolder. + +## usage +```powershell + -a action sort + -d string + work directory (default "./") + -t uint + duplicate throttle, max is 64 (default 5) +``` diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..535920d --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module github.com/fumiama/dupimage + +go 1.18 + +require ( + github.com/corona10/goimagehash v1.1.0 + golang.org/x/image v0.0.0-20220722155232-062f8c9fd539 +) + +require github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..cf7d493 --- /dev/null +++ b/go.sum @@ -0,0 +1,8 @@ +github.com/corona10/goimagehash v1.1.0 h1:teNMX/1e+Wn/AYSbLHX8mj+mF9r60R1kBeqE9MkoYwI= +github.com/corona10/goimagehash v1.1.0/go.mod h1:VkvE0mLn84L4aF8vCb6mafVajEb6QYMHl2ZJLn0mOGI= +github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= +github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +golang.org/x/image v0.0.0-20220722155232-062f8c9fd539 h1:/eM0PCrQI2xd471rI+snWuu251/+/jpBpZqir2mPdnU= +golang.org/x/image v0.0.0-20220722155232-062f8c9fd539/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/main.go b/main.go new file mode 100644 index 0000000..0350002 --- /dev/null +++ b/main.go @@ -0,0 +1,140 @@ +package main + +import ( + "flag" + "fmt" + "image" + "os" + "runtime" + "strconv" + "sync" + + _ "image/gif" + _ "image/jpeg" + _ "image/png" + + _ "golang.org/x/image/webp" + + "github.com/corona10/goimagehash" +) + +type imagecheck struct { + name string + dh *goimagehash.ImageHash +} + +func (ic *imagecheck) String() string { + return ic.name +} + +func main() { + tht := flag.Uint("t", 5, "duplicate throttle, max is 64") + dir := flag.String("d", "./", "work directory") + a := flag.Bool("a", false, "action sort") + flag.Parse() + throttle := *tht + if throttle > 64 { + panic("invalid throttle") + } + err := os.Chdir(*dir) + if err != nil { + panic(err) + } + imgs, err := os.ReadDir("./") + if err != nil { + panic(err) + } + action := *a + chklst := make([]imagecheck, 0, len(imgs)) + fmt.Println("read", len(imgs), "files...") + wg := sync.WaitGroup{} + mu := sync.Mutex{} + part := len(imgs) / runtime.NumCPU() + wg.Add(runtime.NumCPU()) + for i := 0; i < runtime.NumCPU(); i++ { + from := i * part + to := (i + 1) * part + if to > len(imgs) { + to = len(imgs) + } + go func(from, to int) { + for i := from; i < to; i++ { + img := imgs[i] + if !img.IsDir() { + f, err := os.Open(img.Name()) + if err != nil { + panic(err) + } + im, _, err := image.Decode(f) + if err != nil { + panic(err) + } + dh, err := goimagehash.DifferenceHash(im) + if err != nil { + panic(err) + } + mu.Lock() + chklst = append(chklst, imagecheck{ + name: img.Name(), + dh: dh, + }) + fmt.Print("scan: ", len(chklst), " / ", len(imgs), "\r") + mu.Unlock() + _ = f.Close() + } + } + wg.Done() + }(from, to) + } + wg.Wait() + fmt.Println("read file success, comparing...") + dups := make([][]*imagecheck, len(chklst)) + wg.Add(len(chklst)) + for i := 0; i < len(chklst); i++ { + go func(i int) { + for j := len(chklst) - 1; j > i; j-- { + dis, err := chklst[i].dh.Distance(chklst[j].dh) + if err != nil { + panic(err) + } + if uint(dis) < throttle { + mu.Lock() + dups[i] = append(dups[i], &chklst[j]) + mu.Unlock() + } + } + wg.Done() + }(i) + } + wg.Wait() + fmt.Println("compare file success") + hasfound := false + for i, lst := range dups { + if len(lst) > 0 { + dups[i] = append(dups[i], &chklst[i]) + hasfound = true + } + } + if hasfound { + j := 0 + for _, lst := range dups { + if len(lst) > 0 { + j++ + fmt.Println("[", j, "] duplicate: ", lst) + if action { + newdir := strconv.Itoa(j) + err = os.MkdirAll(newdir, 0755) + if err != nil { + panic(err) + } + for _, i := range lst { + err = os.Rename(i.name, newdir+"/"+i.name) + if err != nil { + panic(err) + } + } + } + } + } + } +}