fumiama/dupimage
1
0
Fork 0
mirror of https://github.com/fumiama/dupimage.git synced 2026-06-12 22:40:21 +08:00
Code Activity

去重恢复单线程

Browse Source
This commit is contained in:
源文雨
2022-07-25 23:12:34 +08:00
parent 98697edfc8
commit 4fb796cea5
1 changed files with 40 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
main.go
View File

@@ -106,7 +106,7 @@ func main() {
name: n, name: n,
dh: dh, dh: dh,
}) })
fmt.Print("scan: ", len(chklst), " / ", len(imgs), "\r") fmt.Print("read: ", len(chklst), " / ", len(imgs), "\r")
mu.Unlock() mu.Unlock()
_ = f.Close() _ = f.Close()
} }
@@ -118,40 +118,55 @@ func main() {
fmt.Println("read file success, comparing...") fmt.Println("read file success, comparing...")
duplis := make(map[string]uint, len(chklst)) duplis := make(map[string]uint, len(chklst))
sameset := make([][]uint, 0) sameset := make([][]uint, 0)
wg.Add(len(chklst))
for i := 0; i < len(chklst); i++ { for i := 0; i < len(chklst); i++ {
go func(i int) { fmt.Print("compare: ", i, " / ", len(chklst), "\r")
for j := len(chklst) - 1; j > i; j-- { _, ok := duplis[chklst[i].name]
dis, err := chklst[i].dh.Distance(chklst[j].dh) if ok {
if err != nil { continue
fmt.Println("ERROR:", err) }
continue isfirst := true
} for j := len(chklst) - 1; j > i; j-- {
if uint(dis) < throttle { dis, err := chklst[i].dh.Distance(chklst[j].dh)
mu.Lock() if err != nil {
if x, ok := duplis[chklst[j].name]; ok && x != uint(i) { fmt.Println("ERROR:", err)
LOP: continue
for k, set := range sameset { }
for _, item := range set { if uint(dis) < throttle {
if item == x { x, ok := duplis[chklst[j].name]
if ok {
LOP:
for k, set := range sameset {
for _, item := range set {
if x == item {
if isfirst {
sameset[k] = append(sameset[k], uint(i)) sameset[k] = append(sameset[k], uint(i))
duplis[chklst[i].name] = uint(i) duplis[chklst[i].name] = uint(i)
break LOP isfirst = false
} else {
INNERLOP:
for l, set := range sameset {
for _, item := range set {
if item == uint(i) {
sameset[k] = append(sameset[k], set...)
sameset = append(sameset[:l], sameset[l+1:]...)
break INNERLOP
}
}
}
} }
break LOP
} }
} }
} else if !ok {
sameset = append(sameset, []uint{uint(i)})
duplis[chklst[i].name] = uint(i)
} }
duplis[chklst[j].name] = uint(i) } else if isfirst {
mu.Unlock() sameset = append(sameset, []uint{uint(i)})
duplis[chklst[i].name] = uint(i)
isfirst = false
} }
duplis[chklst[j].name] = uint(i)
} }
wg.Done() }
}(i)
} }
wg.Wait()
fmt.Println("compare file success") fmt.Println("compare file success")
if len(sameset) > 0 { if len(sameset) > 0 {
dupset := make(map[uint][]string, len(sameset)) dupset := make(map[uint][]string, len(sameset))