author | Mikael Berthe <mikael@lilotux.net> |
Sat, 28 Jun 2014 18:21:33 +0200 | |
changeset 1 | a8aa9e54bce4 |
parent 0 | a5642cd03cef |
child 2 | 55098d552ae2 |
permissions | -rw-r--r-- |
0
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
1 |
/* |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
2 |
* Copyright (C) 2014 Mikael Berthe <mikael@lilotux.net> |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
3 |
* |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
4 |
* This program is free software; you can redistribute it and/or modify |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
5 |
* it under the terms of the GNU General Public License as published by |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
6 |
* the Free Software Foundation; either version 2 of the License, or (at |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
7 |
* your option) any later version. |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
8 |
* |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
9 |
* This program is distributed in the hope that it will be useful, but |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
10 |
* WITHOUT ANY WARRANTY; without even the implied warranty of |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
11 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
12 |
* General Public License for more details. |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
13 |
* |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
14 |
* You should have received a copy of the GNU General Public License |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
15 |
* along with this program; if not, write to the Free Software |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
16 |
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
17 |
* USA |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
18 |
*/ |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
19 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
20 |
// This program (Goduf) is a fast duplicate file finder. |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
21 |
// Use goduf --help to get the list of available options. |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
22 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
23 |
package main |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
24 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
25 |
import ( |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
26 |
"crypto/sha1" |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
27 |
"encoding/hex" |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
28 |
"errors" |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
29 |
"flag" |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
30 |
"fmt" |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
31 |
"io" |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
32 |
"log" |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
33 |
"os" |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
34 |
"path/filepath" |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
35 |
"sort" |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
36 |
) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
37 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
38 |
const medsumBytes = 128 |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
39 |
const minSizePartialChecksum = 49152 // Should be > 3*medsumBytes |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
40 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
41 |
type sumType int |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
42 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
43 |
const ( |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
44 |
fullChecksum sumType = iota |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
45 |
partialChecksum |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
46 |
) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
47 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
48 |
type fileObj struct { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
49 |
//Unique bool |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
50 |
FilePath string |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
51 |
os.FileInfo |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
52 |
PartialHash []byte |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
53 |
Hash []byte |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
54 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
55 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
56 |
type FileObjList []*fileObj |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
57 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
58 |
type sizeClass struct { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
59 |
files FileObjList |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
60 |
medsums map[string]FileObjList |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
61 |
fullsums map[string]FileObjList |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
62 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
63 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
64 |
type dataT struct { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
65 |
totalSize uint64 |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
66 |
cmpt uint |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
67 |
sizeGroups map[int64]*sizeClass |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
68 |
emptyFiles FileObjList |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
69 |
ignoreCount int |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
70 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
71 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
72 |
var data dataT |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
73 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
74 |
type myLogT struct { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
75 |
verbosity int |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
76 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
77 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
78 |
var myLog myLogT |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
79 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
80 |
func (l *myLogT) Printf(level int, format string, args ...interface{}) { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
81 |
if level > l.verbosity { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
82 |
return |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
83 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
84 |
if level >= 0 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
85 |
log.Printf(format, args...) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
86 |
return |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
87 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
88 |
// Error message without timestamp |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
89 |
fmt.Fprintf(os.Stderr, format, args...) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
90 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
91 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
92 |
func (l *myLogT) Println(level int, args ...interface{}) { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
93 |
if level > l.verbosity { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
94 |
return |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
95 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
96 |
if level >= 0 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
97 |
log.Println(args...) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
98 |
return |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
99 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
100 |
// Error message without timestamp |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
101 |
fmt.Fprintln(os.Stderr, args...) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
102 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
103 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
104 |
func visit(path string, f os.FileInfo, err error) error { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
105 |
if err != nil { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
106 |
if f == nil { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
107 |
return err |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
108 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
109 |
if f.IsDir() { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
110 |
myLog.Println(-1, "Warning: cannot process directory:", |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
111 |
path) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
112 |
return filepath.SkipDir |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
113 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
114 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
115 |
myLog.Println(-1, "Ignoring ", path, " - ", err) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
116 |
data.ignoreCount++ |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
117 |
return nil |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
118 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
119 |
if f.IsDir() { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
120 |
return nil |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
121 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
122 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
123 |
if mode := f.Mode(); mode&os.ModeType != 0 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
124 |
if mode&os.ModeSymlink != 0 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
125 |
myLog.Println(5, "Ignoring symbolic link", path) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
126 |
} else { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
127 |
myLog.Println(0, "Ignoring special file", path) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
128 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
129 |
data.ignoreCount++ |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
130 |
return nil |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
131 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
132 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
133 |
data.cmpt++ |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
134 |
data.totalSize += uint64(f.Size()) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
135 |
fo := &fileObj{FilePath: path, FileInfo: f} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
136 |
if _, ok := data.sizeGroups[f.Size()]; !ok { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
137 |
data.sizeGroups[f.Size()] = &sizeClass{} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
138 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
139 |
data.sizeGroups[f.Size()].files = |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
140 |
append(data.sizeGroups[f.Size()].files, fo) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
141 |
return nil |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
142 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
143 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
144 |
func (fo *fileObj) CheckSum() error { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
145 |
file, err := os.Open(fo.FilePath) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
146 |
if err != nil { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
147 |
return err |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
148 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
149 |
defer file.Close() |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
150 |
hash := sha1.New() |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
151 |
if size, err := io.Copy(hash, file); size != fo.Size() || err != nil { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
152 |
if err == nil { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
153 |
return errors.New("failed to read the whole file: " + |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
154 |
fo.FilePath) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
155 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
156 |
return err |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
157 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
158 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
159 |
fo.Hash = hash.Sum(nil) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
160 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
161 |
return nil |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
162 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
163 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
164 |
func (fo *fileObj) MedSum() error { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
165 |
file, err := os.Open(fo.FilePath) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
166 |
if err != nil { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
167 |
return err |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
168 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
169 |
defer file.Close() |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
170 |
hash := sha1.New() |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
171 |
// XXX: duplicated code! |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
172 |
// BOF |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
173 |
if _, err := io.CopyN(hash, file, medsumBytes); err != nil { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
174 |
if err == nil { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
175 |
return errors.New("failed to read bytes from file:" + |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
176 |
fo.FilePath) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
177 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
178 |
return err |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
179 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
180 |
/* |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
181 |
// MOF |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
182 |
file.Seek((fo.Size()-medsumBytes)/2, 0) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
183 |
if _, err := io.CopyN(hash, file, medsumBytes); err != nil { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
184 |
if err == nil { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
185 |
return errors.New("failed to read bytes from file:" + |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
186 |
fo.FilePath) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
187 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
188 |
return err |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
189 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
190 |
*/ |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
191 |
// EOF |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
192 |
file.Seek(0-medsumBytes, 2) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
193 |
if _, err := io.CopyN(hash, file, medsumBytes); err != nil { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
194 |
if err == nil { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
195 |
return errors.New("failed to read bytes from file:" + |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
196 |
fo.FilePath) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
197 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
198 |
return err |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
199 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
200 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
201 |
fo.PartialHash = hash.Sum(nil) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
202 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
203 |
return nil |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
204 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
205 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
206 |
func (fo *fileObj) Sum(sType sumType) error { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
207 |
if sType == partialChecksum { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
208 |
return fo.MedSum() |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
209 |
} else if sType == fullChecksum { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
210 |
return fo.CheckSum() |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
211 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
212 |
panic("Internal error: Invalid sType") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
213 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
214 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
215 |
func (data *dataT) dispCount() { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
216 |
if myLog.verbosity < 4 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
217 |
return |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
218 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
219 |
var c1, c2, c3, c4 int |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
220 |
var s4 string |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
221 |
for _, sc := range data.sizeGroups { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
222 |
c1 += len(sc.files) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
223 |
for _, fol := range sc.medsums { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
224 |
c2 += len(fol) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
225 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
226 |
for _, fol := range sc.fullsums { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
227 |
c3 += len(fol) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
228 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
229 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
230 |
c4 = len(data.emptyFiles) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
231 |
if c4 > 0 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
232 |
s4 = fmt.Sprintf("+%d", c4) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
233 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
234 |
myLog.Printf(4, " Current countdown: %d [%d%s/%d/%d]\n", |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
235 |
c1+c2+c3+c4, c1, s4, c2, c3) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
236 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
237 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
238 |
func (data *dataT) filesWithSameHash() (hgroups []FileObjList) { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
239 |
for _, sizeGroup := range data.sizeGroups { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
240 |
for _, l := range sizeGroup.fullsums { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
241 |
hgroups = append(hgroups, l) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
242 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
243 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
244 |
return |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
245 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
246 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
247 |
func createHashFromSum(sType sumType, folist FileObjList, hashes *map[string]FileObjList) { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
248 |
for _, fo := range folist { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
249 |
var hash string |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
250 |
var hbytes []byte |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
251 |
if sType == partialChecksum { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
252 |
hbytes = fo.PartialHash |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
253 |
} else if sType == fullChecksum { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
254 |
hbytes = fo.Hash |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
255 |
} else { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
256 |
panic("Internal error: Invalid sType") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
257 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
258 |
if hbytes == nil { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
259 |
// Could happen if the file was not readable, |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
260 |
// We skip it (already shown as dropped) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
261 |
continue |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
262 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
263 |
hash = hex.EncodeToString(hbytes) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
264 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
265 |
(*hashes)[hash] = append((*hashes)[hash], fo) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
266 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
267 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
268 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
269 |
func (data *dataT) calculateMedSums() { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
270 |
var bigList FileObjList |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
271 |
var droppedGroups int |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
272 |
var droppedFiles int |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
273 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
274 |
// Create a unique list of files to be partially checksummed |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
275 |
for size, sizeGroup := range data.sizeGroups { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
276 |
if size < minSizePartialChecksum { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
277 |
// We do not use MedSums for small files |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
278 |
continue |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
279 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
280 |
bigList = append(bigList, sizeGroup.files...) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
281 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
282 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
283 |
// Sort the list for better efficiency -- that's the whole point of |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
284 |
// the unique big list. |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
285 |
sort.Sort(ByInode(bigList)) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
286 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
287 |
// Compute partial checksums |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
288 |
for _, fo := range bigList { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
289 |
if err := fo.Sum(partialChecksum); err != nil { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
290 |
myLog.Println(0, "Error:", err) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
291 |
droppedFiles++ |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
292 |
// The hash part will be nil and the file will |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
293 |
// be dropped in createHashFromSum() below. |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
294 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
295 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
296 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
297 |
// Reparse size-grouped hashes and use the new hash information |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
298 |
// to build lists of files with the same partial checksum. |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
299 |
for size, sizeGroup := range data.sizeGroups { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
300 |
if size < minSizePartialChecksum { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
301 |
// We do not use MedSums for small files |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
302 |
continue |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
303 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
304 |
hashes := make(map[string]FileObjList) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
305 |
createHashFromSum(partialChecksum, sizeGroup.files, &hashes) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
306 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
307 |
// Let's de-dupe now... |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
308 |
for h, l := range hashes { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
309 |
if len(l) < 2 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
310 |
droppedGroups++ |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
311 |
droppedFiles += len(l) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
312 |
delete(hashes, h) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
313 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
314 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
315 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
316 |
// Done, save the result |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
317 |
data.sizeGroups[size].medsums = hashes |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
318 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
319 |
// We remove the items from "files" since they are in the hash |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
320 |
// tree now. |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
321 |
sizeGroup.files = nil |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
322 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
323 |
if len(hashes) == 0 { // We're done with this size group |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
324 |
delete(data.sizeGroups, size) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
325 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
326 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
327 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
328 |
if droppedFiles > 0 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
329 |
myLog.Printf(3, " Dropped %d files in %d groups\n", |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
330 |
droppedFiles, droppedGroups) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
331 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
332 |
return |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
333 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
334 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
335 |
func (data *dataT) calculateChecksums() { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
336 |
var bigList FileObjList |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
337 |
var droppedGroups int |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
338 |
var droppedFiles int |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
339 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
340 |
// Create a unique list of files to be fully checksummed |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
341 |
for _, sizeGroup := range data.sizeGroups { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
342 |
// #1: small files |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
343 |
bigList = append(bigList, sizeGroup.files...) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
344 |
// #2: files with same partial checksum |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
345 |
for _, l := range sizeGroup.medsums { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
346 |
bigList = append(bigList, l...) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
347 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
348 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
349 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
350 |
// Sort the list for better efficiency -- that's the whole point of |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
351 |
// the unique big list. |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
352 |
sort.Sort(ByInode(bigList)) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
353 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
354 |
// Compute full checksums |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
355 |
for _, fo := range bigList { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
356 |
if err := fo.Sum(fullChecksum); err != nil { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
357 |
myLog.Println(0, "Error:", err) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
358 |
droppedFiles++ |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
359 |
// The hash part will be nil and the file will |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
360 |
// be dropped in createHashFromSum() below. |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
361 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
362 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
363 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
364 |
// Reparse size-grouped hashes and use the new hash information |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
365 |
// to build lists of files with the same checksum. |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
366 |
for size, sizeGroup := range data.sizeGroups { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
367 |
hashes := make(map[string]FileObjList) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
368 |
// #1: small files |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
369 |
createHashFromSum(fullChecksum, sizeGroup.files, &hashes) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
370 |
// #2: files with same partial checksum |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
371 |
for _, l := range sizeGroup.medsums { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
372 |
createHashFromSum(fullChecksum, l, &hashes) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
373 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
374 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
375 |
// Let's de-dupe now... |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
376 |
for h, l := range hashes { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
377 |
if len(l) < 2 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
378 |
droppedGroups++ |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
379 |
droppedFiles += len(l) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
380 |
delete(hashes, h) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
381 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
382 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
383 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
384 |
// Done, save the result |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
385 |
data.sizeGroups[size].fullsums = hashes |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
386 |
data.sizeGroups[size].medsums = nil |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
387 |
sizeGroup.files = nil |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
388 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
389 |
if droppedFiles > 0 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
390 |
myLog.Printf(3, " Dropped %d files in %d groups\n", |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
391 |
droppedFiles, droppedGroups) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
392 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
393 |
return |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
394 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
395 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
396 |
func (data *dataT) dropEmptyFiles(ignoreEmpty bool) (emptyCount int) { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
397 |
sc, ok := data.sizeGroups[0] |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
398 |
if ok == false { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
399 |
return // no empty files |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
400 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
401 |
if !ignoreEmpty { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
402 |
if len(sc.files) > 1 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
403 |
data.emptyFiles = sc.files |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
404 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
405 |
delete(data.sizeGroups, 0) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
406 |
return |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
407 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
408 |
emptyCount = len(sc.files) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
409 |
delete(data.sizeGroups, 0) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
410 |
return |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
411 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
412 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
413 |
func (data *dataT) createSizeHash() (hardLinkCount, uniqueSizeCount int) { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
414 |
for s, sizeGroup := range data.sizeGroups { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
415 |
if len(sizeGroup.files) < 2 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
416 |
delete(data.sizeGroups, s) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
417 |
uniqueSizeCount++ |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
418 |
continue |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
419 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
420 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
421 |
var hardlinksFound bool |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
422 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
423 |
// Check for hardlinks |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
424 |
// TODO: what about symlinks? |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
425 |
// Remove unique dev/inodes |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
426 |
// Instead of this loop, another way would be to use the field |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
427 |
// "Unique" of the fileObj to mark them to be discarded |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
428 |
// and remove them all at the end. |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
429 |
for { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
430 |
if !OSHasInodes() { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
431 |
break |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
432 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
433 |
var hardLinkIndex int |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
434 |
fo := sizeGroup.files[0] |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
435 |
prevDev, prevIno := GetDevIno(fo) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
436 |
//fmt.Printf(" - FO: %#v\n", *fo) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
437 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
438 |
for i, fo := range sizeGroup.files[1:] { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
439 |
//fmt.Printf(" - FO %d : %#v\n", i, *fo) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
440 |
dev, ino := GetDevIno(fo) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
441 |
if dev == prevDev && ino == prevIno { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
442 |
hardLinkIndex = i + 1 |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
443 |
hardLinkCount++ |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
444 |
hardlinksFound = true |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
445 |
break |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
446 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
447 |
prevDev = dev |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
448 |
prevIno = ino |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
449 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
450 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
451 |
if hardLinkIndex == 0 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
452 |
break |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
453 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
454 |
i := hardLinkIndex |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
455 |
// Remove hardink |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
456 |
copy(sizeGroup.files[i:], sizeGroup.files[i+1:]) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
457 |
sizeGroup.files[len(sizeGroup.files)-1] = nil |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
458 |
sizeGroup.files = sizeGroup.files[:len(sizeGroup.files)-1] |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
459 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
460 |
// We have found hard links in this size group, |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
461 |
// maybe we can remove it |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
462 |
if hardlinksFound { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
463 |
if len(sizeGroup.files) < 2 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
464 |
delete(data.sizeGroups, s) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
465 |
uniqueSizeCount++ |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
466 |
continue |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
467 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
468 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
469 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
470 |
return |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
471 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
472 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
473 |
func formatSize(sizeBytes uint64, short bool) string { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
474 |
var units = map[int]string{ |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
475 |
0: "B", |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
476 |
1: "KiB", |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
477 |
2: "MiB", |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
478 |
3: "GiB", |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
479 |
4: "TiB", |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
480 |
5: "PiB", |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
481 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
482 |
humanSize := sizeBytes |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
483 |
var n int |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
484 |
for n < len(units)-1 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
485 |
if humanSize < 10000 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
486 |
break |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
487 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
488 |
humanSize /= 1024 |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
489 |
n++ |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
490 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
491 |
if n < 1 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
492 |
return fmt.Sprintf("%d bytes", sizeBytes) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
493 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
494 |
if short { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
495 |
return fmt.Sprintf("%d %s", humanSize, units[n]) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
496 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
497 |
return fmt.Sprintf("%d bytes (%d %s)", sizeBytes, humanSize, units[n]) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
498 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
499 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
500 |
func main() { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
501 |
var verbose bool |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
502 |
var summary bool |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
503 |
var skipPartial bool |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
504 |
var ignoreEmpty bool |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
505 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
506 |
flag.BoolVar(&verbose, "verbose", false, "Be verbose (verbosity=1)") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
507 |
flag.BoolVar(&verbose, "v", false, "See --verbose") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
508 |
flag.BoolVar(&summary, "summary", false, "Do not display the duplicate list") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
509 |
flag.BoolVar(&summary, "s", false, "See --summary") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
510 |
flag.BoolVar(&skipPartial, "skip-partial", false, "Skip partial checksums") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
511 |
flag.IntVar(&myLog.verbosity, "verbosity", 0, |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
512 |
"Set verbosity level (1-5)") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
513 |
flag.IntVar(&myLog.verbosity, "vl", 0, "See verbosity") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
514 |
timings := flag.Bool("timings", false, "Set detailed log timings") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
515 |
flag.BoolVar(&ignoreEmpty, "no-empty", false, "Ignore empty files") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
516 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
517 |
flag.Parse() |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
518 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
519 |
if myLog.verbosity > 0 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
520 |
verbose = true |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
521 |
} else if verbose == true { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
522 |
myLog.verbosity = 1 |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
523 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
524 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
525 |
if len(flag.Args()) == 0 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
526 |
// TODO: more helpful usage statement |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
527 |
myLog.Println(-1, "Usage:", os.Args[0], |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
528 |
"[options] base_directory") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
529 |
os.Exit(0) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
530 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
531 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
532 |
if *timings { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
533 |
log.SetFlags(log.LstdFlags | log.Lmicroseconds) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
534 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
535 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
536 |
data.sizeGroups = make(map[int64]*sizeClass) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
537 |
myLog.Println(1, "* Reading file metadata") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
538 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
539 |
for _, root := range flag.Args() { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
540 |
if err := filepath.Walk(root, visit); err != nil { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
541 |
myLog.Printf(-1, "* Error: could not read file tree:\n") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
542 |
myLog.Printf(-1, "> %v\n", err) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
543 |
os.Exit(1) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
544 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
545 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
546 |
emptyCount := data.dropEmptyFiles(ignoreEmpty) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
547 |
if verbose { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
548 |
if data.ignoreCount > 0 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
549 |
myLog.Printf(1, " %d special files were ignored\n", |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
550 |
data.ignoreCount) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
551 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
552 |
myLog.Println(2, " Initial counter:", data.cmpt, "files") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
553 |
myLog.Println(2, " Total size:", formatSize(data.totalSize, |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
554 |
false)) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
555 |
if emptyCount > 0 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
556 |
myLog.Printf(1, " %d empty files were ignored\n", |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
557 |
emptyCount) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
558 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
559 |
data.dispCount() |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
560 |
myLog.Println(3, "* Number of size groups:", len(data.sizeGroups)) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
561 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
562 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
563 |
// Remove unique sizes |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
564 |
myLog.Println(1, "* Removing files with unique size, sorting file lists...") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
565 |
hardLinkCount, uniqueSizeCount := data.createSizeHash() |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
566 |
if verbose { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
567 |
myLog.Printf(2, " Dropped %d files with unique size\n", |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
568 |
uniqueSizeCount) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
569 |
myLog.Printf(2, " Dropped %d hard links\n", hardLinkCount) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
570 |
myLog.Println(3, "* Number of size groups:", len(data.sizeGroups)) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
571 |
data.dispCount() |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
572 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
573 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
574 |
// Calculate medsums |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
575 |
if !skipPartial { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
576 |
myLog.Println(1, "* Calculating partial checksums...") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
577 |
data.calculateMedSums() |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
578 |
data.dispCount() |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
579 |
myLog.Println(3, "* Number of size groups:", len(data.sizeGroups)) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
580 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
581 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
582 |
// Calculate checksums |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
583 |
myLog.Println(1, "* Calculating checksums...") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
584 |
data.calculateChecksums() |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
585 |
data.dispCount() |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
586 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
587 |
// Get list of dupes |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
588 |
var result []FileObjList |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
589 |
if len(data.emptyFiles) > 0 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
590 |
result = append(result, data.emptyFiles) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
591 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
592 |
result = append(result, data.filesWithSameHash()...) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
593 |
myLog.Println(3, "* Number of match groups:", len(result)) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
594 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
595 |
// Done! Dump dupes |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
596 |
if len(result) > 0 && !summary { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
597 |
myLog.Println(1, "* Dupes:") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
598 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
599 |
var dupeSize uint64 |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
600 |
data.cmpt = 0 |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
601 |
for i, l := range result { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
602 |
size := uint64(l[0].Size()) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
603 |
// We do not count the size of the 1st item |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
604 |
// so we get only duplicate size. |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
605 |
dupeSize += size * uint64(len(l) - 1) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
606 |
if !summary { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
607 |
fmt.Printf("\nGroup #%d (%d files * %v):\n", i+1, |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
608 |
len(l), formatSize(size, true)) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
609 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
610 |
for _, f := range l { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
611 |
if !summary { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
612 |
fmt.Println(f.FilePath) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
613 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
614 |
data.cmpt++ |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
615 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
616 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
617 |
summaryLevel := 1 // Default verbosity for the summary line |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
618 |
if summary == false { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
619 |
// Add a trailing newline |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
620 |
if len(result) > 0 { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
621 |
fmt.Println("") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
622 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
623 |
} else { |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
624 |
// The summary is requested so we lower the verbosity level |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
625 |
summaryLevel = 0 |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
626 |
} |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
627 |
|
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
628 |
myLog.Println(summaryLevel, "Final count:", data.cmpt, |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
629 |
"duplicate files in", len(result), "sets") |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
630 |
myLog.Println(summaryLevel, "Redundant data size:", |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
631 |
formatSize(dupeSize, false)) |
a5642cd03cef
Goduf - initial version-controlled revision
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
632 |
} |