From: Francois Fleuret <francois@fleuret.org>
Date: Thu, 18 Mar 2010 19:45:18 +0000 (+0100)
Subject: Added MD5 hashing.
X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=commitdiff_plain;h=a61c9478f31b957e0d4007df9feddd6f0139ccf8;p=finddup.git

Added MD5 hashing.
---

diff --git a/Makefile b/Makefile
index d02a4d9..7df06bd 100644
--- a/Makefile
+++ b/Makefile
@@ -24,7 +24,7 @@
 BINARY_PATH = /usr/bin
 MAN_PATH = /usr/share/man/man1
 
-LDFLAGS=
+LDFLAGS=-lssl
 
 UNAME=\"$(shell uname -srmn)\"
 
diff --git a/finddup.1 b/finddup.1
index 18c27e5..9cc21b4 100644
--- a/finddup.1
+++ b/finddup.1
@@ -61,19 +61,23 @@ show the real path of the files
 .TP
 \fB-i\fR, \fB--same-inodes-are-different\fR
 files with same inode are considered as different
+.TP
+\fB-m\fR, \fB--md5\fR
+use MD5 hashing
 
 .SH "BUGS"
 
 None known, probably many. Valgrind does not complain though.
 
+The MD5 hashing often hurts more than it helps, hence it is off by
+default. The only case when it should really be useful is when you
+have plenty of different files of same size, which does not happen
+often.
+
 .SH "WISH LIST"
 
 The format of the output should definitely be improved. Not clear how.
 
-The comparison algorithm could maybe be improved with some MD5 kind of
-signature. However, most of the time is taken by comparison for
-matching files, which are required even when using a hash.
-
 Their could be some fancy option to link two instances of the command
 running on different machines to reduce network disk accesses. Again,
 this may not help much, for the reason given above.
diff --git a/finddup.c b/finddup.c
index aa16dac..ccf21c6 100644
--- a/finddup.c
+++ b/finddup.c
@@ -23,7 +23,7 @@
  *
  */
 
-#define VERSION_NUMBER "0.8"
+#define VERSION_NUMBER "0.9"
 
 #define _BSD_SOURCE
 
@@ -40,6 +40,7 @@
 #include <locale.h>
 #include <getopt.h>
 #include <fcntl.h>
+#include <openssl/md5.h>
 
 /* 1M really helps compared to 64k */
 #define READ_BUFFER_SIZE (1024 * 1024)
@@ -71,6 +72,8 @@ int ignore_same_inode = 0; /* 1 means that comparison between two file
 int tty_width = -1; /* Positive value means what width to use to show
                        the progress bar */
 
+int use_md5 = 0; /* 1 means we keep an MD5 signature for each file */
+
 /********************************************************************/
 
 /* malloc with error checking.  */
@@ -102,6 +105,8 @@ struct file_node {
   ino_t inode;
   int group_id; /* one per identical file content */
   int dir_id; /* 1 for DIR1, and 2 for DIR2 */
+  int md5_computed;
+  unsigned char md5[MD5_DIGEST_LENGTH];
 };
 
 void file_list_delete(struct file_node *head) {
@@ -128,6 +133,21 @@ int file_list_length(struct file_node *head) {
 int same_content(struct file_node *f1, struct file_node *f2,
                  char *buffer1, char *buffer2) {
   int fd1, fd2, s1, s2;
+  MD5_CTX c1, c2;
+
+  if(use_md5) {
+    if(f1->md5_computed && f2->md5_computed &&
+       !memcmp(f1->md5, f2->md5, MD5_DIGEST_LENGTH)) {
+      return 0;
+    } else {
+      if(!f1->md5_computed) {
+        MD5_Init(&c1);
+      }
+      if(!f2->md5_computed) {
+        MD5_Init(&c2);
+      }
+    }
+  }
 
   fd1 = open(f1->name, O_RDONLY);
   fd2 = open(f2->name, O_RDONLY);
@@ -147,8 +167,26 @@ int same_content(struct file_node *f1, struct file_node *f2,
         if(s1 == 0) {
           close(fd1);
           close(fd2);
+          if(use_md5) {
+            if(!f1->md5_computed) {
+              MD5_Final(f1->md5, &c1);
+              f1->md5_computed = 1;
+            }
+            if(!f2->md5_computed) {
+              MD5_Final(f2->md5, &c2);
+              f2->md5_computed = 1;
+            }
+          }
           return 1;
         } else {
+          if(use_md5) {
+            if(!f1->md5_computed) {
+              MD5_Update(&c1, buffer1, s1);
+            }
+            if(!f2->md5_computed) {
+              MD5_Update(&c2, buffer2, s2);
+            }
+          }
           if(memcmp(buffer1, buffer2, s1)) {
             close(fd1);
             close(fd2);
@@ -228,6 +266,7 @@ struct file_node *scan_directory(struct file_node *tail, const char *name) {
           tmp->inode = sb.st_ino;
           tmp->group_id = -1;
           tmp->dir_id = -1;
+          tmp->md5_computed = 0;
           tail = tmp;
         }
       }
@@ -479,6 +518,7 @@ void print_help(FILE *out) {
   fprintf(out, "   -r, --real-paths           show the real file paths\n");
   fprintf(out, "   -i, --same-inodes-are-different\n");
   fprintf(out, "                              consider files with same inode as different\n");
+  fprintf(out, "   -m, --md5                  use MD5 hashing\n");
   fprintf(out, "\n");
   fprintf(out, "Report bugs and comments to <francois@fleuret.org>.\n");
 }
@@ -494,6 +534,7 @@ static struct option long_options[] = {
   { "ignore-dots", no_argument, 0, 'd' },
   { "ignore-empty", no_argument, 0, '0' },
   { "show-progress", no_argument, 0, 'p' },
+  { "md5", no_argument, 0, 'm' },
   { 0, 0, 0, 0 }
 };
 
@@ -502,7 +543,7 @@ int main(int argc, char **argv) {
 
   setlocale (LC_ALL, "");
 
-  while ((c = getopt_long(argc, argv, "hircgd0p",
+  while ((c = getopt_long(argc, argv, "hircgd0pm",
                           long_options, NULL)) != -1) {
     switch (c) {
 
@@ -540,6 +581,10 @@ int main(int argc, char **argv) {
       show_hits = 0;
       break;
 
+    case 'm':
+      use_md5 = 1;
+      break;
+
     default:
       exit(EXIT_FAILURE);
     }