From: Francois Fleuret <francois@fleuret.org>
Date: Wed, 7 Apr 2010 16:56:13 +0000 (+0200)
Subject: Removed the MD5 option which was useless.
X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=commitdiff_plain;h=ab7b6e26f35ac1dfc88d9bf1e09dd289a30ea782;p=finddup.git

Removed the MD5 option which was useless.
---

diff --git a/finddup.1 b/finddup.1
index faaef4d..e262386 100644
--- a/finddup.1
+++ b/finddup.1
@@ -61,30 +61,17 @@ show the real path of the files
 .TP
 \fB-i\fR, \fB--same-inodes-are-different\fR
 files with same inode are considered as different
-.TP
-\fB-m\fR, \fB--md5\fR
-use MD5 hashing (if compiled with the option)
 
 .SH "BUGS"
 
 None known, probably many. Valgrind does not complain though.
 
-The MD5 hashing is not satisfactory. It is computed for a file only if
-the said file has to be read fully for a comparison (i.e. two files
-match and we have to read them completely).
-
-Hence, in practice lot of partial MD5s are computed, which costs a lot
-of cpu and is useless. This often hurts more than it helps. The only
-case when it should really be useful is when you have plenty of
-different files of same size, and lot of similar ones, which does not
-happen often.
-
-Forcing the files to be read fully so that the MD5s are properly
-computed is not okay neither, since it would fully read certain files,
-even if we will never need their MD5s.
-
-Anyway, it has to be compiled in with 'make WITH_MD5=yes', and even in
-that case it will be off by default
+The current algorithm is dumb, that is it does not use any hashing of
+the file content. I tried md5 on the whole file, which is not
+satisfactory because files are often never read entirely hence the md5
+can not be properly computed. I also tried XOR of the first 4, 16 and
+256 bytes with rejection as soon as one does not match. Did not help
+either.
 
 .SH "WISH LIST"
 
diff --git a/finddup.c b/finddup.c
index 5167a59..8f2b6c9 100644
--- a/finddup.c
+++ b/finddup.c
@@ -40,9 +40,6 @@
 #include <locale.h>
 #include <getopt.h>
 #include <fcntl.h>
-#ifdef WITH_MD5
-#include <openssl/md5.h>
-#endif
 
 /* 1M really helps compared to 64k */
 #define READ_BUFFER_SIZE (1024 * 1024)
@@ -74,10 +71,6 @@ int same_inodes_are_different = 0; /* 1 means that comparison between
                                       two files with same inode will
                                       always be false */
 
-#ifdef WITH_MD5
-int use_md5 = 0; /* 1 means we keep an MD5 signature for each file */
-#endif
-
 /********************************************************************/
 
 /* malloc with error checking.  */
@@ -109,10 +102,6 @@ struct file_node {
   ino_t inode;
   int group_id; /* one per identical file content */
   int dir_id; /* 1 for DIR1, and 2 for DIR2 */
-#ifdef WITH_MD5
-  int md5_computed;
-  unsigned char md5[MD5_DIGEST_LENGTH];
-#endif
 };
 
 void file_list_delete(struct file_node *head) {
@@ -140,25 +129,6 @@ int same_content(struct file_node *f1, struct file_node *f2,
                  char *buffer1, char *buffer2) {
   int fd1, fd2, s1, s2;
 
-#ifdef WITH_MD5
-  MD5_CTX c1, c2;
-
-  if(use_md5) {
-    if(f1->md5_computed && f2->md5_computed) {
-      if(!memcmp(f1->md5, f2->md5, MD5_DIGEST_LENGTH)) {
-        return 0;
-      }
-    } else {
-      if(!f1->md5_computed) {
-        MD5_Init(&c1);
-      }
-      if(!f2->md5_computed) {
-        MD5_Init(&c2);
-      }
-    }
-  }
-#endif
-
   fd1 = open(f1->name, O_RDONLY);
   fd2 = open(f2->name, O_RDONLY);
 
@@ -177,35 +147,14 @@ int same_content(struct file_node *f1, struct file_node *f2,
         if(s1 == 0) {
           close(fd1);
           close(fd2);
-#ifdef WITH_MD5
-          if(use_md5) {
-            if(!f1->md5_computed) {
-              MD5_Final(f1->md5, &c1);
-              f1->md5_computed = 1;
-            }
-            if(!f2->md5_computed) {
-              MD5_Final(f2->md5, &c2);
-              f2->md5_computed = 1;
-            }
-          }
-#endif
           return 1;
         } else {
           if(memcmp(buffer1, buffer2, s1)) {
+            /* printf("size_to_read = %d\n", size_to_read); */
             close(fd1);
             close(fd2);
             return 0;
           }
-#ifdef WITH_MD5
-          if(use_md5) {
-            if(!f1->md5_computed) {
-              MD5_Update(&c1, buffer1, s1);
-            }
-            if(!f2->md5_computed) {
-              MD5_Update(&c2, buffer2, s2);
-            }
-          }
-#endif
         }
       } else {
         fprintf(stderr,
@@ -281,9 +230,6 @@ struct file_node *scan_directory(struct file_node *tail, const char *name) {
           tmp->inode = sb.st_ino;
           tmp->group_id = -1;
           tmp->dir_id = -1;
-#ifdef WITH_MD5
-          tmp->md5_computed = 0;
-#endif
           tail = tmp;
         }
       }
@@ -572,9 +518,6 @@ void usage(FILE *out) {
   fprintf(out, "   -r, --real-paths           show the real file paths\n");
   fprintf(out, "   -i, --same-inodes-are-different\n");
   fprintf(out, "                              consider files with same inode as different\n");
-#ifdef WITH_MD5
-  fprintf(out, "   -m, --md5                  use MD5 hashing\n");
-#endif
   fprintf(out, "\n");
   fprintf(out, "Report bugs and comments to <francois@fleuret.org>.\n");
 }
@@ -590,7 +533,6 @@ static struct option long_options[] = {
   { "ignore-dots", no_argument, 0, 'd' },
   { "ignore-empty", no_argument, 0, '0' },
   { "show-progress", no_argument, 0, 'p' },
-  { "md5", no_argument, 0, 'm' },
   { 0, 0, 0, 0 }
 };
 
@@ -637,17 +579,6 @@ int main(int argc, char **argv) {
       show_hits = 0;
       break;
 
-    case 'm':
-#ifdef WITH_MD5
-      use_md5 = 1;
-#else
-      fprintf(stderr,
-              "finddup has not been compiled with MD5 hashing.\n");
-      usage(stderr);
-      exit(EXIT_FAILURE);
-#endif
-      break;
-
     default:
       usage(stderr);
       exit(EXIT_FAILURE);