From: Francois Fleuret Date: Fri, 19 Mar 2010 09:31:46 +0000 (+0100) Subject: Added conditional compilation for the MD5 hashing. X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=commitdiff_plain;h=e4133d06373b48e8509afd0811bb0a726d74f8a8;p=finddup.git Added conditional compilation for the MD5 hashing. --- diff --git a/Makefile b/Makefile index 7df06bd..fd7146a 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,13 @@ BINARY_PATH = /usr/bin MAN_PATH = /usr/share/man/man1 -LDFLAGS=-lssl +ifeq ($(WITH_MD5),yes) + LDFLAGS=-lssl + MD5FLAG=-DWITH_MD5 +else + LDFLAGS= + MD5FLAG= +endif UNAME=\"$(shell uname -srmn)\" @@ -34,7 +40,7 @@ else OPTIMIZE_FLAG = -O2 endif -CFLAGS = -Wall -ansi -pedantic -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -DUNAME="$(UNAME)" $(OPTIMIZE_FLAG) +CFLAGS = -Wall -ansi -pedantic -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -DUNAME="$(UNAME)" $(MD5FLAG) $(OPTIMIZE_FLAG) all: finddup @@ -43,12 +49,12 @@ finddup: finddup.o install: finddup install -m 755 finddup $(BINARY_PATH) - # install -m 644 finddup.1 $(MAN_PATH) - # gzip $(MAN_PATH)/finddup.1 + install -m 644 finddup.1 $(MAN_PATH) + gzip $(MAN_PATH)/finddup.1 uninstall: rm $(BINARY_PATH)/finddup - # rm $(MAN_PATH)/finddup.1.gz + rm $(MAN_PATH)/finddup.1.gz clean: rm -f *.o finddup diff --git a/finddup.1 b/finddup.1 index 9cc21b4..46a4326 100644 --- a/finddup.1 +++ b/finddup.1 @@ -69,10 +69,19 @@ use MD5 hashing None known, probably many. Valgrind does not complain though. -The MD5 hashing often hurts more than it helps, hence it is off by -default. The only case when it should really be useful is when you -have plenty of different files of same size, which does not happen -often. +The MD5 hashing is not satisfactory. It is computed for a file only if +the said file has to be read fully for a comparison (i.e. two files +match and we have to read them completely). + +Hence, in practice lot of partial MD5s are computed, which costs a lot +of cpu and is useless. This often hurts more than it helps, hence it +is off by default. The only case when it should really be useful is +when you have plenty of different files of same size, and lot of +similar ones, which does not happen often. + +Forcing the files to be read fully so that the MD5s are properly +computed is not okay neither, since it would fully read certain files, +even if we will never need their MD5s. .SH "WISH LIST" diff --git a/finddup.c b/finddup.c index ccf21c6..3edfd61 100644 --- a/finddup.c +++ b/finddup.c @@ -40,7 +40,9 @@ #include #include #include +#ifdef WITH_MD5 #include +#endif /* 1M really helps compared to 64k */ #define READ_BUFFER_SIZE (1024 * 1024) @@ -72,7 +74,9 @@ int ignore_same_inode = 0; /* 1 means that comparison between two file int tty_width = -1; /* Positive value means what width to use to show the progress bar */ +#ifdef WITH_MD5 int use_md5 = 0; /* 1 means we keep an MD5 signature for each file */ +#endif /********************************************************************/ @@ -105,8 +109,10 @@ struct file_node { ino_t inode; int group_id; /* one per identical file content */ int dir_id; /* 1 for DIR1, and 2 for DIR2 */ +#ifdef WITH_MD5 int md5_computed; unsigned char md5[MD5_DIGEST_LENGTH]; +#endif }; void file_list_delete(struct file_node *head) { @@ -133,12 +139,15 @@ int file_list_length(struct file_node *head) { int same_content(struct file_node *f1, struct file_node *f2, char *buffer1, char *buffer2) { int fd1, fd2, s1, s2; + +#ifdef WITH_MD5 MD5_CTX c1, c2; if(use_md5) { - if(f1->md5_computed && f2->md5_computed && - !memcmp(f1->md5, f2->md5, MD5_DIGEST_LENGTH)) { - return 0; + if(f1->md5_computed && f2->md5_computed) { + if(!memcmp(f1->md5, f2->md5, MD5_DIGEST_LENGTH)) { + return 0; + } } else { if(!f1->md5_computed) { MD5_Init(&c1); @@ -148,6 +157,7 @@ int same_content(struct file_node *f1, struct file_node *f2, } } } +#endif fd1 = open(f1->name, O_RDONLY); fd2 = open(f2->name, O_RDONLY); @@ -167,6 +177,7 @@ int same_content(struct file_node *f1, struct file_node *f2, if(s1 == 0) { close(fd1); close(fd2); +#ifdef WITH_MD5 if(use_md5) { if(!f1->md5_computed) { MD5_Final(f1->md5, &c1); @@ -177,8 +188,15 @@ int same_content(struct file_node *f1, struct file_node *f2, f2->md5_computed = 1; } } +#endif return 1; } else { + if(memcmp(buffer1, buffer2, s1)) { + close(fd1); + close(fd2); + return 0; + } +#ifdef WITH_MD5 if(use_md5) { if(!f1->md5_computed) { MD5_Update(&c1, buffer1, s1); @@ -187,11 +205,7 @@ int same_content(struct file_node *f1, struct file_node *f2, MD5_Update(&c2, buffer2, s2); } } - if(memcmp(buffer1, buffer2, s1)) { - close(fd1); - close(fd2); - return 0; - } +#endif } } else { fprintf(stderr, @@ -266,7 +280,9 @@ struct file_node *scan_directory(struct file_node *tail, const char *name) { tmp->inode = sb.st_ino; tmp->group_id = -1; tmp->dir_id = -1; +#ifdef WITH_MD5 tmp->md5_computed = 0; +#endif tail = tmp; } } @@ -518,7 +534,9 @@ void print_help(FILE *out) { fprintf(out, " -r, --real-paths show the real file paths\n"); fprintf(out, " -i, --same-inodes-are-different\n"); fprintf(out, " consider files with same inode as different\n"); +#ifdef WITH_MD5 fprintf(out, " -m, --md5 use MD5 hashing\n"); +#endif fprintf(out, "\n"); fprintf(out, "Report bugs and comments to .\n"); } @@ -582,7 +600,13 @@ int main(int argc, char **argv) { break; case 'm': +#ifdef WITH_MD5 use_md5 = 1; +#else + fprintf(stderr, + "finddup has not be compiled with MD5 hashing.\n"); + exit(EXIT_FAILURE); +#endif break; default: