3 * Copyright (c) 2013 Francois Fleuret
4 * Written by Francois Fleuret <francois@fleuret.org>
6 * This file is part of mymail.
8 * mymail is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 3 as
10 * published by the Free Software Foundation.
12 * mymail is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with mymail. If not, see <http://www.gnu.org/licenses/>.
24 This command is a dumb mail indexer. It can either (1) scan
25 directories containing mbox files, and create a db file containing
26 for each mail a list of fields computed from the header, or (2)
27 read such a db file and get all the mails matching regexp-defined
28 conditions on the fields.
30 It is low-tech, simple, light and fast.
47 #define MYMAIL_DB_MAGIC_TOKEN "mymail_index_file"
48 #define VERSION "0.9.1"
50 #define MAX_NB_SEARCH_CONDITIONS 10
52 #define BUFFER_SIZE 65536
55 char *db_filename_regexp_string;
57 char *db_filename_list;
58 char output_filename[PATH_MAX + 1];
64 /********************************************************************/
78 static char *field_names[] = {
89 /********************************************************************/
91 struct search_condition {
97 /********************************************************************/
99 struct parsable_field {
105 static struct parsable_field fields_to_parse[] = {
109 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
114 "^\\([Ff][Rr][Oo][Mm]:\\|[Rr][Ee][Pp][Ll][Yy]-[Tt][Oo]:\\|[Ss][Ee][Nn][Dd][Ee][Rr]:\\)",
115 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
120 "^\\([Tt][Oo]\\|[Cc][Cc]\\|[Bb][Cc][Cc]\\): ",
121 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
126 "^[Ss][Uu][Bb][Jj][Ee][Cc][Tt]: ",
127 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
132 "^[Dd][Aa][Tt][Ee]: ",
133 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
138 /********************************************************************/
140 int xor(int a, int b) {
141 return (a && !b) || (!a && b);
144 char *segment_next_field(char *current) {
145 while(*current && *current != ' ') current++;
146 *current = '\0'; current++;
147 while(*current && *current == ' ') current++;
151 void remove_eof(char *c) {
152 while(*c && *c != '\n' && *c != '\r') c++;
156 /********************************************************************/
158 /* malloc with error checking. */
160 void *safe_malloc(size_t n) {
164 "mymail: cannot allocate memory: %s\n", strerror(errno));
170 /*********************************************************************/
172 void print_version(FILE *out) {
173 fprintf(out, "mymail version %s (%s)\n", VERSION, UNAME);
176 void print_usage(FILE *out) {
178 fprintf(out, "Written by Francois Fleuret <francois@fleuret.org>.\n");
180 fprintf(out, "Usage: mymail [options] [<mbox dir1> [<mbox dir2> ...]|<db file1> [<db file2> ...]]\n");
182 fprintf(out, " -h, --help\n");
183 fprintf(out, " show this help\n");
184 fprintf(out, " -v, --version\n");
185 fprintf(out, " print the version number\n");
186 fprintf(out, " -q, --quiet\n");
187 fprintf(out, " do not print information during search\n");
188 fprintf(out, " -p <db filename pattern>, --db-pattern <db filename pattern>\n");
189 fprintf(out, " set the db filename pattern for recursive search\n");
190 fprintf(out, " -r <db root path>, --db-root <db root path>\n");
191 fprintf(out, " set the db root path for recursive search\n");
192 fprintf(out, " -l <db filename list>, --db-list <db filename list>\n");
193 fprintf(out, " set the semicolon-separated list of db files for search\n");
194 fprintf(out, " -s <search pattern>, --search <search pattern>\n");
195 fprintf(out, " search for matching mails in the db file\n");
196 fprintf(out, " -d <db filename>, --db-file <db filename>\n");
197 fprintf(out, " set the db filename for indexing\n");
198 fprintf(out, " -i, --index\n");
199 fprintf(out, " index mails\n");
200 fprintf(out, " -o <output filename>, --output <output filename>\n");
201 fprintf(out, " set the result file, use stdout if unset\n");
204 /*********************************************************************/
206 int ignore_entry(const char *name) {
208 /* strcmp(name, ".") == 0 || */
209 /* strcmp(name, "..") == 0 || */
210 (name[0] == '.' && name[1] != '/');
213 int mbox_line_match_search(struct search_condition *condition,
214 int mbox_id, char *mbox_value) {
216 (condition->field_id == mbox_id ||
217 (condition->field_id == ID_PARTICIPANT && (mbox_id == ID_FROM || mbox_id == ID_TO)) ||
218 (condition->field_id == ID_FROM && mbox_id == ID_FIRST_LINE))
220 regexec(&condition->regexp, mbox_value, 0, 0, 0) == 0;
223 void search_in_db(FILE *db_file,
224 int nb_search_conditions,
225 struct search_condition *search_conditions,
228 int hits[MAX_NB_SEARCH_CONDITIONS];
229 char raw_db_line[BUFFER_SIZE];
230 char raw_mbox_line[BUFFER_SIZE];
231 char current_mail_filename[PATH_MAX + 1];
232 unsigned long int current_position_in_mail;
233 char *mbox_name, *mbox_value;
235 int already_written, m, n;
236 int last_mbox_line_was_empty;
237 int nb_body_conditions, nb_fulfilled_body_conditions;
239 current_position_in_mail = 0;
242 for(n = 0; n < nb_search_conditions; n++) { hits[n] = 0; }
244 nb_body_conditions = 0;
245 for(n = 0; n < nb_search_conditions; n++) {
246 if(search_conditions[n].field_id == ID_BODY) {
247 nb_body_conditions++;
251 strcpy(current_mail_filename, "");
253 while(fgets(raw_db_line, BUFFER_SIZE, db_file)) {
254 mbox_name = raw_db_line;
255 mbox_value = segment_next_field(raw_db_line);
257 if(strcmp("mail", mbox_name) == 0) {
258 char *position_in_file_string;
261 if(current_mail_filename[0]) {
263 /* We first check all conditions but the body ones */
265 for(n = 0; n < nb_search_conditions &&
266 ((search_conditions[n].field_id == ID_BODY) ||
267 xor(hits[n], search_conditions[n].negation)); n++);
269 if(n == nb_search_conditions) {
271 /* all conditions but the body ones are fine, check the body
274 nb_fulfilled_body_conditions = 0;
276 if(nb_body_conditions > 0) {
281 mail_file = fopen(current_mail_filename, "r");
285 "mymail: Cannot open mbox '%s' for body scan.\n",
286 current_mail_filename);
290 fseek(mail_file, current_position_in_mail, SEEK_SET);
292 if(fgets(raw_mbox_line, BUFFER_SIZE, mail_file)) {
293 while(nb_fulfilled_body_conditions < nb_body_conditions) {
294 last_mbox_line_was_empty = (raw_mbox_line[0] == '\n');
296 if(last_mbox_line_was_empty) { header = 0; }
299 for(n = 0; n < nb_search_conditions; n++) {
300 if(search_conditions[n].field_id == ID_BODY && !hits[n]) {
302 (regexec(&search_conditions[n].regexp, raw_mbox_line, 0, 0, 0) == 0);
304 nb_fulfilled_body_conditions++;
310 if(!fgets(raw_mbox_line, BUFFER_SIZE, mail_file) ||
311 (last_mbox_line_was_empty && strncmp(raw_mbox_line, "From ", 5) == 0))
319 if(nb_body_conditions == nb_fulfilled_body_conditions) {
322 mail_file = fopen(current_mail_filename, "r");
326 "mymail: Cannot open mbox '%s' for mail extraction.\n",
327 current_mail_filename);
331 fseek(mail_file, current_position_in_mail, SEEK_SET);
333 if(fgets(raw_mbox_line, BUFFER_SIZE, mail_file)) {
334 last_mbox_line_was_empty = 1;
335 fprintf(output_file, "%s", raw_mbox_line);
337 if(!fgets(raw_mbox_line, BUFFER_SIZE, mail_file) ||
338 (last_mbox_line_was_empty && strncmp(raw_mbox_line, "From ", 5) == 0))
340 last_mbox_line_was_empty = (raw_mbox_line[0] == '\n');
341 fprintf(output_file, "%s", raw_mbox_line);
350 for(n = 0; n < nb_search_conditions; n++) { hits[n] = 0; }
352 position_in_file_string = mbox_value;
353 mail_filename = segment_next_field(mbox_value);
354 current_position_in_mail = atol(position_in_file_string);
355 strcpy(current_mail_filename, mail_filename);
357 remove_eof(current_mail_filename);
363 for(m = 0; (m < MAX_ID) && mbox_id == -1; m++) {
364 if(strncmp(field_names[m], mbox_name, strlen(mbox_name)) == 0) {
368 for(n = 0; n < nb_search_conditions; n++) {
369 hits[n] |= mbox_line_match_search(&search_conditions[n],
370 mbox_id, mbox_value);
376 void recursive_search_in_db(const char *entry_name, regex_t *db_filename_regexp,
377 int nb_search_conditions,
378 struct search_condition *search_conditions,
381 struct dirent *dir_e;
383 char raw_db_line[BUFFER_SIZE];
384 char subname[PATH_MAX + 1];
386 if(lstat(entry_name, &sb) != 0) {
388 "mymail: Cannot stat \"%s\": %s\n",
394 dir = opendir(entry_name);
397 while((dir_e = readdir(dir))) {
398 if(!ignore_entry(dir_e->d_name)) {
399 snprintf(subname, PATH_MAX, "%s/%s", entry_name, dir_e->d_name);
400 recursive_search_in_db(subname, db_filename_regexp,
401 nb_search_conditions, search_conditions,
409 const char *s = entry_name, *filename = entry_name;
410 while(*s) { if(*s == '/') { filename = s+1; } s++; }
412 if(regexec(db_filename_regexp, filename, 0, 0, 0) == 0) {
413 FILE *db_file = fopen(entry_name, "r");
416 printf("Searching in '%s' ... ", entry_name);
422 "mymail: Cannot open \"%s\" for reading: %s\n",
428 if(fgets(raw_db_line, BUFFER_SIZE, db_file)) {
429 if(strncmp(raw_db_line, MYMAIL_DB_MAGIC_TOKEN, strlen(MYMAIL_DB_MAGIC_TOKEN))) {
431 "mymail: Header line in '%s' does not match the mymail db format.\n",
437 "mymail: Cannot read the header line in '%s'.\n",
442 search_in_db(db_file, nb_search_conditions, search_conditions, output_file);
454 /*********************************************************************/
456 void index_one_mbox_line(int nb_fields_to_parse, struct parsable_field *fields_to_parse,
457 char *raw_mbox_line, FILE *db_file) {
460 for(f = 0; f < nb_fields_to_parse; f++) {
461 if(regexec(&fields_to_parse[f].regexp, raw_mbox_line, 1, &matches, 0) == 0) {
462 fprintf(db_file, "%s %s\n",
463 field_names[fields_to_parse[f].id],
464 raw_mbox_line + matches.rm_eo);
469 void index_mbox(const char *mbox_filename,
470 int nb_fields_to_parse, struct parsable_field *fields_to_parse,
472 char raw_mbox_line[BUFFER_SIZE], full_line[BUFFER_SIZE];
473 char *end_of_full_line;
475 int in_header, new_header, last_mbox_line_was_empty;
476 unsigned long int position_in_file;
478 file = fopen(mbox_filename, "r");
481 fprintf(stderr, "mymail: Cannot open '%s'.\n", mbox_filename);
482 if(paranoid) { exit(EXIT_FAILURE); }
489 position_in_file = 0;
490 end_of_full_line = 0;
492 last_mbox_line_was_empty = 1;
494 while(fgets(raw_mbox_line, BUFFER_SIZE, file)) {
495 if(last_mbox_line_was_empty && strncmp(raw_mbox_line, "From ", 5) == 0) {
498 "Got a ^\"From \" in the header in %s:%lu.\n",
499 mbox_filename, position_in_file);
500 fprintf(stderr, "%s", raw_mbox_line);
501 if(paranoid) { exit(EXIT_FAILURE); }
505 } else if(raw_mbox_line[0] == '\n') {
506 if(in_header) { in_header = 0; }
509 last_mbox_line_was_empty = (raw_mbox_line[0] == '\n');
513 fprintf(db_file, "mail %lu %s\n", position_in_file, mbox_filename);
517 if(raw_mbox_line[0] == ' ' || raw_mbox_line[0] == '\t') {
518 char *start = raw_mbox_line;
519 while(*start == ' ' || *start == '\t') start++;
520 *(end_of_full_line++) = ' ';
521 strcpy(end_of_full_line, start);
522 while(*end_of_full_line && *end_of_full_line != '\n') {
525 *end_of_full_line = '\0';
530 if(!((raw_mbox_line[0] >= 'a' && raw_mbox_line[0] <= 'z') ||
531 (raw_mbox_line[0] >= 'A' && raw_mbox_line[0] <= 'Z'))) {
533 "Header line syntax error %s:%lu.\n",
534 mbox_filename, position_in_file);
535 fprintf(stderr, "%s", raw_mbox_line);
540 index_one_mbox_line(nb_fields_to_parse, fields_to_parse, full_line, db_file);
543 end_of_full_line = full_line;
544 strcpy(end_of_full_line, raw_mbox_line);
545 while(*end_of_full_line && *end_of_full_line != '\n') {
548 *end_of_full_line = '\0';
553 position_in_file += strlen(raw_mbox_line);
559 void recursive_index_mbox(FILE *db_file,
560 const char *entry_name,
561 int nb_fields_to_parse, struct parsable_field *fields_to_parse) {
563 struct dirent *dir_e;
565 char subname[PATH_MAX + 1];
567 if(lstat(entry_name, &sb) != 0) {
569 "mymail: Cannot stat \"%s\": %s\n",
575 dir = opendir(entry_name);
578 while((dir_e = readdir(dir))) {
579 if(!ignore_entry(dir_e->d_name)) {
580 snprintf(subname, PATH_MAX, "%s/%s", entry_name, dir_e->d_name);
581 recursive_index_mbox(db_file, subname, nb_fields_to_parse, fields_to_parse);
586 index_mbox(entry_name, nb_fields_to_parse, fields_to_parse, db_file);
590 /*********************************************************************/
592 /* For long options that have no equivalent short option, use a
593 non-character as a pseudo short option, starting with CHAR_MAX + 1. */
595 OPT_BASH_MODE = CHAR_MAX + 1
598 static struct option long_options[] = {
599 { "help", no_argument, 0, 'h' },
600 { "version", no_argument, 0, 'v' },
601 { "quiet", no_argument, 0, 'q' },
602 { "db-file", 1, 0, 'd' },
603 { "db-pattern", 1, 0, 'p' },
604 { "db-root", 1, 0, 'r' },
605 { "db-list", 1, 0, 'l' },
606 { "search", 1, 0, 's' },
607 { "index", 0, 0, 'i' },
608 { "output", 1, 0, 'o' },
612 /*********************************************************************/
614 int main(int argc, char **argv) {
615 int error = 0, show_help = 0;
616 const int nb_fields_to_parse = sizeof(fields_to_parse) / sizeof(struct parsable_field);
619 int nb_search_conditions;
620 char *search_condition_strings[MAX_NB_SEARCH_CONDITIONS];
627 db_filename_list = 0;
630 setlocale(LC_ALL, "");
632 nb_search_conditions = 0;
634 while ((c = getopt_long(argc, argv, "hvqip:s:d:r:l:o:",
635 long_options, NULL)) != -1) {
644 print_version(stdout);
656 db_filename = strdup(optarg);
660 db_filename_regexp_string = strdup(optarg);
664 strncpy(output_filename, optarg, PATH_MAX);
668 db_root_path = strdup(optarg);
672 db_filename_list = strdup(optarg);
676 if(nb_search_conditions == MAX_NB_SEARCH_CONDITIONS) {
677 fprintf(stderr, "mymail: Too many search patterns.\n");
680 search_condition_strings[nb_search_conditions++] = strdup(optarg);
690 char *default_db_filename = getenv("MYMAIL_DB_FILE");
692 if(!default_db_filename) {
693 default_db_filename = "mymail.db";
696 db_filename = strdup(default_db_filename);
699 if(!db_filename_regexp_string) {
700 char *default_db_filename_regexp_string = getenv("MYMAIL_DB_PATTERN");
702 if(!default_db_filename_regexp_string) {
703 default_db_filename_regexp_string = "^mymail.db$";
706 db_filename_regexp_string = strdup(default_db_filename_regexp_string);
710 char *default_db_root_path = getenv("MYMAIL_DB_ROOT");
712 if(default_db_root_path) {
713 db_root_path = strdup(default_db_root_path);
717 if(!db_filename_list) {
718 char *default_db_filename_list = getenv("MYMAIL_DB_LIST");
720 if(default_db_filename_list) {
721 db_filename_list = strdup(default_db_filename_list);
725 if(output_filename[0]) {
726 output_file = fopen(output_filename, "w");
730 "mymail: Cannot open result file \"%s\" for writing: %s\n",
736 output_file = stdout;
753 db_file = fopen(db_filename, "w");
757 "mymail: Cannot open \"%s\" for writing: %s\n",
763 for(f = 0; f < nb_fields_to_parse; f++) {
764 if(regcomp(&fields_to_parse[f].regexp,
765 fields_to_parse[f].regexp_string,
768 "mymail: Syntax error in regexp \"%s\" for field \"%s\".\n",
769 fields_to_parse[f].regexp_string,
770 field_names[fields_to_parse[f].id]);
775 fprintf(db_file, "%s version_%s raw\n", MYMAIL_DB_MAGIC_TOKEN, VERSION);
777 while(optind < argc) {
778 recursive_index_mbox(db_file,
780 nb_fields_to_parse, fields_to_parse);
786 for(f = 0; f < nb_fields_to_parse; f++) {
787 regfree(&fields_to_parse[f].regexp);
793 if(nb_search_conditions > 0) {
794 struct search_condition search_conditions[MAX_NB_SEARCH_CONDITIONS];
795 char *search_field, *search_regexp_string;
798 for(n = 0; n < nb_search_conditions; n++) {
799 search_field = search_condition_strings[n];
800 search_regexp_string = segment_next_field(search_condition_strings[n]);
802 if(search_field[0] == '!') {
804 search_conditions[n].negation = 1;
806 search_conditions[n].negation = 0;
809 search_conditions[n].field_id = -1;
810 for(m = 0; (m < MAX_ID) && search_conditions[n].field_id == -1; m++) {
811 if(strncmp(field_names[m], search_field, strlen(search_field)) == 0) {
812 search_conditions[n].field_id = m;
816 if(search_conditions[n].field_id == -1) {
818 "mymail: Syntax error in field name \"%s\".\n",
823 if(regcomp(&search_conditions[n].regexp,
824 search_regexp_string,
827 "mymail: Syntax error in regexp \"%s\" for field \"%s\".\n",
828 search_regexp_string,
829 field_names[search_conditions[n].field_id]);
834 /* Recursive search if db_root_path is set */
837 regex_t db_filename_regexp;
838 if(regcomp(&db_filename_regexp,
839 db_filename_regexp_string,
842 "mymail: Syntax error in regexp \"%s\".\n",
843 db_filename_regexp_string);
847 recursive_search_in_db(db_root_path, &db_filename_regexp,
848 nb_search_conditions, search_conditions,
851 regfree(&db_filename_regexp);
854 /* Search in all db files listed in db_filename_list */
856 if(db_filename_list) {
857 char db_filename[PATH_MAX + 1];
861 s = db_filename_list;
865 while(*s == ';') { s++; }
866 while(*s && *s != ';') { *t++ = *s++; }
870 db_file = fopen(db_filename, "r");
874 "mymail: Cannot open \"%s\" for reading: %s\n",
880 search_in_db(db_file, nb_search_conditions, search_conditions, output_file);
887 /* Search in all db files listed in the command arguments */
889 while(optind < argc) {
890 FILE *db_file = fopen(argv[optind], "r");
894 "mymail: Cannot open \"%s\" for reading: %s\n",
900 search_in_db(db_file, nb_search_conditions, search_conditions, output_file);
906 for(n = 0; n < nb_search_conditions; n++) {
907 regfree(&search_conditions[n].regexp);
908 free(search_condition_strings[n]);
913 if(output_file != stdout) {
918 free(db_filename_regexp_string);
920 free(db_filename_list);