3 * Copyright (c) 2013 Francois Fleuret
4 * Written by Francois Fleuret <francois@fleuret.org>
6 * This file is part of mymail.
8 * mymail is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 3 as
10 * published by the Free Software Foundation.
12 * mymail is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with mymail. If not, see <http://www.gnu.org/licenses/>.
24 This command is a dumb mail indexer. It can either (1) scan
25 directories containing mbox files, and create a db file containing
26 for each mail a list of fields computed from the header, or (2)
27 read such a db file and get all the mails matching regexp-defined
28 conditions on the fields.
30 It is low-tech, simple, light and fast.
47 #define MYMAIL_DB_MAGIC_TOKEN "mymail_index_file"
50 #define MAX_NB_SEARCH_REQUESTS 10
52 #define BUFFER_SIZE 65536
55 char *db_filename_regexp_string;
57 char *db_filename_list;
62 /********************************************************************/
74 static char *field_names[] = {
83 /********************************************************************/
85 struct search_request {
91 /********************************************************************/
93 struct parsable_field {
99 static struct parsable_field fields_to_parse[] = {
102 "^\\(From \\|[Ff][Rr][Oo][Mm]:\\|[R][r][E][e][P][p][L][l][Y][y]-[T][t][O][o]:\\)",
103 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
108 "^\\([Tt][Oo]\\|[Cc][Cc]\\|[Bb][Cc][Cc]\\): ",
109 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
114 "^[Ss][Uu][Bb][Jj][Ee][Cc][Tt]: ",
115 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
120 "^[Dd][Aa][Tt][Ee]: ",
121 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
126 /********************************************************************/
128 int xor(int a, int b) {
129 return (a && !b) || (!a && b);
132 char *segment_next_field(char *current) {
133 while(*current && *current != ' ') current++;
134 *current = '\0'; current++;
135 while(*current && *current == ' ') current++;
139 void remove_eof(char *c) {
140 while(*c && *c != '\n' && *c != '\r') c++;
144 /********************************************************************/
146 /* malloc with error checking. */
148 void *safe_malloc(size_t n) {
152 "mymail: cannot allocate memory: %s\n", strerror(errno));
158 /*********************************************************************/
160 void print_version(FILE *out) {
161 fprintf(out, "mymail version %s (%s)\n", VERSION, UNAME);
164 void print_usage(FILE *out) {
166 fprintf(out, "Written by Francois Fleuret <francois@fleuret.org>.\n");
168 fprintf(out, "Usage: mymail [options] [<mbox dir1> [<mbox dir2> ...]|<db file1> [<db file2> ...]]\n");
170 fprintf(out, " -h, --help\n");
171 fprintf(out, " show this help\n");
172 fprintf(out, " -v, --version\n");
173 fprintf(out, " print the version number\n");
174 fprintf(out, " -p <db filename pattern>, --db-pattern <db filename pattern>\n");
175 fprintf(out, " set the db filename pattern for recursive search\n");
176 fprintf(out, " -r <db root path>, --db-root <db root path>\n");
177 fprintf(out, " set the db root path for recursive search\n");
178 fprintf(out, " -l <db filename list>, --db-list <db filename list>\n");
179 fprintf(out, " set the semicolon-separated list of db files for search\n");
180 fprintf(out, " -s <search pattern>, --search <search pattern>\n");
181 fprintf(out, " search for matching mails in the db file\n");
182 fprintf(out, " -d <db filename>, --db-file <db filename>\n");
183 fprintf(out, " set the db filename for indexing\n");
184 fprintf(out, " -i, --index\n");
185 fprintf(out, " index mails\n");
188 /*********************************************************************/
190 int ignore_entry(const char *name) {
192 /* strcmp(name, ".") == 0 || */
193 /* strcmp(name, "..") == 0 || */
194 (name[0] == '.' && name[1] != '/');
197 int mbox_line_match_search(struct search_request *request,
198 int mbox_id, char *mbox_value) {
200 (request->field_id == mbox_id ||
201 (request->field_id == ID_PARTICIPANT && (mbox_id == ID_FROM || mbox_id == ID_TO)))
203 regexec(&request->regexp, mbox_value, 0, 0, 0) == 0;
206 void search_in_db(int nb_search_requests,
207 struct search_request *search_requests,
209 int hits[MAX_NB_SEARCH_REQUESTS];
210 char raw_db_line[BUFFER_SIZE];
211 char raw_mbox_line[BUFFER_SIZE];
212 char current_mail_filename[PATH_MAX + 1];
213 unsigned long int current_position_in_mail;
214 char *mbox_name, *mbox_value;
216 int already_written, m, n;
217 int last_mbox_line_was_empty;
219 current_position_in_mail = 0;
222 for(n = 0; n < nb_search_requests; n++) { hits[n] = 0; }
224 while(fgets(raw_db_line, BUFFER_SIZE, db_file)) {
225 mbox_name = raw_db_line;
226 mbox_value = segment_next_field(raw_db_line);
228 if(strcmp("mail", mbox_name) == 0) {
229 char *position_in_file_string;
232 for(n = 0; n < nb_search_requests && xor(hits[n], search_requests[n].negation); n++);
234 /* for(n = 0; n < nb_search_requests && */
235 /* ((hits[n] && !search_requests[n].negation) || */
236 /* (!hits[n] && search_requests[n].negation)); n++); */
238 if(n == nb_search_requests) {
241 mail_file = fopen(current_mail_filename, "r");
244 fprintf(stderr, "mymail: Cannot open mbox '%s'.\n", current_mail_filename);
248 fseek(mail_file, current_position_in_mail, SEEK_SET);
250 if(fgets(raw_mbox_line, BUFFER_SIZE, mail_file)) {
251 last_mbox_line_was_empty = 1;
252 printf("%s", raw_mbox_line);
254 if(!fgets(raw_mbox_line, BUFFER_SIZE, mail_file) ||
255 (last_mbox_line_was_empty && strncmp(raw_mbox_line, "From ", 5) == 0)) break;
256 last_mbox_line_was_empty = (raw_mbox_line[0] == '\n');
257 printf("%s", raw_mbox_line);
264 for(n = 0; n < nb_search_requests; n++) { hits[n] = 0; }
266 position_in_file_string = mbox_value;
267 mail_filename = segment_next_field(mbox_value);
268 current_position_in_mail = atol(position_in_file_string);
269 strcpy(current_mail_filename, mail_filename);
271 remove_eof(current_mail_filename);
277 for(m = 0; (m < MAX_ID) && mbox_id == -1; m++) {
278 if(strncmp(field_names[m], mbox_name, strlen(mbox_name)) == 0) {
282 for(n = 0; n < nb_search_requests; n++) {
283 hits[n] |= mbox_line_match_search(&search_requests[n],
284 mbox_id, mbox_value);
290 void recursive_search_in_db(const char *entry_name, regex_t *db_filename_regexp,
291 int nb_search_requests, struct search_request *search_requests) {
293 struct dirent *dir_e;
295 char raw_db_line[BUFFER_SIZE];
296 char subname[PATH_MAX + 1];
298 if(lstat(entry_name, &sb) != 0) {
300 "mymail: Cannot stat \"%s\": %s\n",
306 dir = opendir(entry_name);
309 while((dir_e = readdir(dir))) {
310 if(!ignore_entry(dir_e->d_name)) {
311 snprintf(subname, PATH_MAX, "%s/%s", entry_name, dir_e->d_name);
312 recursive_search_in_db(subname, db_filename_regexp,
313 nb_search_requests, search_requests);
320 const char *s = entry_name, *filename = entry_name;
321 while(*s) { if(*s == '/') { filename = s+1; } s++; }
323 if(regexec(db_filename_regexp, filename, 0, 0, 0) == 0) {
324 FILE *db_file = fopen(entry_name, "r");
328 "mymail: Cannot open \"%s\" for reading: %s\n",
334 if(fgets(raw_db_line, BUFFER_SIZE, db_file)) {
335 if(strncmp(raw_db_line, MYMAIL_DB_MAGIC_TOKEN, strlen(MYMAIL_DB_MAGIC_TOKEN))) {
337 "mymail: Header line in '%s' does not match the mymail db format.\n",
343 "mymail: Cannot read the header line in '%s'.\n",
348 search_in_db(nb_search_requests, search_requests, db_file);
355 /*********************************************************************/
357 void index_one_mbox_line(int nb_fields_to_parse, struct parsable_field *fields_to_parse,
358 char *raw_mbox_line, FILE *db_file) {
361 for(f = 0; f < nb_fields_to_parse; f++) {
362 if(regexec(&fields_to_parse[f].regexp, raw_mbox_line, 1, &matches, 0) == 0) {
363 fprintf(db_file, "%s %s\n",
364 field_names[fields_to_parse[f].id],
365 raw_mbox_line + matches.rm_eo);
370 void index_mbox(const char *mbox_filename,
371 int nb_fields_to_parse, struct parsable_field *fields_to_parse,
373 char raw_mbox_line[BUFFER_SIZE], full_line[BUFFER_SIZE];
374 char *end_of_full_line;
376 int in_header, new_header, last_mbox_line_was_empty;
377 unsigned long int position_in_file;
379 file = fopen(mbox_filename, "r");
382 fprintf(stderr, "mymail: Cannot open '%s'.\n", mbox_filename);
383 if(paranoid) { exit(EXIT_FAILURE); }
390 position_in_file = 0;
391 end_of_full_line = 0;
393 last_mbox_line_was_empty = 1;
395 while(fgets(raw_mbox_line, BUFFER_SIZE, file)) {
396 if(last_mbox_line_was_empty && strncmp(raw_mbox_line, "From ", 5) == 0) {
399 "Got a ^\"From \" in the header in %s:%lu.\n",
400 mbox_filename, position_in_file);
401 fprintf(stderr, "%s", raw_mbox_line);
402 if(paranoid) { exit(EXIT_FAILURE); }
406 } else if(raw_mbox_line[0] == '\n') {
407 if(in_header) { in_header = 0; }
410 last_mbox_line_was_empty = (raw_mbox_line[0] == '\n');
414 fprintf(db_file, "mail %lu %s\n", position_in_file, mbox_filename);
418 if(raw_mbox_line[0] == ' ' || raw_mbox_line[0] == '\t') {
419 char *start = raw_mbox_line;
420 while(*start == ' ' || *start == '\t') start++;
421 *(end_of_full_line++) = ' ';
422 strcpy(end_of_full_line, start);
423 while(*end_of_full_line && *end_of_full_line != '\n') {
426 *end_of_full_line = '\0';
431 if(!((raw_mbox_line[0] >= 'a' && raw_mbox_line[0] <= 'z') ||
432 (raw_mbox_line[0] >= 'A' && raw_mbox_line[0] <= 'Z'))) {
434 "Header line syntax error %s:%lu.\n",
435 mbox_filename, position_in_file);
436 fprintf(stderr, "%s", raw_mbox_line);
441 index_one_mbox_line(nb_fields_to_parse, fields_to_parse, full_line, db_file);
444 end_of_full_line = full_line;
445 strcpy(end_of_full_line, raw_mbox_line);
446 while(*end_of_full_line && *end_of_full_line != '\n') {
449 *end_of_full_line = '\0';
454 position_in_file += strlen(raw_mbox_line);
460 void recursive_index_mbox(FILE *db_file,
461 const char *entry_name,
462 int nb_fields_to_parse, struct parsable_field *fields_to_parse) {
464 struct dirent *dir_e;
466 char subname[PATH_MAX + 1];
468 if(lstat(entry_name, &sb) != 0) {
470 "mymail: Cannot stat \"%s\": %s\n",
476 dir = opendir(entry_name);
479 while((dir_e = readdir(dir))) {
480 if(!ignore_entry(dir_e->d_name)) {
481 snprintf(subname, PATH_MAX, "%s/%s", entry_name, dir_e->d_name);
482 recursive_index_mbox(db_file, subname, nb_fields_to_parse, fields_to_parse);
487 index_mbox(entry_name, nb_fields_to_parse, fields_to_parse, db_file);
491 /*********************************************************************/
493 /* For long options that have no equivalent short option, use a
494 non-character as a pseudo short option, starting with CHAR_MAX + 1. */
496 OPT_BASH_MODE = CHAR_MAX + 1
499 static struct option long_options[] = {
500 { "help", no_argument, 0, 'h' },
501 { "version", no_argument, 0, 'v' },
502 { "db-file", 1, 0, 'd' },
503 { "db-pattern", 1, 0, 'p' },
504 { "db-root", 1, 0, 'r' },
505 { "db-list", 1, 0, 'l' },
506 { "search", 1, 0, 's' },
507 { "index", 0, 0, 'i' },
511 /*********************************************************************/
513 int main(int argc, char **argv) {
514 int error = 0, show_help = 0;
515 const int nb_fields_to_parse = sizeof(fields_to_parse) / sizeof(struct parsable_field);
518 int nb_search_requests;
519 char *search_request_strings[MAX_NB_SEARCH_REQUESTS];
521 /* for(f = 0; f < argc; f++) { */
522 /* printf("arg %d \"%s\"\n", f, argv[f]); */
529 db_filename_list = 0;
531 setlocale(LC_ALL, "");
533 nb_search_requests = 0;
535 while ((c = getopt_long(argc, argv, "hvip:s:d:r:l:",
536 long_options, NULL)) != -1) {
545 print_version(stdout);
553 db_filename = strdup(optarg);
557 db_filename_regexp_string = strdup(optarg);
561 db_root_path = strdup(optarg);
565 db_filename_list = strdup(optarg);
569 if(nb_search_requests == MAX_NB_SEARCH_REQUESTS) {
570 fprintf(stderr, "mymail: Too many search patterns.\n");
573 search_request_strings[nb_search_requests++] = strdup(optarg);
583 char *default_db_filename = getenv("MYMAIL_DB_FILE");
585 if(!default_db_filename) {
586 default_db_filename = "mymail.db";
589 db_filename = strdup(default_db_filename);
592 if(!db_filename_regexp_string) {
593 char *default_db_filename_regexp_string = getenv("MYMAIL_DB_PATTERN");
595 if(!default_db_filename_regexp_string) {
596 default_db_filename_regexp_string = "^mymail.db$";
599 db_filename_regexp_string = strdup(default_db_filename_regexp_string);
603 char *default_db_root_path = getenv("MYMAIL_DB_ROOT");
605 if(default_db_root_path) {
606 db_root_path = strdup(default_db_root_path);
610 if(!db_filename_list) {
611 char *default_db_filename_list = getenv("MYMAIL_DB_LIST");
613 if(default_db_filename_list) {
614 db_filename_list = strdup(default_db_filename_list);
631 db_file = fopen(db_filename, "w");
635 "mymail: Cannot open \"%s\" for writing: %s\n",
641 for(f = 0; f < nb_fields_to_parse; f++) {
642 if(regcomp(&fields_to_parse[f].regexp,
643 fields_to_parse[f].regexp_string,
646 "mymail: Syntax error in regexp \"%s\" for field \"%s\".\n",
647 fields_to_parse[f].regexp_string,
648 field_names[fields_to_parse[f].id]);
653 fprintf(db_file, "%s version_%s raw\n", MYMAIL_DB_MAGIC_TOKEN, VERSION);
655 while(optind < argc) {
656 recursive_index_mbox(db_file,
658 nb_fields_to_parse, fields_to_parse);
664 for(f = 0; f < nb_fields_to_parse; f++) {
665 regfree(&fields_to_parse[f].regexp);
671 if(nb_search_requests > 0) {
672 struct search_request search_requests[MAX_NB_SEARCH_REQUESTS];
673 char *search_field, *search_regexp_string;
676 for(n = 0; n < nb_search_requests; n++) {
677 search_field = search_request_strings[n];
678 search_regexp_string = segment_next_field(search_request_strings[n]);
680 if(search_field[0] == '!') {
682 search_requests[n].negation = 1;
684 search_requests[n].negation = 0;
687 search_requests[n].field_id = -1;
688 for(m = 0; (m < MAX_ID) && search_requests[n].field_id == -1; m++) {
689 if(strncmp(field_names[m], search_field, strlen(search_field)) == 0) {
690 search_requests[n].field_id = m;
694 if(search_requests[n].field_id == -1) {
696 "mymail: Syntax error in field name \"%s\".\n",
701 if(regcomp(&search_requests[n].regexp,
702 search_regexp_string,
705 "mymail: Syntax error in regexp \"%s\" for field \"%s\".\n",
706 search_regexp_string,
707 field_names[search_requests[n].field_id]);
712 /* Recursive search if db_root_path is set */
715 regex_t db_filename_regexp;
716 if(regcomp(&db_filename_regexp,
717 db_filename_regexp_string,
720 "mymail: Syntax error in regexp \"%s\".\n",
721 db_filename_regexp_string);
725 recursive_search_in_db(db_root_path, &db_filename_regexp,
726 nb_search_requests, search_requests);
728 regfree(&db_filename_regexp);
731 /* Search in all db files listed in db_filename_list */
733 if(db_filename_list) {
734 char db_filename[PATH_MAX + 1];
738 s = db_filename_list;
742 while(*s == ';') { s++; }
743 while(*s && *s != ';') { *t++ = *s++; }
747 db_file = fopen(db_filename, "r");
751 "mymail: Cannot open \"%s\" for reading: %s\n",
757 search_in_db(nb_search_requests, search_requests, db_file);
764 /* Search in all db files listed in the command arguments */
766 while(optind < argc) {
767 FILE *db_file = fopen(argv[optind], "r");
771 "mymail: Cannot open \"%s\" for reading: %s\n",
777 search_in_db(nb_search_requests, search_requests, db_file);
783 for(n = 0; n < nb_search_requests; n++) {
784 regfree(&search_requests[n].regexp);
785 free(search_request_strings[n]);
791 free(db_filename_regexp_string);
793 free(db_filename_list);