/* Examine 'path' recursively as follows: * * o Ask the filesystem for the mtime of 'path' (fs_mtime) * o Ask the database for its timestamp of 'path' (db_mtime) * * o Ask the filesystem for files and directories within 'path' * (via scandir and stored in fs_entries) * * o Pass 1: For each directory in fs_entries, recursively call into * this same function. * * o Compare fs_mtime to db_mtime. If they are equivalent, terminate * the algorithm at this point, (this directory has not been * updated in the filesystem since the last database scan of PASS * 2). * * o Ask the database for files and directories within 'path' * (db_files and db_subdirs) * * o Pass 2: Walk fs_entries simultaneously with db_files and * db_subdirs. Look for one of three interesting cases: * * 1. Regular file in fs_entries and not in db_files * This is a new file to add_message into the database. * * 2. Filename in db_files not in fs_entries. * This is a file that has been removed from the mail store. * * 3. Directory in db_subdirs not in fs_entries * This is a directory that has been removed from the mail store. * * Note that the addition of a directory is not interesting here, * since that will have been taken care of in pass 1. Also, we * don't immediately act on file/directory removal since we must * ensure that in the case of a rename that the new filename is * added before the old filename is removed, (so that no * information is lost from the database). * * o Tell the database to update its time of 'path' to 'fs_mtime' * if fs_mtime isn't the current wall-clock time. */ static notmuch_status_t add_files_recursive (notmuch_database_t *notmuch, const char *path, add_files_state_t *state) { DIR *dir = NULL; struct dirent *entry = NULL; char *next = NULL; time_t fs_mtime, db_mtime; notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS; notmuch_message_t *message = NULL; struct dirent **fs_entries = NULL; int i, num_fs_entries; notmuch_directory_t *directory; notmuch_filenames_t *db_files = NULL; notmuch_filenames_t *db_subdirs = NULL; time_t stat_time; struct stat st; notmuch_bool_t is_maildir, new_directory; const char **tag; if (stat (path, &st)) { fprintf (stderr, "Error reading directory %s: %s\n", path, strerror (errno)); return NOTMUCH_STATUS_FILE_ERROR; } stat_time = time (NULL); /* This is not an error since we may have recursed based on a * symlink to a regular file, not a directory, and we don't know * that until this stat. */ if (! S_ISDIR (st.st_mode)) return NOTMUCH_STATUS_SUCCESS; fs_mtime = st.st_mtime; directory = notmuch_database_get_directory (notmuch, path); db_mtime = notmuch_directory_get_mtime (directory); new_directory = db_mtime ? FALSE : TRUE; /* XXX This is a temporary workaround. If we don't update the * database mtime until after processing messages in this * directory, then a 0 mtime is *not* sufficient to indicate that * this directory has no messages or subdirs in the database (for * example, if an earlier run skipped the mtime update because * fs_mtime == stat_time, or was interrupted before updating the * mtime at the end). To address this, we record a (bogus) * non-zero value before processing any child messages so that a * later run won't mistake this for a new directory (and, for * example, fail to detect removed files and subdirs). * * A better solution would be for notmuch_database_get_directory * to indicate if it really created a new directory or not, either * by a new out-argument, or by recording this information and * providing an accessor. */ if (new_directory) notmuch_directory_set_mtime (directory, -1); /* If the database knows about this directory, then we sort based * on strcmp to match the database sorting. Otherwise, we can do * inode-based sorting for faster filesystem operation. */ num_fs_entries = scandir (path, &fs_entries, 0, new_directory ? dirent_sort_inode : dirent_sort_strcmp_name); if (num_fs_entries == -1) { fprintf (stderr, "Error opening directory %s: %s\n", path, strerror (errno)); ret = NOTMUCH_STATUS_FILE_ERROR; goto DONE; } /* Pass 1: Recurse into all sub-directories. */ is_maildir = _entries_resemble_maildir (fs_entries, num_fs_entries); for (i = 0; i < num_fs_entries; i++) { if (interrupted) break; entry = fs_entries[i]; /* We only want to descend into directories. * But symlinks can be to directories too, of course. * * And if the filesystem doesn't tell us the file type in the * scandir results, then it might be a directory (and if not, * then we'll stat and return immediately in the next level of * recursion). */ if (entry->d_type != DT_DIR && entry->d_type != DT_LNK && entry->d_type != DT_UNKNOWN) { continue; } /* Ignore special directories to avoid infinite recursion. * Also ignore the .notmuch directory and any "tmp" directory * that appears within a maildir. */ /* XXX: Eventually we'll want more sophistication to let the * user specify files to be ignored. */ if (strcmp (entry->d_name, ".") == 0 || strcmp (entry->d_name, "..") == 0 || (is_maildir && strcmp (entry->d_name, "tmp") == 0) || strcmp (entry->d_name, ".notmuch") ==0) { continue; } next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name); status = add_files_recursive (notmuch, next, state); if (status && ret == NOTMUCH_STATUS_SUCCESS) ret = status; talloc_free (next); next = NULL; } /* If the directory's modification time in the filesystem is the * same as what we recorded in the database the last time we * scanned it, then we can skip the second pass entirely. * * We test for strict equality here to avoid a bug that can happen * if the system clock jumps backward, (preventing new mail from * being discovered until the clock catches up and the directory * is modified again). */ if (fs_mtime == db_mtime) goto DONE; /* new_directory means a directory that the database has never * seen before. In that case, we can simply leave db_files and * db_subdirs NULL. */ if (!new_directory) { db_files = notmuch_directory_get_child_files (directory); db_subdirs = notmuch_directory_get_child_directories (directory); } /* Pass 2: Scan for new files, removed files, and removed directories. */ for (i = 0; i < num_fs_entries; i++) { if (interrupted) break; entry = fs_entries[i]; /* Check if we've walked past any names in db_files or * db_subdirs. If so, these have been deleted. */ while (notmuch_filenames_valid (db_files) && strcmp (notmuch_filenames_get (db_files), entry->d_name) < 0) { char *absolute = talloc_asprintf (state->removed_files, "%s/%s", path, notmuch_filenames_get (db_files)); _filename_list_add (state->removed_files, absolute); notmuch_filenames_move_to_next (db_files); } while (notmuch_filenames_valid (db_subdirs) && strcmp (notmuch_filenames_get (db_subdirs), entry->d_name) <= 0) { const char *filename = notmuch_filenames_get (db_subdirs); if (strcmp (filename, entry->d_name) < 0) { char *absolute = talloc_asprintf (state->removed_directories, "%s/%s", path, filename); _filename_list_add (state->removed_directories, absolute); } notmuch_filenames_move_to_next (db_subdirs); } /* If we're looking at a symlink, we only want to add it if it * links to a regular file, (and not to a directory, say). * * Similarly, if the file is of unknown type (due to filesystem * limitations), then we also need to look closer. * * In either case, a stat does the trick. */ if (entry->d_type == DT_LNK || entry->d_type == DT_UNKNOWN) { int err; next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name); err = stat (next, &st); talloc_free (next); next = NULL; /* Don't emit an error for a link pointing nowhere, since * the directory-traversal pass will have already done * that. */ if (err) continue; if (! S_ISREG (st.st_mode)) continue; } else if (entry->d_type != DT_REG) { continue; } /* Don't add a file that we've added before. */ if (notmuch_filenames_valid (db_files) && strcmp (notmuch_filenames_get (db_files), entry->d_name) == 0) { notmuch_filenames_move_to_next (db_files); continue; } /* We're now looking at a regular file that doesn't yet exist * in the database, so add it. */ next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name); state->processed_files++; if (state->verbose) { if (state->output_is_a_tty) printf("\r\033[K"); printf ("%i/%i: %s", state->processed_files, state->total_files, next); putchar((state->output_is_a_tty) ? '\r' : '\n'); fflush (stdout); } status = notmuch_database_add_message (notmuch, next, &message); switch (status) { /* success */ case NOTMUCH_STATUS_SUCCESS: state->added_messages++; notmuch_message_freeze (message); for (tag=state->new_tags; *tag != NULL; tag++) notmuch_message_add_tag (message, *tag); if (state->synchronize_flags == TRUE) notmuch_message_maildir_flags_to_tags (message); notmuch_message_thaw (message); break; /* Non-fatal issues (go on to next file) */ case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: /* Defer sync of maildir flags until after old filenames * are removed in the case of a rename. */ if (state->synchronize_flags == TRUE) _filename_list_add (state->message_ids_to_sync, notmuch_message_get_message_id (message)); break; case NOTMUCH_STATUS_FILE_NOT_EMAIL: fprintf (stderr, "Note: Ignoring non-mail file: %s\n", next); break; /* Fatal issues. Don't process anymore. */ case NOTMUCH_STATUS_READ_ONLY_DATABASE: case NOTMUCH_STATUS_XAPIAN_EXCEPTION: case NOTMUCH_STATUS_OUT_OF_MEMORY: fprintf (stderr, "Error: %s. Halting processing.\n", notmuch_status_to_string (status)); ret = status; goto DONE; default: case NOTMUCH_STATUS_FILE_ERROR: case NOTMUCH_STATUS_NULL_POINTER: case NOTMUCH_STATUS_TAG_TOO_LONG: case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW: case NOTMUCH_STATUS_LAST_STATUS: INTERNAL_ERROR ("add_message returned unexpected value: %d", status); goto DONE; } if (message) { notmuch_message_destroy (message); message = NULL; } if (do_print_progress) { do_print_progress = 0; generic_print_progress ("Processed", "files", state->tv_start, state->processed_files, state->total_files); } talloc_free (next); next = NULL; } if (interrupted) goto DONE; /* Now that we've walked the whole filesystem list, anything left * over in the database lists has been deleted. */ while (notmuch_filenames_valid (db_files)) { char *absolute = talloc_asprintf (state->removed_files, "%s/%s", path, notmuch_filenames_get (db_files)); _filename_list_add (state->removed_files, absolute); notmuch_filenames_move_to_next (db_files); } while (notmuch_filenames_valid (db_subdirs)) { char *absolute = talloc_asprintf (state->removed_directories, "%s/%s", path, notmuch_filenames_get (db_subdirs)); _filename_list_add (state->removed_directories, absolute); notmuch_filenames_move_to_next (db_subdirs); } /* If the directory's mtime is the same as the wall-clock time * when we stat'ed the directory, we skip updating the mtime in * the database because a message could be delivered later in this * same second. This may lead to unnecessary re-scans, but it * avoids overlooking messages. */ if (! interrupted && fs_mtime != stat_time) { status = notmuch_directory_set_mtime (directory, fs_mtime); if (status && ret == NOTMUCH_STATUS_SUCCESS) ret = status; } DONE: if (next) talloc_free (next); if (entry) free (entry); if (dir) closedir (dir); if (fs_entries) free (fs_entries); if (db_subdirs) notmuch_filenames_destroy (db_subdirs); if (db_files) notmuch_filenames_destroy (db_files); if (directory) notmuch_directory_destroy (directory); return ret; }
int notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[]) { notmuch_status_t status, close_status; notmuch_database_t *notmuch; struct sigaction action; const char *db_path; const char **new_tags; size_t new_tags_length; tag_op_list_t *tag_ops; char *query_string = NULL; const char *folder = NULL; notmuch_bool_t create_folder = FALSE; notmuch_bool_t keep = FALSE; notmuch_bool_t no_hooks = FALSE; notmuch_bool_t synchronize_flags; const char *maildir; char *newpath; int opt_index; unsigned int i; notmuch_opt_desc_t options[] = { { NOTMUCH_OPT_STRING, &folder, "folder", 0, 0 }, { NOTMUCH_OPT_BOOLEAN, &create_folder, "create-folder", 0, 0 }, { NOTMUCH_OPT_BOOLEAN, &keep, "keep", 0, 0 }, { NOTMUCH_OPT_BOOLEAN, &no_hooks, "no-hooks", 'n', 0 }, { NOTMUCH_OPT_END, 0, 0, 0, 0 } }; opt_index = parse_arguments (argc, argv, options, 1); if (opt_index < 0) return EXIT_FAILURE; db_path = notmuch_config_get_database_path (config); new_tags = notmuch_config_get_new_tags (config, &new_tags_length); synchronize_flags = notmuch_config_get_maildir_synchronize_flags (config); tag_ops = tag_op_list_create (config); if (tag_ops == NULL) { fprintf (stderr, "Out of memory.\n"); return EXIT_FAILURE; } for (i = 0; i < new_tags_length; i++) { const char *error_msg; error_msg = illegal_tag (new_tags[i], FALSE); if (error_msg) { fprintf (stderr, "Error: tag '%s' in new.tags: %s\n", new_tags[i], error_msg); return EXIT_FAILURE; } if (tag_op_list_append (tag_ops, new_tags[i], FALSE)) return EXIT_FAILURE; } if (parse_tag_command_line (config, argc - opt_index, argv + opt_index, &query_string, tag_ops)) return EXIT_FAILURE; if (*query_string != '\0') { fprintf (stderr, "Error: unexpected query string: %s\n", query_string); return EXIT_FAILURE; } if (folder == NULL) { maildir = db_path; } else { if (! is_valid_folder_name (folder)) { fprintf (stderr, "Error: invalid folder name: '%s'\n", folder); return EXIT_FAILURE; } maildir = talloc_asprintf (config, "%s/%s", db_path, folder); if (! maildir) { fprintf (stderr, "Out of memory\n"); return EXIT_FAILURE; } if (create_folder && ! maildir_create_folder (config, maildir)) return EXIT_FAILURE; } /* Setup our handler for SIGINT. We do not set SA_RESTART so that copying * from standard input may be interrupted. */ memset (&action, 0, sizeof (struct sigaction)); action.sa_handler = handle_sigint; sigemptyset (&action.sa_mask); action.sa_flags = 0; sigaction (SIGINT, &action, NULL); if (notmuch_database_open (notmuch_config_get_database_path (config), NOTMUCH_DATABASE_MODE_READ_WRITE, ¬much)) return EXIT_FAILURE; /* Write the message to the Maildir new directory. */ newpath = maildir_write_new (config, STDIN_FILENO, maildir); if (! newpath) { notmuch_database_destroy (notmuch); return EXIT_FAILURE; } /* Index the message. */ status = add_file (notmuch, newpath, tag_ops, synchronize_flags, keep); /* Commit changes. */ close_status = notmuch_database_destroy (notmuch); if (close_status) { /* Hold on to the first error, if any. */ if (! status) status = close_status; fprintf (stderr, "%s: failed to commit database changes: %s\n", keep ? "Warning" : "Error", notmuch_status_to_string (close_status)); } if (status) { if (keep) { status = NOTMUCH_STATUS_SUCCESS; } else { /* If maildir flag sync failed, this might fail. */ if (unlink (newpath)) { fprintf (stderr, "Warning: failed to remove '%s' from maildir " "after errors: %s. Please run 'notmuch new' to fix.\n", newpath, strerror (errno)); } } } if (! no_hooks && status == NOTMUCH_STATUS_SUCCESS) { /* Ignore hook failures. */ notmuch_run_hook (db_path, "post-insert"); } return status ? EXIT_FAILURE : EXIT_SUCCESS; }
static int tag_message (notmuch_database_t *notmuch, const char *message_id, char *file_tags, notmuch_bool_t remove_all, notmuch_bool_t synchronize_flags) { notmuch_status_t status; notmuch_tags_t *db_tags; char *db_tags_str; notmuch_message_t *message = NULL; const char *tag; char *next; int ret = 0; status = notmuch_database_find_message (notmuch, message_id, &message); if (status || message == NULL) { fprintf (stderr, "Warning: Cannot apply tags to %smessage: %s\n", message ? "" : "missing ", message_id); if (status) fprintf (stderr, "%s\n", notmuch_status_to_string(status)); return 1; } /* In order to detect missing messages, this check/optimization is * intentionally done *after* first finding the message. */ if (!remove_all && (file_tags == NULL || *file_tags == '\0')) goto DONE; db_tags_str = NULL; for (db_tags = notmuch_message_get_tags (message); notmuch_tags_valid (db_tags); notmuch_tags_move_to_next (db_tags)) { tag = notmuch_tags_get (db_tags); if (db_tags_str) db_tags_str = talloc_asprintf_append (db_tags_str, " %s", tag); else db_tags_str = talloc_strdup (message, tag); } if (((file_tags == NULL || *file_tags == '\0') && (db_tags_str == NULL || *db_tags_str == '\0')) || (file_tags && db_tags_str && strcmp (file_tags, db_tags_str) == 0)) goto DONE; notmuch_message_freeze (message); if (remove_all) notmuch_message_remove_all_tags (message); next = file_tags; while (next) { tag = strsep (&next, " "); if (*tag == '\0') continue; status = notmuch_message_add_tag (message, tag); if (status) { fprintf (stderr, "Error applying tag %s to message %s:\n", tag, message_id); fprintf (stderr, "%s\n", notmuch_status_to_string (status)); ret = 1; } } notmuch_message_thaw (message); if (synchronize_flags) notmuch_message_tags_to_maildir_flags (message); DONE: if (message) notmuch_message_destroy (message); return ret; }
/* * Add the specified message file to the notmuch database, applying * tags in tag_ops. If synchronize_flags is TRUE, the tags are * synchronized to maildir flags (which may result in message file * rename). * * Return NOTMUCH_STATUS_SUCCESS on success, errors otherwise. If keep * is TRUE, errors in tag changes and flag syncing are ignored and * success status is returned; otherwise such errors cause the message * to be removed from the database. Failure to add the message to the * database results in error status regardless of keep. */ static notmuch_status_t add_file (notmuch_database_t *notmuch, const char *path, tag_op_list_t *tag_ops, notmuch_bool_t synchronize_flags, notmuch_bool_t keep) { notmuch_message_t *message; notmuch_status_t status; status = notmuch_database_add_message (notmuch, path, &message); if (status == NOTMUCH_STATUS_SUCCESS) { status = tag_op_list_apply (message, tag_ops, 0); if (status) { fprintf (stderr, "%s: failed to apply tags to file '%s': %s\n", keep ? "Warning" : "Error", path, notmuch_status_to_string (status)); goto DONE; } } else if (status == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) { status = NOTMUCH_STATUS_SUCCESS; } else if (status == NOTMUCH_STATUS_FILE_NOT_EMAIL) { fprintf (stderr, "Error: delivery of non-mail file: '%s'\n", path); goto FAIL; } else { fprintf (stderr, "Error: failed to add '%s' to notmuch database: %s\n", path, notmuch_status_to_string (status)); goto FAIL; } if (synchronize_flags) { status = notmuch_message_tags_to_maildir_flags (message); if (status != NOTMUCH_STATUS_SUCCESS) fprintf (stderr, "%s: failed to sync tags to maildir flags for '%s': %s\n", keep ? "Warning" : "Error", path, notmuch_status_to_string (status)); /* * Note: Unfortunately a failed maildir flag sync might * already have renamed the file, in which case the cleanup * path may fail. */ } DONE: notmuch_message_destroy (message); if (status) { if (keep) { status = NOTMUCH_STATUS_SUCCESS; } else { notmuch_status_t cleanup_status; cleanup_status = notmuch_database_remove_message (notmuch, path); if (cleanup_status != NOTMUCH_STATUS_SUCCESS && cleanup_status != NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) { fprintf (stderr, "Warning: failed to remove '%s' from database " "after errors: %s. Please run 'notmuch new' to fix.\n", path, notmuch_status_to_string (cleanup_status)); } } } FAIL: return status; }