Chris@538: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
Chris@538: 
Chris@538: /*
Chris@538:     EasyMercurial
Chris@538: 
Chris@538:     Based on hgExplorer by Jari Korhonen
Chris@538:     Copyright (c) 2010 Jari Korhonen
Chris@560:     Copyright (c) 2012 Chris Cannam
Chris@560:     Copyright (c) 2012 Queen Mary, University of London
Chris@538:     
Chris@538:     This program is free software; you can redistribute it and/or
Chris@538:     modify it under the terms of the GNU General Public License as
Chris@538:     published by the Free Software Foundation; either version 2 of the
Chris@538:     License, or (at your option) any later version.  See the file
Chris@538:     COPYING included with this distribution for more information.
Chris@538: */
Chris@538: 
Chris@584: #include <QMutexLocker>
Chris@584: #include <QDir>
Chris@584: 
Chris@584: #ifdef Q_OS_MAC
Chris@584: // Must include this before debug.h
Chris@584: #include <CoreServices/CoreServices.h>
Chris@584: #endif
Chris@584: 
Chris@538: #include "fswatcher.h"
Chris@540: #include "debug.h"
Chris@538: 
Chris@538: #include <deque>
Chris@538: 
Chris@585: #define DEBUG_FSWATCHER 1
Chris@540: 
Chris@539: /*
Chris@539:  * Watching the filesystem is trickier than it seems at first glance.
Chris@539:  *
Chris@539:  * We ideally should watch every directory, and every file that is
Chris@539:  * tracked by Hg. If a new file is created in a directory, then we
Chris@539:  * need to respond in order to show it as a potential candidate to be
Chris@539:  * added.
Chris@539:  *
Chris@539:  * Complicating matters though is that Hg itself might modify the
Chris@539:  * filesystem. This can happen even in "read-only" operations: for
Chris@539:  * example, hg stat creates files called hg-checklink and hg-checkexec
Chris@539:  * to test properties of the filesystem. So we need to know to ignore
Chris@539:  * those files; unfortunately, when watching a directory (which is how
Chris@539:  * we find out about the creation of new files) we are notified only
Chris@540:  * that the directory has changed -- we aren't told what changed.
Chris@540:  *
Chris@540:  * This means that, when a directory changes, we need to rescan the
Chris@540:  * directory to learn whether the set of files in it _excluding_ files
Chris@540:  * matching our ignore patterns differs from the previous scan, and
Chris@540:  * ignore the change if it doesn't.
Chris@584:  *
Chris@584:  */
Chris@584: 
Chris@584: /*
Chris@584:  * 20120312 -- Another complication. The documentation for
Chris@584:  * QFileSystemWatcher says:
Chris@584:  *
Chris@584:  *     On Mac OS X 10.4 [...] an open file descriptor is required for
Chris@584:  *     each monitored file. [...] This means that addPath() and
Chris@584:  *     addPaths() will fail if your process tries to add more than 256
Chris@584:  *     files or directories to the file system monitor [...] Mac OS X
Chris@584:  *     10.5 and up use a different backend and do not suffer from this
Chris@584:  *     issue.
Chris@584:  * 
Chris@584:  * Unfortunately, the last sentence above is not true:
Chris@584:  * http://qt.gitorious.org/qt/qt/commit/6d1baf9979346d6f15da81a535becb4046278962
Chris@584:  * ("Removing the usage of FSEvents-based backend for now as it has a
Chris@584:  * few bugs...").  It can't be restored without hacking the Qt source,
Chris@584:  * which we don't want to do in this context. The commit log doesn't
Chris@584:  * make clear how serious the bugs were -- an example is given but it
Chris@584:  * doesn't indicate whether it's an edge case or a common case and
Chris@584:  * whether the result was a crash or failure to notify.
Chris@584:  *
Chris@584:  * This means the Qt class uses kqueue instead on OS/X, but that
Chris@584:  * doesn't really work for us -- it can only monitor 256 files (or
Chris@584:  * whatever the fd ulimit is set to, but that's the default) and it
Chris@584:  * doesn't notify if a file within a directory is modified unless the
Chris@584:  * metadata changes. The main limitation of FSEvents is that it only
Chris@586:  * notifies with directory granularity, but that might be OK for us so
Chris@586:  * long as notifications are actually provoked by file changes as
Chris@586:  * well. (In OS/X 10.7 there appear to be file-level notifications
Chris@586:  * too, but that doesn't help us.)
Chris@584:  *
Chris@584:  * One other problem with FSEvents is that the API only exists on OS/X
Chris@584:  * 10.5 or newer -- on older versions we would have no option but to
Chris@584:  * use kqueue via QFileSystemWatcher. But we can't ship a binary
Chris@584:  * linked with the FSEvents API to run on 10.4 without some fiddling,
Chris@584:  * and I'm not really keen to do that either.  That may be our cue to
Chris@584:  * drop 10.4 support for EasyMercurial.
Chris@539:  */
Chris@539: 
Chris@538: FsWatcher::FsWatcher() :
Chris@538:     m_lastToken(0),
Chris@538:     m_lastCounter(0)
Chris@538: {
Chris@584: #ifdef Q_OS_MAC
Chris@584:     m_stream = 0; // create when we have a path
Chris@584: #else
Chris@538:     connect(&m_watcher, SIGNAL(directoryChanged(QString)),
Chris@538: 	    this, SLOT(fsDirectoryChanged(QString)));
Chris@538:     connect(&m_watcher, SIGNAL(fileChanged(QString)),
Chris@538: 	    this, SLOT(fsFileChanged(QString)));
Chris@584: #endif
Chris@538: }
Chris@538: 
Chris@538: FsWatcher::~FsWatcher()
Chris@538: {
Chris@538: }
Chris@538: 
Chris@538: void
Chris@538: FsWatcher::setWorkDirPath(QString path)
Chris@538: {
Chris@538:     QMutexLocker locker(&m_mutex);
Chris@541:     if (m_workDirPath == path) return;
Chris@584:     clearWatchedPaths();
Chris@584:     m_workDirPath = path;
Chris@584:     addWorkDirectory(path);
Chris@584:     debugPrint();
Chris@584: }
Chris@584: 
Chris@584: void
Chris@584: FsWatcher::clearWatchedPaths()
Chris@584: {
Chris@584: #ifdef Q_OS_MAC
Chris@584:     FSEventStreamRef stream = (FSEventStreamRef)m_stream;
Chris@584:     if (stream) {
Chris@584:         FSEventStreamStop(stream);
Chris@584:         FSEventStreamInvalidate(stream);
Chris@584:         FSEventStreamRelease(stream);
Chris@584:     }
Chris@584:     m_stream = 0;
Chris@584: #else
Chris@562:     // annoyingly, removePaths prints a warning if given an empty list
Chris@562:     if (!m_watcher.directories().empty()) {
Chris@562:         m_watcher.removePaths(m_watcher.directories());
Chris@562:     }
Chris@562:     if (!m_watcher.files().empty()) {
Chris@562:         m_watcher.removePaths(m_watcher.files());
Chris@562:     }
Chris@584: #endif
Chris@538: }
Chris@538: 
Chris@584: #ifdef Q_OS_MAC
Chris@584: static void
Chris@585: fsEventsCallback(ConstFSEventStreamRef streamRef,
Chris@584:                  void *clientCallBackInfo,
Chris@585:                  size_t numEvents,
Chris@585:                  void *paths,
Chris@585:                  const FSEventStreamEventFlags eventFlags[],
Chris@585:                  const FSEventStreamEventId eventIDs[])
Chris@539: {
Chris@585:     FsWatcher *watcher = reinterpret_cast<FsWatcher *>(clientCallBackInfo);
Chris@585:     const char *const *cpaths = reinterpret_cast<const char *const *>(paths);
Chris@585:     for (size_t i = 0; i < numEvents; ++i) {
Chris@586:         std::cerr << "path " << i << " = " << cpaths[i] << std::endl;
Chris@585:         watcher->fsDirectoryChanged(QString::fromLocal8Bit(cpaths[i]));
Chris@585:     }
Chris@539: }
Chris@584: #endif
Chris@539: 
Chris@539: void
Chris@538: FsWatcher::addWorkDirectory(QString path)
Chris@538: {
Chris@584: #ifdef Q_OS_MAC
Chris@585: 
Chris@585:     CFStringRef cfPath = CFStringCreateWithCharacters
Chris@585:         (0, reinterpret_cast<const UniChar *>(path.unicode()),
Chris@585:          path.length());
Chris@585: 
Chris@585:     CFArrayRef cfPaths = CFArrayCreate(0, (const void **)&cfPath, 1, 0);
Chris@585: 
Chris@585:     FSEventStreamContext ctx = { 0, 0, 0, 0, 0 };
Chris@585:     ctx.info = this;
Chris@585: 
Chris@584:     FSEventStreamRef stream =
Chris@584:         FSEventStreamCreate(kCFAllocatorDefault,
Chris@585:                             &fsEventsCallback,
Chris@585:                             &ctx,
Chris@584:                             cfPaths,
Chris@584:                             kFSEventStreamEventIdSinceNow,
Chris@584:                             1.0, // latency, seconds
Chris@584:                             kFSEventStreamCreateFlagNone);
Chris@584: 
Chris@584:     m_stream = stream;
Chris@584:     
Chris@584:     FSEventStreamScheduleWithRunLoop(stream,
Chris@584:                                      CFRunLoopGetCurrent(),
Chris@584:                                      kCFRunLoopDefaultMode);
Chris@584: 
Chris@584:     if (!FSEventStreamStart(stream)) {
Chris@584:         std::cerr << "ERROR: FsWatcher::addWorkDirectory: Failed to start FSEvent stream" << std::endl;
Chris@584:     }
Chris@584: #else
Chris@538:     // QFileSystemWatcher will refuse to add a file or directory to
Chris@538:     // its watch list that it is already watching -- fine -- but it
Chris@538:     // prints a warning when this happens, which we wouldn't want.  So
Chris@538:     // we'll check for duplicates ourselves.
Chris@538:     QSet<QString> alreadyWatched = 
Chris@538: 	QSet<QString>::fromList(m_watcher.directories());
Chris@538:     
Chris@538:     std::deque<QString> pending;
Chris@538:     pending.push_back(path);
Chris@538: 
Chris@538:     while (!pending.empty()) {
Chris@538: 
Chris@538:         QString path = pending.front();
Chris@538:         pending.pop_front();
Chris@538:         if (!alreadyWatched.contains(path)) {
Chris@538:             m_watcher.addPath(path);
Chris@540:             m_dirContents[path] = scanDirectory(path);
Chris@538:         }
Chris@538: 
Chris@538:         QDir d(path);
Chris@538:         if (d.exists()) {
Chris@538:             d.setFilter(QDir::Dirs | QDir::NoDotAndDotDot |
Chris@538:                         QDir::Readable | QDir::NoSymLinks);
Chris@538:             foreach (QString entry, d.entryList()) {
Chris@538:                 if (entry.startsWith('.')) continue;
Chris@538:                 QString entryPath = d.absoluteFilePath(entry);
Chris@538:                 pending.push_back(entryPath);
Chris@538:             }
Chris@538:         }
Chris@538:     }
Chris@584: #endif
Chris@584: }
Chris@584: 
Chris@584: void
Chris@584: FsWatcher::setTrackedFilePaths(QStringList paths)
Chris@584: {
Chris@584: #ifdef Q_OS_MAC
Chris@586: 
Chris@586:     // FSEvents will notify when any file in the directory changes,
Chris@586:     // but we need to be able to check whether the file change was
Chris@586:     // meaningful to us if it didn't result in any files being added
Chris@586:     // or removed -- and we have to do that by examining timestamps on
Chris@586:     // the files we care about
Chris@586:     foreach (QString p, paths) {
Chris@586:         m_trackedFileUpdates[p] = QDateTime::currentDateTime();
Chris@586:     }
Chris@586: 
Chris@584: #else
Chris@586: 
Chris@584:     QMutexLocker locker(&m_mutex);
Chris@584: 
Chris@584:     QSet<QString> alreadyWatched = 
Chris@584: 	QSet<QString>::fromList(m_watcher.files());
Chris@584: 
Chris@584:     foreach (QString path, paths) {
Chris@584:         path = m_workDirPath + QDir::separator() + path;
Chris@584:         if (!alreadyWatched.contains(path)) {
Chris@584:             m_watcher.addPath(path);
Chris@584:         } else {
Chris@584:             alreadyWatched.remove(path);
Chris@584:         }
Chris@584:     }
Chris@584: 
Chris@584:     // Remove the remaining paths, those that were being watched
Chris@584:     // before but that are not in the list we were given
Chris@584:     foreach (QString path, alreadyWatched) {
Chris@584:         m_watcher.removePath(path);
Chris@584:     }
Chris@584: 
Chris@584:     debugPrint();
Chris@586: 
Chris@584: #endif
Chris@538: }
Chris@538: 
Chris@538: void
Chris@538: FsWatcher::setIgnoredFilePrefixes(QStringList prefixes)
Chris@538: {
Chris@538:     QMutexLocker locker(&m_mutex);
Chris@538:     m_ignoredPrefixes = prefixes;
Chris@538: }
Chris@538: 
Chris@538: void
Chris@538: FsWatcher::setIgnoredFileSuffixes(QStringList suffixes)
Chris@538: {
Chris@538:     QMutexLocker locker(&m_mutex);
Chris@538:     m_ignoredSuffixes = suffixes;
Chris@538: }
Chris@538: 
Chris@538: int
Chris@538: FsWatcher::getNewToken()
Chris@538: {
Chris@538:     QMutexLocker locker(&m_mutex);
Chris@538:     int token = ++m_lastToken;
Chris@538:     m_tokenMap[token] = m_lastCounter;
Chris@538:     return token;
Chris@538: }
Chris@538: 
Chris@538: QSet<QString>
Chris@538: FsWatcher::getChangedPaths(int token)
Chris@538: {
Chris@538:     QMutexLocker locker(&m_mutex);
Chris@538:     size_t lastUpdatedAt = m_tokenMap[token];
Chris@538:     QSet<QString> changed;
Chris@538:     for (QHash<QString, size_t>::const_iterator i = m_changes.begin();
Chris@538: 	 i != m_changes.end(); ++i) {
Chris@538: 	if (i.value() > lastUpdatedAt) {
Chris@538: 	    changed.insert(i.key());
Chris@538: 	}
Chris@538:     }
Chris@538:     m_tokenMap[token] = m_lastCounter;
Chris@538:     return changed;
Chris@538: }
Chris@538: 
Chris@538: void
Chris@538: FsWatcher::fsDirectoryChanged(QString path)
Chris@538: {
Chris@586:     bool haveChanges = false;
Chris@586: 
Chris@538:     {
Chris@538: 	QMutexLocker locker(&m_mutex);
Chris@540: 
Chris@538: 	if (shouldIgnore(path)) return;
Chris@540: 
Chris@540:         QSet<QString> files = scanDirectory(path);
Chris@586: 
Chris@540:         if (files == m_dirContents[path]) {
Chris@586: 
Chris@540: #ifdef DEBUG_FSWATCHER
Chris@540:             std::cerr << "FsWatcher: Directory " << path << " has changed, but not in a way that we are monitoring" << std::endl;
Chris@540: #endif
Chris@586: 
Chris@586: #ifdef Q_OS_MAC
Chris@586:             haveChanges = manuallyCheckTrackedFiles();
Chris@586: #endif
Chris@586: 
Chris@541:         } else {
Chris@586: 
Chris@541: #ifdef DEBUG_FSWATCHER
Chris@541:             std::cerr << "FsWatcher: Directory " << path << " has changed" << std::endl;
Chris@541: #endif
Chris@541:             m_dirContents[path] = files;
Chris@586:             size_t counter = ++m_lastCounter;
Chris@586:             m_changes[path] = counter;
Chris@586:             haveChanges = true;
Chris@540:         }
Chris@538:     }
Chris@540: 
Chris@586:     if (haveChanges) {
Chris@586:         emit changed();
Chris@586:     }
Chris@538: }
Chris@538: 
Chris@538: void
Chris@538: FsWatcher::fsFileChanged(QString path)
Chris@538: {
Chris@540:     {
Chris@540:         QMutexLocker locker(&m_mutex);
Chris@540: 
Chris@540:         // We don't check whether the file matches an ignore pattern,
Chris@540:         // because we are only notified for file changes if we are
Chris@540:         // watching the file explicitly, i.e. the file is in the
Chris@586:         // tracked file paths list. So we never want to ignore these
Chris@540: 
Chris@563: #ifdef DEBUG_FSWATCHER
Chris@541:         std::cerr << "FsWatcher: Tracked file " << path << " has changed" << std::endl;
Chris@563: #endif
Chris@541: 
Chris@540:         size_t counter = ++m_lastCounter;
Chris@540:         m_changes[path] = counter;
Chris@540:     }
Chris@540: 
Chris@540:     emit changed();
Chris@538: }
Chris@538: 
Chris@586: #ifdef Q_OS_MAC
Chris@586: bool
Chris@586: FsWatcher::manuallyCheckTrackedFiles()
Chris@586: {
Chris@586:     bool foundChanges = false;
Chris@586: 
Chris@586:     for (PathTimeMap::iterator i = m_trackedFileUpdates.begin();
Chris@586:          i != m_trackedFileUpdates.end(); ++i) {
Chris@586: 
Chris@586:         QString path = i.key();
Chris@586:         QDateTime prevUpdate = i.value();
Chris@586: 
Chris@586:         QFileInfo fi(path);
Chris@586:         QDateTime currUpdate = fi.lastModified();
Chris@586: 
Chris@586:         if (currUpdate > prevUpdate) {
Chris@586: 
Chris@586: #ifdef DEBUG_FSWATCHER
Chris@586:             std::cerr << "FsWatcher: Tracked file " << path << " has been changed since last check" << std::endl;
Chris@586: #endif
Chris@586:             i.value() = currUpdate;
Chris@586:             
Chris@586:             size_t counter = ++m_lastCounter;
Chris@586:             m_changes[path] = counter;
Chris@586:             foundChanges = true;
Chris@586:         }
Chris@586:     }
Chris@586:     
Chris@586:     return foundChanges;
Chris@586: }
Chris@586: #endif
Chris@586: 
Chris@538: bool
Chris@538: FsWatcher::shouldIgnore(QString path)
Chris@538: {
Chris@540:     QFileInfo fi(path);
Chris@540:     QString fn(fi.fileName());
Chris@540:     foreach (QString pfx, m_ignoredPrefixes) {
Chris@541:         if (fn.startsWith(pfx)) {
Chris@563: #ifdef DEBUG_FSWATCHER
Chris@541:             std::cerr << "(ignoring: " << path << ")" << std::endl;
Chris@563: #endif
Chris@541:             return true;
Chris@541:         }
Chris@540:     }
Chris@540:     foreach (QString sfx, m_ignoredSuffixes) {
Chris@541:         if (fn.endsWith(sfx)) {
Chris@563: #ifdef DEBUG_FSWATCHER
Chris@541:             std::cerr << "(ignoring: " << path << ")" << std::endl;
Chris@563: #endif
Chris@541:             return true;
Chris@541:         }
Chris@540:     }
Chris@540:     return false;
Chris@538: }
Chris@538: 
Chris@540: QSet<QString>
Chris@540: FsWatcher::scanDirectory(QString path)
Chris@540: {
Chris@540:     QSet<QString> files;
Chris@540:     QDir d(path);
Chris@540:     if (d.exists()) {
Chris@540:         d.setFilter(QDir::Files | QDir::NoDotAndDotDot |
Chris@540:                     QDir::Readable | QDir::NoSymLinks);
Chris@540:         foreach (QString entry, d.entryList()) {
Chris@540:             if (entry.startsWith('.')) continue;
Chris@540:             if (shouldIgnore(entry)) continue;
Chris@540:             files.insert(entry);
Chris@540:         }
Chris@540:     }
Chris@540:     return files;
Chris@540: }
Chris@540: 
Chris@540: void
Chris@540: FsWatcher::debugPrint()
Chris@540: {
Chris@540: #ifdef DEBUG_FSWATCHER
Chris@585: #ifndef Q_OS_MAC
Chris@540:     std::cerr << "FsWatcher: Now watching " << m_watcher.directories().size()
Chris@540:               << " directories and " << m_watcher.files().size()
Chris@540:               << " files" << std::endl;
Chris@540: #endif
Chris@585: #endif
Chris@540: }