Chris@498: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
Chris@498: 
Chris@498: /*
Chris@498:     Sonic Visualiser
Chris@498:     An audio file viewer and annotation editor.
Chris@498: 
Chris@498:     Sonic Annotator
Chris@498:     A utility for batch feature extraction from audio files.
Chris@498: 
Chris@498:     Mark Levy, Chris Sutton and Chris Cannam, Queen Mary, University of London.
Chris@498:     Copyright 2007-2008 QMUL.
Chris@498: 
Chris@498:     This program is free software; you can redistribute it and/or
Chris@498:     modify it under the terms of the GNU General Public License as
Chris@498:     published by the Free Software Foundation; either version 2 of the
Chris@498:     License, or (at your option) any later version.  See the file
Chris@498:     COPYING included with this distribution for more information.
Chris@498: */
Chris@498: 
Chris@498: #include "FileFeatureWriter.h"
Chris@498: 
Chris@498: #include "base/Exceptions.h"
Chris@498: 
Chris@498: #include <QTextStream>
Chris@498: #include <QFile>
Chris@498: #include <QFileInfo>
Chris@498: #include <QUrl>
Chris@498: #include <QDir>
Chris@498: 
Chris@498: using namespace std;
Chris@498: using namespace Vamp;
Chris@498: 
Chris@498: FileFeatureWriter::FileFeatureWriter(int support,
Chris@498:                                      QString extension) :
Chris@1582:     m_prevstream(nullptr),
Chris@498:     m_support(support),
Chris@498:     m_extension(extension),
Chris@498:     m_manyFiles(false),
Chris@498:     m_stdout(false),
Chris@498:     m_append(false),
Chris@498:     m_force(false)
Chris@498: {
Chris@498:     if (!(m_support & SupportOneFilePerTrack)) {
Chris@498:         if (m_support & SupportOneFilePerTrackTransform) {
Chris@498:             m_manyFiles = true;
Chris@498:         } else if (m_support & SupportOneFileTotal) {
Chris@498:             m_singleFileName = QString("output.%1").arg(m_extension);
Chris@498:         } else {
Chris@1428:             SVCERR << "FileFeatureWriter::FileFeatureWriter: ERROR: Invalid support specification " << support << endl;
Chris@498:         }
Chris@498:     }
Chris@498: }
Chris@498: 
Chris@498: FileFeatureWriter::~FileFeatureWriter()
Chris@498: {
Chris@498:     while (!m_streams.empty()) {
Chris@498:         m_streams.begin()->second->flush();
Chris@498:         delete m_streams.begin()->second;
Chris@498:         m_streams.erase(m_streams.begin());
Chris@498:     }
Chris@498:     while (!m_files.empty()) {
Chris@625:         if (m_files.begin()->second) {
Chris@690:             SVDEBUG << "FileFeatureWriter::~FileFeatureWriter: NOTE: Closing feature file \""
Chris@686:                  << m_files.begin()->second->fileName() << "\"" << endl;
Chris@625:             delete m_files.begin()->second;
Chris@625:         }
Chris@498:         m_files.erase(m_files.begin());
Chris@498:     }
Chris@498: }
Chris@498: 
Chris@498: FileFeatureWriter::ParameterList
Chris@498: FileFeatureWriter::getSupportedParameters() const
Chris@498: {
Chris@498:     ParameterList pl;
Chris@498:     Parameter p;
Chris@498: 
Chris@498:     p.name = "basedir";
Chris@999:     p.description = "Base output directory path. (The default is the same directory as the input file.) The directory must exist already.";
Chris@498:     p.hasArg = true;
Chris@498:     pl.push_back(p);
Chris@498: 
Chris@498:     if (m_support & SupportOneFilePerTrackTransform &&
Chris@498:         m_support & SupportOneFilePerTrack) {
Chris@498:         p.name = "many-files";
Chris@999:         p.description = "Create a separate output file for every combination of input file and transform. The output file names will be based on the input file names. (The default is to create one output file per input audio file, and write all transform results for that input into it.)";
Chris@498:         p.hasArg = false;
Chris@498:         pl.push_back(p);
Chris@498:     }
Chris@498: 
Chris@498:     if (m_support & SupportOneFileTotal) {
Chris@498:         if (m_support & ~SupportOneFileTotal) { // not only option
Chris@498:             p.name = "one-file";
Chris@659:             if (m_support & SupportOneFilePerTrack) {
Chris@999:                 p.description = "Write all transform results for all input files into the single named output file. (The default is to create one output file per input audio file, and write all transform results for that input into it.)";
Chris@659:             } else {
Chris@999:                 p.description = "Write all transform results for all input files into the single named output file. (The default is to create a separate output file for each combination of input audio file and transform.)";
Chris@659:             }                
Chris@498:             p.hasArg = true;
Chris@498:             pl.push_back(p);
Chris@498:         }
Chris@997:     }
Chris@997:     if (m_support & SupportStdOut) {
Chris@498:         p.name = "stdout";
Chris@498:         p.description = "Write all transform results directly to standard output.";
Chris@498:         p.hasArg = false;
Chris@498:         pl.push_back(p);
Chris@498:     }
Chris@498: 
Chris@498:     p.name = "force";
Chris@498:     p.description = "If an output file already exists, overwrite it.";
Chris@498:     p.hasArg = false;
Chris@498:     pl.push_back(p);
Chris@498: 
Chris@498:     p.name = "append";
Chris@498:     p.description = "If an output file already exists, append data to it.";
Chris@498:     p.hasArg = false;
Chris@498:     pl.push_back(p);
Chris@498: 
Chris@498:     return pl;
Chris@498: }
Chris@498: 
Chris@498: void
Chris@498: FileFeatureWriter::setParameters(map<string, string> &params)
Chris@498: {
Chris@498:     for (map<string, string>::iterator i = params.begin();
Chris@498:          i != params.end(); ++i) {
Chris@498:         if (i->first == "basedir") {
Chris@498:             m_baseDir = i->second.c_str();
Chris@498:         } else if (i->first == "many-files") {
Chris@498:             if (m_support & SupportOneFilePerTrackTransform &&
Chris@498:                 m_support & SupportOneFilePerTrack) {
Chris@498:                 if (m_singleFileName != "") {
Chris@1428:                     SVCERR << "FileFeatureWriter::setParameters: WARNING: Both one-file and many-files parameters provided, ignoring many-files" << endl;
Chris@498:                 } else {
Chris@498:                     m_manyFiles = true;
Chris@498:                 }
Chris@498:             }
Chris@498:         } else if (i->first == "one-file") {
Chris@498:             if (m_support & SupportOneFileTotal) {
Chris@498:                 if (m_support & ~SupportOneFileTotal) { // not only option
Chris@659:                     // No, we cannot do this test because m_manyFiles
Chris@659:                     // may be on by default (for any FileFeatureWriter
Chris@659:                     // that supports OneFilePerTrackTransform but not
Chris@659:                     // OneFilePerTrack), so we need to be able to
Chris@659:                     // override it
Chris@659: //                    if (m_manyFiles) {
Chris@1428: //                        SVCERR << "FileFeatureWriter::setParameters: WARNING: Both many-files and one-file parameters provided, ignoring one-file" << endl;
Chris@659: //                    } else {
Chris@498:                         m_singleFileName = i->second.c_str();
Chris@659: //                    }
Chris@498:                 }
Chris@498:             }
Chris@498:         } else if (i->first == "stdout") {
Chris@997:             if (m_support & SupportStdOut) {
Chris@498:                 if (m_singleFileName != "") {
Chris@1428:                     SVCERR << "FileFeatureWriter::setParameters: WARNING: Both stdout and one-file provided, ignoring stdout" << endl;
Chris@498:                 } else {
Chris@498:                     m_stdout = true;
Chris@498:                 }
Chris@498:             }
Chris@498:         } else if (i->first == "append") {
Chris@498:             m_append = true;
Chris@498:         } else if (i->first == "force") {
Chris@498:             m_force = true;
Chris@498:         }
Chris@498:     }
Chris@498: }
Chris@498: 
Chris@625: QString
Chris@999: FileFeatureWriter::createOutputFilename(QString trackId,
Chris@999:                                         TransformId transformId)
Chris@498: {
Chris@498:     if (m_singleFileName != "") {
Chris@498:         if (QFileInfo(m_singleFileName).exists() && !(m_force || m_append)) {
Chris@1428:             SVCERR << endl << "FileFeatureWriter: ERROR: Specified output file \"" << m_singleFileName << "\" exists and neither --" << getWriterTag() << "-force nor --" << getWriterTag() << "-append flag is specified -- not overwriting" << endl;
Chris@1428:             SVCERR << "NOTE: To find out how to fix this problem, read the help for the --" << getWriterTag() << "-force" << endl << "and --" << getWriterTag() << "-append options" << endl;
Chris@498:             return "";
Chris@498:         }
Chris@498:         return m_singleFileName;
Chris@498:     }
Chris@498: 
Chris@999:     if (m_stdout) {
Chris@999:         return "";
Chris@999:     }
Chris@498:     
Christophe@615:     QUrl url(trackId, QUrl::StrictMode);
Chris@498:     QString scheme = url.scheme().toLower();
Chris@498:     bool local = (scheme == "" || scheme == "file" || scheme.length() == 1);
Chris@498: 
Chris@498:     QString dirname, basename;
Chris@498:     QString infilename = url.toLocalFile();
Chris@519:     if (infilename == "") {
Chris@519:         infilename = url.path();
Chris@519:     }
Christophe@615:     basename = QFileInfo(infilename).completeBaseName();
Chris@519:     if (scheme.length() == 1) {
Chris@519:         infilename = scheme + ":" + infilename; // DOS drive!
Chris@519:     }
Chris@507: 
Chris@686: //    cerr << "trackId = " << trackId << ", url = " << url.toString() << ", infilename = "
Chris@686: //         << infilename << ", basename = " << basename << ", m_baseDir = " << m_baseDir << endl;
Chris@498: 
Chris@498:     if (m_baseDir != "") dirname = QFileInfo(m_baseDir).absoluteFilePath();
Chris@498:     else if (local) dirname = QFileInfo(infilename).absolutePath();
Chris@498:     else dirname = QDir::currentPath();
Chris@498: 
Chris@686: //    cerr << "dirname = " << dirname << endl;
Chris@604: 
Chris@498:     QString filename;
Chris@498: 
Chris@498:     if (m_manyFiles && transformId != "") {
Chris@514:         filename = QString("%1_%2.%3").arg(basename).arg(transformId).arg(m_extension);
Chris@498:     } else {
Chris@498:         filename = QString("%1.%2").arg(basename).arg(m_extension);
Chris@498:     }
Chris@498: 
Chris@514:     filename.replace(':', '_'); // ':' not permitted in Windows
Chris@514: 
Chris@498:     filename = QDir(dirname).filePath(filename);
Chris@498: 
Chris@498:     if (QFileInfo(filename).exists() && !(m_force || m_append)) {
Chris@1428:         SVCERR << endl << "FileFeatureWriter: ERROR: Output file \"" << filename << "\" exists (for input file or URL \"" << trackId << "\" and transform \"" << transformId << "\") and neither --" << getWriterTag() << "-force nor --" << getWriterTag() << "-append is specified -- not overwriting" << endl;
Chris@1428:         SVCERR << "NOTE: To find out how to fix this problem, read the help for the --" << getWriterTag() << "-force" << endl << "and --" << getWriterTag() << "-append options" << endl;
Chris@498:         return "";
Chris@498:     }
Chris@498:     
Chris@498:     return filename;
Chris@498: }
Chris@498: 
Chris@625: void
Chris@625: FileFeatureWriter::testOutputFile(QString trackId,
Chris@625:                                   TransformId transformId)
Chris@625: {
Chris@626:     // Obviously, if we're writing to stdout we can't test for an
Chris@626:     // openable output file. But when writing a single file we don't
Chris@626:     // want to either, because this test would fail on the second and
Chris@626:     // subsequent input files (because the file would already exist).
Chris@626:     // getOutputFile does the right thing in this case, so we just
Chris@626:     // leave it to it
Chris@626:     if (m_stdout || m_singleFileName != "") return;
Chris@626: 
Chris@999:     QString filename = createOutputFilename(trackId, transformId);
Chris@625:     if (filename == "") {
Chris@625:         throw FailedToOpenOutputStream(trackId, transformId);
Chris@625:     }
Chris@625: }
Chris@498: 
Chris@999: FileFeatureWriter::TrackTransformPair
Chris@999: FileFeatureWriter::getFilenameKey(QString trackId,
Chris@999:                                   TransformId transformId)
Chris@999: {
Chris@999:     TrackTransformPair key;
Chris@999: 
Chris@999:     if (m_singleFileName != "") {
Chris@999:         key = TrackTransformPair("", "");
Chris@999:     } else if (m_manyFiles) {
Chris@999:         key = TrackTransformPair(trackId, transformId);
Chris@999:     } else {
Chris@999:         key = TrackTransformPair(trackId, "");
Chris@999:     }
Chris@999: 
Chris@999:     return key;
Chris@999: }    
Chris@999: 
Chris@999: QString
Chris@999: FileFeatureWriter::getOutputFilename(QString trackId,
Chris@999:                                      TransformId transformId)
Chris@999: {
Chris@999:     TrackTransformPair key = getFilenameKey(trackId, transformId);
Chris@999:     if (m_filenames.find(key) == m_filenames.end()) {
Chris@999:         m_filenames[key] = createOutputFilename(trackId, transformId);
Chris@999:     }
Chris@999:     return m_filenames[key];
Chris@999: }
Chris@999: 
Chris@625: QFile *
Chris@625: FileFeatureWriter::getOutputFile(QString trackId,
Chris@625:                                  TransformId transformId)
Chris@498: {
Chris@999:     TrackTransformPair key = getFilenameKey(trackId, transformId);
Chris@498: 
Chris@498:     if (m_files.find(key) == m_files.end()) {
Chris@498: 
Chris@999:         QString filename = createOutputFilename(trackId, transformId);
Chris@498: 
Chris@625:         if (filename == "") { // stdout or failure
Chris@1582:             return nullptr;
Chris@498:         }
Chris@498: 
Chris@690:         SVDEBUG << "FileFeatureWriter: NOTE: Using output filename \""
Chris@686:              << filename << "\"" << endl;
Chris@498: 
Chris@591:         if (m_append) {
Chris@690:             SVDEBUG << "FileFeatureWriter: NOTE: Calling reviewFileForAppending" << endl;
Chris@591:             reviewFileForAppending(filename);
Chris@591:         }
Chris@591:         
Chris@498:         QFile *file = new QFile(filename);
Chris@498:         QIODevice::OpenMode mode = (QIODevice::WriteOnly);
Chris@498:         if (m_append) mode |= QIODevice::Append;
Chris@498:                        
Chris@498:         if (!file->open(mode)) {
Chris@1428:             SVCERR << "FileFeatureWriter: ERROR: Failed to open output file \"" << filename
Chris@498:                  << "\" for writing" << endl;
Chris@498:             delete file;
Chris@1582:             m_files[key] = nullptr;
Chris@498:             throw FailedToOpenFile(filename);
Chris@498:         }
Chris@626:         
Chris@498:         m_files[key] = file;
Chris@498:     }
Chris@498: 
Chris@498:     return m_files[key];
Chris@498: }
Chris@498: 
Chris@498: 
Chris@498: QTextStream *FileFeatureWriter::getOutputStream(QString trackId,
Chris@1035:                                                 TransformId transformId,
Chris@1035:                                                 QTextCodec *codec)
Chris@498: {
Chris@498:     QFile *file = getOutputFile(trackId, transformId);
Chris@498:     if (!file && !m_stdout) {
Chris@1582:         return nullptr;
Chris@498:     }
Chris@626:     
Chris@498:     if (m_streams.find(file) == m_streams.end()) {
Chris@498:         if (m_stdout) {
Chris@498:             m_streams[file] = new QTextStream(stdout);
Chris@498:         } else {
Chris@498:             m_streams[file] = new QTextStream(file);
Chris@498:         }
Chris@1035:         m_streams[file]->setCodec(codec);
Chris@498:     }
Chris@498: 
Chris@512:     QTextStream *stream = m_streams[file];
Chris@512: 
Chris@512:     if (m_prevstream && stream != m_prevstream) {
Chris@512:         m_prevstream->flush();
Chris@512:     }
Chris@512:     m_prevstream = stream;
Chris@512: 
Chris@512:     return stream;
Chris@498: }
Chris@498:             
Chris@515: 
Chris@515: void
Chris@515: FileFeatureWriter::flush()
Chris@515: {
Chris@515:     if (m_prevstream) {
Chris@515:         m_prevstream->flush();
Chris@515:     }
Chris@515: }
Chris@515: 
Chris@531: 
Chris@531: void
Chris@531: FileFeatureWriter::finish()
Chris@531: {
Chris@690: //    SVDEBUG << "FileFeatureWriter::finish()" << endl;
Chris@531: 
Chris@531:     if (m_singleFileName != "" || m_stdout) return;
Chris@531: 
Chris@531:     while (!m_streams.empty()) {
Chris@531:         m_streams.begin()->second->flush();
Chris@531:         delete m_streams.begin()->second;
Chris@531:         m_streams.erase(m_streams.begin());
Chris@531:     }
Chris@531:     while (!m_files.empty()) {
Chris@625:         if (m_files.begin()->second) {
Chris@690:             SVDEBUG << "FileFeatureWriter::finish: NOTE: Closing feature file \""
Chris@686:                  << m_files.begin()->second->fileName() << "\"" << endl;
Chris@625:             delete m_files.begin()->second;
Chris@625:         }
Chris@531:         m_files.erase(m_files.begin());
Chris@531:     }
Chris@1582:     m_prevstream = nullptr;
Chris@531: }
Chris@531: