# HG changeset patch # User Chris Cannam # Date 1581591375 0 # Node ID 38cd115c91d413061be266f805f5655da6dd3b63 # Parent 85768d48e6cefc6c88ffa629fbf2e1b3a8860704 Integrate version checker into installer program & build, + win32 signing bits diff -r 85768d48e6ce -r 38cd115c91d4 deploy/linux/generate-qrc --- a/deploy/linux/generate-qrc Wed Feb 12 14:41:09 2020 +0000 +++ b/deploy/linux/generate-qrc Thu Feb 13 10:56:15 2020 +0000 @@ -5,4 +5,4 @@ exit 2 fi set -eu -cat "$qrc".in | sed 's/@SUFFIX@/so/g' > "$qrc" +cat "$qrc".in | sed -e 's/@SUFFIX@/.so/g' -e 's/@EXESUFFIX@//g' > "$qrc" diff -r 85768d48e6ce -r 38cd115c91d4 deploy/osx/generate-qrc --- a/deploy/osx/generate-qrc Wed Feb 12 14:41:09 2020 +0000 +++ b/deploy/osx/generate-qrc Thu Feb 13 10:56:15 2020 +0000 @@ -5,4 +5,4 @@ exit 2 fi set -eu -cat "$qrc".in | sed 's/@SUFFIX@/dylib/g' > "$qrc" +cat "$qrc".in | sed -e 's/@SUFFIX@/.dylib/g' -e 's/@EXESUFFIX@//g' > "$qrc" diff -r 85768d48e6ce -r 38cd115c91d4 deploy/win64/build-64.bat --- a/deploy/win64/build-64.bat Wed Feb 12 14:41:09 2020 +0000 +++ b/deploy/win64/build-64.bat Thu Feb 13 10:56:15 2020 +0000 @@ -18,10 +18,34 @@ @ exit /b 2 ) +set SMLNJDIR=C:\Program Files (x86)\SMLNJ +if not exist "%SMLNJDIR%\bin" ( +@ echo Could not find SML/NJ, required for Repoint +@ exit /b 2 +) + call %vcvarsall% amd64 set ORIGINALPATH=%PATH% -set PATH=%PATH%;C:\Program Files (x86)\SMLNJ\bin;%QTDIR%\bin +set PATH=%PATH%;%SMLNJDIR%\bin;%QTDIR%\bin +set NAME=Open Source Developer, Christopher Cannam + +set ARG=%1 +shift +if "%ARG%" == "sign" ( +@ echo NOTE: sign option specified, will attempt to codesign exe and msi +@ echo NOTE: starting by codesigning an unrelated executable, so we know +@ echo NOTE: whether it'll work before doing the entire build +copy "%SMLNJDIR%\bin\.run\run.x86-win32.exe" signtest.exe +signtool sign /v /n "%NAME%" /t http://time.certum.pl /fd sha1 /a signtest.exe +if errorlevel 1 exit /b %errorlevel% +signtool verify /pa signtest.exe +if errorlevel 1 exit /b %errorlevel% +del signtest.exe +@ echo NOTE: success +) else ( +@ echo NOTE: sign option not specified, will not codesign anything +) cd %STARTPWD% @@ -40,13 +64,22 @@ qmake -spec win32-msvc -r -tp vc ..\plugins.pro if %errorlevel% neq 0 exit /b %errorlevel% +mkdir o + +msbuild get-version.vcxproj /t:Build /p:Configuration=Release +if %errorlevel% neq 0 exit /b %errorlevel% +copy release\out\get-version.exe ..\out\ + msbuild plugins.sln /t:Build /p:Configuration=Release if %errorlevel% neq 0 exit /b %errorlevel% - -rem and sign! copy release\out\*.dll ..\out\ -mkdir o +if "%ARG%" == "sign" ( +@echo Signing plugins and version helper +signtool sign /v /n "%NAME%" /t http://time.certum.pl /fd sha1 /a ..\out\*.dll ..\out\*.exe +signtool verify /pa ..\out\*.dll ..\out\*.exe +) + %QTDIR%\bin\rcc ..\installer.qrc -o o\qrc_installer.cpp qmake -spec win32-msvc -r -tp vc ..\installer.pro @@ -66,6 +99,14 @@ copy %QTDIR%\plugins\platforms\qwindows.dll .\release copy %QTDIR%\plugins\styles\qwindowsvistastyle.dll .\release +if "%ARG%" == "sign" ( +@echo Signing application +signtool sign /v /n "%NAME%" /t http://time.certum.pl /fd sha1 /a release\*.exe release\*.dll +signtool verify /pa "release\Vamp Plugin Pack Installer.exe" +) + cd .. +rem Now what? + set PATH=%ORIGINALPATH% diff -r 85768d48e6ce -r 38cd115c91d4 deploy/win64/generate-qrc.bat --- a/deploy/win64/generate-qrc.bat Wed Feb 12 14:41:09 2020 +0000 +++ b/deploy/win64/generate-qrc.bat Thu Feb 13 10:56:15 2020 +0000 @@ -13,4 +13,4 @@ @echo on -powershell -Command "(Get-Content %IN%) -replace '@SUFFIX@', 'dll' | Out-File -encoding ASCII %QRC%" +powershell -Command "(Get-Content %IN%) -replace '@SUFFIX@', '.dll' -replace '@EXESUFFIX@', '.exe' | Out-File -encoding ASCII %QRC%" diff -r 85768d48e6ce -r 38cd115c91d4 get-version.cpp --- a/get-version.cpp Wed Feb 12 14:41:09 2020 +0000 +++ b/get-version.cpp Thu Feb 13 10:56:15 2020 +0000 @@ -32,6 +32,16 @@ // Undocumented internal API #include +#ifdef _WIN32 +#include +#include +#include +#else +#include +#endif + +#include + #include #include @@ -59,6 +69,49 @@ exit(2); } +// We write our output to stdout, but want to ensure that the plugin +// doesn't write anything itself. To do this we open a null file +// descriptor and dup2() it into place of stdout in the gaps between +// our own output activity. + +static int normalFd = -1; +static int suspendedFd = -1; + +static void initFds() +{ +#ifdef _WIN32 + normalFd = _dup(1); + suspendedFd = _open("NUL", _O_WRONLY); +#else + normalFd = dup(1); + suspendedFd = open("/dev/null", O_WRONLY); +#endif + + if (normalFd < 0 || suspendedFd < 0) { + throw std::runtime_error + ("Failed to initialise fds for stdio suspend/resume"); + } +} + +static void suspendOutput() +{ +#ifdef _WIN32 + _dup2(suspendedFd, 1); +#else + dup2(suspendedFd, 1); +#endif +} + +static void resumeOutput() +{ + fflush(stdout); +#ifdef _WIN32 + _dup2(normalFd, 1); +#else + dup2(normalFd, 1); +#endif +} + int main(int argc, char **argv) { char *scooter = argv[0]; @@ -72,6 +125,9 @@ if (argc != 2) usage(name); if (string(argv[1]) == string("-?")) usage(name); + initFds(); + suspendOutput(); + string libraryPath(argv[1]); void *handle = Files::loadLibrary(libraryPath); @@ -95,8 +151,10 @@ const VampPluginDescriptor *descriptor = 0; while ((descriptor = fn(VAMP_API_VERSION, index))) { + resumeOutput(); cout << descriptor->identifier << ":" << descriptor->pluginVersion << endl; + suspendOutput(); ++index; } diff -r 85768d48e6ce -r 38cd115c91d4 get-version.pro --- a/get-version.pro Wed Feb 12 14:41:09 2020 +0000 +++ b/get-version.pro Thu Feb 13 10:56:15 2020 +0000 @@ -13,7 +13,7 @@ QT += gui widgets svg -TARGET=get-version +TARGET=out/get-version OBJECTS_DIR = o MOC_DIR = o diff -r 85768d48e6ce -r 38cd115c91d4 installer.cpp --- a/installer.cpp Wed Feb 12 14:41:09 2020 +0000 +++ b/installer.cpp Thu Feb 13 10:56:15 2020 +0000 @@ -41,6 +41,10 @@ #include #include #include +#include +#include +#include +#include #include @@ -51,6 +55,8 @@ #include #include +#include + #include "base/Debug.h" using namespace std; @@ -129,6 +135,7 @@ QString maker; QString description; QStringList pluginTitles; + map pluginVersions; // id -> version }; vector @@ -145,7 +152,7 @@ Uri("a"), store.expand("vamp:PluginLibrary"))); - std::map wanted; // basename -> full lib name + map wanted; // basename -> full lib name for (auto lib: libraries) { wanted[QFileInfo(lib).baseName()] = lib; } @@ -208,6 +215,21 @@ if (ptitle.type == Node::Literal) { info.pluginTitles.push_back(ptitle.value); } + + Node pident = store.complete(Triple(p.object(), + store.expand("vamp:identifier"), + Node())); + Node pversion = store.complete(Triple(p.object(), + store.expand("owl:versionInfo"), + Node())); + if (pident.type == Node::Literal && + pversion.type == Node::Literal) { + bool ok = false; + int version = pversion.value.toInt(&ok); + if (ok) { + info.pluginVersions[pident.value] = version; + } + } } results.push_back(info); @@ -222,12 +244,204 @@ return results; } +struct TempFileDeleter { + ~TempFileDeleter() { + if (tempFile != "") { + QFile(tempFile).remove(); + } + } + QString tempFile; +}; + +map +getInstalledLibraryPluginVersions(QString libraryFilePath) +{ + static QMutex mutex; + static QString tempFileName; + static TempFileDeleter deleter; + static bool initHappened = false, initSucceeded = false; + + QMutexLocker locker (&mutex); + + if (!initHappened) { + initHappened = true; + + QTemporaryFile tempFile; + tempFile.setAutoRemove(false); + if (!tempFile.open()) { + SVCERR << "ERROR: Failed to open a temporary file" << endl; + return {}; + } + + // We can't make the QTemporaryFile static, as it will hold + // the file open and that prevents us from executing it. Hence + // the separate deleter. + + tempFileName = tempFile.fileName(); + deleter.tempFile = tempFileName; + +#ifdef Q_OS_WIN32 + QString helperPath = ":out/get-version.exe"; +#else + QString helperPath = ":out/get-version"; +#endif + QFile helper(helperPath); + if (!helper.open(QFile::ReadOnly)) { + SVCERR << "ERROR: Failed to read helper code" << endl; + return {}; + } + QByteArray content = helper.readAll(); + helper.close(); + + if (tempFile.write(content) != content.size()) { + SVCERR << "ERROR: Incomplete write to temporary file" << endl; + return {}; + } + tempFile.close(); + + if (!QFile::setPermissions + (tempFileName, + QFile::ReadOwner | QFile::WriteOwner | QFile::ExeOwner)) { + SVCERR << "ERROR: Failed to set execute permission on helper " + << tempFileName << endl; + return {}; + } + + initSucceeded = true; + } + + if (!initSucceeded) { + return {}; + } + + QProcess process; + process.start(tempFileName, { libraryFilePath }); + + if (!process.waitForStarted()) { + QProcess::ProcessError err = process.error(); + if (err == QProcess::FailedToStart) { + SVCERR << "Unable to start helper process " << tempFileName << endl; + } else if (err == QProcess::Crashed) { + SVCERR << "Helper process " << tempFileName + << " crashed on startup" << endl; + } else { + SVCERR << "Helper process " << tempFileName + << " failed on startup with error code " << err << endl; + } + return {}; + } + process.waitForFinished(); + + QByteArray stdOut = process.readAllStandardOutput(); + QByteArray stdErr = process.readAllStandardError(); + + QString errStr = QString::fromUtf8(stdErr); + if (!errStr.isEmpty()) { + SVCERR << "Note: Helper process stderr follows:" << endl; + SVCERR << errStr << endl; + SVCERR << "Note: Helper process stderr ends" << endl; + } + + QStringList lines = QString::fromUtf8(stdOut).split + (QRegExp("[\\r\\n]+"), QString::SkipEmptyParts); + map versions; + for (QString line: lines) { + QStringList parts = line.split(":"); + if (parts.size() != 2) { + SVCERR << "Unparseable output line: " << line << endl; + continue; + } + bool ok = false; + int version = parts[1].toInt(&ok); + if (!ok) { + SVCERR << "Unparseable version number in line: " << line << endl; + continue; + } + versions[parts[0]] = version; + } + + return versions; +} + +bool isLibraryNewer(map a, map b) +{ + // a and b are maps from plugin id to plugin version for libraries + // A and B. (There is no overarching library version number.) We + // deem library A to be newer than library B if: + // + // 1. A contains a plugin id that is also in B, whose version in + // A is newer than that in B, or + // + // 2. B is not newer than A according to rule 1, and neither A or + // B is empty, and A contains a plugin id that is not in B, and B + // does not contain any plugin id that is not in A + // + // (The not-empty part of rule 2 is just to avoid false positives + // when a library or its metadata could not be read at all.) + + auto containsANewerPlugin = [](const map &m1, + const map &m2) { + for (auto p: m1) { + if (m2.find(p.first) != m2.end() && + p.second > m2.at(p.first)) { + return true; + } + } + return false; + }; + + auto containsANovelPlugin = [](const map &m1, + const map &m2) { + for (auto p: m1) { + if (m2.find(p.first) == m2.end()) { + return true; + } + } + return false; + }; + + if (containsANewerPlugin(a, b)) { + return true; + } + + if (!containsANewerPlugin(b, a) && + !a.empty() && + !b.empty() && + containsANovelPlugin(a, b) && + !containsANovelPlugin(b, a)) { + return true; + } + + return false; +} + +QString +versionsString(const map &vv) +{ + QStringList pv; + for (auto v: vv) { + pv.push_back(QString("%1:%2").arg(v.first).arg(v.second)); + } + return "{ " + pv.join(", ") + " }"; +} + void -installLibrary(QString library, QString target) +installLibrary(QString library, LibraryInfo info, QString target) { QString source = ":out"; QFile f(source + "/" + library); QString destination = target + "/" + library; + + if (QFileInfo(destination).exists()) { + auto installed = getInstalledLibraryPluginVersions(destination); + SVCERR << "Note: comparing installed plugin versions " + << versionsString(installed) + << " to packaged versions " + << versionsString(info.pluginVersions) + << ": isLibraryNewer(installed, packaged) returns " + << isLibraryNewer(installed, info.pluginVersions) + << endl; + } SVCERR << "Copying " << library.toStdString() << " to " << destination.toStdString() << "..." << endl; @@ -262,7 +476,7 @@ } } -QStringList +map getUserApprovedPluginLibraries(vector libraries) { QDialog dialog; @@ -295,9 +509,10 @@ selectionFrame->setLayout(selectionLayout); int selectionRow = 0; - map checkBoxMap; + map checkBoxMap; // filename -> checkbox + map libFileInfo; // filename -> info - map> + map> orderedInfo ([](QString k1, QString k2) { return k1.localeAwareCompare(k2) < 0; @@ -346,6 +561,7 @@ ++selectionRow; checkBoxMap[info.fileName] = cb; + libFileInfo[info.fileName] = info; } scroll->setWidget(selectionFrame); @@ -379,10 +595,10 @@ SVCERR << "rejected" << endl; } - QStringList approved; + map approved; for (const auto &p: checkBoxMap) { if (p.second->isChecked()) { - approved.push_back(p.first); + approved[p.first] = libFileInfo[p.first]; } } @@ -418,7 +634,7 @@ auto info = getLibraryInfo(*rdfStore, libraries); - QStringList toInstall = getUserApprovedPluginLibraries(info); + map toInstall = getUserApprovedPluginLibraries(info); if (!toInstall.empty()) { if (!QDir(target).exists()) { @@ -427,7 +643,7 @@ } for (auto lib: toInstall) { - installLibrary(lib, target); + installLibrary(lib.first, lib.second, target); } return 0; diff -r 85768d48e6ce -r 38cd115c91d4 installer.qrc.in --- a/installer.qrc.in Wed Feb 12 14:41:09 2020 +0000 +++ b/installer.qrc.in Thu Feb 13 10:56:15 2020 +0000 @@ -1,114 +1,115 @@ + out/get-version@EXESUFFIX@ out/azi.cat out/azi_COPYING.txt - out/azi.@SUFFIX@ + out/azi@SUFFIX@ out/bbc-vamp-plugins.cat out/bbc-vamp-plugins_COPYING.txt out/bbc-vamp-plugins.n3 out/bbc-vamp-plugins_README.md - out/bbc-vamp-plugins.@SUFFIX@ + out/bbc-vamp-plugins@SUFFIX@ out/beatroot-vamp.cat out/beatroot-vamp_CITATION.txt out/beatroot-vamp_COPYING.txt out/beatroot-vamp.n3 out/beatroot-vamp_README.txt - out/beatroot-vamp.@SUFFIX@ + out/beatroot-vamp@SUFFIX@ out/cepstral-pitchtracker.cat out/cepstral-pitchtracker.n3 out/cepstral-pitchtracker_README.txt - out/cepstral-pitchtracker.@SUFFIX@ + out/cepstral-pitchtracker@SUFFIX@ out/cqvamp.cat out/cqvamp_CITATION.txt out/cqvamp_COPYING.txt out/cqvamp.n3 out/cqvamp_README.txt - out/cqvamp.@SUFFIX@ + out/cqvamp@SUFFIX@ out/fanchirp.cat out/fanchirp_CITATION.txt out/fanchirp_COPYING.txt out/fanchirp_README.md - out/fanchirp.@SUFFIX@ + out/fanchirp@SUFFIX@ out/match-vamp-plugin.cat out/match-vamp-plugin_CITATION.txt out/match-vamp-plugin_COPYING.txt out/match-vamp-plugin.n3 out/match-vamp-plugin_README.txt - out/match-vamp-plugin.@SUFFIX@ + out/match-vamp-plugin@SUFFIX@ out/mvamp.cat out/mvamp_COPYING.txt out/mvamp.n3 out/mvamp_README.txt - out/mvamp.@SUFFIX@ + out/mvamp@SUFFIX@ out/nnls-chroma.cat out/nnls-chroma_CITATION.txt out/nnls-chroma_COPYING.txt out/nnls-chroma.n3 out/nnls-chroma_README.txt - out/nnls-chroma.@SUFFIX@ + out/nnls-chroma@SUFFIX@ out/pyin.cat out/pyin_COPYING.txt out/pyin.n3 out/pyin_README.txt - out/pyin.@SUFFIX@ + out/pyin@SUFFIX@ out/qm-vamp-plugins.cat out/qm-vamp-plugins_COPYING.txt out/qm-vamp-plugins.n3 out/qm-vamp-plugins_README.md - out/qm-vamp-plugins.@SUFFIX@ + out/qm-vamp-plugins@SUFFIX@ out/segmentino.cat out/segmentino_CITATION.txt out/segmentino_COPYING.txt out/segmentino.n3 out/segmentino_README.txt - out/segmentino.@SUFFIX@ + out/segmentino@SUFFIX@ out/silvet.cat out/silvet_CITATION.txt out/silvet_COPYING.txt out/silvet.n3 out/silvet_README.txt - out/silvet.@SUFFIX@ + out/silvet@SUFFIX@ out/simple-cepstrum.cat out/simple-cepstrum.n3 - out/simple-cepstrum.@SUFFIX@ + out/simple-cepstrum@SUFFIX@ out/tempogram.cat out/tempogram_CITATION.txt out/tempogram_COPYING.txt out/tempogram.n3 out/tempogram_README.txt - out/tempogram.@SUFFIX@ + out/tempogram@SUFFIX@ out/tipic.cat out/tipic_CITATION.txt out/tipic_COPYING.txt out/tipic.n3 out/tipic_README.txt - out/tipic.@SUFFIX@ + out/tipic@SUFFIX@ out/tuning-difference.cat out/tuning-difference_COPYING.txt out/tuning-difference.n3 out/tuning-difference_README.md - out/tuning-difference.@SUFFIX@ + out/tuning-difference@SUFFIX@ out/ua-vamp-plugins.cat out/ua-vamp-plugins_COPYING.txt out/ua-vamp-plugins_README.md - out/ua-vamp-plugins.@SUFFIX@ + out/ua-vamp-plugins@SUFFIX@ out/vamp-aubio.cat out/vamp-aubio.n3 out/vamp-aubio_COPYING.txt out/vamp-aubio_README.md - out/vamp-aubio.@SUFFIX@ + out/vamp-aubio@SUFFIX@ out/vamp-example-plugins.cat out/vamp-example-plugins_COPYING.txt out/vamp-example-plugins.n3 - out/vamp-example-plugins.@SUFFIX@ + out/vamp-example-plugins@SUFFIX@ out/vamp-libxtract.cat out/vamp-libxtract_COPYING.txt out/vamp-libxtract.n3 out/vamp-libxtract_README.txt - out/vamp-libxtract.@SUFFIX@ + out/vamp-libxtract@SUFFIX@ out/vampy_COPYING.txt out/vampy_README.txt - out/vampy.@SUFFIX@ + out/vampy@SUFFIX@ rdf/vamp.n3 rdf/vamp.rdf rdf/plugins/availability.n3